Revert "Merge pull request #4370 from nymtech/jon/reply-on-ipr-version-incompat"

This reverts commit a8dc703399, reversing changes made to e09986e505.
Merge pull request #4370 from nymtech/jon/reply-on-ipr-version-incompat
2024-02-08 21:01:10 +02:00 · 2024-02-07 13:24:14 +01:00 · 2024-02-07 12:52:59 +01:00 · 2024-02-07 11:31:12 +00:00 · 2024-02-07 10:18:23 +01:00 · 2024-02-06 16:03:17 +01:00
739 changed files with 19929 additions and 24865 deletions
@@ -4,18 +4,18 @@ on:
  workflow_dispatch:
  pull_request:
    paths:
-      - 'clients/**'
-      - 'common/**'
-      - 'explorer-api/**'
-      - 'gateway/**'
-      - 'integrations/**'
-      - 'mixnode/**'
-      - 'sdk/rust/nym-sdk/**'
-      - 'service-providers/**'
-      - 'nym-api/**'
-      - 'nym-outfox/**'
-      - 'tools/nym-cli/**'
-      - 'tools/ts-rs-cli/**'
+      - "clients/**"
+      - "common/**"
+      - "explorer-api/**"
+      - "gateway/**"
+      - "integrations/**"
+      - "mixnode/**"
+      - "sdk/rust/nym-sdk/**"
+      - "service-providers/**"
+      - "nym-api/**"
+      - "nym-outfox/**"
+      - "tools/nym-cli/**"
+      - "tools/ts-rs-cli/**"

 jobs:
  publish-nym:
@@ -53,6 +53,16 @@ jobs:
          command: build
          args: --workspace --release

+      - name: Install cargo-deb
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: cargo-deb
+
+      - name: Build deb packages
+        shell: bash
+        run: make deb
+
      - name: Prepare build output
        shell: bash
        env:
@@ -67,6 +77,7 @@ jobs:
          cp target/release/nym-network-statistics $OUTPUT_DIR
          cp target/release/nym-cli $OUTPUT_DIR
          cp target/release/explorer-api $OUTPUT_DIR
+          cp target/debian/*.deb $OUTPUT_DIR

      - name: Deploy branch to CI www
        continue-on-error: true
@@ -1,5 +1,8 @@
 name: ci-cargo-deny
-on: [workflow_dispatch]
+on:
+  workflow_dispatch:
+  pull_request:
+
 jobs:
  cargo-deny:
    runs-on: ubuntu-22.04
@@ -7,10 +10,7 @@ jobs:
      matrix:
        checks:
          # - advisories
-          - licenses
-          - bans sources
-
-    continue-on-error: ${{ matrix.checks == 'licenses' }}
+          - licenses bans sources

    steps:
    - uses: actions/checkout@v3
@@ -17,6 +17,9 @@ jobs:
    steps:
      - uses: actions/checkout@v2

+      - name: install yarn in root
+        run: cd ../.. && yarn install
+
      - name: Install npm
        run: npm install

@@ -1,40 +0,0 @@
-name: ci-nym-vpn-ui-js
-
-on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - 'nym-vpn/ui/src/**'
-      - 'nym-vpn/ui/package.json'
-      - 'nym-vpn/ui/index.html'
-
-jobs:
-  check:
-    runs-on: custom-linux
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18
-      - name: Install Yarn
-        run: npm install -g yarn
-      - name: Install dependencies
-        working-directory: nym-vpn/ui
-        run: yarn
-      - name: Type-check
-        working-directory: nym-vpn/ui
-        run: yarn typecheck
-      - name: Check lint
-        working-directory: nym-vpn/ui
-        run: yarn lint
-      - name: Check formatting
-        working-directory: nym-vpn/ui
-        run: yarn fmt:check
-#      - name: Run tests
-#        working-directory: nym-vpn/ui
-#        run: yarn test
-      - name: Check build
-        working-directory: nym-vpn/ui
-        run: yarn build
@@ -1,63 +0,0 @@
-name: ci-nym-vpn-ui-rust
-
-on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - 'nym-vpn/ui/src-tauri/**'
-
-jobs:
-  build:
-    runs-on: custom-linux
-    env:
-      CARGO_TERM_COLOR: always
-      CARGOTOML_PATH: ./nym-vpn/ui/src-tauri/Cargo.toml
-    steps:
-      - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools libayatana-appindicator3-dev
-        continue-on-error: true
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - name: Prepare build
-        run: mkdir nym-vpn/ui/dist
-
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --manifest-path ${{ env.CARGOTOML_PATH }} --features custom-protocol
-
-#      - name: Run all tests
-#        uses: actions-rs/cargo@v1
-#        with:
-#          command: test
-#          args: --manifest-path ${{ env.CARGOTOML_PATH }}
-
-      - name: Check formatting
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --manifest-path ${{ env.CARGOTOML_PATH }} --all -- --check
-
-      - name: Annotate with clippy checks
-        uses: actions-rs/clippy-check@v1
-        continue-on-error: true
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --manifest-path ${{ env.CARGOTOML_PATH }} --all-features
-
-      - name: Clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --manifest-path ${{ env.CARGOTOML_PATH }} --all-features --all-targets -- -D warnings
@@ -67,6 +67,7 @@ members = [
    "common/nymsphinx/params",
    "common/nymsphinx/routing",
    "common/nymsphinx/types",
+    "common/nyxd-scraper",
    "common/pemstore",
    "common/socks5-client-core",
    "common/socks5/proxy-helpers",
@@ -101,6 +102,7 @@ members = [
    "nym-node",
    "nym-node/nym-node-requests",
    "nym-outfox",
+    "nym-validator-rewarder",
    "tools/internal/ssl-inject",
    "tools/internal/sdk-version-bump",
    "tools/nym-cli",
@@ -123,9 +125,10 @@ default-members = [
    "nym-api",
    "tools/nymvisor",
    "explorer-api",
+    "nym-validator-rewarder",
 ]

-exclude = ["explorer", "contracts", "nym-wallet", "nym-connect/mobile/src-tauri", "nym-connect/desktop", "nym-vpn/ui/src-tauri", "cpu-cycles"]
+exclude = ["explorer", "contracts", "nym-wallet", "nym-connect/mobile/src-tauri", "nym-connect/desktop", "nym-vpn/ui/src-tauri", "cpu-cycles", "sdk/ffi/cpp"]

 [workspace.package]
 authors = ["Nym Technologies SA"]
@@ -159,6 +162,7 @@ reqwest = "0.11.22"
 schemars = "0.8.1"
 serde = "1.0.152"
 serde_json = "1.0.91"
+sqlx = "0.6.3"
 tap = "1.0.1"
 time = "0.3.30"
 thiserror = "1.0.48"
@@ -199,9 +203,12 @@ cw-controllers = { version = "=1.1.0" }

 # cosmrs-related
 bip32 = "0.5.1"
-cosmrs = "=0.15.0"
-tendermint-rpc = "0.34" # same version as used by cosmrs
+
+# temporarily using a fork again (yay.) because we need staking and slashing support
+cosmrs = { git = "https://github.com/jstuczyn/cosmos-rust", branch ="nym-temp/all-validator-features" }
+#cosmrs = {  git = "https://github.com/jstuczyn/cosmos-rust", branch = "nym-temp/all-validator-features" } # unfortuntely we need a fork by yours truly to get the staking support
 tendermint = "0.34" # same version as used by cosmrs
+tendermint-rpc = "0.34" # same version as used by cosmrs
 prost = "0.12"

 # wasm-related dependencies
@@ -12,6 +12,7 @@ help:
 	@echo "  clippy: run clippy for all workspaces"
 	@echo "  test: run clippy, unit tests, and formatting."
 	@echo "  test-all: like test, but also includes the expensive tests"
+	@echo "  deb: build debian packages

 # -----------------------------------------------------------------------------
 # Meta targets
@@ -157,6 +158,12 @@ build-explorer-api:
 build-nym-cli:
 	cargo build -p nym-cli --release

+build-nym-gateway:
+	cargo build -p nym-gateway --release
+
+build-nym-mixnode:
+	cargo build -p nym-mixnode --release
+
 # -----------------------------------------------------------------------------
 # Misc
 # -----------------------------------------------------------------------------
@@ -169,6 +176,13 @@ run-api-tests:
 	cd nym-api/tests/functional_test && yarn test:qa

 # Build debian package, and update PPA
-# Requires base64 encode GPG key to be set up in environment PPA_SIGNING_KEY
-deb:
-	scripts/ppa.sh
+deb-mixnode: build-nym-mixnode
+	cargo deb -p nym-mixnode 
+
+deb-gateway: build-nym-gateway
+	cargo deb -p nym-gateway
+
+deb-cli: build-nym-cli
+	cargo deb -p nym-cli
+
+deb: deb-mixnode deb-gateway deb-cli
@@ -0,0 +1,70 @@
+<html>
+
+<head>
+    <style>
+        @media (prefers-color-scheme: dark) {
+            body {
+                background: #333;
+                color: white;
+            }
+            a {
+                color: skyblue;
+            }
+        }
+        .container {
+            font-family: sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+        }
+        .intro {
+            text-align: center;
+        }
+        .licenses-list {
+            list-style-type: none;
+            margin: 0;
+            padding: 0;
+        }
+        .license-used-by {
+            margin-top: -10px;
+        }
+        .license-text {
+            max-height: 200px;
+            overflow-y: scroll;
+            white-space: pre-wrap;
+        }
+    </style>
+</head>
+
+<body>
+    <main class="container">
+        <div class="intro">
+            <h1>Third Party Licenses</h1>
+            <p>This page lists the licenses of the projects used in cargo-about.</p>
+        </div>
+    
+        <h2>Overview of licenses:</h2>
+        <ul class="licenses-overview">
+            {{#each overview}}
+            <li><a href="#{{id}}">{{name}}</a> ({{count}})</li>
+            {{/each}}
+        </ul>
+
+        <h2>All license text:</h2>
+        <ul class="licenses-list">
+            {{#each licenses}}
+            <li class="license">
+                <h3 id="{{id}}">{{name}}</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    {{#each used_by}}
+                    <li><a href="{{#if crate.repository}} {{crate.repository}} {{else}} https://crates.io/crates/{{crate.name}} {{/if}}">{{crate.name}} {{crate.version}}</a></li>
+                    {{/each}}
+                </ul>
+                <pre class="license-text">{{text}}</pre>
+            </li>
+            {{/each}}
+        </ul>
+    </main>
+</body>
+
+</html>
@@ -0,0 +1,19 @@
+private = { ignore = true }
+
+accepted = [
+    "0BSD",
+    "Apache-2.0",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "CC0-1.0",
+    "ISC",
+    "MIT",
+    "MPL-2.0",
+    "Unicode-DFS-2016",
+    "OpenSSL",
+]
+
+workarounds = [
+    "ring",
+    "rustls",
+]
@@ -1667,9 +1667,9 @@
      }
    },
    "node_modules/follow-redirects": {
-      "version": "1.14.9",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
-      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
+      "version": "1.15.4",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
+      "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
      "dev": true,
      "funding": [
        {
@@ -5800,9 +5800,9 @@
      }
    },
    "follow-redirects": {
-      "version": "1.14.9",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
-      "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
+      "version": "1.15.4",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
+      "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
      "dev": true
    },
    "forwarded": {
@@ -51,7 +51,7 @@ impl InitialisableClient for NativeClientInit {
    }
 }

-#[derive(Args, Clone)]
+#[derive(Args, Clone, Debug)]
 pub(crate) struct Init {
    #[command(flatten)]
    common_args: CommonClientInitArgs,
@@ -51,7 +51,7 @@ impl InitialisableClient for Socks5ClientInit {
    }
 }

-#[derive(Args, Clone)]
+#[derive(Args, Clone, Debug)]
 pub(crate) struct Init {
    #[command(flatten)]
    common_args: CommonClientInitArgs,
@@ -15,4 +15,4 @@ prod:
  mixnode_identity: 3pMCJswCyA19MGYWGDWT5fBk2M8ybSZGXttyAoNY5gBB
  gateway_identity: 2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh
  log_level: error
-  time_zone: utc
+  time_zone: utc
@@ -1,4 +1,6 @@
+import { dir } from "console";
 import { readFileSync } from "fs";
+import { dirname } from "path";
 import { TLogLevelName } from "tslog";

 import YAML from "yaml";
@@ -10,9 +12,11 @@ class ConfigHandler {

  public commonConfig: { request_headers: object };

+  private currentEnvironment: string;
+
  public environment: string;

-  public environmnetConfig: {
+  public environmentConfig: {
    log_level: TLogLevelName;
    time_zone: string;
    api_base_url: string;
@@ -35,8 +39,9 @@ class ConfigHandler {

  private setCommonConfig(): void {
    try {
+      const baseWorkingDirectory = __dirname;
      this.commonConfig = YAML.parse(
-        readFileSync("src/config/config.yaml", "utf8")
+        readFileSync(baseWorkingDirectory + "/config.yaml", "utf8"),
      ).common;
    } catch (error) {
      throw Error(`Error reading common config: (${error})`);
@@ -46,14 +51,24 @@ class ConfigHandler {
  private setEnvironmentConfig(environment: string): void {
    this.ensureEnvironmentIsValid(environment);
    try {
-      this.environmnetConfig = YAML.parse(
-        readFileSync("src/config/config.yaml", "utf8")
+      const baseWorkingDirectory = __dirname;
+      this.environmentConfig = YAML.parse(
+        readFileSync(baseWorkingDirectory + "/config.yaml", "utf8"),
      )[environment];
    } catch (error) {
+      console.log("fadsfasdfasdfsdfsa")
      throw Error(`Error reading environment config: (${error})`);
    }
  }

+  public getEnvironmentConfig(environment: string): any {
+    const baseWorkingDirectory = __dirname;
+    return (
+      this.environmentConfig ||
+      YAML.parse(readFileSync(baseWorkingDirectory + "/config.yaml", "utf8"))[environment]
+    );
+  }
+
  private ensureEnvironmentIsValid(environment: string): void {
    if (this.validEnvironments.indexOf(environment) === -1) {
      throw Error(`Config environment is not valid: "${environment}"`);
@@ -0,0 +1,4 @@
+module.exports = {
+    ConfigHandler: require('./config/configHandler.ts'),
+    RestClient: require('./restClient/RestClient.ts')
+  };
@@ -13,9 +13,9 @@ import ConfigHandler from "../config/configHandler";

 const config = ConfigHandler.getInstance();
 const log = new Logger({
-  minLevel: config.environmnetConfig.log_level,
+  minLevel: config.environmentConfig.log_level,
  dateTimeTimezone:
-    config.environmnetConfig.time_zone ||
+    config.environmentConfig.time_zone ||
    Intl.DateTimeFormat().resolvedOptions().timeZone,
 });

@@ -24,7 +24,7 @@ function isSet(property): boolean {
 }

 export class RestClient {
-  private static authToken: string;
+  public static authToken: string;

  private axiosInstance: AxiosInstance;

@@ -83,7 +83,7 @@ export class RestClient {
        data,
        additionalConfigs,
        params,
-      })
+      }),
    );

    await this.axiosInstance
@@ -214,7 +214,7 @@ export class RestClient {

    if (isSet(additionalConfigs)) {
      logRecord = `${logRecord}\nAdditional Configuration: ${stringify(
-        additionalConfigs
+        additionalConfigs,
      )}`;
    }

@@ -9,6 +9,7 @@ repository = { workspace = true }

 [dependencies]
 atty = "0.2"
+const-str = "0.5.6"
 clap = { workspace = true, features = ["derive"] }
 clap_complete = "4.0"
 clap_complete_fig = "4.0"
@@ -35,9 +36,10 @@ opentelemetry = { version = "0.19.0", optional = true, features = ["rt-tokio"] }


 [build-dependencies]
-vergen = { version = "=7.4.3", default-features = false, features = [
+vergen = { version = "=8.2.6", default-features = false, features = [
    "build",
    "git",
+    "gitcl",
    "rustc",
    "cargo",
 ] }
@@ -1,13 +1,14 @@
 // Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

-use vergen::{vergen, Config};
+use vergen::EmitBuilder;

 fn main() {
-    let mut config = Config::default();
-    if std::env::var("DOCS_RS").is_ok() {
-        // If we don't have access to git information, such as in a docs.rs build, don't error
-        *config.git_mut().skip_if_error_mut() = true;
-    }
-    vergen(config).expect("failed to extract build metadata");
+    EmitBuilder::builder()
+        .all_build()
+        .all_git()
+        .all_rustc()
+        .all_cargo()
+        .emit()
+        .expect("failed to extract build metadata");
 }
@@ -40,14 +40,22 @@ pub struct BinaryBuildInformation {
    /// Provides the rustc channel that was used for the build, for example `nightly`.
    pub rustc_channel: &'static str,

-    // VERGEN_CARGO_PROFILE
-    /// Provides the cargo profile that was used for the build, for example `debug`.
+    // VERGEN_CARGO_DEBUG
+    /// Provides the cargo debug mode that was used for the build.
+    // NOTE: keep the old name cargo_profile instead of cargo_debug for backwards compatibility
    pub cargo_profile: &'static str,
 }

 impl BinaryBuildInformation {
    // explicitly require the build_version to be passed as it's binary specific
    pub const fn new(binary_name: &'static str, build_version: &'static str) -> Self {
+        let cargo_debug = env!("VERGEN_CARGO_DEBUG");
+        let cargo_profile = if const_str::equal!(cargo_debug, "true") {
+            "debug"
+        } else {
+            "release"
+        };
+
        BinaryBuildInformation {
            binary_name,
            build_timestamp: env!("VERGEN_BUILD_TIMESTAMP"),
@@ -57,7 +65,7 @@ impl BinaryBuildInformation {
            commit_branch: env!("VERGEN_GIT_BRANCH"),
            rustc_version: env!("VERGEN_RUSTC_SEMVER"),
            rustc_channel: env!("VERGEN_RUSTC_CHANNEL"),
-            cargo_profile: env!("VERGEN_CARGO_PROFILE"),
+            cargo_profile,
        }
    }

@@ -115,8 +123,9 @@ pub struct BinaryBuildInformationOwned {
    /// Provides the rustc channel that was used for the build, for example `nightly`.
    pub rustc_channel: String,

-    // VERGEN_CARGO_PROFILE
-    /// Provides the cargo profile that was used for the build, for example `debug`.
+    // VERGEN_CARGO_DEBUG
+    /// Provides the cargo debug mode that was used for the build.
+    // NOTE: keep the old name cargo_profile instead of cargo_debug for backwards compatibility
    pub cargo_profile: String,
 }

@@ -46,6 +46,7 @@ nym-validator-client = { path = "../client-libs/validator-client", default-featu
 nym-task = { path = "../task" }
 nym-credential-storage = { path = "../credential-storage" }
 nym-network-defaults = { path = "../network-defaults" }
+si-scale = "0.2.2"

 [target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio-stream]
 version = "0.1.11"
@@ -59,7 +60,7 @@ features = ["time"]
 version = "0.20.1"

 [target."cfg(not(target_arch = \"wasm32\"))".dependencies.sqlx]
-version = "0.6.2"
+workspace = true
 features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"]
 optional = true

@@ -90,7 +91,7 @@ tempfile = "3.1.0"

 [build-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
-sqlx = { version = "0.6.2", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }

 [features]
 default = []
@@ -109,6 +109,8 @@ pub async fn initialise_client<C>(
 ) -> Result<InitResultsWithConfig<C::Config>, C::Error>
 where
    C: InitialisableClient,
+    <C as InitialisableClient>::Config: std::fmt::Debug,
+    <C as InitialisableClient>::InitArgs: std::fmt::Debug,
 {
    info!("initialising {} client", C::NAME);

@@ -140,17 +142,32 @@ where

    // Attempt to use a user-provided gateway, if possible
    let user_chosen_gateway_id = common_args.gateway;
+    log::debug!("User chosen gateway id: {user_chosen_gateway_id:?}");
+
    let selection_spec = GatewaySelectionSpecification::new(
        user_chosen_gateway_id.map(|id| id.to_base58_string()),
        Some(common_args.latency_based_selection),
        false,
    );
+    log::debug!("Gateway selection specification: {selection_spec:?}");

    // Load and potentially override config
+    log::debug!("Init arguments: {init_args:#?}");
    let config = C::construct_config(&init_args);
+    log::debug!("Constructed config: {config:#?}");
    let paths = config.common_paths();
    let core = config.core_config();

+    log::info!(
+        "Using nym-api: {}",
+        core.client
+            .nym_api_urls
+            .iter()
+            .map(|url| url.as_str())
+            .collect::<Vec<&str>>()
+            .join(",")
+    );
+
    // Setup gateway by either registering a new one, or creating a new config from the selected
    // one but with keys kept, or reusing the gateway configuration.
    let key_store = OnDiskKeys::new(paths.keys.clone());
@@ -1,6 +1,7 @@
 // Copyright 2022-2023 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

+use super::packet_statistics_control::PacketStatisticsReporter;
 use super::received_buffer::ReceivedBufferMessage;
 use super::topology_control::geo_aware_provider::GeoAwareTopologyProvider;
 use crate::client::base_client::storage::gateway_details::GatewayDetailsStore;
@@ -10,6 +11,7 @@ use crate::client::inbound_messages::{InputMessage, InputMessageReceiver, InputM
 use crate::client::key_manager::persistence::KeyStore;
 use crate::client::mix_traffic::transceiver::{GatewayReceiver, GatewayTransceiver, RemoteGateway};
 use crate::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
+use crate::client::packet_statistics_control::PacketStatisticsControl;
 use crate::client::real_messages_control;
 use crate::client::real_messages_control::RealMessagesController;
 use crate::client::received_buffer::{
@@ -254,6 +256,7 @@ where
        self_address: Recipient,
        topology_accessor: TopologyAccessor,
        mix_tx: BatchMixMessageSender,
+        stats_tx: PacketStatisticsReporter,
        shutdown: TaskClient,
    ) {
        info!("Starting loop cover traffic stream...");
@@ -266,6 +269,7 @@ where
            topology_accessor,
            debug_config.traffic,
            debug_config.cover_traffic,
+            stats_tx,
        );

        stream.start_with_shutdown(shutdown);
@@ -285,6 +289,7 @@ where
        client_connection_rx: ConnectionCommandReceiver,
        shutdown: TaskClient,
        packet_type: PacketType,
+        stats_tx: PacketStatisticsReporter,
    ) {
        info!("Starting real traffic stream...");

@@ -299,6 +304,7 @@ where
            reply_controller_receiver,
            lane_queue_lengths,
            client_connection_rx,
+            stats_tx,
        )
        .start_with_shutdown(shutdown, packet_type);
    }
@@ -312,6 +318,7 @@ where
        reply_key_storage: SentReplyKeys,
        reply_controller_sender: ReplyControllerSender,
        shutdown: TaskClient,
+        packet_statistics_control: PacketStatisticsReporter,
    ) {
        info!("Starting received messages buffer controller...");
        let controller: ReceivedMessagesBufferController<SphinxMessageReceiver> =
@@ -321,6 +328,7 @@ where
                mixnet_receiver,
                reply_key_storage,
                reply_controller_sender,
+                packet_statistics_control,
            );
        controller.start_with_shutdown(shutdown)
    }
@@ -506,6 +514,13 @@ where
        Ok(())
    }

+    fn start_packet_statistics_control(shutdown: TaskClient) -> PacketStatisticsReporter {
+        info!("Starting packet statistics control...");
+        let (packet_statistics_control, packet_stats_reporter) = PacketStatisticsControl::new();
+        packet_statistics_control.start_with_shutdown(shutdown);
+        packet_stats_reporter
+    }
+
    fn start_mix_traffic_controller(
        gateway_transceiver: Box<dyn GatewayTransceiver + Send>,
        shutdown: TaskClient,
@@ -633,6 +648,9 @@ where
        )
        .await?;

+        let packet_stats_reporter =
+            Self::start_packet_statistics_control(shutdown.fork("packet_statistics_control"));
+
        let gateway_packet_router = PacketRouter::new(
            ack_sender,
            mixnet_messages_sender,
@@ -662,6 +680,7 @@ where
            reply_storage.key_storage(),
            reply_controller_sender.clone(),
            shutdown.fork("received_messages_buffer"),
+            packet_stats_reporter.clone(),
        );

        // The message_sender is the transmitter for any component generating sphinx packets
@@ -700,6 +719,7 @@ where
            client_connection_rx,
            shutdown.fork("real_traffic_controller"),
            self.config.debug.traffic.packet_type,
+            packet_stats_reporter.clone(),
        );

        if !self
@@ -714,6 +734,7 @@ where
                self_address,
                shared_topology_accessor.clone(),
                message_sender,
+                packet_stats_reporter,
                shutdown.fork("cover_traffic_stream"),
            );
        }
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0

 use crate::client::mix_traffic::BatchMixMessageSender;
+use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
 use crate::client::topology_control::TopologyAccessor;
 use crate::{config, spawn_future};
 use futures::task::{Context, Poll};
@@ -61,6 +62,8 @@ where
    secondary_packet_size: Option<PacketSize>,

    packet_type: PacketType,
+
+    stats_tx: PacketStatisticsReporter,
 }

 impl<R> Stream for LoopCoverTrafficStream<R>
@@ -97,7 +100,8 @@ where
 // obviously when we finally make shared rng that is on 'higher' level, this should become
 // generic `R`
 impl LoopCoverTrafficStream<OsRng> {
-    pub fn new(
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
        ack_key: Arc<AckKey>,
        average_ack_delay: Duration,
        mix_tx: BatchMixMessageSender,
@@ -105,6 +109,7 @@ impl LoopCoverTrafficStream<OsRng> {
        topology_access: TopologyAccessor,
        traffic_config: config::Traffic,
        cover_config: config::CoverTraffic,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        let rng = OsRng;

@@ -122,6 +127,7 @@ impl LoopCoverTrafficStream<OsRng> {
            primary_packet_size: traffic_config.primary_packet_size,
            secondary_packet_size: traffic_config.secondary_packet_size,
            packet_type: traffic_config.packet_type,
+            stats_tx,
        }
    }

@@ -191,6 +197,10 @@ impl LoopCoverTrafficStream<OsRng> {
                    log::warn!("Failed to send cover message - channel closed");
                }
            }
+        } else {
+            self.stats_tx.report(PacketStatisticsEvent::CoverPacketSent(
+                cover_traffic_packet_size.size(),
+            ));
        }

        // TODO: I'm not entirely sure whether this is really required, because I'm not 100%
@@ -1,6 +1,8 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

+#![allow(unused_imports)]
+
 use std::time::Duration;

 pub use wasmtimer::{std::Instant, tokio::*};
@@ -7,6 +7,7 @@ pub(crate) mod helpers;
 pub mod inbound_messages;
 pub mod key_manager;
 pub mod mix_traffic;
+pub(crate) mod packet_statistics_control;
 pub mod real_messages_control;
 pub mod received_buffer;
 pub mod replies;
@@ -0,0 +1,498 @@
+use std::{
+    collections::VecDeque,
+    time::{Duration, Instant},
+};
+
+use si_scale::helpers::bibytes2;
+
+use crate::spawn_future;
+
+// Time interval between reporting packet statistics
+const PACKET_REPORT_INTERVAL_SECS: u64 = 2;
+// Interval for taking snapshots of the packet statistics
+const SNAPSHOT_INTERVAL_MS: u64 = 500;
+// When computing rates, we include snapshots that are up to this old. We set it to some odd number
+// a tad larger than an integer number of snapshot intervals, so that we don't have to worry about
+// threshold effects.
+// Also, set it larger than the packet report interval so that we don't miss notable singular events
+const RECORDING_WINDOW_MS: u64 = 2300;
+
+#[derive(Default, Debug, Clone)]
+struct PacketStatistics {
+    // Sent
+    real_packets_sent: u64,
+    real_packets_sent_size: usize,
+    cover_packets_sent: u64,
+    cover_packets_sent_size: usize,
+
+    // Received
+    real_packets_received: u64,
+    real_packets_received_size: usize,
+    cover_packets_received: u64,
+    cover_packets_received_size: usize,
+
+    // Acks
+    total_acks_received: u64,
+    total_acks_received_size: usize,
+    real_acks_received: u64,
+    real_acks_received_size: usize,
+    cover_acks_received: u64,
+    cover_acks_received_size: usize,
+
+    // Types of packets queued
+    // TODO: track the type sent instead
+    real_packets_queued: u64,
+    retransmissions_queued: u64,
+    reply_surbs_queued: u64,
+    additional_reply_surbs_queued: u64,
+}
+
+impl PacketStatistics {
+    fn handle_event(&mut self, event: PacketStatisticsEvent) {
+        match event {
+            PacketStatisticsEvent::RealPacketSent(packet_size) => {
+                self.real_packets_sent += 1;
+                self.real_packets_sent_size += packet_size;
+            }
+            PacketStatisticsEvent::CoverPacketSent(packet_size) => {
+                self.cover_packets_sent += 1;
+                self.cover_packets_sent_size += packet_size;
+            }
+            PacketStatisticsEvent::RealPacketReceived(packet_size) => {
+                self.real_packets_received += 1;
+                self.real_packets_received_size += packet_size;
+            }
+            PacketStatisticsEvent::CoverPacketReceived(packet_size) => {
+                self.cover_packets_received += 1;
+                self.cover_packets_received_size += packet_size;
+            }
+            PacketStatisticsEvent::AckReceived(packet_size) => {
+                self.total_acks_received += 1;
+                self.total_acks_received_size += packet_size;
+            }
+            PacketStatisticsEvent::RealAckReceived(packet_size) => {
+                self.real_acks_received += 1;
+                self.real_acks_received_size += packet_size;
+            }
+            PacketStatisticsEvent::CoverAckReceived(packet_size) => {
+                self.cover_acks_received += 1;
+                self.cover_acks_received_size += packet_size;
+            }
+            PacketStatisticsEvent::RealPacketQueued => {
+                self.real_packets_queued += 1;
+            }
+            PacketStatisticsEvent::RetransmissionQueued => {
+                self.retransmissions_queued += 1;
+            }
+            PacketStatisticsEvent::ReplySurbRequestQueued => {
+                self.reply_surbs_queued += 1;
+            }
+            PacketStatisticsEvent::AdditionalReplySurbRequestQueued => {
+                self.additional_reply_surbs_queued += 1;
+            }
+        }
+    }
+
+    fn summary(&self) -> (String, String) {
+        (
+            format!(
+                "packets sent: {} (real: {}, cover: {}, retransmissions: {})",
+                self.real_packets_sent + self.cover_packets_sent,
+                self.real_packets_sent,
+                self.cover_packets_sent,
+                self.retransmissions_queued,
+            ),
+            format!(
+                "packets received: {}, (real: {}, cover: {}, acks: {}, acks for cover: {})",
+                self.real_packets_received + self.cover_packets_received,
+                self.real_packets_received,
+                self.cover_packets_received,
+                self.real_acks_received,
+                self.cover_acks_received,
+            ),
+        )
+    }
+}
+
+impl std::ops::Sub for PacketStatistics {
+    type Output = Self;
+
+    fn sub(self, rhs: Self) -> Self::Output {
+        Self {
+            real_packets_sent: self.real_packets_sent - rhs.real_packets_sent,
+            real_packets_sent_size: self.real_packets_sent_size - rhs.real_packets_sent_size,
+            cover_packets_sent: self.cover_packets_sent - rhs.cover_packets_sent,
+            cover_packets_sent_size: self.cover_packets_sent_size - rhs.cover_packets_sent_size,
+
+            real_packets_received: self.real_packets_received - rhs.real_packets_received,
+            real_packets_received_size: self.real_packets_received_size
+                - rhs.real_packets_received_size,
+            cover_packets_received: self.cover_packets_received - rhs.cover_packets_received,
+            cover_packets_received_size: self.cover_packets_received_size
+                - rhs.cover_packets_received_size,
+
+            total_acks_received: self.total_acks_received - rhs.total_acks_received,
+            total_acks_received_size: self.total_acks_received_size - rhs.total_acks_received_size,
+            real_acks_received: self.real_acks_received - rhs.real_acks_received,
+            real_acks_received_size: self.real_acks_received_size - rhs.real_acks_received_size,
+            cover_acks_received: self.cover_acks_received - rhs.cover_acks_received,
+            cover_acks_received_size: self.cover_acks_received_size - rhs.cover_acks_received_size,
+
+            real_packets_queued: self.real_packets_queued - rhs.real_packets_queued,
+            retransmissions_queued: self.retransmissions_queued - rhs.retransmissions_queued,
+            reply_surbs_queued: self.reply_surbs_queued - rhs.reply_surbs_queued,
+            additional_reply_surbs_queued: self.additional_reply_surbs_queued
+                - rhs.additional_reply_surbs_queued,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct PacketRates {
+    real_packets_sent: f64,
+    real_packets_sent_size: f64,
+    cover_packets_sent: f64,
+    cover_packets_sent_size: f64,
+
+    real_packets_received: f64,
+    real_packets_received_size: f64,
+    cover_packets_received: f64,
+    cover_packets_received_size: f64,
+
+    total_acks_received: f64,
+    total_acks_received_size: f64,
+    real_acks_received: f64,
+    real_acks_received_size: f64,
+    cover_acks_received: f64,
+    cover_acks_received_size: f64,
+
+    real_packets_queued: f64,
+    retransmissions_queued: f64,
+    reply_surbs_queued: f64,
+    additional_reply_surbs_queued: f64,
+}
+
+impl From<PacketStatistics> for PacketRates {
+    fn from(stats: PacketStatistics) -> Self {
+        Self {
+            real_packets_sent: stats.real_packets_sent as f64,
+            real_packets_sent_size: stats.real_packets_sent_size as f64,
+            cover_packets_sent: stats.cover_packets_sent as f64,
+            cover_packets_sent_size: stats.cover_packets_sent_size as f64,
+
+            real_packets_received: stats.real_packets_received as f64,
+            real_packets_received_size: stats.real_packets_received_size as f64,
+            cover_packets_received: stats.cover_packets_received as f64,
+            cover_packets_received_size: stats.cover_packets_received_size as f64,
+
+            total_acks_received: stats.total_acks_received as f64,
+            total_acks_received_size: stats.total_acks_received_size as f64,
+            real_acks_received: stats.real_acks_received as f64,
+            real_acks_received_size: stats.real_acks_received_size as f64,
+            cover_acks_received: stats.cover_acks_received as f64,
+            cover_acks_received_size: stats.cover_acks_received_size as f64,
+
+            real_packets_queued: stats.real_packets_queued as f64,
+            retransmissions_queued: stats.retransmissions_queued as f64,
+            reply_surbs_queued: stats.reply_surbs_queued as f64,
+            additional_reply_surbs_queued: stats.additional_reply_surbs_queued as f64,
+        }
+    }
+}
+
+impl std::ops::Sub for PacketRates {
+    type Output = Self;
+
+    fn sub(self, rhs: Self) -> Self::Output {
+        Self {
+            real_packets_sent: self.real_packets_sent - rhs.real_packets_sent,
+            real_packets_sent_size: self.real_packets_sent_size - rhs.real_packets_sent_size,
+            cover_packets_sent: self.cover_packets_sent - rhs.cover_packets_sent,
+            cover_packets_sent_size: self.cover_packets_sent_size - rhs.cover_packets_sent_size,
+
+            real_packets_received: self.real_packets_received - rhs.real_packets_received,
+            real_packets_received_size: self.real_packets_received_size
+                - rhs.real_packets_received_size,
+            cover_packets_received: self.cover_packets_received - rhs.cover_packets_received,
+            cover_packets_received_size: self.cover_packets_received_size
+                - rhs.cover_packets_received_size,
+
+            total_acks_received: self.total_acks_received - rhs.total_acks_received,
+            total_acks_received_size: self.total_acks_received_size - rhs.total_acks_received_size,
+            real_acks_received: self.real_acks_received - rhs.real_acks_received,
+            real_acks_received_size: self.real_acks_received_size - rhs.real_acks_received_size,
+            cover_acks_received: self.cover_acks_received - rhs.cover_acks_received,
+            cover_acks_received_size: self.cover_acks_received_size - rhs.cover_acks_received_size,
+
+            real_packets_queued: self.real_packets_queued - rhs.real_packets_queued,
+            retransmissions_queued: self.retransmissions_queued - rhs.retransmissions_queued,
+            reply_surbs_queued: self.reply_surbs_queued - rhs.reply_surbs_queued,
+            additional_reply_surbs_queued: self.additional_reply_surbs_queued
+                - rhs.additional_reply_surbs_queued,
+        }
+    }
+}
+
+impl std::ops::Div<f64> for PacketRates {
+    type Output = Self;
+
+    fn div(self, rhs: f64) -> Self::Output {
+        Self {
+            real_packets_sent: self.real_packets_sent / rhs,
+            real_packets_sent_size: self.real_packets_sent_size / rhs,
+            cover_packets_sent: self.cover_packets_sent / rhs,
+            cover_packets_sent_size: self.cover_packets_sent_size / rhs,
+
+            real_packets_received: self.real_packets_received / rhs,
+            real_packets_received_size: self.real_packets_received_size / rhs,
+            cover_packets_received: self.cover_packets_received / rhs,
+            cover_packets_received_size: self.cover_packets_received_size / rhs,
+
+            total_acks_received: self.total_acks_received / rhs,
+            total_acks_received_size: self.total_acks_received_size / rhs,
+            real_acks_received: self.real_acks_received / rhs,
+            real_acks_received_size: self.real_acks_received_size / rhs,
+            cover_acks_received: self.cover_acks_received / rhs,
+            cover_acks_received_size: self.cover_acks_received_size / rhs,
+
+            real_packets_queued: self.real_packets_queued / rhs,
+            retransmissions_queued: self.retransmissions_queued / rhs,
+            reply_surbs_queued: self.reply_surbs_queued / rhs,
+            additional_reply_surbs_queued: self.additional_reply_surbs_queued / rhs,
+        }
+    }
+}
+
+impl PacketRates {
+    fn summary(&self) -> String {
+        format!(
+            "rx: {}/s (real: {}/s), tx: {}/s (real: {}/s)",
+            bibytes2(self.real_packets_received_size + self.cover_packets_received_size),
+            bibytes2(self.real_packets_received_size),
+            bibytes2(self.real_packets_sent_size + self.cover_packets_sent_size),
+            bibytes2(self.real_packets_sent_size),
+        )
+    }
+
+    fn detailed_summary(&self) -> String {
+        format!(
+            "RX: {:.1} mixpkt/s, {}/s (real: {}/s, acks: {}/s), TX: {:.1} mixpkt/s, {}/s (real: {}/s)",
+            self.real_packets_received + self.cover_packets_received,
+            bibytes2(self.real_packets_received_size + self.cover_packets_received_size),
+            bibytes2(self.real_packets_received_size),
+            bibytes2(self.total_acks_received_size),
+            self.real_packets_sent + self.cover_packets_sent,
+            bibytes2(self.real_packets_sent_size + self.cover_packets_sent_size),
+            bibytes2(self.real_packets_sent_size),
+        )
+    }
+}
+
+pub(crate) enum PacketStatisticsEvent {
+    // The real packets sent. Recall that acks are sent by the gateway, so it's not included here.
+    RealPacketSent(usize),
+    // The cover packets sent
+    CoverPacketSent(usize),
+
+    // Real packets received
+    RealPacketReceived(usize),
+    // Cover packets received
+    CoverPacketReceived(usize),
+
+    // Ack of any type received. This is mostly used as a consistency check, and should be the sum
+    // of real and cover acks received.
+    AckReceived(usize),
+    // Out of the total acks received, this is the subset of those that were real
+    RealAckReceived(usize),
+    // Out of the total acks received, this is the subset of those that were for cover traffic
+    CoverAckReceived(usize),
+
+    // Types of packets queued
+    RealPacketQueued,
+    RetransmissionQueued,
+    ReplySurbRequestQueued,
+    AdditionalReplySurbRequestQueued,
+}
+
+type PacketStatisticsReceiver = tokio::sync::mpsc::UnboundedReceiver<PacketStatisticsEvent>;
+
+#[derive(Clone)]
+pub(crate) struct PacketStatisticsReporter {
+    stats_tx: tokio::sync::mpsc::UnboundedSender<PacketStatisticsEvent>,
+}
+
+impl PacketStatisticsReporter {
+    pub(crate) fn new(stats_tx: tokio::sync::mpsc::UnboundedSender<PacketStatisticsEvent>) -> Self {
+        Self { stats_tx }
+    }
+
+    pub(crate) fn report(&self, event: PacketStatisticsEvent) {
+        self.stats_tx.send(event).unwrap_or_else(|err| {
+            log::error!("Failed to report packet stat: {:?}", err);
+        });
+    }
+}
+
+pub(crate) struct PacketStatisticsControl {
+    // Incoming packet stats events from other tasks
+    stats_rx: PacketStatisticsReceiver,
+
+    // Keep track of packet statistics over time
+    stats: PacketStatistics,
+
+    // We keep snapshots of the statistics over time so we can compute rates, and also keeping the
+    // full history allows for some more fancy averaging if we want to do that.
+    history: VecDeque<(Instant, PacketStatistics)>,
+
+    // Keep previous rates so that we can detect notable events
+    rates: VecDeque<(Instant, PacketRates)>,
+}
+
+impl PacketStatisticsControl {
+    pub(crate) fn new() -> (Self, PacketStatisticsReporter) {
+        let (stats_tx, stats_rx) = tokio::sync::mpsc::unbounded_channel();
+
+        (
+            Self {
+                stats_rx,
+                stats: PacketStatistics::default(),
+                history: VecDeque::new(),
+                rates: VecDeque::new(),
+            },
+            PacketStatisticsReporter::new(stats_tx),
+        )
+    }
+
+    // Add the current stats to the history, and remove old ones.
+    fn update_history(&mut self) {
+        // Update latest
+        self.history.push_back((Instant::now(), self.stats.clone()));
+
+        // Filter out old ones
+        let recording_window = Instant::now() - Duration::from_millis(RECORDING_WINDOW_MS);
+        while self
+            .history
+            .front()
+            .map_or(false, |&(t, _)| t < recording_window)
+        {
+            self.history.pop_front();
+        }
+    }
+
+    fn compute_rates(&self) -> Option<PacketRates> {
+        // NOTE: consider changing this to compute rates over the history instead of using current
+        // stats. Currently it should not make much of a difference since we call this just after
+        // updating the history, but it seems like it could be more internally consistent to do it
+        // that way.
+
+        // Do basic averaging over the entire history, which just uses the first and last
+        if let Some((start, start_stats)) = self.history.front() {
+            let duration_secs = Instant::now().duration_since(*start).as_secs_f64();
+            let delta = self.stats.clone() - start_stats.clone();
+            let rates = PacketRates::from(delta) / duration_secs;
+            Some(rates)
+        } else {
+            None
+        }
+    }
+
+    fn update_rates(&mut self) {
+        // Update latest
+        if let Some(rates) = self.compute_rates() {
+            self.rates.push_back((Instant::now(), rates));
+        }
+
+        // Filter out old ones
+        let recording_window = Instant::now() - Duration::from_millis(RECORDING_WINDOW_MS);
+        while self
+            .rates
+            .front()
+            .map_or(false, |&(t, _)| t < recording_window)
+        {
+            self.rates.pop_front();
+        }
+    }
+
+    fn report_rates(&self) {
+        if let Some((_, rates)) = self.rates.back() {
+            log::info!("{}", rates.summary());
+            log::debug!("{}", rates.detailed_summary());
+        }
+    }
+
+    fn report_counters(&self) {
+        log::trace!("packet statistics: {:?}", &self.stats);
+        let (summary_sent, summary_recv) = self.stats.summary();
+        log::debug!("{}", summary_sent);
+        log::debug!("{}", summary_recv);
+    }
+
+    fn check_for_notable_events(&self) {
+        let Some((_, latest_rates)) = self.rates.back() else {
+            return;
+        };
+
+        // If we get a burst of retransmissions
+        // TODO: consider making this the number of retransmissions since last report instead.
+        if latest_rates.retransmissions_queued > 0.0 {
+            log::debug!(
+                "retransmissions: {:.2} pkt/s",
+                latest_rates.retransmissions_queued
+            );
+
+            // Check what the number of retransmissions was during the recording window
+            if let Some((_, start_stats)) = self.history.front() {
+                let delta = self.stats.clone() - start_stats.clone();
+                log::info!("retransmissions: {}", delta.retransmissions_queued,);
+            } else {
+                log::warn!("Unable to check retransmissions during recording window");
+            }
+        }
+
+        // IDEA: if there is a burst of acks, that could indicate tokio task starvation.
+    }
+
+    pub(crate) async fn run_with_shutdown(&mut self, mut shutdown: nym_task::TaskClient) {
+        log::debug!("Started PacketStatisticsControl with graceful shutdown support");
+
+        let report_interval = Duration::from_secs(PACKET_REPORT_INTERVAL_SECS);
+        let mut report_interval = tokio::time::interval(report_interval);
+        let snapshot_interval = Duration::from_millis(SNAPSHOT_INTERVAL_MS);
+        let mut snapshot_interval = tokio::time::interval(snapshot_interval);
+
+        loop {
+            tokio::select! {
+                stats_event = self.stats_rx.recv() => match stats_event {
+                    Some(stats_event) => {
+                        log::trace!("PacketStatisticsControl: Received stats event");
+                        self.stats.handle_event(stats_event);
+                    },
+                    None => {
+                        log::trace!("PacketStatisticsControl: stopping since stats channel was closed");
+                        break;
+                    }
+                },
+                _ = snapshot_interval.tick() => {
+                    self.update_history();
+                    self.update_rates();
+                }
+                _ = report_interval.tick() => {
+                    self.report_rates();
+                    self.check_for_notable_events();
+                    self.report_counters();
+                }
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("PacketStatisticsControl: Received shutdown");
+                    break;
+                },
+            }
+        }
+        log::debug!("PacketStatisticsControl: Exiting");
+    }
+
+    pub(crate) fn start_with_shutdown(mut self, task_client: nym_task::TaskClient) {
+        spawn_future(async move {
+            self.run_with_shutdown(task_client).await;
+        })
+    }
+}
@@ -1,6 +1,8 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

+use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
+
 use super::action_controller::{AckActionSender, Action};
 use futures::StreamExt;
 use log::*;
@@ -17,6 +19,7 @@ pub(super) struct AcknowledgementListener {
    ack_key: Arc<AckKey>,
    ack_receiver: AcknowledgementReceiver,
    action_sender: AckActionSender,
+    stats_tx: PacketStatisticsReporter,
 }

 impl AcknowledgementListener {
@@ -24,16 +27,21 @@ impl AcknowledgementListener {
        ack_key: Arc<AckKey>,
        ack_receiver: AcknowledgementReceiver,
        action_sender: AckActionSender,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        AcknowledgementListener {
            ack_key,
            ack_receiver,
            action_sender,
+            stats_tx,
        }
    }

    async fn on_ack(&mut self, ack_content: Vec<u8>) {
        trace!("Received an ack");
+        self.stats_tx
+            .report(PacketStatisticsEvent::AckReceived(ack_content.len()));
+
        let frag_id = match recover_identifier(&self.ack_key, &ack_content)
            .map(FragmentIdentifier::try_from_bytes)
        {
@@ -48,11 +56,14 @@ impl AcknowledgementListener {
        // because nothing was inserted in the first place
        if frag_id == COVER_FRAG_ID {
            trace!("Received an ack for a cover message - no need to do anything");
+            self.stats_tx
+                .report(PacketStatisticsEvent::CoverAckReceived(ack_content.len()));
            return;
        }

        trace!("Received {} from the mix network", frag_id);
-
+        self.stats_tx
+            .report(PacketStatisticsEvent::RealAckReceived(ack_content.len()));
        self.action_sender
            .unbounded_send(Action::new_remove(frag_id))
            .unwrap();
@@ -8,6 +8,7 @@ use self::{
    sent_notification_listener::SentNotificationListener,
 };
 use crate::client::inbound_messages::InputMessageReceiver;
+use crate::client::packet_statistics_control::PacketStatisticsReporter;
 use crate::client::real_messages_control::message_handler::MessageHandler;
 use crate::client::replies::reply_controller::ReplyControllerSender;
 use crate::spawn_future;
@@ -208,6 +209,7 @@ where
        connectors: AcknowledgementControllerConnectors,
        message_handler: MessageHandler<R>,
        reply_controller_sender: ReplyControllerSender,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        let (retransmission_tx, retransmission_rx) = mpsc::unbounded();

@@ -224,6 +226,7 @@ where
            Arc::clone(&ack_key),
            connectors.ack_receiver,
            connectors.ack_action_sender.clone(),
+            stats_tx,
        );

        // will listen for any new messages from the client
@@ -35,6 +35,8 @@ use crate::client::replies::reply_controller;
 use crate::config;
 pub(crate) use acknowledgement_control::{AckActionSender, Action};

+use super::packet_statistics_control::PacketStatisticsReporter;
+
 pub(crate) mod acknowledgement_control;
 pub(crate) mod message_handler;
 pub(crate) mod real_traffic_stream;
@@ -143,6 +145,7 @@ impl RealMessagesController<OsRng> {
        reply_controller_receiver: ReplyControllerReceiver,
        lane_queue_lengths: LaneQueueLengths,
        client_connection_rx: ConnectionCommandReceiver,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        let rng = OsRng;

@@ -181,6 +184,7 @@ impl RealMessagesController<OsRng> {
            ack_controller_connectors,
            message_handler.clone(),
            reply_controller_sender,
+            stats_tx.clone(),
        );

        let reply_control = ReplyController::new(
@@ -199,6 +203,7 @@ impl RealMessagesController<OsRng> {
            topology_access,
            lane_queue_lengths,
            client_connection_rx,
+            stats_tx,
        );

        RealMessagesController {
@@ -3,6 +3,7 @@

 use self::sending_delay_controller::SendingDelayController;
 use crate::client::mix_traffic::BatchMixMessageSender;
+use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
 use crate::client::real_messages_control::acknowledgement_control::SentPacketNotificationSender;
 use crate::client::topology_control::TopologyAccessor;
 use crate::client::transmission_buffer::TransmissionBuffer;
@@ -113,6 +114,9 @@ where

    /// Report queue lengths so that upstream can backoff sending data, and keep connections open.
    lane_queue_lengths: LaneQueueLengths,
+
+    /// Channel used for sending statistics events to `PacketStatisticsControl`.
+    stats_tx: PacketStatisticsReporter,
 }

 #[derive(Debug)]
@@ -171,6 +175,7 @@ where
        topology_access: TopologyAccessor,
        lane_queue_lengths: LaneQueueLengths,
        client_connection_rx: ConnectionCommandReceiver,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        OutQueueControl {
            config,
@@ -184,6 +189,7 @@ where
            transmission_buffer: TransmissionBuffer::new(),
            client_connection_rx,
            lane_queue_lengths,
+            stats_tx,
        }
    }

@@ -214,7 +220,7 @@ where
    async fn on_message(&mut self, next_message: StreamMessage) {
        trace!("created new message");

-        let (next_message, fragment_id) = match next_message {
+        let (next_message, fragment_id, packet_size) = match next_message {
            StreamMessage::Cover => {
                let cover_traffic_packet_size = self.loop_cover_message_size();
                trace!("the next loop cover message will be put in a {cover_traffic_packet_size} packet");
@@ -250,15 +256,28 @@ where
                        "Somehow failed to generate a loop cover message with a valid topology",
                    ),
                    None,
+                    cover_traffic_packet_size.size(),
                )
            }
            StreamMessage::Real(real_message) => {
-                (real_message.mix_packet, real_message.fragment_id)
+                let packet_size = real_message.packet_size();
+                (
+                    real_message.mix_packet,
+                    real_message.fragment_id,
+                    packet_size,
+                )
            }
        };

        if let Err(err) = self.mix_tx.send(vec![next_message]).await {
            log::error!("Failed to send: {err}");
+        } else {
+            let event = if fragment_id.is_some() {
+                PacketStatisticsEvent::RealPacketSent(packet_size)
+            } else {
+                PacketStatisticsEvent::CoverPacketSent(packet_size)
+            };
+            self.stats_tx.report(event);
        }

        // notify ack controller about sending our message only after we actually managed to push it
@@ -340,6 +359,28 @@ where
        let lane_length = self.transmission_buffer.lane_length(&lane);
        self.lane_queue_lengths.set(&lane, lane_length);

+        // This is the last step in the pipeline where we know the type of the message, so
+        // lets count the number of retransmissions and reply surb messages sent here.
+        let stat_event = match lane {
+            TransmissionLane::General => None,
+            TransmissionLane::ConnectionId(_) => None,
+            TransmissionLane::ReplySurbRequest => {
+                Some(PacketStatisticsEvent::ReplySurbRequestQueued)
+            }
+            TransmissionLane::AdditionalReplySurbs => {
+                Some(PacketStatisticsEvent::AdditionalReplySurbRequestQueued)
+            }
+            TransmissionLane::Retransmission => Some(PacketStatisticsEvent::RetransmissionQueued),
+        };
+        if let Some(stat_event) = stat_event {
+            self.stats_tx.report(stat_event);
+        }
+        // To avoid comparing apples to oranges when presenting the fraction of packets that are
+        // retransmissions, we also need to keep track to the total number of real messages queued,
+        // even though we also track the actual number of messages sent later in the pipeline.
+        self.stats_tx
+            .report(PacketStatisticsEvent::RealPacketQueued);
+
        Some(real_next)
    }

@@ -433,6 +474,13 @@ where
            Poll::Ready(Some((real_messages, conn_id))) => {
                log::trace!("handling real_messages: size: {}", real_messages.len());

+                // This is the last step in the pipeline where we know the type of the message, so
+                // lets count the number of retransmissions here.
+                if conn_id == TransmissionLane::Retransmission {
+                    self.stats_tx
+                        .report(PacketStatisticsEvent::RetransmissionQueued);
+                }
+
                // First store what we got for the given connection id
                self.transmission_buffer.store(&conn_id, real_messages);
                let real_next = self.pop_next_message().expect("we just added one");
@@ -471,10 +519,10 @@ where
        let mult = self.sending_delay_controller.current_multiplier();
        let delay = self.current_average_message_sending_delay().as_millis();
        let status_str = if self.config.traffic.disable_main_poisson_packet_distribution {
-            format!("Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), no delay")
+            format!("Packet backlog: {backlog:.2} kiB ({packets}), {lanes} lanes, no delay")
        } else {
            format!(
-                "Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), avg delay: {delay}ms ({mult})"
+                "Packet backlog: {backlog:.2} kiB ({packets}), {lanes} lanes, avg delay: {delay}ms ({mult})"
            )
        };
        if packets > 1000 {
@@ -1,8 +1,10 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

-use crate::client::replies::reply_controller::ReplyControllerSender;
-use crate::client::replies::reply_storage::SentReplyKeys;
+use crate::client::{
+    packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter},
+    replies::{reply_controller::ReplyControllerSender, reply_storage::SentReplyKeys},
+};
 use crate::spawn_future;
 use futures::channel::mpsc;
 use futures::lock::Mutex;
@@ -43,15 +45,33 @@ struct ReceivedMessagesBufferInner<R: MessageReceiver> {
    // but perhaps it should be changed to include timestamps of when the message was reconstructed
    // and every now and then remove ids older than X
    recently_reconstructed: HashSet<i32>,
+
+    stats_tx: PacketStatisticsReporter,
 }

 impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
-    fn recover_from_fragment(&mut self, fragment_data: &[u8]) -> Option<NymMessage> {
+    fn recover_from_fragment(
+        &mut self,
+        fragment_data: &[u8],
+        fragment_data_size: usize,
+    ) -> Option<NymMessage> {
        if nym_sphinx::cover::is_cover(fragment_data) {
            trace!("The message was a loop cover message! Skipping it");
+            // NOTE: it's important to note that there is quite a bit of difference in size of
+            // received and sent packets due to the sphinx layers being removed by the exit gateway
+            // before it reaches the mixnet client.
+            self.stats_tx
+                .report(PacketStatisticsEvent::CoverPacketReceived(
+                    fragment_data_size,
+                ));
            return None;
        }

+        self.stats_tx
+            .report(PacketStatisticsEvent::RealPacketReceived(
+                fragment_data_size,
+            ));
+
        let fragment = match self.message_receiver.recover_fragment(fragment_data) {
            Err(err) => {
                warn!("failed to recover fragment from raw data: {err}. The whole underlying message might be corrupted and unrecoverable!");
@@ -103,15 +123,17 @@ impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
        reply_ciphertext: &mut [u8],
        reply_key: SurbEncryptionKey,
    ) -> Result<Option<NymMessage>, MessageRecoveryError> {
+        let reply_ciphertext_size = reply_ciphertext.len();
        // note: this performs decryption IN PLACE without extra allocation
        self.message_receiver
            .recover_plaintext_from_reply(reply_ciphertext, reply_key)?;
        let fragment_data = reply_ciphertext;

-        Ok(self.recover_from_fragment(fragment_data))
+        Ok(self.recover_from_fragment(fragment_data, reply_ciphertext_size))
    }

    fn process_received_regular_packet(&mut self, mut raw_fragment: Vec<u8>) -> Option<NymMessage> {
+        let raw_fragment_size = raw_fragment.len();
        let fragment_data = match self.message_receiver.recover_plaintext_from_regular_packet(
            self.local_encryption_keypair.private_key(),
            &mut raw_fragment,
@@ -123,7 +145,7 @@ impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
            Ok(frag_data) => frag_data,
        };

-        self.recover_from_fragment(fragment_data)
+        self.recover_from_fragment(fragment_data, raw_fragment_size)
    }
 }

@@ -141,6 +163,7 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
        local_encryption_keypair: Arc<encryption::KeyPair>,
        reply_key_storage: SentReplyKeys,
        reply_controller_sender: ReplyControllerSender,
+        stats_tx: PacketStatisticsReporter,
    ) -> Self {
        ReceivedMessagesBuffer {
            inner: Arc::new(Mutex::new(ReceivedMessagesBufferInner {
@@ -149,6 +172,7 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
                message_receiver: R::new(),
                message_sender: None,
                recently_reconstructed: HashSet::new(),
+                stats_tx,
            })),
            reply_key_storage,
            reply_controller_sender,
@@ -353,7 +377,7 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
                };

            if let Some(completed) = completed_message {
-                info!("received {completed}");
+                debug!("received {completed}");
                completed_messages.push(completed)
            }
        }
@@ -480,11 +504,13 @@ impl<R: MessageReceiver + Clone + Send + 'static> ReceivedMessagesBufferControll
        mixnet_packet_receiver: MixnetMessageReceiver,
        reply_key_storage: SentReplyKeys,
        reply_controller_sender: ReplyControllerSender,
+        packet_statistics_reporter: PacketStatisticsReporter,
    ) -> Self {
        let received_buffer = ReceivedMessagesBuffer::new(
            local_encryption_keypair,
            reply_key_storage,
            reply_controller_sender,
+            packet_statistics_reporter,
        );

        ReceivedMessagesBufferController {
@@ -65,7 +65,7 @@ pub async fn current_gateways<R: Rng>(
        .ok_or(ClientCoreError::ListOfNymApisIsEmpty)?;
    let client = nym_validator_client::client::NymApiClient::new(nym_api.clone());

-    log::trace!("Fetching list of gateways from: {nym_api}");
+    log::debug!("Fetching list of gateways from: {nym_api}");

    let gateways = client.get_cached_described_gateways().await?;
    log::debug!("Found {} gateways", gateways.len());
@@ -178,7 +178,7 @@ impl<T> From<PersistedGatewayDetails<T>> for GatewayDetails<T> {
    }
 }

-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum GatewaySelectionSpecification<T = EmptyCustomDetails> {
    /// Uniformly choose a random remote gateway.
    UniformRemote { must_use_tls: bool },
@@ -42,6 +42,14 @@ pub struct Config {
    nyxd_config: nyxd::Config,
 }

+impl TryFrom<NymNetworkDetails> for Config {
+    type Error = ValidatorClientError;
+
+    fn try_from(value: NymNetworkDetails) -> Result<Self, Self::Error> {
+        Config::try_from_nym_network_details(&value)
+    }
+}
+
 impl Config {
    pub fn try_from_nym_network_details(
        details: &NymNetworkDetails,
@@ -5,21 +5,24 @@ use crate::nym_api::error::NymAPIError;
 use crate::nym_api::routes::{CORE_STATUS_COUNT, SINCE_ARG};
 use async_trait::async_trait;
 use http_api_client::{ApiClient, NO_PARAMS};
-use nym_api_requests::coconut::models::{
-    EpochCredentialsResponse, IssuedCredentialResponse, IssuedCredentialsResponse,
+pub use nym_api_requests::{
+    coconut::{
+        models::{
+            EpochCredentialsResponse, IssuedCredential, IssuedCredentialBody,
+            IssuedCredentialResponse, IssuedCredentialsResponse,
+        },
+        BlindSignRequestBody, BlindedSignatureResponse, CredentialsRequestBody,
+        VerifyCredentialBody, VerifyCredentialResponse,
+    },
+    models::{
+        ComputeRewardEstParam, DescribedGateway, GatewayBondAnnotated, GatewayCoreStatusResponse,
+        GatewayStatusReportResponse, GatewayUptimeHistoryResponse, InclusionProbabilityResponse,
+        MixNodeBondAnnotated, MixnodeCoreStatusResponse, MixnodeStatusReportResponse,
+        MixnodeStatusResponse, MixnodeUptimeHistoryResponse, RewardEstimationResponse,
+        StakeSaturationResponse, UptimeResponse,
+    },
 };
-use nym_api_requests::coconut::{
-    BlindSignRequestBody, BlindedSignatureResponse, CredentialsRequestBody, VerifyCredentialBody,
-    VerifyCredentialResponse,
-};
-use nym_api_requests::models::{
-    ComputeRewardEstParam, DescribedGateway, GatewayBondAnnotated, GatewayCoreStatusResponse,
-    GatewayStatusReportResponse, GatewayUptimeHistoryResponse, InclusionProbabilityResponse,
-    MixNodeBondAnnotated, MixnodeCoreStatusResponse, MixnodeStatusReportResponse,
-    MixnodeStatusResponse, MixnodeUptimeHistoryResponse, RewardEstimationResponse,
-    StakeSaturationResponse, UptimeResponse,
-};
-use nym_coconut_dkg_common::types::EpochId;
+pub use nym_coconut_dkg_common::types::EpochId;
 use nym_mixnet_contract_common::mixnode::MixNodeDetails;
 use nym_mixnet_contract_common::{GatewayBond, IdentityKeyRef, MixId};
 use nym_name_service_common::response::NamesListResponse;
@@ -8,6 +8,8 @@ use cosmwasm_std::{Fraction, Uint128};
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use std::ops::Div;
+use std::str::FromStr;
+use thiserror::Error;

 #[derive(Serialize, Deserialize, Clone, Copy, Default, Debug, PartialEq, Eq)]
 pub struct MismatchedDenoms;
@@ -126,6 +128,37 @@ impl From<CosmWasmCoin> for Coin {
    }
 }

+// unfortunately cosmwasm didn't re-export this correct so we just redefine its
+#[derive(Error, Debug, PartialEq, Eq)]
+pub enum CoinFromStrError {
+    #[error("Missing denominator")]
+    MissingDenom,
+    #[error("Missing amount or non-digit characters in amount")]
+    MissingAmount,
+    #[error("Invalid amount: {0}")]
+    InvalidAmount(#[from] std::num::ParseIntError),
+}
+
+impl FromStr for Coin {
+    type Err = CoinFromStrError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let pos = s
+            .find(|c: char| !c.is_ascii_digit())
+            .ok_or(CoinFromStrError::MissingDenom)?;
+        let (amount, denom) = s.split_at(pos);
+
+        if amount.is_empty() {
+            return Err(CoinFromStrError::MissingAmount);
+        }
+
+        Ok(Coin {
+            amount: amount.parse::<u128>()?,
+            denom: denom.to_string(),
+        })
+    }
+}
+
 pub trait CoinConverter {
    type Target;

@@ -7,12 +7,12 @@ use crate::nyxd::error::NyxdError;
 use crate::nyxd::CosmWasmClient;
 use async_trait::async_trait;
 use cosmrs::AccountId;
-use nym_coconut_dkg_common::dealer::{
-    ContractDealing, DealerDetailsResponse, PagedDealerResponse, PagedDealingsResponse,
+use nym_coconut_dkg_common::{
+    dealer::{ContractDealing, DealerDetailsResponse, PagedDealerResponse, PagedDealingsResponse},
+    msg::QueryMsg as DkgQueryMsg,
+    types::{DealerDetails, Epoch, EpochId, InitialReplacementData},
+    verification_key::{ContractVKShare, PagedVKSharesResponse},
 };
-use nym_coconut_dkg_common::msg::QueryMsg as DkgQueryMsg;
-use nym_coconut_dkg_common::types::{DealerDetails, Epoch, EpochId, InitialReplacementData};
-use nym_coconut_dkg_common::verification_key::{ContractVKShare, PagedVKSharesResponse};
 use serde::Deserialize;

 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
@@ -52,10 +52,6 @@ use wasmtimer::tokio::sleep;
 pub const DEFAULT_BROADCAST_POLLING_RATE: Duration = Duration::from_secs(4);
 pub const DEFAULT_BROADCAST_TIMEOUT: Duration = Duration::from_secs(60);

-#[cfg(feature = "http-client")]
-#[async_trait]
-impl CosmWasmClient for cosmrs::rpc::HttpClient {}
-
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait CosmWasmClient: TendermintRpcClient {
@@ -522,3 +518,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
        res.try_into()
    }
 }
+
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+impl<T> CosmWasmClient for T where T: TendermintRpcClient {}
@@ -425,7 +425,7 @@ where
                    amount: amount.into_iter().map(Into::into).collect(),
                }
                .to_any()
-                .map_err(|_| NyxdError::SerializationError("MsgExecuteContract".to_owned()))
+                .map_err(|_| NyxdError::SerializationError("MsgSend".to_owned()))
            })
            .collect::<Result<_, _>>()?;

@@ -1,7 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

-use crate::nyxd::cosmwasm_client::client_traits::{CosmWasmClient, SigningCosmWasmClient};
+use crate::nyxd::cosmwasm_client::client_traits::SigningCosmWasmClient;
 use crate::nyxd::error::NyxdError;
 use crate::nyxd::{Config, GasPrice, Hash, Height};
 use crate::rpc::TendermintRpcClient;
@@ -26,6 +26,7 @@ use cosmrs::rpc::{HttpClient, HttpClientUrl};
 pub mod client_traits;
 mod helpers;
 pub mod logs;
+pub mod module_traits;
 pub mod types;

 #[derive(Debug)]
@@ -329,14 +330,6 @@ where
    }
 }

-#[async_trait]
-impl<C, S> CosmWasmClient for MaybeSigningClient<C, S>
-where
-    C: TendermintRpcClient + Send + Sync,
-    S: Send + Sync,
-{
-}
-
 #[async_trait]
 impl<C, S> SigningCosmWasmClient for MaybeSigningClient<C, S>
 where
@@ -0,0 +1,8 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod slashing;
+pub mod staking;
+
+pub use staking::query::StakingQueryClient;
+// pub use slashing::query
@@ -0,0 +1,4 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod query;
@@ -0,0 +1,2 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,8 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod query;
+
+pub use cosmrs::staking::{
+    QueryHistoricalInfoResponse, QueryValidatorResponse, QueryValidatorsResponse, Validator,
+};
@@ -0,0 +1,78 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{QueryHistoricalInfoResponse, QueryValidatorResponse, QueryValidatorsResponse};
+use crate::nyxd::error::NyxdError;
+use crate::nyxd::{CosmWasmClient, PageRequest};
+use async_trait::async_trait;
+use cosmrs::proto::cosmos::staking::v1beta1::{
+    QueryHistoricalInfoRequest as ProtoQueryHistoricalInfoRequest,
+    QueryHistoricalInfoResponse as ProtoQueryHistoricalInfoResponse,
+    QueryValidatorRequest as ProtoQueryValidatorRequest,
+    QueryValidatorResponse as ProtoQueryValidatorResponse,
+    QueryValidatorsRequest as ProtoQueryValidatorsRequest,
+    QueryValidatorsResponse as ProtoQueryValidatorsResponse,
+};
+use cosmrs::staking::{QueryHistoricalInfoRequest, QueryValidatorRequest, QueryValidatorsRequest};
+use cosmrs::AccountId;
+
+// TODO: change trait restriction from `CosmWasmClient` to `TendermintRpcClient`
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+pub trait StakingQueryClient: CosmWasmClient {
+    async fn historical_info(&self, height: i64) -> Result<QueryHistoricalInfoResponse, NyxdError> {
+        let path = Some("/cosmos.staking.v1beta1.Query/HistoricalInfo".to_owned());
+
+        let req = QueryHistoricalInfoRequest { height };
+
+        let res = self
+            .make_abci_query::<ProtoQueryHistoricalInfoRequest, ProtoQueryHistoricalInfoResponse>(
+                path,
+                req.into(),
+            )
+            .await?;
+
+        Ok(res.try_into()?)
+    }
+
+    async fn validator(
+        &self,
+        validator_addr: AccountId,
+    ) -> Result<QueryValidatorResponse, NyxdError> {
+        let path = Some("/cosmos.staking.v1beta1.Query/Validator".to_owned());
+
+        let req = QueryValidatorRequest { validator_addr };
+
+        let res = self
+            .make_abci_query::<ProtoQueryValidatorRequest, ProtoQueryValidatorResponse>(
+                path,
+                req.into(),
+            )
+            .await?;
+
+        Ok(res.try_into()?)
+    }
+
+    async fn validators(
+        &self,
+        status: String,
+        pagination: Option<PageRequest>,
+    ) -> Result<QueryValidatorsResponse, NyxdError> {
+        let path = Some("/cosmos.staking.v1beta1.Query/Validators".to_owned());
+
+        let req = QueryValidatorsRequest { status, pagination };
+
+        let res = self
+            .make_abci_query::<ProtoQueryValidatorsRequest, ProtoQueryValidatorsResponse>(
+                path,
+                req.into(),
+            )
+            .await?;
+
+        Ok(res.try_into()?)
+    }
+}
+
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+impl<T> StakingQueryClient for T where T: CosmWasmClient {}
@@ -29,23 +29,30 @@ use tendermint_rpc::endpoint::*;
 use tendermint_rpc::{Error as TendermintRpcError, Order};
 use url::Url;

-pub use crate::nyxd::cosmwasm_client::client_traits::{CosmWasmClient, SigningCosmWasmClient};
-pub use crate::nyxd::fee::Fee;
+pub use crate::nyxd::{
+    cosmwasm_client::{
+        client_traits::{CosmWasmClient, SigningCosmWasmClient},
+        module_traits::{self, StakingQueryClient},
+    },
+    fee::Fee,
+};
 pub use crate::rpc::TendermintRpcClient;
 pub use coin::Coin;
-pub use cosmrs::bank::MsgSend;
-pub use cosmrs::tendermint::abci::{
-    response::DeliverTx, types::ExecTxResult, Event, EventAttribute,
+pub use cosmrs::{
+    bank::MsgSend,
+    bip32,
+    crypto::PublicKey,
+    query::{PageRequest, PageResponse},
+    tendermint::{
+        abci::{response::DeliverTx, types::ExecTxResult, Event, EventAttribute},
+        block::Height,
+        hash::{self, Algorithm, Hash},
+        validator::Info as TendermintValidatorInfo,
+        Time as TendermintTime,
+    },
+    tx::{self, Msg},
+    AccountId, Any, Coin as CosmosCoin, Denom, Gas,
 };
-pub use cosmrs::tendermint::block::Height;
-pub use cosmrs::tendermint::hash::{self, Algorithm, Hash};
-pub use cosmrs::tendermint::validator::Info as TendermintValidatorInfo;
-pub use cosmrs::tendermint::Time as TendermintTime;
-pub use cosmrs::tx::Msg;
-pub use cosmrs::tx::{self};
-pub use cosmrs::Coin as CosmosCoin;
-pub use cosmrs::Gas;
-pub use cosmrs::{bip32, AccountId, Denom};
 pub use cosmwasm_std::Coin as CosmWasmCoin;
 pub use cw2;
 pub use cw3;
@@ -55,9 +62,8 @@ pub use fee::{gas_price::GasPrice, GasAdjustable, GasAdjustment};
 pub use tendermint_rpc::{
    endpoint::{tx::Response as TxResponse, validators::Response as ValidatorResponse},
    query::Query,
-    Paging,
+    Paging, Request, Response, SimpleRequest,
 };
-pub use tendermint_rpc::{Request, Response, SimpleRequest};

 #[cfg(feature = "http-client")]
 use crate::http_client;
@@ -97,6 +103,14 @@ impl Config {
    }
 }

+impl TryFrom<NymNetworkDetails> for Config {
+    type Error = NyxdError;
+
+    fn try_from(value: NymNetworkDetails) -> Result<Self, Self::Error> {
+        Config::try_from_nym_network_details(&value)
+    }
+}
+
 #[derive(Debug)]
 pub struct NyxdClient<C, S = NoSigner> {
    client: MaybeSigningClient<C, S>,
@@ -728,7 +742,7 @@ where
    where
        H: Into<Height> + Send,
    {
-        self.client.validators(height, paging).await
+        TendermintRpcClient::validators(&self.client, height, paging).await
    }

    async fn latest_consensus_params(
@@ -803,14 +817,6 @@ where
    }
 }

-#[async_trait]
-impl<C, S> CosmWasmClient for NyxdClient<C, S>
-where
-    C: TendermintRpcClient + Send + Sync,
-    S: Send + Sync,
-{
-}
-
 impl<C, S> OfflineSigner for NyxdClient<C, S>
 where
    S: OfflineSigner,
@@ -73,7 +73,7 @@ where
    P: AsRef<Path>,
 {
    let path = path.as_ref();
-    log::debug!("trying to save config file to {}", path.display());
+    log::info!("saving config file to {}", path.display());

    if let Some(parent) = path.parent() {
        create_dir_all(parent)?;
@@ -226,4 +226,8 @@ impl EpochState {
    pub fn is_final(&self) -> bool {
        *self == EpochState::InProgress
    }
+
+    pub fn is_in_progress(&self) -> bool {
+        matches!(self, EpochState::InProgress)
+    }
 }
@@ -14,14 +14,14 @@ thiserror = { workspace = true }
 tokio = { version = "1.24.1", features = ["sync"]}

 [target."cfg(not(target_arch = \"wasm32\"))".dependencies.sqlx]
-version = "0.5"
+workspace = true
 features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"]

 [target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio]
-version = "1.24.1"
+workspace = true
 features = [ "rt-multi-thread", "net", "signal", "fs" ]


 [build-dependencies]
-sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
 tokio = { version = "1.24.1", features = ["rt-multi-thread", "macros"] }
@@ -81,6 +81,15 @@ ExitPolicy accept6 *6:119
 ExitPolicy accept *4:120
 ExitPolicy reject6 [FC00::]/7:*

+# Portless
+ExitPolicy accept *:0
+ExitPolicy accept *4:0
+ExitPolicy accept *6:0
+
+ExitPolicy reject *:0
+ExitPolicy reject *4:0
+ExitPolicy reject *6:0
+
 #ExitPolicy accept *:8080 #and another comment here

 ExitPolicy reject FE80:0000:0000:0000:0202:B3FF:FE1E:8329:*
@@ -184,6 +193,60 @@ ExitPolicy reject *:*
            },
        );

+        // ExitPolicy accept *:0
+        expected.push(
+            Accept,
+            AddressPortPattern {
+                ip_pattern: IpPattern::Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
+        // ExitPolicy accept *4:0
+        expected.push(
+            Accept,
+            AddressPortPattern {
+                ip_pattern: IpPattern::V4Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
+        // ExitPolicy accept *6:0
+        expected.push(
+            Accept,
+            AddressPortPattern {
+                ip_pattern: IpPattern::V6Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
+        // ExitPolicy reject *:0
+        expected.push(
+            Reject,
+            AddressPortPattern {
+                ip_pattern: IpPattern::Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
+        // ExitPolicy reject *4:0
+        expected.push(
+            Reject,
+            AddressPortPattern {
+                ip_pattern: IpPattern::V4Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
+        // ExitPolicy reject *6:0
+        expected.push(
+            Reject,
+            AddressPortPattern {
+                ip_pattern: IpPattern::V6Star,
+                ports: PortRange::new_zero(),
+            },
+        );
+
        // ExitPolicy FE80:0000:0000:0000:0202:B3FF:FE1E:8329:*
        expected.push(
            Reject,
@@ -264,7 +264,13 @@ mod stringified_ip_pattern {
 impl AddressPortPattern {
    /// Return true iff this pattern matches a given address and port.
    pub fn matches(&self, addr: &IpAddr, port: u16) -> bool {
-        self.ip_pattern.matches(addr) && self.ports.contains(port)
+        // For backward compatibility, we treat port 0 as a wildcard until all gateways have
+        // upgraded, at which point we can add *:0 to the policy list.
+        if port == 0 {
+            self.ip_pattern.matches(addr)
+        } else {
+            self.ip_pattern.matches(addr) && self.ports.contains(port)
+        }
    }

    /// As matches, but accept a SocketAddr.
@@ -395,19 +401,9 @@ fn parse_addr(s: &str) -> Result<IpAddr, PolicyError> {
    })
 }

-/// Helper: try to parse a port making sure it's non-zero
 fn parse_port(s: &str) -> Result<u16, PolicyError> {
-    let port = s
-        .parse::<u16>()
-        .map_err(|_| PolicyError::InvalidPort { raw: s.to_string() })?;
-
-    if port == 0 {
-        Err(PolicyError::InvalidPort {
-            raw: port.to_string(),
-        })
-    } else {
-        Ok(port)
-    }
+    s.parse::<u16>()
+        .map_err(|_| PolicyError::InvalidPort { raw: s.to_string() })
 }

 impl FromStr for IpPattern {
@@ -494,6 +490,10 @@ impl PortRange {
        PortRange::new_unchecked(1, 65535)
    }

+    pub fn new_zero() -> Self {
+        PortRange { start: 0, end: 0 }
+    }
+
    /// Create a new PortRange.
    ///
    /// The Portrange contains all ports between `start` and `end` inclusive.
@@ -574,6 +574,7 @@ mod test {

        check("marzipan:80");
        check("1.2.3.4:90-80");
+        check("1.2.3.4:0-80");
        check("1.2.3.4/100:8888");
        check("[1.2.3.4]/16:80");
        check("[::1]/130:8888");
@@ -612,6 +613,22 @@ mod test {

        check("0.0.0.0/0:*", &["127.0.0.1:80"], &["[f00b::]:80"]);
        check("[::]/0:*", &["[f00b::]:80"], &["127.0.0.1:80"]);
+
+        check(
+            "*:0",
+            &["1.2.3.4:0", "[::1]:0", "9.0.0.0:0"],
+            &["1.2.3.4:443", "[::1]:500", "9.0.0.0:80", "[::1]:80"],
+        );
+        check(
+            "*4:0",
+            &["1.2.3.4:0", "9.0.0.0:0"],
+            &["1.2.3.4:443", "9.0.0.0:80", "[::1]:0", "[::1]:80"],
+        );
+        check(
+            "*6:0",
+            &["[::1]:0"],
+            &["[::1]:80", "1.2.3.4:0", "1.2.3.4:443"],
+        );
    }

    #[test]
@@ -620,6 +637,7 @@ mod test {
        policy.push(AddressPolicyAction::Accept, "*:443".parse()?);
        policy.push(AddressPolicyAction::Accept, "[::1]:80".parse()?);
        policy.push(AddressPolicyAction::Reject, "*:80".parse()?);
+        policy.push(AddressPolicyAction::Accept, "*:0".parse()?);

        let policy = policy; // drop mut
        assert!(policy
@@ -640,6 +658,9 @@ mod test {
        assert!(policy
            .allows_sockaddr(&"127.0.0.1:66".parse().unwrap())
            .is_none());
+        assert!(policy
+            .allows_sockaddr(&"127.0.0.1:0".parse().unwrap())
+            .unwrap());
        Ok(())
    }

@@ -672,7 +693,6 @@ mod test {
        assert_eq!("*".parse::<PortRange>().unwrap(), PortRange::new_all());

        assert!("hello".parse::<PortRange>().is_err());
-        assert!("0".parse::<PortRange>().is_err());
        assert!("65536".parse::<PortRange>().is_err());
        assert!("65537".parse::<PortRange>().is_err());
        assert!("1-2-3".parse::<PortRange>().is_err());
@@ -680,6 +700,9 @@ mod test {
        assert!("1-".parse::<PortRange>().is_err());
        assert!("-2".parse::<PortRange>().is_err());
        assert!("-".parse::<PortRange>().is_err());
+
+        assert_eq!("0".parse::<PortRange>().unwrap(), PortRange::new_zero(),);
+        assert!("0-1".parse::<PortRange>().is_err());
    }

    #[test]
@@ -11,6 +11,7 @@ repository.workspace = true
 cfg-if = { workspace = true }
 dotenvy = { workspace = true }
 hex-literal = "0.3.3"
+log = { workspace = true }
 once_cell = { workspace = true }
 schemars = { workspace = true, features = ["preserve_order"] }
 serde = { workspace = true, features = ["derive"]}
@@ -111,6 +111,7 @@ impl NymNetworkDetails {
            .with_additional_validator_endpoint(ValidatorDetails::new(
                var(var_names::NYXD).expect("nyxd validator not set"),
                Some(var(var_names::NYM_API).expect("nym api not set")),
+                get_optional_env(var_names::NYXD_WEBSOCKET),
            ))
            .with_mixnet_contract(Some(
                var(var_names::MIXNET_CONTRACT_ADDRESS).expect("mixnet contract not set"),
@@ -340,6 +341,9 @@ impl DenomDetailsOwned {
 pub struct ValidatorDetails {
    // it is assumed those values are always valid since they're being provided in our defaults file
    pub nyxd_url: String,
+    //
+    pub websocket_url: Option<String>,
+
    // Right now api_url is optional as we are not running the api reliably on all validators
    // however, later on it should be a mandatory field
    pub api_url: Option<String>,
@@ -347,9 +351,10 @@ pub struct ValidatorDetails {
 }

 impl ValidatorDetails {
-    pub fn new<S: Into<String>>(nyxd_url: S, api_url: Option<S>) -> Self {
+    pub fn new<S: Into<String>>(nyxd_url: S, api_url: Option<S>, websocket_url: Option<S>) -> Self {
        ValidatorDetails {
            nyxd_url: nyxd_url.into(),
+            websocket_url: websocket_url.map(Into::into),
            api_url: api_url.map(Into::into),
        }
    }
@@ -357,6 +362,7 @@ impl ValidatorDetails {
    pub fn new_nyxd_only<S: Into<String>>(nyxd_url: S) -> Self {
        ValidatorDetails {
            nyxd_url: nyxd_url.into(),
+            websocket_url: None,
            api_url: None,
        }
    }
@@ -372,6 +378,12 @@ impl ValidatorDetails {
            .as_ref()
            .map(|url| url.parse().expect("the provided api url is invalid!"))
    }
+
+    pub fn websocket_url(&self) -> Option<Url> {
+        self.websocket_url
+            .as_ref()
+            .map(|url| url.parse().expect("the provided websocket url is invalid!"))
+    }
 }

 fn fix_deprecated_environmental_variables() {
@@ -388,11 +400,25 @@ fn fix_deprecated_environmental_variables() {
    }
 }

+// Read the variables from the file and log what the corresponding values in the environment are.
+fn print_env_vars_with_keys_in_file<P: AsRef<Path> + Copy>(config_env_file: P) {
+    let items = dotenvy::from_path_iter(config_env_file)
+        .expect("Invalid path to environment configuration file");
+    for item in items {
+        let (key, val) = item.expect("Invalid item in environment configuration file");
+        log::debug!("{}: {}", key, val);
+    }
+}
+
 pub fn setup_env<P: AsRef<Path>>(config_env_file: Option<P>) {
    match std::env::var(var_names::CONFIGURED) {
        // if the configuration is not already set in the env vars
        Err(std::env::VarError::NotPresent) => {
-            if let Some(config_env_file) = config_env_file {
+            if let Some(config_env_file) = &config_env_file {
+                log::debug!(
+                    "Loading environment variables from {:?}",
+                    config_env_file.as_ref()
+                );
                dotenvy::from_path(config_env_file)
                    .expect("Invalid path to environment configuration file");
                fix_deprecated_environmental_variables();
@@ -400,17 +426,25 @@ pub fn setup_env<P: AsRef<Path>>(config_env_file: Option<P>) {
                // if nothing is set, the use mainnet defaults
                // if the user has not set `CONFIGURED`, then even if they set any of the env variables,
                // overwrite them
+                log::debug!("Loading mainnet defaults");
                crate::mainnet::export_to_env();
            }
        }
-        Err(_) => crate::mainnet::export_to_env(),
+        Err(_) => {
+            log::debug!("Environment variables already set. Using them");
+            crate::mainnet::export_to_env()
+        }
        _ => {
            fix_deprecated_environmental_variables();
        }
    }

    // if we haven't explicitly defined any of the constants, fallback to defaults
-    crate::mainnet::export_to_env_if_not_set()
+    crate::mainnet::export_to_env_if_not_set();
+
+    if let Some(config_env_file) = &config_env_file {
+        print_env_vars_with_keys_in_file(config_env_file);
+    }
 }

 // Name of the event triggered by the eth contract. If the event name is changed,
@@ -3,6 +3,7 @@

 use crate::var_names;
 use crate::{DenomDetails, ValidatorDetails};
+use std::str::FromStr;

 pub const NETWORK_NAME: &str = "mainnet";

@@ -25,6 +26,7 @@ pub const REWARDING_VALIDATOR_ADDRESS: &str = "n10yyd98e2tuwu0f7ypz9dy3hhjw7v772
 pub const STATISTICS_SERVICE_DOMAIN_ADDRESS: &str = "https://mainnet-stats.nymte.ch:8090/";
 pub const NYXD_URL: &str = "https://rpc.nymtech.net";
 pub const NYM_API: &str = "https://validator.nymtech.net/api/";
+pub const NYXD_WS: &str = "wss://rpc.nymtech.net/websocket";
 pub const EXPLORER_API: &str = "https://explorer.nymtech.net/api/";

 // I'm making clippy mad on purpose, because that url HAS TO be updated and deployed before merging
@@ -32,7 +34,11 @@ pub const EXIT_POLICY_URL: &str =
    "https://nymtech.net/.wellknown/network-requester/exit-policy.txt";

 pub(crate) fn validators() -> Vec<ValidatorDetails> {
-    vec![ValidatorDetails::new(NYXD_URL, Some(NYM_API))]
+    vec![ValidatorDetails::new(
+        NYXD_URL,
+        Some(NYM_API),
+        Some(NYXD_WS),
+    )]
 }

 const DEFAULT_SUFFIX: &str = "_MAINNET_DEFAULT";
@@ -60,6 +66,12 @@ pub fn read_var_if_not_default(var: &str) -> Option<String> {
    }
 }

+pub fn read_parsed_var_if_not_default<T: FromStr>(var: &str) -> Option<Result<T, T::Err>> {
+    read_var_if_not_default(var)
+        .as_deref()
+        .map(FromStr::from_str)
+}
+
 pub fn export_to_env() {
    set_var_to_default(var_names::CONFIGURED, "true");
    set_var_to_default(var_names::NETWORK_NAME, NETWORK_NAME);
@@ -104,6 +116,7 @@ pub fn export_to_env() {
    );
    set_var_to_default(var_names::NYXD, NYXD_URL);
    set_var_to_default(var_names::NYM_API, NYM_API);
+    set_var_to_default(var_names::NYXD_WEBSOCKET, NYXD_WS);
    set_var_to_default(var_names::EXPLORER_API, EXPLORER_API);
    set_var_to_default(var_names::EXIT_POLICY_URL, EXIT_POLICY_URL);
 }
@@ -152,6 +165,7 @@ pub fn export_to_env_if_not_set() {
    );
    set_var_conditionally_to_default(var_names::NYXD, NYXD_URL);
    set_var_conditionally_to_default(var_names::NYM_API, NYM_API);
+    set_var_conditionally_to_default(var_names::NYXD_WEBSOCKET, NYXD_WS);
    set_var_conditionally_to_default(var_names::EXPLORER_API, EXPLORER_API);
    set_var_conditionally_to_default(var_names::EXIT_POLICY_URL, EXIT_POLICY_URL);
 }
@@ -26,6 +26,7 @@ pub const SERVICE_PROVIDER_DIRECTORY_CONTRACT_ADDRESS: &str =
 pub const NAME_SERVICE_CONTRACT_ADDRESS: &str = "NAME_SERVICE_CONTRACT_ADDRESS";
 pub const NYXD: &str = "NYXD";
 pub const NYM_API: &str = "NYM_API";
+pub const NYXD_WEBSOCKET: &str = "NYXD_WS";
 pub const EXPLORER_API: &str = "EXPLORER_API";
 pub const EXIT_POLICY_URL: &str = "EXIT_POLICY";

@@ -255,7 +255,7 @@ fn bench_coconut(c: &mut Criterion) {
            b.iter(|| {
                verify_partial_blind_signature(
                    &params,
-                    &blind_sign_request,
+                    blind_sign_request.get_private_attributes_pedersen_commitments(),
                    &public_attributes,
                    random_blind_signature,
                    partial_verification_key,
@@ -43,3 +43,5 @@ mod utils;
 pub type Attribute = bls12_381::Scalar;
 pub type PrivateAttribute = Attribute;
 pub type PublicAttribute = Attribute;
+
+pub use bls12_381::G1Projective;
@@ -362,12 +362,12 @@ pub fn blind_sign(
 /// The function returns `true` if the partial blind signature is valid, and `false` otherwise.
 pub fn verify_partial_blind_signature(
    params: &Parameters,
-    blind_sign_request: &BlindSignRequest,
+    private_attribute_commitments: &[G1Projective],
    public_attributes: &[&Attribute],
    blind_sig: &BlindedSignature,
    partial_verification_key: &VerificationKey,
 ) -> bool {
-    let num_private_attributes = blind_sign_request.private_attributes_commitments.len();
+    let num_private_attributes = private_attribute_commitments.len();
    if num_private_attributes + public_attributes.len() > partial_verification_key.beta_g2.len() {
        return false;
    }
@@ -388,8 +388,7 @@ pub fn verify_partial_blind_signature(
    ];

    // for each private attribute, add (cm_i, beta_i) to the miller terms
-    for (private_attr_commit, beta_g2) in blind_sign_request
-        .private_attributes_commitments
+    for (private_attr_commit, beta_g2) in private_attribute_commitments
        .iter()
        .zip(&partial_verification_key.beta_g2)
    {
@@ -422,7 +421,7 @@ pub fn verify_partial_blind_signature(
    // is equivalent to checking e(a, b) • e(c, d)^{-1} == id
    // and thus to e(a, b) • e(c^{-1}, d) == id
    //
-    // compute e(c^1, g2) • e(s, alpha) • e(cm_0, beta_0) • e(cm_i, beta_i) • (s^pub_0, beta_{i+1}) (s^pub_j, beta_{i + j})
+    // compute e(c^{-1}, g2) • e(s, alpha) • e(cm_0, beta_0) • e(cm_i, beta_i) • (s^pub_0, beta_{i+1}) (s^pub_j, beta_{i + j})
    multi_miller_loop(&terms_refs)
        .final_exponentiation()
        .is_identity()
@@ -519,7 +518,7 @@ mod tests {

        assert!(verify_partial_blind_signature(
            &params,
-            &request,
+            &request.private_attributes_commitments,
            &public_attributes,
            &blind_sig,
            validator_keypair.verification_key()
@@ -539,7 +538,7 @@ mod tests {

        assert!(verify_partial_blind_signature(
            &params,
-            &request,
+            &request.private_attributes_commitments,
            &[],
            &blind_sig,
            validator_keypair.verification_key()
@@ -568,7 +567,7 @@ mod tests {
        // this assertion should fail, as we try to verify with a wrong validator key
        assert!(!verify_partial_blind_signature(
            &params,
-            &request,
+            &request.private_attributes_commitments,
            &public_attributes,
            &blind_sig,
            validator2_keypair.verification_key()
@@ -108,6 +108,17 @@ pub fn transpose_matrix<T: Debug>(matrix: Vec<Vec<T>>) -> Vec<Vec<T>> {
        .collect::<Vec<_>>()
 }

+#[macro_export]
+macro_rules! random_scalars_refs {
+    ( $x: ident, $params: expr, $n: expr ) => {
+        let _vec = $params.n_random_scalars($n);
+        #[allow(clippy::map_identity)]
+        let $x = _vec.iter().collect::<Vec<_>>();
+    };
+}
+
+pub use random_scalars_refs;
+
 #[cfg(test)]
 pub mod tests {
    use super::*;
@@ -170,14 +181,3 @@ pub mod tests {
            .collect()
    }
 }
-
-#[macro_export]
-macro_rules! random_scalars_refs {
-    ( $x: ident, $params: expr, $n: expr ) => {
-        let _vec = $params.n_random_scalars($n);
-        #[allow(clippy::map_identity)]
-        let $x = _vec.iter().collect::<Vec<_>>();
-    };
-}
-
-pub use random_scalars_refs;
@@ -239,8 +239,15 @@ impl NymMessage {
        let (packets_used, space_left) =
            chunking::number_of_required_fragments(total_required_bytes, plaintext_per_packet);

-        let wasted_space = space_left as f32 / (bytes.len() + 1 + space_left) as f32;
-        log::trace!("Padding {self_display}: {} of raw plaintext bytes are required. They're going to be put into {packets_used} sphinx packets with {space_left} bytes of leftover space. {wasted_space}% of packet capacity is going to be wasted.", bytes.len() + 1);
+        let wasted_space_percentage =
+            (space_left as f32 / (bytes.len() + 1 + space_left) as f32) * 100.0;
+        log::trace!(
+            "Padding {self_display}: {} of raw plaintext bytes are required. \
+            They're going to be put into {packets_used} sphinx packets with {space_left} bytes \
+            of leftover space. {wasted_space_percentage:.1}% of packet capacity is going to \
+            be wasted.",
+            bytes.len() + 1
+        );

        bytes
            .into_iter()
@@ -0,0 +1,37 @@
+[package]
+name = "nyxd-scraper"
+version = "0.1.0"
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+edition.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait.workspace = true
+const_format = "0.2.32"
+cosmrs.workspace = true
+eyre = "0.6.9"
+futures.workspace = true
+sha2 = "0.10.8"
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "time"] }
+tendermint.workspace = true
+tendermint-rpc = { workspace = true, features = ["websocket-client", "http-client"] }
+thiserror.workspace = true
+tokio = { workspace = true, features = ["full"] }
+tokio-stream = "0.1.14"
+tokio-util = { version = "0.7.10", features = ["rt"]}
+tracing.workspace = true
+url.workspace = true
+
+
+# TEMP
+nym-bin-common = { path = "../bin-common", features = ["basic_tracing"]}
+
+
+[build-dependencies]
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
@@ -0,0 +1,28 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#[tokio::main]
+async fn main() {
+    use sqlx::{Connection, SqliteConnection};
+    use std::env;
+
+    let out_dir = env::var("OUT_DIR").unwrap();
+    let database_path = format!("{out_dir}/scraper-example.sqlite");
+
+    let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc"))
+        .await
+        .expect("Failed to create SQLx database connection");
+
+    sqlx::migrate!("./sql_migrations")
+        .run(&mut conn)
+        .await
+        .expect("Failed to perform SQLx migrations");
+
+    #[cfg(target_family = "unix")]
+    println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
+
+    #[cfg(target_family = "windows")]
+    // for some strange reason we need to add a leading `/` to the windows path even though it's
+    // not a valid windows path... but hey, it works...
+    println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
+}
@@ -0,0 +1,10 @@
+/*
+ * Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+CREATE TABLE METADATA
+(
+    id                    INTEGER PRIMARY KEY CHECK (id = 0),
+    last_processed_height INTEGER NOT NULL
+);
@@ -0,0 +1,76 @@
+CREATE TABLE validator
+(
+    consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */
+    consensus_pubkey  TEXT NOT NULL UNIQUE /* Validator consensus public key */
+);
+
+CREATE TABLE pre_commit
+(
+    validator_address TEXT                        NOT NULL REFERENCES validator (consensus_address),
+    height            BIGINT                      NOT NULL,
+    timestamp         TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+    voting_power      BIGINT                      NOT NULL,
+    proposer_priority BIGINT                      NOT NULL,
+    UNIQUE (validator_address, timestamp)
+);
+CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address);
+CREATE INDEX pre_commit_height_index ON pre_commit (height);
+
+CREATE TABLE block
+(
+    height           BIGINT  UNIQUE PRIMARY KEY,
+    hash             TEXT    NOT NULL UNIQUE,
+    num_txs          INTEGER DEFAULT 0,
+    total_gas        BIGINT  DEFAULT 0,
+    proposer_address TEXT REFERENCES validator (consensus_address),
+    timestamp        TIMESTAMP WITHOUT TIME ZONE NOT NULL
+);
+CREATE INDEX block_height_index ON block (height);
+CREATE INDEX block_hash_index ON block (hash);
+CREATE INDEX block_proposer_address_index ON block (proposer_address);
+
+-- no JSONB in sqlite (yet) : )
+CREATE TABLE "transaction"
+(
+    hash         TEXT    UNIQUE NOT NULL,
+    height       BIGINT  NOT NULL REFERENCES block (height),
+    "index"      INTEGER NOT NULL,
+    success      BOOLEAN NOT NULL,
+
+    /* Body */
+    num_messages INTEGER NOT NULL,
+--     messages     JSONB   NOT NULL,-- DEFAULT '[]'::JSONB,
+    memo         TEXT,
+--     signatures   TEXT[]  NOT NULL,
+
+    /* AuthInfo */
+--     signer_infos JSONB   NOT NULL,-- DEFAULT '[]'::JSONB,
+--     fee          JSONB   NOT NULL,-- DEFAULT '{}'::JSONB,
+
+    /* Tx response */
+    gas_wanted   BIGINT           DEFAULT 0,
+    gas_used     BIGINT           DEFAULT 0,
+    raw_log      TEXT
+--     logs         JSONB
+);
+CREATE INDEX transaction_hash_index ON "transaction" (hash);
+CREATE INDEX transaction_height_index ON "transaction" (height);
+
+CREATE TABLE message
+(
+    transaction_hash            TEXT   NOT NULL REFERENCES "transaction" (hash),
+    "index"                       BIGINT NOT NULL,
+    type                        TEXT   NOT NULL,
+--     value                       JSONB  NOT NULL,
+--     involved_accounts_addresses TEXT[] NOT NULL,
+
+    height                      BIGINT NOT NULL,
+    CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, "index")
+);
+CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
+CREATE INDEX message_type_index ON message (type);
+
+CREATE TABLE pruning
+(
+    last_pruned_height BIGINT NOT NULL
+);
@@ -0,0 +1,65 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::MAX_RANGE_SIZE;
+use std::cmp::min;
+use std::collections::VecDeque;
+use std::ops::Range;
+
+pub(crate) fn split_request_range(request_range: Range<u32>) -> VecDeque<Range<u32>> {
+    let mut requests = VecDeque::new();
+
+    let mut start = request_range.start;
+    let mut end = min(request_range.end, start + MAX_RANGE_SIZE as u32);
+
+    loop {
+        requests.push_back(start..end);
+        start = min(start + MAX_RANGE_SIZE as u32, request_range.end);
+        end = min(end + MAX_RANGE_SIZE as u32, request_range.end);
+
+        if start == end {
+            break;
+        }
+    }
+
+    requests
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn splitting_request_range() {
+        let range = 0..100;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        expected.push_back(30..60);
+        expected.push_back(60..90);
+        expected.push_back(90..100);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..30;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..60;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..30);
+        expected.push_back(30..60);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 0..5;
+        let mut expected = VecDeque::new();
+        expected.push_back(0..5);
+        assert_eq!(expected, split_request_range(range));
+
+        let range = 123..200;
+        let mut expected = VecDeque::new();
+        expected.push_back(123..153);
+        expected.push_back(153..183);
+        expected.push_back(183..200);
+        assert_eq!(expected, split_request_range(range));
+    }
+}
@@ -0,0 +1,332 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::helpers::split_request_range;
+use crate::block_processor::types::BlockToProcess;
+use crate::block_requester::BlockRequest;
+use crate::error::ScraperError;
+use crate::modules::{BlockModule, MsgModule, TxModule};
+use crate::rpc_client::RpcClient;
+use crate::storage::{persist_block, ScraperStorage};
+use futures::StreamExt;
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::ops::{Add, Range};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::mpsc::{Sender, UnboundedReceiver};
+use tokio::sync::Notify;
+use tokio::time::{interval_at, Instant};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, info, warn};
+
+mod helpers;
+pub(crate) mod types;
+
+const MISSING_BLOCKS_CHECK_INTERVAL: Duration = Duration::from_secs(30);
+const MAX_MISSING_BLOCKS_DELAY: Duration = Duration::from_secs(15);
+const MAX_RANGE_SIZE: usize = 30;
+
+#[derive(Debug, Default)]
+struct PendingSync {
+    request_in_flight: HashSet<u32>,
+    queued_requests: VecDeque<Range<u32>>,
+}
+
+impl PendingSync {
+    fn is_empty(&self) -> bool {
+        self.request_in_flight.is_empty() && self.queued_requests.is_empty()
+    }
+}
+
+pub struct BlockProcessor {
+    cancel: CancellationToken,
+    synced: Arc<Notify>,
+    last_processed_height: u32,
+    last_processed_at: Instant,
+    pending_sync: PendingSync,
+    queued_blocks: BTreeMap<u32, BlockToProcess>,
+
+    rpc_client: RpcClient,
+    incoming: UnboundedReceiverStream<BlockToProcess>,
+    block_requester: Sender<BlockRequest>,
+    storage: ScraperStorage,
+
+    // future work: rather than sending each msg to every msg module,
+    // let them subscribe based on `type_url` inside the message itself
+    // (like "/cosmwasm.wasm.v1.MsgExecuteContract")
+    block_modules: Vec<Box<dyn BlockModule + Send>>,
+    tx_modules: Vec<Box<dyn TxModule + Send>>,
+    msg_modules: Vec<Box<dyn MsgModule + Send>>,
+}
+
+impl BlockProcessor {
+    pub async fn new(
+        cancel: CancellationToken,
+        synced: Arc<Notify>,
+        incoming: UnboundedReceiver<BlockToProcess>,
+        block_requester: Sender<BlockRequest>,
+        storage: ScraperStorage,
+        rpc_client: RpcClient,
+    ) -> Result<Self, ScraperError> {
+        let last_processed = storage.get_last_processed_height().await?;
+
+        Ok(BlockProcessor {
+            cancel,
+            synced,
+            last_processed_height: last_processed.try_into().unwrap_or_default(),
+            last_processed_at: Instant::now(),
+            pending_sync: Default::default(),
+            queued_blocks: Default::default(),
+            rpc_client,
+            incoming: incoming.into(),
+            block_requester,
+            storage,
+            block_modules: vec![],
+            tx_modules: vec![],
+            msg_modules: vec![],
+        })
+    }
+
+    async fn process_block(&mut self, block: BlockToProcess) -> Result<(), ScraperError> {
+        info!("processing block at height {}", block.height);
+
+        let full_info = self.rpc_client.try_get_full_details(block).await?;
+
+        debug!(
+            "this block has {} transaction(s)",
+            full_info.transactions.len()
+        );
+        for tx in &full_info.transactions {
+            debug!("{} has {} message(s)", tx.hash, tx.tx.body.messages.len());
+            for (index, msg) in tx.tx.body.messages.iter().enumerate() {
+                debug!("{index}: {:?}", msg.type_url)
+            }
+        }
+
+        // process the entire block as a transaction so that if anything fails,
+        // we won't end up with a corrupted storage.
+        let mut tx = self.storage.begin_processing_tx().await?;
+
+        persist_block(&full_info, &mut tx).await?;
+
+        // let the modules do whatever they want
+        // the ones wanting the full block:
+        for block_module in &mut self.block_modules {
+            block_module.handle_block(&full_info, &mut tx).await?;
+        }
+
+        // the ones wanting transactions:
+        for block_tx in full_info.transactions {
+            for tx_module in &mut self.tx_modules {
+                tx_module.handle_tx(&block_tx, &mut tx).await?;
+            }
+            // the ones concerned with individual messages
+            for (index, msg) in block_tx.tx.body.messages.iter().enumerate() {
+                for msg_module in &mut self.msg_modules {
+                    msg_module
+                        .handle_msg(index, msg, &block_tx, &mut tx)
+                        .await?
+                }
+            }
+        }
+
+        tx.commit()
+            .await
+            .map_err(|source| ScraperError::StorageTxCommitFailure { source })?;
+
+        self.last_processed_height = full_info.block.header.height.value() as u32;
+        self.last_processed_at = Instant::now();
+
+        Ok(())
+    }
+
+    pub fn set_block_modules(&mut self, modules: Vec<Box<dyn BlockModule + Send>>) {
+        self.block_modules = modules;
+    }
+
+    pub fn set_tx_modules(&mut self, modules: Vec<Box<dyn TxModule + Send>>) {
+        self.tx_modules = modules;
+    }
+
+    pub fn set_msg_modules(&mut self, modules: Vec<Box<dyn MsgModule + Send>>) {
+        self.msg_modules = modules;
+    }
+
+    async fn maybe_request_missing_blocks(&mut self) -> Result<(), ScraperError> {
+        // we're still processing, so we're good
+        if self.last_processed_at.elapsed() < MAX_MISSING_BLOCKS_DELAY {
+            debug!("no need to request missing blocks");
+            return Ok(());
+        }
+
+        if self.try_request_pending().await {
+            return Ok(());
+        }
+
+        // TODO: properly fill in the gaps later with BlockRequest::Specific,
+        let request_range = if let Some((next_available, _)) = self.queued_blocks.first_key_value()
+        {
+            self.last_processed_height + 1..*next_available
+        } else {
+            let current_height = self.rpc_client.current_block_height().await? as u32;
+            self.last_processed_height + 1..current_height + 1
+        };
+
+        self.request_missing_blocks(request_range).await?;
+
+        Ok(())
+    }
+
+    async fn request_missing_blocks(
+        &mut self,
+        request_range: Range<u32>,
+    ) -> Result<(), ScraperError> {
+        let request_range = if request_range.len() > MAX_RANGE_SIZE {
+            let mut split = split_request_range(request_range);
+
+            // SAFETY: we know that after the split of a non-empty range we have AT LEAST one value
+            #[allow(clippy::unwrap_used)]
+            let first = split.pop_front().unwrap();
+            self.pending_sync.queued_requests = split;
+            self.pending_sync.request_in_flight = first.clone().collect();
+
+            first
+        } else {
+            request_range
+        };
+
+        self.send_blocks_request(request_range).await
+    }
+
+    // technically we're not mutating self here,
+    // but we need it to help the compiler figure out the future is `Send`
+    async fn send_blocks_request(&mut self, request_range: Range<u32>) -> Result<(), ScraperError> {
+        debug!("requesting missing blocks: {request_range:?}");
+
+        self.block_requester
+            .send(BlockRequest::Range(request_range))
+            .await?;
+        Ok(())
+    }
+
+    async fn next_incoming(&mut self, block: BlockToProcess) {
+        let height = block.height;
+
+        self.pending_sync.request_in_flight.remove(&height);
+
+        if self.last_processed_height == 0 {
+            // this is the first time we've started up the process
+            debug!("setting up initial processing height");
+            self.last_processed_height = height - 1
+        }
+
+        if height <= self.last_processed_height {
+            warn!("we have already processed block for height {height}");
+            return;
+        }
+
+        if self.last_processed_height + 1 != height {
+            if self.queued_blocks.insert(height, block).is_some() {
+                warn!("we have already queued up block for height {height}");
+            }
+            return;
+        }
+
+        if let Err(err) = self.process_block(block).await {
+            error!("failed to process block at height {height}: {err}");
+            return;
+        }
+
+        // process as much as we can from the queue
+        let mut next = height + 1;
+        while let Some(next_block) = self.queued_blocks.remove(&next) {
+            if let Err(err) = self.process_block(next_block).await {
+                error!("failed to process queued-up block at height {next}: {err}")
+            }
+            next += 1;
+        }
+
+        self.try_request_pending().await;
+
+        if self.pending_sync.is_empty() {
+            self.synced.notify_one();
+        }
+    }
+
+    async fn try_request_pending(&mut self) -> bool {
+        if self.pending_sync.request_in_flight.is_empty() {
+            if let Some(next_sync) = self.pending_sync.queued_requests.pop_front() {
+                debug!(
+                    "current request range has been resolved. requesting another bunch of blocks"
+                );
+                if let Err(err) = self.send_blocks_request(next_sync.clone()).await {
+                    error!("failed to request resync blocks: {err}");
+                    self.pending_sync.queued_requests.push_front(next_sync);
+                } else {
+                    self.pending_sync.request_in_flight = next_sync.collect()
+                }
+
+                return true;
+            }
+        }
+
+        false
+    }
+
+    // technically we're not mutating self here,
+    // but we need it to help the compiler figure out the future is `Send`
+    async fn startup_resync(&mut self) -> Result<(), ScraperError> {
+        assert!(self.pending_sync.is_empty());
+
+        let latest_block = self.rpc_client.current_block_height().await? as u32;
+        if latest_block > self.last_processed_height && self.last_processed_height != 0 {
+            let request_range = self.last_processed_height + 1..latest_block + 1;
+            info!("we need to request {request_range:?} to resync");
+            self.request_missing_blocks(request_range).await?;
+        }
+
+        Ok(())
+    }
+
+    pub(crate) async fn run(&mut self) {
+        info!("starting processing loop");
+
+        // sure, we could be more efficient and reset it on every processed block,
+        // but the overhead is so minimal that it doesn't matter
+        let mut missing_check_interval = interval_at(
+            Instant::now().add(MISSING_BLOCKS_CHECK_INTERVAL),
+            MISSING_BLOCKS_CHECK_INTERVAL,
+        );
+
+        if let Err(err) = self.startup_resync().await {
+            error!("failed to perform startup sync: {err}");
+            self.cancel.cancel();
+            return;
+        };
+
+        loop {
+            tokio::select! {
+                _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    break
+                }
+                _ = missing_check_interval.tick() => {
+                    if let Err(err) = self.maybe_request_missing_blocks().await {
+                        error!("failed to request missing blocks: {err}")
+                    }
+                }
+                block = self.incoming.next() => {
+                    match block {
+                        Some(block) => self.next_incoming(block).await,
+                        None => {
+                            warn!("stopped receiving new blocks");
+                            self.cancel.cancel();
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,121 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::ScraperError;
+use crate::helpers;
+use tendermint::{abci, block, tx, Block, Hash};
+use tendermint_rpc::endpoint::{block as block_endpoint, block_results, validators};
+use tendermint_rpc::event::{Event, EventData};
+
+// just get all everything out of tx::Response, but parse raw `tx` bytes
+#[derive(Clone, Debug)]
+pub struct ParsedTransactionResponse {
+    /// The hash of the transaction.
+    ///
+    /// Deserialized from a hex-encoded string (there is a discrepancy between
+    /// the format used for the request and the format used for the response in
+    /// the Tendermint RPC).
+    pub hash: Hash,
+
+    pub height: block::Height,
+
+    pub index: u32,
+
+    pub tx_result: abci::types::ExecTxResult,
+
+    pub tx: cosmrs::tx::Tx,
+
+    pub proof: Option<tx::Proof>,
+}
+
+#[derive(Debug)]
+pub struct FullBlockInformation {
+    /// Basic block information, including its signers.
+    pub block: Block,
+
+    /// All of the emitted events alongside any tx results.
+    pub results: block_results::Response,
+
+    /// Validator set for this particular block
+    pub validators: validators::Response,
+
+    /// Transaction results from this particular block
+    pub transactions: Vec<ParsedTransactionResponse>,
+}
+
+impl FullBlockInformation {
+    pub fn ensure_proposer(&self) -> Result<(), ScraperError> {
+        let block_proposer = self.block.header.proposer_address;
+        if !self
+            .validators
+            .validators
+            .iter()
+            .any(|v| v.address == block_proposer)
+        {
+            let proposer = helpers::validator_consensus_address(block_proposer)?;
+            return Err(ScraperError::BlockProposerNotInValidatorSet {
+                height: self.block.header.height.value() as u32,
+                proposer: proposer.to_string(),
+            });
+        }
+        Ok(())
+    }
+}
+
+pub(crate) struct BlockToProcess {
+    pub(crate) height: u32,
+    pub(crate) block: Block,
+}
+
+impl From<Block> for BlockToProcess {
+    fn from(block: Block) -> Self {
+        BlockToProcess {
+            height: block.header.height.value() as u32,
+            block,
+        }
+    }
+}
+
+impl TryFrom<Event> for BlockToProcess {
+    type Error = ScraperError;
+
+    fn try_from(event: Event) -> Result<Self, Self::Error> {
+        let query = event.query.clone();
+
+        // TODO: we're losing `result_begin_block` and `result_end_block` here but maybe that's fine?
+        let maybe_block = match event.data {
+            // we don't care about `NewBlock` until CometBFT 0.38, i.e. until we upgrade to wasmd 0.50
+            EventData::NewBlock { .. } => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "NewBlock".to_string(),
+                })
+            }
+            EventData::LegacyNewBlock { block, .. } => block,
+            EventData::Tx { .. } => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "Tx".to_string(),
+                })
+            }
+            EventData::GenericJsonEvent(_) => {
+                return Err(ScraperError::InvalidSubscriptionEvent {
+                    query,
+                    kind: "GenericJsonEvent".to_string(),
+                })
+            }
+        };
+
+        let Some(block) = maybe_block else {
+            return Err(ScraperError::EmptyBlockData { query });
+        };
+
+        Ok((*block).into())
+    }
+}
+
+impl From<block_endpoint::Response> for BlockToProcess {
+    fn from(value: block_endpoint::Response) -> Self {
+        value.block.into()
+    }
+}
@@ -0,0 +1,91 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::BlockToProcess;
+use crate::error::ScraperError;
+use crate::rpc_client::RpcClient;
+use futures::StreamExt;
+use std::ops::Range;
+use tokio::sync::mpsc::{Receiver, UnboundedSender};
+use tokio_stream::wrappers::ReceiverStream;
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info, instrument, warn};
+
+#[derive(Debug)]
+pub enum BlockRequest {
+    Range(Range<u32>),
+
+    // UNIMPLEMENTED:
+    #[allow(dead_code)]
+    Specific(Vec<u32>),
+}
+
+pub(crate) struct BlockRequester {
+    cancel: CancellationToken,
+    rpc_client: RpcClient,
+    requests: ReceiverStream<BlockRequest>,
+    blocks: UnboundedSender<BlockToProcess>,
+}
+
+impl BlockRequester {
+    pub(crate) fn new(
+        cancel: CancellationToken,
+        rpc_client: RpcClient,
+        requests: Receiver<BlockRequest>,
+        blocks: UnboundedSender<BlockToProcess>,
+    ) -> Self {
+        BlockRequester {
+            cancel,
+            rpc_client,
+            requests: requests.into(),
+            blocks,
+        }
+    }
+
+    async fn request_and_send(&self, height: u32) -> Result<(), ScraperError> {
+        let block = self.rpc_client.get_basic_block_details(height).await?;
+        self.blocks.send(block.into())?;
+        Ok(())
+    }
+
+    async fn request_blocks<I: IntoIterator<Item = u32>>(&self, heights: I) {
+        futures::stream::iter(heights)
+            .for_each_concurrent(4, |height| async move {
+                if let Err(err) = self.request_and_send(height).await {
+                    error!("failed to request block data: {err}")
+                }
+            })
+            .await
+    }
+
+    #[instrument(skip(self))]
+    async fn handle_blocks_request(&self, request: BlockRequest) {
+        info!("received request for missed blocks");
+
+        match request {
+            BlockRequest::Range(range) => self.request_blocks(range).await,
+            BlockRequest::Specific(heights) => self.request_blocks(heights).await,
+        }
+    }
+
+    pub(crate) async fn run(&mut self) {
+        loop {
+            tokio::select! {
+               _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    break
+                }
+                maybe_request = self.requests.next() => {
+                    match maybe_request {
+                        Some(request) => self.handle_blocks_request(request).await,
+                        None => {
+                            warn!("stopped receiving new requests");
+                            self.cancel.cancel();
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,47 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use const_format::concatcp;
+
+// TODO: make those configurable via 'NymNetworkDetails'
+
+// BECH32_PREFIX defines the main SDK Bech32 prefix of an account's address
+pub const BECH32_PREFIX: &str = "n";
+
+// ACCOUNT_PREFIX is the prefix for account keys
+pub const ACCOUNT_PREFIX: &str = "acc";
+// VALIDATOR_PREFIX is the prefix for validator keys
+pub const VALIDATOR_PREFIX: &str = "val";
+// CONSENSUS_PREFIX is the prefix for consensus keys
+pub const CONSENSUS_PREFIX: &str = "cons";
+// PUBKEY_PREFIX is the prefix for public keys
+pub const PUBKEY_PREFIX: &str = "pub";
+// OPERATOR_PREFIX is the prefix for operator keys
+pub const OPERATOR_PREFIX: &str = "oper";
+// ADDRESS_PREFIX is the prefix for addresses
+pub const ADDRESS_PREFIX: &str = "addr";
+
+// BECH32_ACCOUNT_ADDRESS_PREFIX defines the Bech32 prefix of an account's address
+pub const BECH32_ACCOUNT_ADDRESS_PREFIX: &str = BECH32_PREFIX;
+// BECH32_ACCOUNT_PUBKEY_PREFIX defines the Bech32 prefix of an account's public key
+pub const BECH32_ACCOUNT_PUBKEY_PREFIX: &str = concatcp!(BECH32_PREFIX, PUBKEY_PREFIX);
+// BECH32_VALIDATOR_ADDRESS_PREFIX defines the Bech32 prefix of a validator's operator address
+pub const BECH32_VALIDATOR_ADDRESS_PREFIX: &str =
+    concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, OPERATOR_PREFIX);
+// BECH32_VALIDATOR_PUBKEY_PREFIX defines the Bech32 prefix of a validator's operator public key
+pub const BECH32_VALIDATOR_PUBKEY_PREFIX: &str = concatcp!(
+    BECH32_PREFIX,
+    VALIDATOR_PREFIX,
+    OPERATOR_PREFIX,
+    PUBKEY_PREFIX
+);
+// BECH32_CONSENSUS_ADDRESS_PREFIX defines the Bech32 prefix of a consensus node address
+pub const BECH32_CONSENSUS_ADDRESS_PREFIX: &str =
+    concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, CONSENSUS_PREFIX);
+// BECH32_CONESNSUS_PUBKEY_PREFIX defines the Bech32 prefix of a consensus node public key
+pub const BECH32_CONESNSUS_PUBKEY_PREFIX: &str = concatcp!(
+    BECH32_PREFIX,
+    VALIDATOR_PREFIX,
+    CONSENSUS_PREFIX,
+    PUBKEY_PREFIX
+);
@@ -0,0 +1,131 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use tendermint::Hash;
+use thiserror::Error;
+use tokio::sync::mpsc::error::SendError;
+
+#[derive(Debug, Error)]
+pub enum ScraperError {
+    #[error("experienced internal database error: {0}")]
+    InternalDatabaseError(#[from] sqlx::Error),
+
+    #[error("failed to perform startup SQL migration: {0}")]
+    StartupMigrationFailure(#[from] sqlx::migrate::MigrateError),
+
+    #[error("can't add any modules to the scraper as it's already running")]
+    ScraperAlreadyRunning,
+
+    #[error("failed to establish websocket connection to {url}: {source}")]
+    WebSocketConnectionFailure {
+        url: String,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("failed to establish rpc connection to {url}: {source}")]
+    HttpConnectionFailure {
+        url: String,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("failed to create chain subscription: {source}")]
+    ChainSubscriptionFailure {
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain basic block information at height: {height}: {source}")]
+    BlockQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain block results information at height: {height}: {source}")]
+    BlockResultsQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain validators information at height: {height}: {source}")]
+    ValidatorsQueryFailure {
+        height: u32,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain tx results for tx: {hash}: {source}")]
+    TxResultsQueryFailure {
+        hash: Hash,
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not obtain current abci info: {source}")]
+    AbciInfoQueryFailure {
+        #[source]
+        source: tendermint_rpc::Error,
+    },
+
+    #[error("could not parse tx {hash}: {source}")]
+    TxParseFailure {
+        hash: Hash,
+        #[source]
+        source: cosmrs::ErrorReport,
+    },
+
+    #[error("received an invalid chain subscription event of kind {kind} while we were waiting for new block data (query: '{query}')")]
+    InvalidSubscriptionEvent { query: String, kind: String },
+
+    #[error("received block data was empty (query: '{query}')")]
+    EmptyBlockData { query: String },
+
+    #[error("reached maximum number of allowed errors for subscription events")]
+    MaximumSubscriptionFailures,
+
+    #[error("failed to begin storage tx: {source}")]
+    StorageTxBeginFailure {
+        #[source]
+        source: sqlx::Error,
+    },
+
+    #[error("failed to commit storage tx: {source}")]
+    StorageTxCommitFailure {
+        #[source]
+        source: sqlx::Error,
+    },
+
+    #[error("failed to send on a closed channel")]
+    ClosedChannelError,
+
+    #[error("failed to parse validator's address: {source}")]
+    MalformedValidatorAddress {
+        #[source]
+        source: eyre::Report,
+    },
+
+    #[error("failed to parse validator's address: {source}")]
+    MalformedValidatorPubkey {
+        #[source]
+        source: eyre::Report,
+    },
+
+    #[error(
+        "could not find the block proposer ('{proposer}') for height {height} in the validator set"
+    )]
+    BlockProposerNotInValidatorSet { height: u32, proposer: String },
+
+    #[error(
+        "could not find validator information for {address}; the validator has signed a commit"
+    )]
+    MissingValidatorInfoCommitted { address: String },
+}
+
+impl<T> From<SendError<T>> for ScraperError {
+    fn from(_: SendError<T>) -> Self {
+        ScraperError::ClosedChannelError
+    }
+}
@@ -0,0 +1,46 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX};
+use crate::error::ScraperError;
+use cosmrs::AccountId;
+use sha2::{Digest, Sha256};
+use tendermint::{account, PublicKey};
+use tendermint::{validator, Hash};
+use tendermint_rpc::endpoint::validators;
+
+pub(crate) fn tx_hash<M: AsRef<[u8]>>(raw_tx: M) -> Hash {
+    Hash::Sha256(Sha256::digest(raw_tx).into())
+}
+
+pub(crate) fn validator_pubkey_to_bech32(pubkey: PublicKey) -> Result<AccountId, ScraperError> {
+    // TODO: this one seem to attach additional prefix to they pubkeys, is that what we want instead maybe?
+    // Ok(pubkey.to_bech32(BECH32_CONESNSUS_PUBKEY_PREFIX))
+    AccountId::new(BECH32_CONESNSUS_PUBKEY_PREFIX, &pubkey.to_bytes())
+        .map_err(|source| ScraperError::MalformedValidatorPubkey { source })
+}
+
+pub(crate) fn validator_consensus_address(id: account::Id) -> Result<AccountId, ScraperError> {
+    AccountId::new(BECH32_CONSENSUS_ADDRESS_PREFIX, id.as_ref())
+        .map_err(|source| ScraperError::MalformedValidatorAddress { source })
+}
+
+pub(crate) fn tx_gas_sum(txs: &[ParsedTransactionResponse]) -> i64 {
+    txs.iter().map(|tx| tx.tx_result.gas_used).sum()
+}
+
+pub(crate) fn validator_info(
+    id: account::Id,
+    validators: &validators::Response,
+) -> Result<&validator::Info, ScraperError> {
+    match validators.validators.iter().find(|v| v.address == id) {
+        Some(info) => Ok(info),
+        None => {
+            let addr = validator_consensus_address(id)?;
+            Err(ScraperError::MissingValidatorInfoCommitted {
+                address: addr.to_string(),
+            })
+        }
+    }
+}
@@ -0,0 +1,19 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#![warn(clippy::expect_used)]
+#![warn(clippy::unwrap_used)]
+
+pub(crate) mod block_processor;
+pub(crate) mod block_requester;
+pub mod constants;
+pub mod error;
+pub(crate) mod helpers;
+pub mod modules;
+pub(crate) mod rpc_client;
+pub(crate) mod scraper;
+pub mod storage;
+
+pub use modules::{BlockModule, MsgModule, TxModule};
+pub use scraper::{Config, NyxdScraper};
+pub use storage::models;
@@ -0,0 +1,16 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::FullBlockInformation;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait BlockModule {
+    async fn handle_block(
+        &mut self,
+        block: &FullBlockInformation,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}
@@ -0,0 +1,10 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+mod block_module;
+mod msg_module;
+mod tx_module;
+
+pub use block_module::BlockModule;
+pub use msg_module::MsgModule;
+pub use tx_module::TxModule;
@@ -0,0 +1,19 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+use cosmrs::Any;
+
+#[async_trait]
+pub trait MsgModule {
+    async fn handle_msg(
+        &mut self,
+        index: usize,
+        msg: &Any,
+        tx: &ParsedTransactionResponse,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}
@@ -0,0 +1,16 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::ParsedTransactionResponse;
+use crate::error::ScraperError;
+use crate::storage::StorageTransaction;
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait TxModule {
+    async fn handle_tx(
+        &mut self,
+        tx: &ParsedTransactionResponse,
+        storage_tx: &mut StorageTransaction,
+    ) -> Result<(), ScraperError>;
+}
@@ -0,0 +1,175 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::{
+    BlockToProcess, FullBlockInformation, ParsedTransactionResponse,
+};
+use crate::error::ScraperError;
+use crate::helpers::tx_hash;
+use futures::future::join3;
+use futures::StreamExt;
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use tendermint::Hash;
+use tendermint_rpc::endpoint::{block, block_results, tx, validators};
+use tendermint_rpc::{Client, HttpClient, Paging};
+use tokio::sync::Mutex;
+use tracing::{debug, instrument};
+use url::Url;
+
+#[derive(Clone)]
+pub struct RpcClient {
+    // right now I don't care about anything nym specific, so a simple http client is sufficient,
+    // once this is inadequate, we can switch to a NyxdClient
+    inner: Arc<HttpClient>,
+}
+
+impl RpcClient {
+    pub fn new(url: &Url) -> Result<Self, ScraperError> {
+        let http_client = HttpClient::new(url.as_str()).map_err(|source| {
+            ScraperError::HttpConnectionFailure {
+                url: url.to_string(),
+                source,
+            }
+        })?;
+
+        Ok(RpcClient {
+            inner: Arc::new(http_client),
+        })
+    }
+
+    #[instrument(skip(self, block), fields(height = block.height))]
+    pub async fn try_get_full_details(
+        &self,
+        block: BlockToProcess,
+    ) -> Result<FullBlockInformation, ScraperError> {
+        debug!("getting complete block details");
+        let height = block.height;
+
+        // make all the http requests concurrently
+        let (results, validators, raw_transactions) = join3(
+            self.get_block_results(height),
+            self.get_validators_details(height),
+            self.get_transaction_results(&block.block.data),
+        )
+        .await;
+
+        let raw_transactions = raw_transactions?;
+        let mut transactions = Vec::with_capacity(raw_transactions.len());
+        for tx in raw_transactions {
+            transactions.push(ParsedTransactionResponse {
+                hash: tx.hash,
+                height: tx.height,
+                index: tx.index,
+                tx_result: tx.tx_result,
+                tx: cosmrs::Tx::from_bytes(&tx.tx).map_err(|source| {
+                    ScraperError::TxParseFailure {
+                        hash: tx.hash,
+                        source,
+                    }
+                })?,
+                proof: tx.proof,
+            })
+        }
+
+        Ok(FullBlockInformation {
+            block: block.block,
+            results: results?,
+            validators: validators?,
+            transactions,
+        })
+    }
+
+    #[instrument(skip(self), err(Display))]
+    pub async fn get_basic_block_details(
+        &self,
+        height: u32,
+    ) -> Result<block::Response, ScraperError> {
+        debug!("getting basic block details");
+
+        self.inner
+            .block(height)
+            .await
+            .map_err(|source| ScraperError::BlockQueryFailure { height, source })
+    }
+
+    #[instrument(skip(self), err(Display))]
+    pub async fn get_block_results(
+        &self,
+        height: u32,
+    ) -> Result<block_results::Response, ScraperError> {
+        debug!("getting block results");
+
+        self.inner
+            .block_results(height)
+            .await
+            .map_err(|source| ScraperError::BlockResultsQueryFailure { height, source })
+    }
+
+    pub(crate) async fn current_block_height(&self) -> Result<u64, ScraperError> {
+        debug!("getting current block height");
+
+        let info = self
+            .inner
+            .abci_info()
+            .await
+            .map_err(|source| ScraperError::AbciInfoQueryFailure { source })?;
+        Ok(info.last_block_height.value())
+    }
+
+    async fn get_transaction_results(
+        &self,
+        raw: &[Vec<u8>],
+    ) -> Result<Vec<tx::Response>, ScraperError> {
+        let ordered_results = Arc::new(Mutex::new(BTreeMap::new()));
+
+        // "Data is just a wrapper for a list of transactions, where transactions are arbitrary byte arrays"
+        // source: https://github.com/tendermint/spec/blob/d46cd7f573a2c6a2399fcab2cde981330aa63f37/spec/core/data_structures.md#data
+        //
+        // I hate that zip as much as you, dear reader, but for some reason the compiler didn't let me remove the `move`
+        futures::stream::iter(
+            raw.iter()
+                .map(tx_hash)
+                .enumerate()
+                .zip(std::iter::repeat(ordered_results.clone())),
+        )
+        .for_each_concurrent(4, |((id, tx_hash), ordered_results)| async move {
+            let res = self.get_transaction_result(tx_hash).await;
+            ordered_results.lock().await.insert(id, res);
+        })
+        .await;
+
+        // safety the futures have completed so we MUST have the only arc reference
+        #[allow(clippy::unwrap_used)]
+        let inner = Arc::into_inner(ordered_results).unwrap().into_inner();
+
+        // BTreeMap is ordered by its keys so we're guaranteed to get txs in correct order
+        inner.into_values().collect()
+    }
+
+    #[instrument(skip(self, tx_hash), fields(tx_hash = %tx_hash), err(Display))]
+    async fn get_transaction_result(&self, tx_hash: Hash) -> Result<tx::Response, ScraperError> {
+        debug!("getting tx results");
+
+        self.inner
+            .tx(tx_hash, false)
+            .await
+            .map_err(|source| ScraperError::TxResultsQueryFailure {
+                hash: tx_hash,
+                source,
+            })
+    }
+
+    #[instrument(skip(self))]
+    pub async fn get_validators_details(
+        &self,
+        height: u32,
+    ) -> Result<validators::Response, ScraperError> {
+        debug!("getting validators set");
+
+        self.inner
+            .validators(height, Paging::All)
+            .await
+            .map_err(|source| ScraperError::ValidatorsQueryFailure { height, source })
+    }
+}
@@ -0,0 +1,210 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::BlockProcessor;
+use crate::block_requester::BlockRequester;
+use crate::error::ScraperError;
+use crate::modules::{BlockModule, MsgModule, TxModule};
+use crate::rpc_client::RpcClient;
+use crate::scraper::subscriber::{run_websocket_driver, ChainSubscriber};
+use crate::storage::ScraperStorage;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tendermint_rpc::WebSocketClientDriver;
+use tokio::sync::mpsc::{channel, unbounded_channel};
+use tokio::sync::Notify;
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+use tracing::info;
+use url::Url;
+
+mod subscriber;
+
+pub struct Config {
+    /// Url to the websocket endpoint of a validator, for example `wss://rpc.nymtech.net/websocket`
+    pub websocket_url: Url,
+
+    /// Url to the rpc endpoint of a validator, for example `https://rpc.nymtech.net/`
+    pub rpc_url: Url,
+
+    pub database_path: PathBuf,
+}
+
+pub struct NyxdScraperBuilder {
+    config: Config,
+
+    block_modules: Vec<Box<dyn BlockModule + Send>>,
+    tx_modules: Vec<Box<dyn TxModule + Send>>,
+    msg_modules: Vec<Box<dyn MsgModule + Send>>,
+}
+
+impl NyxdScraperBuilder {
+    pub async fn build_and_start(self) -> Result<NyxdScraper, ScraperError> {
+        let scraper = NyxdScraper::new(self.config).await?;
+
+        let (processing_tx, processing_rx) = unbounded_channel();
+        let (req_tx, req_rx) = channel(5);
+
+        let rpc_client = RpcClient::new(&scraper.config.rpc_url)?;
+
+        // create the tasks
+        let block_requester = BlockRequester::new(
+            scraper.cancel_token.clone(),
+            rpc_client.clone(),
+            req_rx,
+            processing_tx.clone(),
+        );
+        let mut block_processor = BlockProcessor::new(
+            scraper.cancel_token.clone(),
+            scraper.startup_sync.clone(),
+            processing_rx,
+            req_tx,
+            scraper.storage.clone(),
+            rpc_client,
+        )
+        .await?;
+        block_processor.set_block_modules(self.block_modules);
+        block_processor.set_tx_modules(self.tx_modules);
+        block_processor.set_msg_modules(self.msg_modules);
+
+        let mut chain_subscriber = ChainSubscriber::new(
+            &scraper.config.websocket_url,
+            scraper.cancel_token.clone(),
+            processing_tx,
+        )
+        .await?;
+        let ws_driver = chain_subscriber.ws_driver();
+
+        scraper.start_tasks(
+            block_requester,
+            block_processor,
+            chain_subscriber,
+            ws_driver,
+        );
+
+        Ok(scraper)
+    }
+
+    pub fn new(config: Config) -> Self {
+        NyxdScraperBuilder {
+            config,
+            block_modules: vec![],
+            tx_modules: vec![],
+            msg_modules: vec![],
+        }
+    }
+
+    pub fn with_block_module<M: BlockModule + Send + 'static>(mut self, module: M) -> Self {
+        self.block_modules.push(Box::new(module));
+        self
+    }
+
+    pub fn with_tx_module<M: TxModule + Send + 'static>(mut self, module: M) -> Self {
+        self.tx_modules.push(Box::new(module));
+        self
+    }
+
+    pub fn with_msg_module<M: MsgModule + Send + 'static>(mut self, module: M) -> Self {
+        self.msg_modules.push(Box::new(module));
+        self
+    }
+}
+
+pub struct NyxdScraper {
+    config: Config,
+
+    task_tracker: TaskTracker,
+    cancel_token: CancellationToken,
+    startup_sync: Arc<Notify>,
+    pub storage: ScraperStorage,
+}
+
+impl NyxdScraper {
+    pub fn builder(config: Config) -> NyxdScraperBuilder {
+        NyxdScraperBuilder::new(config)
+    }
+
+    pub async fn new(config: Config) -> Result<Self, ScraperError> {
+        let storage = ScraperStorage::init(&config.database_path).await?;
+
+        Ok(NyxdScraper {
+            config,
+            task_tracker: TaskTracker::new(),
+            cancel_token: CancellationToken::new(),
+            startup_sync: Arc::new(Default::default()),
+            storage,
+        })
+    }
+
+    fn start_tasks(
+        &self,
+        mut block_requester: BlockRequester,
+        mut block_processor: BlockProcessor,
+        mut chain_subscriber: ChainSubscriber,
+        ws_driver: WebSocketClientDriver,
+    ) {
+        self.task_tracker
+            .spawn(async move { block_requester.run().await });
+        self.task_tracker
+            .spawn(async move { block_processor.run().await });
+        self.task_tracker
+            .spawn(async move { chain_subscriber.run().await });
+        self.task_tracker
+            .spawn(run_websocket_driver(ws_driver, self.cancel_token.clone()));
+        self.task_tracker.close();
+    }
+
+    pub async fn start(&self) -> Result<(), ScraperError> {
+        let (processing_tx, processing_rx) = unbounded_channel();
+        let (req_tx, req_rx) = channel(5);
+
+        let rpc_client = RpcClient::new(&self.config.rpc_url)?;
+
+        // create the tasks
+        let block_requester = BlockRequester::new(
+            self.cancel_token.clone(),
+            rpc_client.clone(),
+            req_rx,
+            processing_tx.clone(),
+        );
+        let block_processor = BlockProcessor::new(
+            self.cancel_token.clone(),
+            self.startup_sync.clone(),
+            processing_rx,
+            req_tx,
+            self.storage.clone(),
+            rpc_client,
+        )
+        .await?;
+        let mut chain_subscriber = ChainSubscriber::new(
+            &self.config.websocket_url,
+            self.cancel_token.clone(),
+            processing_tx,
+        )
+        .await?;
+        let ws_driver = chain_subscriber.ws_driver();
+
+        // spawn them
+        self.start_tasks(
+            block_requester,
+            block_processor,
+            chain_subscriber,
+            ws_driver,
+        );
+
+        Ok(())
+    }
+
+    pub async fn wait_for_startup_sync(&self) {
+        info!("awaiting startup chain sync");
+        self.startup_sync.notified().await
+    }
+
+    pub async fn stop(self) {
+        info!("stopping the chain scrapper");
+        assert!(self.task_tracker.is_closed());
+
+        self.cancel_token.cancel();
+        self.task_tracker.wait().await
+    }
+}
@@ -0,0 +1,129 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::BlockToProcess;
+use crate::error::ScraperError;
+use tendermint_rpc::event::Event;
+use tendermint_rpc::query::EventType;
+use tendermint_rpc::{SubscriptionClient, WebSocketClient, WebSocketClientDriver};
+use tokio::sync::mpsc::UnboundedSender;
+use tokio_stream::StreamExt;
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info, warn};
+use url::Url;
+
+const MAX_FAILURES: usize = 10;
+
+pub struct ChainSubscriber {
+    cancel: CancellationToken,
+    block_sender: UnboundedSender<BlockToProcess>,
+
+    websocket_client: WebSocketClient,
+    websocket_driver: Option<WebSocketClientDriver>,
+}
+
+impl ChainSubscriber {
+    pub async fn new(
+        websocket_endpoint: &Url,
+        cancel: CancellationToken,
+        block_sender: UnboundedSender<BlockToProcess>,
+    ) -> Result<Self, ScraperError> {
+        // sure, we could have just used websocket client entirely, but let's keep the logic for
+        // getting current blocks and historical blocks completely separate with the dual connection
+        let (client, driver) = WebSocketClient::new(websocket_endpoint.as_str())
+            .await
+            .map_err(|source| ScraperError::WebSocketConnectionFailure {
+                url: websocket_endpoint.to_string(),
+                source,
+            })?;
+
+        Ok(ChainSubscriber {
+            cancel,
+            block_sender,
+            websocket_client: client,
+            websocket_driver: Some(driver),
+        })
+    }
+
+    fn handle_new_event(&mut self, event: Event) -> Result<(), ScraperError> {
+        if let Err(err) = self.block_sender.send(event.try_into()?) {
+            // this error has nothing to do with the websocket or chain
+            error!("failed to send block for processing: {err} - are we shutting down?")
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn run(&mut self) -> Result<(), ScraperError> {
+        let _drop_guard = self.cancel.clone().drop_guard();
+
+        info!("creating chain subscription");
+        let mut subs = self
+            .websocket_client
+            .subscribe(EventType::NewBlock.into())
+            .await
+            .map_err(|source| ScraperError::ChainSubscriptionFailure { source })?;
+
+        let mut failures = 0;
+
+        info!("starting processing loop");
+        loop {
+            tokio::select! {
+                _ = self.cancel.cancelled() => {
+                    info!("received cancellation token");
+                    break
+                }
+                maybe_event = subs.next() => {
+                    let Some(maybe_event) = maybe_event else {
+                        warn!("stopped receiving new events");
+                        break;
+                    };
+                    match maybe_event {
+                        Ok(event) => {
+                            if let Err(err) = self.handle_new_event(event) {
+                                error!("failed to process received block: {err}");
+                                failures += 1
+                            } else {
+                                failures = 0;
+                            }
+                        }
+                        Err(err) => {
+                            error!("failed to receive a valid subscription event: {err}");
+                            failures += 1
+                        }
+                    }
+                    if failures >= MAX_FAILURES {
+                        // note: the drop_guard will get dropped and thus cause a shutdown
+                        return Err(ScraperError::MaximumSubscriptionFailures);
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn ws_driver(&mut self) -> WebSocketClientDriver {
+        #[allow(clippy::expect_used)]
+        self.websocket_driver
+            .take()
+            .expect("websocket driver has already been started!")
+    }
+}
+
+pub async fn run_websocket_driver(driver: WebSocketClientDriver, cancel: CancellationToken) {
+    info!("starting websocket driver");
+    tokio::select! {
+        _ = cancel.cancelled() => {
+            info!("received cancellation token")
+        }
+        res = driver.run() => {
+            match res {
+                Ok(_) => info!("our websocket driver has finished execution"),
+                Err(err) => {
+                    // TODO: in the future just attempt to reconnect
+                    error!("our websocket driver has errored out: {err}")
+                }
+            }
+            cancel.cancel()
+        }
+    }
+}
@@ -0,0 +1,2 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,352 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::storage::models::{CommitSignature, Validator};
+use sqlx::types::time::OffsetDateTime;
+use sqlx::{Executor, Sqlite};
+use tracing::{instrument, trace};
+
+#[derive(Clone)]
+pub(crate) struct StorageManager {
+    pub(crate) connection_pool: sqlx::SqlitePool,
+}
+
+impl StorageManager {
+    pub(crate) async fn set_initial_metadata(&self) -> Result<(), sqlx::Error> {
+        if sqlx::query("SELECT * from metadata")
+            .fetch_optional(&self.connection_pool)
+            .await?
+            .is_none()
+        {
+            sqlx::query("INSERT INTO metadata (id, last_processed_height) VALUES (0, 0)")
+                .execute(&self.connection_pool)
+                .await?;
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn get_first_block_height_after(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, sqlx::Error> {
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT height
+                FROM block
+                WHERE timestamp > ?
+                ORDER BY timestamp
+                LIMIT 1
+            "#,
+            time
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+
+        if let Some(row) = maybe_record {
+            Ok(row.height)
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_last_block_height_before(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, sqlx::Error> {
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT height
+                FROM block
+                WHERE timestamp < ?
+                ORDER BY timestamp DESC
+                LIMIT 1
+            "#,
+            time
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+
+        if let Some(row) = maybe_record {
+            Ok(row.height)
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub(crate) async fn get_signed_between(
+        &self,
+        consensus_address: &str,
+        start_height: i64,
+        end_height: i64,
+    ) -> Result<i32, sqlx::Error> {
+        let count = sqlx::query!(
+            r#"
+                SELECT COUNT(*) as count FROM pre_commit
+                WHERE 
+                    validator_address == ?
+                    AND height >= ? 
+                    AND height <= ?
+            "#,
+            consensus_address,
+            start_height,
+            end_height
+        )
+        .fetch_one(&self.connection_pool)
+        .await?
+        .count;
+
+        Ok(count)
+    }
+
+    pub(crate) async fn get_precommit(
+        &self,
+        consensus_address: &str,
+        height: i64,
+    ) -> Result<Option<CommitSignature>, sqlx::Error> {
+        sqlx::query_as(
+            r#"
+                SELECT * FROM pre_commit
+                WHERE validator_address = ? 
+                AND height = ?
+            "#,
+        )
+        .bind(consensus_address)
+        .bind(height)
+        .fetch_optional(&self.connection_pool)
+        .await
+    }
+
+    pub(crate) async fn get_block_validators(
+        &self,
+        height: i64,
+    ) -> Result<Vec<Validator>, sqlx::Error> {
+        sqlx::query_as!(
+            Validator,
+            r#"
+                SELECT * FROM validator 
+                WHERE EXISTS (
+                    SELECT 1 FROM pre_commit
+                    WHERE height == ?
+                    AND pre_commit.validator_address = validator.consensus_address
+                )
+            "#,
+            height
+        )
+        .fetch_all(&self.connection_pool)
+        .await
+    }
+
+    pub(crate) async fn get_validators(&self) -> Result<Vec<Validator>, sqlx::Error> {
+        sqlx::query_as("SELECT * FROM validator")
+            .fetch_all(&self.connection_pool)
+            .await
+    }
+
+    pub(crate) async fn get_last_processed_height(&self) -> Result<i64, sqlx::Error> {
+        let maybe_record = sqlx::query!(
+            r#"
+                SELECT last_processed_height FROM metadata
+            "#
+        )
+        .fetch_optional(&self.connection_pool)
+        .await?;
+
+        if let Some(row) = maybe_record {
+            Ok(row.last_processed_height)
+        } else {
+            Ok(-1)
+        }
+    }
+}
+
+// make those generic over executor so that they could be performed over connection pool and a tx
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_validator<'a, E>(
+    consensus_address: String,
+    consensus_pubkey: String,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("insert validator");
+
+    sqlx::query!(
+        r#"
+            INSERT INTO validator (consensus_address, consensus_pubkey)
+            VALUES (?, ?)
+            ON CONFLICT DO NOTHING
+        "#,
+        consensus_address,
+        consensus_pubkey
+    )
+    .execute(executor)
+    .await?;
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_block<'a, E>(
+    height: i64,
+    hash: String,
+    num_txs: u32,
+    total_gas: i64,
+    proposer_address: String,
+    timestamp: OffsetDateTime,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("insert block");
+
+    sqlx::query!(
+        r#"
+            INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)
+            VALUES (?, ?, ?, ?, ?, ?)
+            ON CONFLICT DO NOTHING
+        "#,
+        height,
+        hash,
+        num_txs,
+        total_gas,
+        proposer_address,
+        timestamp
+    )
+    .execute(executor)
+    .await?;
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_precommit<'a, E>(
+    validator_address: String,
+    height: i64,
+    timestamp: OffsetDateTime,
+    voting_power: i64,
+    proposer_priority: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("insert precommit");
+
+    sqlx::query!(
+        r#"
+            INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT (validator_address, timestamp) DO NOTHING
+        "#,
+        validator_address,
+        height,
+        timestamp,
+        voting_power,
+        proposer_priority
+    )
+    .execute(executor)
+    .await?;
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn insert_transaction<'a, E>(
+    hash: String,
+    height: i64,
+    index: i64,
+    success: bool,
+    messages: i64,
+    memo: String,
+    gas_wanted: i64,
+    gas_used: i64,
+    raw_log: String,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("insert transaction");
+
+    sqlx::query!(
+        r#"
+            INSERT INTO "transaction" (hash, height, "index", success, num_messages, memo, gas_wanted, gas_used, raw_log)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+               ON CONFLICT (hash) DO UPDATE
+               SET height = excluded.height,
+               "index" = excluded."index",
+               success = excluded.success,
+               num_messages = excluded.num_messages,
+               memo = excluded.memo,
+               gas_wanted = excluded.gas_wanted,
+               gas_used = excluded.gas_used,
+               raw_log = excluded.raw_log
+        "#,
+            hash,
+            height,
+            index,
+            success,
+            messages,
+            memo,
+            gas_wanted,
+            gas_used,
+            raw_log,
+    )
+        .execute(executor)
+        .await?;
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn insert_message<'a, E>(
+    transaction_hash: String,
+    index: i64,
+    typ: String,
+    height: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("insert message");
+
+    sqlx::query!(
+        r#"
+            INSERT INTO message (transaction_hash, "index", type, height)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT (transaction_hash, "index") DO UPDATE
+                SET height = excluded.height,
+                type = excluded.type
+        "#,
+        transaction_hash,
+        index,
+        typ,
+        height
+    )
+    .execute(executor)
+    .await?;
+
+    Ok(())
+}
+
+#[instrument(skip(executor))]
+pub(crate) async fn update_last_processed<'a, E>(
+    height: i64,
+    executor: E,
+) -> Result<(), sqlx::Error>
+where
+    E: Executor<'a, Database = Sqlite>,
+{
+    trace!("update_last_processed");
+
+    sqlx::query!("UPDATE metadata SET last_processed_height = ?", height)
+        .execute(executor)
+        .await?;
+
+    Ok(())
+}
@@ -0,0 +1,327 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::block_processor::types::{FullBlockInformation, ParsedTransactionResponse};
+use crate::error::ScraperError;
+use crate::storage::manager::{
+    insert_block, insert_message, insert_precommit, insert_transaction, insert_validator,
+    update_last_processed, StorageManager,
+};
+use crate::storage::models::{CommitSignature, Validator};
+use sqlx::types::time::OffsetDateTime;
+use sqlx::{ConnectOptions, Sqlite, Transaction};
+use std::fmt::Debug;
+use std::path::Path;
+use tendermint::block::{Commit, CommitSig};
+use tendermint::Block;
+use tendermint_rpc::endpoint::validators;
+use tracing::{debug, error, info, instrument, trace, warn};
+
+mod helpers;
+mod manager;
+pub mod models;
+
+pub type StorageTransaction = Transaction<'static, Sqlite>;
+
+#[derive(Clone)]
+pub struct ScraperStorage {
+    pub(crate) manager: StorageManager,
+}
+
+impl ScraperStorage {
+    #[instrument]
+    pub async fn init<P: AsRef<Path> + Debug>(database_path: P) -> Result<Self, ScraperError> {
+        let mut opts = sqlx::sqlite::SqliteConnectOptions::new()
+            .filename(database_path)
+            .create_if_missing(true);
+
+        // TODO: do we want auto_vacuum ?
+
+        opts.disable_statement_logging();
+
+        let connection_pool = match sqlx::SqlitePool::connect_with(opts).await {
+            Ok(db) => db,
+            Err(err) => {
+                error!("Failed to connect to SQLx database: {err}");
+                return Err(err.into());
+            }
+        };
+
+        if let Err(err) = sqlx::migrate!("./sql_migrations")
+            .run(&connection_pool)
+            .await
+        {
+            error!("Failed to initialize SQLx database: {err}");
+            return Err(err.into());
+        }
+
+        info!("Database migration finished!");
+
+        let manager = StorageManager { connection_pool };
+        manager.set_initial_metadata().await?;
+
+        let storage = ScraperStorage { manager };
+
+        Ok(storage)
+    }
+
+    #[instrument(skip_all)]
+    pub async fn begin_processing_tx(&self) -> Result<StorageTransaction, ScraperError> {
+        debug!("starting storage tx");
+        self.manager
+            .connection_pool
+            .begin()
+            .await
+            .map_err(|source| ScraperError::StorageTxBeginFailure { source })
+    }
+
+    pub async fn get_first_block_height_after(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, ScraperError> {
+        Ok(self.manager.get_first_block_height_after(time).await?)
+    }
+
+    pub async fn get_last_block_height_before(
+        &self,
+        time: OffsetDateTime,
+    ) -> Result<Option<i64>, ScraperError> {
+        Ok(self.manager.get_last_block_height_before(time).await?)
+    }
+
+    pub async fn get_blocks_between(
+        &self,
+        start_time: OffsetDateTime,
+        end_time: OffsetDateTime,
+    ) -> Result<i64, ScraperError> {
+        let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
+            return Ok(0);
+        };
+        let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
+            return Ok(0);
+        };
+
+        Ok(block_end - block_start)
+    }
+
+    pub async fn get_signed_between(
+        &self,
+        consensus_address: &str,
+        start_height: i64,
+        end_height: i64,
+    ) -> Result<i32, ScraperError> {
+        Ok(self
+            .manager
+            .get_signed_between(consensus_address, start_height, end_height)
+            .await?)
+    }
+
+    pub async fn get_signed_between_times(
+        &self,
+        consensus_address: &str,
+        start_time: OffsetDateTime,
+        end_time: OffsetDateTime,
+    ) -> Result<i32, ScraperError> {
+        let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
+            return Ok(0);
+        };
+        let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
+            return Ok(0);
+        };
+
+        self.get_signed_between(consensus_address, block_start, block_end)
+            .await
+    }
+
+    pub async fn get_precommit(
+        &self,
+        consensus_address: &str,
+        height: i64,
+    ) -> Result<Option<CommitSignature>, ScraperError> {
+        Ok(self
+            .manager
+            .get_precommit(consensus_address, height)
+            .await?)
+    }
+
+    pub async fn get_block_signers(&self, height: i64) -> Result<Vec<Validator>, ScraperError> {
+        Ok(self.manager.get_block_validators(height).await?)
+    }
+
+    pub async fn get_all_known_validators(&self) -> Result<Vec<Validator>, ScraperError> {
+        Ok(self.manager.get_validators().await?)
+    }
+
+    pub async fn get_last_processed_height(&self) -> Result<i64, ScraperError> {
+        Ok(self.manager.get_last_processed_height().await?)
+    }
+}
+
+pub async fn persist_block(
+    block: &FullBlockInformation,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    let total_gas = crate::helpers::tx_gas_sum(&block.transactions);
+
+    // SANITY CHECK: make sure the block proposer is present in the validator set
+    block.ensure_proposer()?;
+
+    // persist validators
+    persist_validators(&block.validators, tx).await?;
+
+    // persist block data
+    persist_block_data(&block.block, total_gas, tx).await?;
+
+    // persist commits
+    if let Some(commit) = &block.block.last_commit {
+        persist_commits(commit, &block.validators, tx).await?;
+    } else {
+        warn!("no commits for block {}", block.block.header.height)
+    }
+
+    // persist txs
+    persist_txs(&block.transactions, tx).await?;
+
+    // persist messages (inside the transactions)
+    persist_messages(&block.transactions, tx).await?;
+
+    update_last_processed(block.block.header.height.into(), tx).await?;
+
+    Ok(())
+}
+
+async fn persist_validators(
+    validators: &validators::Response,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting {} validators", validators.total);
+    for validator in &validators.validators {
+        let consensus_address = crate::helpers::validator_consensus_address(validator.address)?;
+        let consensus_pubkey = crate::helpers::validator_pubkey_to_bech32(validator.pub_key)?;
+
+        insert_validator(
+            consensus_address.to_string(),
+            consensus_pubkey.to_string(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+async fn persist_block_data(
+    block: &Block,
+    total_gas: i64,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    let proposer_address =
+        crate::helpers::validator_consensus_address(block.header.proposer_address)?.to_string();
+
+    insert_block(
+        block.header.height.into(),
+        block.header.hash().to_string(),
+        block.data.len() as u32,
+        total_gas,
+        proposer_address,
+        block.header.time.into(),
+        tx,
+    )
+    .await?;
+    Ok(())
+}
+
+async fn persist_commits(
+    commits: &Commit,
+    validators: &validators::Response,
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting up to {} commits", commits.signatures.len());
+    let height: i64 = commits.height.into();
+
+    for commit_sig in &commits.signatures {
+        let (validator_id, timestamp, signature) = match commit_sig {
+            CommitSig::BlockIdFlagAbsent => {
+                trace!("absent signature");
+                continue;
+            }
+            CommitSig::BlockIdFlagCommit {
+                validator_address,
+                timestamp,
+                signature,
+            } => (validator_address, timestamp, signature),
+            CommitSig::BlockIdFlagNil {
+                validator_address,
+                timestamp,
+                signature,
+            } => (validator_address, timestamp, signature),
+        };
+
+        let validator = crate::helpers::validator_info(*validator_id, validators)?;
+        let validator_address = crate::helpers::validator_consensus_address(*validator_id)?;
+
+        if signature.is_none() {
+            warn!("empty signature for {validator_address} at height {height}");
+            continue;
+        }
+
+        insert_precommit(
+            validator_address.to_string(),
+            height,
+            (*timestamp).into(),
+            validator.power.into(),
+            validator.proposer_priority.value(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+async fn persist_txs(
+    txs: &[ParsedTransactionResponse],
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting {} txs", txs.len());
+
+    for chain_tx in txs {
+        insert_transaction(
+            chain_tx.hash.to_string(),
+            chain_tx.height.into(),
+            chain_tx.index as i64,
+            chain_tx.tx_result.code.is_ok(),
+            chain_tx.tx.body.messages.len() as i64,
+            chain_tx.tx.body.memo.clone(),
+            chain_tx.tx_result.gas_wanted,
+            chain_tx.tx_result.gas_used,
+            chain_tx.tx_result.log.clone(),
+            &mut *tx,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+async fn persist_messages(
+    txs: &[ParsedTransactionResponse],
+    tx: &mut StorageTransaction,
+) -> Result<(), ScraperError> {
+    debug!("persisting messages");
+
+    for chain_tx in txs {
+        for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() {
+            insert_message(
+                chain_tx.hash.to_string(),
+                index as i64,
+                msg.type_url.clone(),
+                chain_tx.height.into(),
+                &mut *tx,
+            )
+            .await?
+        }
+    }
+
+    Ok(())
+}
@@ -0,0 +1,30 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use sqlx::types::time::OffsetDateTime;
+use sqlx::FromRow;
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)]
+pub struct Validator {
+    pub consensus_address: String,
+    pub consensus_pubkey: String,
+}
+
+#[derive(Debug, Clone, FromRow)]
+pub struct Block {
+    pub height: i64,
+    pub hash: String,
+    pub num_txs: u32,
+    pub total_gas: i64,
+    pub proposer_address: String,
+    pub timestamp: OffsetDateTime,
+}
+
+#[derive(Debug, Clone, FromRow)]
+pub struct CommitSignature {
+    pub height: i64,
+    pub validator_address: String,
+    pub voting_power: i64,
+    pub proposer_priority: i64,
+    pub timestamp: OffsetDateTime,
+}
@@ -15,6 +15,6 @@ log = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "chrono"]}
+sqlx = { workspace = true, features = ["runtime-tokio-rustls", "chrono"]}
 thiserror = { workspace = true }
 tokio = { version = "1.24.1", features = [ "time" ] }
@@ -116,8 +116,8 @@ impl TaskManager {
    }

    #[cfg(not(target_arch = "wasm32"))]
-    pub async fn catch_interrupt(mut self) -> Result<(), SentError> {
-        let res = crate::wait_for_signal_and_error(&mut self).await;
+    pub async fn catch_interrupt(&mut self) -> Result<(), SentError> {
+        let res = crate::wait_for_signal_and_error(self).await;

        log::info!("Sending shutdown");
        self.signal_shutdown().ok();
@@ -629,7 +629,7 @@ impl TaskHandle {
    #[cfg(not(target_arch = "wasm32"))]
    pub async fn wait_for_shutdown(self) -> Result<(), SentError> {
        match self {
-            TaskHandle::Internal(task_manager) => task_manager.catch_interrupt().await,
+            TaskHandle::Internal(mut task_manager) => task_manager.catch_interrupt().await,
            TaskHandle::External(mut task_client) => {
                task_client.recv().await;
                Ok(())
@@ -16,10 +16,12 @@ base64 = "0.21.3"
 # version mismatch with x25519-dalek/curve25519-dalek that is resolved in the
 # latest commit. So pick that for now.
 x25519-dalek = "2.0.0"
-defguard_wireguard_rs = { git = "https://github.com/neacsu/wireguard-rs.git", rev = "c2cd0c1119f699f4bc43f5e6ffd6fc242caa42ed" }
 ip_network = "0.4.1"
 log.workspace = true
 nym-network-defaults = { path = "../network-defaults" }
 nym-task = { path = "../task" }
 nym-wireguard-types = { path = "../wireguard-types" }
 tokio = { workspace = true, features = ["rt-multi-thread", "net", "io-util"] }
+
+[target."cfg(target_os = \"linux\")".dependencies]
+defguard_wireguard_rs = { git = "https://github.com/neacsu/wireguard-rs.git", rev = "c2cd0c1119f699f4bc43f5e6ffd6fc242caa42ed" }
@@ -5,26 +5,20 @@

 pub mod setup;

-use nym_wireguard_types::registration::GatewayClientRegistry;
-use std::sync::Arc;
-
-// Currently the module related to setting up the virtual network device is platform specific.
-#[cfg(target_os = "linux")]
-use crate::setup::{peer_allowed_ips, peer_static_public_key, PRIVATE_KEY};
-use defguard_wireguard_rs::WGApi;
-#[cfg(target_os = "linux")]
-use defguard_wireguard_rs::{
-    host::Peer, key::Key, net::IpAddrMask, InterfaceConfiguration, WireguardInterfaceApi,
-};
-#[cfg(target_os = "linux")]
-use nym_network_defaults::{WG_PORT, WG_TUN_DEVICE_ADDRESS};
-
 /// Start wireguard device
 #[cfg(target_os = "linux")]
 pub async fn start_wireguard(
    mut task_client: nym_task::TaskClient,
-    _gateway_client_registry: Arc<GatewayClientRegistry>,
-) -> Result<WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
+    _gateway_client_registry: std::sync::Arc<
+        nym_wireguard_types::registration::GatewayClientRegistry,
+    >,
+) -> Result<defguard_wireguard_rs::WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
+    use crate::setup::{peer_allowed_ips, peer_static_public_key, PRIVATE_KEY};
+    use defguard_wireguard_rs::{
+        host::Peer, key::Key, net::IpAddrMask, InterfaceConfiguration, WGApi, WireguardInterfaceApi,
+    };
+    use nym_network_defaults::{WG_PORT, WG_TUN_DEVICE_ADDRESS};
+
    let ifname = String::from("wg0");
    let wgapi = WGApi::new(ifname.clone(), false)?;
    wgapi.create_interface()?;
@@ -48,10 +42,8 @@ pub async fn start_wireguard(

    Ok(wgapi)
 }
+
 #[cfg(not(target_os = "linux"))]
-pub async fn start_wireguard(
-    _task_client: nym_task::TaskClient,
-    _gateway_client_registry: Arc<GatewayClientRegistry>,
-) -> Result<WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
-    todo!("WireGuard is currently only supported on Linux")
+pub async fn start_wireguard() {
+    todo!("WireGuard is currently only supported on Linux");
 }
@@ -4,9 +4,10 @@ version = "0.1.0"
 edition = "2021"
 build = "build.rs"
 links = "cpucycles"
+license = "LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT"

 [dependencies]
 libc = "0.2.140"

 [build-dependencies]
-cfg-if = "1"
+cfg-if = "1"
@@ -114,6 +114,7 @@ allow = [
    "CC0-1.0",
    "Unicode-DFS-2016",
    "OpenSSL",
+    "Zlib",
 ]
 # List of explicitly disallowed licenses
 # See https://spdx.org/licenses/ for list of possible licenses
@@ -51,10 +51,19 @@ assets_version = "3.0.0" # do not edit: managed by `mdbook-admonish install`
 minimum_rust_version = "1.66"
 wallet_release_version = "1.2.8"

+# nym-vpn related variables
+nym_vpn_latest_binary_url = "https://github.com/nymtech/nym/releases/tag/nym-vpn-alpha-0.0.3"
+nym_vpn_form_url = "https://opnform.com/forms/nymvpn-user-research-at-37c3-yccqko-2"
+
 [preprocessor.last-changed]
 command = "mdbook-last-changed"
 renderer = ["html"]

+# used for grabbing output of binary commands for automation
+# https://github.com/FauconFan/mdbook-cmdrun
+[preprocessor.cmdrun]
+
+
 #########
 # BUILD #
 #########
@@ -18,13 +18,23 @@

 # User Manuals

+- [NymVPN alpha](nymvpn/intro.md)
+    - [GUI](nymvpn/gui.md)
+        - [Linux](nymvpn/gui-linux.md)
+        - [MacOS](nymvpn/gui-mac.md)
+    - [CLI](nymvpn/cli.md)
+        - [Linux](nymvpn/cli-linux.md)
+        - [MacOS](nymvpn/cli-mac.md)
+    - [Testing](nymvpn/testing.md)
+    - [Troubleshooting](nymvpn/troubleshooting.md)
+    - [NymVPN FAQ](nymvpn/faq.md)
 - [NymConnect X Monero](tutorials/monero.md)
 - [NymConnect X Matrix](tutorials/matrix.md)
 - [NymConnect X Telegram](tutorials/telegram.md)
 - [NymConnect X Electrum](tutorials/electrum.md)
 - [NymConnect X Firo wallet](tutorials/firo.md)

-# Code Examples 
+# Code Examples

 - [Custom Service Providers](examples/custom-services.md)
 - [Apps Using Network Requesters](examples/using-nrs.md)
@@ -49,9 +59,9 @@
 		- [Preparing Your Service](tutorials/cosmos-service/service.md)
 		- [Preparing Your Service pt2](tutorials/cosmos-service/service-src.md)
 		- [Querying the Chain](tutorials/cosmos-service/querying.md)
-   
+
 - [Typescript](tutorials/typescript.md)
-    - [Simple Service Provider](tutorials/simple-service-provider/simple-service-provider.md) 
+    - [Simple Service Provider](tutorials/simple-service-provider/simple-service-provider.md)
      	- [Tutorial Overview](tutorials/simple-service-provider/overview.md)
      	- [Preparing Your User Client Environment](tutorials/simple-service-provider/preparating-env.md)
      	- [Building Your User Client](tutorials/simple-service-provider/user-client.md)
@@ -83,7 +93,7 @@
 - [Community Applications and Guides](community-resources/community-applications-and-guides.md)
 - [Change Service Grantee Information](info-request.md)

--- 
+---
 # Misc.
 - [Code of Conduct](coc.md)
 - [Licensing](licensing.md)
@@ -0,0 +1,116 @@
+# NymVPN alpha CLI: Guide for GNU/Linux
+
+```admonish info
+NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
+```
+
+## Installation
+
+> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
+
+1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for Debian based Linux
+2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
+```sh
+echo "<SHA_STRING>" | shasum -a 256 -c
+
+# choose a correct one according to your binary, this is just an example
+echo "0e4abb461e86b2c168577e0294112a3bacd3a24bf8565b49783bfebd9b530e23  nym-vpn-cli_0.1.0_ubuntu-22.04_amd64.zip" | shasum -a 256 -c
+```
+3. Extract files with `unzip` command or manually as you are used to
+4. Make executable by running:
+```sh
+chmod u+x ./nym-vpn-cli
+```
+5. Create Sandbox environment config file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the same directory as your NymVPN binaries by running:
+```sh
+curl -o sandbox.env -L https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env
+```
+
+## Run NymVPN
+
+**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
+
+Make sure your terminal is open in the same directory as your `nym-vpn-cli` binary.
+
+1. Go to [nymvpn.com/en/alpha](https://nymvpn.com/en/alpha) to get the entire command with all the needed arguments' values and your wireguard private key for testing purposes
+2. Run it as root with `sudo` - the command will look like this with specified arguments:
+```sh
+sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WIREGUARD_IP>
+```
+3. To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways) and pick one
+4. See all possibilities in [command explanation](#cli-commands-and-options) section below
+
+In case of errors, see [troubleshooting section](troubleshooting.md).
+
+### CLI Commands and Options
+
+The basic syntax of `nym-vpn-cli` is:
+```sh
+sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WG_IP>
+```
+* To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways)
+* To see all possibilities run with `--help` flag:
+```sh
+./nym-vpn-cli --help
+```
+~~~admonish example collapsible=true title="Console output"
+```
+Usage: nym-vpn-cli [OPTIONS]
+
+Options:
+  -c, --config-env-file <CONFIG_ENV_FILE>
+          Path pointing to an env file describing the network
+      --mixnet-client-path <MIXNET_CLIENT_PATH>
+          Path to the data directory of a previously initialised mixnet client, where the keys reside
+      --entry-gateway-id <ENTRY_GATEWAY_ID>
+          Mixnet public ID of the entry gateway
+      --entry-gateway-country <ENTRY_GATEWAY_COUNTRY>
+          Auto-select entry gateway by country ISO
+      --exit-router-address <EXIT_ROUTER_ADDRESS>
+          Mixnet recipient address
+      --exit-gateway-id <EXIT_GATEWAY_ID>
+
+      --exit-router-country <EXIT_ROUTER_COUNTRY>
+          Mixnet recipient address
+      --enable-wireguard
+          Enable the wireguard traffic between the client and the entry gateway
+      --private-key <PRIVATE_KEY>
+          Associated private key
+      --wg-ip <WG_IP>
+          The IP address of the wireguard interface used for the first hop to the entry gateway
+      --nym-ip <NYM_IP>
+          The IP address of the nym TUN device that wraps IP packets in sphinx packets
+      --nym-mtu <NYM_MTU>
+          The MTU of the nym TUN device that wraps IP packets in sphinx packets
+      --disable-routing
+          Disable routing all traffic through the nym TUN device. When the flag is set, the nym TUN device will be created, but to route traffic through it you will need to do it manually, e.g. ping -Itun0
+      --enable-two-hop
+          Enable two-hop mixnet traffic. This means that traffic jumps directly from entry gateway to exit gateway
+      --enable-poisson-rate
+          Enable Poisson process rate limiting of outbound traffic
+  -h, --help
+          Print help
+  -V, --version
+          Print version
+
+```
+~~~
+
+Here is a list of the options and their descriptions. Some are essential, some are more technical and not needed to be adjusted by users.
+
+**Fundamental commands and arguments**
+
+- `-c` is a path to the [Sandbox config](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) file saved as `sandbox.env`
+- `--entry-gateway-id`: paste one of the values labeled with a key `"identityKey"` (without `" "`) from [here](https://nymvpn.com/en/alpha/api/gateways)
+- `--exit-router-address`: paste one of the values labeled with a key `"address"` (without `" "`) from here [here](https://nymvpn.com/en/alpha/api/gateways)
+- `--enable-wireguard`: Enable the wireguard traffic between the client and the entry gateway. NymVPN uses Mullvad libraries for wrapping `wireguard-go` and to setup local routing rules to route all traffic to the TUN virtual network device
+- `--wg-ip`: The address of the wireguard interface, you can get it [here](https://nymvpn.com/en/alpha)
+- `--private-key`: get your private key for testing purposes [here](https://nymvpn.com/en/alpha)
+- `--enable-two-hop` is a faster setup where the traffic is routed from the client to Entry Gateway and directly to Exit Gateway (default is 5-hops)
+
+**Advanced options**
+
+- `--enable-poisson`: Enables process rate limiting of outbound traffic (disabled by default). It means that NymVPN client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
+- `--ip` is the IP address of the TUN device. That is the IP address of the local private network that is set up between local client and the Exit Gateway.
+- `--mtu`: The MTU of the TUN device. That is the max IP packet size of the local private network that is set up between local client and the Exit Gateway.
+- `--disable-routing`: Disable routing all traffic through the VPN TUN device.
@@ -0,0 +1,117 @@
+# NymVPN alpha CLI: Guide for MacOS
+
+```admonish info
+NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
+```
+
+## Preparation
+
+> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
+
+1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for MacOS
+2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
+```sh
+echo "<SHA_STRING>" | shasum -a 256 -c
+
+# choose a correct one according to your binary, this is just an example
+echo "96623ccc69bc4cc0e4e3e18528b6dae6be69f645d0a592d926a3158ce2d0c269  nym-vpn-cli_0.1.0_macos_x86_64.zip" | shasum -a 256 -c
+```
+5. Extract files with `unzip` command or manually as you are used to
+6. Make executable by running:
+```sh
+chmod u+x ./nym-vpn-cli
+```
+7. Create Sandbox environment config file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the same directory as your NymVPN binaries by running:
+```sh
+curl -L "https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env" -o sandbox.env
+```
+
+## Run NymVPN
+
+**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
+
+Make sure your terminal is open in the same directory as your `nym-vpn-cli` binary.
+
+
+1. Go to [nymvpn.com/en/alpha](https://nymvpn.com/en/alpha) to get the entire command with all the needed arguments' values and your wireguard private key for testing purposes
+2. Run it as root with `sudo` - the command will look like this with specified arguments:
+```sh
+sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WIREGUARD_IP>
+```
+3. To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways) and pick one
+4. See all possibilities in [command explanation](#cli-commands-and-options) section below
+
+In case of errors, see [troubleshooting section](troubleshooting.md).
+
+### CLI Commands and Options
+
+The basic syntax of `nym-vpn-cli` is:
+```sh
+sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WG_IP>
+```
+* To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways)
+* To see all possibilities run with `--help` flag:
+```sh
+./nym-vpn-cli --help
+```
+~~~admonish example collapsible=true title="Console output"
+```
+Usage: nym-vpn-cli [OPTIONS]
+
+Options:
+  -c, --config-env-file <CONFIG_ENV_FILE>
+          Path pointing to an env file describing the network
+      --mixnet-client-path <MIXNET_CLIENT_PATH>
+          Path to the data directory of a previously initialised mixnet client, where the keys reside
+      --entry-gateway-id <ENTRY_GATEWAY_ID>
+          Mixnet public ID of the entry gateway
+      --entry-gateway-country <ENTRY_GATEWAY_COUNTRY>
+          Auto-select entry gateway by country ISO
+      --exit-router-address <EXIT_ROUTER_ADDRESS>
+          Mixnet recipient address
+      --exit-gateway-id <EXIT_GATEWAY_ID>
+
+      --exit-router-country <EXIT_ROUTER_COUNTRY>
+          Mixnet recipient address
+      --enable-wireguard
+          Enable the wireguard traffic between the client and the entry gateway
+      --private-key <PRIVATE_KEY>
+          Associated private key
+      --wg-ip <WG_IP>
+          The IP address of the wireguard interface used for the first hop to the entry gateway
+      --nym-ip <NYM_IP>
+          The IP address of the nym TUN device that wraps IP packets in sphinx packets
+      --nym-mtu <NYM_MTU>
+          The MTU of the nym TUN device that wraps IP packets in sphinx packets
+      --disable-routing
+          Disable routing all traffic through the nym TUN device. When the flag is set, the nym TUN device will be created, but to route traffic through it you will need to do it manually, e.g. ping -Itun0
+      --enable-two-hop
+          Enable two-hop mixnet traffic. This means that traffic jumps directly from entry gateway to exit gateway
+      --enable-poisson-rate
+          Enable Poisson process rate limiting of outbound traffic
+  -h, --help
+          Print help
+  -V, --version
+          Print version
+
+```
+~~~
+
+Here is a list of the options and their descriptions. Some are essential, some are more technical and not needed to be adjusted by users.
+
+**Fundamental commands and arguments**
+
+- `-c` is a path to the [Sandbox config](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) file saved as `sandbox.env`
+- `--entry-gateway-id`: paste one of the values labeled with a key `"identityKey"` (without `" "`) from [here](https://nymvpn.com/en/alpha/api/gateways)
+- `--exit-router-address`: paste one of the values labeled with a key `"address"` (without `" "`) from here [here](https://nymvpn.com/en/alpha/api/gateways)
+- `--enable-wireguard`: Enable the wireguard traffic between the client and the entry gateway. NymVPN uses Mullvad libraries for wrapping `wireguard-go` and to setup local routing rules to route all traffic to the TUN virtual network device
+- `--wg-ip`: The address of the wireguard interface, you can get it [here](https://nymvpn.com/en/alpha)
+- `--private-key`: get your private key for testing purposes [here](https://nymvpn.com/en/alpha)
+- `--enable-two-hop` is a faster setup where the traffic is routed from the client to Entry Gateway and directly to Exit Gateway (default is 5-hops)
+
+**Advanced options**
+
+- `--enable-poisson`: Enables process rate limiting of outbound traffic (disabled by default). It means that NymVPN client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
+- `--ip` is the IP address of the TUN device. That is the IP address of the local private network that is set up between local client and the Exit Gateway.
+- `--mtu`: The MTU of the TUN device. That is the max IP packet size of the local private network that is set up between local client and the Exit Gateway.
+- `--disable-routing`: Disable routing all traffic through the VPN TUN device.
@@ -0,0 +1,46 @@
+# NymVPN Command Line Interface (CLI)
+
+```admonish info
+Our alpha testing round is done with participants at live workshop events. This guide will not work for everyone, as the NymVPN source code is not yet publicly accessible. The alpha testing is done on Nym testnet Sandbox environment, this configuration is limited and will not work in the future.
+
+**If you commit to test NymVPN alpha, please start with the [user research form]({{nym_vpn_form_url}}) where all the steps will be provided**. If you disagree with any of the conditions listed, please leave this page.
+```
+
+NymVPN CLI is a fundamental way to run the client for different purposes, currently it is a must for users who want to run the [testing scripts](testing.md).
+
+Follow the simple [automated script](#automated-script-for-cli-installation) below to install and run NymVPN CLI. If you prefer to do a manual setup follow the steps in the guide for [Linux](cli-linux.md) or [MacOS](cli-mac.md).
+
+## Automated Script for CLI Installation
+
+We wrote a [script](https://gist.github.com/tommyv1987/87267ded27e1eb7651aa9cc745ddf4af) which does download of the CLI, sha256 verification, extraction, installation and configuration for Linux and MacOS users automatically following the steps below:
+
+1. Open a terminal window in a directory where you want the script and NymVPN CLI binary be downloaded and run
+```sh
+curl -o execute-nym-vpn-cli-binary.sh -L https://gist.githubusercontent.com/tommyv1987/87267ded27e1eb7651aa9cc745ddf4af/raw/2a8b703655549f2e515ef1960b5f6dc54adc02fa/execute-nym-vpn-cli-binary.sh
+```
+
+2. Make the script executable
+```sh
+chmod u+x execute-nym-vpn-cli-binary.sh
+```
+
+3. Start the script as root, turn off any VPN and run
+```sh
+sudo ./execute-nym-vpn-cli-binary.sh
+```
+
+4. Follow the prompts in the program
+
+5. The script will automatically start the client. Make sure to **turn off any other VPNs** and follow the prompts:
+
+* It prints a JSON view of existing Gateways and prompt you to:
+    - *Make sure to use two different Gateways for entry and exit!*
+    - `enter a gateway ID:` paste one of the values labeled with a key `"identityKey"` printed above (without `" "`)
+    - `enter an exit address:` paste one of the values labeled with a key `"address"` printed above (without `" "`)
+    - `do you want five hop or two hop?`: type `five` or `two`
+    - `enable WireGuard? (yes/no):` if you chose yes, find your private key and wireguard IP [here](https://nymvpn.com/en/alpha)
+
+To run `nym-vpn-cli` again, reconnect your wifi, move to the directory of your CLI binary `cd ~/nym-vpn-cli-dir` and follow the guide for [Linux](cli-linux.md#run-nymvpn) or [MacOS](cli-mac.md#run-nymvpn). If you find it too difficult, just run this script again - like in step \#3 above.
+
+In case of errors check out the [troubleshooting](troubleshooting.md) section.
+
@@ -0,0 +1,71 @@
+# Frequently Asked Questions
+
+**NymVPN [*Support & FAQ page*](https://nymvpn.com/en/support)** contains all essential FAQs regarding the client. This page (below) is a source of additional information often seeked by users, operators and developers testing NymVPN.
+
+If you interested to read more about Nym platform, you can have a look at [Nym general FAQ](https://nymtech.net/developers/faq/general-faq.html) and read through Nym's technical [documentation](https://nymtech.net/docs), [Developer Portal](https://nymtech.net/developers) and [Operators Guide](https://nymtech.net/operators).
+
+## NymVPN
+
+If this your first time hearing about NymVPN, make sure you visit [NymVPN webpage](https://nymvpn.com/en), the official NymVPN [support & FAQ page](https://nymvpn.com/en/support) and the proceed to the introduction and guide on how to [install, run and test](intro.md#nymvpn-guides) the client.
+
+Below are some extra FAQs which came out during the previous alpha testing rounds.
+
+### What's the difference between 2-hops and 5-hops
+
+The default is 5-hops (including Entry and Exit Gateways), which means that the traffic goes from the local client to Entry Gateway -> through 3 layers of Mix Nodes -> to Exit Gateway -> internet. this option uses all the Nym Mixnet features for maximum privacy.
+
+```
+                      ┌─►mix──┐  mix     mix
+                      │       │
+            Entry     │       │                   Exit
+client ───► Gateway ──┘  mix  │  mix  ┌─►mix ───► Gateway ───► internet
+                              │       │
+                              │       │
+                         mix  └─►mix──┘  mix
+```
+
+The 2-hop option is going from the local client -> Entry Gateway -> directly to Exit Gateway -> internet. This option is good for operations demanding faster connection. Keep in mind that this setup by-passes the 3 layers of Mix Nodes. The anonymising features done by your local client like breaking data into same-size packets with inserting additional "dummy" ones to break the time and volume patterns is done in both options.
+
+```
+            Entry         Exit
+client ───► Gateway ────► Gateway ───► internet
+```
+
+We highly recommend to read more about [Nym network overview](https://nymtech.net/docs/architecture/network-overview.html) and the [Mixnet traffic flow](https://nymtech.net/docs/architecture/traffic-flow.html).
+
+### Why do I see different sizes of packets in my terminal log?
+
+One of features of Nym Mixnet's clients is to break data into the same size packets called Sphinx, which is currently ~2kb. When running NymVPN, the data log shows payload sizes, which are the raw sizes of the IP packets, not Sphinx. The payload sizes will be capped by the configured MTU, which is set around 1500 bytes.
+
+### What is 'poisson filter' about?
+
+By default `--enable-poisson` is disabled and packets are sent from the local client to the Entry Gateway as quickly as possible. With the poisson process enabled the Nym client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
+
+Enabling the poisson filter is one of the key mechanisms to de-correlate input and output traffic to the Mixnet. The performance impact however is dramatic:
+1 packer per 20ms is 50 packets / sec so ballpark 100kb/s.
+For mobile clients that means constantly sending data eating up data allowance.
+
+
+## Nym Mixnet Architecture and Rewards
+
+We have a list of questions related to Nym Nodes and the incentives behind running them under [FAQ pages](https://nymtech.net/operators/faq/mixnodes-faq.html) in our [Operators Guide](https://nymtech.net/operators). For better knowledge about Nym architecture we recommend to read [Nym network overview](https://nymtech.net/docs/architecture/network-overview.html) and the [Mixnet traffic flow](https://nymtech.net/docs/architecture/traffic-flow.html) in our [technical documentation](https://nymtech.net/docs).
+
+## Project Smoosh
+
+Project Smoosh is a code name for a process in which different components of Nym Mixnet architecture get *smooshed* into one binary. Check out [Smoosh FAQ](https://nymtech.net/operators/faq/smoosh-faq.html) in Operators Guide to read more.
+
+## Exit Gateway
+
+Part of the the transition under code name [Project Smoosh](#project-smoosh) is a creation of [Nym Exit Gateway](https://nymtech.net/operators/legal/exit-gateway.html) functionality. The operators running Gateways would have to “open” their nodes to a wider range of online services, in a similar fashion to Tor exit relays. The main change will be to expand the original short [allowed.list](https://nymtech.net/.wellknown/network-requester/standard-allowed-list.txt) to a more permissive setup. An [exit policy](https://nymtech.net/.wellknown/network-requester/exit-policy.txt) will constrain the hosts that the users of the Nym VPN and Mixnet can connect to. This will be done in an effort to protect the operators, as Gateways will act both as SOCKS5 Network Requesters, and exit nodes for IP traffic from Nym VPN and Mixnet clients.
+
+* Read more how the exit policy gets implemented [here](https://nymtech.net/operators/faq/smoosh-faq.html#how-will-the-exit-policy-be-implemented)
+* Check out [Nym Operators Legal Forum](https://nymtech.net/operators/legal/exit-gateway.html)
+* Do reach out to us with any experiences you may have running Tor Exit relays or legal findings and suggestions for Nym Exit Gateway operators
+
+## Nym Integrations and SDKs
+
+If you are a dev who is interested to integrate Nym, have a look on our SDK tutorials:
+
+* [Rust SDKs](https://nymtech.net/developers/tutorials/cosmos-service/intro.html)
+* [TypeScript SDKs](https://sdk.nymtech.net/)
+* [Integration FAQ](https://nymtech.net/developers/faq/integrations-faq.html)
@@ -0,0 +1,73 @@
+# NymVPN alpha GUI: Guide for GNU/Linux
+
+<div style="padding:56.25% 0 0 0;position:relative;"><iframe src="https://player.vimeo.com/video/908221306?h=404b2bbdc8" style="position:absolute;top:0;left:0;width:100%;height:100%;" frameborder="0" allow="autoplay; fullscreen; picture-in-picture" allowfullscreen></iframe></div><script src="https://player.vimeo.com/api/player.js"></script>
+
+```admonish info
+NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
+```
+
+## Preparation
+
+> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
+
+### Installation
+
+1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for Debian based Linux
+2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
+```sh
+echo "<SHA_STRING>" | shasum -a 256 -c
+
+# choose a correct one according to your binary, this is just an example
+echo "a5f91f20d587975e30b6a75d3a9e195234cf1269eac278139a5b9c39b039e807  nym-vpn-desktop_0.0.3_ubuntu-22.04_x86_64.zip" | shasum -a 256 -c
+```
+3. Extract files with `unzip` command or manually as you are used to
+4. If you prefer to run `.AppImage` make executable by running:
+```sh
+chmod u+x ./appimage/nym-vpn_0.0.2_amd64.AppImage
+```
+5. If you prefer to use the `.deb` version for installation (works on Debian based Linux only), open terminal in the same directory and run:
+```sh
+cd deb
+
+sudo dpkg -i ./nym-vpn_0.0.3_amd64.deb
+# or
+sudo apt-get install -f ./nym-vpn_0.0.3_amd64.deb
+```
+
+### Configuration
+
+6. Create a NymVPN config directory called `nym-vpn` in your `~/.config`, either manually or by a command:
+```sh
+mkdir $HOME/.config/nym-vpn/
+```
+7. Create the network config by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the directory `~/.config/nym-vpn/` you just created by running:
+```sh
+curl -o $HOME/.config/nym-vpn/sandbox.env -L https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env
+```
+
+8. Create NymVPN main config file called `config.toml` in the same directory `~/.config/nym-vpn/` with this content:
+```toml
+# change <USER> to your username
+env_config_file = "/home/<USER>/.config/nym-vpn/sandbox.env"
+entry_node_location = "DE" # two letters country code
+# You can choose different entry by entering one of the following two letter country codes:
+# DE, UK, FR, IE
+```
+
+## Run NymVPN
+
+**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
+
+In case you used `.deb` package and installed the client, you may be able to have a NymVPN application icon in your app menu. However this may not work as the application needs root permission.
+
+Open terminal and run:
+
+```sh
+# .AppImage must be run from the same directory as the binary
+sudo -E ./nym-vpn_0.0.2_amd64.AppImage
+
+# .deb installation shall be executable from anywhere as
+sudo -E nym-vpn
+```
+
+In case of errors, see [troubleshooting section](troubleshooting.md).
@@ -0,0 +1,59 @@
+# NymVPN alpha GUI: Guide for Mac OS
+
+```admonish info
+NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
+```
+
+## Preparation
+
+> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
+
+### Installation
+
+1. Create a directory `~/nym-vpn-latest`
+```sh
+mkdir -p "$HOME/nym-vpn-latest"
+```
+2. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for MacOS
+3. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
+```sh
+echo "<SHA_STRING>" | shasum -a 256 -c
+
+# choose a correct one according to your binary, this is just an example
+echo "da4c0bf8e8b52658312d341fa3581954cfcb6efd516d9a448c76d042a454b5df  nym-vpn-desktop_0.0.3_macos_x86_64.zip" | shasum -a 256 -c
+```
+4. Extract files with `unzip` command or manually as you are used to
+5. Move to the application directory and make executable
+```sh
+cd "macos/nym-vpn.app/Contents/MacOS"
+
+chmod u+x nym-vpn
+```
+6. Move `nym-vpn` to your `~/nym-vpn-latest` directory
+```sh
+mv nym-vpn "$HOME/nym-vpn-latest"
+```
+
+### Configuration
+
+7. Create the configuration file by opening a text editor and saving the lines below as `config.toml` in the same directory `~/nym-vpn-latest`
+```toml
+env_config_file = ".env"
+entry_node_location = "DE" # two letters country code
+# You can choose different entry by entering one of the following two letter country codes:
+# DE, UK, FR, IE
+```
+8. Create testnet configuration file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `.env` in the same directory `~/nym-vpn-latest`
+```sh
+curl -L "https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env" -o "$HOME/nym-vpn-latest/.env"
+```
+## Run NymVPN
+
+**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
+
+Open terminal in your `~/nym-vpn-latest` directory and run:
+```sh
+sudo ./nym-vpn
+```
+
+In case of errors check out the [troubleshooting](troubleshooting.html#installing-gui-on-macos-not-working) section.
@@ -0,0 +1,43 @@
+# NymVPN Application (GUI)
+
+```admonish info
+Our alpha testing round is done with participants at live workshop events. This guide will not work for everyone, as the NymVPN source code is not yet publicly accessible. The alpha testing is done on Nym testnet Sandbox environment, this configuration is limited and will not work in the future.
+
+**If you commit to test NymVPN alpha, please start with the [user research form]({{nym_vpn_form_url}}) where all the steps will be provided**. If you disagree with any of the conditions listed, please leave this page.
+```
+
+This is the alpha version of NymVPN application - the GUI. A demo of how the client will look like for majority of day-to-day users. For qualitative testing the [CLI](cli.md) is a necessity but to run the GUI holds the same importance as it provides the user with an experience of the actual app and the developers with a valuable feedback from the users.
+
+Follow the simple [automated script](#automated-script-for-gui-installation) below to install and run NymVPN GUI. If you prefer to do a manual setup follow the steps in the guide for [Linux](gui-linux.md) or [MacOS](gui-mac.md).
+
+## Automated Script for GUI Installation
+
+We wrote a [script](https://gist.github.com/serinko/e0a9f7ff3d79e974ec6f6783caa1137e) which does download of dependencies and the application, sha256 verification, extraction, installation and configuration for Linux and MacOS users automatically following the steps below:
+
+1. Open a terminal window in a directory where you want the script and NymVPN binary be downloaded and run
+```sh
+curl -o nym-vpn-desktop-install-run.sh -L https://gist.githubusercontent.com/serinko/e0a9f7ff3d79e974ec6f6783caa1137e/raw/064c0d42af6a187b4f62998a0c2e6a84c319eeef/nym-vpn-desktop-install-run.sh
+```
+
+2. Make the script executable
+```sh
+chmod u+x nym-vpn-desktop-install-run.sh
+```
+
+3. Start the script as root, turn off any VPN and run
+```sh
+sudo -E ./nym-vpn-desktop-install-run.sh
+```
+
+4. Follow the prompts in the program
+
+To start the application again, reconnect your wifi and run
+```sh
+# Linux
+sudo -E ~/nym-vpn-latest/nym-vpn_0.0.3_amd64.AppImage
+
+# MacOS
+sudo $nym_vpn_dir/nym-vpn
+```
+
+In case of errors check out the [troubleshooting](troubleshooting.md#installing-gui-on-macos-not-working) section.
--- a/Show More
+++ b/Show More