Add Matrix link to navbar

* Initial commit

* Cherry pick from develop

* Keep track of fragments

* A bunch of data formats, graphs

* Use mix_id for display

* Proper API routes

* Add openapi + swagger ui

* Update locustfile

* Add node stats endpoint

* Add Swagger and locust to readme

* All node stats endpoint

* Update dependencies to use workspace

* Bunch of pedantic fixes

* More version updates, fmt

* More lints

* Add new_from_env for NymTopology

* Nym API endpoint to submit monitoring results (#4616)

* Nym API endpoint to submit monitoring results

* Add gateway monitoring results

* Cleanup, ergonomics

* Weaponize

* Finalize results submissions

* Monitor message signing and verification

* Update README

* Axum graceful shutdown

* More grtacefulness

* Restructure result submission

* Less fragile routes

* Remove gateway unique index on node_id

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.github
+.gitignore
+**/node_modules
+**/target
+dist
+documentation

.gitignore

@@ -48,4 +48,7 @@ foxyfox.env
 
 .next
 ppa-private-key.b64
-ppa-private-key.asc
+ppa-private-key.asc
+nym-network-monitor/topology.json
+nym-network-monitor/__pycache__
+nym-network-monitor/*.key

Cargo.lock

@@ -347,6 +347,14 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
 
+[[package]]
+name = "autodoc"
+version = "0.1.0"
+dependencies = [
+ "env_logger 0.11.5",
+ "log",
+]
+
 [[package]]
 name = "axum"
 version = "0.6.20"
@@ -1858,6 +1866,7 @@ dependencies = [
  "lock_api",
  "once_cell",
  "parking_lot_core 0.9.10",
+ "serde",
 ]
 
 [[package]]
@@ -2184,6 +2193,16 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "env_filter"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+dependencies = [
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.7.1"
@@ -2207,6 +2226,19 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "env_logger"
+version = "0.11.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "env_filter",
+ "humantime 2.1.0",
+ "log",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -2397,6 +2429,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
 [[package]]
 name = "flate2"
 version = "1.0.30"
@@ -4253,6 +4291,7 @@ dependencies = [
  "nym-sphinx",
  "nym-task",
  "nym-topology",
+ "nym-types",
  "nym-validator-client",
  "nym-vesting-contract-common",
  "okapi",
@@ -4579,6 +4618,7 @@ dependencies = [
  "nym-topology",
  "nym-validator-client",
  "rand 0.8.5",
+ "rand_chacha 0.3.1",
  "serde",
  "serde_json",
  "sha2 0.10.8",
@@ -5410,6 +5450,36 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "nym-network-monitor"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "axum 0.7.5",
+ "clap 4.5.7",
+ "dashmap",
+ "futures",
+ "log",
+ "nym-bin-common",
+ "nym-crypto",
+ "nym-network-defaults",
+ "nym-sdk",
+ "nym-sphinx",
+ "nym-topology",
+ "nym-types",
+ "nym-validator-client",
+ "petgraph",
+ "rand 0.8.5",
+ "rand_chacha 0.3.1",
+ "reqwest 0.12.4",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tokio-util",
+ "utoipa",
+ "utoipa-swagger-ui",
+]
+
 [[package]]
 name = "nym-network-requester"
 version = "1.1.40"
@@ -5575,6 +5645,7 @@ dependencies = [
  "nym-task",
  "nym-topology",
  "rand 0.8.5",
+ "rand_chacha 0.3.1",
  "serde",
  "serde_json",
  "thiserror",
@@ -5858,6 +5929,7 @@ version = "0.1.0"
 dependencies = [
  "log",
  "nym-crypto",
+ "nym-metrics",
  "nym-mixnet-contract-common",
  "nym-sphinx-acknowledgements",
  "nym-sphinx-addressing",
@@ -5871,6 +5943,7 @@ dependencies = [
  "nym-sphinx-types",
  "nym-topology",
  "rand 0.8.5",
+ "rand_chacha 0.3.1",
  "rand_distr",
  "thiserror",
  "tokio",
@@ -5927,12 +6000,17 @@ dependencies = [
 name = "nym-sphinx-chunking"
 version = "0.1.0"
 dependencies = [
+ "dashmap",
  "log",
+ "nym-crypto",
+ "nym-metrics",
  "nym-sphinx-addressing",
  "nym-sphinx-params",
  "nym-sphinx-types",
  "rand 0.8.5",
+ "serde",
  "thiserror",
+ "utoipa",
 ]
 
 [[package]]
@@ -6046,6 +6124,7 @@ dependencies = [
  "nym-sphinx-routing",
  "nym-sphinx-types",
  "rand 0.8.5",
+ "reqwest 0.12.4",
  "semver 0.11.0",
  "serde",
  "serde_json",
@@ -6682,6 +6761,16 @@ dependencies = [
  "sha2 0.10.8",
 ]
 
+[[package]]
+name = "petgraph"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
+dependencies = [
+ "fixedbitset",
+ "indexmap 2.2.6",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.5"

Cargo.toml

@@ -14,7 +14,6 @@ panic = "abort"
 opt-level = 3
 
 [workspace]
-
 resolver = "2"
 members = [
     "clients/native",
@@ -94,6 +93,7 @@ members = [
     "common/wasm/utils",
     "common/wireguard",
     "common/wireguard-types",
+    "documentation/autodoc",
     "explorer-api",
     "explorer-api/explorer-api-requests",
     "explorer-api/explorer-client",
@@ -106,6 +106,7 @@ members = [
     "service-providers/common",
     "service-providers/ip-packet-router",
     "service-providers/network-requester",
+    "nym-network-monitor",
     "nym-api",
     "nym-browser-extension/storage",
     "nym-api/nym-api-requests",
@@ -159,6 +160,7 @@ homepage = "https://nymtech.net"
 documentation = "https://nymtech.net"
 edition = "2021"
 license = "Apache-2.0"
+rust-version = "1.80"
 
 [workspace.dependencies]
 addr = "0.15.6"

common/client-core/Cargo.toml

@@ -19,6 +19,7 @@ futures = { workspace = true }
 humantime-serde = { workspace = true }
 log = { workspace = true }
 rand = { workspace = true }
+rand_chacha = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 sha2 = { workspace = true }

common/client-core/src/client/packet_statistics_control.rs

@@ -458,7 +458,7 @@ impl PacketStatisticsControl {
 
     fn report_rates(&self) {
         if let Some((_, rates)) = self.rates.back() {
-            log::info!("{}", rates.summary());
+            log::debug!("{}", rates.summary());
             log::debug!("{}", rates.detailed_summary());
         }
     }
@@ -486,7 +486,7 @@ impl PacketStatisticsControl {
             // Check what the number of retransmissions was during the recording window
             if let Some((_, start_stats)) = self.history.front() {
                 let delta = self.stats.clone() - start_stats.clone();
-                log::info!(
+                log::debug!(
                     "mix packet retransmissions/real mix packets: {}/{}",
                     delta.retransmissions_queued,
                     delta.real_packets_queued,

common/client-core/src/client/real_messages_control/message_handler.rs

@@ -453,6 +453,7 @@ where
 
         let mut pending_acks = Vec::with_capacity(fragments.len());
         let mut real_messages = Vec::with_capacity(fragments.len());
+        debug!("Splitting message into {} fragments", fragments.len());
         for fragment in fragments {
             // we need to clone it because we need to keep it in memory in case we had to retransmit
             // it. And then we'd need to recreate entire ACK again.

common/client-libs/validator-client/Cargo.toml

@@ -90,4 +90,6 @@ default = ["http-client"]
 http-client = ["cosmrs/rpc"]
 generate-ts = []
 contract-testing = ["nym-mixnet-contract-common/contract-testing"]
-
+# Features below are added to make clippy happy, it seems like they're unused we should remove them
+tendermint-rpc-http-client = ["tendermint-rpc/http-client"]
+tendermint-rpc-websocket-client = ["tendermint-rpc/websocket-client"]

common/client-libs/validator-client/src/nym_api/routes.rs

@@ -49,5 +49,7 @@ pub const COMPUTE_REWARD_ESTIMATION: &str = "compute-reward-estimation";
 pub const AVG_UPTIME: &str = "avg_uptime";
 pub const STAKE_SATURATION: &str = "stake-saturation";
 pub const INCLUSION_CHANCE: &str = "inclusion-probability";
+pub const SUBMIT_GATEWAY: &str = "submit-gateway-monitoring-results";
+pub const SUBMIT_NODE: &str = "submit-node-monitoring-results";
 
 pub const SERVICE_PROVIDERS: &str = "services";

common/client-libs/validator-client/src/nyxd/cosmwasm_client/mod.rs

@@ -300,8 +300,8 @@ where
     }
 
     #[cfg(any(
-        feature = "tendermint-rpc/http-client",
-        feature = "tendermint-rpc/websocket-client"
+        feature = "tendermint-rpc-http-client",
+        feature = "tendermint-rpc-websocket-client"
     ))]
     async fn wait_until_healthy<T>(&self, timeout: T) -> Result<(), Error>
     where

common/client-libs/validator-client/src/nyxd/mod.rs

@@ -820,8 +820,8 @@ where
     }
 
     #[cfg(any(
-        feature = "tendermint-rpc/http-client",
-        feature = "tendermint-rpc/websocket-client"
+        feature = "tendermint-rpc-http-client",
+        feature = "tendermint-rpc-websocket-client"
     ))]
     async fn wait_until_healthy<T>(&self, timeout: T) -> Result<(), Error>
     where

common/client-libs/validator-client/src/rpc/mod.rs

@@ -300,8 +300,8 @@ pub trait TendermintRpcClient {
     }
 
     #[cfg(any(
-        feature = "tendermint-rpc/http-client",
-        feature = "tendermint-rpc/websocket-client"
+        feature = "tendermint-rpc-http-client",
+        feature = "tendermint-rpc-websocket-client"
     ))]
     /// Poll the `/health` endpoint until it returns a successful result or
     /// the given `timeout` has elapsed.
@@ -518,8 +518,8 @@ mod non_wasm {
         }
 
         #[cfg(any(
-            feature = "tendermint-rpc/http-client",
-            feature = "tendermint-rpc/websocket-client"
+            feature = "tendermint-rpc-http-client",
+            feature = "tendermint-rpc-websocket-client"
         ))]
         async fn wait_until_healthy<T>(&self, timeout: T) -> Result<(), Error>
         where

common/gateway-storage/src/lib.rs

@@ -22,7 +22,7 @@ use tracing::{debug, error};
 pub mod bandwidth;
 pub mod error;
 mod inboxes;
-pub(crate) mod models;
+pub mod models;
 mod shared_keys;
 mod tickets;
 #[cfg(feature = "wireguard")]

common/node-tester-utils/Cargo.toml

@@ -9,11 +9,12 @@ license.workspace = true
 [dependencies]
 futures = { workspace = true }
 rand = { workspace = true }
+rand_chacha = { workspace = true }
 
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["macros"]}
+tokio = { workspace = true, features = ["macros"] }
 
 nym-crypto = { path = "../crypto", features = ["asymmetric"] }
 nym-task = { path = "../task" }

common/node-tester-utils/src/tester.rs

@@ -294,4 +294,8 @@ impl<R: CryptoRng + Rng> FragmentPreparer for NodeTester<R> {
     fn average_ack_delay(&self) -> Duration {
         self.average_ack_delay
     }
+
+    fn nonce(&self) -> i32 {
+        1
+    }
 }

common/nymsphinx/Cargo.toml

@@ -11,6 +11,7 @@ repository = { workspace = true }
 log = { workspace = true }
 rand = { workspace = true }
 rand_distr = { workspace = true }
+rand_chacha = { workspace = true }
 thiserror = { workspace = true }
 
 nym-sphinx-acknowledgements = { path = "acknowledgements" }
@@ -27,10 +28,13 @@ nym-sphinx-types = { path = "types" }
 # to separate crate?
 nym-crypto = { path = "../crypto", version = "0.4.0" }
 nym-topology = { path = "../topology" }
+nym-metrics = { path = "../nym-metrics" }
 
 [dev-dependencies]
 nym-mixnet-contract-common = { path = "../cosmwasm-smart-contracts/mixnet-contract" }
-nym-crypto = { path = "../crypto", version = "0.4.0", features = ["asymmetric"] }
+nym-crypto = { path = "../crypto", version = "0.4.0", features = [
+    "asymmetric",
+] }
 
 # do not include this when compiling into wasm as it somehow when combined together with reqwest, it will require
 # net2 via tokio-util -> tokio -> mio -> net2
@@ -43,5 +47,13 @@ features = ["sync"]
 
 [features]
 default = ["sphinx"]
-sphinx = ["nym-crypto/sphinx", "nym-sphinx-params/sphinx", "nym-sphinx-types/sphinx"]
-outfox = ["nym-crypto/outfox", "nym-sphinx-params/outfox", "nym-sphinx-types/outfox"]
+sphinx = [
+    "nym-crypto/sphinx",
+    "nym-sphinx-params/sphinx",
+    "nym-sphinx-types/sphinx",
+]
+outfox = [
+    "nym-crypto/outfox",
+    "nym-sphinx-params/outfox",
+    "nym-sphinx-types/outfox",
+]

common/nymsphinx/chunking/Cargo.toml

@@ -13,7 +13,14 @@ repository = { workspace = true }
 log = { workspace = true }
 rand = { workspace = true }
 thiserror = { workspace = true }
+dashmap = { workspace = true, features = ["serde"] }
+serde = { workspace = true, features = ["derive"] }
+utoipa = { workspace = true }
 
 nym-sphinx-addressing = { path = "../addressing" }
 nym-sphinx-params = { path = "../params" }
 nym-sphinx-types = { path = "../types" }
+nym-metrics = { path = "../../nym-metrics" }
+nym-crypto = { path = "../../crypto", version = "0.4.0", features = [
+    "asymmetric",
+] }

common/nymsphinx/chunking/src/fragment.rs

@@ -3,6 +3,8 @@
 
 use crate::ChunkingError;
 use nym_sphinx_params::{SerializedFragmentIdentifier, FRAG_ID_LEN};
+use serde::Serialize;
+use utoipa::ToSchema;
 
 use std::fmt::{self, Debug, Formatter};
 
@@ -58,7 +60,7 @@ pub const COVER_FRAG_ID: FragmentIdentifier = FragmentIdentifier {
 /// and u8 position of the `Fragment` in the set.
 // TODO: this should really be redesigned, especially how cover and reply messages are really
 // "abusing" this. They should work with it natively instead.
-#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize)]
 pub struct FragmentIdentifier {
     set_id: i32,
     fragment_position: u8,
@@ -75,6 +77,10 @@ impl fmt::Display for FragmentIdentifier {
 }
 
 impl FragmentIdentifier {
+    pub fn set_id(&self) -> i32 {
+        self.set_id
+    }
+
     pub fn to_bytes(self) -> SerializedFragmentIdentifier {
         debug_assert_eq!(FRAG_ID_LEN, 5);
 
@@ -125,6 +131,10 @@ impl Debug for Fragment {
 }
 
 impl Fragment {
+    pub fn header(&self) -> FragmentHeader {
+        self.header.clone()
+    }
+
     /// Tries to encapsulate provided payload slice and metadata into a `Fragment`.
     /// It can fail if payload would not fully fit in a single `Fragment` or some of the metadata
     /// is malformed or self-contradictory, for example if current_fragment > total_fragments.
@@ -216,6 +226,10 @@ impl Fragment {
         }
     }
 
+    pub fn seed(&self) -> i32 {
+        self.header().seed()
+    }
+
     /// Gets the size of payload contained in this `Fragment`.
     pub fn payload_size(&self) -> usize {
         self.payload.len()
@@ -297,8 +311,8 @@ impl Fragment {
 /// there is 7 bytes of overhead inside each sphinx packet sent
 /// and for the longest messages, without upper bound, there is usually also only 7 bytes
 /// of overhead apart from first and last fragments in each set that instead have 10 bytes of overhead.
-#[derive(PartialEq, Clone, Debug)]
-pub(crate) struct FragmentHeader {
+#[derive(PartialEq, Clone, Debug, Serialize, ToSchema)]
+pub struct FragmentHeader {
     /// ID associated with `FragmentSet` to which this particular `Fragment` belongs.
     /// Its value is restricted to (0, i32::MAX].
     /// Note that it *excludes* 0, but *includes* i32::MAX.
@@ -324,6 +338,20 @@ pub(crate) struct FragmentHeader {
 }
 
 impl FragmentHeader {
+    pub fn seed(&self) -> i32 {
+        let mut seed = self.id;
+        seed = seed.wrapping_mul(self.total_fragments as i32);
+        seed = seed.wrapping_mul(self.current_fragment as i32);
+        seed
+    }
+
+    pub fn total_fragments(&self) -> u8 {
+        self.total_fragments
+    }
+
+    pub fn current_fragment(&self) -> u8 {
+        self.current_fragment
+    }
     /// Tries to create a new `FragmentHeader` using provided metadata. Bunch of logical
     /// checks are performed to see if the data is not self-contradictory,
     /// for example if current_fragment > total_fragments.

common/nymsphinx/chunking/src/lib.rs

@@ -1,9 +1,16 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::sync::LazyLock;
+
 use crate::fragment::{linked_fragment_payload_max_len, unlinked_fragment_payload_max_len};
+use dashmap::DashMap;
+use fragment::{Fragment, FragmentHeader};
+use nym_crypto::asymmetric::ed25519::PublicKey;
+use serde::Serialize;
 pub use set::split_into_sets;
 use thiserror::Error;
+use utoipa::ToSchema;
 
 pub const MIN_PADDING_OVERHEAD: usize = 1;
 
@@ -22,6 +29,118 @@ pub mod fragment;
 pub mod reconstruction;
 pub mod set;
 
+#[derive(Debug, Clone)]
+pub struct FragmentMixParams {
+    destination: PublicKey,
+    hops: u8,
+}
+
+impl FragmentMixParams {
+    pub fn destination(&self) -> &PublicKey {
+        &self.destination
+    }
+
+    pub fn hops(&self) -> u8 {
+        self.hops
+    }
+}
+
+#[derive(Debug, Clone, Serialize, ToSchema)]
+pub struct SentFragment {
+    header: FragmentHeader,
+    at: u64,
+    client_nonce: i32,
+    #[serde(skip)]
+    mixnet_params: FragmentMixParams,
+}
+
+impl SentFragment {
+    fn new(
+        header: FragmentHeader,
+        at: u64,
+        client_nonce: i32,
+        destination: PublicKey,
+        hops: u8,
+    ) -> Self {
+        let mixnet_params = FragmentMixParams { destination, hops };
+        SentFragment {
+            header,
+            at,
+            client_nonce,
+            mixnet_params,
+        }
+    }
+
+    pub fn header(&self) -> FragmentHeader {
+        self.header.clone()
+    }
+
+    pub fn at(&self) -> u64 {
+        self.at
+    }
+
+    pub fn client_nonce(&self) -> i32 {
+        self.client_nonce
+    }
+
+    pub fn seed(&self) -> i32 {
+        self.header().seed().wrapping_mul(self.client_nonce())
+    }
+
+    pub fn mixnet_params(&self) -> FragmentMixParams {
+        self.mixnet_params.clone()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, ToSchema)]
+pub struct ReceivedFragment {
+    header: FragmentHeader,
+    at: u64,
+}
+
+impl ReceivedFragment {
+    fn new(header: FragmentHeader, at: u64) -> Self {
+        ReceivedFragment { header, at }
+    }
+
+    pub fn header(&self) -> FragmentHeader {
+        self.header.clone()
+    }
+
+    pub fn at(&self) -> u64 {
+        self.at
+    }
+}
+
+pub static FRAGMENTS_RECEIVED: LazyLock<DashMap<i32, Vec<ReceivedFragment>>> =
+    LazyLock::new(DashMap::new);
+
+pub static FRAGMENTS_SENT: LazyLock<DashMap<i32, Vec<SentFragment>>> = LazyLock::new(DashMap::new);
+
+#[macro_export]
+macro_rules! now {
+    () => {
+        match std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) {
+            Ok(n) => n.as_secs(),
+            Err(_) => 0,
+        }
+    };
+}
+
+pub fn fragment_received(fragment: &Fragment) {
+    let id = fragment.fragment_identifier().set_id();
+    let mut entry = FRAGMENTS_RECEIVED.entry(id).or_default();
+    let r = ReceivedFragment::new(fragment.header(), now!());
+    entry.push(r);
+}
+
+pub fn fragment_sent(fragment: &Fragment, client_nonce: i32, destination: PublicKey, hops: u8) {
+    let id = fragment.fragment_identifier().set_id();
+    let mut entry = FRAGMENTS_SENT.entry(id).or_default();
+    let s = SentFragment::new(fragment.header(), now!(), client_nonce, destination, hops);
+    entry.push(s);
+}
+
 /// The idea behind the process of chunking is to incur as little data overhead as possible due
 /// to very computationally costly sphinx encapsulation procedure.
 ///

common/nymsphinx/chunking/src/reconstruction.rs

@@ -1,7 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 use crate::fragment::Fragment;
-use crate::ChunkingError;
+use crate::{fragment_received, ChunkingError};
 use log::*;
 use std::collections::HashMap;
 
@@ -66,6 +66,12 @@ impl ReconstructionBuffer {
         // if the set is complete.
         debug_assert!(self.is_complete);
 
+        debug!(
+            "Got {} fragments for set id {}",
+            self.fragments.len(),
+            self.fragments[0].as_ref().unwrap().id()
+        );
+
         self.fragments
             .into_iter()
             .map(|fragment| fragment.unwrap().extract_payload())
@@ -104,6 +110,8 @@ impl ReconstructionBuffer {
             }
         });
 
+        fragment_received(&fragment);
+
         let fragment_index = fragment.current_fragment() as usize - 1;
         if self.fragments[fragment_index].is_some() {
             // TODO: what to do in that case? give up on the message? overwrite it? panic?

common/nymsphinx/src/preparer/mod.rs

@@ -3,6 +3,7 @@
 
 use crate::message::{NymMessage, ACK_OVERHEAD, OUTFOX_ACK_OVERHEAD};
 use crate::NymPayloadBuilder;
+use log::debug;
 use nym_crypto::asymmetric::encryption;
 use nym_crypto::Digest;
 use nym_sphinx_acknowledgements::surb_ack::SurbAck;
@@ -11,12 +12,14 @@ use nym_sphinx_addressing::clients::Recipient;
 use nym_sphinx_addressing::nodes::NymNodeRoutingAddress;
 use nym_sphinx_anonymous_replies::reply_surb::ReplySurb;
 use nym_sphinx_chunking::fragment::{Fragment, FragmentIdentifier};
+use nym_sphinx_chunking::fragment_sent;
 use nym_sphinx_forwarding::packet::MixPacket;
 use nym_sphinx_params::packet_sizes::PacketSize;
 use nym_sphinx_params::{PacketType, ReplySurbKeyDigestAlgorithm, DEFAULT_NUM_MIX_HOPS};
 use nym_sphinx_types::{Delay, NymPacket};
 use nym_topology::{NymTopology, NymTopologyError};
-use rand::{CryptoRng, Rng};
+use rand::{CryptoRng, Rng, SeedableRng};
+use rand_chacha::ChaCha20Rng;
 
 use std::time::Duration;
 
@@ -49,6 +52,7 @@ pub trait FragmentPreparer {
     type Rng: CryptoRng + Rng;
 
     fn rng(&mut self) -> &mut Self::Rng;
+    fn nonce(&self) -> i32;
     fn num_mix_hops(&self) -> u8;
     fn average_packet_delay(&self) -> Duration;
     fn average_ack_delay(&self) -> Duration;
@@ -192,9 +196,18 @@ pub trait FragmentPreparer {
         packet_type: PacketType,
         mix_hops: Option<u8>,
     ) -> Result<PreparedFragment, NymTopologyError> {
+        debug!("Preparing chunk for sending");
         // each plain or repliable packet (i.e. not a reply) attaches an ephemeral public key so that the recipient
         // could perform diffie-hellman with its own keys followed by a kdf to re-derive
         // the packet encryption key
+
+        let seed = fragment.seed().wrapping_mul(self.nonce());
+        let mut rng = ChaCha20Rng::seed_from_u64(seed as u64);
+
+        let destination = packet_recipient.gateway();
+        let hops = mix_hops.unwrap_or(self.num_mix_hops());
+        fragment_sent(&fragment, self.nonce(), *destination, hops);
+
         let non_reply_overhead = encryption::PUBLIC_KEY_SIZE;
         let expected_plaintext = match packet_type {
             PacketType::Outfox => {
@@ -228,10 +241,8 @@ pub trait FragmentPreparer {
         };
 
         // generate pseudorandom route for the packet
-        let hops = mix_hops.unwrap_or(self.num_mix_hops());
         log::trace!("Preparing chunk for sending with {} mix hops", hops);
-        let route =
-            topology.random_route_to_gateway(self.rng(), hops, packet_recipient.gateway())?;
+        let route = topology.random_route_to_gateway(&mut rng, hops, destination)?;
         let destination = packet_recipient.as_sphinx_destination();
 
         // including set of delays
@@ -313,6 +324,8 @@ pub struct MessagePreparer<R> {
     /// Number of mix hops each packet ('real' message, ack, reply) is expected to take.
     /// Note that it does not include gateway hops.
     num_mix_hops: u8,
+
+    nonce: i32,
 }
 
 impl<R> MessagePreparer<R>
@@ -325,12 +338,15 @@ where
         average_packet_delay: Duration,
         average_ack_delay: Duration,
     ) -> Self {
+        let mut rng = rng;
+        let nonce = rng.gen();
         MessagePreparer {
             rng,
             sender_address,
             average_packet_delay,
             average_ack_delay,
             num_mix_hops: DEFAULT_NUM_MIX_HOPS,
+            nonce,
         }
     }
 
@@ -454,6 +470,10 @@ impl<R: CryptoRng + Rng> FragmentPreparer for MessagePreparer<R> {
     fn average_ack_delay(&self) -> Duration {
         self.average_ack_delay
     }
+
+    fn nonce(&self) -> i32 {
+        self.nonce
+    }
 }
 
 /*

common/socks5/proxy-helpers/src/proxy_runner/outbound.rs

@@ -60,7 +60,7 @@ pub(super) async fn run_outbound(
 
     loop {
         select! {
-            connection_message = &mut mix_receiver.next() => {
+            connection_message = mix_receiver.next() => {
                 if let Some(connection_message) = connection_message {
                     if deal_with_message(connection_message, &mut writer, &local_destination_address, &remote_source_address, connection_id).await {
                         break;

common/topology/Cargo.toml

@@ -5,7 +5,6 @@ edition = { workspace = true }
 authors = { workspace = true }
 license = { workspace = true }
 repository = { workspace = true }
-readme = { workspace = true }
 homepage = { workspace = true }
 documentation = { workspace = true }
 
@@ -15,9 +14,10 @@ documentation = { workspace = true }
 bs58 = { workspace = true }
 log = { workspace = true }
 rand = { workspace = true }
+reqwest = { workspace = true, features = ["json"] }
 thiserror = { workspace = true }
 async-trait = { workspace = true, optional = true }
-semver = "0.11"
+semver = { version = "0.11" }
 
 # 'serializable' feature
 serde = { workspace = true, features = ["derive"], optional = true }
@@ -28,20 +28,22 @@ tsify = { workspace = true, features = ["js"], optional = true }
 wasm-bindgen = { workspace = true, optional = true }
 
 ## internal
+nym-bin-common = { path = "../bin-common" }
+nym-config = { path = "../config" }
 nym-crypto = { path = "../crypto", features = ["sphinx", "outfox"] }
 nym-mixnet-contract-common = { path = "../cosmwasm-smart-contracts/mixnet-contract" }
 nym-sphinx-addressing = { path = "../nymsphinx/addressing" }
-nym-sphinx-types = { path = "../nymsphinx/types", features = ["sphinx", "outfox"] }
+nym-sphinx-types = { path = "../nymsphinx/types", features = [
+    "sphinx",
+    "outfox",
+] }
 nym-sphinx-routing = { path = "../nymsphinx/routing" }
-nym-bin-common = { path = "../bin-common" }
+
 
 # I'm not sure how to feel about pulling in this dependency here...
 nym-api-requests = { path = "../../nym-api/nym-api-requests" }
 
 
-# 'serializable' feature
-nym-config = { path = "../config", optional = true }
-
 # 'wasm-serde-types' feature
 wasm-utils = { path = "../wasm/utils", default-features = false, optional = true }
 
@@ -49,4 +51,5 @@ wasm-utils = { path = "../wasm/utils", default-features = false, optional = true
 default = ["provider-trait"]
 provider-trait = ["async-trait"]
 wasm-serde-types = ["tsify", "wasm-bindgen", "wasm-utils"]
-serializable = ["serde", "nym-config", "serde_json"]
+serializable = ["serde", "serde_json"]
+outfox = []

common/topology/src/error.rs

@@ -51,4 +51,16 @@ pub enum NymTopologyError {
 
     #[error("{0}")]
     PacketError(#[from] NymPacketError),
+
+    #[error("{0}")]
+    ReqwestError(#[from] reqwest::Error),
+
+    #[error("{0}")]
+    MixnodeConversionError(#[from] crate::mix::MixnodeConversionError),
+
+    #[error("{0}")]
+    GatewayConversionError(#[from] crate::gateway::GatewayConversionError),
+
+    #[error("{0}")]
+    VarError(#[from] std::env::VarError),
 }

common/topology/src/lib.rs

@@ -6,7 +6,9 @@
 
 use crate::filter::VersionFilterable;
 pub use error::NymTopologyError;
-use log::{debug, warn};
+use log::{debug, info, warn};
+use mix::Node;
+use nym_config::defaults::var_names::NYM_API;
 use nym_mixnet_contract_common::mixnode::MixNodeDetails;
 use nym_mixnet_contract_common::{GatewayBond, IdentityKeyRef, MixId};
 use nym_sphinx_addressing::nodes::NodeIdentity;
@@ -116,13 +118,40 @@ impl Display for NetworkAddress {
 
 pub type MixLayer = u8;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct NymTopology {
     mixes: BTreeMap<MixLayer, Vec<mix::Node>>,
     gateways: Vec<gateway::Node>,
 }
 
 impl NymTopology {
+    pub async fn new_from_env() -> Result<Self, NymTopologyError> {
+        let api_url = std::env::var(NYM_API)?;
+
+        info!("Generating topology from {}", api_url);
+
+        let mixnodes = reqwest::get(&format!("{}/v1/mixnodes", api_url))
+            .await?
+            .json::<Vec<MixNodeDetails>>()
+            .await?
+            .into_iter()
+            .map(|details| details.bond_information)
+            .map(mix::Node::try_from)
+            .filter(Result::is_ok)
+            .collect::<Result<Vec<_>, _>>()?;
+
+        let gateways = reqwest::get(&format!("{}/v1/gateways", api_url))
+            .await?
+            .json::<Vec<GatewayBond>>()
+            .await?
+            .into_iter()
+            .map(gateway::Node::try_from)
+            .filter(Result::is_ok)
+            .collect::<Result<Vec<_>, _>>()?;
+        let topology = NymTopology::new_unordered(mixnodes, gateways);
+        Ok(topology)
+    }
+
     pub fn new(mixes: BTreeMap<MixLayer, Vec<mix::Node>>, gateways: Vec<gateway::Node>) -> Self {
         NymTopology { mixes, gateways }
     }
@@ -270,7 +299,7 @@ impl NymTopology {
         &self,
         rng: &mut R,
         num_mix_hops: u8,
-    ) -> Result<Vec<SphinxNode>, NymTopologyError>
+    ) -> Result<Vec<Node>, NymTopologyError>
     where
         R: Rng + CryptoRng + ?Sized,
     {
@@ -295,12 +324,32 @@ impl NymTopology {
             let random_mix = layer_mixes
                 .choose(rng)
                 .ok_or(NymTopologyError::EmptyMixLayer { layer })?;
-            route.push(random_mix.into());
+            route.push(random_mix.clone());
         }
 
         Ok(route)
     }
 
+    pub fn random_path_to_gateway<R>(
+        &self,
+        rng: &mut R,
+        num_mix_hops: u8,
+        gateway_identity: &NodeIdentity,
+    ) -> Result<(Vec<mix::Node>, gateway::Node), NymTopologyError>
+    where
+        R: Rng + CryptoRng + ?Sized,
+    {
+        let gateway = self.get_gateway(gateway_identity).ok_or(
+            NymTopologyError::NonExistentGatewayError {
+                identity_key: gateway_identity.to_base58_string(),
+            },
+        )?;
+
+        let path = self.random_mix_route(rng, num_mix_hops)?;
+
+        Ok((path, gateway.clone()))
+    }
+
     /// Tries to create a route to the specified gateway, such that it goes through mixnode on layer 1,
     /// mixnode on layer2, .... mixnode on layer n and finally the target gateway
     pub fn random_route_to_gateway<R>(
@@ -321,6 +370,7 @@ impl NymTopology {
         Ok(self
             .random_mix_route(rng, num_mix_hops)?
             .into_iter()
+            .map(|node| SphinxNode::from(&node))
             .chain(std::iter::once(gateway.into()))
             .collect())
     }

common/types/Cargo.toml

@@ -4,7 +4,7 @@ version = "1.0.0"
 description = "Nym common types"
 authors.workspace = true
 edition = "2021"
-rust-version = "1.58"
+rust-version.workspace = true
 license.workspace = true
 
 [dependencies]

common/types/src/lib.rs

@@ -11,6 +11,7 @@ pub mod gas;
 pub mod gateway;
 pub mod helpers;
 pub mod mixnode;
+pub mod monitoring;
 pub mod pending_events;
 pub mod transaction;
 pub mod vesting;

common/types/src/monitoring.rs

@@ -0,0 +1,131 @@
+use std::{collections::HashSet, sync::LazyLock, time::SystemTime};
+
+use nym_crypto::asymmetric::identity::{PrivateKey, PublicKey, Signature};
+use nym_mixnet_contract_common::MixId;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+static NETWORK_MONITORS: LazyLock<HashSet<String>> = LazyLock::new(|| {
+    let mut nm = HashSet::new();
+    nm.insert("5VsPyLbsBCq9PAMWmjKkToteVAKNabNqex6QwDf5fWzt".to_string());
+    nm
+});
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub struct NodeResult {
+    pub node_id: MixId,
+    pub identity: String,
+    pub reliability: u8,
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone)]
+pub struct MixnodeResult {
+    pub mix_id: MixId,
+    pub identity: String,
+    pub owner: String,
+    pub reliability: u8,
+}
+
+impl MixnodeResult {
+    pub fn new(mix_id: MixId, identity: String, owner: String, reliability: u8) -> Self {
+        MixnodeResult {
+            mix_id,
+            identity,
+            owner,
+            reliability,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct GatewayResult {
+    pub identity: String,
+    pub owner: String,
+    pub reliability: u8,
+    pub mix_id: MixId,
+}
+
+impl GatewayResult {
+    pub fn new(identity: String, owner: String, reliability: u8) -> Self {
+        GatewayResult {
+            identity,
+            owner,
+            reliability,
+            mix_id: 0,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, JsonSchema)]
+#[serde(untagged)]
+pub enum MonitorResults {
+    Mixnode(Vec<MixnodeResult>),
+    Gateway(Vec<GatewayResult>),
+}
+
+#[derive(Serialize, Deserialize, JsonSchema)]
+pub struct MonitorMessage {
+    results: Vec<NodeResult>,
+    signature: String,
+    signer: String,
+    timestamp: i64,
+}
+
+impl MonitorMessage {
+    fn message_to_sign(results: &[NodeResult], timestamp: i64) -> Vec<u8> {
+        let mut msg = match serde_json::to_vec(results) {
+            Ok(msg) => msg,
+            Err(_) => Vec::new(),
+        };
+        msg.extend_from_slice(&timestamp.to_le_bytes());
+        msg
+    }
+
+    pub fn timely(&self) -> bool {
+        let now = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs() as i64;
+
+        now - self.timestamp < 5
+    }
+
+    pub fn new(results: Vec<NodeResult>, private_key: &PrivateKey) -> Self {
+        let timestamp = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs() as i64;
+
+        let msg = Self::message_to_sign(&results, timestamp);
+        let signature = private_key.sign(&msg);
+        let public_key = private_key.public_key();
+
+        MonitorMessage {
+            results,
+            signature: signature.to_base58_string(),
+            signer: public_key.to_base58_string(),
+            timestamp,
+        }
+    }
+
+    pub fn from_allowed(&self) -> bool {
+        NETWORK_MONITORS.contains(&self.signer)
+    }
+
+    pub fn results(&self) -> &[NodeResult] {
+        &self.results
+    }
+
+    pub fn verify(&self) -> bool {
+        let msg = Self::message_to_sign(&self.results, self.timestamp);
+
+        let signature = match Signature::from_base58_string(&self.signature) {
+            Ok(sig) => sig,
+            Err(_) => return false,
+        };
+
+        PublicKey::from_base58_string(&self.signer)
+            .map(|pk| pk.verify(msg, &signature).is_ok())
+            .unwrap_or(false)
+    }
+}

common/wasm/utils/Cargo.toml

@@ -30,13 +30,9 @@ workspace = true
 optional = true
 
 [features]
-default = ["sleep", "console_error_panic_hook"]
+default = ["sleep"]
 sleep = ["web-sys", "web-sys/Window"]
-websocket = [
-    "getrandom",
-    "tungstenite",
-    "gloo-net"
-]
+websocket = ["getrandom", "tungstenite", "gloo-net"]
 crypto = [
     "web-sys",
     "web-sys/Crypto",

common/wireguard/src/lib.rs

@@ -84,6 +84,7 @@ pub struct WireguardData {
 #[cfg(target_os = "linux")]
 pub async fn start_wireguard<St: nym_gateway_storage::Storage + 'static>(
     storage: St,
+    all_peers: Vec<nym_gateway_storage::models::WireguardPeer>,
     task_client: nym_task::TaskClient,
     wireguard_data: WireguardData,
     control_tx: UnboundedSender<peer_controller::PeerControlResponse>,
@@ -95,7 +96,7 @@ pub async fn start_wireguard<St: nym_gateway_storage::Storage + 'static>(
 
     let mut peers = vec![];
     let mut suspended_peers = vec![];
-    for storage_peer in storage.get_all_wireguard_peers().await? {
+    for storage_peer in all_peers {
         let suspended = storage_peer.suspended;
         let peer = Peer::try_from(storage_peer)?;
         if suspended {

documentation/autodoc/.gitignore

@@ -0,0 +1 @@
+/autodoc-generated-markdown/*

documentation/autodoc/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "autodoc"
+version = "0.1.0"
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+env_logger = "0.11.3"
+log.workspace = true

documentation/autodoc/README.md

@@ -0,0 +1,4 @@
+# `autodoc`
+
+Command output documentation generator WIP 
+

documentation/autodoc/src/main.rs

@@ -0,0 +1,261 @@
+use log::{debug, info};
+use std::fs::File;
+use std::io::{self, Write};
+use std::process::{Command, Output};
+use std::{fs, vec};
+
+const WRITE_PATH: &str = "./autodoc-generated-markdown/";
+
+fn main() -> io::Result<()> {
+    env_logger::init();
+
+    // TODO if this balloons write automated way of grabbing commands from crates.
+    let commands_with_subcommands = vec![
+        (
+            "../../target/release/nym-api",
+            vec!["init", "run", "build-info"],
+        ),
+        (
+            "../../target/release/nym-client",
+            vec![
+                "init",
+                "run",
+                "import-credential",
+                "list-gateways",
+                "switch-gateway",
+                "build-info",
+                "completions",
+                "generate-fig-spec",
+            ],
+        ),
+        (
+            "../../target/release/nym-socks5-client",
+            vec![
+                "init",
+                "run",
+                "import-credential",
+                "list-gateways",
+                "add-gateway",
+                "build-info",
+                "completions",
+                "generate-fig-spec",
+            ],
+        ),
+        (
+            "../../target/release/nym-node",
+            vec![
+                "build-info",
+                "bonding-information",
+                "node-details",
+                "migrate",
+                "run",
+                "sign",
+            ],
+        ),
+        (
+            "../../target/release/nymvisor",
+            vec![
+                "init",
+                "run",
+                "build-info",
+                "daemon-build-info",
+                "add-upgrade",
+                "config",
+            ],
+        ),
+    ];
+
+    let commands_with_subsubcommands = vec![(
+        "../../target/release/nym-cli",
+        vec![
+            (
+                "account",
+                vec!["create", "balance", "pub-key", "send", "send-multiple"],
+            ),
+            ("signature", vec!["sign", "verify"]),
+            (
+                "ecash",
+                vec![
+                    "issue-ticket-book",
+                    "recover-ticket-book",
+                    "import-ticket-book",
+                ],
+            ),
+            (
+                "coconut",
+                vec![
+                    "generate-freepass",
+                    "issue-credentials",
+                    "recover-credentials",
+                    "import-credential",
+                ],
+            ),
+            ("block", vec!["get", "time", "current-height"]),
+            (
+                "cosmwasm",
+                vec![
+                    "upload",
+                    "init",
+                    "generate-init-message",
+                    "migrate",
+                    "execute",
+                ],
+            ),
+            ("tx", vec!["get", "query"]),
+            (
+                "vesting-schedule",
+                vec!["create", "query", "vested-balance", "withdraw-vested"],
+            ),
+            ("mixnet", vec!["query", "delegators", "operators"]),
+            ("generate-fig", vec![""]),
+        ],
+    )];
+
+    for (main_command, subcommands) in commands_with_subcommands {
+        let last_word = get_last_word_from_filepath(main_command);
+        debug!("now running {last_word:#?}");
+
+        if !fs::metadata(WRITE_PATH)
+            .map(|metadata| metadata.is_dir())
+            .unwrap_or(false)
+        {
+            fs::create_dir_all(WRITE_PATH)?;
+        }
+
+        let mut file = File::create(format!("{}/{}-commands.md", WRITE_PATH, last_word.unwrap()))?;
+        writeln!(
+            file,
+            "# {} Binary Commands",
+            format!("`{}`", last_word.unwrap())
+        )?;
+        writeln!(
+            file,
+            "\nThese docs are autogenerated by the `autodocs` script.
+            \n**TODO add link**"
+        )?;
+        let output = Command::new(main_command).arg("--help").output()?;
+        write_output_to_file(&mut file, output)?;
+
+        for subcommand in subcommands {
+            execute_command(&mut file, main_command, subcommand, None)?;
+        }
+    }
+
+    // nym-cli has subsubcommands so needs its own loop
+    for (main_command, subcommands) in &commands_with_subsubcommands {
+        let last_word = get_last_word_from_filepath(main_command);
+        debug!("now running {last_word:#?}");
+        let mut file = File::create(format!("{}/{}-commands.md", WRITE_PATH, last_word.unwrap()))?;
+        writeln!(
+            file,
+            "# {} Binary Commands",
+            format!("`{}`", last_word.unwrap())
+        )?;
+        writeln!(
+            file,
+            "\nThese docs are autogenerated by the `autodocs` script.
+            \n**TODO add link**"
+        )?;
+        let output = Command::new(main_command).arg("--help").output()?;
+
+        write_output_to_file(&mut file, output)?;
+
+        for (subcommand, subsubcommands) in subcommands {
+            writeln!(file, "\n### `{}` ", subcommand)?;
+            let output = Command::new(main_command)
+                .arg(subcommand)
+                .arg("--help")
+                .output()?;
+            if !output.stdout.is_empty() {
+                write_output_to_file(&mut file, output)?;
+            } else {
+                debug!("empty stdout - nothing to write");
+            }
+            for subsubcommand in subsubcommands {
+                execute_command(&mut file, main_command, subcommand, Some(subsubcommand))?;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn get_last_word_from_filepath(filepath: &str) -> Option<&str> {
+    let parts: Vec<&str> = filepath.split('/').collect();
+    parts.last().copied()
+}
+
+fn execute_command(
+    file: &mut File,
+    main_command: &str,
+    subcommand: &str,
+    subsubcommand: Option<&str>,
+) -> io::Result<()> {
+    // checking for the nym-cli subsubcommands
+    if subsubcommand.is_some() {
+        writeln!(file, "\n### `{} {}`", subcommand, subsubcommand.unwrap())?;
+
+        info!("executing {} {} --help ", main_command, subcommand);
+        let output = Command::new(main_command)
+            .arg(subcommand)
+            .arg(subsubcommand.unwrap())
+            .arg("--help")
+            .output()?;
+        if !output.stdout.is_empty() {
+            write_output_to_file(file, output)?;
+        } else {
+            debug!("empty stdout - nothing to write");
+        }
+    // just subcommands
+    } else {
+        writeln!(file, "\n### `{}`", subcommand)?;
+
+        // execute help
+        let output = Command::new(main_command)
+            .arg(subcommand)
+            .arg("--help")
+            .output()?;
+        if !output.stdout.is_empty() {
+            write_output_to_file(file, output)?;
+        } else {
+            debug!("empty stdout - nothing to write");
+        }
+
+        // then execute w/out help: the majority of functions will fail since you're not passing
+        // required params but thats fine as we can just not render stderr into the final file.
+        //
+        // this check is basically checking for the rare commands (rn just one) that start a process with no params
+        // perhaps if this list grows we could just add a timeout and shunt the running and writing
+        // into a thread with a timeout or something but for right now its fine / thats overkill
+        if get_last_word_from_filepath(main_command).unwrap() == "nym-node"
+            || get_last_word_from_filepath(main_command).unwrap() == "nym-api"
+            || get_last_word_from_filepath(main_command).unwrap() == "nymvisor"
+                && subcommand == "run"
+        {
+            info!("SKIPPING {} {}", main_command, subcommand);
+        } else {
+            info!("executing {} {}", main_command, subcommand);
+            let output = Command::new(main_command).arg(subcommand).output()?;
+            if !output.stdout.is_empty() {
+                writeln!(file, "Example output:")?;
+                write_output_to_file(file, output)?;
+            } else {
+                debug!("empty stdout - nothing to write");
+                if !&output.stderr.is_empty() {
+                    debug!("stderr: {:#?}", String::from_utf8_lossy(&output.stderr));
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+fn write_output_to_file(file: &mut File, output: Output) -> io::Result<()> {
+    writeln!(file, "```")?;
+    file.write_all(&output.stdout)?;
+    writeln!(file, "```")?;
+
+    if !&output.stderr.is_empty() {
+        debug!("stderr: {:#?}", String::from_utf8_lossy(&output.stderr));
+    }
+    Ok(())
+}

documentation/docs/.gitignore

@@ -1,14 +1,6 @@
-# mdbook files
-book/
-
-# Compiled assets
-.sass-cache
-_site
-
-# Developing
-todo.md
-.idea
-
-# OSX
-.DS_Store
+.next
+node_modules
+out
 
+# the lock file will break Vercel because it may get committed from a machine with a different build architecture
+package-lock.json

documentation/docs/README.md

@@ -1,42 +1,24 @@
-# Nym Documentation
-Documentation for the Nym privacy platform built using the [mdBook](https://rust-lang.github.io/mdBook/) docs framework.
+# Nym Docs v2
 
-Documentation can be viewed at https://nymtech.net/docs
+New consolidated version of the nym docs.
 
-## Contributing
-Contributions to our documentation are very welcome. Please work on your contribution in either a `feature/<feature-name>` or `chore/<chore-name>` branch from `master` and target your pull request at `master`.
+## Local development
 
-Since these docs autogenerate command output and import docs from binaries in `target/release` on `build` make sure you're branching off of `master` when making your branch.
+```
+npm i
+npm run dev
+```
 
-Changes merged to `master` will be autodeployed to the production site.
+Open http://localhost:3000 to browse the output that will hot-reload when you make changes.
 
-### Contributing a new translation
-To contribute tranlsations in a new language, please get in touch via [Matrix](https://matrix.to/#/#general:nymtech.chat) or [Discord](https://nymtech.net/go/discord).
+## Build
 
-### Variables
-There are some variables that are shared across the entire docs site, such as the current latest software version.
+```
+npm run build
+```
 
-Variables are denoted in the `.md` files wrapped in `{{}}` (e.g `{{wallet_release_version}}`), and are located in the `book.toml` file under the `[preprocessor.variables.variables]` heading. If you are changing something like the software release version, minimum code versions in prerequisites, etc, **check in here first!**
+The static output will be in `./out`;
 
-### Diagrams
-Most diagrams are simply ascii. Copies are kept in `/diagrams/` for ease of reproducability. Created using [textik](https://textik.com/#).
+## Template details
 
-### Importing files and auto-generated command output
-
-Example files are inserted as per normal with mdbook.
-
-Some binary command outputs are generated using the [`cmdrun`](https://docs.rs/mdbook-cmdrun/latest/mdbook_cmdrun/) mdbook plugin.
-
-## Building
-When working locally, it is recommended that you use `mdbook serve` to have a local version of the docs served on `localhost:3000`, with hot reloading on any changes made to files in the `src/` directory.
-
-You can find other commands in the [mdBook CLI tool docs](https://rust-lang.github.io/mdBook/cli/index.html).
-
-### I tried to edit files in `theme/` and they aren't taking effect / `mdbook serve` causes a looping reload on file changes after changing fields in `[preprocessor.theme]` config
-
-Looping reload is a known issue with the `mdbook-theme` preprocessor used for the table of contents and layout of these docs. As outlined in the `mdbook-theme` [readme](https://github.com/zjp-CN/mdbook-theme#avoid-repeating-call-on-this-tool-when-mdbook-watch) one way to mitigate this is to set `turn-off = true` under `[preprocessor.theme]`. This means that `mdbook serve` or `mdbook watch` ignores changes to the `theme/` directory, which is the source of the looping reload. If you have changed or commented out this line, reintroduce it to remove the looping reload. If you are trying to edit the theme of the docs and want to apply the change, see [here](https://github.com/zjp-CN/mdbook-theme#avoid-repeating-call-on-this-tool-when-mdbook-watch) for more info on how to remove the block, change the theme, and reintroduce the block.
-
-### Checking the mdBook version
-To check the version of mdBook installed on your system, you can use the `mdbook --version` command. This will print the version number of mdBook installed on your system in the terminal.
-
-The latest release of the binary of the pre-compiled binaries can be found on [GitHub](https://github.com/rust-lang/mdBook/releases).
+This documentation was made with [Nextra](https://nextra.site) using the template from here https://github.com/shuding/nextra-docs-template.

documentation/docs/components/landing-page.tsx

@@ -0,0 +1,90 @@
+import React from "react";
+import { Ghost, Zap, Code, Soup } from "lucide-react";
+
+export const LandingPage = () => {
+  const squares = [
+    { text: "Network Docs", href: "/network", Icon: Ghost },
+    {
+      text: "Developers: Core Concepts, Integration Overview, Tools & Tutorials",
+      href: "/developers",
+      Icon: Zap,
+    },
+    { text: "SDKs", href: "/developers/rust", Icon: Code },
+    {
+      text: "Operators: Setup Guides & Maintenance",
+      href: "/operators",
+      Icon: Soup,
+    },
+  ];
+
+  return (
+    <div className="landing-page-container">
+      <p className="landing-page-intro">
+        Nym is a privacy platform. It provides strong network-level privacy
+        against sophisticated end-to-end attackers, and anonymous access control
+        using blinded, re-randomizable, decentralized credentials. Our goal is
+        to allow developers to build new applications, or upgrade existing apps,
+        with privacy features unavailable in other systems.
+      </p>
+
+      <div className="landing-page-grid">
+        {squares.map((square, index) => (
+          <a key={index} href={square.href} className="landing-page-square">
+            <square.Icon className="landing-page-icon" />
+            <span className="landing-page-text">{square.text}</span>
+          </a>
+        ))}
+      </div>
+
+      <style jsx>{`
+        .landing-page-container {
+          max-width: 1200px;
+          margin: 0 auto;
+          padding: 2rem 1rem;
+        }
+        .landing-page-intro {
+          font-size: 1.125rem;
+          margin-bottom: 2rem;
+        }
+        .landing-page-grid {
+          display: grid;
+          grid-template-columns: repeat(2, 1fr);
+          gap: 2rem;
+          max-width: 36rem;
+          margin: 0 auto;
+        }
+        .landing-page-square {
+          background-color: #000000;
+          color: white;
+          padding: 1rem;
+          border-radius: 0.5rem;
+          text-align: center;
+          cursor: pointer;
+          text-decoration: none;
+          transition: box-shadow 0.3s ease;
+          aspect-ratio: 1 / 1;
+          display: flex;
+          flex-direction: column;
+          align-items: center;
+          justify-content: center;
+          border: 3px solid #ff6600;
+          box-shadow: 0 0 10px #ff6600;
+        }
+        .landing-page-square:hover {
+          box-shadow: 0 0 20px #ff6600;
+        }
+        .landing-page-icon {
+          width: 12.5rem;
+          height: 12.5rem;
+          margin-bottom: 0.75rem;
+          color: #ff6600;
+        }
+        .landing-page-text {
+          font-size: 0.875rem;
+          font-weight: 600;
+          color: #ff6600;
+        }
+      `}</style>
+    </div>
+  );
+};

documentation/docs/components/my-tab.tsx

@@ -0,0 +1,6 @@
+import { Tabs } from "nextra/components";
+export const MyTab = ({ name, children }) => (
+  <Tabs.Tab>
+    {name} {children}
+  </Tabs.Tab>
+);

documentation/docs/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@nymproject/ts-sdk-docs",
-  "version": "1.3.0-rc.0",
-  "description": "Nym Typescript SDK Docs",
+  "name": "@nymproject/docs",
+  "version": "1.0.0",
+  "description": "Nym Docs Monorepo",
   "license": "Apache-2.0",
   "author": "Nym Technologies SA",
   "scripts": {
@@ -34,6 +34,7 @@
     "@nymproject/sdk-full-fat": ">=1.2.4-rc.2 || ^1",
     "chain-registry": "^1.19.0",
     "cosmjs-types": "^0.9.0",
+    "lucide-react": "^0.438.0",
     "next": "^13.4.19",
     "nextra": "latest",
     "nextra-theme-docs": "latest",

documentation/docs/pages/_meta.json

@@ -0,0 +1,31 @@
+{
+  "index": {
+    "display": "hidden"
+  },
+  "network": {
+    "title": "Network",
+    "type": "page"
+  },
+  "developers": {
+    "title": "Developers",
+    "type": "menu",
+    "items": {
+      "developers": {
+        "title": "Developer Overview",
+        "href": "/developers/"
+      },
+      "rust": {
+        "title": "Rust SDK",
+        "href": "/developers/rust"
+      },
+      "typescript": {
+        "title": "Typescript SDK",
+        "href": "/developers/typescript"
+      }
+    }
+  },
+  "operators": {
+    "title": "Operators",
+    "type": "page"
+  }
+}

documentation/docs/pages/_meta.json.bak

@@ -0,0 +1,42 @@
+{
+  "network": {
+    "title": "Network",
+    "type": "page"
+  },
+  "developers": {
+    "title": "Developers",
+    "type": "menu",
+    "items": {
+      "concepts": {
+        "title": "Core Concepts",
+        "href": "/developers/concepts"
+      },
+      "integrations": {
+        "title": "Integrations",
+        "href": "/developers/integrations"
+      },
+      "tools": {
+        "title": "Tools",
+        "href": "/developers/tools"
+      }
+    }
+  },
+  "sdk": {
+    "title": "SDKs",
+    "type": "menu",
+    "items": {
+      "rust": {
+        "title": "Rust SDK",
+        "href": "/sdk/rust"
+      },
+      "typescript": {
+        "title": "Typescript SDK",
+        "href": "/sdk/typescript"
+      }
+    }
+  },
+  "operators": {
+    "title": "Operators",
+    "type": "page"
+  }
+}

documentation/docs/pages/_meta.json.bak.bak

@@ -0,0 +1,31 @@
+{
+  "index": {
+    "display": "hidden"
+  },
+  "network": {
+    "title": "Network",
+    "type": "page"
+  },
+  "developers": {
+    "title": "Developers",
+    "type": "page"
+  },
+  "sdk": {
+    "title": "SDKs",
+    "type": "menu",
+    "items": {
+      "rust": {
+        "title": "Rust SDK",
+        "href": "/sdk/rust"
+      },
+      "typescript": {
+        "title": "Typescript SDK",
+        "href": "/sdk/typescript"
+      }
+    }
+  },
+  "operators": {
+    "title": "Operators",
+    "type": "page"
+  }
+}

documentation/docs/pages/developers/_meta.json

@@ -0,0 +1,17 @@
+{
+  "index": "Introduction",
+  "concepts": "Core Concepts",
+  "integrations": "Integration Options",
+  "clients": "Clients",
+  "tools": "Tools",
+  "chain": "Interacting with Nyx",
+  "--": {
+    "type": "separator"
+  },
+  "rust": "Rust SDK",
+  "typescript": "Typescript SDK",
+  "---": {
+    "type": "separator"
+  },
+  "licensing": "Licensing"
+}

documentation/docs/pages/developers/clients.mdx

@@ -0,0 +1,21 @@
+# Types of Nym clients
+At present, there are three standalone Nym clients. These are built as standalone binaries when building our codebase, but are most easily accessed through one of our SDKs:
+
+- the websocket (native) client - most easily accessed via the [Rust SDK]() TODO LINK and Go/C++ FFI TODO LINK
+- the SOCKS5 client - most easily accessed via the [Rust SDK]() TODO LINK and Go/C++ FFI TODO LINK
+- the wasm (webassembly) client - most easily via the [Typescript SDK]() TODO LINK
+
+> For information about the role that clients play within the Nym system and their role when communicating with the Mixnet, see the [Client network docs](../network/architecture/mixnet/clients).
+
+### The websocket client
+This is a compiled program that can run on Linux, Mac OS X, and Windows machines. It can be run as a persistent process on a desktop or server machine. You can connect to it with **any language that supports websockets**.
+
+### The webassembly client
+If you're working in JavaScript or Typescript in the browser, or building an [edge computing](https://en.wikipedia.org/wiki/Edge_computing) app, you'll likely want to choose the webassembly client.
+
+It's packaged and [available on the npm registry](https://www.npmjs.com/package/@nymproject/nym-client-wasm), so you can `npm install` it into your JavaScript or TypeScript application.
+
+### The SOCKS5 client
+The `nym-socks5-client` is useful for allowing existing applications to use the Nym mixnet without any code changes. All that's necessary is that they can use one of the SOCKS5, SOCKS4a, or SOCKS4 proxy protocols (which many applications can - crypto wallets, browsers, chat applications etc).
+
+When used as a standalone client, it's less flexible as a way of writing custom applications than the other clients, but able to be used to proxy application traffic through the mixnet without having to make any code changes.

documentation/docs/pages/developers/clients/socks5.md

@@ -0,0 +1,212 @@
+# Socks5 Client
+
+TODO REMOVE THIS REPLACE WITH A BUILD INSTRUCTIONS DROPDOWN MENU COMPONENT
+> The Nym Socks5 Client was built in the [building nym](../binaries/building-nym.md) section. If you haven't yet built Nym and want to run the code on this page, go there first.
+
+## Current version
+TODO
+```
+<!-- cmdrun ../../../../target/release/nym-socks5-client --version | grep "Build Version" | cut -b 21-26  -->
+```
+
+## What is this client for?
+Many existing applications are able to use either the SOCKS4, SOCKS4A, or SOCKS5 proxy protocols. If you want to send such an application's traffic through the mixnet, you can use the `nym-socks5-client` to bounce network traffic through the Nym network, like this:
+
+```
+                                                                              External Systems:
+                                                                                     +--------------------+
+                                                                             |------>| Monero blockchain  |
+                                                                             |       +--------------------+
+                                                                             |       +--------------------+
+                                                                             |------>|    Email server    |
+                                                                             |       +--------------------+
+                                                                             |       +--------------------+
+                                                                             |------>|    RPC endpoint    |
+                                                                             |       +--------------------+
+                                                                             |       +--------------------+
+                                                                             |------>|       Website      |
+                                                                             |       +--------------------+
+                                                                             |       +--------------------+
+  +----------------------------------+                                       |------>|       etc...       |
+  | Mixnet:                          |                                       |       +--------------------+
+  |       * Gateway your client is   |                                       |
+  |       connected to               |          +--------------------+       |
+  |       * Mix nodes 1 -> 3         |<-------->| Network requester  |<------+
+  |       * Gateway that network     |          +--------------------+
+  |       requester is connected to  |
+  +----------------------------------+
+           ^
+           |
+           |
+           |
+           |
+           v
+ +-------------------+
+ | +---------------+ |
+ | |  Nym client   | |
+ | +---------------+ |
+ |         ^         |
+ |         |         |
+ |         |         |
+ |         |         |
+ |         v         |
+ | +---------------+ |
+ | | Your app code | |
+ | +---------------+ |
+ +-------------------+
+  Your Local Machine
+```
+
+There are 2 pieces of software that work together to send SOCKS traffic through the mixnet: the `nym-socks5-client`, and the `nym-network-requester`.
+
+The `nym-socks5-client` allows you to do the following from your local machine:
+* Take a TCP data stream from a application that can send traffic via SOCKS5.
+* Chop up the TCP stream into multiple Sphinx packets, assigning sequence numbers to them, while leaving the TCP connection open for more data
+* Send the Sphinx packets through the Nym Network. Packets are shuffled and mixed as they transit the mixnet.
+
+The `nym-network-requester` then reassembles the original TCP stream using the packets' sequence numbers, and make the intended request. It will then chop up the response into Sphinx packets and send them back through the mixnet to your  `nym-socks5-client`. The application will then receive its data, without even noticing that it wasn't talking to a "normal" SOCKS5 proxy!
+
+## Client setup
+### Viewing command help
+
+You can check that your binaries are properly compiled with:
+
+```
+./nym-socks5-client --help
+```
+
+~~~admonish example collapsible=true title="Console output"
+```
+<!-- cmdrun ../../../../target/release/nym-socks5-client --help -->
+```
+~~~
+
+You can check the necessary parameters for the available commands by running:
+
+```
+./nym-client <COMMAND> --help
+```
+
+### Initialising a new client instance
+Before you can use the client, you need to initalise a new instance of it, which can be done with the following command:
+
+```
+./nym-socks5-client init --id docs-example --use-reply-surbs true --provider Entztfv6Uaz2hpYHQJ6JKoaCTpDL5dja18SuQWVJAmmx.Cvhn9rBJw5Ay9wgHcbgCnVg89MPSV5s2muPV2YF1BXYu@Fo4f4SQLdoyoGkFae5TpVhRVoXCF8UiypLVGtGjujVPf
+```
+
+~~~admonish example collapsible=true title="Console output"
+```
+<!-- cmdrun ../../../../target/release/nym-socks5-client init --id docs-example --provider Entztfv6Uaz2hpYHQJ6JKoaCTpDL5dja18SuQWVJAmmx.Cvhn9rBJw5Ay9wgHcbgCnVg89MPSV5s2muPV2YF1BXYu@Fo4f4SQLdoyoGkFae5TpVhRVoXCF8UiypLVGtGjujVPf -->
+```
+~~~
+
+The `--id` in the example above is a local identifier so that you can name your clients and keep track of them on your local system; it is **never** transmitted over the network.
+
+The `--use-reply-surbs` field denotes whether you wish to send [SURBs](https://nymtech.net/docs/architecture/traffic-flow.md#private-replies-using-surbs) along with your request. It defaults to `false`, we are explicitly setting it as `true`. It defaults to `false` for compatibility with older versions of the [Network Requester](https://nymtech.net/nodes/network-requester.md).
+
+The `--provider` field needs to be filled with the Nym address of a Network Requester that can make network requests on your behalf. If you don't want to [run your own](https://nymtech.net/operators/nodes/network-requester.md) you can select one from the [mixnet explorer](https://explorer.nymtech.net/network-components/service-providers) by copying its `Client ID` and using this as the value of the `--provider` flag. Alternatively, you could use [this list](https://harbourmaster.nymtech.net/).
+
+#### Choosing a Gateway
+By default - as in the example above - your client will choose a random gateway to connect to.
+
+However, there are several options for choosing a gateway, if you do not want one that is randomly assigned to your client:
+* If you wish to connect to a specific gateway, you can specify this with the `--gateway` flag when running `init`.
+* You can also choose a gateway based on its location relative to your client. This can be done by appending the `--latency-based-selection` flag to your `init` command. This command means that to select a gateway, your client will:
+	* fetch a list of all availiable gateways
+	* send few ping messages to all of them, and measure response times.
+	* create a weighted distribution to randomly choose one, favouring ones with lower latency.
+
+> Note this doesn't mean that your client will pick the closest gateway to you, but it will be far more likely to connect to gateway with a 20ms ping rather than 200ms
+
+### Configuring your client
+When you initalise a client instance, a configuration directory will be generated and stored in `$HOME_DIR/.nym/socks5-clients/<client-name>/`.
+
+```
+tree $HOME/<user>/.nym/socks5-clients/docs-example
+├── config
+│   └── config.toml
+└── data
+    ├── ack_key.pem
+    ├── credentials_database.db
+    ├── gateway_shared.pem
+    ├── persistent_reply_store.sqlite
+    ├── private_encryption.pem
+    ├── private_identity.pem
+    ├── public_encryption.pem
+    └── public_identity.pem
+
+```
+
+The `config.toml` file contains client configuration options, while the two `pem` files contain client key information.
+
+The generated files contain the client name, public/private keypairs, and gateway address. The name `<client_id>` in the example above is just a local identifier so that you can name your clients.
+
+#### Configuring your client for Docker
+By default, the native client listens to host `127.0.0.1`. However this can be an issue if you wish to run a client in a Dockerized environment, where it can be convenenient to listen on a different host such as `0.0.0.0`.
+
+You can set this via the `--host` flag during either the `init` or `run` commands.
+
+Alternatively, a custom host can be set in the `config.toml` file under the `socket` section. If you do this, remember to restart your client process.
+
+### Running the socks5 client
+
+You can run the initialised client by doing this:
+
+```
+./nym-socks5-client run --id docs-example
+```
+
+## Automating your socks5 client with systemd
+
+Create a service file for the socks5 client at `/etc/systemd/system/nym-socks5-client.service`:
+
+```ini
+[Unit]
+Description=Nym Socks5 Client
+StartLimitInterval=350
+StartLimitBurst=10
+
+[Service]
+User=nym # replace this with whatever user you wish
+LimitNOFILE=65536
+ExecStart=/home/nym/nym-socks5-client run --id <your_id>
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Now enable and start your socks5 client:
+
+```
+systemctl enable nym-socks5-client.service
+systemctl start nym-socks5-client.service
+# you can always check your socks5 client has succesfully started with:
+systemctl status nym-socks5-client.service
+```
+
+## Using your Socks5 Client
+
+After completing the steps above, your local Socks5 Client will be listening on `localhost:1080` ready to proxy traffic to the Network Requester set as the `--provider` when initialising.
+
+When trying to connect your app, generally the proxy settings are found in `settings->advanced` or `settings->connection`.
+
+Here is an example of setting the proxy connecting in Blockstream Green:
+
+![Blockstream Green settings](../images/wallet-proxy-settings/blockstream-green.gif)
+
+Most wallets and other applications will work basically the same way: find the network proxy settings, enter the proxy url (host: **localhost**, port: **1080**).
+
+In some other applications, this might be written as **localhost:1080** if there's only one proxy entry field.
+
+## Useful Commands
+
+**no-banner**
+
+Adding `--no-banner` startup flag will prevent Nym banner being printed even if run in tty environment.
+
+**build-info**
+
+A `build-info` command prints the build information like commit hash, rust version, binary version just like what command `--version` does. However, you can also specify an `--output=json` flag that will format the whole output as a json, making it an order of magnitude easier to parse.

documentation/docs/pages/developers/concepts.mdx

@@ -0,0 +1,5 @@
+# Core Concepts
+
+We are working on making the Mixnet as easy as possible to use; that said, it is at the moment still quite an unusual system in terms of how it works and the required parts.
+
+We are working on abstractions to emulate more traditional networking such as reading from / writing to TCP sockets, and have plans to modify the internals of the clients to remove some of the required architecture and slim down the parts required to use the Mixnet.

documentation/docs/pages/developers/concepts/_meta.json

@@ -0,0 +1,6 @@
+{
+  "required-architecture": "Required Architecture",
+  "messages": "Message Based Paradigm",
+  "message-queue": "Message Queue",
+  "abstractions": "Connection Abstractions"
+}