Made vesting_contract_address argument peroperly optional (#1769)

* Made vesting_contract_address argument peroperly optional

* Updated changelog

.github/workflows/audit.yml

@@ -1,36 +1,52 @@
 name: Daily security audit
 
-on: workflow_dispatch
+on:
+  schedule:
+    - cron: '5 9 * * *'
 jobs:
-  security_audit:
+  cargo-deny:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/audit-check@v1
+      - name: Checkout repository code
+        uses: actions/checkout@v2
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
+          toolchain: stable
+      - name: Install cargo deny
+        run: cargo install --locked cargo-deny
+      - name: Run cargo deny
+        run: |
+          find . -name Cargo.toml -exec cargo deny --manifest-path {} check \
+            advisories -A advisory-not-detected  --hide-inclusion-graph \; &> \
+            >(uniq  &> .github/workflows/support-files/notifications/deny.message )
+      - uses: actions/upload-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications/deny.message
   notification:
-    if: ${{ failure() }}
-    needs: security_audit
+    needs: cargo-deny
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: Download report from previous job
+        uses: actions/download-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications
       - name: Keybase - Node Install
         run: npm install
         working-directory: .github/workflows/support-files
       - name: Keybase - Send Notification
         env:
-          NYM_NOTIFICATION_KIND: nightly
-          NYM_PROJECT_NAME: "Nym daily audit"
+          NYM_NOTIFICATION_KIND: security
+          NYM_PROJECT_NAME: "Daily security report"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-          GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
-          GIT_BRANCH: "${GITHUB_REF##*/}"
           KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
           KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
-          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBTECH_TEAM }}"
-          KEYBASE_NYM_CHANNEL: "test"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "security"
         uses: docker://keybaseio/client:stable-node
         with:
           args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/build_on_dispatch.yml

@@ -1,72 +0,0 @@
-name: Continuous integration on dispatch
-
-on: workflow_dispatch
-
-jobs:
-  build:
-    runs-on: [ self-hosted, custom-linux ]
-    # Enable sccache via environment variable
-    env:
-      RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
-    steps:
-      - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
-
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Install rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - name: Build all binaries
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace
-
-      - name: Run all tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --all-features
-
-      - name: Check formatting
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - uses: actions-rs/clippy-check@v1
-        name: Clippy checks
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
-
-      - name: Run clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --workspace -- -D warnings
-
-      - name: Build all binaries with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --features=coconut
-
-      - name: Run all tests with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --features=coconut
-
-      - name: Run clippy with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --features=coconut -- -D warnings

.github/workflows/network-explorer-api.yml

@@ -0,0 +1,56 @@
+name: CI for Network Explorer API
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+    
+env:
+  NETWORK: mainnet    
+
+jobs:
+  publish-nym:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+
+      - name: Check the release tag starts with `nym-explorer-api-`
+        if: startsWith(github.ref, 'refs/tags/nym-explorer-api-') == false && github.event_name != 'workflow_dispatch'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Release tag did not start with nym-explorer-api-...')
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build all explorer-api
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path explorer-api/Cargo.toml --workspace --release
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: my-artifact
+          path: |
+            target/release/explorer-api
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            target/release/explorer-api

.github/workflows/nightly_build_release.yml

@@ -1,4 +1,4 @@
-name: Nightly builds on dispatch
+name: Nightly builds on latest release
 
 on: workflow_dispatch
 jobs:
@@ -7,26 +7,40 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      # creates the matrix strategy from nightly_build_matrix_includes.json
-      - uses: actions/checkout@v2
+      # creates the matrix strategy from nightly_build_release_matrix.json
+      - uses: actions/checkout@v3
       - id: set-matrix
         uses: JoshuaTheMiller/conditional-build-matrix@main
         with:
-          inputFile: '.github/workflows/nightly_build_matrix_on_dispatch.json'
+          inputFile: '.github/workflows/nightly_build_release_matrix.json'
           filter: '[?runOnEvent==`${{ github.event_name }}` || runOnEvent==`always`]'
-  build:
+  get_release:
+    runs-on: ubuntu-latest
     needs: matrix_prep
+    outputs:
+      output1: ${{ steps.step2.outputs.lastest_release }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Set output variable to latest release branch
+        id: step2
+        run: echo "latest_release=$(git branch -r | grep -E 'release/v[0-9]+\.[0-9]+\.[0-9]+' | tail -n 1 | sed 's/origin\///')" >> $GITHUB_OUTPUT
+  build:
+    needs: [get_release,matrix_prep]
     strategy:
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
     steps:
       - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
         if: matrix.os == 'ubuntu-latest'
 
-      - name: Check out repository code
-        uses: actions/checkout@v2
+      - name: Check out latest release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{needs.get_release.outputs.output1}}
 
       - name: Install rust toolchain
         uses: actions-rs/toolchain@v1
@@ -42,6 +56,12 @@ jobs:
           command: build
           args: --workspace
 
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
       - name: Run all tests
         uses: actions-rs/cargo@v1
         with:
@@ -99,6 +119,12 @@ jobs:
           command: build
           args: --workspace --features=coconut
 
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
       - name: Run all tests with coconut enabled
         uses: actions-rs/cargo@v1
         with:
@@ -151,7 +177,7 @@ jobs:
       - name: Collect jobs status
         uses: technote-space/workflow-conclusion-action@v2
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Keybase - Node Install
         if: env.WORKFLOW_CONCLUSION == 'failure'
         run: npm install
@@ -160,14 +186,14 @@ jobs:
         if: env.WORKFLOW_CONCLUSION == 'failure'
         env:
           NYM_NOTIFICATION_KIND: nightly
-          NYM_PROJECT_NAME: "Nym nightly build"
+          NYM_PROJECT_NAME: "Nym nightly build on latest release"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
           GIT_BRANCH: "${GITHUB_REF##*/}"
           KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
           KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
-          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMTECH_TEAM }}"
-          KEYBASE_NYM_CHANNEL: "${{ secrets.KEYBASE_CHANNEL_DEV_CORE_ID }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "ci-nightly-release"
           IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
         uses: docker://keybaseio/client:stable-node
         with:

.github/workflows/nym-release-publish.yml

@@ -25,6 +25,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+
       - name: Check the release tag starts with `nym-binaries-`
         if: startsWith(github.ref, 'refs/tags/nym-binaries-') == false && github.event_name != 'workflow_dispatch'
         uses: actions/github-script@v3

.github/workflows/support-files/notifications/send_message.js

@@ -3,7 +3,7 @@ require('dotenv').config();
 const Bot = require('keybase-bot');
 
 let context = {
-  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly', 'nym-connect'],
+  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly', 'nym-connect','security'],
 };
 
 /**

.github/workflows/support-files/security/index.js

@@ -0,0 +1,24 @@
+const Handlebars = require('handlebars');
+const fs = require('fs');
+const path = require('path');
+const { Octokit, App } = require('octokit');
+
+async function addToContextAndValidate(context) {
+  return
+}
+
+async function getMessageBody(context) {
+  try {
+    const source = fs
+    .readFileSync("deny.message").toString();
+    return source;
+} catch (error) {
+    console.error(error);
+}
+
+}
+
+module.exports = {
+  addToContextAndValidate,
+  getMessageBody,
+};

CHANGELOG.md

@@ -6,28 +6,49 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
 
 ### Added
 
-- nym-cli: added CLI tool for interacting with the Nyx blockchain and Nym mixnet smart contracts ([#1577])
-- validator-client: added `query_contract_smart` and `query_contract_raw` on `NymdClient` ([#1558])
-- network-requester: added additional Blockstream Green wallet endpoint to `example.allowed.list` ([#1611](https://github.com/nymtech/nym/pull/1611))
-- common/ledger: new library for communicating with a Ledger device ([#1640])
-- native-client/socks5-client: `disable_loop_cover_traffic_stream` Debug config option to disable the separate loop cover traffic stream ([#1666])
-- native-client/socks5-client: `disable_main_poisson_packet_distribution` Debug config option to make the client ignore poisson distribution in the main packet stream and ONLY send real message (and as fast as they come) ([#1664])
-- native-client/socks5-client: `use_extended_packet_size` Debug config option to make the client use 'ExtendedPacketSize' for its traffic (32kB as opposed to 2kB in 1.0.2) ([#1671])
-- wasm-client: uses updated wasm-compatible `client-core` so that it's now capable of packet retransmission, cover traffic and poisson delay (among other things!) ([#1673])
-- validator-api: add `interval_operating_cost` and `profit_margin_percent` to cmpute reward estimation endpoint
-
-### Fixed
-
-- validator-api, mixnode, gateway should now prefer values in config.toml over mainnet defaults ([#1645])
-- socks5-client: fix bug where in some cases packet reordering could trigger a connection being closed too early ([#1702])
+- binaries: add `-c` shortform for `--config-env-file`
 
 ### Changed
 
-- validator-client: made `fee` argument optional for `execute` and `execute_multiple` ([#1541])
-- socks5 client: graceful shutdown should fix error on disconnect in nym-connect ([#1591])
-- wasm-client: fixed build errors on MacOS and changed example JS code to use mainnet ([#1585])
-- gateway-client: will attempt to read now as many as 8 websocket messages at once, assuming they're already available on the socket ([#1669])
+- clients: add concept of transmission lanes to better handle multiple data streams ([#1720])
 
+[#1720]: https://github.com/nymtech/nym/pull/1720
+
+
+## [v1.1.0](https://github.com/nymtech/nym/tree/v1.1.0) (2022-11-09)
+
+### Added
+
+- clients: add testing-only support for two more extended packet sizes (8kb and 16kb).
+- common/ledger: new library for communicating with a Ledger device ([#1640])
+- native-client/socks5-client/wasm-client: `disable_loop_cover_traffic_stream` Debug config option to disable the separate loop cover traffic stream ([#1666])
+- native-client/socks5-client/wasm-client: `disable_main_poisson_packet_distribution` Debug config option to make the client ignore poisson distribution in the main packet stream and ONLY send real message (and as fast as they come) ([#1664])
+- native-client/socks5-client/wasm-client: `use_extended_packet_size` Debug config option to make the client use 'ExtendedPacketSize' for its traffic (32kB as opposed to 2kB in 1.0.2) ([#1671])
+- network-requester: added additional Blockstream Green wallet endpoint to `example.allowed.list` ([#1611])
+- validator-api: add `interval_operating_cost` and `profit_margin_percent` to compute reward estimation endpoint
+- validator-client: added `query_contract_smart` and `query_contract_raw` on `NymdClient` ([#1558])
+- wasm-client: uses updated wasm-compatible `client-core` so that it's now capable of packet retransmission, cover traffic and poisson delay (among other things!) ([#1673])
+
+### Fixed
+
+- socks5-client: fix bug where in some cases packet reordering could trigger a connection being closed too early ([#1702],[#1724])
+- validator-api: mixnode, gateway should now prefer values in config.toml over mainnet defaults ([#1645])
+- validator-api: should now correctly update historical uptimes for all mixnodes and gateways every 24h ([#1721])
+
+### Changed
+
+- clients: bound the sphinx packet channel and reduce sending rate if gateway can't keep up ([#1703],[#1725])
+- gateway-client: will attempt to read now as many as 8 websocket messages at once, assuming they're already available on the socket ([#1669])
+- moved `Percent` struct to `contracts-common`, change affects explorer-api
+- socks5 client: graceful shutdown should fix error on disconnect in nym-connect ([#1591])
+- validator-api: changed error serialization on `inclusion_probability`, `stake-saturation` and `reward-estimation` endpoints to provide more accurate information ([#1681]) 
+- validator-client: made `fee` argument optional for `execute` and `execute_multiple` ([#1541])
+- wasm-client: fixed build errors on MacOS and changed example JS code to use mainnet ([#1585])
+- validator-api: changes to internal SQL schema due to the mixnet contract revamp ([#1472])
+- validator-api: changes to internal data structures due to the mixnet contract revamp ([#1472])
+- validator-api: split epoch-operations into multiple separate transactions ([#1472])
+
+[#1472]: https://github.com/nymtech/nym/pull/1472
 [#1541]: https://github.com/nymtech/nym/pull/1541
 [#1558]: https://github.com/nymtech/nym/pull/1558
 [#1577]: https://github.com/nymtech/nym/pull/1577
@@ -35,12 +56,18 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
 [#1591]: https://github.com/nymtech/nym/pull/1591
 [#1640]: https://github.com/nymtech/nym/pull/1640
 [#1645]: https://github.com/nymtech/nym/pull/1645
+[#1611]: https://github.com/nymtech/nym/pull/1611
 [#1664]: https://github.com/nymtech/nym/pull/1664
 [#1666]: https://github.com/nymtech/nym/pull/1645
 [#1669]: https://github.com/nymtech/nym/pull/1669
 [#1671]: https://github.com/nymtech/nym/pull/1671
 [#1673]: https://github.com/nymtech/nym/pull/1673
+[#1681]: https://github.com/nymtech/nym/pull/1681
 [#1702]: https://github.com/nymtech/nym/pull/1702
+[#1703]: https://github.com/nymtech/nym/pull/1703
+[#1721]: https://github.com/nymtech/nym/pull/1721
+[#1724]: https://github.com/nymtech/nym/pull/1724
+[#1725]: https://github.com/nymtech/nym/pull/1725
 
 
 ## [nym-binaries-1.0.2](https://github.com/nymtech/nym/tree/nym-binaries-1.0.2)

Cargo.lock

@@ -577,9 +577,17 @@ dependencies = [
 ]
 
 [[package]]
-name = "client-core"
-version = "1.0.1"
+name = "client-connections"
+version = "0.1.0"
 dependencies = [
+ "futures",
+]
+
+[[package]]
+name = "client-core"
+version = "1.1.0"
+dependencies = [
+ "client-connections",
  "config",
  "crypto",
  "dirs",
@@ -737,7 +745,10 @@ name = "contracts-common"
 version = "0.1.0"
 dependencies = [
  "cosmwasm-std",
+ "schemars",
  "serde",
+ "serde_json",
+ "thiserror",
 ]
 
 [[package]]
@@ -847,9 +858,9 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.2"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
+checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
 dependencies = [
  "libc",
 ]
@@ -1579,15 +1590,17 @@ dependencies = [
 
 [[package]]
 name = "explorer-api"
-version = "1.0.1"
+version = "1.1.0"
 dependencies = [
  "chrono",
  "clap 3.2.8",
+ "contracts-common",
  "dotenv",
  "humantime-serde",
  "isocountry",
  "itertools",
  "log",
+ "logging",
  "maxminddb",
  "mixnet-contract-common",
  "network-defaults",
@@ -2726,6 +2739,14 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "logging"
+version = "0.1.0"
+dependencies = [
+ "log",
+ "pretty_env_logger",
+]
+
 [[package]]
 name = "loom"
 version = "0.5.4"
@@ -3057,7 +3078,7 @@ dependencies = [
 
 [[package]]
 name = "nym-cli"
-version = "1.0.0"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "base64",
@@ -3068,6 +3089,7 @@ dependencies = [
  "clap_complete_fig",
  "dotenv",
  "log",
+ "logging",
  "network-defaults",
  "nym-cli-commands",
  "pretty_env_logger",
@@ -3108,9 +3130,10 @@ dependencies = [
 
 [[package]]
 name = "nym-client"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "clap 3.2.8",
+ "client-connections",
  "client-core",
  "coconut-interface",
  "completions",
@@ -3123,6 +3146,7 @@ dependencies = [
  "gateway-client",
  "gateway-requests",
  "log",
+ "logging",
  "network-defaults",
  "nymsphinx",
  "pemstore",
@@ -3132,6 +3156,7 @@ dependencies = [
  "serde_json",
  "sled",
  "task",
+ "thiserror",
  "tokio",
  "tokio-tungstenite 0.14.0",
  "topology",
@@ -3144,7 +3169,7 @@ dependencies = [
 
 [[package]]
 name = "nym-gateway"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3164,6 +3189,7 @@ dependencies = [
  "gateway-requests",
  "humantime-serde",
  "log",
+ "logging",
  "mixnet-client",
  "mixnode-common",
  "network-defaults",
@@ -3190,7 +3216,7 @@ dependencies = [
 
 [[package]]
 name = "nym-mixnode"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "bs58",
@@ -3206,6 +3232,7 @@ dependencies = [
  "humantime-serde",
  "lazy_static",
  "log",
+ "logging",
  "mixnet-client",
  "mixnode-common",
  "nonexhaustive-delayqueue",
@@ -3231,15 +3258,17 @@ dependencies = [
 
 [[package]]
 name = "nym-network-requester"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "async-trait",
  "clap 3.2.8",
+ "client-connections",
  "completions",
  "dirs",
  "futures",
  "ipnetwork 0.20.0",
  "log",
+ "logging",
  "network-defaults",
  "nymsphinx",
  "ordered-buffer",
@@ -3265,6 +3294,7 @@ version = "1.0.2"
 dependencies = [
  "dirs",
  "log",
+ "logging",
  "pretty_env_logger",
  "rocket",
  "serde",
@@ -3276,9 +3306,10 @@ dependencies = [
 
 [[package]]
 name = "nym-socks5-client"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "clap 3.2.8",
+ "client-connections",
  "client-core",
  "coconut-interface",
  "completions",
@@ -3291,6 +3322,7 @@ dependencies = [
  "gateway-client",
  "gateway-requests",
  "log",
+ "logging",
  "network-defaults",
  "nymsphinx",
  "ordered-buffer",
@@ -3303,6 +3335,7 @@ dependencies = [
  "snafu 0.6.10",
  "socks5-requests",
  "task",
+ "thiserror",
  "tokio",
  "topology",
  "url",
@@ -3339,7 +3372,7 @@ dependencies = [
 
 [[package]]
 name = "nym-validator-api"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3349,6 +3382,7 @@ dependencies = [
  "coconut-interface",
  "config",
  "console-subscriber",
+ "contracts-common",
  "cosmwasm-std",
  "credential-storage",
  "credentials",
@@ -3363,6 +3397,7 @@ dependencies = [
  "humantime-serde",
  "inclusion-probability",
  "log",
+ "logging",
  "mixnet-contract-common",
  "multisig-contract-common",
  "nymcoconut",
@@ -4099,6 +4134,7 @@ name = "proxy-helpers"
 version = "0.1.0"
 dependencies = [
  "bytes",
+ "client-connections",
  "futures",
  "log",
  "ordered-buffer",
@@ -6386,6 +6422,7 @@ dependencies = [
  "coconut-interface",
  "colored",
  "config",
+ "contracts-common",
  "cosmrs",
  "cosmwasm-std",
  "cw3",
@@ -6474,12 +6511,14 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 name = "vesting-contract"
 version = "1.1.0"
 dependencies = [
+ "contracts-common",
  "cosmwasm-std",
  "cw-storage-plus",
  "mixnet-contract-common",
  "schemars",
  "serde",
  "thiserror",
+ "vergen 5.1.17",
  "vesting-contract-common",
 ]
 
@@ -6487,6 +6526,7 @@ dependencies = [
 name = "vesting-contract-common"
 version = "0.1.0"
 dependencies = [
+ "contracts-common",
  "cosmwasm-std",
  "mixnet-contract-common",
  "schemars",

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "common/client-libs/gateway-client",
     "common/client-libs/mixnet-client",
     "common/client-libs/validator-client",
+    "common/client-connections",
     "common/coconut-interface",
     "common/commands",
     "common/config",
@@ -41,6 +42,7 @@ members = [
     "common/execute",
     "common/inclusion-probability",
     "common/ledger",
+    "common/logging",
     "common/mixnode-common",
     "common/network-defaults",
     "common/nonexhaustive-delayqueue",

Makefile

@@ -92,6 +92,9 @@ build-wallet:
 build-connect:
 	cargo build --manifest-path nym-connect/Cargo.toml --workspace
 
+build-explorer-api:
+	cargo build --manifest-path explorer-api/Cargo.toml --workspace
+
 build-wasm-client:
 	cargo build --manifest-path clients/webassembly/Cargo.toml --workspace --target wasm32-unknown-unknown
 

clients/client-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "client-core"
-version = "1.0.1"
+version = "1.1.0"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 edition = "2021"
 
@@ -14,11 +14,13 @@ log = "0.4"
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
 serde = { version = "1.0", features = ["derive"] }
 sled = { version = "0.34", optional = true }
+tap = "1.0.1"
 thiserror = "1.0.34"
 url = { version ="2.2", features = ["serde"] }
 
 # internal
 config = { path = "../../common/config" }
+client-connections = { path = "../../common/client-connections" }
 crypto = { path = "../../common/crypto" }
 gateway-client = { path = "../../common/client-libs/gateway-client" }
 #gateway-client = { path = "../../common/client-libs/gateway-client", default-features = false, features = ["wasm", "coconut"] }
@@ -28,7 +30,6 @@ nymsphinx = { path = "../../common/nymsphinx" }
 pemstore = { path = "../../common/pemstore" }
 topology = { path = "../../common/topology" }
 validator-client = { path = "../../common/client-libs/validator-client", default-features = false }
-tap = "1.0.1"
 
 tokio = { version = "1.21.2", features = ["time", "macros"]}
 
@@ -56,4 +57,5 @@ tempfile = "3.1.0"
 default = ["reply-surb"]
 wasm = ["gateway-client/wasm"]
 coconut = ["gateway-client/coconut", "gateway-requests/coconut"]
-reply-surb = ["sled"]
+reply-surb = ["sled"]
+

clients/client-core/src/client/cover_traffic_stream.rs

@@ -16,6 +16,7 @@ use rand::{rngs::OsRng, CryptoRng, Rng};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
+use tokio::sync::mpsc::error::TrySendError;
 
 #[cfg(not(target_arch = "wasm32"))]
 use tokio::time;
@@ -171,11 +172,22 @@ impl LoopCoverTrafficStream<OsRng> {
         )
         .expect("Somehow failed to generate a loop cover message with a valid topology");
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        self.mix_tx.unbounded_send(vec![cover_message]).unwrap();
+        if let Err(err) = self.mix_tx.try_send(vec![cover_message]) {
+            match err {
+                TrySendError::Full(_) => {
+                    // This isn't a problem, if the channel is full means we're already sending the
+                    // max amount of messages downstream can handle.
+                    log::debug!("Failed to send cover message - channel full");
+                    // However it's still useful to alert the user that the gateway or the link to
+                    // the gateway can't keep up. Either due to insufficient bandwidth on the
+                    // client side, or that the gateway is overloaded.
+                    log::warn!("Failed to send: gateway appears to not keep up");
+                }
+                TrySendError::Closed(_) => {
+                    log::warn!("Failed to send cover message - channel closed");
+                }
+            }
+        }
 
         // TODO: I'm not entirely sure whether this is really required, because I'm not 100%
         // sure how `yield_now()` works - whether it just notifies the scheduler or whether it

clients/client-core/src/client/inbound_messages.rs

@@ -1,3 +1,4 @@
+use client_connections::TransmissionLane;
 use futures::channel::mpsc;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::anonymous_replies::ReplySurb;
@@ -11,6 +12,7 @@ pub enum InputMessage {
         recipient: Recipient,
         data: Vec<u8>,
         with_reply_surb: bool,
+        lane: TransmissionLane,
     },
     Reply {
         reply_surb: ReplySurb,
@@ -19,11 +21,17 @@ pub enum InputMessage {
 }
 
 impl InputMessage {
-    pub fn new_fresh(recipient: Recipient, data: Vec<u8>, with_reply_surb: bool) -> Self {
+    pub fn new_fresh(
+        recipient: Recipient,
+        data: Vec<u8>,
+        with_reply_surb: bool,
+        lane: TransmissionLane,
+    ) -> Self {
         InputMessage::Fresh {
             recipient,
             data,
             with_reply_surb,
+            lane,
         }
     }
 

clients/client-core/src/client/mix_traffic.rs

@@ -2,15 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::spawn_future;
-use futures::channel::mpsc;
-use futures::StreamExt;
 use gateway_client::GatewayClient;
 use log::*;
 use nymsphinx::forwarding::packet::MixPacket;
 
-pub type BatchMixMessageSender = mpsc::UnboundedSender<Vec<MixPacket>>;
-pub type BatchMixMessageReceiver = mpsc::UnboundedReceiver<Vec<MixPacket>>;
+pub type BatchMixMessageSender = tokio::sync::mpsc::Sender<Vec<MixPacket>>;
+pub type BatchMixMessageReceiver = tokio::sync::mpsc::Receiver<Vec<MixPacket>>;
 
+// We remind ourselves that 32 x 32kb = 1024kb, a reasonable size for a network buffer.
+pub const MIX_MESSAGE_RECEIVER_BUFFER_SIZE: usize = 32;
 const MAX_FAILURE_COUNT: usize = 100;
 
 pub struct MixTrafficController {
@@ -25,15 +25,17 @@ pub struct MixTrafficController {
 }
 
 impl MixTrafficController {
-    pub fn new(
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-    ) -> MixTrafficController {
-        MixTrafficController {
-            gateway_client,
-            mix_rx,
-            consecutive_gateway_failure_count: 0,
-        }
+    pub fn new(gateway_client: GatewayClient) -> (MixTrafficController, BatchMixMessageSender) {
+        let (sphinx_message_sender, sphinx_message_receiver) =
+            tokio::sync::mpsc::channel(MIX_MESSAGE_RECEIVER_BUFFER_SIZE);
+        (
+            MixTrafficController {
+                gateway_client,
+                mix_rx: sphinx_message_receiver,
+                consecutive_gateway_failure_count: 0,
+            },
+            sphinx_message_sender,
+        )
     }
 
     async fn on_messages(&mut self, mut mix_packets: Vec<MixPacket>) {
@@ -72,7 +74,7 @@ impl MixTrafficController {
 
             while !shutdown.is_shutdown() {
                 tokio::select! {
-                    mix_packets = self.mix_rx.next() => match mix_packets {
+                    mix_packets = self.mix_rx.recv() => match mix_packets {
                         Some(mix_packets) => {
                             self.on_messages(mix_packets).await;
                         },
@@ -96,7 +98,7 @@ impl MixTrafficController {
         spawn_future(async move {
             debug!("Started MixTrafficController without graceful shutdown support");
 
-            while let Some(mix_packets) = self.mix_rx.next().await {
+            while let Some(mix_packets) = self.mix_rx.recv().await {
                 self.on_messages(mix_packets).await;
             }
         })

clients/client-core/src/client/real_messages_control/acknowledgement_control/acknowledgement_listener.rs

@@ -33,7 +33,7 @@ impl AcknowledgementListener {
     }
 
     async fn on_ack(&mut self, ack_content: Vec<u8>) {
-        debug!("Received an ack");
+        trace!("Received an ack");
         let frag_id = match recover_identifier(&self.ack_key, &ack_content)
             .map(FragmentIdentifier::try_from_bytes)
         {

clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs

@@ -8,6 +8,7 @@ use crate::client::{
     real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
     topology_control::TopologyAccessor,
 };
+use client_connections::TransmissionLane;
 use futures::StreamExt;
 use log::*;
 use nymsphinx::anonymous_replies::ReplySurb;
@@ -104,6 +105,7 @@ where
         content: Vec<u8>,
         with_reply_surb: bool,
     ) -> Option<Vec<RealMessage>> {
+        log::trace!("handling msg size: {}", content.len());
         let topology_permit = self.topology_access.get_read_permit().await;
         let topology = match topology_permit
             .try_get_valid_topology_ref(&self.ack_recipient, Some(&recipient))
@@ -164,26 +166,30 @@ where
     }
 
     async fn on_input_message(&mut self, msg: InputMessage) {
-        let real_messages = match msg {
+        let (real_messages, lane) = match msg {
             InputMessage::Fresh {
                 recipient,
                 data,
                 with_reply_surb,
-            } => {
+                lane,
+            } => (
                 self.handle_fresh_message(recipient, data, with_reply_surb)
+                    .await,
+                lane,
+            ),
+            InputMessage::Reply { reply_surb, data } => (
+                self.handle_reply(reply_surb, data)
                     .await
-            }
-            InputMessage::Reply { reply_surb, data } => self
-                .handle_reply(reply_surb, data)
-                .await
-                .map(|message| vec![message]),
+                    .map(|message| vec![message]),
+                TransmissionLane::Reply,
+            ),
         };
 
         // there's no point in trying to send nothing
         if let Some(real_messages) = real_messages {
             // tells real message sender (with the poisson timer) to send this to the mix network
             self.real_message_sender
-                .unbounded_send(real_messages)
+                .unbounded_send((real_messages, lane))
                 .unwrap();
         }
     }

clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs

@@ -1,17 +1,21 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
-use super::PendingAcknowledgement;
-use super::RetransmissionRequestReceiver;
+use super::{
+    action_controller::{Action, ActionSender},
+    PendingAcknowledgement, RetransmissionRequestReceiver,
+};
 use crate::client::{
     real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
     topology_control::TopologyAccessor,
 };
+
+use client_connections::TransmissionLane;
 use futures::StreamExt;
 use log::*;
-use nymsphinx::preparer::MessagePreparer;
-use nymsphinx::{acknowledgements::AckKey, addressing::clients::Recipient};
+use nymsphinx::{
+    acknowledgements::AckKey, addressing::clients::Recipient, preparer::MessagePreparer,
+};
 use rand::{CryptoRng, Rng};
 use std::sync::{Arc, Weak};
 
@@ -113,10 +117,10 @@ where
 
         // send to `OutQueueControl` to eventually send to the mix network
         self.real_message_sender
-            .unbounded_send(vec![RealMessage::new(
-                prepared_fragment.mix_packet,
-                frag_id,
-            )])
+            .unbounded_send((
+                vec![RealMessage::new(prepared_fragment.mix_packet, frag_id)],
+                TransmissionLane::Retransmission,
+            ))
             .unwrap();
     }
 

clients/client-core/src/client/real_messages_control/mod.rs

@@ -14,6 +14,7 @@ use crate::client::{
     topology_control::TopologyAccessor,
 };
 use crate::spawn_future;
+use client_connections::ClosedConnectionReceiver;
 use futures::channel::mpsc;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
@@ -110,6 +111,7 @@ impl RealMessagesController<OsRng> {
         mix_sender: BatchMixMessageSender,
         topology_access: TopologyAccessor,
         #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
+        closed_connection_rx: ClosedConnectionReceiver,
     ) -> Self {
         let rng = OsRng;
 
@@ -159,6 +161,7 @@ impl RealMessagesController<OsRng> {
             rng,
             config.self_recipient,
             topology_access,
+            closed_connection_rx,
         );
 
         RealMessagesController {

clients/client-core/src/client/real_messages_control/real_traffic_stream.rs

@@ -4,6 +4,7 @@
 use crate::client::mix_traffic::BatchMixMessageSender;
 use crate::client::real_messages_control::acknowledgement_control::SentPacketNotificationSender;
 use crate::client::topology_control::TopologyAccessor;
+use client_connections::{ClosedConnectionReceiver, ConnectionId, TransmissionLane};
 use futures::channel::mpsc;
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
@@ -16,7 +17,6 @@ use nymsphinx::forwarding::packet::MixPacket;
 use nymsphinx::params::PacketSize;
 use nymsphinx::utils::sample_poisson_duration;
 use rand::{CryptoRng, Rng};
-use std::collections::VecDeque;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
@@ -27,6 +27,23 @@ use tokio::time;
 #[cfg(target_arch = "wasm32")]
 use wasm_timer;
 
+use self::{
+    sending_delay_controller::SendingDelayController, transmission_buffer::TransmissionBuffer,
+};
+
+mod sending_delay_controller;
+mod transmission_buffer;
+
+#[cfg(not(target_arch = "wasm32"))]
+fn get_time_now() -> time::Instant {
+    time::Instant::now()
+}
+
+#[cfg(target_arch = "wasm32")]
+fn get_time_now() -> wasm_timer::Instant {
+    wasm_timer::Instant::now()
+}
+
 /// Configurable parameters of the `OutQueueControl`
 pub(crate) struct Config {
     /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
@@ -89,6 +106,10 @@ where
     #[cfg(target_arch = "wasm32")]
     next_delay: Option<Pin<Box<wasm_timer::Delay>>>,
 
+    // To make sure we don't overload the mix_tx channel, we limit the rate we are pushing
+    // messages.
+    sending_delay_controller: SendingDelayController,
+
     /// Channel used for sending prepared sphinx packets to `MixTrafficController` that sends them
     /// out to the network without any further delays.
     mix_tx: BatchMixMessageSender,
@@ -106,8 +127,13 @@ where
     /// Accessor to the common instance of network topology.
     topology_access: TopologyAccessor,
 
-    /// Buffer containing all real messages received. It is first exhausted before more are pulled.
-    received_buffer: VecDeque<RealMessage>,
+    /// Buffer containing all incoming real messages keyed by transmission lane, that we will send
+    /// out to the mixnet.
+    transmission_buffer: TransmissionBuffer,
+
+    /// Incoming channel for being notified of closed connections, so that we can close lanes
+    /// corresponding to connections. To avoid sending traffic unnecessary
+    closed_connection_rx: ClosedConnectionReceiver,
 }
 
 pub(crate) struct RealMessage {
@@ -126,8 +152,9 @@ impl RealMessage {
 
 // messages are already prepared, etc. the real point of it is to forward it to mix_traffic
 // after sufficient delay
-pub(crate) type BatchRealMessageSender = mpsc::UnboundedSender<Vec<RealMessage>>;
-type BatchRealMessageReceiver = mpsc::UnboundedReceiver<Vec<RealMessage>>;
+pub(crate) type BatchRealMessageSender =
+    mpsc::UnboundedSender<(Vec<RealMessage>, TransmissionLane)>;
+type BatchRealMessageReceiver = mpsc::UnboundedReceiver<(Vec<RealMessage>, TransmissionLane)>;
 
 pub(crate) enum StreamMessage {
     Cover,
@@ -150,18 +177,21 @@ where
         rng: R,
         our_full_destination: Recipient,
         topology_access: TopologyAccessor,
+        closed_connection_rx: ClosedConnectionReceiver,
     ) -> Self {
         OutQueueControl {
             config,
             ack_key,
             sent_notifier,
             next_delay: None,
+            sending_delay_controller: Default::default(),
             mix_tx,
             real_receiver,
             our_full_destination,
             rng,
             topology_access,
-            received_buffer: VecDeque::with_capacity(0), // we won't be putting any data into this guy directly
+            transmission_buffer: Default::default(),
+            closed_connection_rx,
         }
     }
 
@@ -176,7 +206,7 @@ where
     async fn on_message(&mut self, next_message: StreamMessage) {
         trace!("created new message");
 
-        let next_message = match next_message {
+        let (next_message, fragment_id) = match next_message {
             StreamMessage::Cover => {
                 // TODO for way down the line: in very rare cases (during topology update) we might have
                 // to wait a really tiny bit before actually obtaining the permit hence messing with our
@@ -195,34 +225,41 @@ where
                 }
                 let topology_ref = topology_ref_option.unwrap();
 
-                generate_loop_cover_packet(
-                    &mut self.rng,
-                    topology_ref,
-                    &self.ack_key,
-                    &self.our_full_destination,
-                    self.config.average_ack_delay,
-                    self.config.average_packet_delay,
-                    self.config.cover_packet_size,
+                (
+                    generate_loop_cover_packet(
+                        &mut self.rng,
+                        topology_ref,
+                        &self.ack_key,
+                        &self.our_full_destination,
+                        self.config.average_ack_delay,
+                        self.config.average_packet_delay,
+                        self.config.cover_packet_size,
+                    )
+                    .expect(
+                        "Somehow failed to generate a loop cover message with a valid topology",
+                    ),
+                    None,
                 )
-                .expect("Somehow failed to generate a loop cover message with a valid topology")
             }
             StreamMessage::Real(real_message) => {
-                self.sent_notify(real_message.fragment_id);
-                real_message.mix_packet
+                (real_message.mix_packet, Some(real_message.fragment_id))
             }
         };
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        if let Err(err) = self.mix_tx.unbounded_send(vec![next_message]) {
-            log::warn!(
-                "Failed to send {} packets (possible process shutdown?)",
-                err.into_inner().len()
-            );
+        if let Err(err) = self.mix_tx.send(vec![next_message]).await {
+            log::error!("Failed to send - channel closed: {}", err);
         }
 
+        // notify ack controller about sending our message only after we actually managed to push it
+        // through the channel
+        if let Some(fragment_id) = fragment_id {
+            self.sent_notify(fragment_id);
+        }
+
+        // In addition to closing connections on receiving messages throught closed_connection_rx,
+        // also close connections when sufficiently stale.
+        self.transmission_buffer.prune_stale_connections();
+
         // JS: Not entirely sure why or how it fixes stuff, but without the yield call,
         // the UnboundedReceiver [of mix_rx] will not get a chance to read anything
         // JS2: Basically it was the case that with high enough rate, the stream had already a next value
@@ -234,7 +271,57 @@ where
         tokio::task::yield_now().await;
     }
 
+    fn on_close_connection(&mut self, connection_id: ConnectionId) {
+        log::debug!("Removing lane for connection: {connection_id}");
+        self.transmission_buffer
+            .remove(&TransmissionLane::ConnectionId(connection_id));
+    }
+
+    fn current_average_message_sending_delay(&self) -> Duration {
+        self.config.average_message_sending_delay
+            * self.sending_delay_controller.current_multiplier()
+    }
+
+    fn adjust_current_average_message_sending_delay(&mut self) {
+        let used_slots = self.mix_tx.max_capacity() - self.mix_tx.capacity();
+        log::trace!(
+            "used_slots: {used_slots}, current_multiplier: {}",
+            self.sending_delay_controller.current_multiplier()
+        );
+
+        // Even just a single used slot is enough to signal backpressure
+        if used_slots > 0 {
+            log::trace!("Backpressure detected");
+            self.sending_delay_controller.record_backpressure_detected();
+        }
+
+        // If the buffer is running out, slow down the sending rate
+        if self.mix_tx.capacity() == 0
+            && self.sending_delay_controller.not_increased_delay_recently()
+        {
+            self.sending_delay_controller.increase_delay_multiplier();
+        }
+
+        // Very carefully step up the sending rate in case it seems like we can solidly handle the
+        // current rate.
+        if self.sending_delay_controller.is_sending_reliable() {
+            self.sending_delay_controller.decrease_delay_multiplier();
+        }
+    }
+
     fn poll_poisson(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
+        // The average delay could change depending on if backpressure in the downstream channel
+        // (mix_tx) was detected.
+        self.adjust_current_average_message_sending_delay();
+        let avg_delay = self.current_average_message_sending_delay();
+
+        // Start by checking if we have any incoming messages about closed connections
+        // NOTE: this feels a bit iffy, the `OutQueueControl` is getting ripe for a rewrite to
+        // something simpler.
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.closed_connection_rx).poll_next(cx) {
+            self.on_close_connection(id);
+        }
+
         if let Some(ref mut next_delay) = &mut self.next_delay {
             // it is not yet time to return a message
             if next_delay.as_mut().poll(cx).is_pending() {
@@ -243,7 +330,6 @@ where
 
             // we know it's time to send a message, so let's prepare delay for the next one
             // Get the `now` by looking at the current `delay` deadline
-            let avg_delay = self.config.average_message_sending_delay;
             let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
 
             // The next interval value is `next_poisson_delay` after the one that just
@@ -260,28 +346,35 @@ where
                 next_delay.as_mut().reset(next_poisson_delay);
             }
 
-            // check if we have anything immediately available
-            if let Some(real_available) = self.received_buffer.pop_front() {
-                return Poll::Ready(Some(StreamMessage::Real(Box::new(real_available))));
-            }
-
-            // decide what kind of message to send
+            // On every iteration we get new messages from upstream. Given that these come bunched
+            // in `Vec`, this ensures that on average we will fetch messages faster than we can
+            // send, which is a condition for being able to multiplex sphinx packets from multiple
+            // data streams.
             match Pin::new(&mut self.real_receiver).poll_next(cx) {
                 // in the case our real message channel stream was closed, we should also indicate we are closed
                 // (and whoever is using the stream should panic)
                 Poll::Ready(None) => Poll::Ready(None),
 
-                // if there are more messages available, return first one and store the rest
-                Poll::Ready(Some(real_messages)) => {
-                    self.received_buffer = real_messages.into();
-                    // we MUST HAVE received at least ONE message
-                    Poll::Ready(Some(StreamMessage::Real(Box::new(
-                        self.received_buffer.pop_front().unwrap(),
-                    ))))
+                Poll::Ready(Some((real_messages, conn_id))) => {
+                    log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                    self.transmission_buffer.store(&conn_id, real_messages);
+                    let real_next = self
+                        .transmission_buffer
+                        .pop_next_message_at_random()
+                        .expect("we just added one");
+
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
                 }
 
-                // otherwise construct a dummy one
-                Poll::Pending => Poll::Ready(Some(StreamMessage::Cover)),
+                Poll::Pending => {
+                    if let Some(real_next) = self.transmission_buffer.pop_next_message_at_random() {
+                        Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                    } else {
+                        // otherwise construct a dummy one
+                        Poll::Ready(Some(StreamMessage::Cover))
+                    }
+                }
             }
         } else {
             // we never set an initial delay - let's do it now
@@ -303,14 +396,9 @@ where
     }
 
     fn poll_immediate(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
-        // check if we have anything immediately available
-        if let Some(real_available) = self.received_buffer.pop_front() {
-            // if there are more messages immediately available, notify the runtime
-            // because we should be polled again
-            if !self.received_buffer.is_empty() {
-                cx.waker().wake_by_ref()
-            }
-            return Poll::Ready(Some(StreamMessage::Real(Box::new(real_available))));
+        // Start by checking if we have any incoming messages about closed connections
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.closed_connection_rx).poll_next(cx) {
+            self.on_close_connection(id);
         }
 
         match Pin::new(&mut self.real_receiver).poll_next(cx) {
@@ -318,17 +406,26 @@ where
             // (and whoever is using the stream should panic)
             Poll::Ready(None) => Poll::Ready(None),
 
-            // if there are more messages available, return first one and store the rest
-            Poll::Ready(Some(real_messages)) => {
-                self.received_buffer = real_messages.into();
-                // we MUST HAVE received at least ONE message
-                Poll::Ready(Some(StreamMessage::Real(Box::new(
-                    self.received_buffer.pop_front().unwrap(),
-                ))))
+            Poll::Ready(Some((real_messages, conn_id))) => {
+                log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                // First store what we got for the given connection id
+                self.transmission_buffer.store(&conn_id, real_messages);
+                let real_next = self
+                    .transmission_buffer
+                    .pop_next_message_at_random()
+                    .expect("we just added one");
+
+                Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
             }
 
-            // if there's nothing, then there's nothing
-            Poll::Pending => Poll::Pending,
+            Poll::Pending => {
+                if let Some(real_next) = self.transmission_buffer.pop_next_message_at_random() {
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                } else {
+                    Poll::Pending
+                }
+            }
         }
     }
 
@@ -343,24 +440,47 @@ where
         }
     }
 
+    #[cfg(not(target_arch = "wasm32"))]
+    fn log_status(&self) {
+        let packets = self.transmission_buffer.total_size();
+        let backlog = self.transmission_buffer.total_size_in_bytes() as f64 / 1024.0;
+        let lanes = self.transmission_buffer.num_lanes();
+        let mult = self.sending_delay_controller.current_multiplier();
+        let delay = self.current_average_message_sending_delay().as_millis();
+        if self.config.disable_poisson_packet_distribution {
+            log::info!(
+                "Status: {lanes} lanes, backlog: {:.2} kiB ({packets}), no delay",
+                backlog
+            );
+        } else {
+            log::info!(
+                "Status: {lanes} lanes, backlog: {:.2} kiB ({packets}), avg delay: {}ms ({mult})",
+                backlog,
+                delay
+            );
+        }
+    }
+
     #[cfg(not(target_arch = "wasm32"))]
     pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
         debug!("Started OutQueueControl with graceful shutdown support");
 
+        let mut status_timer = tokio::time::interval(Duration::from_secs(1));
+
         while !shutdown.is_shutdown() {
             tokio::select! {
                 biased;
                 _ = shutdown.recv() => {
                     log::trace!("OutQueueControl: Received shutdown");
                 }
-                next_message = self.next() => match next_message {
-                    Some(next_message) => {
-                        self.on_message(next_message).await;
-                    },
-                    None => {
-                        log::trace!("OutQueueControl: Stopping since channel closed");
-                        break;
-                    }
+                _ = status_timer.tick() => {
+                    self.log_status();
+                }
+                next_message = self.next() => if let Some(next_message) = next_message {
+                    self.on_message(next_message).await;
+                } else {
+                    log::trace!("OutQueueControl: Stopping since channel closed");
+                    break;
                 }
             }
         }

clients/client-core/src/client/real_messages_control/real_traffic_stream/sending_delay_controller.rs

@@ -0,0 +1,124 @@
+// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use super::get_time_now;
+use std::time::Duration;
+
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::time;
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+// The minimum time between increasing the average delay between packets. If we hit the ceiling in
+// the available buffer space we want to take somewhat swift action, but we still need to give a
+// short time to give the channel a chance reduce pressure.
+const INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 1;
+// The minimum time between decreasing the average delay between packets. We don't want to change
+// to quickly to keep things somewhat stable. Also there are buffers downstreams meaning we need to
+// wait a little to see the effect before we decrease further.
+const DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 30;
+// If we enough time passes without any sign of backpressure in the channel, we can consider
+// lowering the average delay. The goal is to keep somewhat stable, rather than maxing out
+// bandwidth at all times.
+const ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS: u64 = 30;
+// The maximum multiplier we apply to the base average Poisson delay.
+const MAX_DELAY_MULTIPLIER: u32 = 6;
+// The minium multiplier we apply to the base average Poisson delay.
+const MIN_DELAY_MULTIPLIER: u32 = 1;
+
+pub(crate) struct SendingDelayController {
+    /// Multiply the average sending delay.
+    /// This is normally set to unity, but if we detect backpressure we increase this
+    /// multiplier. We use discrete steps.
+    current_multiplier: u32,
+
+    /// Maximum delay multiplier
+    upper_bound: u32,
+
+    /// Minimum delay multiplier
+    lower_bound: u32,
+
+    /// To make sure we don't change the multiplier to fast, we limit a change to some duration
+    #[cfg(not(target_arch = "wasm32"))]
+    time_when_changed: time::Instant,
+
+    #[cfg(target_arch = "wasm32")]
+    time_when_changed: wasm_timer::Instant,
+
+    /// If we have a long enough time without any backpressure detected we try reducing the sending
+    /// delay multiplier
+    #[cfg(not(target_arch = "wasm32"))]
+    time_when_backpressure_detected: time::Instant,
+
+    #[cfg(target_arch = "wasm32")]
+    time_when_backpressure_detected: wasm_timer::Instant,
+}
+
+impl Default for SendingDelayController {
+    fn default() -> Self {
+        SendingDelayController::new(MIN_DELAY_MULTIPLIER, MAX_DELAY_MULTIPLIER)
+    }
+}
+
+impl SendingDelayController {
+    pub(crate) fn new(lower_bound: u32, upper_bound: u32) -> Self {
+        assert!(lower_bound <= upper_bound);
+        let now = get_time_now();
+        SendingDelayController {
+            current_multiplier: MIN_DELAY_MULTIPLIER,
+            upper_bound,
+            lower_bound,
+            time_when_changed: now,
+            time_when_backpressure_detected: now,
+        }
+    }
+
+    pub(crate) fn current_multiplier(&self) -> u32 {
+        self.current_multiplier
+    }
+
+    pub(crate) fn increase_delay_multiplier(&mut self) {
+        if self.current_multiplier < self.upper_bound {
+            self.current_multiplier =
+                (self.current_multiplier + 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::debug!(
+                "Increasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        } else {
+            log::warn!("Trying to increase delay multipler higher than allowed");
+        }
+    }
+
+    pub(crate) fn decrease_delay_multiplier(&mut self) {
+        if self.current_multiplier > self.lower_bound {
+            self.current_multiplier =
+                (self.current_multiplier - 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::debug!(
+                "Decreasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        }
+    }
+
+    pub(crate) fn record_backpressure_detected(&mut self) {
+        self.time_when_backpressure_detected = get_time_now();
+    }
+
+    pub(crate) fn not_increased_delay_recently(&self) -> bool {
+        get_time_now()
+            > self.time_when_changed + Duration::from_secs(INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS)
+    }
+
+    pub(crate) fn is_sending_reliable(&self) -> bool {
+        let now = get_time_now();
+        let delay_change_interval = Duration::from_secs(DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS);
+        let acceptable_time_without_backpressure =
+            Duration::from_secs(ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS);
+
+        now > self.time_when_backpressure_detected + acceptable_time_without_backpressure
+            && now > self.time_when_changed + delay_change_interval
+    }
+}

clients/client-core/src/client/real_messages_control/real_traffic_stream/transmission_buffer.rs

@@ -0,0 +1,207 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use client_connections::TransmissionLane;
+use rand::seq::SliceRandom;
+use std::{
+    collections::{HashMap, HashSet, VecDeque},
+    time::Duration,
+};
+
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::time;
+
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+use super::{get_time_now, RealMessage};
+
+// The number of lanes included in the oldest set. Used when we need to prioritize traffic.
+const OLDEST_LANE_SET_SIZE: usize = 5;
+// As a way of prune connections we also check for timeouts.
+const MSG_CONSIDERED_STALE_AFTER_SECS: u64 = 10 * 60;
+
+#[derive(Default)]
+pub(crate) struct TransmissionBuffer {
+    buffer: HashMap<TransmissionLane, LaneBufferEntry>,
+}
+
+impl TransmissionBuffer {
+    #[allow(unused)]
+    pub(crate) fn is_empty(&self) -> bool {
+        self.buffer.is_empty()
+    }
+
+    pub(crate) fn remove(&mut self, lane: &TransmissionLane) -> Option<LaneBufferEntry> {
+        self.buffer.remove(lane)
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn num_lanes(&self) -> usize {
+        self.buffer.keys().count()
+    }
+
+    #[allow(unused)]
+    pub(crate) fn connections(&self) -> HashSet<u64> {
+        self.buffer
+            .keys()
+            .filter_map(|lane| match lane {
+                TransmissionLane::ConnectionId(id) => Some(id),
+                _ => None,
+            })
+            .copied()
+            .collect()
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn total_size(&self) -> usize {
+        self.buffer.values().map(LaneBufferEntry::len).sum()
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn total_size_in_bytes(&self) -> usize {
+        self.buffer
+            .values()
+            .map(|lane_buffer_entry| {
+                lane_buffer_entry
+                    .real_messages
+                    .iter()
+                    .map(|real_message| real_message.mix_packet.sphinx_packet().len())
+                    .sum::<usize>()
+            })
+            .sum()
+    }
+
+    fn get_oldest_set(&self) -> Vec<TransmissionLane> {
+        let mut buffer: Vec<_> = self
+            .buffer
+            .iter()
+            .map(|(k, v)| (k, v.messages_transmitted))
+            .collect();
+        buffer.sort_by_key(|v| v.1);
+        buffer
+            .iter()
+            .rev()
+            .map(|(k, _)| *k)
+            .take(OLDEST_LANE_SET_SIZE)
+            .copied()
+            .collect()
+    }
+
+    pub(crate) fn store(&mut self, lane: &TransmissionLane, real_messages: Vec<RealMessage>) {
+        if let Some(lane_buffer_entry) = self.buffer.get_mut(lane) {
+            lane_buffer_entry.append(real_messages);
+        } else {
+            self.buffer
+                .insert(*lane, LaneBufferEntry::new(real_messages));
+        }
+    }
+
+    fn pick_random_lane(&self) -> Option<&TransmissionLane> {
+        let lanes: Vec<&TransmissionLane> = self.buffer.keys().collect();
+        lanes.choose(&mut rand::thread_rng()).copied()
+    }
+
+    fn pick_random_small_lane(&self) -> Option<&TransmissionLane> {
+        let lanes: Vec<&TransmissionLane> = self
+            .buffer
+            .iter()
+            .filter(|(_, v)| v.is_small())
+            .map(|(k, _)| k)
+            .collect();
+        lanes.choose(&mut rand::thread_rng()).copied()
+    }
+
+    fn pick_random_old_lane(&self) -> Option<TransmissionLane> {
+        let lanes = self.get_oldest_set();
+        lanes.choose(&mut rand::thread_rng()).copied()
+    }
+
+    fn pop_front_from_lane(&mut self, lane: &TransmissionLane) -> Option<RealMessage> {
+        let real_msgs_queued = self.buffer.get_mut(lane)?;
+        let real_next = real_msgs_queued.pop_front()?;
+        real_msgs_queued.messages_transmitted += 1;
+        if real_msgs_queued.is_empty() {
+            self.buffer.remove(lane);
+        }
+        Some(real_next)
+    }
+
+    pub(crate) fn pop_next_message_at_random(&mut self) -> Option<RealMessage> {
+        if self.buffer.is_empty() {
+            return None;
+        }
+
+        // Very basic heuristic where we prioritize according to small lanes first, the older lanes
+        // to try to finish lanes when possible, then the rest.
+        let lane = if let Some(small_lane) = self.pick_random_small_lane() {
+            *small_lane
+        } else if let Some(old_lane) = self.pick_random_old_lane() {
+            old_lane
+        } else {
+            *self.pick_random_lane()?
+        };
+
+        log::trace!("picking to send from lane: {:?}", lane);
+        self.pop_front_from_lane(&lane)
+    }
+
+    pub(crate) fn prune_stale_connections(&mut self) {
+        let stale_entries: Vec<_> = self
+            .buffer
+            .iter()
+            .filter_map(|(lane, entry)| if entry.is_stale() { Some(lane) } else { None })
+            .copied()
+            .collect();
+
+        for lane in stale_entries {
+            self.remove(&lane);
+        }
+    }
+}
+
+pub(crate) struct LaneBufferEntry {
+    pub real_messages: VecDeque<RealMessage>,
+    pub messages_transmitted: usize,
+    #[cfg(not(target_arch = "wasm32"))]
+    pub time_for_last_activity: time::Instant,
+    #[cfg(target_arch = "wasm32")]
+    pub time_for_last_activity: wasm_timer::Instant,
+}
+
+impl LaneBufferEntry {
+    fn new(real_messages: Vec<RealMessage>) -> Self {
+        LaneBufferEntry {
+            real_messages: real_messages.into(),
+            messages_transmitted: 0,
+            time_for_last_activity: get_time_now(),
+        }
+    }
+
+    fn append(&mut self, real_messages: Vec<RealMessage>) {
+        self.real_messages.append(&mut real_messages.into());
+        self.time_for_last_activity = get_time_now();
+    }
+
+    fn pop_front(&mut self) -> Option<RealMessage> {
+        self.real_messages.pop_front()
+    }
+
+    fn is_small(&self) -> bool {
+        self.real_messages.len() < 100
+    }
+
+    fn is_stale(&self) -> bool {
+        get_time_now() - self.time_for_last_activity
+            > Duration::from_secs(MSG_CONSIDERED_STALE_AFTER_SECS)
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn len(&self) -> usize {
+        self.real_messages.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.real_messages.is_empty()
+    }
+}

clients/client-core/src/client/received_buffer.rs

@@ -208,7 +208,7 @@ impl ReceivedMessagesBuffer {
     }
 
     async fn handle_new_received(&mut self, msgs: Vec<Vec<u8>>) {
-        debug!(
+        trace!(
             "Processing {:?} new message that might get added to the buffer!",
             msgs.len()
         );

clients/client-core/src/client/topology_control.rs

@@ -57,24 +57,15 @@ impl<'a> TopologyReadPermit<'a> {
     ) -> Option<&'a NymTopology> {
         // Note: implicit deref with Deref for TopologyReadPermit is happening here
         let topology_ref_option = self.permit.as_ref();
-        match topology_ref_option {
-            None => None,
-            Some(topology_ref) => {
-                // see if it's possible to route the packet to both gateways
-                if !topology_ref.can_construct_path_through(DEFAULT_NUM_MIX_HOPS)
-                    || !topology_ref.gateway_exists(ack_recipient.gateway())
-                    || if let Some(packet_recipient) = packet_recipient {
-                        !topology_ref.gateway_exists(packet_recipient.gateway())
-                    } else {
-                        false
-                    }
-                {
-                    None
+        topology_ref_option.as_ref().filter(|topology_ref| {
+            !(!topology_ref.can_construct_path_through(DEFAULT_NUM_MIX_HOPS)
+                || !topology_ref.gateway_exists(ack_recipient.gateway())
+                || if let Some(packet_recipient) = packet_recipient {
+                    !topology_ref.gateway_exists(packet_recipient.gateway())
                 } else {
-                    Some(topology_ref)
-                }
-            }
-        }
+                    false
+                })
+        })
     }
 }
 

clients/client-core/src/config/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use config::NymConfig;
+use nymsphinx::params::PacketSize;
 use serde::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -247,8 +248,8 @@ impl<T: NymConfig> Config<T> {
         self.debug.disable_main_poisson_packet_distribution
     }
 
-    pub fn get_use_extended_packet_size(&self) -> bool {
-        self.debug.use_extended_packet_size
+    pub fn get_use_extended_packet_size(&self) -> Option<ExtendedPacketSize> {
+        self.debug.use_extended_packet_size.clone()
     }
 
     pub fn get_version(&self) -> &str {
@@ -470,8 +471,16 @@ pub struct Debug {
     /// poisson distribution.
     pub disable_main_poisson_packet_distribution: bool,
 
-    /// Controls whether the sent sphinx packet use the NON-DEFAULT bigger size.
-    pub use_extended_packet_size: bool,
+    /// Controls whether the sent sphinx packet use a NON-DEFAULT bigger size.
+    pub use_extended_packet_size: Option<ExtendedPacketSize>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ExtendedPacketSize {
+    Extended8,
+    Extended16,
+    Extended32,
 }
 
 impl Default for Debug {
@@ -488,7 +497,17 @@ impl Default for Debug {
             topology_resolution_timeout: DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT,
             disable_loop_cover_traffic_stream: false,
             disable_main_poisson_packet_distribution: false,
-            use_extended_packet_size: false,
+            use_extended_packet_size: None,
+        }
+    }
+}
+
+impl From<ExtendedPacketSize> for PacketSize {
+    fn from(size: ExtendedPacketSize) -> PacketSize {
+        match size {
+            ExtendedPacketSize::Extended8 => PacketSize::ExtendedPacket8,
+            ExtendedPacketSize::Extended16 => PacketSize::ExtendedPacket16,
+            ExtendedPacketSize::Extended32 => PacketSize::ExtendedPacket32,
         }
     }
 }

clients/client-core/src/error.rs

@@ -24,4 +24,6 @@ pub enum ClientCoreError {
     ListOfValidatorApisIsEmpty,
     #[error("Could not load existing gateway configuration: {0}")]
     CouldNotLoadExistingGatewayConfiguration(std::io::Error),
+    #[error("The current network topology seem to be insufficient to route any packets through")]
+    InsufficientNetworkTopology,
 }

clients/credential/src/main.rs

@@ -22,7 +22,7 @@ cfg_if::cfg_if! {
         #[clap(author = "Nymtech", version, about)]
         struct Cli {
             /// Path pointing to an env file that configures the client.
-            #[clap(long)]
+            #[clap(short, long)]
             pub(crate) config_env_file: Option<std::path::PathBuf>,
 
             /// Path where the sqlite credental database will be located.

clients/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nym-client"
-version = "1.0.2"
+version = "1.1.0"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 description = "Implementation of the Nym Client"
 edition = "2021"
@@ -27,17 +27,20 @@ pretty_env_logger = "0.4" # for formatting log messages
 rand = { version = "0.7.3", features = ["wasm-bindgen"] } # rng-related traits + some rng implementation to use
 serde = { version = "1.0.104", features = ["derive"] } # for config serialization/deserialization
 sled = "0.34" # for storage of replySURB decryption keys
+thiserror = "1.0.34"
 tokio = { version = "1.21.2", features = ["rt-multi-thread", "net", "signal"] } # async runtime
 tokio-tungstenite = "0.14" # websocket
 
 ## internal
 client-core = { path = "../client-core" }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 config = { path = "../../common/config" }
 completions = { path = "../../common/completions" }
 credential-storage = { path = "../../common/credential-storage" }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
+logging = { path = "../../common/logging"}
 gateway-client = { path = "../../common/client-libs/gateway-client" }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 network-defaults = { path = "../../common/network-defaults" }

clients/native/examples/websocket_binarysend.rs

@@ -43,6 +43,7 @@ async fn send_file_with_reply() {
         recipient,
         message: read_data,
         with_reply_surb: true,
+        connection_id: 0,
     };
 
     println!("sending content of 'dummy_file' over the mix network...");
@@ -91,6 +92,7 @@ async fn send_file_without_reply() {
         recipient,
         message: read_data,
         with_reply_surb: false,
+        connection_id: 0,
     };
 
     println!("sending content of 'dummy_file' over the mix network...");

clients/native/src/client/mod.rs

@@ -1,14 +1,13 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use client_connections::{ClosedConnectionReceiver, ClosedConnectionSender, TransmissionLane};
 use client_core::client::cover_traffic_stream::LoopCoverTrafficStream;
 use client_core::client::inbound_messages::{
     InputMessage, InputMessageReceiver, InputMessageSender,
 };
 use client_core::client::key_manager::KeyManager;
-use client_core::client::mix_traffic::{
-    BatchMixMessageReceiver, BatchMixMessageSender, MixTrafficController,
-};
+use client_core::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
 use client_core::client::real_messages_control;
 use client_core::client::real_messages_control::RealMessagesController;
 use client_core::client::received_buffer::{
@@ -20,6 +19,7 @@ use client_core::client::topology_control::{
     TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
 };
 use client_core::config::persistence::key_pathfinder::ClientKeyPathfinder;
+use client_core::error::ClientCoreError;
 use crypto::asymmetric::identity;
 use futures::channel::mpsc;
 use gateway_client::bandwidth::BandwidthController;
@@ -31,11 +31,11 @@ use log::*;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::addressing::nodes::NodeIdentity;
 use nymsphinx::anonymous_replies::ReplySurb;
-use nymsphinx::params::PacketSize;
 use nymsphinx::receiver::ReconstructedMessage;
 use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
 
 use crate::client::config::{Config, SocketType};
+use crate::error::ClientError;
 use crate::websocket;
 
 pub(crate) mod config;
@@ -103,13 +103,15 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
         }
 
         stream.start_with_shutdown(shutdown);
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn start_real_traffic_controller(
         &self,
         topology_accessor: TopologyAccessor,
@@ -117,6 +119,7 @@ impl NymClient {
         ack_receiver: AcknowledgementReceiver,
         input_receiver: InputMessageReceiver,
         mix_sender: BatchMixMessageSender,
+        closed_connection_rx: ClosedConnectionReceiver,
         shutdown: ShutdownListener,
     ) {
         let mut controller_config = real_messages_control::Config::new(
@@ -132,8 +135,9 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
         }
 
         info!("Starting real traffic stream...");
@@ -145,6 +149,7 @@ impl NymClient {
             mix_sender,
             topology_accessor,
             reply_key_storage,
+            closed_connection_rx,
         )
         .start_with_shutdown(shutdown);
     }
@@ -233,7 +238,7 @@ impl NymClient {
         &mut self,
         topology_accessor: TopologyAccessor,
         shutdown: ShutdownListener,
-    ) {
+    ) -> Result<(), ClientError> {
         let topology_refresher_config = TopologyRefresherConfig::new(
             self.config.get_base().get_validator_api_endpoints(),
             self.config.get_base().get_topology_refresh_rate(),
@@ -248,14 +253,16 @@ impl NymClient {
 
         // TODO: a slightly more graceful termination here
         if !topology_refresher.is_topology_routable().await {
-            panic!(
-                "The current network topology seem to be insufficient to route any packets through\
+            log::error!(
+                "The current network topology seem to be insufficient to route any packets through \
                 - check if enough nodes and a gateway are online"
             );
+            return Err(ClientCoreError::InsufficientNetworkTopology.into());
         }
 
         info!("Starting topology refresher...");
         topology_refresher.start_with_shutdown(shutdown);
+        Ok(())
     }
 
     // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
@@ -263,24 +270,29 @@ impl NymClient {
     // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
     // requests?
     fn start_mix_traffic_controller(
-        &mut self,
-        mix_rx: BatchMixMessageReceiver,
         gateway_client: GatewayClient,
         shutdown: ShutdownListener,
-    ) {
+    ) -> BatchMixMessageSender {
         info!("Starting mix traffic controller...");
-        MixTrafficController::new(mix_rx, gateway_client).start_with_shutdown(shutdown);
+        let (mix_traffic_controller, mix_tx) = MixTrafficController::new(gateway_client);
+        mix_traffic_controller.start_with_shutdown(shutdown);
+        mix_tx
     }
 
     fn start_websocket_listener(
         &self,
         buffer_requester: ReceivedBufferRequestSender,
         msg_input: InputMessageSender,
+        closed_connection_tx: ClosedConnectionSender,
     ) {
         info!("Starting websocket listener...");
 
-        let websocket_handler =
-            websocket::Handler::new(msg_input, buffer_requester, self.as_mix_recipient());
+        let websocket_handler = websocket::Handler::new(
+            msg_input,
+            closed_connection_tx,
+            buffer_requester,
+            &self.as_mix_recipient(),
+        );
 
         websocket::Listener::new(self.config.get_listening_port()).start(websocket_handler);
     }
@@ -289,7 +301,8 @@ impl NymClient {
     /// It's untested and there are absolutely no guarantees about it (but seems to have worked
     /// well enough in local tests)
     pub fn send_message(&mut self, recipient: Recipient, message: Vec<u8>, with_reply_surb: bool) {
-        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb);
+        let lane = TransmissionLane::General;
+        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb, lane);
 
         self.input_tx
             .as_ref()
@@ -329,8 +342,8 @@ impl NymClient {
     }
 
     /// blocking version of `start` method. Will run forever (or until SIGINT is sent)
-    pub async fn run_forever(&mut self) {
-        let shutdown = self.start().await;
+    pub async fn run_forever(&mut self) -> Result<(), ClientError> {
+        let shutdown = self.start().await?;
         wait_for_signal().await;
 
         println!(
@@ -347,20 +360,16 @@ impl NymClient {
         //shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-client");
+        Ok(())
     }
 
-    pub async fn start(&mut self) -> ShutdownNotifier {
+    pub async fn start(&mut self) -> Result<ShutdownNotifier, ClientError> {
         info!("Starting nym client");
         // channels for inter-component communication
         // TODO: make the channels be internally created by the relevant components
         // rather than creating them here, so say for example the buffer controller would create the request channels
         // and would allow anyone to clone the sender channel
 
-        // sphinx_message_sender is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
-        // they are used by cover traffic stream and real traffic stream
-        // sphinx_message_receiver is the receiver used by MixTrafficController that sends the actual traffic
-        let (sphinx_message_sender, sphinx_message_receiver) = mpsc::unbounded();
-
         // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
         // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
         let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
@@ -385,7 +394,7 @@ impl NymClient {
         // the components are started in very specific order. Unless you know what you are doing,
         // do not change that.
         self.start_topology_refresher(shared_topology_accessor.clone(), shutdown.subscribe())
-            .await;
+            .await?;
         self.start_received_messages_buffer_controller(
             received_buffer_request_receiver,
             mixnet_messages_receiver,
@@ -397,17 +406,24 @@ impl NymClient {
             .start_gateway_client(mixnet_messages_sender, ack_sender, shutdown.subscribe())
             .await;
 
-        self.start_mix_traffic_controller(
-            sphinx_message_receiver,
-            gateway_client,
-            shutdown.subscribe(),
-        );
+        // The sphinx_message_sender is the transmitter for any component generating sphinx packets
+        // that are to be sent to the mixnet. They are used by cover traffic stream and real
+        // traffic stream.
+        // The MixTrafficController then sends the actual traffic
+        let sphinx_message_sender =
+            Self::start_mix_traffic_controller(gateway_client, shutdown.subscribe());
+
+        // Channels that the websocket listener can use to signal downstream to the real traffic
+        // controller that connections are closed.
+        let (closed_connection_tx, closed_connection_rx) = mpsc::unbounded();
+
         self.start_real_traffic_controller(
             shared_topology_accessor.clone(),
             reply_key_storage,
             ack_receiver,
             input_receiver,
             sphinx_message_sender.clone(),
+            closed_connection_rx,
             shutdown.subscribe(),
         );
 
@@ -424,9 +440,11 @@ impl NymClient {
         }
 
         match self.config.get_socket_type() {
-            SocketType::WebSocket => {
-                self.start_websocket_listener(received_buffer_request_sender, input_sender)
-            }
+            SocketType::WebSocket => self.start_websocket_listener(
+                received_buffer_request_sender,
+                input_sender,
+                closed_connection_tx,
+            ),
             SocketType::None => {
                 // if we did not start the socket, it means we're running (supposedly) in the native mode
                 // and hence we should announce 'ourselves' to the buffer
@@ -447,6 +465,6 @@ impl NymClient {
         info!("Client startup finished!");
         info!("The address of this client is: {}", self.as_mix_recipient());
 
-        shutdown
+        Ok(shutdown)
     }
 }

clients/native/src/commands/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::client::config::{Config, SocketType};
+use crate::error::ClientError;
 use clap::CommandFactory;
 use clap::{Parser, Subcommand};
 use completions::{fig_generate, ArgShell};
@@ -49,7 +50,7 @@ fn long_version_static() -> &'static str {
 #[clap(author = "Nymtech", version, long_version = long_version_static(), about)]
 pub(crate) struct Cli {
     /// Path pointing to an env file that configures the client.
-    #[clap(long)]
+    #[clap(short, long)]
     pub(crate) config_env_file: Option<std::path::PathBuf>,
 
     #[clap(subcommand)]
@@ -83,16 +84,17 @@ pub(crate) struct OverrideConfig {
     enabled_credentials_mode: bool,
 }
 
-pub(crate) async fn execute(args: &Cli) {
+pub(crate) async fn execute(args: &Cli) -> Result<(), ClientError> {
     let bin_name = "nym-native-client";
 
     match &args.command {
         Commands::Init(m) => init::execute(m).await,
-        Commands::Run(m) => run::execute(m).await,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::Upgrade(m) => upgrade::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::into_app(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::into_app(), bin_name),
     }
+    Ok(())
 }
 
 pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Config {

clients/native/src/commands/run.rs

@@ -4,6 +4,7 @@
 use crate::{
     client::{config::Config, NymClient},
     commands::{override_config, OverrideConfig},
+    error::ClientError,
 };
 
 use clap::Args;
@@ -73,14 +74,14 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) {
+pub(crate) async fn execute(args: &Run) -> Result<(), ClientError> {
     let id = &args.id;
 
     let mut config = match Config::load_from_file(Some(id)) {
         Ok(cfg) => cfg,
         Err(err) => {
             error!("Failed to load config for {}. Are you sure you have run `init` before? (Error was: {})", id, err);
-            return;
+            return Err(ClientError::FailedToLoadConfig(id.to_string()));
         }
     };
 
@@ -89,8 +90,8 @@ pub(crate) async fn execute(args: &Run) {
 
     if !version_check(&config) {
         error!("failed the local version check");
-        return;
+        return Err(ClientError::FailedLocalVersionCheck);
     }
 
-    NymClient::new(config).run_forever().await;
+    NymClient::new(config).run_forever().await
 }

clients/native/src/error.rs

@@ -0,0 +1,23 @@
+use client_core::error::ClientCoreError;
+use crypto::asymmetric::identity::Ed25519RecoveryError;
+use gateway_client::error::GatewayClientError;
+use validator_client::ValidatorClientError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum ClientError {
+    #[error("I/O error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("Gateway client error: {0}")]
+    GatewayClientError(#[from] GatewayClientError),
+    #[error("Ed25519 error: {0}")]
+    Ed25519RecoveryError(#[from] Ed25519RecoveryError),
+    #[error("Validator client error: {0}")]
+    ValidatorClientError(#[from] ValidatorClientError),
+    #[error("client-core error: {0}")]
+    ClientCoreError(#[from] ClientCoreError),
+
+    #[error("Failed to load config for: {0}")]
+    FailedToLoadConfig(String),
+    #[error("Failed local version check, client and config mismatch")]
+    FailedLocalVersionCheck,
+}

clients/native/src/lib.rs

@@ -2,4 +2,5 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod client;
+pub mod error;
 pub mod websocket;

clients/native/src/main.rs

@@ -2,20 +2,23 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use clap::{crate_version, Parser};
+use error::ClientError;
+use logging::setup_logging;
 use network_defaults::setup_env;
 
 pub mod client;
 pub mod commands;
+pub mod error;
 pub mod websocket;
 
 #[tokio::main]
-async fn main() {
+async fn main() -> Result<(), ClientError> {
     setup_logging();
     println!("{}", banner());
 
     let args = commands::Cli::parse();
     setup_env(args.config_env_file.clone());
-    commands::execute(&args).await;
+    commands::execute(&args).await
 }
 
 fn banner() -> String {
@@ -34,25 +37,3 @@ fn banner() -> String {
         crate_version!()
     )
 }
-
-fn setup_logging() {
-    let mut log_builder = pretty_env_logger::formatted_timed_builder();
-    if let Ok(s) = ::std::env::var("RUST_LOG") {
-        log_builder.parse_filters(&s);
-    } else {
-        // default to 'Info'
-        log_builder.filter(None, log::LevelFilter::Info);
-    }
-
-    log_builder
-        .filter_module("hyper", log::LevelFilter::Warn)
-        .filter_module("tokio_reactor", log::LevelFilter::Warn)
-        .filter_module("reqwest", log::LevelFilter::Warn)
-        .filter_module("mio", log::LevelFilter::Warn)
-        .filter_module("want", log::LevelFilter::Warn)
-        .filter_module("tungstenite", log::LevelFilter::Warn)
-        .filter_module("tokio_tungstenite", log::LevelFilter::Warn)
-        .filter_module("handlebars", log::LevelFilter::Warn)
-        .filter_module("sled", log::LevelFilter::Warn)
-        .init();
-}

clients/native/src/websocket/handler.rs

@@ -1,6 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use client_connections::{ClosedConnectionSender, TransmissionLane};
 use client_core::client::{
     inbound_messages::{InputMessage, InputMessageSender},
     received_buffer::{
@@ -34,6 +35,7 @@ impl Default for ReceivedResponseType {
 
 pub(crate) struct Handler {
     msg_input: InputMessageSender,
+    closed_connection_tx: ClosedConnectionSender,
     buffer_requester: ReceivedBufferRequestSender,
     self_full_address: Recipient,
     socket: Option<WebSocketStream<TcpStream>>,
@@ -45,6 +47,7 @@ impl Clone for Handler {
     fn clone(&self) -> Self {
         Handler {
             msg_input: self.msg_input.clone(),
+            closed_connection_tx: self.closed_connection_tx.clone(),
             buffer_requester: self.buffer_requester.clone(),
             self_full_address: self.self_full_address,
             socket: None,
@@ -64,13 +67,15 @@ impl Drop for Handler {
 impl Handler {
     pub(crate) fn new(
         msg_input: InputMessageSender,
+        closed_connection_tx: ClosedConnectionSender,
         buffer_requester: ReceivedBufferRequestSender,
-        self_full_address: Recipient,
+        self_full_address: &Recipient,
     ) -> Self {
         Handler {
             msg_input,
+            closed_connection_tx,
             buffer_requester,
-            self_full_address,
+            self_full_address: *self_full_address,
             socket: None,
             received_response_type: Default::default(),
         }
@@ -78,12 +83,14 @@ impl Handler {
 
     fn handle_send(
         &mut self,
-        recipient: Recipient,
+        recipient: &Recipient,
         message: Vec<u8>,
         with_reply_surb: bool,
+        connection_id: u64,
     ) -> Option<ServerResponse> {
         // the ack control is now responsible for chunking, etc.
-        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb);
+        let lane = TransmissionLane::ConnectionId(connection_id);
+        let input_msg = InputMessage::new_fresh(*recipient, message, with_reply_surb, lane);
         self.msg_input.unbounded_send(input_msg).unwrap();
 
         None
@@ -104,18 +111,27 @@ impl Handler {
         ServerResponse::SelfAddress(self.self_full_address)
     }
 
+    fn handle_closed_connection(&self, connection_id: u64) -> Option<ServerResponse> {
+        self.closed_connection_tx
+            .unbounded_send(connection_id)
+            .unwrap();
+        None
+    }
+
     fn handle_request(&mut self, request: ClientRequest) -> Option<ServerResponse> {
         match request {
             ClientRequest::Send {
                 recipient,
                 message,
                 with_reply_surb,
-            } => self.handle_send(recipient, message, with_reply_surb),
+                connection_id,
+            } => self.handle_send(&recipient, message, with_reply_surb, connection_id),
             ClientRequest::Reply {
                 message,
                 reply_surb,
             } => self.handle_reply(reply_surb, message),
             ClientRequest::SelfAddress => Some(self.handle_self_address()),
+            ClientRequest::ClosedConnection(id) => self.handle_closed_connection(id),
         }
     }
 
@@ -134,7 +150,7 @@ impl Handler {
         response.map(|resp| WsMessage::text(resp.into_text()))
     }
 
-    fn handle_binary_message(&mut self, msg: Vec<u8>) -> Option<WsMessage> {
+    fn handle_binary_message(&mut self, msg: &[u8]) -> Option<WsMessage> {
         debug!("Handling binary message request");
 
         self.received_response_type = ReceivedResponseType::Binary;
@@ -154,37 +170,11 @@ impl Handler {
         // old version of this file.
         match raw_request {
             WsMessage::Text(text_message) => self.handle_text_message(text_message),
-            WsMessage::Binary(binary_message) => self.handle_binary_message(binary_message),
+            WsMessage::Binary(binary_message) => self.handle_binary_message(&binary_message),
             _ => None,
         }
     }
 
-    // I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
-    // let's just play along for now
-    fn prepare_reconstructed_binary(
-        &self,
-        reconstructed_messages: Vec<ReconstructedMessage>,
-    ) -> Vec<Result<WsMessage, WsError>> {
-        reconstructed_messages
-            .into_iter()
-            .map(ServerResponse::Received)
-            .map(|resp| Ok(WsMessage::Binary(resp.into_binary())))
-            .collect()
-    }
-
-    // I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
-    // let's just play along for now
-    fn prepare_reconstructed_text(
-        &self,
-        reconstructed_messages: Vec<ReconstructedMessage>,
-    ) -> Vec<Result<WsMessage, WsError>> {
-        reconstructed_messages
-            .into_iter()
-            .map(ServerResponse::Received)
-            .map(|resp| Ok(WsMessage::Text(resp.into_text())))
-            .collect()
-    }
-
     async fn push_websocket_received_plaintexts(
         &mut self,
         reconstructed_messages: Vec<ReconstructedMessage>,
@@ -193,10 +183,8 @@ impl Handler {
         // if it's text or binary, but for time being we use the naive assumption that if
         // client is sending Message::Text it expects text back. Same for Message::Binary
         let response_messages = match self.received_response_type {
-            ReceivedResponseType::Binary => {
-                self.prepare_reconstructed_binary(reconstructed_messages)
-            }
-            ReceivedResponseType::Text => self.prepare_reconstructed_text(reconstructed_messages),
+            ReceivedResponseType::Binary => prepare_reconstructed_binary(reconstructed_messages),
+            ReceivedResponseType::Text => prepare_reconstructed_text(reconstructed_messages),
         };
 
         let mut send_stream = futures::stream::iter(response_messages);
@@ -291,3 +279,27 @@ impl Handler {
         self.listen_for_requests(reconstructed_receiver).await;
     }
 }
+
+// I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
+// let's just play along for now
+fn prepare_reconstructed_binary(
+    reconstructed_messages: Vec<ReconstructedMessage>,
+) -> Vec<Result<WsMessage, WsError>> {
+    reconstructed_messages
+        .into_iter()
+        .map(ServerResponse::Received)
+        .map(|resp| Ok(WsMessage::Binary(resp.into_binary())))
+        .collect()
+}
+
+// I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
+// let's just play along for now
+fn prepare_reconstructed_text(
+    reconstructed_messages: Vec<ReconstructedMessage>,
+) -> Vec<Result<WsMessage, WsError>> {
+    reconstructed_messages
+        .into_iter()
+        .map(ServerResponse::Received)
+        .map(|resp| Ok(WsMessage::Text(resp.into_text())))
+        .collect()
+}

clients/native/websocket-requests/src/requests.rs

@@ -20,6 +20,9 @@ pub const REPLY_REQUEST_TAG: u8 = 0x01;
 /// Value tag representing [`SelfAddress`] variant of the [`ClientRequest`]
 pub const SELF_ADDRESS_REQUEST_TAG: u8 = 0x02;
 
+/// Value tag representing [`ClosedConnection`] variant of the [`ClientRequest`]
+pub const CLOSED_CONNECTION_REQUEST_TAG: u8 = 0x03;
+
 #[allow(non_snake_case)]
 #[derive(Debug)]
 pub enum ClientRequest {
@@ -28,32 +31,41 @@ pub enum ClientRequest {
         message: Vec<u8>,
         // Perhaps we could change it to a number to indicate how many reply_SURBs we want to include?
         with_reply_surb: bool,
+        connection_id: u64,
     },
     Reply {
         message: Vec<u8>,
         reply_surb: ReplySurb,
     },
     SelfAddress,
+    ClosedConnection(u64),
 }
 
 // we could have been parsing it directly TryFrom<WsMessage>, but we want to retain
 // information about whether it came from binary or text to send appropriate response back
 impl ClientRequest {
-    // SEND_REQUEST_TAG || with_surb || recipient || data_len || data
-    fn serialize_send(recipient: Recipient, data: Vec<u8>, with_reply_surb: bool) -> Vec<u8> {
+    // SEND_REQUEST_TAG || with_surb || recipient || conn_id || data_len || data
+    fn serialize_send(
+        recipient: Recipient,
+        data: Vec<u8>,
+        with_reply_surb: bool,
+        connection_id: u64,
+    ) -> Vec<u8> {
         let data_len_bytes = (data.len() as u64).to_be_bytes();
+        let conn_id_bytes = connection_id.to_be_bytes();
         std::iter::once(SEND_REQUEST_TAG)
             .chain(std::iter::once(with_reply_surb as u8))
             .chain(recipient.to_bytes().iter().cloned()) // will not be length prefixed because the length is constant
+            .chain(conn_id_bytes.iter().cloned())
             .chain(data_len_bytes.iter().cloned())
             .chain(data.into_iter())
             .collect()
     }
 
-    // SEND_REQUEST_TAG || with_reply || recipient || data_len || data
+    // SEND_REQUEST_TAG || with_reply || recipient || conn_id || data_len || data
     fn deserialize_send(b: &[u8]) -> Result<Self, error::Error> {
-        // we need to have at least 1 (tag) + 1 (reply flag) + Recipient::LEN + sizeof<u64> bytes
-        if b.len() < 2 + Recipient::LEN + size_of::<u64>() {
+        // we need to have at least 1 (tag) + 1 (reply flag) + Recipient::LEN + 2*sizeof<u64> bytes
+        if b.len() < 2 + Recipient::LEN + 2 * size_of::<u64>() {
             return Err(error::Error::new(
                 ErrorKind::TooShortRequest,
                 "not enough data provided to recover 'send'".to_string(),
@@ -86,9 +98,15 @@ impl ClientRequest {
             }
         };
 
-        let data_len_bytes = &b[2 + Recipient::LEN..2 + Recipient::LEN + size_of::<u64>()];
+        let mut connection_id_bytes = [0u8; size_of::<u64>()];
+        connection_id_bytes
+            .copy_from_slice(&b[2 + Recipient::LEN..2 + Recipient::LEN + size_of::<u64>()]);
+        let connection_id = u64::from_be_bytes(connection_id_bytes);
+
+        let data_len_bytes =
+            &b[2 + Recipient::LEN + size_of::<u64>()..2 + Recipient::LEN + 2 * size_of::<u64>()];
         let data_len = u64::from_be_bytes(data_len_bytes.try_into().unwrap());
-        let data = &b[2 + Recipient::LEN + size_of::<u64>()..];
+        let data = &b[2 + Recipient::LEN + 2 * size_of::<u64>()..];
         if data.len() as u64 != data_len {
             return Err(error::Error::new(
                 ErrorKind::MalformedRequest,
@@ -104,11 +122,12 @@ impl ClientRequest {
             with_reply_surb,
             recipient,
             message: data.to_vec(),
+            connection_id,
         })
     }
 
     // REPLY_REQUEST_TAG || surb_len || surb || message_len || message
-    fn serialize_reply(message: Vec<u8>, reply_surb: ReplySurb) -> Vec<u8> {
+    fn serialize_reply(message: Vec<u8>, reply_surb: &ReplySurb) -> Vec<u8> {
         let reply_surb_bytes = reply_surb.to_bytes();
         let surb_len_bytes = (reply_surb_bytes.len() as u64).to_be_bytes();
         let message_len_bytes = (message.len() as u64).to_be_bytes();
@@ -202,20 +221,43 @@ impl ClientRequest {
         ClientRequest::SelfAddress
     }
 
+    // CLOSED_CONNECTION_REQUEST_TAG
+    fn serialize_closed_connection(connection_id: u64) -> Vec<u8> {
+        let conn_id_bytes = connection_id.to_be_bytes();
+        std::iter::once(CLOSED_CONNECTION_REQUEST_TAG)
+            .chain(conn_id_bytes.iter().copied())
+            .collect()
+    }
+
+    // CLOSED_CONNECTION_REQUEST_TAG
+    fn deserialize_closed_connection(b: &[u8]) -> Self {
+        // this MUST match because it was called by 'deserialize'
+        debug_assert_eq!(b[0], CLOSED_CONNECTION_REQUEST_TAG);
+
+        let mut connection_id_bytes = [0u8; size_of::<u64>()];
+        connection_id_bytes.copy_from_slice(&b[1..=size_of::<u64>()]);
+        let connection_id = u64::from_be_bytes(connection_id_bytes);
+
+        ClientRequest::ClosedConnection(connection_id)
+    }
+
     pub fn serialize(self) -> Vec<u8> {
         match self {
             ClientRequest::Send {
                 recipient,
                 message,
                 with_reply_surb,
-            } => Self::serialize_send(recipient, message, with_reply_surb),
+                connection_id,
+            } => Self::serialize_send(recipient, message, with_reply_surb, connection_id),
 
             ClientRequest::Reply {
                 message,
                 reply_surb,
-            } => Self::serialize_reply(message, reply_surb),
+            } => Self::serialize_reply(message, &reply_surb),
 
             ClientRequest::SelfAddress => Self::serialize_self_address(),
+
+            ClientRequest::ClosedConnection(id) => Self::serialize_closed_connection(id),
         }
     }
 
@@ -245,15 +287,16 @@ impl ClientRequest {
             SEND_REQUEST_TAG => Self::deserialize_send(b),
             REPLY_REQUEST_TAG => Self::deserialize_reply(b),
             SELF_ADDRESS_REQUEST_TAG => Ok(Self::deserialize_self_address(b)),
+            CLOSED_CONNECTION_REQUEST_TAG => Ok(Self::deserialize_closed_connection(b)),
             n => Err(error::Error::new(
                 ErrorKind::UnknownRequest,
-                format!("type {}", n),
+                format!("type {n}"),
             )),
         }
     }
 
-    pub fn try_from_binary(raw_req: Vec<u8>) -> Result<Self, error::Error> {
-        Self::deserialize(&raw_req)
+    pub fn try_from_binary(raw_req: &[u8]) -> Result<Self, error::Error> {
+        Self::deserialize(raw_req)
     }
 
     pub fn try_from_text(raw_req: String) -> Result<Self, error::Error> {
@@ -280,6 +323,7 @@ mod tests {
             recipient,
             message: b"foomp".to_vec(),
             with_reply_surb: false,
+            connection_id: 42,
         };
 
         let bytes = send_request_no_surb.serialize();
@@ -289,10 +333,12 @@ mod tests {
                 recipient,
                 message,
                 with_reply_surb,
+                connection_id,
             } => {
                 assert_eq!(recipient.to_string(), recipient_string);
                 assert_eq!(message, b"foomp".to_vec());
-                assert!(!with_reply_surb)
+                assert!(!with_reply_surb);
+                assert_eq!(connection_id, 42)
             }
             _ => unreachable!(),
         }
@@ -301,6 +347,7 @@ mod tests {
             recipient,
             message: b"foomp".to_vec(),
             with_reply_surb: true,
+            connection_id: 213,
         };
 
         let bytes = send_request_surb.serialize();
@@ -310,10 +357,12 @@ mod tests {
                 recipient,
                 message,
                 with_reply_surb,
+                connection_id,
             } => {
                 assert_eq!(recipient.to_string(), recipient_string);
                 assert_eq!(message, b"foomp".to_vec());
-                assert!(with_reply_surb)
+                assert!(with_reply_surb);
+                assert_eq!(connection_id, 213)
             }
             _ => unreachable!(),
         }
@@ -352,4 +401,15 @@ mod tests {
             _ => unreachable!(),
         }
     }
+
+    #[test]
+    fn close_connection_request_serialization_works() {
+        let close_connection_request = ClientRequest::ClosedConnection(42);
+        let bytes = close_connection_request.serialize();
+        let recovered = ClientRequest::deserialize(&bytes).unwrap();
+        match recovered {
+            ClientRequest::ClosedConnection(id) => assert_eq!(id, 42),
+            _ => unreachable!(),
+        }
+    }
 }

clients/native/websocket-requests/src/text.rs

@@ -20,6 +20,7 @@ pub(super) enum ClientRequestText {
         message: String,
         recipient: String,
         with_reply_surb: bool,
+        connection_id: u64,
     },
     SelfAddress,
     #[serde(rename_all = "camelCase")]
@@ -46,6 +47,7 @@ impl TryInto<ClientRequest> for ClientRequestText {
                 message,
                 recipient,
                 with_reply_surb,
+                connection_id,
             } => {
                 let message_bytes = message.into_bytes();
                 let recipient = Recipient::try_from_base58_string(recipient).map_err(|err| {
@@ -56,6 +58,7 @@ impl TryInto<ClientRequest> for ClientRequestText {
                     message: message_bytes,
                     recipient,
                     with_reply_surb,
+                    connection_id,
                 })
             }
             ClientRequestText::SelfAddress => Ok(ClientRequest::SelfAddress),

clients/socks5/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nym-socks5-client"
-version = "1.0.2"
+version = "1.1.0"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 description = "A SOCKS5 localhost proxy that converts incoming messages to Sphinx and sends them to a Nym address"
 edition = "2021"
@@ -20,17 +20,20 @@ pretty_env_logger = "0.4"
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
 serde = { version = "1.0", features = ["derive"] } # for config serialization/deserialization
 snafu = "0.6"
+thiserror = "1.0.34"
 tokio = { version = "1.21.2", features = ["rt-multi-thread", "net", "signal"] }
 url = "2.2"
 
 # internal
 client-core = { path = "../client-core" }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 config = { path = "../../common/config" }
 completions = { path = "../../common/completions" }
 credential-storage = { path = "../../common/credential-storage" }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
+logging = { path = "../../common/logging"}
 gateway-client = { path = "../../common/client-libs/gateway-client" }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 network-defaults = { path = "../../common/network-defaults" }

clients/socks5/src/client/mod.rs

@@ -4,18 +4,18 @@
 use std::sync::atomic::Ordering;
 
 use crate::client::config::Config;
+use crate::error::Socks5ClientError;
 use crate::socks::{
     authentication::{AuthenticationMethods, Authenticator, User},
     server::SphinxSocksServer,
 };
+use client_connections::{ClosedConnectionReceiver, ClosedConnectionSender};
 use client_core::client::cover_traffic_stream::LoopCoverTrafficStream;
 use client_core::client::inbound_messages::{
     InputMessage, InputMessageReceiver, InputMessageSender,
 };
 use client_core::client::key_manager::KeyManager;
-use client_core::client::mix_traffic::{
-    BatchMixMessageReceiver, BatchMixMessageSender, MixTrafficController,
-};
+use client_core::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
 use client_core::client::real_messages_control::RealMessagesController;
 use client_core::client::received_buffer::{
     ReceivedBufferRequestReceiver, ReceivedBufferRequestSender, ReceivedMessagesBufferController,
@@ -25,6 +25,7 @@ use client_core::client::topology_control::{
     TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
 };
 use client_core::config::persistence::key_pathfinder::ClientKeyPathfinder;
+use client_core::error::ClientCoreError;
 use crypto::asymmetric::identity;
 use futures::channel::mpsc;
 use futures::StreamExt;
@@ -36,7 +37,6 @@ use gateway_client::{
 use log::*;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::addressing::nodes::NodeIdentity;
-use nymsphinx::params::PacketSize;
 use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
 
 pub mod config;
@@ -103,13 +103,15 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
         }
 
         stream.start_with_shutdown(shutdown);
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn start_real_traffic_controller(
         &self,
         topology_accessor: TopologyAccessor,
@@ -117,6 +119,7 @@ impl NymClient {
         ack_receiver: AcknowledgementReceiver,
         input_receiver: InputMessageReceiver,
         mix_sender: BatchMixMessageSender,
+        closed_connection_rx: ClosedConnectionReceiver,
         shutdown: ShutdownListener,
     ) {
         let mut controller_config = client_core::client::real_messages_control::Config::new(
@@ -132,8 +135,9 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
         }
 
         info!("Starting real traffic stream...");
@@ -145,6 +149,7 @@ impl NymClient {
             mix_sender,
             topology_accessor,
             reply_key_storage,
+            closed_connection_rx,
         )
         .start_with_shutdown(shutdown);
     }
@@ -233,7 +238,7 @@ impl NymClient {
         &mut self,
         topology_accessor: TopologyAccessor,
         shutdown: ShutdownListener,
-    ) {
+    ) -> Result<(), Socks5ClientError> {
         let topology_refresher_config = TopologyRefresherConfig::new(
             self.config.get_base().get_validator_api_endpoints(),
             self.config.get_base().get_topology_refresh_rate(),
@@ -248,14 +253,16 @@ impl NymClient {
 
         // TODO: a slightly more graceful termination here
         if !topology_refresher.is_topology_routable().await {
-            panic!(
-                "The current network topology seem to be insufficient to route any packets through\
+            log::error!(
+                "The current network topology seem to be insufficient to route any packets through \
                 - check if enough nodes and a gateway are online"
             );
+            return Err(ClientCoreError::InsufficientNetworkTopology.into());
         }
 
         info!("Starting topology refresher...");
         topology_refresher.start_with_shutdown(shutdown);
+        Ok(())
     }
 
     // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
@@ -263,19 +270,20 @@ impl NymClient {
     // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
     // requests?
     fn start_mix_traffic_controller(
-        &mut self,
-        mix_rx: BatchMixMessageReceiver,
         gateway_client: GatewayClient,
         shutdown: ShutdownListener,
-    ) {
+    ) -> BatchMixMessageSender {
         info!("Starting mix traffic controller...");
-        MixTrafficController::new(mix_rx, gateway_client).start_with_shutdown(shutdown);
+        let (mix_traffic_controller, mix_tx) = MixTrafficController::new(gateway_client);
+        mix_traffic_controller.start_with_shutdown(shutdown);
+        mix_tx
     }
 
     fn start_socks5_listener(
         &self,
         buffer_requester: ReceivedBufferRequestSender,
         msg_input: InputMessageSender,
+        closed_connection_tx: ClosedConnectionSender,
         shutdown: ShutdownListener,
     ) {
         info!("Starting socks5 listener...");
@@ -290,12 +298,16 @@ impl NymClient {
             self.as_mix_recipient(),
             shutdown,
         );
-        tokio::spawn(async move { sphinx_socks.serve(msg_input, buffer_requester).await });
+        tokio::spawn(async move {
+            sphinx_socks
+                .serve(msg_input, buffer_requester, closed_connection_tx)
+                .await
+        });
     }
 
     /// blocking version of `start` method. Will run forever (or until SIGINT is sent)
-    pub async fn run_forever(&mut self) {
-        let mut shutdown = self.start().await;
+    pub async fn run_forever(&mut self) -> Result<(), Socks5ClientError> {
+        let mut shutdown = self.start().await?;
         wait_for_signal().await;
 
         log::info!("Sending shutdown");
@@ -306,11 +318,15 @@ impl NymClient {
         shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-socks5-client");
+        Ok(())
     }
 
     // Variant of `run_forever` that listends for remote control messages
-    pub async fn run_and_listen(&mut self, mut receiver: Socks5ControlMessageReceiver) {
-        let mut shutdown = self.start().await;
+    pub async fn run_and_listen(
+        &mut self,
+        mut receiver: Socks5ControlMessageReceiver,
+    ) -> Result<(), Socks5ClientError> {
+        let mut shutdown = self.start().await?;
         tokio::select! {
             message = receiver.next() => {
                 log::debug!("Received message: {:?}", message);
@@ -336,20 +352,16 @@ impl NymClient {
         shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-socks5-client");
+        Ok(())
     }
 
-    pub async fn start(&mut self) -> ShutdownNotifier {
+    pub async fn start(&mut self) -> Result<ShutdownNotifier, Socks5ClientError> {
         info!("Starting nym client");
         // channels for inter-component communication
         // TODO: make the channels be internally created by the relevant components
         // rather than creating them here, so say for example the buffer controller would create the request channels
         // and would allow anyone to clone the sender channel
 
-        // sphinx_message_sender is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
-        // they are used by cover traffic stream and real traffic stream
-        // sphinx_message_receiver is the receiver used by MixTrafficController that sends the actual traffic
-        let (sphinx_message_sender, sphinx_message_receiver) = mpsc::unbounded();
-
         // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
         // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
         let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
@@ -374,7 +386,7 @@ impl NymClient {
         // the components are started in very specific order. Unless you know what you are doing,
         // do not change that.
         self.start_topology_refresher(shared_topology_accessor.clone(), shutdown.subscribe())
-            .await;
+            .await?;
         self.start_received_messages_buffer_controller(
             received_buffer_request_receiver,
             mixnet_messages_receiver,
@@ -386,17 +398,24 @@ impl NymClient {
             .start_gateway_client(mixnet_messages_sender, ack_sender, shutdown.subscribe())
             .await;
 
-        self.start_mix_traffic_controller(
-            sphinx_message_receiver,
-            gateway_client,
-            shutdown.subscribe(),
-        );
+        // The sphinx_message_sender is the transmitter for any component generating sphinx packets
+        // that are to be sent to the mixnet. They are used by cover traffic stream and real
+        // traffic stream.
+        // The MixTrafficController then sends the actual traffic
+        let sphinx_message_sender =
+            Self::start_mix_traffic_controller(gateway_client, shutdown.subscribe());
+
+        // Channel for announcing closed (socks5) connections by the controller.
+        // This will be forwarded to `OutQueueControl`
+        let (closed_connection_tx, closed_connection_rx) = mpsc::unbounded();
+
         self.start_real_traffic_controller(
             shared_topology_accessor.clone(),
             reply_key_storage,
             ack_receiver,
             input_receiver,
             sphinx_message_sender.clone(),
+            closed_connection_rx,
             shutdown.subscribe(),
         );
 
@@ -415,12 +434,13 @@ impl NymClient {
         self.start_socks5_listener(
             received_buffer_request_sender,
             input_sender,
+            closed_connection_tx,
             shutdown.subscribe(),
         );
 
         info!("Client startup finished!");
         info!("The address of this client is: {}", self.as_mix_recipient());
 
-        shutdown
+        Ok(shutdown)
     }
 }

clients/socks5/src/commands/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::client::config::Config;
+use crate::error::Socks5ClientError;
 use clap::CommandFactory;
 use clap::{Parser, Subcommand};
 use completions::{fig_generate, ArgShell};
@@ -50,7 +51,7 @@ fn long_version_static() -> &'static str {
 #[clap(author = "Nymtech", version, long_version = long_version_static(), about)]
 pub(crate) struct Cli {
     /// Path pointing to an env file that configures the client.
-    #[clap(long)]
+    #[clap(short, long)]
     pub(crate) config_env_file: Option<std::path::PathBuf>,
 
     #[clap(subcommand)]
@@ -61,8 +62,10 @@ pub(crate) struct Cli {
 pub(crate) enum Commands {
     /// Initialise a Nym client. Do this first!
     Init(init::Init),
+
     /// Run the Nym client with provided configuration client optionally overriding set parameters
     Run(run::Run),
+
     /// Try to upgrade the client
     Upgrade(upgrade::Upgrade),
 
@@ -83,16 +86,17 @@ pub(crate) struct OverrideConfig {
     enabled_credentials_mode: bool,
 }
 
-pub(crate) async fn execute(args: &Cli) {
+pub(crate) async fn execute(args: &Cli) -> Result<(), Socks5ClientError> {
     let bin_name = "nym-socks5-client";
 
     match &args.command {
         Commands::Init(m) => init::execute(m).await,
-        Commands::Run(m) => run::execute(m).await,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::Upgrade(m) => upgrade::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::into_app(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::into_app(), bin_name),
     }
+    Ok(())
 }
 
 pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Config {

clients/socks5/src/commands/run.rs

@@ -4,6 +4,7 @@
 use crate::{
     client::{config::Config, NymClient},
     commands::{override_config, OverrideConfig},
+    error::Socks5ClientError,
 };
 
 use clap::Args;
@@ -80,14 +81,14 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) {
+pub(crate) async fn execute(args: &Run) -> Result<(), Socks5ClientError> {
     let id = &args.id;
 
     let mut config = match Config::load_from_file(Some(id)) {
         Ok(cfg) => cfg,
         Err(err) => {
             error!("Failed to load config for {}. Are you sure you have run `init` before? (Error was: {})", id, err);
-            return;
+            return Err(Socks5ClientError::FailedToLoadConfig(id.to_string()));
         }
     };
 
@@ -96,8 +97,8 @@ pub(crate) async fn execute(args: &Run) {
 
     if !version_check(&config) {
         error!("failed the local version check");
-        return;
+        return Err(Socks5ClientError::FailedLocalVersionCheck);
     }
 
-    NymClient::new(config).run_forever().await;
+    NymClient::new(config).run_forever().await
 }

clients/socks5/src/error.rs

@@ -0,0 +1,23 @@
+use client_core::error::ClientCoreError;
+use crypto::asymmetric::identity::Ed25519RecoveryError;
+use gateway_client::error::GatewayClientError;
+use validator_client::ValidatorClientError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Socks5ClientError {
+    #[error("I/O error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("Gateway client error: {0}")]
+    GatewayClientError(#[from] GatewayClientError),
+    #[error("Ed25519 error: {0}")]
+    Ed25519RecoveryError(#[from] Ed25519RecoveryError),
+    #[error("Validator client error: {0}")]
+    ValidatorClientError(#[from] ValidatorClientError),
+    #[error("client-core error: {0}")]
+    ClientCoreError(#[from] ClientCoreError),
+
+    #[error("Failed to load config for: {0}")]
+    FailedToLoadConfig(String),
+    #[error("Failed local version check, client and config mismatch")]
+    FailedLocalVersionCheck,
+}

clients/socks5/src/lib.rs

@@ -2,4 +2,5 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod client;
+pub mod error;
 pub mod socks;

clients/socks5/src/main.rs

@@ -2,20 +2,23 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use clap::{crate_version, Parser};
+use error::Socks5ClientError;
+use logging::setup_logging;
 use network_defaults::setup_env;
 
 pub mod client;
 mod commands;
+pub mod error;
 pub mod socks;
 
 #[tokio::main]
-async fn main() {
+async fn main() -> Result<(), Socks5ClientError> {
     setup_logging();
     println!("{}", banner());
 
     let args = commands::Cli::parse();
     setup_env(args.config_env_file.clone());
-    commands::execute(&args).await;
+    commands::execute(&args).await
 }
 
 fn banner() -> String {
@@ -34,23 +37,3 @@ fn banner() -> String {
         crate_version!()
     )
 }
-
-fn setup_logging() {
-    let mut log_builder = pretty_env_logger::formatted_timed_builder();
-    if let Ok(s) = ::std::env::var("RUST_LOG") {
-        log_builder.parse_filters(&s);
-    } else {
-        // default to 'Info'
-        log_builder.filter(None, log::LevelFilter::Info);
-    }
-
-    log_builder
-        .filter_module("hyper", log::LevelFilter::Warn)
-        .filter_module("tokio_reactor", log::LevelFilter::Warn)
-        .filter_module("reqwest", log::LevelFilter::Warn)
-        .filter_module("mio", log::LevelFilter::Warn)
-        .filter_module("want", log::LevelFilter::Warn)
-        .filter_module("tungstenite", log::LevelFilter::Warn)
-        .filter_module("tokio_tungstenite", log::LevelFilter::Warn)
-        .init();
-}

clients/socks5/src/socks/client.rs

@@ -4,8 +4,8 @@ use super::authentication::{AuthenticationMethods, Authenticator, User};
 use super::request::{SocksCommand, SocksRequest};
 use super::types::{ResponseCode, SocksProxyError};
 use super::{RESERVED, SOCKS_VERSION};
-use client_core::client::inbound_messages::InputMessage;
-use client_core::client::inbound_messages::InputMessageSender;
+use client_connections::TransmissionLane;
+use client_core::client::inbound_messages::{InputMessage, InputMessageSender};
 use futures::channel::mpsc;
 use futures::task::{Context, Poll};
 use log::*;
@@ -226,17 +226,21 @@ impl SocksClient {
         }
     }
 
-    async fn send_connect_to_mixnet(&mut self, remote_address: RemoteAddress) {
+    fn send_connect_to_mixnet(&mut self, remote_address: RemoteAddress) {
         let req = Request::new_connect(self.connection_id, remote_address, self.self_address);
         let msg = Message::Request(req);
 
-        let input_message = InputMessage::new_fresh(self.service_provider, msg.into_bytes(), false);
+        let input_message = InputMessage::new_fresh(
+            self.service_provider,
+            msg.into_bytes(),
+            false,
+            TransmissionLane::ConnectionId(self.connection_id),
+        );
         self.input_sender.unbounded_send(input_message).unwrap();
     }
 
     async fn run_proxy(&mut self, conn_receiver: ConnectionReceiver, remote_proxy_target: String) {
-        self.send_connect_to_mixnet(remote_proxy_target.clone())
-            .await;
+        self.send_connect_to_mixnet(remote_proxy_target.clone());
 
         let stream = self.stream.run_proxy();
         let local_stream_remote = stream
@@ -259,7 +263,8 @@ impl SocksClient {
         .run(move |conn_id, read_data, socket_closed| {
             let provider_request = Request::new_send(conn_id, read_data, socket_closed);
             let provider_message = Message::Request(provider_request);
-            InputMessage::new_fresh(recipient, provider_message.into_bytes(), false)
+            let lane = TransmissionLane::ConnectionId(conn_id);
+            InputMessage::new_fresh(recipient, provider_message.into_bytes(), false, lane)
         })
         .await
         .into_inner();

clients/socks5/src/socks/server.rs

@@ -4,6 +4,7 @@ use super::{
     mixnet_responses::MixnetResponseListener,
     types::{ResponseCode, SocksProxyError},
 };
+use client_connections::ClosedConnectionSender;
 use client_core::client::{
     inbound_messages::InputMessageSender, received_buffer::ReceivedBufferRequestSender,
 };
@@ -51,13 +52,14 @@ impl SphinxSocksServer {
         &mut self,
         input_sender: InputMessageSender,
         buffer_requester: ReceivedBufferRequestSender,
+        closed_connection_tx: ClosedConnectionSender,
     ) -> Result<(), SocksProxyError> {
         let listener = TcpListener::bind(self.listening_address).await.unwrap();
         info!("Serving Connections...");
 
         // controller for managing all active connections
         let (mut active_streams_controller, controller_sender) =
-            Controller::new(self.shutdown.clone());
+            Controller::new(closed_connection_tx, self.shutdown.clone());
         tokio::spawn(async move {
             active_streams_controller.run().await;
         });

clients/webassembly/Cargo.toml

@@ -29,6 +29,7 @@ url = "2.2"
 
 # internal
 client-core = { path = "../client-core", default-features = false, features = ["wasm"] }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
@@ -59,4 +60,4 @@ wasm-opt = true
 
 [profile.release]
 lto = true
-opt-level = 'z'
+opt-level = 'z'

clients/webassembly/Makefile

@@ -0,0 +1,5 @@
+clippy:
+	cargo clippy --all-features --target wasm32-unknown-unknown -- -D warnings
+
+test:
+	wasm-pack test --node

clients/webassembly/src/binary_message_helper.rs

@@ -0,0 +1,217 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsCast;
+
+#[wasm_bindgen(typescript_custom_section)]
+const TS_DEFS: &'static str = r#"
+export interface BinaryMessage {
+    kind: number,
+    payload: Uint8Array;
+    headers: string,
+}
+
+export interface StringMessage {
+    kind: number,
+    payload: string;
+}
+"#;
+
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen(typescript_type = "BinaryMessage")]
+    pub type IBinaryMessage;
+    #[wasm_bindgen(typescript_type = "StringMessage")]
+    pub type IStringMessage;
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BinaryMessage {
+    pub kind: u8,
+    pub payload: Vec<u8>,
+    pub headers: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct StringMessage {
+    pub kind: u8,
+    pub payload: String,
+}
+
+/// Create a new binary message with a user-specified `kind`.
+#[wasm_bindgen]
+pub fn create_binary_message(kind: u8, payload: Vec<u8>) -> Vec<u8> {
+    create_binary_message_with_headers(kind, payload, "".to_string())
+}
+
+/// Create a new message with a UTF-8 encoded string `payload` and a user-specified `kind`.
+#[wasm_bindgen]
+pub fn create_binary_message_from_string(kind: u8, payload: String) -> Vec<u8> {
+    create_binary_message_with_headers(kind, payload.as_bytes().to_vec(), "".to_string())
+}
+
+/// Create a new binary message with a user-specified `kind`, and `headers` as a string.
+#[wasm_bindgen]
+pub fn create_binary_message_with_headers(kind: u8, payload: Vec<u8>, headers: String) -> Vec<u8> {
+    let headers = headers.as_bytes().to_vec();
+    let size = (headers.len() as u64).to_be_bytes().to_vec();
+    vec![vec![kind], size, headers, payload].concat()
+}
+
+/// Parse the `kind` and byte array `payload` from a byte array
+#[wasm_bindgen]
+pub async fn parse_binary_message(message: Vec<u8>) -> Result<IBinaryMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, _headers, payload) = parse_binary_payload(&message);
+
+    Ok(serde_wasm_bindgen::to_value(&BinaryMessage {
+        kind,
+        payload: payload.to_vec(),
+        headers: "".to_string(),
+    })
+    .unwrap()
+    .unchecked_into::<IBinaryMessage>())
+}
+
+/// Parse the `kind` and byte array `payload` from a byte array with headers
+#[wasm_bindgen]
+pub async fn parse_binary_message_with_headers(
+    message: Vec<u8>,
+) -> Result<IBinaryMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, headers, payload) = parse_binary_payload(&message);
+
+    Ok(serde_wasm_bindgen::to_value(&BinaryMessage {
+        kind,
+        payload: payload.to_vec(),
+        headers,
+    })
+    .unwrap()
+    .unchecked_into::<IBinaryMessage>())
+}
+
+/// Parse the `kind` and UTF-8 string `payload` from a byte array with headers
+#[wasm_bindgen]
+pub async fn parse_string_message_with_headers(
+    message: Vec<u8>,
+) -> Result<IStringMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, _headers, payload) = parse_binary_payload(&message);
+    let payload = String::from_utf8_lossy(payload).into_owned();
+
+    Ok(
+        serde_wasm_bindgen::to_value(&StringMessage { kind, payload })
+            .unwrap()
+            .unchecked_into::<IStringMessage>(),
+    )
+}
+pub(crate) fn parse_binary_payload(message: &[u8]) -> (u8, String, &[u8]) {
+    // 1st byte is the kind
+    let kind = message[0];
+
+    // then the size as u64 big endian
+    let mut size = [0u8; 8];
+    size.clone_from_slice(&message[1..9]);
+    let size = u64::from_be_bytes(size) as usize;
+
+    // then the headers
+    let headers = String::from_utf8_lossy(&message[9..9 + size]).into_owned();
+
+    // finally the payload
+    let payload = &message[9 + size..];
+
+    (kind, headers, payload)
+}
+
+/// Parse the `kind` and UTF-8 string `payload` from a byte array
+#[wasm_bindgen]
+pub async fn parse_string_message(message: Vec<u8>) -> Result<IStringMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let kind = message[0];
+    let payload = String::from_utf8_lossy(&message[1..]).into_owned();
+
+    Ok(
+        serde_wasm_bindgen::to_value(&StringMessage { kind, payload })
+            .unwrap()
+            .unchecked_into::<IStringMessage>(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{create_binary_message_with_headers, parse_binary_payload};
+    use wasm_bindgen_test::*;
+
+    #[wasm_bindgen_test]
+    fn test_binary_with_headers() {
+        let message_as_bytes = create_binary_message_with_headers(
+            42u8,
+            vec![0u8, 1u8, 2u8],
+            "test headers".to_string(),
+        );
+
+        // calculate header size
+        let headers = "test headers".as_bytes().to_vec();
+        let size = headers.len();
+
+        // the expected size
+        let expected_size = 12;
+        assert_eq!(size, expected_size);
+
+        assert_eq!(message_as_bytes[0], 42u8);
+        assert_eq!(message_as_bytes[1..9], 12u64.to_be_bytes());
+        assert_eq!(
+            message_as_bytes[9 + expected_size..9 + expected_size + 3],
+            vec![0u8, 1u8, 2u8]
+        );
+
+        let res = parse_binary_payload(&message_as_bytes);
+
+        assert_eq!(res.0, 42u8);
+        assert_eq!(res.1, "test headers".to_string());
+        assert_eq!(res.2, vec![0u8, 1u8, 2u8]);
+    }
+
+    #[wasm_bindgen_test]
+    fn test_binary_with_empty_headers() {
+        let message_as_bytes =
+            create_binary_message_with_headers(42u8, vec![0u8, 1u8, 2u8], "".to_string());
+
+        let expected_size = 0;
+
+        assert_eq!(message_as_bytes[0], 42u8);
+        assert_eq!(message_as_bytes[1..9], 0u64.to_be_bytes());
+        assert_eq!(
+            message_as_bytes[9 + expected_size..9 + expected_size + 3],
+            vec![0u8, 1u8, 2u8]
+        );
+
+        let res = parse_binary_payload(&message_as_bytes);
+
+        assert_eq!(res.0, 42u8);
+        assert_eq!(res.1, "".to_string());
+        assert_eq!(res.2, vec![0u8, 1u8, 2u8]);
+    }
+}

clients/webassembly/src/client/config.rs

@@ -4,7 +4,7 @@
 // due to expansion of #[wasm_bindgen] macro on `Debug` Config struct
 #![allow(clippy::drop_non_drop)]
 
-use client_core::config::{Debug as ConfigDebug, GatewayEndpoint};
+use client_core::config::{Debug as ConfigDebug, ExtendedPacketSize, GatewayEndpoint};
 use std::time::Duration;
 use url::Url;
 use wasm_bindgen::prelude::*;
@@ -107,6 +107,11 @@ pub struct Debug {
 
 impl From<Debug> for ConfigDebug {
     fn from(debug: Debug) -> Self {
+        // For now we just always use the (older) 32kb extended size
+        let use_extended_packet_size = debug
+            .use_extended_packet_size
+            .then(|| ExtendedPacketSize::Extended32);
+
         ConfigDebug {
             average_packet_delay: Duration::from_millis(debug.average_packet_delay_ms),
             average_ack_delay: Duration::from_millis(debug.average_ack_delay_ms),
@@ -126,7 +131,7 @@ impl From<Debug> for ConfigDebug {
             disable_loop_cover_traffic_stream: debug.disable_loop_cover_traffic_stream,
             disable_main_poisson_packet_distribution: debug
                 .disable_main_poisson_packet_distribution,
-            use_extended_packet_size: debug.use_extended_packet_size,
+            use_extended_packet_size,
         }
     }
 }
@@ -148,7 +153,7 @@ impl From<ConfigDebug> for Debug {
             disable_loop_cover_traffic_stream: debug.disable_loop_cover_traffic_stream,
             disable_main_poisson_packet_distribution: debug
                 .disable_main_poisson_packet_distribution,
-            use_extended_packet_size: debug.use_extended_packet_size,
+            use_extended_packet_size: debug.use_extended_packet_size.is_some(),
         }
     }
 }

clients/webassembly/src/client/mod.rs

@@ -2,11 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use self::config::Config;
+use client_connections::{ClosedConnectionReceiver, TransmissionLane};
 use client_core::client::{
     cover_traffic_stream::LoopCoverTrafficStream,
     inbound_messages::{InputMessage, InputMessageReceiver, InputMessageSender},
     key_manager::KeyManager,
-    mix_traffic::{BatchMixMessageReceiver, BatchMixMessageSender, MixTrafficController},
+    mix_traffic::{BatchMixMessageSender, MixTrafficController},
     real_messages_control::{self, RealMessagesController},
     received_buffer::{
         ReceivedBufferMessage, ReceivedBufferRequestReceiver, ReceivedBufferRequestSender,
@@ -22,11 +23,10 @@ use gateway_client::{
     MixnetMessageSender,
 };
 use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::params::PacketSize;
 use rand::rngs::OsRng;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::spawn_local;
-use wasm_utils::{console_log, console_warn};
+use wasm_utils::console_log;
 
 pub mod config;
 
@@ -45,6 +45,7 @@ pub struct NymClient {
 
     // callbacks
     on_message: Option<js_sys::Function>,
+    on_binary_message: Option<js_sys::Function>,
     on_gateway_connect: Option<js_sys::Function>,
 }
 
@@ -56,6 +57,7 @@ impl NymClient {
             config,
             key_manager: Self::setup_key_manager(),
             on_message: None,
+            on_binary_message: None,
             on_gateway_connect: None,
             input_tx: None,
         }
@@ -73,8 +75,11 @@ impl NymClient {
         self.on_message = Some(on_message);
     }
 
+    pub fn set_on_binary_message(&mut self, on_binary_message: js_sys::Function) {
+        self.on_binary_message = Some(on_binary_message);
+    }
+
     pub fn set_on_gateway_connect(&mut self, on_connect: js_sys::Function) {
-        console_log!("setting on connect...");
         self.on_gateway_connect = Some(on_connect)
     }
 
@@ -110,8 +115,8 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.debug.use_extended_packet_size {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = &self.config.debug.use_extended_packet_size {
+            stream.set_custom_packet_size(size.clone().into());
         }
 
         stream.start();
@@ -123,6 +128,7 @@ impl NymClient {
         ack_receiver: AcknowledgementReceiver,
         input_receiver: InputMessageReceiver,
         mix_sender: BatchMixMessageSender,
+        closed_connection_rx: ClosedConnectionReceiver,
     ) {
         let mut controller_config = real_messages_control::Config::new(
             self.key_manager.ack_key(),
@@ -135,8 +141,8 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.debug.use_extended_packet_size {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = &self.config.debug.use_extended_packet_size {
+            controller_config.set_custom_packet_size(size.clone().into());
         }
 
         console_log!("Starting real traffic stream...");
@@ -147,6 +153,7 @@ impl NymClient {
             input_receiver,
             mix_sender,
             topology_accessor,
+            closed_connection_rx,
         )
         .start();
     }
@@ -253,13 +260,11 @@ impl NymClient {
     // TODO: if we want to send control messages to gateway_client, this CAN'T take the ownership
     // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
     // requests?
-    fn start_mix_traffic_controller(
-        &mut self,
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-    ) {
+    fn start_mix_traffic_controller(gateway_client: GatewayClient) -> BatchMixMessageSender {
         console_log!("Starting mix traffic controller...");
-        MixTrafficController::new(mix_rx, gateway_client).start();
+        let (mix_traffic_controller, mix_tx) = MixTrafficController::new(gateway_client);
+        mix_traffic_controller.start();
+        mix_tx
     }
 
     // TODO: this procedure is extremely overcomplicated, because it's based off native client's behaviour
@@ -269,6 +274,7 @@ impl NymClient {
         received_buffer_request_sender: ReceivedBufferRequestSender,
     ) {
         let on_message = self.on_message.take();
+        let on_binary_message = self.on_binary_message.take();
 
         spawn_local(async move {
             let (reconstructed_sender, mut reconstructed_receiver) = mpsc::unbounded();
@@ -283,8 +289,14 @@ impl NymClient {
             let this = JsValue::null();
 
             while let Some(reconstructed) = reconstructed_receiver.next().await {
-                if let Some(ref callback) = on_message {
-                    for msg in reconstructed {
+                for msg in reconstructed {
+                    if let Some(ref callback_binary) = on_binary_message {
+                        let arg1 = serde_wasm_bindgen::to_value(&msg.message).unwrap();
+                        callback_binary
+                            .call1(&this, &arg1)
+                            .expect("on binary message failed!");
+                    }
+                    if let Some(ref callback) = on_message {
                         if msg.reply_surb.is_some() {
                             console_log!("the received message contained a reply-surb that we do not know how to handle (yet)")
                         }
@@ -292,9 +304,6 @@ impl NymClient {
                         let arg1 = serde_wasm_bindgen::to_value(&stringified).unwrap();
                         callback.call1(&this, &arg1).expect("on message failed!");
                     }
-                } else {
-                    console_warn!("no on_message callback was specified. the received message content is getting dropped");
-                    console_log!("the raw messages: {:?}", reconstructed)
                 }
             }
         });
@@ -307,11 +316,6 @@ impl NymClient {
         // rather than creating them here, so say for example the buffer controller would create the request channels
         // and would allow anyone to clone the sender channel
 
-        // sphinx_message_sender is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
-        // they are used by cover traffic stream and real traffic stream
-        // sphinx_message_receiver is the receiver used by MixTrafficController that sends the actual traffic
-        let (sphinx_message_sender, sphinx_message_receiver) = mpsc::unbounded();
-
         // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
         // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
         let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
@@ -326,6 +330,10 @@ impl NymClient {
         let (ack_sender, ack_receiver) = mpsc::unbounded();
         let shared_topology_accessor = TopologyAccessor::new();
 
+        // Channel that the real traffix controller can listed to for closing connections.
+        // Currently unused in the wasm client.
+        let (_closed_connection_tx, closed_connection_rx) = mpsc::unbounded();
+
         // the components are started in very specific order. Unless you know what you are doing,
         // do not change that.
         self.start_topology_refresher(shared_topology_accessor.clone())
@@ -339,12 +347,18 @@ impl NymClient {
             .start_gateway_client(mixnet_messages_sender, ack_sender)
             .await;
 
-        self.start_mix_traffic_controller(sphinx_message_receiver, gateway_client);
+        // The sphinx_message_sender is the transmitter for any component generating sphinx packets
+        // that are to be sent to the mixnet. They are used by cover traffic stream and real
+        // traffic stream.
+        // The MixTrafficController then sends the actual traffic
+        let sphinx_message_sender = Self::start_mix_traffic_controller(gateway_client);
+
         self.start_real_traffic_controller(
             shared_topology_accessor.clone(),
             ack_receiver,
             input_receiver,
             sphinx_message_sender.clone(),
+            closed_connection_rx,
         );
 
         if !self.config.debug.disable_loop_cover_traffic_stream {
@@ -370,8 +384,9 @@ impl NymClient {
         console_log!("Sending {} bytes to {}", message.len(), recipient);
 
         let recipient = Recipient::try_from_base58_string(recipient).unwrap();
+        let lane = TransmissionLane::General;
 
-        let input_msg = InputMessage::new_fresh(recipient, message, false);
+        let input_msg = InputMessage::new_fresh(recipient, message, false, lane);
 
         self.input_tx
             .as_ref()

clients/webassembly/src/lib.rs

@@ -3,6 +3,8 @@
 
 use wasm_bindgen::prelude::*;
 
+#[cfg(target_arch = "wasm32")]
+pub mod binary_message_helper;
 #[cfg(target_arch = "wasm32")]
 mod client;
 #[cfg(target_arch = "wasm32")]

common/client-connections/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "client-connections"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+futures = "0.3"

common/client-connections/src/lib.rs

@@ -0,0 +1,21 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use futures::channel::mpsc;
+
+pub type ConnectionId = u64;
+
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
+pub enum TransmissionLane {
+    General,
+    Reply,
+    Retransmission,
+    Control,
+    ConnectionId(ConnectionId),
+}
+
+/// Announce connections that are closed, for whoever is interested.
+/// One usecase is that the network-requester and socks5-client wants to know about this, so that
+/// they can forward this to the `OutQueueControl` (via `ClientRequest` for the network-requester)
+pub type ClosedConnectionSender = mpsc::UnboundedSender<ConnectionId>;
+pub type ClosedConnectionReceiver = mpsc::UnboundedReceiver<ConnectionId>;

common/client-libs/gateway-client/src/packet_router.rs

@@ -62,9 +62,19 @@ impl PacketRouter {
                 trace!("routing regular packet");
                 received_messages.push(received_packet);
             } else if received_packet.len()
-                == PacketSize::ExtendedPacket.plaintext_size() - ack_overhead
+                == PacketSize::ExtendedPacket8.plaintext_size() - ack_overhead
             {
-                trace!("routing extended packet");
+                trace!("routing extended8 packet");
+                received_messages.push(received_packet);
+            } else if received_packet.len()
+                == PacketSize::ExtendedPacket16.plaintext_size() - ack_overhead
+            {
+                trace!("routing extended16 packet");
+                received_messages.push(received_packet);
+            } else if received_packet.len()
+                == PacketSize::ExtendedPacket32.plaintext_size() - ack_overhead
+            {
+                trace!("routing extended32 packet");
                 received_messages.push(received_packet);
             } else {
                 // this can happen if other clients are not padding their messages

common/client-libs/mixnet-client/src/client.rs

@@ -23,6 +23,7 @@ pub struct Config {
     maximum_reconnection_backoff: Duration,
     initial_connection_timeout: Duration,
     maximum_connection_buffer_size: usize,
+    use_legacy_version: bool,
 }
 
 impl Config {
@@ -31,12 +32,14 @@ impl Config {
         maximum_reconnection_backoff: Duration,
         initial_connection_timeout: Duration,
         maximum_connection_buffer_size: usize,
+        use_legacy_version: bool,
     ) -> Self {
         Config {
             initial_reconnection_backoff,
             maximum_reconnection_backoff,
             initial_connection_timeout,
             maximum_connection_buffer_size,
+            use_legacy_version,
         }
     }
 }
@@ -201,7 +204,8 @@ impl SendWithoutResponse for Client {
         packet_mode: PacketMode,
     ) -> io::Result<()> {
         trace!("Sending packet to {:?}", address);
-        let framed_packet = FramedSphinxPacket::new(packet, packet_mode);
+        let framed_packet =
+            FramedSphinxPacket::new(packet, packet_mode, self.config.use_legacy_version);
 
         if let Some(sender) = self.conn_new.get_mut(&address) {
             if let Err(err) = sender.channel.try_send(framed_packet) {
@@ -259,6 +263,7 @@ mod tests {
             maximum_reconnection_backoff: Duration::from_millis(300_000),
             initial_connection_timeout: Duration::from_millis(1_500),
             maximum_connection_buffer_size: 128,
+            use_legacy_version: false,
         })
     }
 

common/client-libs/mixnet-client/src/forwarder.rs

@@ -24,12 +24,14 @@ impl PacketForwarder {
         maximum_reconnection_backoff: Duration,
         initial_connection_timeout: Duration,
         maximum_connection_buffer_size: usize,
+        use_legacy_version: bool,
     ) -> (PacketForwarder, MixForwardingSender) {
         let client_config = Config::new(
             initial_reconnection_backoff,
             maximum_reconnection_backoff,
             initial_connection_timeout,
             maximum_connection_buffer_size,
+            use_legacy_version,
         );
 
         let (packet_sender, packet_receiver) = mpsc::unbounded();

common/client-libs/validator-client/Cargo.toml

@@ -13,6 +13,7 @@ colored = "2.0"
 cw3 = "0.13.1"
 mixnet-contract-common = { path= "../../cosmwasm-smart-contracts/mixnet-contract" }
 vesting-contract-common = { path= "../../cosmwasm-smart-contracts/vesting-contract" }
+contracts-common = { path = "../../cosmwasm-smart-contracts/contracts-common" }
 coconut-bandwidth-contract-common = { path= "../../cosmwasm-smart-contracts/coconut-bandwidth-contract" }
 multisig-contract-common = { path = "../../cosmwasm-smart-contracts/multisig-contract" }
 vesting-contract = { path = "../../../contracts/vesting" }

common/client-libs/validator-client/src/nymd/mod.rs

@@ -15,14 +15,12 @@ use cosmrs::rpc::query::Query;
 use cosmrs::rpc::Error as TendermintRpcError;
 use cosmrs::rpc::HttpClientUrl;
 use cosmrs::tx::Msg;
-use cosmwasm_std::Uint128;
 use execute::execute;
 use network_defaults::{ChainDetails, NymNetworkDetails};
 use serde::{Deserialize, Serialize};
 use std::convert::TryInto;
 use std::time::SystemTime;
 use vesting_contract_common::ExecuteMsg as VestingExecuteMsg;
-use vesting_contract_common::QueryMsg as VestingQueryMsg;
 
 pub use crate::nymd::cosmwasm_client::client::CosmWasmClient;
 pub use crate::nymd::cosmwasm_client::signing_client::SigningCosmWasmClient;
@@ -47,6 +45,7 @@ pub use fee::{gas_price::GasPrice, GasAdjustable, GasAdjustment};
 use mixnet_contract_common::MixId;
 pub use signing_client::Client as SigningNymdClient;
 pub use traits::{VestingQueryClient, VestingSigningClient};
+use vesting_contract_common::PledgeCap;
 
 pub mod coin;
 pub mod cosmwasm_client;
@@ -482,16 +481,6 @@ impl<C> NymdClient<C> {
         self.client.get_total_supply().await
     }
 
-    pub async fn vesting_get_locked_pledge_cap(&self) -> Result<Uint128, NymdError>
-    where
-        C: CosmWasmClient + Sync,
-    {
-        let request = VestingQueryMsg::GetLockedPledgeCap {};
-        self.client
-            .query_contract_smart(self.vesting_contract_address(), &request)
-            .await
-    }
-
     pub async fn simulate<I, M>(&self, messages: I) -> Result<SimulateResponse, NymdError>
     where
         C: SigningCosmWasmClient + Sync,
@@ -737,12 +726,16 @@ impl<C> NymdClient<C> {
     #[execute("vesting")]
     fn _vesting_update_locked_pledge_cap(
         &self,
-        amount: Uint128,
+        address: String,
+        cap: PledgeCap,
         fee: Option<Fee>,
     ) -> (VestingExecuteMsg, Option<Fee>)
     where
         C: SigningCosmWasmClient + Sync,
     {
-        (VestingExecuteMsg::UpdateLockedPledgeCap { amount }, fee)
+        (
+            VestingExecuteMsg::UpdateLockedPledgeCap { address, cap },
+            fee,
+        )
     }
 }

common/client-libs/validator-client/src/nymd/traits/vesting_query_client.rs

@@ -6,8 +6,10 @@ pub use crate::nymd::cosmwasm_client::client::CosmWasmClient;
 use crate::nymd::error::NymdError;
 use crate::nymd::NymdClient;
 use async_trait::async_trait;
+use contracts_common::ContractBuildInformation;
 use cosmwasm_std::{Coin as CosmWasmCoin, Timestamp};
 use mixnet_contract_common::MixId;
+use serde::Deserialize;
 use vesting_contract::vesting::Account;
 use vesting_contract_common::{
     messages::QueryMsg as VestingQueryMsg, AllDelegationsResponse, DelegationTimesResponse,
@@ -16,6 +18,15 @@ use vesting_contract_common::{
 
 #[async_trait]
 pub trait VestingQueryClient {
+    async fn query_vesting_contract<T>(&self, query: VestingQueryMsg) -> Result<T, NymdError>
+    where
+        for<'a> T: Deserialize<'a>;
+
+    async fn get_vesting_contract_version(&self) -> Result<ContractBuildInformation, NymdError> {
+        self.query_vesting_contract(VestingQueryMsg::GetContractVersion {})
+            .await
+    }
+
     async fn locked_coins(
         &self,
         address: &str,
@@ -107,6 +118,15 @@ pub trait VestingQueryClient {
 
 #[async_trait]
 impl<C: CosmWasmClient + Sync + Send> VestingQueryClient for NymdClient<C> {
+    async fn query_vesting_contract<T>(&self, query: VestingQueryMsg) -> Result<T, NymdError>
+    where
+        for<'a> T: Deserialize<'a>,
+    {
+        self.client
+            .query_contract_smart(self.vesting_contract_address(), &query)
+            .await
+    }
+
     async fn locked_coins(
         &self,
         vesting_account_address: &str,

common/client-libs/validator-client/src/nymd/traits/vesting_signing_client.rs

@@ -9,6 +9,7 @@ use async_trait::async_trait;
 use mixnet_contract_common::mixnode::{MixNodeConfigUpdate, MixNodeCostParams};
 use mixnet_contract_common::{Gateway, MixId, MixNode};
 use vesting_contract_common::messages::{ExecuteMsg as VestingExecuteMsg, VestingSpecification};
+use vesting_contract_common::PledgeCap;
 
 #[async_trait]
 pub trait VestingSigningClient {
@@ -105,6 +106,7 @@ pub trait VestingSigningClient {
         staking_address: Option<String>,
         vesting_spec: Option<VestingSpecification>,
         amount: Coin,
+        cap: Option<PledgeCap>,
         fee: Option<Fee>,
     ) -> Result<ExecuteResult, NymdError>;
 }
@@ -382,6 +384,7 @@ impl<C: SigningCosmWasmClient + Sync + Send> VestingSigningClient for NymdClient
         staking_address: Option<String>,
         vesting_spec: Option<VestingSpecification>,
         amount: Coin,
+        cap: Option<PledgeCap>,
         fee: Option<Fee>,
     ) -> Result<ExecuteResult, NymdError> {
         let fee = fee.unwrap_or(Fee::Auto(Some(self.simulated_gas_multiplier)));
@@ -389,6 +392,7 @@ impl<C: SigningCosmWasmClient + Sync + Send> VestingSigningClient for NymdClient
             owner_address: owner_address.to_string(),
             staking_address,
             vesting_spec,
+            cap,
         };
         self.client
             .execute(

common/client-libs/validator-client/src/validator_api/error.rs

@@ -1,4 +1,5 @@
 use thiserror::Error;
+use validator_api_requests::models::RequestError;
 
 #[derive(Error, Debug)]
 pub enum ValidatorAPIError {
@@ -10,4 +11,7 @@ pub enum ValidatorAPIError {
 
     #[error("Request failed with error message - {0}")]
     GenericRequestFailure(String),
+
+    #[error("The validator API has failed to resolve our request. It returned status code {status} and additional error message: {}", error.message())]
+    ApiRequestFailure { status: u16, error: RequestError },
 }

common/client-libs/validator-client/src/validator_api/mod.rs

@@ -5,6 +5,7 @@ use crate::validator_api::error::ValidatorAPIError;
 use crate::validator_api::routes::{CORE_STATUS_COUNT, SINCE_ARG};
 use mixnet_contract_common::mixnode::MixNodeDetails;
 use mixnet_contract_common::{GatewayBond, IdentityKeyRef, MixId};
+use reqwest::Response;
 use serde::{Deserialize, Serialize};
 use url::Url;
 use validator_api_requests::coconut::{
@@ -12,9 +13,10 @@ use validator_api_requests::coconut::{
     VerifyCredentialBody, VerifyCredentialResponse,
 };
 use validator_api_requests::models::{
-    GatewayCoreStatusResponse, InclusionProbabilityResponse, MixNodeBondAnnotated,
-    MixnodeCoreStatusResponse, MixnodeStatusResponse, RewardEstimationResponse,
-    StakeSaturationResponse, UptimeResponse,
+    GatewayCoreStatusResponse, GatewayStatusReportResponse, GatewayUptimeHistoryResponse,
+    InclusionProbabilityResponse, MixNodeBondAnnotated, MixnodeCoreStatusResponse,
+    MixnodeStatusReportResponse, MixnodeStatusResponse, MixnodeUptimeHistoryResponse, RequestError,
+    RewardEstimationResponse, StakeSaturationResponse, UptimeResponse,
 };
 
 pub mod error;
@@ -47,6 +49,19 @@ impl Client {
         &self.url
     }
 
+    async fn send_get_request<K, V>(
+        &self,
+        path: PathSegments<'_>,
+        params: Params<'_, K, V>,
+    ) -> Result<Response, ValidatorAPIError>
+    where
+        K: AsRef<str>,
+        V: AsRef<str>,
+    {
+        let url = create_api_url(&self.url, path, params);
+        Ok(self.reqwest_client.get(url).send().await?)
+    }
+
     async fn query_validator_api<T, K, V>(
         &self,
         path: PathSegments<'_>,
@@ -57,8 +72,36 @@ impl Client {
         K: AsRef<str>,
         V: AsRef<str>,
     {
-        let url = create_api_url(&self.url, path, params);
-        Ok(self.reqwest_client.get(url).send().await?.json().await?)
+        let res = self.send_get_request(path, params).await?;
+        if res.status().is_success() {
+            Ok(res.json().await?)
+        } else {
+            Err(ValidatorAPIError::GenericRequestFailure(res.text().await?))
+        }
+    }
+
+    // This works for endpoints returning Result<Json<T>, ErrorResponse>
+    async fn query_validator_api_fallible<T, K, V>(
+        &self,
+        path: PathSegments<'_>,
+        params: Params<'_, K, V>,
+    ) -> Result<T, ValidatorAPIError>
+    where
+        for<'a> T: Deserialize<'a>,
+        K: AsRef<str>,
+        V: AsRef<str>,
+    {
+        let res = self.send_get_request(path, params).await?;
+        let status = res.status();
+        if res.status().is_success() {
+            Ok(res.json().await?)
+        } else {
+            let request_error: RequestError = res.json().await?;
+            Err(ValidatorAPIError::ApiRequestFailure {
+                status: status.as_u16(),
+                error: request_error,
+            })
+        }
     }
 
     async fn post_validator_api<B, T, K, V>(
@@ -135,6 +178,74 @@ impl Client {
         .await
     }
 
+    pub async fn get_mixnode_report(
+        &self,
+        mix_id: MixId,
+    ) -> Result<MixnodeStatusReportResponse, ValidatorAPIError> {
+        self.query_validator_api(
+            &[
+                routes::API_VERSION,
+                routes::STATUS,
+                routes::MIXNODE,
+                &mix_id.to_string(),
+                routes::REPORT,
+            ],
+            NO_PARAMS,
+        )
+        .await
+    }
+
+    pub async fn get_gateway_report(
+        &self,
+        identity: IdentityKeyRef<'_>,
+    ) -> Result<GatewayStatusReportResponse, ValidatorAPIError> {
+        self.query_validator_api(
+            &[
+                routes::API_VERSION,
+                routes::STATUS,
+                routes::GATEWAY,
+                identity,
+                routes::REPORT,
+            ],
+            NO_PARAMS,
+        )
+        .await
+    }
+
+    pub async fn get_mixnode_history(
+        &self,
+        mix_id: MixId,
+    ) -> Result<MixnodeUptimeHistoryResponse, ValidatorAPIError> {
+        self.query_validator_api(
+            &[
+                routes::API_VERSION,
+                routes::STATUS,
+                routes::MIXNODE,
+                &mix_id.to_string(),
+                routes::HISTORY,
+            ],
+            NO_PARAMS,
+        )
+        .await
+    }
+
+    pub async fn get_gateway_history(
+        &self,
+        identity: IdentityKeyRef<'_>,
+    ) -> Result<GatewayUptimeHistoryResponse, ValidatorAPIError> {
+        self.query_validator_api(
+            &[
+                routes::API_VERSION,
+                routes::STATUS,
+                routes::GATEWAY,
+                identity,
+                routes::HISTORY,
+            ],
+            NO_PARAMS,
+        )
+        .await
+    }
+
     pub async fn get_rewarded_mixnodes_detailed(
         &self,
     ) -> Result<Vec<MixNodeBondAnnotated>, ValidatorAPIError> {
@@ -234,7 +345,7 @@ impl Client {
         &self,
         mix_id: MixId,
     ) -> Result<RewardEstimationResponse, ValidatorAPIError> {
-        self.query_validator_api(
+        self.query_validator_api_fallible(
             &[
                 routes::API_VERSION,
                 routes::STATUS_ROUTES,
@@ -251,7 +362,7 @@ impl Client {
         &self,
         mix_id: MixId,
     ) -> Result<StakeSaturationResponse, ValidatorAPIError> {
-        self.query_validator_api(
+        self.query_validator_api_fallible(
             &[
                 routes::API_VERSION,
                 routes::STATUS_ROUTES,
@@ -268,7 +379,7 @@ impl Client {
         &self,
         mix_id: MixId,
     ) -> Result<InclusionProbabilityResponse, ValidatorAPIError> {
-        self.query_validator_api(
+        self.query_validator_api_fallible(
             &[
                 routes::API_VERSION,
                 routes::STATUS_ROUTES,
@@ -285,7 +396,7 @@ impl Client {
         &self,
         mix_id: MixId,
     ) -> Result<UptimeResponse, ValidatorAPIError> {
-        self.query_validator_api(
+        self.query_validator_api_fallible(
             &[
                 routes::API_VERSION,
                 routes::STATUS_ROUTES,

common/client-libs/validator-client/src/validator_api/routes.rs

@@ -10,7 +10,6 @@ pub const GATEWAYS: &str = "gateways";
 pub const DETAILED: &str = "detailed";
 pub const ACTIVE: &str = "active";
 pub const REWARDED: &str = "rewarded";
-
 pub const COCONUT_ROUTES: &str = "coconut";
 pub const BANDWIDTH: &str = "bandwidth";
 
@@ -28,6 +27,8 @@ pub const CORE_STATUS_COUNT: &str = "core-status-count";
 pub const SINCE_ARG: &str = "since";
 
 pub const STATUS: &str = "status";
+pub const REPORT: &str = "report";
+pub const HISTORY: &str = "history";
 pub const REWARD_ESTIMATION: &str = "reward-estimation";
 pub const AVG_UPTIME: &str = "avg_uptime";
 pub const STAKE_SATURATION: &str = "stake-saturation";

common/commands/src/validator/signature/sign.rs

@@ -13,7 +13,7 @@ use validator_client::nymd::wallet::DirectSecp256k1HdWallet;
 pub struct SignatureOutputJson {
     pub account_id: String,
     pub public_key: PublicKey,
-    pub signature: String,
+    pub signature_as_hex: String,
 }
 
 #[derive(Debug, Parser)]
@@ -46,7 +46,7 @@ pub fn sign(args: Args, prefix: &str, mnemonic: Option<bip39::Mnemonic>) {
                             let output = SignatureOutputJson {
                                 account_id: account.address().to_string(),
                                 public_key: account.public_key(),
-                                signature: signature.to_string(),
+                                signature_as_hex: signature.to_string(),
                             };
                             println!("{}", json!(output));
                         }

common/commands/src/validator/vesting/create_vesting_schedule.rs

@@ -12,6 +12,7 @@ use validator_client::nymd::AccountId;
 use validator_client::nymd::VestingSigningClient;
 use validator_client::nymd::{CosmosCoin, Denom};
 use vesting_contract_common::messages::VestingSpecification;
+use vesting_contract_common::PledgeCap;
 
 use crate::context::SigningClient;
 
@@ -34,6 +35,12 @@ pub struct Args {
 
     #[clap(long)]
     pub staking_address: Option<String>,
+
+    #[clap(
+        long,
+        help = "Pledge cap as either absolute uNYM value or percentage, floats need to be in the 0.0 to 1.0 range and will be parsed as percentages, integers will be parsed as uNYM"
+    )]
+    pub pledge_cap: Option<PledgeCap>,
 }
 
 pub async fn create(args: Args, client: SigningClient, network_details: &NymNetworkDetails) {
@@ -55,6 +62,7 @@ pub async fn create(args: Args, client: SigningClient, network_details: &NymNetw
             args.staking_address,
             Some(vesting),
             coin.into(),
+            args.pledge_cap,
             None,
         )
         .await

common/cosmwasm-smart-contracts/contracts-common/Cargo.toml

@@ -9,3 +9,8 @@ edition = "2021"
 [dependencies]
 cosmwasm-std = "1.0.0"
 serde = { version = "1.0", features = ["derive"] }
+schemars = "0.8"
+thiserror = "1"
+
+[dev-dependencies]
+serde_json = "1.0.0"

common/cosmwasm-smart-contracts/contracts-common/src/types.rs

@@ -1,7 +1,128 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use serde::{Deserialize, Serialize};
+use cosmwasm_std::Decimal;
+use cosmwasm_std::Uint128;
+use schemars::JsonSchema;
+use serde::de::Error;
+use serde::{Deserialize, Deserializer, Serialize};
+use std::fmt::{self, Display, Formatter};
+use std::ops::Mul;
+use std::str::FromStr;
+use thiserror::Error;
+
+pub fn truncate_decimal(amount: Decimal) -> Uint128 {
+    amount * Uint128::new(1)
+}
+
+#[derive(Error, Debug)]
+pub enum ContractsCommonError {
+    #[error("Provided percent value ({0}) is greater than 100%")]
+    InvalidPercent(Decimal),
+
+    #[error("{source}")]
+    StdErr {
+        #[from]
+        source: cosmwasm_std::StdError,
+    },
+}
+
+/// Percent represents a value between 0 and 100%
+/// (i.e. between 0.0 and 1.0)
+#[derive(
+    Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Serialize, Deserialize, JsonSchema,
+)]
+pub struct Percent(#[serde(deserialize_with = "de_decimal_percent")] Decimal);
+
+impl Percent {
+    pub fn new(value: Decimal) -> Result<Self, ContractsCommonError> {
+        if value > Decimal::one() {
+            Err(ContractsCommonError::InvalidPercent(value))
+        } else {
+            Ok(Percent(value))
+        }
+    }
+
+    pub fn is_zero(&self) -> bool {
+        self.0 == Decimal::zero()
+    }
+
+    pub fn zero() -> Self {
+        Self(Decimal::zero())
+    }
+
+    pub fn hundred() -> Self {
+        Self(Decimal::one())
+    }
+
+    pub fn from_percentage_value(value: u64) -> Result<Self, ContractsCommonError> {
+        Percent::new(Decimal::percent(value))
+    }
+
+    pub fn value(&self) -> Decimal {
+        self.0
+    }
+
+    pub fn round_to_integer(&self) -> u8 {
+        let hundred = Decimal::from_ratio(100u32, 1u32);
+        // we know the cast from u128 to u8 is a safe one since the internal value must be within 0 - 1 range
+        truncate_decimal(hundred * self.0).u128() as u8
+    }
+}
+
+impl Display for Percent {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        let adjusted = Decimal::from_ratio(100u32, 1u32) * self.0;
+        write!(f, "{}%", adjusted)
+    }
+}
+
+impl FromStr for Percent {
+    type Err = ContractsCommonError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Percent::new(Decimal::from_str(s)?)
+    }
+}
+
+impl Mul<Decimal> for Percent {
+    type Output = Decimal;
+
+    fn mul(self, rhs: Decimal) -> Self::Output {
+        self.0 * rhs
+    }
+}
+
+impl Mul<Percent> for Decimal {
+    type Output = Decimal;
+
+    fn mul(self, rhs: Percent) -> Self::Output {
+        rhs * self
+    }
+}
+
+impl Mul<Uint128> for Percent {
+    type Output = Uint128;
+
+    fn mul(self, rhs: Uint128) -> Self::Output {
+        self.0 * rhs
+    }
+}
+
+// implement custom Deserialize because we want to validate Percent has the correct range
+fn de_decimal_percent<'de, D>(deserializer: D) -> Result<Decimal, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let v = Decimal::deserialize(deserializer)?;
+    if v > Decimal::one() {
+        Err(D::Error::custom(
+            "provided decimal percent is larger than 100%",
+        ))
+    } else {
+        Ok(v)
+    }
+}
 
 // TODO: there's no reason this couldn't be used for proper binaries, but in that case
 // perhaps the struct should get renamed and moved to a "more" common crate
@@ -31,3 +152,47 @@ pub struct ContractBuildInformation {
     /// Provides the rustc version that was used for the build, for example `1.52.0-nightly`.
     pub rustc_version: String,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn percent_serde() {
+        let valid_value = Percent::from_percentage_value(80).unwrap();
+        let serialized = serde_json::to_string(&valid_value).unwrap();
+
+        let deserialized: Percent = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(valid_value, deserialized);
+
+        let invalid_values = vec!["\"42\"", "\"1.1\"", "\"1.00000001\"", "\"foomp\"", "\"1a\""];
+        for invalid_value in invalid_values {
+            assert!(serde_json::from_str::<'_, Percent>(invalid_value).is_err())
+        }
+        assert_eq!(
+            serde_json::from_str::<'_, Percent>("\"0.95\"").unwrap(),
+            Percent::from_percentage_value(95).unwrap()
+        )
+    }
+
+    #[test]
+    fn percent_to_absolute_integer() {
+        let p = serde_json::from_str::<'_, Percent>("\"0.0001\"").unwrap();
+        assert_eq!(p.round_to_integer(), 0);
+
+        let p = serde_json::from_str::<'_, Percent>("\"0.0099\"").unwrap();
+        assert_eq!(p.round_to_integer(), 0);
+
+        let p = serde_json::from_str::<'_, Percent>("\"0.0199\"").unwrap();
+        assert_eq!(p.round_to_integer(), 1);
+
+        let p = serde_json::from_str::<'_, Percent>("\"0.45123\"").unwrap();
+        assert_eq!(p.round_to_integer(), 45);
+
+        let p = serde_json::from_str::<'_, Percent>("\"0.999999999\"").unwrap();
+        assert_eq!(p.round_to_integer(), 99);
+
+        let p = serde_json::from_str::<'_, Percent>("\"1.00\"").unwrap();
+        assert_eq!(p.round_to_integer(), 100);
+    }
+}

common/cosmwasm-smart-contracts/mixnet-contract/src/error.rs

@@ -13,9 +13,6 @@ pub enum MixnetContractError {
         source: cosmwasm_std::StdError,
     },
 
-    #[error("Provided percent value is greater than 100%")]
-    InvalidPercent,
-
     #[error("Attempted to subtract decimals with overflow ({minuend}.sub({subtrahend}))")]
     OverflowDecimalSubtraction {
         minuend: Decimal,

common/cosmwasm-smart-contracts/mixnet-contract/src/interval.rs

@@ -73,6 +73,9 @@ pub struct Interval {
     // TODO add a better TS type generation
     #[cfg_attr(feature = "generate-ts", ts(type = "string"))]
     #[serde(with = "string_rfc3339_offset_date_time")]
+    // note: the `ts-rs failed to parse this attribute. It will be ignored.` warning emitted during
+    // compilation is fine (I guess). `ts-rs` can't handle `with` serde attribute, but that's okay
+    // since we explicitly specified this field should correspond to typescript's string
     current_epoch_start: OffsetDateTime,
     current_epoch_id: EpochId,
     #[cfg_attr(feature = "generate-ts", ts(type = "{ secs: number; nanos: number; }"))]

common/cosmwasm-smart-contracts/mixnet-contract/src/msg.rs

@@ -33,6 +33,7 @@ pub struct InitialRewardingParams {
     pub initial_reward_pool: Decimal,
     pub initial_staking_supply: Decimal,
 
+    pub staking_supply_scale_factor: Percent,
     pub sybil_resistance: Percent,
     pub active_set_work_factor: Decimal,
     pub interval_pool_emission: Percent,
@@ -56,6 +57,7 @@ impl InitialRewardingParams {
             interval: IntervalRewardParams {
                 reward_pool: self.initial_reward_pool,
                 staking_supply: self.initial_staking_supply,
+                staking_supply_scale_factor: self.staking_supply_scale_factor,
                 epoch_reward_budget,
                 stake_saturation_point,
                 sybil_resistance: self.sybil_resistance,
@@ -411,4 +413,6 @@ pub enum QueryMsg {
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "snake_case")]
-pub struct MigrateMsg {}
+pub struct MigrateMsg {
+    pub vesting_contract_address: Option<String>,
+}

common/cosmwasm-smart-contracts/mixnet-contract/src/reward_params.rs

@@ -27,6 +27,11 @@ pub struct IntervalRewardParams {
     #[cfg_attr(feature = "generate-ts", ts(type = "string"))]
     pub staking_supply: Decimal,
 
+    /// Defines the percentage of stake needed to reach saturation for all of the nodes in the rewarded set.
+    /// Also known as `beta`.
+    #[cfg_attr(feature = "generate-ts", ts(type = "string"))]
+    pub staking_supply_scale_factor: Percent,
+
     // computed values
     /// Current value of the computed reward budget per epoch, per node.
     /// It is expected to be constant throughout the interval.
@@ -174,6 +179,10 @@ impl RewardingParams {
             self.interval.staking_supply = staking_supply;
         }
 
+        if let Some(staking_supply_scale_factor) = updates.staking_supply_scale_factor {
+            self.interval.staking_supply_scale_factor = staking_supply_scale_factor
+        }
+
         if let Some(sybil_resistance_percent) = updates.sybil_resistance_percent {
             self.interval.sybil_resistance = sybil_resistance_percent;
         }
@@ -245,6 +254,9 @@ pub struct IntervalRewardingParamsUpdate {
     #[cfg_attr(feature = "generate-ts", ts(type = "string | null"))]
     pub staking_supply: Option<Decimal>,
 
+    #[cfg_attr(feature = "generate-ts", ts(type = "string | null"))]
+    pub staking_supply_scale_factor: Option<Percent>,
+
     #[cfg_attr(feature = "generate-ts", ts(type = "string | null"))]
     pub sybil_resistance_percent: Option<Percent>,
 
@@ -262,6 +274,7 @@ impl IntervalRewardingParamsUpdate {
         // essentially at least a single field has to be a `Some`
         self.reward_pool.is_some()
             || self.staking_supply.is_some()
+            || self.staking_supply_scale_factor.is_some()
             || self.sybil_resistance_percent.is_some()
             || self.active_set_work_factor.is_some()
             || self.interval_pool_emission.is_some()

common/cosmwasm-smart-contracts/mixnet-contract/src/rewarding/helpers.rs

@@ -1,6 +1,7 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use contracts_common::truncate_decimal;
 use cosmwasm_std::{Coin, Decimal, Uint128};
 
 /// Truncates all decimal points so that the reward would fit in a `Coin` and so that we would
@@ -17,7 +18,3 @@ pub fn truncate_reward(reward: Decimal, denom: impl Into<String>) -> Coin {
 pub fn truncate_reward_amount(reward: Decimal) -> Uint128 {
     truncate_decimal(reward)
 }
-
-pub fn truncate_decimal(amount: Decimal) -> Uint128 {
-    amount * Uint128::new(1)
-}

common/cosmwasm-smart-contracts/mixnet-contract/src/rewarding/simulator/mod.rs

@@ -41,7 +41,12 @@ impl Simulator {
         if self.interval.current_interval_id() + 1 == updated.current_interval_id() {
             let old = self.system_rewarding_params.interval;
             let reward_pool = old.reward_pool - self.pending_reward_pool_emission;
-            let staking_supply = old.staking_supply + self.pending_reward_pool_emission;
+            let staking_supply = old.staking_supply
+                + self
+                    .system_rewarding_params
+                    .interval
+                    .staking_supply_scale_factor
+                    * self.pending_reward_pool_emission;
             let epoch_reward_budget = reward_pool
                 / self.interval.epochs_in_interval().into_base_decimal()?
                 * old.interval_pool_emission.value();
@@ -55,6 +60,7 @@ impl Simulator {
                 interval: IntervalRewardParams {
                     reward_pool,
                     staking_supply,
+                    staking_supply_scale_factor: old.staking_supply_scale_factor,
                     epoch_reward_budget,
                     stake_saturation_point,
                     sybil_resistance: old.sybil_resistance,
@@ -250,6 +256,7 @@ mod tests {
                 interval: IntervalRewardParams {
                     reward_pool: Decimal::from_atomics(reward_pool, 0).unwrap(), // 250M * 1M (we're expressing it all in base tokens)
                     staking_supply: Decimal::from_atomics(staking_supply, 0).unwrap(), // 100M * 1M
+                    staking_supply_scale_factor: Percent::hundred(),
                     epoch_reward_budget,
                     stake_saturation_point,
                     sybil_resistance: Percent::from_percentage_value(30).unwrap(),
@@ -613,6 +620,7 @@ mod tests {
             interval: IntervalRewardParams {
                 reward_pool: Decimal::from_atomics(reward_pool, 0).unwrap(),
                 staking_supply: Decimal::from_atomics(staking_supply, 0).unwrap(),
+                staking_supply_scale_factor: Percent::hundred(),
                 epoch_reward_budget,
                 stake_saturation_point,
                 sybil_resistance: Percent::from_percentage_value(30).unwrap(),

common/cosmwasm-smart-contracts/mixnet-contract/src/types.rs

@@ -2,15 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::error::MixnetContractError;
-use crate::rewarding::helpers::truncate_decimal;
 use crate::{Layer, RewardedSetNodeStatus};
-use cosmwasm_std::{Addr, Uint128};
-use cosmwasm_std::{Coin, Decimal};
+use cosmwasm_std::Addr;
+use cosmwasm_std::Coin;
 use schemars::JsonSchema;
-use serde::de::Error;
-use serde::{Deserialize, Deserializer, Serialize};
-use std::fmt::{self, Display, Formatter};
-use std::ops::{Index, Mul};
+use serde::{Deserialize, Serialize};
+use std::ops::Index;
 
 // type aliases for better reasoning about available data
 pub type IdentityKey = String;
@@ -24,87 +21,6 @@ pub type BlockHeight = u64;
 pub type EpochEventId = u32;
 pub type IntervalEventId = u32;
 
-/// Percent represents a value between 0 and 100%
-/// (i.e. between 0.0 and 1.0)
-#[derive(
-    Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Serialize, Deserialize, JsonSchema,
-)]
-pub struct Percent(#[serde(deserialize_with = "de_decimal_percent")] Decimal);
-
-impl Percent {
-    pub fn new(value: Decimal) -> Result<Self, MixnetContractError> {
-        if value > Decimal::one() {
-            Err(MixnetContractError::InvalidPercent)
-        } else {
-            Ok(Percent(value))
-        }
-    }
-
-    pub fn is_zero(&self) -> bool {
-        self.0 == Decimal::zero()
-    }
-
-    pub fn from_percentage_value(value: u64) -> Result<Self, MixnetContractError> {
-        Percent::new(Decimal::percent(value))
-    }
-
-    pub fn value(&self) -> Decimal {
-        self.0
-    }
-
-    pub fn round_to_integer(&self) -> u8 {
-        let hundred = Decimal::from_ratio(100u32, 1u32);
-        // we know the cast from u128 to u8 is a safe one since the internal value must be within 0 - 1 range
-        truncate_decimal(hundred * self.0).u128() as u8
-    }
-}
-
-impl Display for Percent {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        let adjusted = Decimal::from_ratio(100u32, 1u32) * self.0;
-        write!(f, "{}%", adjusted)
-    }
-}
-
-impl Mul<Decimal> for Percent {
-    type Output = Decimal;
-
-    fn mul(self, rhs: Decimal) -> Self::Output {
-        self.0 * rhs
-    }
-}
-
-impl Mul<Percent> for Decimal {
-    type Output = Decimal;
-
-    fn mul(self, rhs: Percent) -> Self::Output {
-        rhs * self
-    }
-}
-
-impl Mul<Uint128> for Percent {
-    type Output = Uint128;
-
-    fn mul(self, rhs: Uint128) -> Self::Output {
-        self.0 * rhs
-    }
-}
-
-// implement custom Deserialize because we want to validate Percent has the correct range
-fn de_decimal_percent<'de, D>(deserializer: D) -> Result<Decimal, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    let v = Decimal::deserialize(deserializer)?;
-    if v > Decimal::one() {
-        Err(D::Error::custom(
-            "provided decimal percent is larger than 100%",
-        ))
-    } else {
-        Ok(v)
-    }
-}
-
 #[derive(Debug, Default, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
 pub struct LayerDistribution {
     pub layer1: u64,
@@ -210,48 +126,3 @@ pub struct PagedRewardedSetResponse {
     pub nodes: Vec<(MixId, RewardedSetNodeStatus)>,
     pub start_next_after: Option<MixId>,
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn percent_serde() {
-        let valid_value = Percent::from_percentage_value(80).unwrap();
-        let serialized = serde_json::to_string(&valid_value).unwrap();
-
-        println!("{}", serialized);
-        let deserialized: Percent = serde_json::from_str(&serialized).unwrap();
-        assert_eq!(valid_value, deserialized);
-
-        let invalid_values = vec!["\"42\"", "\"1.1\"", "\"1.00000001\"", "\"foomp\"", "\"1a\""];
-        for invalid_value in invalid_values {
-            assert!(serde_json::from_str::<'_, Percent>(invalid_value).is_err())
-        }
-        assert_eq!(
-            serde_json::from_str::<'_, Percent>("\"0.95\"").unwrap(),
-            Percent::from_percentage_value(95).unwrap()
-        )
-    }
-
-    #[test]
-    fn percent_to_absolute_integer() {
-        let p = serde_json::from_str::<'_, Percent>("\"0.0001\"").unwrap();
-        assert_eq!(p.round_to_integer(), 0);
-
-        let p = serde_json::from_str::<'_, Percent>("\"0.0099\"").unwrap();
-        assert_eq!(p.round_to_integer(), 0);
-
-        let p = serde_json::from_str::<'_, Percent>("\"0.0199\"").unwrap();
-        assert_eq!(p.round_to_integer(), 1);
-
-        let p = serde_json::from_str::<'_, Percent>("\"0.45123\"").unwrap();
-        assert_eq!(p.round_to_integer(), 45);
-
-        let p = serde_json::from_str::<'_, Percent>("\"0.999999999\"").unwrap();
-        assert_eq!(p.round_to_integer(), 99);
-
-        let p = serde_json::from_str::<'_, Percent>("\"1.00\"").unwrap();
-        assert_eq!(p.round_to_integer(), 100);
-    }
-}

common/cosmwasm-smart-contracts/vesting-contract/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 cosmwasm-std = "1.0.0"
 mixnet-contract-common = { path = "../mixnet-contract" }
+contracts-common = { path = "../contracts-common" }
 serde = { version = "1.0", features = ["derive"] }
 schemars = "0.8"
 ts-rs = {version = "6.1.2", optional = true}

common/cosmwasm-smart-contracts/vesting-contract/src/lib.rs

@@ -4,10 +4,12 @@
 #![warn(clippy::expect_used)]
 #![warn(clippy::unwrap_used)]
 
+use contracts_common::Percent;
 use cosmwasm_std::{Addr, Coin, Timestamp, Uint128};
 use mixnet_contract_common::MixId;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
+use std::str::FromStr;
 
 pub use messages::{ExecuteMsg, InitMsg, MigrateMsg, QueryMsg};
 
@@ -46,6 +48,35 @@ impl PledgeData {
     }
 }
 
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
+pub enum PledgeCap {
+    Percent(Percent),
+    Absolute(Uint128), // This has to be in unym
+}
+
+impl FromStr for PledgeCap {
+    type Err = String;
+
+    fn from_str(cap: &str) -> Result<Self, Self::Err> {
+        let cap = cap.replace('_', "").replace(',', ".");
+        match Percent::from_str(&cap) {
+            Ok(p) => Ok(PledgeCap::Percent(p)),
+            Err(_) => match cap.parse::<u128>() {
+                Ok(i) => Ok(PledgeCap::Absolute(Uint128::from(i))),
+                Err(_e) => Err(format!("Could not parse {} as Percent or Uint128", cap)),
+            },
+        }
+    }
+}
+
+impl Default for PledgeCap {
+    fn default() -> Self {
+        #[allow(clippy::expect_used)]
+        PledgeCap::Percent(Percent::from_percentage_value(10).expect("This can never fail!"))
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema)]
 pub struct OriginalVestingResponse {
     pub amount: Coin,
@@ -102,3 +133,32 @@ pub struct AllDelegationsResponse {
     pub delegations: Vec<VestingDelegation>,
     pub start_next_after: Option<(u32, MixId, u64)>,
 }
+
+#[cfg(test)]
+mod test {
+    use contracts_common::Percent;
+    use cosmwasm_std::Uint128;
+    use std::str::FromStr;
+
+    use crate::PledgeCap;
+
+    #[test]
+    fn test_pledge_cap_from_str() {
+        assert_eq!(
+            PledgeCap::from_str("0.1").unwrap(),
+            PledgeCap::Percent(Percent::from_percentage_value(10).unwrap())
+        );
+        assert_eq!(
+            PledgeCap::from_str("0,1").unwrap(),
+            PledgeCap::Percent(Percent::from_percentage_value(10).unwrap())
+        );
+        assert_eq!(
+            PledgeCap::from_str("100_000_000_000").unwrap(),
+            PledgeCap::Absolute(Uint128::new(100_000_000_000))
+        );
+        assert_eq!(
+            PledgeCap::from_str("100000000000").unwrap(),
+            PledgeCap::Absolute(Uint128::new(100_000_000_000))
+        );
+    }
+}

common/cosmwasm-smart-contracts/vesting-contract/src/messages.rs

@@ -1,4 +1,4 @@
-use cosmwasm_std::{Coin, Timestamp, Uint128};
+use cosmwasm_std::{Coin, Timestamp};
 use mixnet_contract_common::{
     mixnode::{MixNodeConfigUpdate, MixNodeCostParams},
     Gateway, MixId, MixNode,
@@ -6,6 +6,8 @@ use mixnet_contract_common::{
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
+use crate::PledgeCap;
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "snake_case")]
 pub struct InitMsg {
@@ -83,6 +85,7 @@ pub enum ExecuteMsg {
         owner_address: String,
         staking_address: Option<String>,
         vesting_spec: Option<VestingSpecification>,
+        cap: Option<PledgeCap>,
     },
     WithdrawVestedCoins {
         amount: Coin,
@@ -120,7 +123,8 @@ pub enum ExecuteMsg {
         to_address: Option<String>,
     },
     UpdateLockedPledgeCap {
-        amount: Uint128,
+        address: String,
+        cap: PledgeCap,
     },
 }
 
@@ -156,6 +160,7 @@ impl ExecuteMsg {
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
 #[serde(rename_all = "snake_case")]
 pub enum QueryMsg {
+    GetContractVersion {},
     LockedCoins {
         vesting_account_address: String,
         block_time: Option<Timestamp>,
@@ -201,7 +206,6 @@ pub enum QueryMsg {
     GetCurrentVestingPeriod {
         address: String,
     },
-    GetLockedPledgeCap {},
     GetDelegationTimes {
         address: String,
         mix_id: MixId,

common/logging/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "logging"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4.0"
+pretty_env_logger = "0.4.0"

common/logging/src/lib.rs

@@ -0,0 +1,25 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+// I'd argue we should start transitioning from `log` to `tracing`
+pub fn setup_logging() {
+    let mut log_builder = pretty_env_logger::formatted_timed_builder();
+    if let Ok(s) = ::std::env::var("RUST_LOG") {
+        log_builder.parse_filters(&s);
+    } else {
+        // default to 'Info'
+        log_builder.filter(None, log::LevelFilter::Info);
+    }
+
+    log_builder
+        .filter_module("hyper", log::LevelFilter::Warn)
+        .filter_module("tokio_reactor", log::LevelFilter::Warn)
+        .filter_module("reqwest", log::LevelFilter::Warn)
+        .filter_module("mio", log::LevelFilter::Warn)
+        .filter_module("want", log::LevelFilter::Warn)
+        .filter_module("tungstenite", log::LevelFilter::Warn)
+        .filter_module("tokio_tungstenite", log::LevelFilter::Warn)
+        .filter_module("handlebars", log::LevelFilter::Warn)
+        .filter_module("sled", log::LevelFilter::Warn)
+        .init();
+}

common/mixnode-common/src/packet_processor/processor.rs

@@ -116,7 +116,10 @@ impl SphinxPacketProcessor {
                 trace!("received an ack packet!");
                 Ok((None, data))
             }
-            PacketSize::RegularPacket | PacketSize::ExtendedPacket => {
+            PacketSize::RegularPacket
+            | PacketSize::ExtendedPacket8
+            | PacketSize::ExtendedPacket16
+            | PacketSize::ExtendedPacket32 => {
                 trace!("received a normal packet!");
                 let (ack_data, message) = self.split_hop_data_into_ack_and_message(data)?;
                 let (ack_first_hop, ack_packet) = SurbAck::try_recover_first_hop_packet(&ack_data)?;

common/network-defaults/src/mainnet.rs

@@ -10,7 +10,7 @@ pub const MIX_DENOM: DenomDetails = DenomDetails::new("unym", "nym", 6);
 pub const STAKE_DENOM: DenomDetails = DenomDetails::new("unyx", "nyx", 6);
 
 pub(crate) const MIXNET_CONTRACT_ADDRESS: &str =
-    "n14hj2tavq8fpesdwxxcu44rty3hh90vhujrvcmstl4zr3txmfvw9sjyvg3g";
+    "n17srjznxl9dvzdkpwpw24gg668wc73val88a6m5ajg6ankwvz9wtst0cznr";
 pub(crate) const VESTING_CONTRACT_ADDRESS: &str =
     "n1nc5tatafv6eyq7llkr2gv50ff9e22mnf70qgjlv737ktmt4eswrq73f2nw";
 pub(crate) const BANDWIDTH_CLAIM_CONTRACT_ADDRESS: &str =

common/nymcoconut/src/scheme/keygen.rs

@@ -85,7 +85,7 @@ impl SecretKey {
     // x || ys.len() || ys
     pub fn to_bytes(&self) -> Vec<u8> {
         let ys_len = self.ys.len();
-        let mut bytes = Vec::with_capacity(8 + (ys_len + 1) as usize * 32);
+        let mut bytes = Vec::with_capacity(8 + (ys_len + 1) * 32);
 
         bytes.extend_from_slice(&self.x.to_bytes());
         bytes.extend_from_slice(&ys_len.to_le_bytes());
@@ -162,7 +162,7 @@ impl TryFrom<&[u8]> for VerificationKey {
         let mut beta_g1_end: u64 = 0;
         for i in 0..betas_len {
             let start = (104 + i * 48) as usize;
-            let end = (start + 48) as usize;
+            let end = start + 48;
             let beta_i_bytes = bytes[start..end].try_into().unwrap();
             let beta_i = try_deserialize_g1_projective(
                 &beta_i_bytes,
@@ -178,7 +178,7 @@ impl TryFrom<&[u8]> for VerificationKey {
         let mut beta_g2 = Vec::with_capacity(betas_len as usize);
         for i in 0..betas_len {
             let start = (beta_g1_end + i * 96) as usize;
-            let end = (start + 96) as usize;
+            let end = start + 96;
             let beta_i_bytes = bytes[start..end].try_into().unwrap();
             let beta_i = try_deserialize_g2_projective(
                 &beta_i_bytes,

common/nymsphinx/chunking/src/set.rs

@@ -94,10 +94,10 @@ fn prepare_unlinked_fragmented_set(
 
     for i in 1..(pre_casted_frags + 1) {
         // we can't use u8 directly here as upper (NON-INCLUSIVE, so it would always fit) bound could be u8::max_value() + 1
-        let lb = (i as usize - 1) * unlinked_fragment_payload_max_len(max_plaintext_size);
+        let lb = (i - 1) * unlinked_fragment_payload_max_len(max_plaintext_size);
         let ub = usize::min(
             message.len(),
-            i as usize * unlinked_fragment_payload_max_len(max_plaintext_size),
+            i * unlinked_fragment_payload_max_len(max_plaintext_size),
         );
         fragments.push(
             Fragment::try_new(

common/nymsphinx/forwarding/src/packet.rs

@@ -25,8 +25,10 @@ impl Display for MixPacketFormattingError {
             InvalidPacketSize(actual) =>
                 write!(
                     f,
-                    "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {} (EXTENDED))",
-                    actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(), PacketSize::ExtendedPacket.size()
+                    "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {}, {}, {} (EXTENDED))",
+                    actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(),
+                    PacketSize::ExtendedPacket8.size(), PacketSize::ExtendedPacket16.size(),
+                    PacketSize::ExtendedPacket32.size()
                 ),
             MalformedSphinxPacket => write!(f, "received sphinx packet was malformed"),
             InvalidPacketMode => write!(f, "provided packet mode is invalid")

common/nymsphinx/framing/src/codec.rs

@@ -6,7 +6,6 @@ use bytes::{Buf, BufMut, BytesMut};
 use nymsphinx_params::packet_modes::InvalidPacketMode;
 use nymsphinx_params::packet_sizes::{InvalidPacketSize, PacketSize};
 use nymsphinx_types::SphinxPacket;
-use std::convert::TryFrom;
 use std::io;
 use tokio_util::codec::{Decoder, Encoder};
 
@@ -75,7 +74,7 @@ impl Decoder for SphinxCodec {
         if src.is_empty() {
             // can't do anything if we have no bytes, but let's reserve enough for the most
             // conservative case, i.e. receiving an ack packet
-            src.reserve(Header::SIZE + PacketSize::AckPacket.size());
+            src.reserve(Header::LEGACY_SIZE + PacketSize::AckPacket.size());
             return Ok(None);
         }
 
@@ -87,7 +86,7 @@ impl Decoder for SphinxCodec {
         };
 
         let sphinx_packet_size = header.packet_size.size();
-        let frame_len = Header::SIZE + sphinx_packet_size;
+        let frame_len = header.size() + sphinx_packet_size;
 
         if src.len() < frame_len {
             // we don't have enough bytes to read the rest of frame
@@ -96,7 +95,7 @@ impl Decoder for SphinxCodec {
         }
 
         // advance buffer past the header - at this point we have enough bytes
-        src.advance(Header::SIZE);
+        src.advance(header.size());
         let sphinx_packet_bytes = src.split_to(sphinx_packet_size);
         let sphinx_packet = match SphinxPacket::from_bytes(&sphinx_packet_bytes) {
             Ok(sphinx_packet) => sphinx_packet,
@@ -115,21 +114,27 @@ impl Decoder for SphinxCodec {
         // has appropriate capacity in anticipation of future calls to decode.
         // Failing to do so leads to inefficiency.
 
-        // if we have at least one more byte available, we can reserve enough bytes for
+        // if we have enough bytes to decode the header of the next packet, we can reserve enough bytes for
         // the entire next frame, if not, we assume the next frame is an ack packet and
         // reserve for that.
+        // we also assume the next packet coming from the same client will use exactly the same versioning
+        // as the current packet
+        let mut allocate_for_next_packet = header.size() + PacketSize::AckPacket.size();
         if !src.is_empty() {
-            let next_packet_len = match PacketSize::try_from(src[0]) {
-                Ok(next_packet_len) => next_packet_len,
+            match Header::decode(src) {
+                Ok(Some(next_header)) => {
+                    allocate_for_next_packet = next_header.size() + next_header.packet_size.size();
+                }
+                Ok(None) => {
+                    // we don't have enough information to know how much to reserve, fallback to the ack case
+                }
+
                 // the next frame will be malformed but let's leave handling the error to the next
                 // call to 'decode', as presumably, the current sphinx packet is still valid
                 Err(_) => return Ok(Some(nymsphinx_packet)),
             };
-            let next_frame_len = next_packet_len.size() + Header::SIZE;
-            src.reserve(next_frame_len - 1);
-        } else {
-            src.reserve(Header::SIZE + PacketSize::AckPacket.size());
         }
+        src.reserve(allocate_for_next_packet);
 
         Ok(Some(nymsphinx_packet))
     }
@@ -199,6 +204,8 @@ mod packet_encoding {
     #[cfg(test)]
     mod decode_will_allocate_enough_bytes_for_next_call {
         use super::*;
+        use nymsphinx_params::packet_version::PacketVersion;
+        use nymsphinx_params::PacketMode;
 
         #[test]
         fn for_empty_bytes() {
@@ -207,20 +214,23 @@ mod packet_encoding {
             assert!(SphinxCodec.decode(&mut empty_bytes).unwrap().is_none());
             assert_eq!(
                 empty_bytes.capacity(),
-                Header::SIZE + PacketSize::AckPacket.size()
+                Header::LEGACY_SIZE + PacketSize::AckPacket.size()
             );
         }
 
         #[test]
-        fn for_bytes_with_header() {
+        fn for_bytes_with_legacy_header() {
             // if header gets decoded there should be enough bytes for the entire frame
             let packet_sizes = vec![
                 PacketSize::AckPacket,
                 PacketSize::RegularPacket,
-                PacketSize::ExtendedPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
             ];
             for packet_size in packet_sizes {
                 let header = Header {
+                    packet_version: PacketVersion::Legacy,
                     packet_size,
                     packet_mode: Default::default(),
                 };
@@ -228,12 +238,60 @@ mod packet_encoding {
                 header.encode(&mut bytes);
                 assert!(SphinxCodec.decode(&mut bytes).unwrap().is_none());
 
-                assert_eq!(bytes.capacity(), Header::SIZE + packet_size.size())
+                assert_eq!(bytes.capacity(), Header::LEGACY_SIZE + packet_size.size())
             }
         }
 
         #[test]
-        fn for_full_frame() {
+        fn for_bytes_with_versioned_header() {
+            // if header gets decoded there should be enough bytes for the entire frame
+            let packet_sizes = vec![
+                PacketSize::AckPacket,
+                PacketSize::RegularPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
+            ];
+            for packet_size in packet_sizes {
+                let header = Header {
+                    packet_version: PacketVersion::Versioned(123),
+                    packet_size,
+                    packet_mode: Default::default(),
+                };
+                let mut bytes = BytesMut::new();
+                header.encode(&mut bytes);
+                assert!(SphinxCodec.decode(&mut bytes).unwrap().is_none());
+
+                assert_eq!(
+                    bytes.capacity(),
+                    Header::VERSIONED_SIZE + packet_size.size()
+                )
+            }
+        }
+
+        #[test]
+        fn for_full_frame_with_legacy_header() {
+            // if full frame is used exactly, there should be enough space for header + ack packet
+            let packet = FramedSphinxPacket {
+                header: Header {
+                    packet_version: PacketVersion::Legacy,
+                    packet_size: Default::default(),
+                    packet_mode: Default::default(),
+                },
+                packet: make_valid_sphinx_packet(Default::default()),
+            };
+
+            let mut bytes = BytesMut::new();
+            SphinxCodec.encode(packet, &mut bytes).unwrap();
+            assert!(SphinxCodec.decode(&mut bytes).unwrap().is_some());
+            assert_eq!(
+                bytes.capacity(),
+                Header::LEGACY_SIZE + PacketSize::AckPacket.size()
+            );
+        }
+
+        #[test]
+        fn for_full_frame_with_versioned_header() {
             // if full frame is used exactly, there should be enough space for header + ack packet
             let packet = FramedSphinxPacket {
                 header: Header::default(),
@@ -245,17 +303,50 @@ mod packet_encoding {
             assert!(SphinxCodec.decode(&mut bytes).unwrap().is_some());
             assert_eq!(
                 bytes.capacity(),
-                Header::SIZE + PacketSize::AckPacket.size()
+                Header::VERSIONED_SIZE + PacketSize::AckPacket.size()
             );
         }
 
         #[test]
-        fn for_full_frame_with_extra_byte() {
-            // if there was at least 1 byte left, there should be enough space for entire next frame
+        fn for_full_frame_with_extra_bytes_with_legacy_header() {
+            // if there was at least 2 byte left, there should be enough space for entire next frame
             let packet_sizes = vec![
                 PacketSize::AckPacket,
                 PacketSize::RegularPacket,
-                PacketSize::ExtendedPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
+            ];
+
+            for packet_size in packet_sizes {
+                let first_packet = FramedSphinxPacket {
+                    header: Header {
+                        packet_version: PacketVersion::Legacy,
+                        packet_size: Default::default(),
+                        packet_mode: Default::default(),
+                    },
+                    packet: make_valid_sphinx_packet(Default::default()),
+                };
+
+                let mut bytes = BytesMut::new();
+                SphinxCodec.encode(first_packet, &mut bytes).unwrap();
+                bytes.put_u8(packet_size as u8);
+                bytes.put_u8(PacketMode::default() as u8);
+                assert!(SphinxCodec.decode(&mut bytes).unwrap().is_some());
+
+                assert!(bytes.capacity() >= Header::LEGACY_SIZE + packet_size.size())
+            }
+        }
+
+        #[test]
+        fn for_full_frame_with_extra_bytes_with_versioned_header() {
+            // if there was at least 3 byte left, there should be enough space for entire next frame
+            let packet_sizes = vec![
+                PacketSize::AckPacket,
+                PacketSize::RegularPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
             ];
 
             for packet_size in packet_sizes {
@@ -266,10 +357,12 @@ mod packet_encoding {
 
                 let mut bytes = BytesMut::new();
                 SphinxCodec.encode(first_packet, &mut bytes).unwrap();
+                bytes.put_u8(PacketVersion::new_versioned(123).as_u8().unwrap());
                 bytes.put_u8(packet_size as u8);
+                bytes.put_u8(PacketMode::default() as u8);
                 assert!(SphinxCodec.decode(&mut bytes).unwrap().is_some());
 
-                assert!(bytes.capacity() >= Header::SIZE + packet_size.size())
+                assert!(bytes.capacity() >= Header::VERSIONED_SIZE + packet_size.size())
             }
         }
     }

common/nymsphinx/framing/src/packet.rs

@@ -4,6 +4,7 @@
 use crate::codec::SphinxCodecError;
 use bytes::{BufMut, BytesMut};
 use nymsphinx_params::packet_sizes::PacketSize;
+use nymsphinx_params::packet_version::PacketVersion;
 use nymsphinx_params::PacketMode;
 use nymsphinx_types::SphinxPacket;
 use std::convert::TryFrom;
@@ -17,12 +18,14 @@ pub struct FramedSphinxPacket {
 }
 
 impl FramedSphinxPacket {
-    pub fn new(packet: SphinxPacket, packet_mode: PacketMode) -> Self {
+    pub fn new(packet: SphinxPacket, packet_mode: PacketMode, use_legacy_version: bool) -> Self {
         // If this fails somebody is using the library in a super incorrect way, because they
         // already managed to somehow create a sphinx packet
         let packet_size = PacketSize::get_type(packet.len()).unwrap();
+
         FramedSphinxPacket {
             header: Header {
+                packet_version: PacketVersion::new(use_legacy_version),
                 packet_size,
                 packet_mode,
             },
@@ -48,6 +51,9 @@ impl FramedSphinxPacket {
 // but would that really be worth it?
 #[derive(Debug, Default, PartialEq, Eq, Copy, Clone)]
 pub struct Header {
+    /// Represents the wire format version used to construct this packet.
+    pub(crate) packet_version: PacketVersion,
+
     /// Represents type and consequently size of the included SphinxPacket.
     pub(crate) packet_size: PacketSize,
 
@@ -64,11 +70,25 @@ pub struct Header {
 }
 
 impl Header {
-    pub(crate) const SIZE: usize = 2;
+    pub(crate) const LEGACY_SIZE: usize = 2;
+    pub(crate) const VERSIONED_SIZE: usize = 3;
+
+    pub(crate) fn size(&self) -> usize {
+        if self.packet_version.is_legacy() {
+            Self::LEGACY_SIZE
+        } else {
+            Self::VERSIONED_SIZE
+        }
+    }
 
     pub(crate) fn encode(&self, dst: &mut BytesMut) {
         // we reserve one byte for `packet_size` and the other for `mode`
-        dst.reserve(Self::SIZE);
+        dst.reserve(Self::LEGACY_SIZE);
+        if let Some(version) = self.packet_version.as_u8() {
+            dst.reserve(Self::VERSIONED_SIZE);
+            dst.put_u8(version)
+        }
+
         dst.put_u8(self.packet_size as u8);
         dst.put_u8(self.packet_mode as u8);
         // reserve bytes for the actual packet
@@ -76,16 +96,30 @@ impl Header {
     }
 
     pub(crate) fn decode(src: &mut BytesMut) -> Result<Option<Self>, SphinxCodecError> {
-        if src.len() < Self::SIZE {
+        if src.len() < Self::LEGACY_SIZE {
             // can't do anything if we don't have enough bytes - but reserve enough for the next call
-            src.reserve(Self::SIZE);
+            src.reserve(Self::LEGACY_SIZE);
             return Ok(None);
         }
 
-        Ok(Some(Header {
-            packet_size: PacketSize::try_from(src[0])?,
-            packet_mode: PacketMode::try_from(src[1])?,
-        }))
+        let packet_version = PacketVersion::from(src[0]);
+        if packet_version.is_legacy() {
+            Ok(Some(Header {
+                packet_version,
+                packet_size: PacketSize::try_from(src[0])?,
+                packet_mode: PacketMode::try_from(src[1])?,
+            }))
+        } else if src.len() < Self::VERSIONED_SIZE {
+            // we're missing that 1 byte to read the full header...
+            src.reserve(Self::VERSIONED_SIZE);
+            Ok(None)
+        } else {
+            Ok(Some(Header {
+                packet_version,
+                packet_size: PacketSize::try_from(src[1])?,
+                packet_mode: PacketMode::try_from(src[2])?,
+            }))
+        }
     }
 }
 
@@ -108,7 +142,16 @@ mod header_encoding {
         // make sure this is still 'unknown' for if we make changes in the future
         assert!(PacketSize::try_from(unknown_packet_size).is_err());
 
-        let mut bytes = BytesMut::from([unknown_packet_size, PacketMode::default() as u8].as_ref());
+        // unfortunately this will only work for the 'versioned' variant
+        // due to the hack used to get legacy mode compatibility
+        let mut bytes = BytesMut::from(
+            [
+                PacketVersion::new_versioned(123).as_u8().unwrap(),
+                unknown_packet_size,
+                PacketMode::default() as u8,
+            ]
+            .as_ref(),
+        );
         assert!(Header::decode(&mut bytes).is_err())
     }
 
@@ -127,23 +170,47 @@ mod header_encoding {
         let mut empty_bytes = BytesMut::new();
         let decode_attempt_1 = Header::decode(&mut empty_bytes).unwrap();
         assert!(decode_attempt_1.is_none());
-        assert!(empty_bytes.capacity() > Header::SIZE);
+        assert!(empty_bytes.capacity() > Header::LEGACY_SIZE);
 
         let mut empty_bytes = BytesMut::with_capacity(1);
         let decode_attempt_2 = Header::decode(&mut empty_bytes).unwrap();
         assert!(decode_attempt_2.is_none());
-        assert!(empty_bytes.capacity() > Header::SIZE);
+        assert!(empty_bytes.capacity() > Header::LEGACY_SIZE);
     }
 
     #[test]
-    fn header_encoding_reserves_enough_bytes_for_full_sphinx_packet() {
+    fn header_encoding_reserves_enough_bytes_for_full_sphinx_packet_in_legacy_mode() {
         let packet_sizes = vec![
             PacketSize::AckPacket,
             PacketSize::RegularPacket,
-            PacketSize::ExtendedPacket,
+            PacketSize::ExtendedPacket8,
+            PacketSize::ExtendedPacket16,
+            PacketSize::ExtendedPacket32,
         ];
         for packet_size in packet_sizes {
             let header = Header {
+                packet_version: PacketVersion::Legacy,
+                packet_size,
+                packet_mode: Default::default(),
+            };
+            let mut bytes = BytesMut::new();
+            header.encode(&mut bytes);
+            assert_eq!(bytes.capacity(), bytes.len() + packet_size.size())
+        }
+    }
+
+    #[test]
+    fn header_encoding_reserves_enough_bytes_for_full_sphinx_packet_in_versioned_mode() {
+        let packet_sizes = vec![
+            PacketSize::AckPacket,
+            PacketSize::RegularPacket,
+            PacketSize::ExtendedPacket8,
+            PacketSize::ExtendedPacket16,
+            PacketSize::ExtendedPacket32,
+        ];
+        for packet_size in packet_sizes {
+            let header = Header {
+                packet_version: PacketVersion::Versioned(123),
                 packet_size,
                 packet_mode: Default::default(),
             };

common/nymsphinx/params/src/lib.rs

@@ -13,6 +13,7 @@ pub use packet_sizes::PacketSize;
 
 pub mod packet_modes;
 pub mod packet_sizes;
+pub mod packet_version;
 
 // If somebody can provide an argument why it might be reasonable to have more than 255 mix hops,
 // I will change this to [`usize`]
@@ -24,6 +25,17 @@ pub const DEFAULT_NUM_MIX_HOPS: u8 = 3;
 pub const FRAG_ID_LEN: usize = 5;
 pub type SerializedFragmentIdentifier = [u8; FRAG_ID_LEN];
 
+// wait, wait, but why are we starting with version 7?
+// when packet header gets serialized, the following bytes (in that order) are put onto the wire:
+// - packet_version (starting with v1.1.0)
+// - packet_size indicator
+// - packet_mode
+// it also just so happens that the only valid values for packet_size indicator include values 1-6
+// therefore if we receive byte `7` (or larger than that) we'll know we received a versioned packet,
+// otherwise we should treat it as legacy
+/// Increment it whenever we perform any breaking change in the wire format!
+const CURRENT_PACKET_VERSION_NUMBER: u8 = 7;
+
 // TODO: ask @AP about the choice of below algorithms
 
 /// Hashing algorithm used during hkdf for ephemeral shared key generation per sphinx packet payload.

common/nymsphinx/params/src/packet_sizes.rs

@@ -5,6 +5,7 @@ use crate::FRAG_ID_LEN;
 use nymsphinx_types::header::HEADER_SIZE;
 use nymsphinx_types::PAYLOAD_OVERHEAD_SIZE;
 use std::convert::TryFrom;
+use std::str::FromStr;
 
 // it's up to the smart people to figure those values out : )
 const REGULAR_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 2 * 1024;
@@ -15,11 +16,16 @@ const REGULAR_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 2 * 102
 const ACK_IV_SIZE: usize = 16;
 
 const ACK_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + ACK_IV_SIZE + FRAG_ID_LEN;
-const EXTENDED_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 32 * 1024;
+const EXTENDED_PACKET_SIZE_8: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 8 * 1024;
+const EXTENDED_PACKET_SIZE_16: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 16 * 1024;
+const EXTENDED_PACKET_SIZE_32: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 32 * 1024;
 
 #[derive(Debug)]
 pub struct InvalidPacketSize;
 
+#[derive(Debug)]
+pub struct InvalidExtendedPacketSize;
+
 #[repr(u8)]
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum PacketSize {
@@ -30,7 +36,28 @@ pub enum PacketSize {
     AckPacket = 2,
 
     // for example for streaming fast and furious in uncompressed 10bit 4K HDR quality
-    ExtendedPacket = 3,
+    ExtendedPacket32 = 3,
+
+    // for example for streaming fast and furious in heavily compressed lossy RealPlayer quality
+    ExtendedPacket8 = 4,
+
+    // for example for streaming fast and furious in compressed XviD quality
+    ExtendedPacket16 = 5,
+}
+
+impl FromStr for PacketSize {
+    type Err = InvalidPacketSize;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "regular" => Ok(Self::RegularPacket),
+            "ack" => Ok(Self::AckPacket),
+            "extended8" => Ok(Self::ExtendedPacket8),
+            "extended16" => Ok(Self::ExtendedPacket16),
+            "extended32" => Ok(Self::ExtendedPacket32),
+            _ => Err(InvalidPacketSize),
+        }
+    }
 }
 
 impl TryFrom<u8> for PacketSize {
@@ -40,7 +67,9 @@ impl TryFrom<u8> for PacketSize {
         match value {
             _ if value == (PacketSize::RegularPacket as u8) => Ok(Self::RegularPacket),
             _ if value == (PacketSize::AckPacket as u8) => Ok(Self::AckPacket),
-            _ if value == (PacketSize::ExtendedPacket as u8) => Ok(Self::ExtendedPacket),
+            _ if value == (PacketSize::ExtendedPacket8 as u8) => Ok(Self::ExtendedPacket8),
+            _ if value == (PacketSize::ExtendedPacket16 as u8) => Ok(Self::ExtendedPacket16),
+            _ if value == (PacketSize::ExtendedPacket32 as u8) => Ok(Self::ExtendedPacket32),
             _ => Err(InvalidPacketSize),
         }
     }
@@ -51,7 +80,9 @@ impl PacketSize {
         match self {
             PacketSize::RegularPacket => REGULAR_PACKET_SIZE,
             PacketSize::AckPacket => ACK_PACKET_SIZE,
-            PacketSize::ExtendedPacket => EXTENDED_PACKET_SIZE,
+            PacketSize::ExtendedPacket8 => EXTENDED_PACKET_SIZE_8,
+            PacketSize::ExtendedPacket16 => EXTENDED_PACKET_SIZE_16,
+            PacketSize::ExtendedPacket32 => EXTENDED_PACKET_SIZE_32,
         }
     }
 
@@ -68,12 +99,33 @@ impl PacketSize {
             Ok(PacketSize::RegularPacket)
         } else if PacketSize::AckPacket.size() == size {
             Ok(PacketSize::AckPacket)
-        } else if PacketSize::ExtendedPacket.size() == size {
-            Ok(PacketSize::ExtendedPacket)
+        } else if PacketSize::ExtendedPacket8.size() == size {
+            Ok(PacketSize::ExtendedPacket8)
+        } else if PacketSize::ExtendedPacket16.size() == size {
+            Ok(PacketSize::ExtendedPacket16)
+        } else if PacketSize::ExtendedPacket32.size() == size {
+            Ok(PacketSize::ExtendedPacket32)
         } else {
             Err(InvalidPacketSize)
         }
     }
+
+    pub fn is_extended_size(&self) -> bool {
+        match self {
+            PacketSize::RegularPacket | PacketSize::AckPacket => false,
+            PacketSize::ExtendedPacket8
+            | PacketSize::ExtendedPacket16
+            | PacketSize::ExtendedPacket32 => true,
+        }
+    }
+
+    pub fn as_extended_size(self) -> Option<Self> {
+        if self.is_extended_size() {
+            Some(self)
+        } else {
+            None
+        }
+    }
 }
 
 impl Default for PacketSize {

common/nymsphinx/params/src/packet_version.rs

@@ -0,0 +1,59 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{PacketSize, CURRENT_PACKET_VERSION_NUMBER};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PacketVersion {
+    // this will allow updated mixnodes to still understand packets from before the update
+    Legacy,
+    Versioned(u8),
+}
+
+impl PacketVersion {
+    pub fn new(use_legacy: bool) -> Self {
+        if use_legacy {
+            Self::new_legacy()
+        } else {
+            Self::new_versioned(CURRENT_PACKET_VERSION_NUMBER)
+        }
+    }
+
+    pub fn new_legacy() -> Self {
+        PacketVersion::Legacy
+    }
+
+    pub fn new_versioned(version: u8) -> Self {
+        PacketVersion::Versioned(version)
+    }
+
+    pub fn is_legacy(&self) -> bool {
+        matches!(self, PacketVersion::Legacy)
+    }
+
+    pub fn as_u8(&self) -> Option<u8> {
+        match self {
+            PacketVersion::Legacy => None,
+            PacketVersion::Versioned(version) => Some(*version),
+        }
+    }
+}
+
+impl From<u8> for PacketVersion {
+    fn from(v: u8) -> Self {
+        match v {
+            n if n == PacketSize::RegularPacket as u8 => PacketVersion::Legacy,
+            n if n == PacketSize::AckPacket as u8 => PacketVersion::Legacy,
+            n if n == PacketSize::ExtendedPacket8 as u8 => PacketVersion::Legacy,
+            n if n == PacketSize::ExtendedPacket16 as u8 => PacketVersion::Legacy,
+            n if n == PacketSize::ExtendedPacket32 as u8 => PacketVersion::Legacy,
+            n => PacketVersion::Versioned(n),
+        }
+    }
+}
+
+impl Default for PacketVersion {
+    fn default() -> Self {
+        PacketVersion::Versioned(CURRENT_PACKET_VERSION_NUMBER)
+    }
+}

common/socks5/proxy-helpers/Cargo.toml

@@ -14,8 +14,11 @@ tokio-util = { version = "0.7.3", features = [ "io" ] } # reason for getting thi
 # In the long run, the dependency should probably get removed in favour of pure-tokio implementation, but for time being it's fine.
 futures = "0.3"
 log = "0.4"
-socks5-requests = { path = "../requests" }
+
+# internal
+client-connections = { path = "../../client-connections" }
 ordered-buffer = { path = "../ordered-buffer" }
+socks5-requests = { path = "../requests" }
 task = { path = "../../task" }
 
 [dev-dependencies]

common/socks5/proxy-helpers/src/connection_controller.rs

@@ -1,6 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use client_connections::ClosedConnectionSender;
 use futures::channel::mpsc;
 use futures::StreamExt;
 use log::*;
@@ -73,6 +74,9 @@ pub struct Controller {
     // to avoid memory issues
     recently_closed: HashSet<ConnectionId>,
 
+    // Broadcast closed connections
+    closed_connection_tx: ClosedConnectionSender,
+
     // TODO: this can potentially be abused to ddos and kill provider. Not sure at this point
     // how to handle it more gracefully
 
@@ -84,13 +88,17 @@ pub struct Controller {
 }
 
 impl Controller {
-    pub fn new(shutdown: ShutdownListener) -> (Self, ControllerSender) {
+    pub fn new(
+        closed_connection_tx: ClosedConnectionSender,
+        shutdown: ShutdownListener,
+    ) -> (Self, ControllerSender) {
         let (sender, receiver) = mpsc::unbounded();
         (
             Controller {
                 active_connections: HashMap::new(),
                 receiver,
                 recently_closed: HashSet::new(),
+                closed_connection_tx,
                 pending_messages: HashMap::new(),
                 shutdown,
             },
@@ -127,6 +135,9 @@ impl Controller {
             )
         }
         self.recently_closed.insert(conn_id);
+
+        // Announce closed connections, currently used by the `OutQueueControl`.
+        self.closed_connection_tx.unbounded_send(conn_id).unwrap();
     }
 
     fn send_to_connection(&mut self, conn_id: ConnectionId, payload: Vec<u8>, is_closed: bool) {
@@ -172,11 +183,15 @@ impl Controller {
             pending.push((payload, is_closed));
         } else if !is_closed {
             error!(
-                "Tried to write to closed connection ({} bytes were 'lost)",
+                "Tried to write to closed connection {} ({} bytes were 'lost)",
+                conn_id,
                 payload.len()
             );
         } else {
-            debug!("Tried to write to closed connection, but remote is already closed")
+            debug!(
+                "Tried to write to closed connection {}, but remote is already closed",
+                conn_id
+            )
         }
     }
 

common/socks5/proxy-helpers/src/proxy_runner/inbound.rs

@@ -63,6 +63,10 @@ where
 
     // if we're sending through the mixnet increase the sequence number...
     let ordered_msg = message_sender.wrap_message(read_data.to_vec()).into_bytes();
+    log::trace!(
+        "pushing data down the input sender: size: {}",
+        ordered_msg.len()
+    );
     mix_sender
         .unbounded_send(adapter_fn(connection_id, ordered_msg, is_finished))
         .unwrap();

common/types/src/delegation.rs

@@ -62,6 +62,7 @@ pub struct DelegationWithEverything {
 
     // DEPRECATED, IF POSSIBLE TRY TO DISCONTINUE USE OF IT!
     pub pending_events: Vec<DelegationEvent>,
+    pub mixnode_is_unbonding: Option<bool>,
 }
 
 #[cfg_attr(feature = "generate-ts", derive(ts_rs::TS))]

contracts/CHANGELOG.md

@@ -1,14 +1,55 @@
+## Unreleased
+
+## Added
+
+- Added migration code to the mixnet contract to allow updating stored vesting contract address to make it easier to deploy any future environments ([#1759],[#1769])
+
+[#1759]: https://github.com/nymtech/nym/pull/1759
+[#1769]: https://github.com/nymtech/nym/pull/1769
+
+## [nym-contracts-v1.1.0](https://github.com/nymtech/nym/tree/nym-contracts-v1.1.0) (2022-11-09)
+
+### Changed
+- mixnet-contract: rework of rewarding ([#1472]), which includes, but is not limited to:
+  - internal reward accounting was modified to be similar to the ideas presented in Cosmos' F1 paper, which results in throughput gains and no storage or gas cost bloat over time,
+  - introduced internal queues for pending epoch and interval events that only get resolved once relevant epoch/interval rolls over
+  - the contract no longer stores any historical information regarding past epochs/parameters/stake state for the purposes of rewarding
+  - a lot of queries got renamed to keep naming more consistent,
+  - introduced new utility-based queries such as a query for reward estimation for the current epoch,
+  - mixnodes are now identified by a monotonously increasing `mix_id`
+  - bonding now results in getting fresh `mix_id` and thus if given node decides to unbond and rebond, it will lose all its delegations,
+  - mixnode operators are now allowed to set their operating costs as opposed to having fixed value of 40nym/interval
+  - rewarding parameters are now correctly updated at an **interval** end
+  - rewarding parameters now include a staking supply scale factor attribute (beta in the tokenomics paper)
+  - node performance can now be more granular with internal `Decimal` representation as opposed to an `u8`
+  - node profit margin can now be more granular with internal `Decimal` representation as opposed to an `u8`
+  - mixnode operators are now allowed to change their configuration options, such as port information, without having to unbond
+  - mixnode unbonding is no longer instantaneous, instead it happens once an epoch rolls over
+  - it is now possible to query for operator and node history to see how often (and with what parameters) they rebonded
+  - other minor bugfixes and changes
+  - ...
+  - new exciting bugs to find and squash
+
+- vesting-contract: optional locked token pledge cap per account ([#1687]), defaults to 10%
+- vesting-contract: updated internal delegation storage due to mixnet contract revamp ([#1472])
+
+### Added
+- vesting-contract: added query for obtaining contract build information ([#1726])
+
+[#1472]: https://github.com/nymtech/nym/pull/1472
+[#1687]: https://github.com/nymtech/nym/pull/1687
+[#1726]: https://github.com/nymtech/nym/pull/1726
+
+
 ## [nym-contracts-v1.0.2](https://github.com/nymtech/nym/tree/nym-contracts-v1.0.2) (2022-09-13)
 
 ### Added
 
 - vesting-contract: added queries for delegation timestamps and paged query for all vesting delegations in the contract ([#1569])
-- all binaries: added shell completion and [Fig](fig.io) spec generation ([#1638])
 
 ### Changed
 
 - mixnet-contract: compounding delegator rewards now happens instantaneously as opposed to having to wait for the current epoch to finish ([#1571])
-- network-requester: updated CLI to use `clap` macros ([#1638])
 
 ### Fixed
 
@@ -18,9 +59,8 @@
 
 [#1544]: https://github.com/nymtech/nym/pull/1544
 [#1569]: https://github.com/nymtech/nym/pull/1569
-[#1569]: https://github.com/nymtech/nym/pull/1571
+[#1571]: https://github.com/nymtech/nym/pull/1571
 [#1613]: https://github.com/nymtech/nym/pull/1613
-[#1638]: https://github.com/nymtech/nym/pull/1638
 
 ## [nym-contracts-v1.0.1](https://github.com/nymtech/nym/tree/nym-contracts-v1.0.1) (2022-06-22)
 

contracts/Cargo.lock

@@ -221,7 +221,9 @@ name = "contracts-common"
 version = "0.1.0"
 dependencies = [
  "cosmwasm-std",
+ "schemars",
  "serde",
+ "thiserror",
 ]
 
 [[package]]
@@ -285,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.1"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
+checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
 dependencies = [
  "libc",
 ]
@@ -928,7 +930,7 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
 
 [[package]]
 name = "mixnet-contract"
-version = "1.0.2"
+version = "1.1.0"
 dependencies = [
  "bs58",
  "cosmwasm-schema",
@@ -1606,12 +1608,14 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 name = "vesting-contract"
 version = "1.1.0"
 dependencies = [
+ "contracts-common",
  "cosmwasm-std",
  "cw-storage-plus",
  "mixnet-contract-common",
  "schemars",
  "serde",
  "thiserror",
+ "vergen",
  "vesting-contract-common",
 ]
 
@@ -1619,6 +1623,7 @@ dependencies = [
 name = "vesting-contract-common"
 version = "0.1.0"
 dependencies = [
+ "contracts-common",
  "cosmwasm-std",
  "mixnet-contract-common",
  "schemars",

contracts/mixnet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mixnet-contract"
-version = "1.0.2"
+version = "1.1.0"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 edition = "2021"
 

contracts/mixnet/src/contract.rs

@@ -467,10 +467,20 @@ pub fn query(
 
 #[entry_point]
 pub fn migrate(
-    _deps: DepsMut<'_>,
+    deps: DepsMut<'_>,
     _env: Env,
-    _msg: MigrateMsg,
+    msg: MigrateMsg,
 ) -> Result<Response, MixnetContractError> {
+    // due to circular dependency on contract addresses (i.e. mixnet contract requiring vesting contract address
+    // and vesting contract requiring the mixnet contract address), if we ever want to deploy any new fresh
+    // environment, one of the contracts will HAVE TO go through a migration
+    if let Some(vesting_contract_address) = msg.vesting_contract_address {
+        let mut current_state = mixnet_params_storage::CONTRACT_STATE.load(deps.storage)?;
+        current_state.vesting_contract_address =
+            deps.api.addr_validate(&vesting_contract_address)?;
+        mixnet_params_storage::CONTRACT_STATE.save(deps.storage, &current_state)?;
+    }
+
     Ok(Default::default())
 }
 
@@ -497,6 +507,7 @@ mod tests {
             initial_rewarding_params: InitialRewardingParams {
                 initial_reward_pool: Decimal::from_atomics(100_000_000_000_000u128, 0).unwrap(),
                 initial_staking_supply: Decimal::from_atomics(123_456_000_000_000u128, 0).unwrap(),
+                staking_supply_scale_factor: Percent::hundred(),
                 sybil_resistance: Percent::from_percentage_value(23).unwrap(),
                 active_set_work_factor: Decimal::from_atomics(10u32, 0).unwrap(),
                 interval_pool_emission: Percent::from_percentage_value(1).unwrap(),
@@ -535,6 +546,7 @@ mod tests {
             interval: IntervalRewardParams {
                 reward_pool: Decimal::from_atomics(100_000_000_000_000u128, 0).unwrap(),
                 staking_supply: Decimal::from_atomics(123_456_000_000_000u128, 0).unwrap(),
+                staking_supply_scale_factor: Percent::hundred(),
                 epoch_reward_budget: expected_epoch_reward_budget,
                 stake_saturation_point: expected_stake_saturation_point,
                 sybil_resistance: Percent::from_percentage_value(23).unwrap(),