Merge pull request #1874 from nymtech/bug-fix/hide-display-mnemonic

Bug fix/hide display mnemonic

.env.sample-dev

@@ -3,3 +3,21 @@
 
 RUST_LOG=info
 RUST_BACKTRACE=1
+
+#########################################
+# geoipupdate (needed for explorer-api) #
+#########################################
+# MaxMind account ID (change it to a valid account ID)
+GEOIPUPDATE_ACCOUNT_ID=xxx
+# MaxMind license key (change it to a valid license key)
+GEOIPUPDATE_LICENSE_KEY=xxx
+# List of space-separated database edition IDs. Edition IDs may
+# consist of letters, digits, and dashes. For example, GeoIP2-City
+# would download the GeoIP2 City database (GeoIP2-City).
+GEOIPUPDATE_EDITION_IDS=GeoLite2-Country
+# The number of hours between geoipupdate runs. If this is not set
+# or is set to 0, geoipupdate will run once and exit.
+GEOIPUPDATE_FREQUENCY=72
+# The path to the directory where geoipupdate will download the
+# database.
+GEOIP_DB_DIRECTORY=./explorer-api/geo_ip

.github/workflows/audit.yml

@@ -1,36 +1,52 @@
 name: Daily security audit
 
-on: workflow_dispatch
+on:
+  schedule:
+    - cron: '5 9 * * *'
 jobs:
-  security_audit:
-    runs-on: ubuntu-latest
+  cargo-deny:
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/audit-check@v1
+      - name: Checkout repository code
+        uses: actions/checkout@v2
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
+          toolchain: stable
+      - name: Install cargo deny
+        run: cargo install --locked cargo-deny
+      - name: Run cargo deny
+        run: |
+          find . -name Cargo.toml -exec cargo deny --manifest-path {} check \
+            advisories -A advisory-not-detected  --hide-inclusion-graph \; &> \
+            >(uniq  &> .github/workflows/support-files/notifications/deny.message )
+      - uses: actions/upload-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications/deny.message
   notification:
-    if: ${{ failure() }}
-    needs: security_audit
-    runs-on: ubuntu-latest
+    needs: cargo-deny
+    runs-on: ubuntu-20.04
     steps:
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: Download report from previous job
+        uses: actions/download-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications
       - name: Keybase - Node Install
         run: npm install
         working-directory: .github/workflows/support-files
       - name: Keybase - Send Notification
         env:
-          NYM_NOTIFICATION_KIND: nightly
-          NYM_PROJECT_NAME: "Nym daily audit"
+          NYM_NOTIFICATION_KIND: security
+          NYM_PROJECT_NAME: "Daily security report"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-          GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
-          GIT_BRANCH: "${GITHUB_REF##*/}"
           KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
           KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
-          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBTECH_TEAM }}"
-          KEYBASE_NYM_CHANNEL: "test"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "security"
         uses: docker://keybaseio/client:stable-node
         with:
           args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/build.yml

@@ -29,6 +29,12 @@ jobs:
           override: true
           components: rustfmt, clippy
 
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
       - name: Build all binaries
         uses: actions-rs/cargo@v1
         with:
@@ -48,12 +54,6 @@ jobs:
           command: test
           args: --workspace --all-features -- --ignored
 
-      - name: Check formatting
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
       - uses: actions-rs/clippy-check@v1
         name: Clippy checks
         with:
@@ -66,6 +66,8 @@ jobs:
           command: clippy
           args: --workspace -- -D warnings
 
+      # COCONUT stuff
+
       - name: Build all binaries with coconut enabled
         uses: actions-rs/cargo@v1
         with:
@@ -82,4 +84,4 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --features=coconut -- -D warnings
+          args: --all-targets --features=coconut -- -D warnings

.github/workflows/build_matrix_includes.json

@@ -1,6 +1,6 @@
 [
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"stable",
     "runOnEvent":"always"
   },

.github/workflows/build_on_dispatch.yml

@@ -1,72 +0,0 @@
-name: Continuous integration on dispatch
-
-on: workflow_dispatch
-
-jobs:
-  build:
-    runs-on: [ self-hosted, custom-linux ]
-    # Enable sccache via environment variable
-    env:
-      RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
-    steps:
-      - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
-
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Install rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - name: Build all binaries
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace
-
-      - name: Run all tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --all-features
-
-      - name: Check formatting
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - uses: actions-rs/clippy-check@v1
-        name: Clippy checks
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
-
-      - name: Run clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --workspace -- -D warnings
-
-      - name: Build all binaries with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --features=coconut
-
-      - name: Run all tests with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --features=coconut
-
-      - name: Run clippy with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --features=coconut -- -D warnings

.github/workflows/contracts-build.yml

@@ -1,16 +1,21 @@
 name: Build release of Nym smart contracts
 on:
   workflow_dispatch:
-
-defaults:
-  run:
-    working-directory: contracts
+  release:
+    types: [created]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
+      
+      - name: Check the release tag starts with `nym-contracts-`
+        if: startsWith(github.ref, 'refs/tags/nym-contracts-') == false && github.event_name != 'workflow_dispatch'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Release tag did not start with nym-contracts-...')      
 
       - name: Install Rust stable
         uses: actions-rs/toolchain@v1
@@ -21,7 +26,7 @@ jobs:
           components: rustfmt, clippy
 
       - name: Build release contracts
-        run: RUSTFLAGS='-C link-arg=-s' cargo build --release --target wasm32-unknown-unknown
+        run: make wasm
 
       - name: Upload Mixnet Contract Artifact
         uses: actions/upload-artifact@v3
@@ -36,3 +41,11 @@ jobs:
           name: vesting_contract.wasm
           path: contracts/target/wasm32-unknown-unknown/release/vesting_contract.wasm
           retention-days: 5
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            contracts/target/wasm32-unknown-unknown/release/vesting_contract.wasm
+            contracts/target/wasm32-unknown-unknown/release/mixnet_contract.wasm

.github/workflows/contracts.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   matrix_prep:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -24,7 +24,7 @@ jobs:
   contracts:
     # since it's going to be compiled into wasm, there's absolutely
     # no point in running CI on different OS-es
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.rust == 'nightly' }}
     needs: matrix_prep
     strategy:

.github/workflows/network-explorer-api.yml

@@ -0,0 +1,56 @@
+name: CI for Network Explorer API
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+    
+env:
+  NETWORK: mainnet    
+
+jobs:
+  publish-nym:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+
+      - name: Check the release tag starts with `nym-explorer-api-`
+        if: startsWith(github.ref, 'refs/tags/nym-explorer-api-') == false && github.event_name != 'workflow_dispatch'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Release tag did not start with nym-explorer-api-...')
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build all explorer-api
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path explorer-api/Cargo.toml --workspace --release
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: my-artifact
+          path: |
+            target/release/explorer-api
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            target/release/explorer-api

.github/workflows/nightly_build.yml

@@ -5,7 +5,7 @@ on:
     - cron: '14 1 * * *'
 jobs:
   matrix_prep:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -25,7 +25,7 @@ jobs:
     steps:
       - name: Install Dependencies (Linux)
         run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-20.04'
 
       - name: Check out repository code
         uses: actions/checkout@v2
@@ -96,7 +96,7 @@ jobs:
 
       - name: Reclaim some disk space
         uses: actions-rs/cargo@v1
-        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-latest' }}
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
         with:
           command: clean
 
@@ -160,7 +160,7 @@ jobs:
 
   notification:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Collect jobs status
         uses: technote-space/workflow-conclusion-action@v2

.github/workflows/nightly_build_matrix_includes.json

@@ -1,6 +1,6 @@
 [
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"stable",
     "runOnEvent":"schedule"
   },
@@ -17,7 +17,7 @@
   },
 
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"beta",
     "runOnEvent":"schedule"
   },
@@ -33,7 +33,7 @@
   },
 
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"nightly",
     "runOnEvent":"schedule"
   },

.github/workflows/nightly_build_matrix_on_dispatch.json

@@ -1,50 +0,0 @@
-[
-  {
-    "os":"ubuntu-latest",
-    "rust":"stable",
-    "runOnEvent":"workflow_dispatch"
-  },
-
-  {
-    "os":"windows-latest",
-    "rust":"stable",
-    "runOnEvent":"workflow_dispatch"
-  },
-  {
-    "os":"macos-latest",
-    "rust":"stable",
-    "runOnEvent":"workflow_dispatch"
-  },
-
-  {
-    "os":"ubuntu-latest",
-    "rust":"beta",
-    "runOnEvent":"workflow_dispatch"
-  },
-  {
-    "os":"windows-latest",
-    "rust":"beta",
-    "runOnEvent":"workflow_dispatch"
-  },
-  {
-    "os":"macos-latest",
-    "rust":"beta",
-    "runOnEvent":"workflow_dispatch"
-  },
-
-  {
-    "os":"ubuntu-latest",
-    "rust":"nightly",
-    "runOnEvent":"workflow_dispatch"
-  },
-  {
-    "os":"windows-latest",
-    "rust":"nightly",
-    "runOnEvent":"workflow_dispatch"
-  },
-  {
-    "os":"macos-latest",
-    "rust":"nightly",
-    "runOnEvent":"workflow_dispatch"
-  }
-]

.github/workflows/nightly_build_release.yml

@@ -1,32 +1,49 @@
-name: Nightly builds on dispatch
+name: Nightly builds on latest release
 
-on: workflow_dispatch
+on:
+  schedule:
+    - cron: '14 2 * * *'
 jobs:
   matrix_prep:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       # creates the matrix strategy from nightly_build_matrix_includes.json
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - id: set-matrix
         uses: JoshuaTheMiller/conditional-build-matrix@main
         with:
-          inputFile: '.github/workflows/nightly_build_matrix_on_dispatch.json'
+          inputFile: '.github/workflows/nightly_build_matrix_includes.json'
           filter: '[?runOnEvent==`${{ github.event_name }}` || runOnEvent==`always`]'
-  build:
+  get_release:
+    runs-on: ubuntu-20.04
     needs: matrix_prep
+    outputs:
+      output1: ${{ steps.step2.outputs.latest_release }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Fetch all branches
+        run: git fetch --all
+      - name: Set output variable to latest release branch
+        id: step2
+        run: echo "latest_release=$(git branch -r | grep -E 'release/v[0-9]+\.[0-9]+\.[0-9]+' | tail -n 1 | sed 's/  origin\///')" >> $GITHUB_OUTPUT
+  build:
+    needs: [get_release,matrix_prep]
     strategy:
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
     steps:
       - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
-        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        if: matrix.os == 'ubuntu-20.04'
 
-      - name: Check out repository code
-        uses: actions/checkout@v2
+      - name: Check out latest release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{needs.get_release.outputs.output1}}
 
       - name: Install rust toolchain
         uses: actions-rs/toolchain@v1
@@ -42,6 +59,12 @@ jobs:
           command: build
           args: --workspace
 
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
       - name: Run all tests
         uses: actions-rs/cargo@v1
         with:
@@ -88,7 +111,7 @@ jobs:
 
       - name: Reclaim some disk space
         uses: actions-rs/cargo@v1
-        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-latest' }}
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
         with:
           command: clean
 
@@ -99,6 +122,12 @@ jobs:
           command: build
           args: --workspace --features=coconut
 
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
       - name: Run all tests with coconut enabled
         uses: actions-rs/cargo@v1
         with:
@@ -145,13 +174,13 @@ jobs:
           args: --manifest-path nym-wallet/Cargo.toml --workspace --all-targets -- -D warnings
 
   notification:
-    needs: build
-    runs-on: ubuntu-latest
+    needs: [build,get_release]
+    runs-on: ubuntu-20.04
     steps:
       - name: Collect jobs status
         uses: technote-space/workflow-conclusion-action@v2
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Keybase - Node Install
         if: env.WORKFLOW_CONCLUSION == 'failure'
         run: npm install
@@ -160,14 +189,14 @@ jobs:
         if: env.WORKFLOW_CONCLUSION == 'failure'
         env:
           NYM_NOTIFICATION_KIND: nightly
-          NYM_PROJECT_NAME: "Nym nightly build"
+          NYM_PROJECT_NAME: "Nym nightly build on latest release"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
-          GIT_BRANCH: "${GITHUB_REF##*/}"
+          GIT_BRANCH_NAME: "${{needs.get_release.outputs.output1}}"
           KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
           KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
-          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMTECH_TEAM }}"
-          KEYBASE_NYM_CHANNEL: "${{ secrets.KEYBASE_CHANNEL_DEV_CORE_ID }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "ci-nightly-release"
           IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
         uses: docker://keybaseio/client:stable-node
         with:

.github/workflows/nightly_build_release2.yml

@@ -0,0 +1,203 @@
+name: Nightly builds on second latest release
+
+on:
+  schedule:
+    - cron: '24 2 * * *'
+jobs:
+  matrix_prep:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      # creates the matrix strategy from nightly_build_matrix_includes.json
+      - uses: actions/checkout@v3
+      - id: set-matrix
+        uses: JoshuaTheMiller/conditional-build-matrix@main
+        with:
+          inputFile: '.github/workflows/nightly_build_matrix_includes.json'
+          filter: '[?runOnEvent==`${{ github.event_name }}` || runOnEvent==`always`]'
+  get_release:
+    runs-on: ubuntu-20.04
+    needs: matrix_prep
+    outputs:
+      output1: ${{ steps.step2.outputs.latest_release }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Fetch all branches
+        run: git fetch --all
+      - name: Set output variable to latest release branch
+        id: step2
+        run: echo "latest_release=$(git branch -r | grep -E 'release/v[0-9]+\.[0-9]+\.[0-9]+' | tail -n 2 | head -n 1 | sed 's/  origin\///')" >> $GITHUB_OUTPUT
+  build:
+    needs: [get_release,matrix_prep]
+    strategy:
+      matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
+    steps:
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        if: matrix.os == 'ubuntu-20.04'
+
+      - name: Check out latest release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{needs.get_release.outputs.output1}}
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run expensive tests
+        if: github.ref == 'refs/heads/develop' || github.event.pull_request.base.ref == 'develop' || github.event.pull_request.base.ref == 'master'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --all-features -- --ignored
+
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy checks
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
+
+      - name: Run clippy
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --workspace --all-targets -- -D warnings
+
+      - name: Reclaim some disk space
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
+        with:
+          command: clean
+
+      # COCONUT stuff
+      - name: Build all binaries with coconut enabled
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --features=coconut
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run all tests with coconut enabled
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --features=coconut
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run clippy with coconut enabled
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --workspace --all-targets --features=coconut -- -D warnings
+
+      # nym-wallet (the rust part)
+      - name: Build nym-wallet rust code
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Run nym-wallet tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Check nym-wallet formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-wallet/Cargo.toml --all -- --check
+
+      - name: Run clippy for nym-wallet
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-targets -- -D warnings
+
+  notification:
+    needs: [build,get_release]
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Collect jobs status
+        uses: technote-space/workflow-conclusion-action@v2
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Keybase - Node Install
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: npm install
+        working-directory: .github/workflows/support-files
+      - name: Keybase - Send Notification
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        env:
+          NYM_NOTIFICATION_KIND: nightly
+          NYM_PROJECT_NAME: "Nym nightly build on latest release"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+          GIT_BRANCH_NAME: "${{needs.get_release.outputs.output1}}"
+          KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+          KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "ci-nightly-release"
+          IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
+        uses: docker://keybaseio/client:stable-node
+        with:
+          args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nym-cli-publish.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, windows-latest, macos-latest]
+        platform: [ubuntu-20.04, windows-latest, macos-latest]
 
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/nym-connect-publish-ubuntu.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/nym-connect.yml

@@ -41,19 +41,19 @@ jobs:
     - name: Keybase - Node Install
       run: npm install
       working-directory: .github/workflows/support-files
-#    - name: Keybase - Send Notification
-#      env:
-#        NYM_NOTIFICATION_KIND: nym-connect
-#        NYM_PROJECT_NAME: "nym-connect"
-#        NYM_CI_WWW_BASE: "${{ secrets.NYM_CI_WWW_BASE }}"
-#        NYM_CI_WWW_LOCATION: "nym-connect-${{ env.GITHUB_REF_SLUG }}"
-#        GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
-#        GIT_BRANCH: "${GITHUB_REF##*/}"
-#        KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
-#        KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
-#        KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
-#        KEYBASE_NYM_CHANNEL: "ci-nym-connect"
-#        IS_SUCCESS: "${{ job.status == 'success' }}"
-#      uses: docker://keybaseio/client:stable-node
-#      with:
-#        args: .github/workflows/support-files/notifications/entry_point.sh
+    - name: Keybase - Send Notification
+      env:
+        NYM_NOTIFICATION_KIND: nym-connect
+        NYM_PROJECT_NAME: "nym-connect"
+        NYM_CI_WWW_BASE: "${{ secrets.NYM_CI_WWW_BASE }}"
+        NYM_CI_WWW_LOCATION: "nym-connect-${{ env.GITHUB_REF_SLUG }}"
+        GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+        GIT_BRANCH: "${GITHUB_REF##*/}"
+        KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+        KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+        KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+        KEYBASE_NYM_CHANNEL: "ci-nym-connect"
+        IS_SUCCESS: "${{ job.status == 'success' }}"
+      uses: docker://keybaseio/client:stable-node
+      with:
+        args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nym-release-publish.yml

@@ -19,12 +19,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
     steps:
       - uses: actions/checkout@v3
 
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+
       - name: Check the release tag starts with `nym-binaries-`
         if: startsWith(github.ref, 'refs/tags/nym-binaries-') == false && github.event_name != 'workflow_dispatch'
         uses: actions/github-script@v3

.github/workflows/nym-wallet-publish-ubuntu.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/nym-wallet-release.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [ubuntu-20.04]
     runs-on: ${{ matrix.platform }}
     steps:
       - uses: actions/checkout@v2

.github/workflows/nym_wallet.yml

@@ -12,7 +12,7 @@ defaults:
 jobs:
   test:
     name: wallet tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/support-files/nightly/index.js

@@ -85,7 +85,7 @@ async function getMessageBody(context) {
             ...
           ],
           check_run_url: 'https://api.github.com/repos/nymtech/nym/check-runs/5182940024',
-          labels: [ 'ubuntu-latest' ],
+          labels: [ 'ubuntu-20.04' ],
           runner_id: 1,
           runner_name: 'Hosted Agent',
           runner_group_id: 2,

.github/workflows/support-files/notifications/send_message.js

@@ -3,7 +3,7 @@ require('dotenv').config();
 const Bot = require('keybase-bot');
 
 let context = {
-  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly', 'nym-connect'],
+  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly', 'nym-connect','security'],
 };
 
 /**

.github/workflows/support-files/security/index.js

@@ -0,0 +1,24 @@
+const Handlebars = require('handlebars');
+const fs = require('fs');
+const path = require('path');
+const { Octokit, App } = require('octokit');
+
+async function addToContextAndValidate(context) {
+  return
+}
+
+async function getMessageBody(context) {
+  try {
+    const source = fs
+    .readFileSync("deny.message").toString();
+    return source;
+} catch (error) {
+    console.error(error);
+}
+
+}
+
+module.exports = {
+  addToContextAndValidate,
+  getMessageBody,
+};

.github/workflows/wasm_client_build.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   wasm:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
 

CHANGELOG.md

@@ -2,25 +2,103 @@
 
 Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased
+## [Unreleased]
 
-### Added
-
-- nym-cli: added CLI tool for interacting with the Nyx blockchain and Nym mixnet smart contracts ([#1577])
-- validator-client: added `query_contract_smart` and `query_contract_raw` on `NymdClient` ([#1558])
-- network-requester: added additional Blockstream Green wallet endpoint to `example.allowed.list` ([#1611](https://github.com/nymtech/nym/pull/1611))
-- common/ledger: new library for communicating with a Ledger device ([#1640])
-
-### Fixed
-
-- validator-api, mixnode, gateway should now prefer values in config.toml over mainnet defaults ([#1645])
+## [v1.1.3] (2022-12-13)
 
 ### Changed
 
-- validator-client: made `fee` argument optional for `execute` and `execute_multiple` ([#1541])
-- socks5 client: graceful shutdown should fix error on disconnect in nym-connect ([#1591])
-- wasm-client: fixed build errors on MacOS and changed example JS code to use mainnet ([#1585])
+- validator-api: can recover from shutdown during DKG process ([#1872])
+- clients: deduplicate gateway inititialization, part of work towards a rust-sdk
+- clients: keep all transmission lanes going at all times by making priority probabilistic
 
+### Fixed
+
+- network-requester: fix bug where websocket connection disconnect resulted in success error code
+- clients: fix a few panics handling the gateway-client
+- mixnode, gateway, validator-api: Use mainnet values as defaults for URLs and mixnet contract  ([#1884])
+
+[#1872]: https://github.com/nymtech/nym/pull/1872
+[#1884]: https://github.com/nymtech/nym/pull/1884
+
+
+## [v1.1.2]
+
+### Changed
+
+- gateway: Renamed flag from `enabled/disabled_credentials_mode` to `only-coconut-credentials`
+- "Family" feature for node families + layers
+- Initial coconut functionality including credentials and distributed key generation 
+
+## [v1.1.1](https://github.com/nymtech/nym/tree/v1.1.1) (2022-11-29)
+
+### Added
+
+- binaries: add `-c` shortform for `--config-env-file`
+- websocket-requests: add server response signalling current packet queue length in the client
+- contracts: DKG contract that handles coconut key generation ([#1678][#1708][#1747])
+- validator-api: generate coconut keys interactively, using DKG and multisig contracts ([#1678][#1708][#1747])
+
+### Changed
+
+- clients: add concept of transmission lanes to better handle multiple data streams ([#1720])
+- clients,validator-api: take coconut signers from the chain instead of specifying them via CLI ([#1747])
+- multisig contract: add DKG contract to the list of addresses that can create proposals ([#1747])
+- socks5-client: wait closing inbound connection until data is sent, and throttle incoming data in general ([#1783])
+- nym-cli: improve error reporting/handling and changed `vesting-schedule` queries to use query client instead of signing client
+
+### Fixed
+
+- gateway-client: fix decrypting stored messages on reconnect ([#1786])
+
+### Fixed
+
+- gateway-client: fix decrypting stored messages on reconnect ([#1786])
+- socks5-client: fix shutting down all tasks if anyone of them panics or errors out ([#1805])
+
+[#1678]: https://github.com/nymtech/nym/pull/1678
+[#1708]: https://github.com/nymtech/nym/pull/1708
+[#1720]: https://github.com/nymtech/nym/pull/1720
+[#1747]: https://github.com/nymtech/nym/pull/1747
+[#1783]: https://github.com/nymtech/nym/pull/1783
+[#1786]: https://github.com/nymtech/nym/pull/1786
+[#1805]: https://github.com/nymtech/nym/pull/1805
+
+
+## [v1.1.0](https://github.com/nymtech/nym/tree/v1.1.0) (2022-11-09)
+
+### Added
+
+- clients: add testing-only support for two more extended packet sizes (8kb and 16kb).
+- common/ledger: new library for communicating with a Ledger device ([#1640])
+- native-client/socks5-client/wasm-client: `disable_loop_cover_traffic_stream` Debug config option to disable the separate loop cover traffic stream ([#1666])
+- native-client/socks5-client/wasm-client: `disable_main_poisson_packet_distribution` Debug config option to make the client ignore poisson distribution in the main packet stream and ONLY send real message (and as fast as they come) ([#1664])
+- native-client/socks5-client/wasm-client: `use_extended_packet_size` Debug config option to make the client use 'ExtendedPacketSize' for its traffic (32kB as opposed to 2kB in 1.0.2) ([#1671])
+- network-requester: added additional Blockstream Green wallet endpoint to `example.allowed.list` ([#1611])
+- validator-api: add `interval_operating_cost` and `profit_margin_percent` to compute reward estimation endpoint
+- validator-client: added `query_contract_smart` and `query_contract_raw` on `NymdClient` ([#1558])
+- wasm-client: uses updated wasm-compatible `client-core` so that it's now capable of packet retransmission, cover traffic and poisson delay (among other things!) ([#1673])
+
+### Fixed
+
+- socks5-client: fix bug where in some cases packet reordering could trigger a connection being closed too early ([#1702],[#1724])
+- validator-api: mixnode, gateway should now prefer values in config.toml over mainnet defaults ([#1645])
+- validator-api: should now correctly update historical uptimes for all mixnodes and gateways every 24h ([#1721])
+
+### Changed
+
+- clients: bound the sphinx packet channel and reduce sending rate if gateway can't keep up ([#1703],[#1725])
+- gateway-client: will attempt to read now as many as 8 websocket messages at once, assuming they're already available on the socket ([#1669])
+- moved `Percent` struct to `contracts-common`, change affects explorer-api
+- socks5 client: graceful shutdown should fix error on disconnect in nym-connect ([#1591])
+- validator-api: changed error serialization on `inclusion_probability`, `stake-saturation` and `reward-estimation` endpoints to provide more accurate information ([#1681]) 
+- validator-client: made `fee` argument optional for `execute` and `execute_multiple` ([#1541])
+- wasm-client: fixed build errors on MacOS and changed example JS code to use mainnet ([#1585])
+- validator-api: changes to internal SQL schema due to the mixnet contract revamp ([#1472])
+- validator-api: changes to internal data structures due to the mixnet contract revamp ([#1472])
+- validator-api: split epoch-operations into multiple separate transactions ([#1472])
+
+[#1472]: https://github.com/nymtech/nym/pull/1472
 [#1541]: https://github.com/nymtech/nym/pull/1541
 [#1558]: https://github.com/nymtech/nym/pull/1558
 [#1577]: https://github.com/nymtech/nym/pull/1577
@@ -28,6 +106,18 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
 [#1591]: https://github.com/nymtech/nym/pull/1591
 [#1640]: https://github.com/nymtech/nym/pull/1640
 [#1645]: https://github.com/nymtech/nym/pull/1645
+[#1611]: https://github.com/nymtech/nym/pull/1611
+[#1664]: https://github.com/nymtech/nym/pull/1664
+[#1666]: https://github.com/nymtech/nym/pull/1645
+[#1669]: https://github.com/nymtech/nym/pull/1669
+[#1671]: https://github.com/nymtech/nym/pull/1671
+[#1673]: https://github.com/nymtech/nym/pull/1673
+[#1681]: https://github.com/nymtech/nym/pull/1681
+[#1702]: https://github.com/nymtech/nym/pull/1702
+[#1703]: https://github.com/nymtech/nym/pull/1703
+[#1721]: https://github.com/nymtech/nym/pull/1721
+[#1724]: https://github.com/nymtech/nym/pull/1724
+[#1725]: https://github.com/nymtech/nym/pull/1725
 
 
 ## [nym-binaries-1.0.2](https://github.com/nymtech/nym/tree/nym-binaries-1.0.2)

Cargo.toml

@@ -26,10 +26,12 @@ members = [
     "common/client-libs/gateway-client",
     "common/client-libs/mixnet-client",
     "common/client-libs/validator-client",
+    "common/client-connections",
     "common/coconut-interface",
     "common/commands",
     "common/config",
     "common/cosmwasm-smart-contracts/coconut-bandwidth-contract",
+    "common/cosmwasm-smart-contracts/coconut-dkg",
     "common/cosmwasm-smart-contracts/contracts-common",
     "common/cosmwasm-smart-contracts/mixnet-contract",
     "common/cosmwasm-smart-contracts/multisig-contract",
@@ -41,6 +43,7 @@ members = [
     "common/execute",
     "common/inclusion-probability",
     "common/ledger",
+    "common/logging",
     "common/mixnode-common",
     "common/network-defaults",
     "common/nonexhaustive-delayqueue",
@@ -67,6 +70,7 @@ members = [
     "explorer-api",
     "gateway",
     "gateway/gateway-requests",
+    "integrations/bity",
     "mixnode",
     "service-providers/network-requester",
     "service-providers/network-statistics",

Makefile

@@ -2,12 +2,12 @@ test: clippy-all cargo-test wasm fmt
 test-all: test cargo-test-expensive
 no-clippy: build cargo-test wasm fmt
 happy: fmt clippy-happy test
-clippy-all: clippy-main clippy-coconut clippy-all-contracts clippy-all-wallet clippy-all-connect
+clippy-all: clippy-main clippy-coconut clippy-all-contracts clippy-all-wallet clippy-all-connect clippy-all-wasm-client
 clippy-happy: clippy-happy-main clippy-happy-contracts clippy-happy-wallet clippy-happy-connect
-cargo-test: test-main test-contracts test-wallet test-connect test-coconut
+cargo-test: test-main test-contracts test-wallet test-connect test-coconut test-wasm-client
 cargo-test-expensive: test-main-expensive test-contracts-expensive test-wallet-expensive test-connect-expensive test-coconut-expensive
-build: build-contracts build-wallet build-main build-connect
-fmt: fmt-main fmt-contracts fmt-wallet fmt-connect
+build: build-contracts build-wallet build-main build-connect build-wasm-client
+fmt: fmt-main fmt-contracts fmt-wallet fmt-connect fmt-wasm-client
 
 clippy-happy-main:
 	cargo clippy
@@ -40,6 +40,9 @@ clippy-all-wallet:
 clippy-all-connect:
 	cargo clippy --workspace --manifest-path nym-connect/Cargo.toml --all-features -- -D warnings
 
+clippy-all-wasm-client:
+	cargo clippy --workspace --manifest-path clients/webassembly/Cargo.toml --all-features --target wasm32-unknown-unknown -- -D warnings
+
 test-main:
 	cargo test --workspace
 
@@ -68,6 +71,9 @@ test-wallet:
 test-wallet-expensive:
 	cargo test --manifest-path nym-wallet/Cargo.toml --all-features -- --ignored
 
+test-wasm-client:
+	cargo test --workspace --manifest-path clients/webassembly/Cargo.toml --all-features
+
 test-connect:
 	cargo test --manifest-path nym-connect/Cargo.toml --all-features
 
@@ -86,6 +92,12 @@ build-wallet:
 build-connect:
 	cargo build --manifest-path nym-connect/Cargo.toml --workspace
 
+build-explorer-api:
+	cargo build --manifest-path explorer-api/Cargo.toml --workspace
+
+build-wasm-client:
+	cargo build --manifest-path clients/webassembly/Cargo.toml --workspace --target wasm32-unknown-unknown
+
 build-nym-cli:
 	cargo build --release --manifest-path tools/nym-cli/Cargo.toml
 
@@ -101,9 +113,15 @@ fmt-wallet:
 fmt-connect:
 	cargo fmt --manifest-path nym-connect/Cargo.toml --all
 
+fmt-wasm-client:
+	cargo fmt --manifest-path clients/webassembly/Cargo.toml --all
+
 wasm:
 	RUSTFLAGS='-C link-arg=-s' cargo build --manifest-path contracts/Cargo.toml --release --target wasm32-unknown-unknown
 
+mixnet-opt: wasm
+	cd contracts/mixnet && make opt
+
 generate-typescript:
 	cd tools/ts-rs-cli && cargo run && cd ../..
 	yarn types:lint:fix

clients/client-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "client-core"
-version = "1.0.1"
+version = "1.1.3"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 edition = "2021"
 
@@ -13,26 +13,54 @@ humantime-serde = "1.0"
 log = "0.4"
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
 serde = { version = "1.0", features = ["derive"] }
-sled = "0.34"
+serde_json = "1.0.89"
+sled = { version = "0.34", optional = true }
+tap = "1.0.1"
 thiserror = "1.0.34"
-tokio = { version = "1.19.1", features = ["macros"] }
+tokio = { version = "1.21.2", features = ["time", "macros"]}
 url = { version ="2.2", features = ["serde"] }
 
 # internal
 config = { path = "../../common/config" }
+client-connections = { path = "../../common/client-connections" }
 crypto = { path = "../../common/crypto" }
 gateway-client = { path = "../../common/client-libs/gateway-client" }
+#gateway-client = { path = "../../common/client-libs/gateway-client", default-features = false, features = ["wasm", "coconut"] }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 nonexhaustive-delayqueue = { path = "../../common/nonexhaustive-delayqueue" }
 nymsphinx = { path = "../../common/nymsphinx" }
 pemstore = { path = "../../common/pemstore" }
-task = { path = "../../common/task" }
 topology = { path = "../../common/topology" }
-validator-client = { path = "../../common/client-libs/validator-client" }
-tap = "1.0.1"
+validator-client = { path = "../../common/client-libs/validator-client", default-features = false }
+task = { path = "../../common/task" }
+
+[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio-stream]
+version = "0.1.9"
+features = ["time"]
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen-futures]
+version = "0.4"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen]
+version = "0.2.83"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-timer]
+git = "https://github.com/mmsinclair/wasm-timer"
+rev = "b9d1a54ad514c2f230a026afe0dde341e98cd7b6"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.gloo-timers]
+version = "0.2.4"
+features = ["futures"]
+
+#[target."cfg(not(target_arch = \"wasm32\"))".dependencies.task]
+#path = "../../common/task"
 
 [dev-dependencies]
 tempfile = "3.1.0"
 
 [features]
+default = ["reply-surb"]
+wasm = ["gateway-client/wasm"]
 coconut = ["gateway-client/coconut", "gateway-requests/coconut"]
+reply-surb = ["sled"]
+

clients/client-core/src/client/base_client/mod.rs

@@ -0,0 +1,441 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::cover_traffic_stream::LoopCoverTrafficStream;
+use crate::client::inbound_messages::{InputMessage, InputMessageReceiver, InputMessageSender};
+use crate::client::key_manager::KeyManager;
+use crate::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
+use crate::client::real_messages_control;
+use crate::client::real_messages_control::RealMessagesController;
+use crate::client::received_buffer::{
+    ReceivedBufferRequestReceiver, ReceivedBufferRequestSender, ReceivedMessagesBufferController,
+};
+use crate::client::topology_control::{
+    TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
+};
+use crate::config::{Config, DebugConfig, GatewayEndpointConfig};
+use crate::error::ClientCoreError;
+use client_connections::{ConnectionCommandReceiver, ConnectionCommandSender, LaneQueueLengths};
+use crypto::asymmetric::identity;
+use futures::channel::mpsc;
+use gateway_client::bandwidth::BandwidthController;
+use gateway_client::{
+    AcknowledgementReceiver, AcknowledgementSender, GatewayClient, MixnetMessageReceiver,
+    MixnetMessageSender,
+};
+use log::{debug, info};
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::addressing::nodes::NodeIdentity;
+#[cfg(feature = "reply-surb")]
+use std::path::PathBuf;
+use tap::TapFallible;
+use task::{ShutdownListener, ShutdownNotifier};
+use url::Url;
+
+// it's fine to do this disgusting compilation flag business here as this problem
+// is going to go away in 1.2.0
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorage;
+
+pub struct ClientInput {
+    pub shared_lane_queue_lengths: LaneQueueLengths,
+    pub connection_command_sender: ConnectionCommandSender,
+    pub input_sender: InputMessageSender,
+}
+
+pub struct ClientOutput {
+    pub received_buffer_request_sender: ReceivedBufferRequestSender,
+}
+
+pub enum ClientInputStatus {
+    AwaitingProducer { client_input: ClientInput },
+    Connected,
+}
+
+impl ClientInputStatus {
+    pub fn register_producer(&mut self) -> ClientInput {
+        match std::mem::replace(self, ClientInputStatus::Connected) {
+            ClientInputStatus::AwaitingProducer { client_input } => client_input,
+            ClientInputStatus::Connected => panic!("producer was already registered before"),
+        }
+    }
+}
+
+pub enum ClientOutputStatus {
+    AwaitingConsumer { client_output: ClientOutput },
+    Connected,
+}
+
+impl ClientOutputStatus {
+    pub fn register_consumer(&mut self) -> ClientOutput {
+        match std::mem::replace(self, ClientOutputStatus::Connected) {
+            ClientOutputStatus::AwaitingConsumer { client_output } => client_output,
+            ClientOutputStatus::Connected => panic!("consumer was already registered before"),
+        }
+    }
+}
+
+pub struct BaseClientBuilder<'a> {
+    // due to wasm limitations I had to split it like this : (
+    gateway_config: &'a GatewayEndpointConfig,
+    debug_config: &'a DebugConfig,
+    disabled_credentials: bool,
+    validator_api_endpoints: Vec<Url>,
+
+    #[cfg(feature = "reply-surb")]
+    reply_surb_keys_store_path: PathBuf,
+
+    bandwidth_controller: Option<BandwidthController>,
+    key_manager: KeyManager,
+}
+
+impl<'a> BaseClientBuilder<'a> {
+    pub fn new_from_base_config<T>(
+        base_config: &'a Config<T>,
+        key_manager: KeyManager,
+        bandwidth_controller: Option<BandwidthController>,
+    ) -> BaseClientBuilder<'a> {
+        BaseClientBuilder {
+            gateway_config: base_config.get_gateway_endpoint_config(),
+            debug_config: base_config.get_debug_config(),
+            disabled_credentials: base_config.get_disabled_credentials_mode(),
+            validator_api_endpoints: base_config.get_validator_api_endpoints(),
+            bandwidth_controller,
+            key_manager,
+            #[cfg(feature = "reply-surb")]
+            reply_surb_keys_store_path: base_config.get_reply_encryption_key_store_path(),
+        }
+    }
+
+    pub fn new(
+        gateway_config: &'a GatewayEndpointConfig,
+        debug_config: &'a DebugConfig,
+        key_manager: KeyManager,
+        bandwidth_controller: Option<BandwidthController>,
+        disabled_credentials: bool,
+        validator_api_endpoints: Vec<Url>,
+        #[cfg(feature = "reply-surb")] reply_surb_keys_store_path: PathBuf,
+    ) -> BaseClientBuilder<'a> {
+        BaseClientBuilder {
+            gateway_config,
+            debug_config,
+            disabled_credentials,
+            validator_api_endpoints,
+            bandwidth_controller,
+            key_manager,
+            #[cfg(feature = "reply-surb")]
+            reply_surb_keys_store_path,
+        }
+    }
+
+    pub fn as_mix_recipient(&self) -> Recipient {
+        Recipient::new(
+            *self.key_manager.identity_keypair().public_key(),
+            *self.key_manager.encryption_keypair().public_key(),
+            // TODO: below only works under assumption that gateway address == gateway id
+            // (which currently is true)
+            NodeIdentity::from_base58_string(&self.gateway_config.gateway_id).unwrap(),
+        )
+    }
+
+    // future constantly pumping loop cover traffic at some specified average rate
+    // the pumped traffic goes to the MixTrafficController
+    fn start_cover_traffic_stream(
+        &self,
+        topology_accessor: TopologyAccessor,
+        mix_tx: BatchMixMessageSender,
+        shutdown: ShutdownListener,
+    ) {
+        info!("Starting loop cover traffic stream...");
+
+        let mut stream = LoopCoverTrafficStream::new(
+            self.key_manager.ack_key(),
+            self.debug_config.average_ack_delay,
+            self.debug_config.average_packet_delay,
+            self.debug_config.loop_cover_traffic_average_delay,
+            mix_tx,
+            self.as_mix_recipient(),
+            topology_accessor,
+        );
+
+        if let Some(size) = self.debug_config.use_extended_packet_size {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
+        }
+
+        stream.start_with_shutdown(shutdown);
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn start_real_traffic_controller(
+        &self,
+        topology_accessor: TopologyAccessor,
+        ack_receiver: AcknowledgementReceiver,
+        input_receiver: InputMessageReceiver,
+        mix_sender: BatchMixMessageSender,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
+        shutdown: ShutdownListener,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
+    ) {
+        let mut controller_config = real_messages_control::Config::new(
+            self.key_manager.ack_key(),
+            self.debug_config.ack_wait_multiplier,
+            self.debug_config.ack_wait_addition,
+            self.debug_config.average_ack_delay,
+            self.debug_config.message_sending_average_delay,
+            self.debug_config.average_packet_delay,
+            self.debug_config.disable_main_poisson_packet_distribution,
+            self.as_mix_recipient(),
+        );
+
+        if let Some(size) = self.debug_config.use_extended_packet_size {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
+        }
+
+        info!("Starting real traffic stream...");
+
+        RealMessagesController::new(
+            controller_config,
+            ack_receiver,
+            input_receiver,
+            mix_sender,
+            topology_accessor,
+            lane_queue_lengths,
+            client_connection_rx,
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage,
+        )
+        .start_with_shutdown(shutdown);
+    }
+
+    // buffer controlling all messages fetched from provider
+    // required so that other components would be able to use them (say the websocket)
+    fn start_received_messages_buffer_controller(
+        &self,
+        query_receiver: ReceivedBufferRequestReceiver,
+        mixnet_receiver: MixnetMessageReceiver,
+        shutdown: ShutdownListener,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
+    ) {
+        info!("Starting received messages buffer controller...");
+        ReceivedMessagesBufferController::new(
+            self.key_manager.encryption_keypair(),
+            query_receiver,
+            mixnet_receiver,
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage,
+        )
+        .start_with_shutdown(shutdown)
+    }
+
+    async fn start_gateway_client(
+        &mut self,
+        mixnet_message_sender: MixnetMessageSender,
+        ack_sender: AcknowledgementSender,
+        shutdown: ShutdownListener,
+    ) -> Result<GatewayClient, ClientCoreError> {
+        let gateway_id = self.gateway_config.gateway_id.clone();
+        if gateway_id.is_empty() {
+            return Err(ClientCoreError::GatewayIdUnknown);
+        }
+        let gateway_owner = self.gateway_config.gateway_owner.clone();
+        if gateway_owner.is_empty() {
+            return Err(ClientCoreError::GatewayOwnerUnknown);
+        }
+        let gateway_address = self.gateway_config.gateway_listener.clone();
+        if gateway_address.is_empty() {
+            return Err(ClientCoreError::GatwayAddressUnknown);
+        }
+
+        let gateway_identity = identity::PublicKey::from_base58_string(gateway_id)
+            .map_err(ClientCoreError::UnableToCreatePublicKeyFromGatewayId)?;
+
+        // disgusting wasm workaround since there's no key persistence there (nor `client init`)
+        let shared_key = if self.key_manager.gateway_key_set() {
+            Some(self.key_manager.gateway_shared_key())
+        } else {
+            None
+        };
+
+        let mut gateway_client = GatewayClient::new(
+            gateway_address,
+            self.key_manager.identity_keypair(),
+            gateway_identity,
+            gateway_owner,
+            shared_key,
+            mixnet_message_sender,
+            ack_sender,
+            self.debug_config.gateway_response_timeout,
+            self.bandwidth_controller.take(),
+            shutdown,
+        );
+
+        gateway_client.set_disabled_credentials_mode(self.disabled_credentials);
+
+        gateway_client
+            .authenticate_and_start()
+            .await
+            .tap_err(|err| {
+                log::error!("Could not authenticate and start up the gateway connection - {err}")
+            })?;
+        Ok(gateway_client)
+    }
+
+    // future responsible for periodically polling directory server and updating
+    // the current global view of topology
+    async fn start_topology_refresher(
+        &mut self,
+        topology_accessor: TopologyAccessor,
+        shutdown: ShutdownListener,
+    ) -> Result<(), ClientCoreError> {
+        let topology_refresher_config = TopologyRefresherConfig::new(
+            self.validator_api_endpoints.clone(),
+            self.debug_config.topology_refresh_rate,
+            env!("CARGO_PKG_VERSION").to_string(),
+        );
+        let mut topology_refresher =
+            TopologyRefresher::new(topology_refresher_config, topology_accessor);
+        // before returning, block entire runtime to refresh the current network view so that any
+        // components depending on topology would see a non-empty view
+        info!("Obtaining initial network topology");
+        topology_refresher.refresh().await;
+
+        // TODO: a slightly more graceful termination here
+        if !topology_refresher.is_topology_routable().await {
+            log::error!(
+                "The current network topology seem to be insufficient to route any packets through \
+                - check if enough nodes and a gateway are online"
+            );
+            return Err(ClientCoreError::InsufficientNetworkTopology);
+        }
+
+        info!("Starting topology refresher...");
+        topology_refresher.start_with_shutdown(shutdown);
+        Ok(())
+    }
+
+    // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
+    // TODO: if we want to send control messages to gateway_client, this CAN'T take the ownership
+    // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
+    // requests?
+    fn start_mix_traffic_controller(
+        gateway_client: GatewayClient,
+        shutdown: ShutdownListener,
+    ) -> BatchMixMessageSender {
+        info!("Starting mix traffic controller...");
+        let (mix_traffic_controller, mix_tx) = MixTrafficController::new(gateway_client);
+        mix_traffic_controller.start_with_shutdown(shutdown);
+        mix_tx
+    }
+
+    pub async fn start_base(mut self) -> Result<BaseClient, ClientCoreError> {
+        info!("Starting nym client");
+        // channels for inter-component communication
+        // TODO: make the channels be internally created by the relevant components
+        // rather than creating them here, so say for example the buffer controller would create the request channels
+        // and would allow anyone to clone the sender channel
+
+        // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
+        // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
+        let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
+
+        // used for announcing connection or disconnection of a channel for pushing re-assembled messages to
+        let (received_buffer_request_sender, received_buffer_request_receiver) = mpsc::unbounded();
+
+        // channels responsible for controlling real messages
+        let (input_sender, input_receiver) = tokio::sync::mpsc::channel::<InputMessage>(1);
+
+        // channels responsible for controlling ack messages
+        let (ack_sender, ack_receiver) = mpsc::unbounded();
+        let shared_topology_accessor = TopologyAccessor::new();
+
+        #[cfg(feature = "reply-surb")]
+        let reply_key_storage =
+            ReplyKeyStorage::load(&self.reply_surb_keys_store_path).tap_err(|err| {
+                log::error!("Failed to load reply key storage - is it perhaps already in use?");
+                log::error!("{:?}", err);
+            })?;
+
+        // Shutdown notifier for signalling tasks to stop
+        let shutdown = ShutdownNotifier::default();
+
+        // the components are started in very specific order. Unless you know what you are doing,
+        // do not change that.
+        self.start_topology_refresher(shared_topology_accessor.clone(), shutdown.subscribe())
+            .await?;
+        self.start_received_messages_buffer_controller(
+            received_buffer_request_receiver,
+            mixnet_messages_receiver,
+            shutdown.subscribe(),
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage.clone(),
+        );
+
+        let gateway_client = self
+            .start_gateway_client(mixnet_messages_sender, ack_sender, shutdown.subscribe())
+            .await?;
+
+        // The sphinx_message_sender is the transmitter for any component generating sphinx packets
+        // that are to be sent to the mixnet. They are used by cover traffic stream and real
+        // traffic stream.
+        // The MixTrafficController then sends the actual traffic
+        let sphinx_message_sender =
+            Self::start_mix_traffic_controller(gateway_client, shutdown.subscribe());
+
+        // Channels that the websocket listener can use to signal downstream to the real traffic
+        // controller that connections are closed.
+        let (client_connection_tx, client_connection_rx) = mpsc::unbounded();
+
+        // Shared queue length data. Published by the `OutQueueController` in the client, and used
+        // primarily to throttle incoming connections (e.g socks5 for attached network-requesters)
+        let shared_lane_queue_lengths = LaneQueueLengths::new();
+
+        self.start_real_traffic_controller(
+            shared_topology_accessor.clone(),
+            ack_receiver,
+            input_receiver,
+            sphinx_message_sender.clone(),
+            shared_lane_queue_lengths.clone(),
+            client_connection_rx,
+            shutdown.subscribe(),
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage,
+        );
+
+        if !self.debug_config.disable_loop_cover_traffic_stream {
+            self.start_cover_traffic_stream(
+                shared_topology_accessor,
+                sphinx_message_sender,
+                shutdown.subscribe(),
+            );
+        }
+
+        debug!("Core client startup finished!");
+        debug!("The address of this client is: {}", self.as_mix_recipient());
+
+        Ok(BaseClient {
+            client_input: ClientInputStatus::AwaitingProducer {
+                client_input: ClientInput {
+                    shared_lane_queue_lengths,
+                    connection_command_sender: client_connection_tx,
+                    input_sender,
+                },
+            },
+            client_output: ClientOutputStatus::AwaitingConsumer {
+                client_output: ClientOutput {
+                    received_buffer_request_sender,
+                },
+            },
+            shutdown_notifier: shutdown,
+        })
+    }
+}
+
+pub struct BaseClient {
+    pub client_input: ClientInputStatus,
+    pub client_output: ClientOutputStatus,
+
+    pub shutdown_notifier: ShutdownNotifier,
+}

clients/client-core/src/client/cover_traffic_stream.rs

@@ -3,20 +3,27 @@
 
 use crate::client::mix_traffic::BatchMixMessageSender;
 use crate::client::topology_control::TopologyAccessor;
+use crate::spawn_future;
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
 use log::*;
 use nymsphinx::acknowledgements::AckKey;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::cover::generate_loop_cover_packet;
+use nymsphinx::params::PacketSize;
 use nymsphinx::utils::sample_poisson_duration;
 use rand::{rngs::OsRng, CryptoRng, Rng};
 use std::pin::Pin;
 use std::sync::Arc;
-use task::ShutdownListener;
-use tokio::task::JoinHandle;
+use std::time::Duration;
+use tokio::sync::mpsc::error::TrySendError;
+
+#[cfg(not(target_arch = "wasm32"))]
 use tokio::time;
 
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
 pub struct LoopCoverTrafficStream<R>
 where
     R: CryptoRng + Rng,
@@ -25,18 +32,22 @@ where
     ack_key: Arc<AckKey>,
 
     /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
-    average_ack_delay: time::Duration,
+    average_ack_delay: Duration,
 
     /// Average delay a data packet is going to get delay at a single mixnode.
-    average_packet_delay: time::Duration,
+    average_packet_delay: Duration,
 
     /// Average delay between sending subsequent cover packets.
-    average_cover_message_sending_delay: time::Duration,
+    average_cover_message_sending_delay: Duration,
 
     /// Internal state, determined by `average_message_sending_delay`,
     /// used to keep track of when a next packet should be sent out.
+    #[cfg(not(target_arch = "wasm32"))]
     next_delay: Pin<Box<time::Sleep>>,
 
+    #[cfg(target_arch = "wasm32")]
+    next_delay: Pin<Box<wasm_timer::Delay>>,
+
     /// Channel used for sending prepared sphinx packets to `MixTrafficController` that sends them
     /// out to the network without any further delays.
     mix_tx: BatchMixMessageSender,
@@ -50,8 +61,8 @@ where
     /// Accessor to the common instance of network topology.
     topology_access: TopologyAccessor,
 
-    /// Listen to shutdown signals.
-    shutdown: ShutdownListener,
+    /// Predefined packet size used for the loop cover messages.
+    packet_size: PacketSize,
 }
 
 impl<R> Stream for LoopCoverTrafficStream<R>
@@ -73,13 +84,21 @@ where
         // we know it's time to send a message, so let's prepare delay for the next one
         // Get the `now` by looking at the current `delay` deadline
         let avg_delay = self.average_cover_message_sending_delay;
-        let now = self.next_delay.deadline();
         let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
 
         // The next interval value is `next_poisson_delay` after the one that just
         // yielded.
-        let next = now + next_poisson_delay;
-        self.next_delay.as_mut().reset(next);
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let now = self.next_delay.deadline();
+            let next = now + next_poisson_delay;
+            self.next_delay.as_mut().reset(next);
+        }
+
+        #[cfg(target_arch = "wasm32")]
+        {
+            self.next_delay.as_mut().reset(next_poisson_delay);
+        }
 
         Poll::Ready(Some(()))
     }
@@ -91,30 +110,49 @@ impl LoopCoverTrafficStream<OsRng> {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         ack_key: Arc<AckKey>,
-        average_ack_delay: time::Duration,
-        average_packet_delay: time::Duration,
-        average_cover_message_sending_delay: time::Duration,
+        average_ack_delay: Duration,
+        average_packet_delay: Duration,
+        average_cover_message_sending_delay: Duration,
         mix_tx: BatchMixMessageSender,
         our_full_destination: Recipient,
         topology_access: TopologyAccessor,
-        shutdown: ShutdownListener,
     ) -> Self {
         let rng = OsRng;
 
+        #[cfg(not(target_arch = "wasm32"))]
+        let next_delay = Box::pin(time::sleep(Default::default()));
+
+        #[cfg(target_arch = "wasm32")]
+        let next_delay = Box::pin(wasm_timer::Delay::new(Default::default()));
+
         LoopCoverTrafficStream {
             ack_key,
             average_ack_delay,
             average_packet_delay,
             average_cover_message_sending_delay,
-            next_delay: Box::pin(time::sleep(Default::default())),
+            next_delay,
             mix_tx,
             our_full_destination,
             rng,
             topology_access,
-            shutdown,
+            packet_size: Default::default(),
         }
     }
 
+    pub fn set_custom_packet_size(&mut self, packet_size: PacketSize) {
+        self.packet_size = packet_size;
+    }
+
+    fn set_next_delay(&mut self, amount: Duration) {
+        #[cfg(not(target_arch = "wasm32"))]
+        let next_delay = Box::pin(time::sleep(amount));
+
+        #[cfg(target_arch = "wasm32")]
+        let next_delay = Box::pin(wasm_timer::Delay::new(amount));
+
+        self.next_delay = next_delay;
+    }
+
     async fn on_new_message(&mut self) {
         trace!("next cover message!");
 
@@ -140,14 +178,26 @@ impl LoopCoverTrafficStream<OsRng> {
             &self.our_full_destination,
             self.average_ack_delay,
             self.average_packet_delay,
+            self.packet_size,
         )
         .expect("Somehow failed to generate a loop cover message with a valid topology");
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        self.mix_tx.unbounded_send(vec![cover_message]).unwrap();
+        if let Err(err) = self.mix_tx.try_send(vec![cover_message]) {
+            match err {
+                TrySendError::Full(_) => {
+                    // This isn't a problem, if the channel is full means we're already sending the
+                    // max amount of messages downstream can handle.
+                    log::debug!("Failed to send cover message - channel full");
+                    // However it's still useful to alert the user that the gateway or the link to
+                    // the gateway can't keep up. Either due to insufficient bandwidth on the
+                    // client side, or that the gateway is overloaded.
+                    log::warn!("Failed to send sphinx packet - gateway or connection to gatway can't keep up");
+                }
+                TrySendError::Closed(_) => {
+                    log::warn!("Failed to send cover message - channel closed");
+                }
+            }
+        }
 
         // TODO: I'm not entirely sure whether this is really required, because I'm not 100%
         // sure how `yield_now()` works - whether it just notifies the scheduler or whether it
@@ -156,40 +206,54 @@ impl LoopCoverTrafficStream<OsRng> {
 
         // JS: due to identical logical structure to OutQueueControl::on_message(), this is also
         // presumably required to prevent bugs in the future. Exact reason is still unknown to me.
+
+        // TODO: temporary and BAD workaround for wasm (we should find a way to yield here in wasm)
+        #[cfg(not(target_arch = "wasm32"))]
         tokio::task::yield_now().await;
     }
 
-    async fn run(&mut self) {
+    pub fn start_with_shutdown(mut self, mut shutdown: task::ShutdownListener) {
         // we should set initial delay only when we actually start the stream
-        self.next_delay = Box::pin(time::sleep(sample_poisson_duration(
-            &mut self.rng,
-            self.average_cover_message_sending_delay,
-        )));
+        let sampled =
+            sample_poisson_duration(&mut self.rng, self.average_cover_message_sending_delay);
+        self.set_next_delay(sampled);
 
-        let mut shutdown = self.shutdown.clone();
-        while !shutdown.is_shutdown() {
-            tokio::select! {
-                biased;
-                _ = shutdown.recv() => {
-                    log::trace!("LoopCoverTrafficStream: Received shutdown");
-                }
-                next = self.next() => {
-                    if next.is_some() {
-                        self.on_new_message().await;
-                    } else {
-                        log::trace!("LoopCoverTrafficStream: Stopping since channel closed");
-                        break;
+        spawn_future(async move {
+            debug!("Started LoopCoverTrafficStream with graceful shutdown support");
+
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv() => {
+                        log::trace!("LoopCoverTrafficStream: Received shutdown");
+                    }
+                    next = self.next() => {
+                        if next.is_some() {
+                            self.on_new_message().await;
+                        } else {
+                            log::trace!("LoopCoverTrafficStream: Stopping since channel closed");
+                            break;
+                        }
                     }
                 }
             }
-        }
-        assert!(self.shutdown.is_shutdown_poll());
-        log::debug!("LoopCoverTrafficStream: Exiting");
+            shutdown.recv_timeout().await;
+            log::debug!("LoopCoverTrafficStream: Exiting");
+        })
     }
 
-    pub fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            self.run().await;
+    pub fn start(mut self) {
+        // we should set initial delay only when we actually start the stream
+        let sampled =
+            sample_poisson_duration(&mut self.rng, self.average_cover_message_sending_delay);
+        self.set_next_delay(sampled);
+
+        spawn_future(async move {
+            debug!("Started LoopCoverTrafficStream without graceful shutdown support");
+
+            while self.next().await.is_some() {
+                self.on_new_message().await;
+            }
         })
     }
 }

clients/client-core/src/client/inbound_messages.rs

@@ -1,9 +1,9 @@
-use futures::channel::mpsc;
+use client_connections::TransmissionLane;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::anonymous_replies::ReplySurb;
 
-pub type InputMessageSender = mpsc::UnboundedSender<InputMessage>;
-pub type InputMessageReceiver = mpsc::UnboundedReceiver<InputMessage>;
+pub type InputMessageSender = tokio::sync::mpsc::Sender<InputMessage>;
+pub type InputMessageReceiver = tokio::sync::mpsc::Receiver<InputMessage>;
 
 #[derive(Debug)]
 pub enum InputMessage {
@@ -11,6 +11,7 @@ pub enum InputMessage {
         recipient: Recipient,
         data: Vec<u8>,
         with_reply_surb: bool,
+        lane: TransmissionLane,
     },
     Reply {
         reply_surb: ReplySurb,
@@ -19,11 +20,17 @@ pub enum InputMessage {
 }
 
 impl InputMessage {
-    pub fn new_fresh(recipient: Recipient, data: Vec<u8>, with_reply_surb: bool) -> Self {
+    pub fn new_fresh(
+        recipient: Recipient,
+        data: Vec<u8>,
+        with_reply_surb: bool,
+        lane: TransmissionLane,
+    ) -> Self {
         InputMessage::Fresh {
             recipient,
             data,
             with_reply_surb,
+            lane,
         }
     }
 

clients/client-core/src/client/key_manager.rs

@@ -149,6 +149,10 @@ impl KeyManager {
         )
     }
 
+    pub fn gateway_key_set(&self) -> bool {
+        self.gateway_shared_key.is_some()
+    }
+
     /// Gets an atomically reference counted pointer to [`AckKey`].
     pub fn ack_key(&self) -> Arc<AckKey> {
         Arc::clone(&self.ack_key)

clients/client-core/src/client/mix_traffic.rs

@@ -1,17 +1,16 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use futures::channel::mpsc;
-use futures::StreamExt;
+use crate::spawn_future;
 use gateway_client::GatewayClient;
 use log::*;
 use nymsphinx::forwarding::packet::MixPacket;
-use task::ShutdownListener;
-use tokio::task::JoinHandle;
 
-pub type BatchMixMessageSender = mpsc::UnboundedSender<Vec<MixPacket>>;
-pub type BatchMixMessageReceiver = mpsc::UnboundedReceiver<Vec<MixPacket>>;
+pub type BatchMixMessageSender = tokio::sync::mpsc::Sender<Vec<MixPacket>>;
+pub type BatchMixMessageReceiver = tokio::sync::mpsc::Receiver<Vec<MixPacket>>;
 
+// We remind ourselves that 32 x 32kb = 1024kb, a reasonable size for a network buffer.
+pub const MIX_MESSAGE_RECEIVER_BUFFER_SIZE: usize = 32;
 const MAX_FAILURE_COUNT: usize = 100;
 
 pub struct MixTrafficController {
@@ -19,7 +18,6 @@ pub struct MixTrafficController {
     // later on gateway_client will need to be accessible by other entities
     gateway_client: GatewayClient,
     mix_rx: BatchMixMessageReceiver,
-    shutdown: ShutdownListener,
 
     // TODO: this is temporary work-around.
     // in long run `gateway_client` will be moved away from `MixTrafficController` anyway.
@@ -27,17 +25,17 @@ pub struct MixTrafficController {
 }
 
 impl MixTrafficController {
-    pub fn new(
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-        shutdown: ShutdownListener,
-    ) -> MixTrafficController {
-        MixTrafficController {
-            gateway_client,
-            mix_rx,
-            shutdown,
-            consecutive_gateway_failure_count: 0,
-        }
+    pub fn new(gateway_client: GatewayClient) -> (MixTrafficController, BatchMixMessageSender) {
+        let (sphinx_message_sender, sphinx_message_receiver) =
+            tokio::sync::mpsc::channel(MIX_MESSAGE_RECEIVER_BUFFER_SIZE);
+        (
+            MixTrafficController {
+                gateway_client,
+                mix_rx: sphinx_message_receiver,
+                consecutive_gateway_failure_count: 0,
+            },
+            sphinx_message_sender,
+        )
     }
 
     async fn on_messages(&mut self, mut mix_packets: Vec<MixPacket>) {
@@ -69,30 +67,38 @@ impl MixTrafficController {
         }
     }
 
-    pub async fn run(&mut self) {
-        while !self.shutdown.is_shutdown() {
-            tokio::select! {
-                mix_packets = self.mix_rx.next() => match mix_packets {
-                    Some(mix_packets) => {
-                        self.on_messages(mix_packets).await;
+    pub fn start_with_shutdown(mut self, mut shutdown: task::ShutdownListener) {
+        spawn_future(async move {
+            debug!("Started MixTrafficController with graceful shutdown support");
+
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    mix_packets = self.mix_rx.recv() => match mix_packets {
+                        Some(mix_packets) => {
+                            self.on_messages(mix_packets).await;
+                        },
+                        None => {
+                            log::trace!("MixTrafficController: Stopping since channel closed");
+                            break;
+                        }
                     },
-                    None => {
-                        log::trace!("MixTrafficController: Stopping since channel closed");
-                        break;
+                    _ = shutdown.recv() => {
+                        log::trace!("MixTrafficController: Received shutdown");
                     }
-                },
-                _ = self.shutdown.recv() => {
-                    log::trace!("MixTrafficController: Received shutdown");
                 }
             }
-        }
-        assert!(self.shutdown.is_shutdown_poll());
-        log::debug!("MixTrafficController: Exiting");
+            shutdown.recv_timeout().await;
+            log::debug!("MixTrafficController: Exiting");
+        })
     }
 
-    pub fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            self.run().await;
+    pub fn start(mut self) {
+        spawn_future(async move {
+            debug!("Started MixTrafficController without graceful shutdown support");
+
+            while let Some(mix_packets) = self.mix_rx.recv().await {
+                self.on_messages(mix_packets).await;
+            }
         })
     }
 }

clients/client-core/src/client/mod.rs

@@ -1,17 +1,13 @@
-use std::sync::atomic::AtomicBool;
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
 
+pub mod base_client;
 pub mod cover_traffic_stream;
 pub mod inbound_messages;
 pub mod key_manager;
 pub mod mix_traffic;
 pub mod real_messages_control;
 pub mod received_buffer;
+#[cfg(feature = "reply-surb")]
 pub mod reply_key_storage;
 pub mod topology_control;
-
-// This is *NOT* used to signal shutdown.
-// It's critical that we don't have any tasks finishing early, this is an additional safety check
-// that tasks exiting are doing so because shutdown has been signalled, and no other reason.
-// In particular for tasks that rely on their associated channel being closed to signal shutdown,
-// and don't have access to a shutdown listener channel.
-pub static SHUTDOWN_HAS_BEEN_SIGNALLED: AtomicBool = AtomicBool::new(false);

clients/client-core/src/client/real_messages_control/acknowledgement_control/acknowledgement_listener.rs

@@ -10,7 +10,6 @@ use nymsphinx::{
     chunking::fragment::{FragmentIdentifier, COVER_FRAG_ID},
 };
 use std::sync::Arc;
-use task::ShutdownListener;
 
 /// Module responsible for listening for any data resembling acknowledgements from the network
 /// and firing actions to remove them from the 'Pending' state.
@@ -18,7 +17,6 @@ pub(super) struct AcknowledgementListener {
     ack_key: Arc<AckKey>,
     ack_receiver: AcknowledgementReceiver,
     action_sender: ActionSender,
-    shutdown: ShutdownListener,
 }
 
 impl AcknowledgementListener {
@@ -26,18 +24,16 @@ impl AcknowledgementListener {
         ack_key: Arc<AckKey>,
         ack_receiver: AcknowledgementReceiver,
         action_sender: ActionSender,
-        shutdown: ShutdownListener,
     ) -> Self {
         AcknowledgementListener {
             ack_key,
             ack_receiver,
             action_sender,
-            shutdown,
         }
     }
 
     async fn on_ack(&mut self, ack_content: Vec<u8>) {
-        debug!("Received an ack");
+        trace!("Received an ack");
         let frag_id = match recover_identifier(&self.ack_key, &ack_content)
             .map(FragmentIdentifier::try_from_bytes)
         {
@@ -67,28 +63,41 @@ impl AcknowledgementListener {
             .unwrap();
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started AcknowledgementListener");
-        while !self.shutdown.is_shutdown() {
+    async fn handle_ack_receiver_item(&mut self, item: Vec<Vec<u8>>) {
+        // realistically we would only be getting one ack at the time
+        for ack in item {
+            self.on_ack(ack).await;
+        }
+    }
+
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started AcknowledgementListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
             tokio::select! {
                 acks = self.ack_receiver.next() => match acks {
-                    Some(acks) => {
-                        // realistically we would only be getting one ack at the time
-                        for ack in acks {
-                            self.on_ack(ack).await;
-                        }
-                    },
+                    Some(acks) => self.handle_ack_receiver_item(acks).await,
                     None => {
                         log::trace!("AcknowledgementListener: Stopping since channel closed");
                         break;
                     }
                 },
-                _ = self.shutdown.recv() => {
+                _ = shutdown.recv() => {
                     log::trace!("AcknowledgementListener: Received shutdown");
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        shutdown.recv_timeout().await;
         log::debug!("AcknowledgementListener: Exiting");
     }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started AcknowledgementListener without graceful shutdown support");
+
+        while let Some(acks) = self.ack_receiver.next().await {
+            self.handle_ack_receiver_item(acks).await
+        }
+    }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/action_controller.rs

@@ -12,7 +12,6 @@ use nymsphinx::Delay as SphinxDelay;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
-use task::ShutdownListener;
 
 pub(crate) type ActionSender = UnboundedSender<Action>;
 
@@ -100,16 +99,12 @@ pub(super) struct ActionController {
 
     /// Channel for notifying `RetransmissionRequestListener` about expired acknowledgements.
     retransmission_sender: RetransmissionRequestSender,
-
-    /// Listen for shutdown notifications
-    shutdown: ShutdownListener,
 }
 
 impl ActionController {
     pub(super) fn new(
         config: Config,
         retransmission_sender: RetransmissionRequestSender,
-        shutdown: ShutdownListener,
     ) -> (Self, ActionSender) {
         let (sender, receiver) = mpsc::unbounded();
         (
@@ -119,7 +114,6 @@ impl ActionController {
                 pending_acks_timers: NonExhaustiveDelayQueue::new(),
                 incoming_actions: receiver,
                 retransmission_sender,
-                shutdown,
             },
             sender,
         )
@@ -251,8 +245,10 @@ impl ActionController {
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        while !self.shutdown.is_shutdown() {
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started ActionController with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
             tokio::select! {
                 action = self.incoming_actions.next() => match action {
                     Some(action) => self.process_action(action),
@@ -270,12 +266,28 @@ impl ActionController {
                         break;
                     }
                 },
-                _ = self.shutdown.recv() => {
+                _ = shutdown.recv() => {
                     log::trace!("ActionController: Received shutdown");
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        #[cfg(not(target_arch = "wasm32"))]
+        tokio::time::timeout(Duration::from_secs(5), shutdown.recv())
+            .await
+            .expect("Task stopped without shutdown called");
         log::debug!("ActionController: Exiting");
     }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started ActionController without graceful shutdown support");
+
+        loop {
+            tokio::select! {
+                action = self.incoming_actions.next() => self.process_action(action.unwrap()),
+                expired_ack = self.pending_acks_timers.next() => self.handle_expired_ack_timer(expired_ack.unwrap())
+            }
+        }
+    }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs

@@ -3,20 +3,21 @@
 
 use super::action_controller::{Action, ActionSender};
 use super::PendingAcknowledgement;
-use crate::client::reply_key_storage::ReplyKeyStorage;
 use crate::client::{
     inbound_messages::{InputMessage, InputMessageReceiver},
     real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
     topology_control::TopologyAccessor,
 };
-use futures::StreamExt;
+use client_connections::TransmissionLane;
 use log::*;
 use nymsphinx::anonymous_replies::ReplySurb;
 use nymsphinx::preparer::MessagePreparer;
 use nymsphinx::{acknowledgements::AckKey, addressing::clients::Recipient};
 use rand::{CryptoRng, Rng};
 use std::sync::Arc;
-use task::ShutdownListener;
+
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorage;
 
 /// Module responsible for dealing with the received messages: splitting them, creating acknowledgements,
 /// putting everything into sphinx packets, etc.
@@ -32,8 +33,8 @@ where
     action_sender: ActionSender,
     real_message_sender: BatchRealMessageSender,
     topology_access: TopologyAccessor,
+    #[cfg(feature = "reply-surb")]
     reply_key_storage: ReplyKeyStorage,
-    shutdown: ShutdownListener,
 }
 
 impl<R> InputMessageListener<R>
@@ -51,8 +52,7 @@ where
         action_sender: ActionSender,
         real_message_sender: BatchRealMessageSender,
         topology_access: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
-        shutdown: ShutdownListener,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
     ) -> Self {
         InputMessageListener {
             ack_key,
@@ -62,8 +62,8 @@ where
             action_sender,
             real_message_sender,
             topology_access,
+            #[cfg(feature = "reply-surb")]
             reply_key_storage,
-            shutdown,
         }
     }
 
@@ -104,6 +104,7 @@ where
         content: Vec<u8>,
         with_reply_surb: bool,
     ) -> Option<Vec<RealMessage>> {
+        log::trace!("handling msg size: {}", content.len());
         let topology_permit = self.topology_access.get_read_permit().await;
         let topology = match topology_permit
             .try_get_valid_topology_ref(&self.ack_recipient, Some(&recipient))
@@ -121,12 +122,16 @@ where
             .prepare_and_split_message(content, with_reply_surb, topology)
             .expect("somehow the topology was invalid after all!");
 
+        #[cfg(feature = "reply-surb")]
         if let Some(reply_key) = reply_key {
             self.reply_key_storage
                 .insert_encryption_key(reply_key)
                 .expect("Failed to insert surb reply key to the store!")
         }
 
+        #[cfg(not(feature = "reply-surb"))]
+        let _reply_key = reply_key;
+
         // encrypt chunks, put them inside sphinx packets and generate acks
         let mut pending_acks = Vec::with_capacity(split_message.len());
         let mut real_messages = Vec::with_capacity(split_message.len());
@@ -160,35 +165,41 @@ where
     }
 
     async fn on_input_message(&mut self, msg: InputMessage) {
-        let real_messages = match msg {
+        let (real_messages, lane) = match msg {
             InputMessage::Fresh {
                 recipient,
                 data,
                 with_reply_surb,
-            } => {
+                lane,
+            } => (
                 self.handle_fresh_message(recipient, data, with_reply_surb)
+                    .await,
+                lane,
+            ),
+            InputMessage::Reply { reply_surb, data } => (
+                self.handle_reply(reply_surb, data)
                     .await
-            }
-            InputMessage::Reply { reply_surb, data } => self
-                .handle_reply(reply_surb, data)
-                .await
-                .map(|message| vec![message]),
+                    .map(|message| vec![message]),
+                TransmissionLane::Reply,
+            ),
         };
 
         // there's no point in trying to send nothing
         if let Some(real_messages) = real_messages {
             // tells real message sender (with the poisson timer) to send this to the mix network
             self.real_message_sender
-                .unbounded_send(real_messages)
-                .unwrap();
+                .send((real_messages, lane))
+                .await
+                .expect("BatchRealMessageReceiver has stopped receiving!");
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started InputMessageListener");
-        while !self.shutdown.is_shutdown() {
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started InputMessageListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
             tokio::select! {
-                input_msg = self.input_receiver.next() => match input_msg {
+                input_msg = self.input_receiver.recv() => match input_msg {
                     Some(input_msg) => {
                         self.on_input_message(input_msg).await;
                     },
@@ -197,12 +208,21 @@ where
                         break;
                     }
                 },
-                _ = self.shutdown.recv() => {
+                _ = shutdown.recv() => {
                     log::trace!("InputMessageListener: Received shutdown");
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        shutdown.recv_timeout().await;
         log::debug!("InputMessageListener: Exiting");
     }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started InputMessageListener without graceful shutdown support");
+        while let Some(input_msg) = self.input_receiver.recv().await {
+            self.on_input_message(input_msg).await;
+        }
+    }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/mod.rs

@@ -8,11 +8,12 @@ use self::{
     sent_notification_listener::SentNotificationListener,
 };
 use super::real_traffic_stream::BatchRealMessageSender;
-use crate::client::reply_key_storage::ReplyKeyStorage;
 use crate::client::{inbound_messages::InputMessageReceiver, topology_control::TopologyAccessor};
+use crate::spawn_future;
 use futures::channel::mpsc;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
+use nymsphinx::params::PacketSize;
 use nymsphinx::{
     acknowledgements::AckKey,
     addressing::clients::Recipient,
@@ -25,8 +26,9 @@ use std::{
     sync::{Arc, Weak},
     time::Duration,
 };
-use task::ShutdownListener;
-use tokio::task::JoinHandle;
+
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorage;
 
 mod acknowledgement_listener;
 mod action_controller;
@@ -120,6 +122,9 @@ pub(super) struct Config {
 
     /// Average delay a data packet is going to get delayed at a single mixnode.
     average_packet_delay: Duration,
+
+    /// Predefined packet size used for the encapsulated messages.
+    packet_size: PacketSize,
 }
 
 impl Config {
@@ -134,19 +139,25 @@ impl Config {
             ack_wait_multiplier,
             average_ack_delay,
             average_packet_delay,
+            packet_size: Default::default(),
         }
     }
+
+    pub fn with_custom_packet_size(mut self, packet_size: PacketSize) -> Self {
+        self.packet_size = packet_size;
+        self
+    }
 }
 
 pub(super) struct AcknowledgementController<R>
 where
     R: CryptoRng + Rng,
 {
-    acknowledgement_listener: Option<AcknowledgementListener>,
-    input_message_listener: Option<InputMessageListener<R>>,
-    retransmission_request_listener: Option<RetransmissionRequestListener<R>>,
-    sent_notification_listener: Option<SentNotificationListener>,
-    action_controller: Option<ActionController>,
+    acknowledgement_listener: AcknowledgementListener,
+    input_message_listener: InputMessageListener<R>,
+    retransmission_request_listener: RetransmissionRequestListener<R>,
+    sent_notification_listener: SentNotificationListener,
+    action_controller: ActionController,
 }
 
 impl<R> AcknowledgementController<R>
@@ -160,30 +171,29 @@ where
         topology_access: TopologyAccessor,
         ack_key: Arc<AckKey>,
         ack_recipient: Recipient,
-        reply_key_storage: ReplyKeyStorage,
         connectors: AcknowledgementControllerConnectors,
-        shutdown: ShutdownListener,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
     ) -> Self {
         let (retransmission_tx, retransmission_rx) = mpsc::unbounded();
 
         let action_config =
             action_controller::Config::new(config.ack_wait_addition, config.ack_wait_multiplier);
         let (action_controller, action_sender) =
-            ActionController::new(action_config, retransmission_tx, shutdown.clone());
+            ActionController::new(action_config, retransmission_tx);
 
         let message_preparer = MessagePreparer::new(
             rng,
             ack_recipient,
             config.average_packet_delay,
             config.average_ack_delay,
-        );
+        )
+        .with_custom_real_message_packet_size(config.packet_size);
 
         // will listen for any acks coming from the network
         let acknowledgement_listener = AcknowledgementListener::new(
             Arc::clone(&ack_key),
             connectors.ack_receiver,
             action_sender.clone(),
-            shutdown.clone(),
         );
 
         // will listen for any new messages from the client
@@ -195,8 +205,8 @@ where
             action_sender.clone(),
             connectors.real_message_sender.clone(),
             topology_access.clone(),
+            #[cfg(feature = "reply-surb")]
             reply_key_storage,
-            shutdown.clone(),
         );
 
         // will listen for any ack timeouts and trigger retransmission
@@ -208,75 +218,95 @@ where
             connectors.real_message_sender,
             retransmission_rx,
             topology_access,
-            shutdown.clone(),
         );
 
         // will listen for events indicating the packet was sent through the network so that
         // the retransmission timer should be started.
         let sent_notification_listener =
-            SentNotificationListener::new(connectors.sent_notifier, action_sender, shutdown);
+            SentNotificationListener::new(connectors.sent_notifier, action_sender);
 
         AcknowledgementController {
-            acknowledgement_listener: Some(acknowledgement_listener),
-            input_message_listener: Some(input_message_listener),
-            retransmission_request_listener: Some(retransmission_request_listener),
-            sent_notification_listener: Some(sent_notification_listener),
-            action_controller: Some(action_controller),
+            acknowledgement_listener,
+            input_message_listener,
+            retransmission_request_listener,
+            sent_notification_listener,
+            action_controller,
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        let mut acknowledgement_listener = self.acknowledgement_listener.take().unwrap();
-        let mut input_message_listener = self.input_message_listener.take().unwrap();
-        let mut retransmission_request_listener =
-            self.retransmission_request_listener.take().unwrap();
-        let mut sent_notification_listener = self.sent_notification_listener.take().unwrap();
-        let mut action_controller = self.action_controller.take().unwrap();
+    pub(super) fn start_with_shutdown(self, shutdown: task::ShutdownListener) {
+        let mut acknowledgement_listener = self.acknowledgement_listener;
+        let mut input_message_listener = self.input_message_listener;
+        let mut retransmission_request_listener = self.retransmission_request_listener;
+        let mut sent_notification_listener = self.sent_notification_listener;
+        let mut action_controller = self.action_controller;
 
-        // the below are log messages are errors as at the current stage we do not expect any of
-        // the task to ever finish. This will of course change once we introduce
-        // graceful shutdowns.
-        let ack_listener_fut = tokio::spawn(async move {
-            acknowledgement_listener.run().await;
-            debug!("The acknowledgement listener has finished execution!");
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
             acknowledgement_listener
-        });
-        let input_listener_fut = tokio::spawn(async move {
-            input_message_listener.run().await;
-            debug!("The input listener has finished execution!");
-            input_message_listener
-        });
-        let retransmission_req_fut = tokio::spawn(async move {
-            retransmission_request_listener.run().await;
-            debug!("The retransmission request listener has finished execution!");
-            retransmission_request_listener
-        });
-        let sent_notification_fut = tokio::spawn(async move {
-            sent_notification_listener.run().await;
-            debug!("The sent notification listener has finished execution!");
-            sent_notification_listener
-        });
-        let action_controller_fut = tokio::spawn(async move {
-            action_controller.run().await;
-            debug!("The controller has finished execution!");
-            action_controller
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The acknowledgement listener has finished execution!");
         });
 
-        // technically we don't have to bring `AcknowledgementController` back to a valid state
-        // but we can do it, so why not? Perhaps it might be useful if we wanted to allow
-        // for restarts of certain modules without killing the entire process.
-        self.acknowledgement_listener = Some(ack_listener_fut.await.unwrap());
-        self.input_message_listener = Some(input_listener_fut.await.unwrap());
-        self.retransmission_request_listener = Some(retransmission_req_fut.await.unwrap());
-        self.sent_notification_listener = Some(sent_notification_fut.await.unwrap());
-        self.action_controller = Some(action_controller_fut.await.unwrap());
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            input_message_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The input listener has finished execution!");
+        });
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            retransmission_request_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The retransmission request listener has finished execution!");
+        });
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            sent_notification_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The sent notification listener has finished execution!");
+        });
+
+        spawn_future(async move {
+            action_controller.run_with_shutdown(shutdown).await;
+            debug!("The controller has finished execution!");
+        });
     }
 
+    // todo: think whether this is still required
     #[allow(dead_code)]
-    pub(super) fn start(mut self) -> JoinHandle<Self> {
-        tokio::spawn(async move {
-            self.run().await;
-            self
-        })
+    pub(super) fn start(self) {
+        let mut acknowledgement_listener = self.acknowledgement_listener;
+        let mut input_message_listener = self.input_message_listener;
+        let mut retransmission_request_listener = self.retransmission_request_listener;
+        let mut sent_notification_listener = self.sent_notification_listener;
+        let mut action_controller = self.action_controller;
+
+        spawn_future(async move {
+            acknowledgement_listener.run().await;
+            error!("The acknowledgement listener has finished execution!");
+        });
+        spawn_future(async move {
+            input_message_listener.run().await;
+            error!("The input listener has finished execution!");
+        });
+        spawn_future(async move {
+            retransmission_request_listener.run().await;
+            error!("The retransmission request listener has finished execution!");
+        });
+        spawn_future(async move {
+            sent_notification_listener.run().await;
+            error!("The sent notification listener has finished execution!");
+        });
+        spawn_future(async move {
+            action_controller.run().await;
+            error!("The controller has finished execution!");
+        });
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs

@@ -1,20 +1,23 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
-use super::PendingAcknowledgement;
-use super::RetransmissionRequestReceiver;
+use super::{
+    action_controller::{Action, ActionSender},
+    PendingAcknowledgement, RetransmissionRequestReceiver,
+};
 use crate::client::{
     real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
     topology_control::TopologyAccessor,
 };
+
+use client_connections::TransmissionLane;
 use futures::StreamExt;
 use log::*;
-use nymsphinx::preparer::MessagePreparer;
-use nymsphinx::{acknowledgements::AckKey, addressing::clients::Recipient};
+use nymsphinx::{
+    acknowledgements::AckKey, addressing::clients::Recipient, preparer::MessagePreparer,
+};
 use rand::{CryptoRng, Rng};
 use std::sync::{Arc, Weak};
-use task::ShutdownListener;
 
 // responsible for packet retransmission upon fired timer
 pub(super) struct RetransmissionRequestListener<R>
@@ -28,7 +31,6 @@ where
     real_message_sender: BatchRealMessageSender,
     request_receiver: RetransmissionRequestReceiver,
     topology_access: TopologyAccessor,
-    shutdown: ShutdownListener,
 }
 
 impl<R> RetransmissionRequestListener<R>
@@ -44,7 +46,6 @@ where
         real_message_sender: BatchRealMessageSender,
         request_receiver: RetransmissionRequestReceiver,
         topology_access: TopologyAccessor,
-        shutdown: ShutdownListener,
     ) -> Self {
         RetransmissionRequestListener {
             ack_key,
@@ -54,7 +55,6 @@ where
             real_message_sender,
             request_receiver,
             topology_access,
-            shutdown,
         }
     }
 
@@ -117,17 +117,18 @@ where
 
         // send to `OutQueueControl` to eventually send to the mix network
         self.real_message_sender
-            .unbounded_send(vec![RealMessage::new(
-                prepared_fragment.mix_packet,
-                frag_id,
-            )])
-            .unwrap();
+            .send((
+                vec![RealMessage::new(prepared_fragment.mix_packet, frag_id)],
+                TransmissionLane::Retransmission,
+            ))
+            .await
+            .expect("BatchRealMessageReceiver has stopped receiving!");
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started RetransmissionRequestListener");
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started RetransmissionRequestListener with graceful shutdown support");
 
-        while !self.shutdown.is_shutdown() {
+        while !shutdown.is_shutdown() {
             tokio::select! {
                 timed_out_ack = self.request_receiver.next() => match timed_out_ack {
                     Some(timed_out_ack) => self.on_retransmission_request(timed_out_ack).await,
@@ -136,12 +137,22 @@ where
                         break;
                     }
                 },
-                _ = self.shutdown.recv() => {
+                _ = shutdown.recv() => {
                     log::trace!("RetransmissionRequestListener: Received shutdown");
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        shutdown.recv_timeout().await;
         log::debug!("RetransmissionRequestListener: Exiting");
     }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started RetransmissionRequestListener without graceful shutdown support");
+
+        while let Some(timed_out_ack) = self.request_receiver.next().await {
+            self.on_retransmission_request(timed_out_ack).await;
+        }
+    }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/sent_notification_listener.rs

@@ -6,7 +6,6 @@ use super::SentPacketNotificationReceiver;
 use futures::StreamExt;
 use log::*;
 use nymsphinx::chunking::fragment::{FragmentIdentifier, COVER_FRAG_ID};
-use task::ShutdownListener;
 
 /// Module responsible for starting up retransmission timers.
 /// It is required because when we send our packet to the `real traffic stream` controlled
@@ -15,19 +14,16 @@ use task::ShutdownListener;
 pub(super) struct SentNotificationListener {
     sent_notifier: SentPacketNotificationReceiver,
     action_sender: ActionSender,
-    shutdown: ShutdownListener,
 }
 
 impl SentNotificationListener {
     pub(super) fn new(
         sent_notifier: SentPacketNotificationReceiver,
         action_sender: ActionSender,
-        shutdown: ShutdownListener,
     ) -> Self {
         SentNotificationListener {
             sent_notifier,
             action_sender,
-            shutdown,
         }
     }
 
@@ -46,9 +42,10 @@ impl SentNotificationListener {
             .unwrap();
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started SentNotificationListener");
-        while !self.shutdown.is_shutdown() {
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started SentNotificationListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
             tokio::select! {
                 frag_id = self.sent_notifier.next() => match frag_id {
                     Some(frag_id) => {
@@ -59,12 +56,22 @@ impl SentNotificationListener {
                         break;
                     }
                 },
-                _ = self.shutdown.recv() => {
+                _ = shutdown.recv() => {
                     log::trace!("SentNotificationListener: Received shutdown");
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        assert!(shutdown.is_shutdown_poll());
         log::debug!("SentNotificationListener: Exiting");
     }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started SentNotificationListener without graceful shutdown support");
+
+        while let Some(frag_id) = self.sent_notifier.next().await {
+            self.on_sent_message(frag_id).await;
+        }
+    }
 }

clients/client-core/src/client/real_messages_control/mod.rs

@@ -8,22 +8,27 @@
 use self::{
     acknowledgement_control::AcknowledgementController, real_traffic_stream::OutQueueControl,
 };
-use crate::client::real_messages_control::acknowledgement_control::AcknowledgementControllerConnectors;
-use crate::client::reply_key_storage::ReplyKeyStorage;
-use crate::client::{
-    inbound_messages::InputMessageReceiver, mix_traffic::BatchMixMessageSender,
-    topology_control::TopologyAccessor,
+use crate::{
+    client::{
+        inbound_messages::InputMessageReceiver, mix_traffic::BatchMixMessageSender,
+        real_messages_control::acknowledgement_control::AcknowledgementControllerConnectors,
+        topology_control::TopologyAccessor,
+    },
+    spawn_future,
 };
+use client_connections::{ConnectionCommandReceiver, LaneQueueLengths};
 use futures::channel::mpsc;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
 use nymsphinx::acknowledgements::AckKey;
 use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::params::PacketSize;
 use rand::{rngs::OsRng, CryptoRng, Rng};
 use std::sync::Arc;
 use std::time::Duration;
-use task::ShutdownListener;
-use tokio::task::JoinHandle;
+
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorage;
 
 mod acknowledgement_control;
 mod real_traffic_stream;
@@ -50,9 +55,18 @@ pub struct Config {
 
     /// Average delay an acknowledgement packet is going to get delayed at a single mixnode.
     average_ack_delay_duration: Duration,
+
+    /// Controls whether the main packet stream constantly produces packets according to the predefined
+    /// poisson distribution.
+    disable_main_poisson_packet_distribution: bool,
+
+    /// Predefined packet size used for the encapsulated messages.
+    packet_size: PacketSize,
 }
 
 impl Config {
+    // TODO: change the config into a builder
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         ack_key: Arc<AckKey>,
         ack_wait_multiplier: f64,
@@ -60,6 +74,7 @@ impl Config {
         average_ack_delay_duration: Duration,
         average_message_sending_delay: Duration,
         average_packet_delay_duration: Duration,
+        disable_main_poisson_packet_distribution: bool,
         self_recipient: Recipient,
     ) -> Self {
         Config {
@@ -70,33 +85,41 @@ impl Config {
             average_message_sending_delay,
             average_packet_delay_duration,
             average_ack_delay_duration,
+            disable_main_poisson_packet_distribution,
+            packet_size: Default::default(),
         }
     }
+
+    pub fn set_custom_packet_size(&mut self, packet_size: PacketSize) {
+        self.packet_size = packet_size;
+    }
 }
 
 pub struct RealMessagesController<R>
 where
     R: CryptoRng + Rng,
 {
-    out_queue_control: Option<OutQueueControl<R>>,
-    ack_control: Option<AcknowledgementController<R>>,
+    out_queue_control: OutQueueControl<R>,
+    ack_control: AcknowledgementController<R>,
 }
 
 // obviously when we finally make shared rng that is on 'higher' level, this should become
 // generic `R`
 impl RealMessagesController<OsRng> {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         config: Config,
         ack_receiver: AcknowledgementReceiver,
         input_receiver: InputMessageReceiver,
         mix_sender: BatchMixMessageSender,
         topology_access: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
-        shutdown: ShutdownListener,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
     ) -> Self {
         let rng = OsRng;
 
-        let (real_message_sender, real_message_receiver) = mpsc::unbounded();
+        let (real_message_sender, real_message_receiver) = tokio::sync::mpsc::channel(1);
         let (sent_notifier_tx, sent_notifier_rx) = mpsc::unbounded();
 
         let ack_controller_connectors = AcknowledgementControllerConnectors::new(
@@ -111,7 +134,8 @@ impl RealMessagesController<OsRng> {
             config.ack_wait_multiplier,
             config.average_ack_delay_duration,
             config.average_packet_delay_duration,
-        );
+        )
+        .with_custom_packet_size(config.packet_size);
 
         let ack_control = AcknowledgementController::new(
             ack_control_config,
@@ -119,16 +143,18 @@ impl RealMessagesController<OsRng> {
             topology_access.clone(),
             Arc::clone(&config.ack_key),
             config.self_recipient,
-            reply_key_storage,
             ack_controller_connectors,
-            shutdown.clone(),
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage,
         );
 
         let out_queue_config = real_traffic_stream::Config::new(
             config.average_ack_delay_duration,
             config.average_packet_delay_duration,
             config.average_message_sending_delay,
-        );
+            config.disable_main_poisson_packet_distribution,
+        )
+        .with_custom_cover_packet_size(config.packet_size);
 
         let out_queue_control = OutQueueControl::new(
             out_queue_config,
@@ -139,44 +165,37 @@ impl RealMessagesController<OsRng> {
             rng,
             config.self_recipient,
             topology_access,
-            shutdown,
+            lane_queue_lengths,
+            client_connection_rx,
         );
 
         RealMessagesController {
-            out_queue_control: Some(out_queue_control),
-            ack_control: Some(ack_control),
+            out_queue_control,
+            ack_control,
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        let mut out_queue_control = self.out_queue_control.take().unwrap();
-        let mut ack_control = self.ack_control.take().unwrap();
+    pub fn start_with_shutdown(self, shutdown: task::ShutdownListener) {
+        let mut out_queue_control = self.out_queue_control;
+        let ack_control = self.ack_control;
 
-        // the below are log messages are errors as at the current stage we do not expect any of
-        // the task to ever finish. This will of course change once we introduce
-        // graceful shutdowns.
-        let out_queue_control_fut = tokio::spawn(async move {
-            out_queue_control.run_out_queue_control().await;
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            out_queue_control.run_with_shutdown(shutdown_handle).await;
             debug!("The out queue controller has finished execution!");
-            out_queue_control
         });
-        let ack_control_fut = tokio::spawn(async move {
-            ack_control.run().await;
-            debug!("The acknowledgement controller has finished execution!");
-            ack_control
-        });
-
-        // technically we don't have to bring `RealMessagesController` back to a valid state
-        // but we can do it, so why not? Perhaps it might be useful if we wanted to allow
-        // for restarts of certain modules without killing the entire process.
-        self.out_queue_control = Some(out_queue_control_fut.await.unwrap());
-        self.ack_control = Some(ack_control_fut.await.unwrap());
+        ack_control.start_with_shutdown(shutdown);
     }
 
-    pub fn start(mut self) -> JoinHandle<Self> {
-        tokio::spawn(async move {
-            self.run().await;
-            self
-        })
+    #[cfg(target_arch = "wasm32")]
+    pub fn start(self) {
+        let mut out_queue_control = self.out_queue_control;
+        let ack_control = self.ack_control;
+
+        spawn_future(async move {
+            out_queue_control.run().await;
+            debug!("The out queue controller has finished execution!");
+        });
+        ack_control.start();
     }
 }

clients/client-core/src/client/real_messages_control/real_traffic_stream.rs

@@ -4,7 +4,9 @@
 use crate::client::mix_traffic::BatchMixMessageSender;
 use crate::client::real_messages_control::acknowledgement_control::SentPacketNotificationSender;
 use crate::client::topology_control::TopologyAccessor;
-use futures::channel::mpsc;
+use client_connections::{
+    ConnectionCommand, ConnectionCommandReceiver, ConnectionId, LaneQueueLengths, TransmissionLane,
+};
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
 use log::*;
@@ -13,15 +15,36 @@ use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::chunking::fragment::FragmentIdentifier;
 use nymsphinx::cover::generate_loop_cover_packet;
 use nymsphinx::forwarding::packet::MixPacket;
+use nymsphinx::params::PacketSize;
 use nymsphinx::utils::sample_poisson_duration;
 use rand::{CryptoRng, Rng};
-use std::collections::VecDeque;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
-use task::ShutdownListener;
+
+#[cfg(not(target_arch = "wasm32"))]
 use tokio::time;
 
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+use self::{
+    sending_delay_controller::SendingDelayController, transmission_buffer::TransmissionBuffer,
+};
+
+mod sending_delay_controller;
+mod transmission_buffer;
+
+#[cfg(not(target_arch = "wasm32"))]
+fn get_time_now() -> time::Instant {
+    time::Instant::now()
+}
+
+#[cfg(target_arch = "wasm32")]
+fn get_time_now() -> wasm_timer::Instant {
+    wasm_timer::Instant::now()
+}
+
 /// Configurable parameters of the `OutQueueControl`
 pub(crate) struct Config {
     /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
@@ -32,6 +55,13 @@ pub(crate) struct Config {
 
     /// Average delay between sending subsequent packets.
     average_message_sending_delay: Duration,
+
+    /// Controls whether the stream constantly produces packets according to the predefined
+    /// poisson distribution.
+    disable_poisson_packet_distribution: bool,
+
+    /// Predefined packet size used for the loop cover messages.
+    cover_packet_size: PacketSize,
 }
 
 impl Config {
@@ -39,13 +69,21 @@ impl Config {
         average_ack_delay: Duration,
         average_packet_delay: Duration,
         average_message_sending_delay: Duration,
+        disable_poisson_packet_distribution: bool,
     ) -> Self {
         Config {
             average_ack_delay,
             average_packet_delay,
             average_message_sending_delay,
+            disable_poisson_packet_distribution,
+            cover_packet_size: Default::default(),
         }
     }
+
+    pub fn with_custom_cover_packet_size(mut self, packet_size: PacketSize) -> Self {
+        self.cover_packet_size = packet_size;
+        self
+    }
 }
 
 pub(crate) struct OutQueueControl<R>
@@ -63,7 +101,15 @@ where
 
     /// Internal state, determined by `average_message_sending_delay`,
     /// used to keep track of when a next packet should be sent out.
-    next_delay: Pin<Box<time::Sleep>>,
+    #[cfg(not(target_arch = "wasm32"))]
+    next_delay: Option<Pin<Box<time::Sleep>>>,
+
+    #[cfg(target_arch = "wasm32")]
+    next_delay: Option<Pin<Box<wasm_timer::Delay>>>,
+
+    // To make sure we don't overload the mix_tx channel, we limit the rate we are pushing
+    // messages.
+    sending_delay_controller: SendingDelayController,
 
     /// Channel used for sending prepared sphinx packets to `MixTrafficController` that sends them
     /// out to the network without any further delays.
@@ -82,13 +128,19 @@ where
     /// Accessor to the common instance of network topology.
     topology_access: TopologyAccessor,
 
-    /// Buffer containing all real messages received. It is first exhausted before more are pulled.
-    received_buffer: VecDeque<RealMessage>,
+    /// Buffer containing all incoming real messages keyed by transmission lane, that we will send
+    /// out to the mixnet.
+    transmission_buffer: TransmissionBuffer,
 
-    /// Listens for shutdown signals
-    shutdown: ShutdownListener,
+    /// Incoming channel for being notified of closed connections, so that we can close lanes
+    /// corresponding to connections. To avoid sending traffic unnecessary
+    client_connection_rx: ConnectionCommandReceiver,
+
+    /// Report queue lengths so that upstream can backoff sending data, and keep connections open.
+    lane_queue_lengths: LaneQueueLengths,
 }
 
+#[derive(Debug)]
 pub(crate) struct RealMessage {
     mix_packet: MixPacket,
     fragment_id: FragmentIdentifier,
@@ -105,63 +157,15 @@ impl RealMessage {
 
 // messages are already prepared, etc. the real point of it is to forward it to mix_traffic
 // after sufficient delay
-pub(crate) type BatchRealMessageSender = mpsc::UnboundedSender<Vec<RealMessage>>;
-type BatchRealMessageReceiver = mpsc::UnboundedReceiver<Vec<RealMessage>>;
+pub(crate) type BatchRealMessageSender =
+    tokio::sync::mpsc::Sender<(Vec<RealMessage>, TransmissionLane)>;
+type BatchRealMessageReceiver = tokio::sync::mpsc::Receiver<(Vec<RealMessage>, TransmissionLane)>;
 
 pub(crate) enum StreamMessage {
     Cover,
     Real(Box<RealMessage>),
 }
 
-impl<R> Stream for OutQueueControl<R>
-where
-    R: CryptoRng + Rng + Unpin,
-{
-    type Item = StreamMessage;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        // it is not yet time to return a message
-        if self.next_delay.as_mut().poll(cx).is_pending() {
-            return Poll::Pending;
-        };
-
-        // we know it's time to send a message, so let's prepare delay for the next one
-        // Get the `now` by looking at the current `delay` deadline
-        let avg_delay = self.config.average_message_sending_delay;
-        let now = self.next_delay.deadline();
-        let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
-
-        // The next interval value is `next_poisson_delay` after the one that just
-        // yielded.
-        let next = now + next_poisson_delay;
-        self.next_delay.as_mut().reset(next);
-
-        // check if we have anything immediately available
-        if let Some(real_available) = self.received_buffer.pop_front() {
-            return Poll::Ready(Some(StreamMessage::Real(Box::new(real_available))));
-        }
-
-        // decide what kind of message to send
-        match Pin::new(&mut self.real_receiver).poll_next(cx) {
-            // in the case our real message channel stream was closed, we should also indicate we are closed
-            // (and whoever is using the stream should panic)
-            Poll::Ready(None) => Poll::Ready(None),
-
-            // if there are more messages available, return first one and store the rest
-            Poll::Ready(Some(real_messages)) => {
-                self.received_buffer = real_messages.into();
-                // we MUST HAVE received at least ONE message
-                Poll::Ready(Some(StreamMessage::Real(Box::new(
-                    self.received_buffer.pop_front().unwrap(),
-                ))))
-            }
-
-            // otherwise construct a dummy one
-            Poll::Pending => Poll::Ready(Some(StreamMessage::Cover)),
-        }
-    }
-}
-
 impl<R> OutQueueControl<R>
 where
     R: CryptoRng + Rng + Unpin,
@@ -178,20 +182,23 @@ where
         rng: R,
         our_full_destination: Recipient,
         topology_access: TopologyAccessor,
-        shutdown: ShutdownListener,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
     ) -> Self {
         OutQueueControl {
             config,
             ack_key,
             sent_notifier,
-            next_delay: Box::pin(time::sleep(Default::default())),
+            next_delay: None,
+            sending_delay_controller: Default::default(),
             mix_tx,
             real_receiver,
             our_full_destination,
             rng,
             topology_access,
-            received_buffer: VecDeque::with_capacity(0), // we won't be putting any data into this guy directly
-            shutdown,
+            transmission_buffer: Default::default(),
+            client_connection_rx,
+            lane_queue_lengths,
         }
     }
 
@@ -206,7 +213,7 @@ where
     async fn on_message(&mut self, next_message: StreamMessage) {
         trace!("created new message");
 
-        let next_message = match next_message {
+        let (next_message, fragment_id) = match next_message {
             StreamMessage::Cover => {
                 // TODO for way down the line: in very rare cases (during topology update) we might have
                 // to wait a really tiny bit before actually obtaining the permit hence messing with our
@@ -225,76 +232,340 @@ where
                 }
                 let topology_ref = topology_ref_option.unwrap();
 
-                generate_loop_cover_packet(
-                    &mut self.rng,
-                    topology_ref,
-                    &self.ack_key,
-                    &self.our_full_destination,
-                    self.config.average_ack_delay,
-                    self.config.average_packet_delay,
+                (
+                    generate_loop_cover_packet(
+                        &mut self.rng,
+                        topology_ref,
+                        &self.ack_key,
+                        &self.our_full_destination,
+                        self.config.average_ack_delay,
+                        self.config.average_packet_delay,
+                        self.config.cover_packet_size,
+                    )
+                    .expect(
+                        "Somehow failed to generate a loop cover message with a valid topology",
+                    ),
+                    None,
                 )
-                .expect("Somehow failed to generate a loop cover message with a valid topology")
             }
             StreamMessage::Real(real_message) => {
-                self.sent_notify(real_message.fragment_id);
-                real_message.mix_packet
+                (real_message.mix_packet, Some(real_message.fragment_id))
             }
         };
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        if let Err(err) = self.mix_tx.unbounded_send(vec![next_message]) {
-            if self.shutdown.is_shutdown_poll() {
-                log::info!("Failed to send (shutdown detected)");
-            } else {
-                // We don't try to limp along, panic to avoid continuing in a potentially
-                // inconsistent state
-                panic!("{err}");
-            }
+        if let Err(err) = self.mix_tx.send(vec![next_message]).await {
+            log::error!("Failed to send: {}", err);
         }
 
+        // notify ack controller about sending our message only after we actually managed to push it
+        // through the channel
+        if let Some(fragment_id) = fragment_id {
+            self.sent_notify(fragment_id);
+        }
+
+        // In addition to closing connections on receiving messages throught client_connection_rx,
+        // also close connections when sufficiently stale.
+        self.transmission_buffer.prune_stale_connections();
+
         // JS: Not entirely sure why or how it fixes stuff, but without the yield call,
         // the UnboundedReceiver [of mix_rx] will not get a chance to read anything
         // JS2: Basically it was the case that with high enough rate, the stream had already a next value
         // ready and hence was immediately re-scheduled causing other tasks to be starved;
         // yield makes it go back the scheduling queue regardless of its value availability
+
+        // TODO: temporary and BAD workaround for wasm (we should find a way to yield here in wasm)
+        #[cfg(not(target_arch = "wasm32"))]
         tokio::task::yield_now().await;
     }
 
-    // Send messages at certain rate and if no real traffic is available, send cover message.
-    async fn run_normal_out_queue(&mut self) {
-        // we should set initial delay only when we actually start the stream
-        self.next_delay = Box::pin(time::sleep(sample_poisson_duration(
-            &mut self.rng,
-            self.config.average_message_sending_delay,
-        )));
+    fn on_close_connection(&mut self, connection_id: ConnectionId) {
+        log::debug!("Removing lane for connection: {connection_id}");
+        self.transmission_buffer
+            .remove(&TransmissionLane::ConnectionId(connection_id));
+    }
 
-        let mut shutdown = self.shutdown.clone();
-        while !shutdown.is_shutdown() {
-            tokio::select! {
-                biased;
-                _ = shutdown.recv() => {
-                    log::trace!("OutQueueControl: Received shutdown");
+    fn current_average_message_sending_delay(&self) -> Duration {
+        self.config.average_message_sending_delay
+            * self.sending_delay_controller.current_multiplier()
+    }
+
+    fn adjust_current_average_message_sending_delay(&mut self) {
+        let used_slots = self.mix_tx.max_capacity() - self.mix_tx.capacity();
+        log::trace!(
+            "used_slots: {used_slots}, current_multiplier: {}",
+            self.sending_delay_controller.current_multiplier()
+        );
+
+        // Even just a single used slot is enough to signal backpressure
+        if used_slots > 0 {
+            log::trace!("Backpressure detected");
+            self.sending_delay_controller.record_backpressure_detected();
+        }
+
+        // If the buffer is running out, slow down the sending rate
+        if self.mix_tx.capacity() == 0
+            && self.sending_delay_controller.not_increased_delay_recently()
+        {
+            self.sending_delay_controller.increase_delay_multiplier();
+        }
+
+        // Very carefully step up the sending rate in case it seems like we can solidly handle the
+        // current rate.
+        if self.sending_delay_controller.is_sending_reliable() {
+            self.sending_delay_controller.decrease_delay_multiplier();
+        }
+    }
+
+    fn pop_next_message(&mut self) -> Option<RealMessage> {
+        // Pop the next message from the transmission buffer
+        let (lane, real_next) = self.transmission_buffer.pop_next_message_at_random()?;
+
+        // Update the published queue length
+        let lane_length = self.transmission_buffer.lane_length(&lane);
+        self.lane_queue_lengths.set(&lane, lane_length);
+
+        Some(real_next)
+    }
+
+    fn poll_poisson(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
+        // The average delay could change depending on if backpressure in the downstream channel
+        // (mix_tx) was detected.
+        self.adjust_current_average_message_sending_delay();
+        let avg_delay = self.current_average_message_sending_delay();
+
+        // Start by checking if we have any incoming messages about closed connections
+        // NOTE: this feels a bit iffy, the `OutQueueControl` is getting ripe for a rewrite to
+        // something simpler.
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.client_connection_rx).poll_next(cx) {
+            match id {
+                ConnectionCommand::Close(id) => self.on_close_connection(id),
+                ConnectionCommand::ActiveConnections(_) => panic!(),
+            }
+        }
+
+        if let Some(ref mut next_delay) = &mut self.next_delay {
+            // it is not yet time to return a message
+            if next_delay.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
+            };
+
+            // we know it's time to send a message, so let's prepare delay for the next one
+            // Get the `now` by looking at the current `delay` deadline
+            let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
+
+            // The next interval value is `next_poisson_delay` after the one that just
+            // yielded.
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                let now = next_delay.deadline();
+                let next = now + next_poisson_delay;
+                next_delay.as_mut().reset(next);
+            }
+
+            #[cfg(target_arch = "wasm32")]
+            {
+                next_delay.as_mut().reset(next_poisson_delay);
+            }
+
+            // On every iteration we get new messages from upstream. Given that these come bunched
+            // in `Vec`, this ensures that on average we will fetch messages faster than we can
+            // send, which is a condition for being able to multiplex sphinx packets from multiple
+            // data streams.
+            match Pin::new(&mut self.real_receiver).poll_recv(cx) {
+                // in the case our real message channel stream was closed, we should also indicate we are closed
+                // (and whoever is using the stream should panic)
+                Poll::Ready(None) => Poll::Ready(None),
+
+                Poll::Ready(Some((real_messages, conn_id))) => {
+                    log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                    self.transmission_buffer.store(&conn_id, real_messages);
+                    let real_next = self.pop_next_message().expect("Just stored one");
+
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
                 }
-                next_message = self.next() => match next_message {
-                    Some(next_message) => {
+
+                Poll::Pending => {
+                    if let Some(real_next) = self.pop_next_message() {
+                        Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                    } else {
+                        // otherwise construct a dummy one
+                        Poll::Ready(Some(StreamMessage::Cover))
+                    }
+                }
+            }
+        } else {
+            // we never set an initial delay - let's do it now
+            cx.waker().wake_by_ref();
+
+            let sampled =
+                sample_poisson_duration(&mut self.rng, self.config.average_message_sending_delay);
+
+            #[cfg(not(target_arch = "wasm32"))]
+            let next_delay = Box::pin(time::sleep(sampled));
+
+            #[cfg(target_arch = "wasm32")]
+            let next_delay = Box::pin(wasm_timer::Delay::new(sampled));
+
+            self.next_delay = Some(next_delay);
+
+            Poll::Pending
+        }
+    }
+
+    fn poll_immediate(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
+        // Start by checking if we have any incoming messages about closed connections
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.client_connection_rx).poll_next(cx) {
+            match id {
+                ConnectionCommand::Close(id) => self.on_close_connection(id),
+                ConnectionCommand::ActiveConnections(_) => panic!(),
+            }
+        }
+
+        match Pin::new(&mut self.real_receiver).poll_recv(cx) {
+            // in the case our real message channel stream was closed, we should also indicate we are closed
+            // (and whoever is using the stream should panic)
+            Poll::Ready(None) => Poll::Ready(None),
+
+            Poll::Ready(Some((real_messages, conn_id))) => {
+                log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                // First store what we got for the given connection id
+                self.transmission_buffer.store(&conn_id, real_messages);
+                let real_next = self.pop_next_message().expect("we just added one");
+
+                Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+            }
+
+            Poll::Pending => {
+                if let Some(real_next) = self.pop_next_message() {
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                } else {
+                    Poll::Pending
+                }
+            }
+        }
+    }
+
+    fn poll_next_message(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<StreamMessage>> {
+        if self.config.disable_poisson_packet_distribution {
+            self.poll_immediate(cx)
+        } else {
+            self.poll_poisson(cx)
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn log_status(&self) {
+        let packets = self.transmission_buffer.total_size();
+        let backlog = self.transmission_buffer.total_size_in_bytes() as f64 / 1024.0;
+        let lanes = self.transmission_buffer.num_lanes();
+        let mult = self.sending_delay_controller.current_multiplier();
+        let delay = self.current_average_message_sending_delay().as_millis();
+        let status_str = if self.config.disable_poisson_packet_distribution {
+            format!(
+                "Status: {lanes} lanes, backlog: {:.2} kiB ({packets}), no delay",
+                backlog
+            )
+        } else {
+            format!(
+                "Status: {lanes} lanes, backlog: {:.2} kiB ({packets}), avg delay: {}ms ({mult})",
+                backlog, delay
+            )
+        };
+        if packets > 1000 {
+            log::warn!("{status_str}");
+        } else if packets > 0 {
+            log::info!("{status_str}");
+        } else {
+            log::debug!("{status_str}");
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn log_status_infrequent(&self) {
+        if self.sending_delay_controller.current_multiplier() > 1 {
+            log::warn!(
+                "Unable to send packets at the default rate - rate reduced by setting the delay multiplier set to: {}",
+                self.sending_delay_controller.current_multiplier()
+            );
+        }
+    }
+
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started OutQueueControl with graceful shutdown support");
+
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let mut status_timer = tokio::time::interval(Duration::from_secs(5));
+            let mut infrequent_status_timer = tokio::time::interval(Duration::from_secs(60));
+
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv() => {
+                        log::trace!("OutQueueControl: Received shutdown");
+                    }
+                    _ = status_timer.tick() => {
+                        self.log_status();
+                    }
+                    _ = infrequent_status_timer.tick() => {
+                        self.log_status_infrequent();
+                    }
+                    next_message = self.next() => if let Some(next_message) = next_message {
                         self.on_message(next_message).await;
-                    },
-                    None => {
+                    } else {
+                        log::trace!("OutQueueControl: Stopping since channel closed");
+                        break;
+                    }
+                }
+            }
+            tokio::time::timeout(Duration::from_secs(5), shutdown.recv())
+                .await
+                .expect("Task stopped without shutdown called");
+        }
+
+        #[cfg(target_arch = "wasm32")]
+        {
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv() => {
+                        log::trace!("OutQueueControl: Received shutdown");
+                    }
+                    next_message = self.next() => if let Some(next_message) = next_message {
+                        self.on_message(next_message).await;
+                    } else {
                         log::trace!("OutQueueControl: Stopping since channel closed");
                         break;
                     }
                 }
             }
         }
-        assert!(shutdown.is_shutdown_poll());
         log::debug!("OutQueueControl: Exiting");
     }
 
-    pub(crate) async fn run_out_queue_control(&mut self) {
-        debug!("Starting out queue controller...");
-        self.run_normal_out_queue().await
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    pub(super) async fn run(&mut self) {
+        debug!("Started OutQueueControl without graceful shutdown support");
+
+        while let Some(next_message) = self.next().await {
+            self.on_message(next_message).await;
+        }
+    }
+}
+
+impl<R> Stream for OutQueueControl<R>
+where
+    R: CryptoRng + Rng + Unpin,
+{
+    type Item = StreamMessage;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.poll_next_message(cx)
     }
 }

clients/client-core/src/client/real_messages_control/real_traffic_stream/sending_delay_controller.rs

@@ -0,0 +1,124 @@
+// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use super::get_time_now;
+use std::time::Duration;
+
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::time;
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+// The minimum time between increasing the average delay between packets. If we hit the ceiling in
+// the available buffer space we want to take somewhat swift action, but we still need to give a
+// short time to give the channel a chance reduce pressure.
+const INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 1;
+// The minimum time between decreasing the average delay between packets. We don't want to change
+// to quickly to keep things somewhat stable. Also there are buffers downstreams meaning we need to
+// wait a little to see the effect before we decrease further.
+const DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 30;
+// If we enough time passes without any sign of backpressure in the channel, we can consider
+// lowering the average delay. The goal is to keep somewhat stable, rather than maxing out
+// bandwidth at all times.
+const ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS: u64 = 30;
+// The maximum multiplier we apply to the base average Poisson delay.
+const MAX_DELAY_MULTIPLIER: u32 = 6;
+// The minium multiplier we apply to the base average Poisson delay.
+const MIN_DELAY_MULTIPLIER: u32 = 1;
+
+pub(crate) struct SendingDelayController {
+    /// Multiply the average sending delay.
+    /// This is normally set to unity, but if we detect backpressure we increase this
+    /// multiplier. We use discrete steps.
+    current_multiplier: u32,
+
+    /// Maximum delay multiplier
+    upper_bound: u32,
+
+    /// Minimum delay multiplier
+    lower_bound: u32,
+
+    /// To make sure we don't change the multiplier to fast, we limit a change to some duration
+    #[cfg(not(target_arch = "wasm32"))]
+    time_when_changed: time::Instant,
+
+    #[cfg(target_arch = "wasm32")]
+    time_when_changed: wasm_timer::Instant,
+
+    /// If we have a long enough time without any backpressure detected we try reducing the sending
+    /// delay multiplier
+    #[cfg(not(target_arch = "wasm32"))]
+    time_when_backpressure_detected: time::Instant,
+
+    #[cfg(target_arch = "wasm32")]
+    time_when_backpressure_detected: wasm_timer::Instant,
+}
+
+impl Default for SendingDelayController {
+    fn default() -> Self {
+        SendingDelayController::new(MIN_DELAY_MULTIPLIER, MAX_DELAY_MULTIPLIER)
+    }
+}
+
+impl SendingDelayController {
+    pub(crate) fn new(lower_bound: u32, upper_bound: u32) -> Self {
+        assert!(lower_bound <= upper_bound);
+        let now = get_time_now();
+        SendingDelayController {
+            current_multiplier: MIN_DELAY_MULTIPLIER,
+            upper_bound,
+            lower_bound,
+            time_when_changed: now,
+            time_when_backpressure_detected: now,
+        }
+    }
+
+    pub(crate) fn current_multiplier(&self) -> u32 {
+        self.current_multiplier
+    }
+
+    pub(crate) fn increase_delay_multiplier(&mut self) {
+        if self.current_multiplier < self.upper_bound {
+            self.current_multiplier =
+                (self.current_multiplier + 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::warn!(
+                "Increasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        } else {
+            log::warn!("Trying to increase delay multipler higher than allowed");
+        }
+    }
+
+    pub(crate) fn decrease_delay_multiplier(&mut self) {
+        if self.current_multiplier > self.lower_bound {
+            self.current_multiplier =
+                (self.current_multiplier - 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::debug!(
+                "Decreasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        }
+    }
+
+    pub(crate) fn record_backpressure_detected(&mut self) {
+        self.time_when_backpressure_detected = get_time_now();
+    }
+
+    pub(crate) fn not_increased_delay_recently(&self) -> bool {
+        get_time_now()
+            > self.time_when_changed + Duration::from_secs(INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS)
+    }
+
+    pub(crate) fn is_sending_reliable(&self) -> bool {
+        let now = get_time_now();
+        let delay_change_interval = Duration::from_secs(DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS);
+        let acceptable_time_without_backpressure =
+            Duration::from_secs(ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS);
+
+        now > self.time_when_backpressure_detected + acceptable_time_without_backpressure
+            && now > self.time_when_changed + delay_change_interval
+    }
+}

clients/client-core/src/client/real_messages_control/real_traffic_stream/transmission_buffer.rs

@@ -0,0 +1,217 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use client_connections::TransmissionLane;
+use rand::{seq::SliceRandom, Rng};
+use std::{
+    collections::{HashMap, HashSet, VecDeque},
+    time::Duration,
+};
+
+#[cfg(not(target_arch = "wasm32"))]
+use tokio::time;
+
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+use super::{get_time_now, RealMessage};
+
+// The number of lanes included in the oldest set. Used when we need to prioritize traffic.
+const OLDEST_LANE_SET_SIZE: usize = 5;
+// As a way of prune connections we also check for timeouts.
+const MSG_CONSIDERED_STALE_AFTER_SECS: u64 = 10 * 60;
+
+#[derive(Default)]
+pub(crate) struct TransmissionBuffer {
+    buffer: HashMap<TransmissionLane, LaneBufferEntry>,
+}
+
+impl TransmissionBuffer {
+    #[allow(unused)]
+    pub(crate) fn is_empty(&self) -> bool {
+        self.buffer.is_empty()
+    }
+
+    pub(crate) fn remove(&mut self, lane: &TransmissionLane) -> Option<LaneBufferEntry> {
+        self.buffer.remove(lane)
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn num_lanes(&self) -> usize {
+        self.buffer.keys().count()
+    }
+
+    pub(crate) fn lane_length(&self, lane: &TransmissionLane) -> Option<usize> {
+        self.buffer.get(lane).map(LaneBufferEntry::len)
+    }
+
+    #[allow(unused)]
+    pub(crate) fn connections(&self) -> HashSet<u64> {
+        self.buffer
+            .keys()
+            .filter_map(|lane| match lane {
+                TransmissionLane::ConnectionId(id) => Some(id),
+                _ => None,
+            })
+            .copied()
+            .collect()
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn total_size(&self) -> usize {
+        self.buffer.values().map(LaneBufferEntry::len).sum()
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn total_size_in_bytes(&self) -> usize {
+        self.buffer
+            .values()
+            .map(|lane_buffer_entry| {
+                lane_buffer_entry
+                    .real_messages
+                    .iter()
+                    .map(|real_message| real_message.mix_packet.sphinx_packet().len())
+                    .sum::<usize>()
+            })
+            .sum()
+    }
+
+    fn get_oldest_set(&self) -> Vec<TransmissionLane> {
+        let mut buffer: Vec<_> = self
+            .buffer
+            .iter()
+            .map(|(k, v)| (k, v.messages_transmitted))
+            .collect();
+        buffer.sort_by_key(|v| v.1);
+        buffer
+            .iter()
+            .rev()
+            .map(|(k, _)| *k)
+            .take(OLDEST_LANE_SET_SIZE)
+            .copied()
+            .collect()
+    }
+
+    pub(crate) fn store(&mut self, lane: &TransmissionLane, real_messages: Vec<RealMessage>) {
+        if let Some(lane_buffer_entry) = self.buffer.get_mut(lane) {
+            lane_buffer_entry.append(real_messages);
+        } else {
+            self.buffer
+                .insert(*lane, LaneBufferEntry::new(real_messages));
+        }
+    }
+
+    fn pick_random_lane(&self) -> Option<&TransmissionLane> {
+        let lanes: Vec<&TransmissionLane> = self.buffer.keys().collect();
+        lanes.choose(&mut rand::thread_rng()).copied()
+    }
+
+    fn pick_random_small_lane(&self) -> Option<&TransmissionLane> {
+        let lanes: Vec<&TransmissionLane> = self
+            .buffer
+            .iter()
+            .filter(|(_, v)| v.is_small())
+            .map(|(k, _)| k)
+            .collect();
+        lanes.choose(&mut rand::thread_rng()).copied()
+    }
+
+    // 2/3 chance to pick from the old lanes
+    fn pick_random_old_lane(&self) -> Option<TransmissionLane> {
+        let rand = &mut rand::thread_rng();
+        if rand.gen_ratio(2, 3) {
+            let lanes = self.get_oldest_set();
+            lanes.choose(rand).copied()
+        } else {
+            self.pick_random_lane().copied()
+        }
+    }
+
+    fn pop_front_from_lane(&mut self, lane: &TransmissionLane) -> Option<RealMessage> {
+        let real_msgs_queued = self.buffer.get_mut(lane)?;
+        let real_next = real_msgs_queued.pop_front()?;
+        real_msgs_queued.messages_transmitted += 1;
+        if real_msgs_queued.is_empty() {
+            self.buffer.remove(lane);
+        }
+        Some(real_next)
+    }
+
+    pub(crate) fn pop_next_message_at_random(&mut self) -> Option<(TransmissionLane, RealMessage)> {
+        if self.buffer.is_empty() {
+            return None;
+        }
+
+        // Very basic heuristic where we prioritize according to small lanes first, the older lanes
+        // to try to finish lanes when possible, then the rest.
+        let lane = if let Some(small_lane) = self.pick_random_small_lane() {
+            *small_lane
+        } else if let Some(old_lane) = self.pick_random_old_lane() {
+            old_lane
+        } else {
+            *self.pick_random_lane()?
+        };
+
+        let msg = self.pop_front_from_lane(&lane)?;
+        log::trace!("picking to send from lane: {:?}", lane);
+        Some((lane, msg))
+    }
+
+    pub(crate) fn prune_stale_connections(&mut self) {
+        let stale_entries: Vec<_> = self
+            .buffer
+            .iter()
+            .filter_map(|(lane, entry)| if entry.is_stale() { Some(lane) } else { None })
+            .copied()
+            .collect();
+
+        for lane in stale_entries {
+            self.remove(&lane);
+        }
+    }
+}
+
+pub(crate) struct LaneBufferEntry {
+    pub real_messages: VecDeque<RealMessage>,
+    pub messages_transmitted: usize,
+    #[cfg(not(target_arch = "wasm32"))]
+    pub time_for_last_activity: time::Instant,
+    #[cfg(target_arch = "wasm32")]
+    pub time_for_last_activity: wasm_timer::Instant,
+}
+
+impl LaneBufferEntry {
+    fn new(real_messages: Vec<RealMessage>) -> Self {
+        LaneBufferEntry {
+            real_messages: real_messages.into(),
+            messages_transmitted: 0,
+            time_for_last_activity: get_time_now(),
+        }
+    }
+
+    fn append(&mut self, real_messages: Vec<RealMessage>) {
+        self.real_messages.append(&mut real_messages.into());
+        self.time_for_last_activity = get_time_now();
+    }
+
+    fn pop_front(&mut self) -> Option<RealMessage> {
+        self.real_messages.pop_front()
+    }
+
+    fn is_small(&self) -> bool {
+        self.real_messages.len() < 100
+    }
+
+    fn is_stale(&self) -> bool {
+        get_time_now() - self.time_for_last_activity
+            > Duration::from_secs(MSG_CONSIDERED_STALE_AFTER_SECS)
+    }
+
+    fn len(&self) -> usize {
+        self.real_messages.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.real_messages.is_empty()
+    }
+}

clients/client-core/src/client/received_buffer.rs

@@ -1,24 +1,25 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::client::reply_key_storage::ReplyKeyStorage;
-use crate::client::SHUTDOWN_HAS_BEEN_SIGNALLED;
+use crate::spawn_future;
 use crypto::asymmetric::encryption;
-use crypto::symmetric::stream_cipher;
-use crypto::Digest;
 use futures::channel::mpsc;
 use futures::lock::Mutex;
 use futures::StreamExt;
 use gateway_client::MixnetMessageReceiver;
 use log::*;
-use nymsphinx::anonymous_replies::{encryption_key::EncryptionKeyDigest, SurbEncryptionKey};
-use nymsphinx::params::{ReplySurbEncryptionAlgorithm, ReplySurbKeyDigestAlgorithm};
 use nymsphinx::receiver::{MessageReceiver, MessageRecoveryError, ReconstructedMessage};
 use std::collections::HashSet;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
-use task::ShutdownListener;
-use tokio::task::JoinHandle;
+
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorage;
+#[cfg(feature = "reply-surb")]
+use crypto::{symmetric::stream_cipher, Digest};
+#[cfg(feature = "reply-surb")]
+use nymsphinx::anonymous_replies::{encryption_key::EncryptionKeyDigest, SurbEncryptionKey};
+#[cfg(feature = "reply-surb")]
+use nymsphinx::params::{ReplySurbEncryptionAlgorithm, ReplySurbKeyDigestAlgorithm};
 
 // Buffer Requests to say "hey, send any reconstructed messages to this channel"
 // or to say "hey, I'm going offline, don't send anything more to me. Just buffer them instead"
@@ -116,13 +117,14 @@ struct ReceivedMessagesBuffer {
 
     /// Storage containing keys to all [`ReplySURB`]s ever sent out that we did not receive back.
     // There's no need to put it behind a Mutex since it's already properly concurrent
+    #[cfg(feature = "reply-surb")]
     reply_key_storage: ReplyKeyStorage,
 }
 
 impl ReceivedMessagesBuffer {
     fn new(
         local_encryption_keypair: Arc<encryption::KeyPair>,
-        reply_key_storage: ReplyKeyStorage,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
     ) -> Self {
         ReceivedMessagesBuffer {
             inner: Arc::new(Mutex::new(ReceivedMessagesBufferInner {
@@ -132,6 +134,7 @@ impl ReceivedMessagesBuffer {
                 message_sender: None,
                 recently_reconstructed: HashSet::new(),
             })),
+            #[cfg(feature = "reply-surb")]
             reply_key_storage,
         }
     }
@@ -180,6 +183,7 @@ impl ReceivedMessagesBuffer {
         self.inner.lock().await.messages.extend(msgs)
     }
 
+    #[cfg(feature = "reply-surb")]
     fn process_received_reply(
         reply_ciphertext: &[u8],
         reply_key: SurbEncryptionKey,
@@ -204,7 +208,7 @@ impl ReceivedMessagesBuffer {
     }
 
     async fn handle_new_received(&mut self, msgs: Vec<Vec<u8>>) {
-        debug!(
+        trace!(
             "Processing {:?} new message that might get added to the buffer!",
             msgs.len()
         );
@@ -212,38 +216,50 @@ impl ReceivedMessagesBuffer {
         let mut completed_messages = Vec::new();
         let mut inner_guard = self.inner.lock().await;
 
-        let reply_surb_digest_size = ReplySurbKeyDigestAlgorithm::output_size();
-
         // first check if this is a reply or a chunked message
         // TODO: verify with @AP if this way of doing it is safe or whether it could
         // cause some attacks due to, I don't know, stupid edge case collisions?
         // Update: this DOES introduce a possible leakage: https://github.com/nymtech/nym/issues/296
         for msg in msgs {
-            let possible_key_digest =
-                EncryptionKeyDigest::clone_from_slice(&msg[..reply_surb_digest_size]);
+            // TODO:
+            // 1. make it nicer
+            // 2. make it not feature-locked
 
-            // check first `HasherOutputSize` bytes if they correspond to known encryption key
-            // if yes - this is a reply message
-
-            // TODO: this might be a bottleneck - since the keys are stored on disk we, presumably,
-            // are doing a disk operation every single received fragment
-            if let Some(reply_encryption_key) = self
-                .reply_key_storage
-                .get_and_remove_encryption_key(possible_key_digest)
-                .expect("storage operation failed!")
+            #[cfg(feature = "reply-surb")]
             {
-                if let Some(completed_message) = Self::process_received_reply(
-                    &msg[reply_surb_digest_size..],
-                    reply_encryption_key,
-                ) {
-                    completed_messages.push(completed_message)
-                }
-            } else {
-                // otherwise - it's a 'normal' message
-                if let Some(completed_message) = inner_guard.process_received_fragment(msg) {
-                    completed_messages.push(completed_message)
+                let reply_surb_digest_size = ReplySurbKeyDigestAlgorithm::output_size();
+
+                let possible_key_digest =
+                    EncryptionKeyDigest::clone_from_slice(&msg[..reply_surb_digest_size]);
+
+                // check first `HasherOutputSize` bytes if they correspond to known encryption key
+                // if yes - this is a reply message
+
+                // TODO: this might be a bottleneck - since the keys are stored on disk we, presumably,
+                // are doing a disk operation every single received fragment
+                if let Some(reply_encryption_key) = self
+                    .reply_key_storage
+                    .get_and_remove_encryption_key(possible_key_digest)
+                    .expect("storage operation failed!")
+                {
+                    if let Some(completed_message) = Self::process_received_reply(
+                        &msg[reply_surb_digest_size..],
+                        reply_encryption_key,
+                    ) {
+                        completed_messages.push(completed_message)
+                    }
+                } else {
+                    // otherwise - it's a 'normal' message
+                    if let Some(completed_message) = inner_guard.process_received_fragment(msg) {
+                        completed_messages.push(completed_message)
+                    }
                 }
             }
+
+            #[cfg(not(feature = "reply-surb"))]
+            if let Some(completed_message) = inner_guard.process_received_fragment(msg) {
+                completed_messages.push(completed_message)
+            }
         }
 
         if !completed_messages.is_empty() {
@@ -293,72 +309,97 @@ impl RequestReceiver {
         }
     }
 
-    fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            loop {
-                tokio::select! {
-                    request = self.query_receiver.next() => {
-                        match request {
-                            Some(ReceivedBufferMessage::ReceiverAnnounce(sender)) => {
-                                self.received_buffer.connect_sender(sender).await;
-                            }
-                            Some(ReceivedBufferMessage::ReceiverDisconnect) => {
-                                self.received_buffer.disconnect_sender().await
-                            }
-                            None => {
-                                log::trace!("RequestReceiver: Stopping since channel closed");
-                                break;
-                            },
-                        }
-                    },
-                };
+    async fn handle_message(&mut self, message: ReceivedBufferMessage) {
+        match message {
+            ReceivedBufferMessage::ReceiverAnnounce(sender) => {
+                self.received_buffer.connect_sender(sender).await;
             }
+            ReceivedBufferMessage::ReceiverDisconnect => {
+                self.received_buffer.disconnect_sender().await
+            }
+        }
+    }
 
-            assert!(SHUTDOWN_HAS_BEEN_SIGNALLED.load(Ordering::Relaxed));
-            log::debug!("RequestReceiver: Exiting");
-        })
+    async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started RequestReceiver with graceful shutdown support");
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                biased;
+                _ = shutdown.recv() => {
+                    log::trace!("RequestReceiver: Received shutdown");
+                }
+                request = self.query_receiver.next() => {
+                    match request {
+                        Some(message) => self.handle_message(message).await,
+                        None => {
+                            log::trace!("RequestReceiver: Stopping since channel closed");
+                            break;
+                        },
+                    }
+                },
+            }
+        }
+        shutdown.recv_timeout().await;
+        log::debug!("RequestReceiver: Exiting");
+    }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    async fn run(&mut self) {
+        debug!("Started RequestReceiver without graceful shutdown support");
+
+        while let Some(message) = self.query_receiver.next().await {
+            self.handle_message(message).await
+        }
     }
 }
 
 struct FragmentedMessageReceiver {
     received_buffer: ReceivedMessagesBuffer,
     mixnet_packet_receiver: MixnetMessageReceiver,
-    shutdown: ShutdownListener,
 }
 
 impl FragmentedMessageReceiver {
     fn new(
         received_buffer: ReceivedMessagesBuffer,
         mixnet_packet_receiver: MixnetMessageReceiver,
-        shutdown: ShutdownListener,
     ) -> Self {
         FragmentedMessageReceiver {
             received_buffer,
             mixnet_packet_receiver,
-            shutdown,
         }
     }
-    fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            while !self.shutdown.is_shutdown() {
-                tokio::select! {
-                    new_messages = self.mixnet_packet_receiver.next() => match new_messages {
-                        Some(new_messages) => {
-                            self.received_buffer.handle_new_received(new_messages).await;
-                        }
-                        None => {
-                            log::trace!("FragmentedMessageReceiver: Stopping since channel closed");
-                            break;
-                        }
-                    },
-                    _ = self.shutdown.recv() => {
-                        log::trace!("FragmentedMessageReceiver: Received shutdown");
+
+    async fn run_with_shutdown(&mut self, mut shutdown: task::ShutdownListener) {
+        debug!("Started FragmentedMessageReceiver with graceful shutdown support");
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                new_messages = self.mixnet_packet_receiver.next() => match new_messages {
+                    Some(new_messages) => {
+                        self.received_buffer.handle_new_received(new_messages).await;
                     }
+                    None => {
+                        log::trace!("FragmentedMessageReceiver: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv() => {
+                    log::trace!("FragmentedMessageReceiver: Received shutdown");
                 }
             }
-            assert!(self.shutdown.is_shutdown_poll());
-            log::debug!("FragmentedMessageReceiver: Exiting");
-        })
+        }
+        shutdown.recv_timeout().await;
+        log::debug!("FragmentedMessageReceiver: Exiting");
+    }
+
+    // todo: think whether this is still required
+    #[allow(dead_code)]
+    async fn run(&mut self) {
+        debug!("Started FragmentedMessageReceiver without graceful shutdown support");
+
+        while let Some(new_messages) = self.mixnet_packet_receiver.next().await {
+            self.received_buffer.handle_new_received(new_messages).await;
+        }
     }
 }
 
@@ -372,25 +413,47 @@ impl ReceivedMessagesBufferController {
         local_encryption_keypair: Arc<encryption::KeyPair>,
         query_receiver: ReceivedBufferRequestReceiver,
         mixnet_packet_receiver: MixnetMessageReceiver,
-        reply_key_storage: ReplyKeyStorage,
-        shutdown: ShutdownListener,
+        #[cfg(feature = "reply-surb")] reply_key_storage: ReplyKeyStorage,
     ) -> Self {
-        let received_buffer =
-            ReceivedMessagesBuffer::new(local_encryption_keypair, reply_key_storage);
+        let received_buffer = ReceivedMessagesBuffer::new(
+            local_encryption_keypair,
+            #[cfg(feature = "reply-surb")]
+            reply_key_storage,
+        );
 
         ReceivedMessagesBufferController {
             fragmented_message_receiver: FragmentedMessageReceiver::new(
                 received_buffer.clone(),
                 mixnet_packet_receiver,
-                shutdown,
             ),
             request_receiver: RequestReceiver::new(received_buffer, query_receiver),
         }
     }
 
+    pub fn start_with_shutdown(self, shutdown: task::ShutdownListener) {
+        let mut fragmented_message_receiver = self.fragmented_message_receiver;
+        let mut request_receiver = self.request_receiver;
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            fragmented_message_receiver
+                .run_with_shutdown(shutdown_handle)
+                .await;
+        });
+        spawn_future(async move {
+            request_receiver.run_with_shutdown(shutdown).await;
+        });
+    }
+
+    #[cfg(target_arch = "wasm32")]
     pub fn start(self) {
-        // TODO: should we do anything with JoinHandle(s) returned by start methods?
-        self.fragmented_message_receiver.start();
-        self.request_receiver.start();
+        let mut fragmented_message_receiver = self.fragmented_message_receiver;
+        let mut request_receiver = self.request_receiver;
+        spawn_future(async move {
+            fragmented_message_receiver.run().await;
+        });
+        spawn_future(async move {
+            request_receiver.run().await;
+        });
     }
 }

clients/client-core/src/client/reply_key_storage.rs

@@ -8,10 +8,13 @@ use nymsphinx::anonymous_replies::{
 };
 use std::path::Path;
 
-#[derive(Debug)]
+#[derive(Debug, thiserror::Error)]
 pub enum ReplyKeyStorageError {
+    #[error("DB Read Error: {0}")]
     DbReadError(sled::Error),
+    #[error("DB Write Error: {0}")]
     DbWriteError(sled::Error),
+    #[error("DB Open Error: {0}")]
     DbOpenError(sled::Error),
 }
 

clients/client-core/src/client/topology_control.rs

@@ -1,6 +1,8 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use crate::spawn_future;
+use futures::StreamExt;
 use log::*;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::params::DEFAULT_NUM_MIX_HOPS;
@@ -10,9 +12,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 use std::time;
 use std::time::Duration;
-use task::ShutdownListener;
 use tokio::sync::{RwLock, RwLockReadGuard};
-use tokio::task::JoinHandle;
 use topology::{nym_topology_from_detailed, NymTopology};
 use url::Url;
 
@@ -58,24 +58,15 @@ impl<'a> TopologyReadPermit<'a> {
     ) -> Option<&'a NymTopology> {
         // Note: implicit deref with Deref for TopologyReadPermit is happening here
         let topology_ref_option = self.permit.as_ref();
-        match topology_ref_option {
-            None => None,
-            Some(topology_ref) => {
-                // see if it's possible to route the packet to both gateways
-                if !topology_ref.can_construct_path_through(DEFAULT_NUM_MIX_HOPS)
-                    || !topology_ref.gateway_exists(ack_recipient.gateway())
-                    || if let Some(packet_recipient) = packet_recipient {
-                        !topology_ref.gateway_exists(packet_recipient.gateway())
-                    } else {
-                        false
-                    }
-                {
-                    None
+        topology_ref_option.as_ref().filter(|topology_ref| {
+            !(!topology_ref.can_construct_path_through(DEFAULT_NUM_MIX_HOPS)
+                || !topology_ref.gateway_exists(ack_recipient.gateway())
+                || if let Some(packet_recipient) = packet_recipient {
+                    !topology_ref.gateway_exists(packet_recipient.gateway())
                 } else {
-                    Some(topology_ref)
-                }
-            }
-        }
+                    false
+                })
+        })
     }
 }
 
@@ -148,7 +139,7 @@ impl TopologyRefresherConfig {
 }
 
 pub struct TopologyRefresher {
-    validator_client: validator_client::ApiClient,
+    validator_client: validator_client::client::ApiClient,
     client_version: String,
 
     validator_api_urls: Vec<Url>,
@@ -164,7 +155,9 @@ impl TopologyRefresher {
         cfg.validator_api_urls.shuffle(&mut thread_rng());
 
         TopologyRefresher {
-            validator_client: validator_client::ApiClient::new(cfg.validator_api_urls[0].clone()),
+            validator_client: validator_client::client::ApiClient::new(
+                cfg.validator_api_urls[0].clone(),
+            ),
             client_version: cfg.client_version,
             validator_api_urls: cfg.validator_api_urls,
             topology_accessor,
@@ -194,13 +187,10 @@ impl TopologyRefresher {
     /// # Arguments
     ///
     /// * `topology`: active topology constructed from validator api data
-    /// * `mixnodes_count`: total number of active mixnodes
-    fn check_layer_distribution(
-        &self,
-        active_topology: &NymTopology,
-        mixnodes_count: usize,
-    ) -> bool {
+    fn check_layer_distribution(&self, active_topology: &NymTopology) -> bool {
         let mixes = active_topology.mixes();
+        let mixnodes_count = active_topology.num_mixnodes();
+
         if active_topology.gateways().is_empty() {
             return false;
         }
@@ -265,11 +255,10 @@ impl TopologyRefresher {
             Ok(gateways) => gateways,
         };
 
-        let mixnodes_count = mixnodes.len();
         let topology = nym_topology_from_detailed(mixnodes, gateways)
             .filter_system_version(&self.client_version);
 
-        if !self.check_layer_distribution(&topology, mixnodes_count) {
+        if !self.check_layer_distribution(&topology) {
             warn!("The current filtered active topology has extremely skewed layer distribution. It cannot be used.");
             None
         } else {
@@ -304,11 +293,22 @@ impl TopologyRefresher {
         self.topology_accessor.is_routable().await
     }
 
-    pub fn start(mut self, mut shutdown: ShutdownListener) -> JoinHandle<()> {
-        tokio::spawn(async move {
+    pub fn start_with_shutdown(mut self, mut shutdown: task::ShutdownListener) {
+        spawn_future(async move {
+            debug!("Started TopologyRefresher with graceful shutdown support");
+
+            #[cfg(not(target_arch = "wasm32"))]
+            let mut interval = tokio_stream::wrappers::IntervalStream::new(tokio::time::interval(
+                self.refresh_rate,
+            ));
+
+            #[cfg(target_arch = "wasm32")]
+            let mut interval =
+                gloo_timers::future::IntervalStream::new(self.refresh_rate.as_millis() as u32);
+
             while !shutdown.is_shutdown() {
                 tokio::select! {
-                    _ = tokio::time::sleep(self.refresh_rate) => {
+                    _ = interval.next() => {
                         self.refresh().await;
                     },
                     _ = shutdown.recv() => {
@@ -316,8 +316,25 @@ impl TopologyRefresher {
                     },
                 }
             }
-            assert!(shutdown.is_shutdown_poll());
+            shutdown.recv_timeout().await;
             log::debug!("TopologyRefresher: Exiting");
         })
     }
+
+    pub fn start(mut self) {
+        spawn_future(async move {
+            #[cfg(not(target_arch = "wasm32"))]
+            let mut interval = tokio_stream::wrappers::IntervalStream::new(tokio::time::interval(
+                self.refresh_rate,
+            ));
+
+            #[cfg(target_arch = "wasm32")]
+            let mut interval =
+                gloo_timers::future::IntervalStream::new(self.refresh_rate.as_millis() as u32);
+
+            while (interval.next().await).is_some() {
+                self.refresh().await;
+            }
+        })
+    }
 }

clients/client-core/src/config/mod.rs

@@ -1,13 +1,17 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use config::NymConfig;
+use config::{NymConfig, DB_FILE_NAME};
+use nymsphinx::params::PacketSize;
 use serde::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::path::PathBuf;
 use std::time::Duration;
 use url::Url;
 
+#[cfg(target_arch = "wasm32")]
+use wasm_bindgen::prelude::*;
+
 pub mod persistence;
 
 pub const MISSING_VALUE: &str = "MISSING VALUE";
@@ -30,7 +34,11 @@ pub fn missing_string_value() -> String {
     MISSING_VALUE.to_string()
 }
 
-#[derive(Debug, Deserialize, PartialEq, Serialize)]
+pub trait ClientCoreConfigTrait {
+    fn get_gateway_endpoint(&self) -> &GatewayEndpointConfig;
+}
+
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
 #[serde(deny_unknown_fields)]
 pub struct Config<T> {
     client: Client<T>,
@@ -38,17 +46,29 @@ pub struct Config<T> {
     #[serde(default)]
     logging: Logging,
     #[serde(default)]
-    debug: Debug,
+    debug: DebugConfig,
 }
 
-impl<T: NymConfig> Config<T> {
-    pub fn new<S: Into<String>>(id: S) -> Self {
+impl<T> ClientCoreConfigTrait for Config<T> {
+    fn get_gateway_endpoint(&self) -> &GatewayEndpointConfig {
+        &self.client.gateway_endpoint
+    }
+}
+
+impl<T> Config<T> {
+    pub fn new<S: Into<String>>(id: S) -> Self
+    where
+        T: NymConfig,
+    {
         let mut cfg = Config::default();
         cfg.with_id(id);
         cfg
     }
 
-    pub fn with_id<S: Into<String>>(&mut self, id: S) {
+    pub fn with_id<S: Into<String>>(&mut self, id: S)
+    where
+        T: NymConfig,
+    {
         let id = id.into();
 
         // identity key setting
@@ -113,7 +133,7 @@ impl<T: NymConfig> Config<T> {
         self.client.disabled_credentials_mode = disabled_credentials_mode;
     }
 
-    pub fn with_gateway_endpoint(&mut self, gateway_endpoint: GatewayEndpoint) {
+    pub fn with_gateway_endpoint(&mut self, gateway_endpoint: GatewayEndpointConfig) {
         self.client.gateway_endpoint = gateway_endpoint;
     }
 
@@ -121,14 +141,25 @@ impl<T: NymConfig> Config<T> {
         self.client.gateway_endpoint.gateway_id = id.into();
     }
 
+    pub fn set_custom_validators(&mut self, validator_urls: Vec<Url>) {
+        self.client.validator_urls = validator_urls;
+    }
+
     pub fn set_custom_validator_apis(&mut self, validator_api_urls: Vec<Url>) {
         self.client.validator_api_urls = validator_api_urls;
     }
 
     pub fn set_high_default_traffic_volume(&mut self) {
         self.debug.average_packet_delay = Duration::from_millis(10);
-        self.debug.loop_cover_traffic_average_delay = Duration::from_millis(2_000_000); // basically don't really send cover messages
-        self.debug.message_sending_average_delay = Duration::from_millis(4); // 250 "real" messages / s
+        // basically don't really send cover messages
+        self.debug.loop_cover_traffic_average_delay = Duration::from_millis(2_000_000);
+        // 250 "real" messages / s
+        self.debug.message_sending_average_delay = Duration::from_millis(4);
+    }
+
+    pub fn set_no_cover_traffic(&mut self) {
+        self.debug.disable_loop_cover_traffic_stream = true;
+        self.debug.disable_main_poisson_packet_distribution = true;
     }
 
     pub fn set_custom_version(&mut self, version: &str) {
@@ -175,6 +206,10 @@ impl<T: NymConfig> Config<T> {
         self.client.ack_key_file.clone()
     }
 
+    pub fn get_validator_endpoints(&self) -> Vec<Url> {
+        self.client.validator_urls.clone()
+    }
+
     pub fn get_validator_api_endpoints(&self) -> Vec<Url> {
         self.client.validator_api_urls.clone()
     }
@@ -191,7 +226,7 @@ impl<T: NymConfig> Config<T> {
         self.client.gateway_endpoint.gateway_listener.clone()
     }
 
-    pub fn get_gateway_endpoint(&self) -> &GatewayEndpoint {
+    pub fn get_gateway_endpoint_config(&self) -> &GatewayEndpointConfig {
         &self.client.gateway_endpoint
     }
 
@@ -200,6 +235,10 @@ impl<T: NymConfig> Config<T> {
     }
 
     // Debug getters
+    pub fn get_debug_config(&self) -> &DebugConfig {
+        &self.debug
+    }
+
     pub fn get_average_packet_delay(&self) -> Duration {
         self.debug.average_packet_delay
     }
@@ -236,6 +275,18 @@ impl<T: NymConfig> Config<T> {
         self.debug.topology_resolution_timeout
     }
 
+    pub fn get_disabled_loop_cover_traffic_stream(&self) -> bool {
+        self.debug.disable_loop_cover_traffic_stream
+    }
+
+    pub fn get_disabled_main_poisson_packet_distribution(&self) -> bool {
+        self.debug.disable_main_poisson_packet_distribution
+    }
+
+    pub fn get_use_extended_packet_size(&self) -> Option<ExtendedPacketSize> {
+        self.debug.use_extended_packet_size
+    }
+
     pub fn get_version(&self) -> &str {
         &self.client.version
     }
@@ -252,7 +303,8 @@ impl<T: NymConfig> Default for Config<T> {
 }
 
 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
-pub struct GatewayEndpoint {
+#[cfg_attr(target_arch = "wasm32", wasm_bindgen(getter_with_clone))]
+pub struct GatewayEndpointConfig {
     /// gateway_id specifies ID of the gateway to which the client should send messages.
     /// If initially omitted, a random gateway will be chosen from the available topology.
     pub gateway_id: String,
@@ -264,10 +316,10 @@ pub struct GatewayEndpoint {
     pub gateway_listener: String,
 }
 
-impl From<topology::gateway::Node> for GatewayEndpoint {
-    fn from(node: topology::gateway::Node) -> GatewayEndpoint {
+impl From<topology::gateway::Node> for GatewayEndpointConfig {
+    fn from(node: topology::gateway::Node) -> GatewayEndpointConfig {
         let gateway_listener = node.clients_address();
-        GatewayEndpoint {
+        GatewayEndpointConfig {
             gateway_id: node.identity_key.to_base58_string(),
             gateway_owner: node.owner,
             gateway_listener,
@@ -275,7 +327,7 @@ impl From<topology::gateway::Node> for GatewayEndpoint {
     }
 }
 
-#[derive(Debug, Deserialize, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)]
 pub struct Client<T> {
     /// Version of the client for which this configuration was created.
     #[serde(default = "missing_string_value")]
@@ -289,6 +341,10 @@ pub struct Client<T> {
     #[serde(default)]
     disabled_credentials_mode: bool,
 
+    /// Addresses to nymd validators via which the client can communicate with the chain.
+    #[serde(default)]
+    validator_urls: Vec<Url>,
+
     /// Addresses to APIs running on validator from which the client gets the view of the network.
     validator_api_urls: Vec<Url>,
 
@@ -317,7 +373,7 @@ pub struct Client<T> {
     reply_encryption_key_store_path: PathBuf,
 
     /// Information regarding how the client should send data to gateway.
-    gateway_endpoint: GatewayEndpoint,
+    gateway_endpoint: GatewayEndpointConfig,
 
     /// Path to the database containing bandwidth credentials of this client.
     database_path: PathBuf,
@@ -337,6 +393,7 @@ impl<T: NymConfig> Default for Client<T> {
             version: env!("CARGO_PKG_VERSION").to_string(),
             id: "".to_string(),
             disabled_credentials_mode: true,
+            validator_urls: vec![],
             validator_api_urls: vec![],
             private_identity_key_file: Default::default(),
             public_identity_key_file: Default::default(),
@@ -382,74 +439,93 @@ impl<T: NymConfig> Client<T> {
         T::default_data_directory(Some(id)).join("reply_key_store")
     }
     fn default_database_path(id: &str) -> PathBuf {
-        T::default_data_directory(Some(id)).join("db.sqlite")
+        T::default_data_directory(Some(id)).join(DB_FILE_NAME)
     }
 }
 
-#[derive(Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
 pub struct Logging {}
 
-#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
 #[serde(default, deny_unknown_fields)]
-pub struct Debug {
+pub struct DebugConfig {
     /// The parameter of Poisson distribution determining how long, on average,
     /// sent packet is going to be delayed at any given mix node.
     /// So for a packet going through three mix nodes, on average, it will take three times this value
     /// until the packet reaches its destination.
     #[serde(with = "humantime_serde")]
-    average_packet_delay: Duration,
+    pub average_packet_delay: Duration,
 
     /// The parameter of Poisson distribution determining how long, on average,
     /// sent acknowledgement is going to be delayed at any given mix node.
     /// So for an ack going through three mix nodes, on average, it will take three times this value
     /// until the packet reaches its destination.
     #[serde(with = "humantime_serde")]
-    average_ack_delay: Duration,
+    pub average_ack_delay: Duration,
 
     /// Value multiplied with the expected round trip time of an acknowledgement packet before
     /// it is assumed it was lost and retransmission of the data packet happens.
     /// In an ideal network with 0 latency, this value would have been 1.
-    ack_wait_multiplier: f64,
+    pub ack_wait_multiplier: f64,
 
     /// Value added to the expected round trip time of an acknowledgement packet before
     /// it is assumed it was lost and retransmission of the data packet happens.
     /// In an ideal network with 0 latency, this value would have been 0.
     #[serde(with = "humantime_serde")]
-    ack_wait_addition: Duration,
+    pub ack_wait_addition: Duration,
 
     /// The parameter of Poisson distribution determining how long, on average,
     /// it is going to take for another loop cover traffic message to be sent.
     #[serde(with = "humantime_serde")]
-    loop_cover_traffic_average_delay: Duration,
+    pub loop_cover_traffic_average_delay: Duration,
 
     /// The parameter of Poisson distribution determining how long, on average,
     /// it is going to take another 'real traffic stream' message to be sent.
     /// If no real packets are available and cover traffic is enabled,
     /// a loop cover message is sent instead in order to preserve the rate.
     #[serde(with = "humantime_serde")]
-    message_sending_average_delay: Duration,
+    pub message_sending_average_delay: Duration,
 
     /// How long we're willing to wait for a response to a message sent to the gateway,
     /// before giving up on it.
     #[serde(with = "humantime_serde")]
-    gateway_response_timeout: Duration,
+    pub gateway_response_timeout: Duration,
 
     /// The uniform delay every which clients are querying the directory server
     /// to try to obtain a compatible network topology to send sphinx packets through.
     #[serde(with = "humantime_serde")]
-    topology_refresh_rate: Duration,
+    pub topology_refresh_rate: Duration,
 
     /// During topology refresh, test packets are sent through every single possible network
     /// path. This timeout determines waiting period until it is decided that the packet
     /// did not reach its destination.
     #[serde(with = "humantime_serde")]
-    topology_resolution_timeout: Duration,
+    pub topology_resolution_timeout: Duration,
+
+    /// Controls whether the dedicated loop cover traffic stream should be enabled.
+    /// (and sending packets, on average, every [Self::loop_cover_traffic_average_delay])
+    pub disable_loop_cover_traffic_stream: bool,
+
+    /// Controls whether the main packet stream constantly produces packets according to the predefined
+    /// poisson distribution.
+    pub disable_main_poisson_packet_distribution: bool,
+
+    /// Controls whether the sent sphinx packet use a NON-DEFAULT bigger size.
+    pub use_extended_packet_size: Option<ExtendedPacketSize>,
 }
 
-impl Default for Debug {
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ExtendedPacketSize {
+    Extended8,
+    Extended16,
+    Extended32,
+}
+
+impl Default for DebugConfig {
     fn default() -> Self {
-        Debug {
+        DebugConfig {
             average_packet_delay: DEFAULT_AVERAGE_PACKET_DELAY,
             average_ack_delay: DEFAULT_AVERAGE_PACKET_DELAY,
             ack_wait_multiplier: DEFAULT_ACK_WAIT_MULTIPLIER,
@@ -459,6 +535,19 @@ impl Default for Debug {
             gateway_response_timeout: DEFAULT_GATEWAY_RESPONSE_TIMEOUT,
             topology_refresh_rate: DEFAULT_TOPOLOGY_REFRESH_RATE,
             topology_resolution_timeout: DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT,
+            disable_loop_cover_traffic_stream: false,
+            disable_main_poisson_packet_distribution: false,
+            use_extended_packet_size: None,
+        }
+    }
+}
+
+impl From<ExtendedPacketSize> for PacketSize {
+    fn from(size: ExtendedPacketSize) -> PacketSize {
+        match size {
+            ExtendedPacketSize::Extended8 => PacketSize::ExtendedPacket8,
+            ExtendedPacketSize::Extended16 => PacketSize::ExtendedPacket16,
+            ExtendedPacketSize::Extended32 => PacketSize::ExtendedPacket32,
         }
     }
 }

clients/client-core/src/error.rs

@@ -1,6 +1,8 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+#[cfg(feature = "reply-surb")]
+use crate::client::reply_key_storage::ReplyKeyStorageError;
 use crypto::asymmetric::identity::Ed25519RecoveryError;
 use gateway_client::error::GatewayClientError;
 use validator_client::ValidatorClientError;
@@ -16,12 +18,33 @@ pub enum ClientCoreError {
     #[error("Validator client error: {0}")]
     ValidatorClientError(#[from] ValidatorClientError),
 
+    #[cfg(feature = "reply-surb")]
+    #[error("Reply key storage error: {0}")]
+    ReplyKeyStorageError(#[from] ReplyKeyStorageError),
+
     #[error("No gateway with id: {0}")]
     NoGatewayWithId(String),
     #[error("No gateways on network")]
     NoGatewaysOnNetwork,
+    #[error("Failed to setup gateway")]
+    FailedToSetupGateway,
     #[error("List of validator apis is empty")]
     ListOfValidatorApisIsEmpty,
     #[error("Could not load existing gateway configuration: {0}")]
     CouldNotLoadExistingGatewayConfiguration(std::io::Error),
+    #[error("The current network topology seem to be insufficient to route any packets through")]
+    InsufficientNetworkTopology,
+
+    #[error("The gateway id is invalid - {0}")]
+    UnableToCreatePublicKeyFromGatewayId(Ed25519RecoveryError),
+
+    #[error("The identity of the gateway is unknwown - did you run init?")]
+    GatewayIdUnknown,
+    #[error("The owner of the gateway is unknown - did you run init?")]
+    GatewayOwnerUnknown,
+    #[error("The address of the gateway is unknown - did you run init?")]
+    GatwayAddressUnknown,
+
+    #[error("Unexpected exit")]
+    UnexpectedExit,
 }

clients/client-core/src/init/helpers.rs

@@ -1,22 +1,14 @@
-// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
-// SPDX-License-Identifier: Apache-2.0
-
-//! Collection of initialization steps used by client implementations
-
 use std::{sync::Arc, time::Duration};
 
+use rand::{rngs::OsRng, seq::SliceRandom, thread_rng};
+use tap::TapFallible;
+use url::Url;
+
 use config::NymConfig;
-use crypto::asymmetric::{encryption, identity};
+use crypto::asymmetric::identity;
 use gateway_client::GatewayClient;
 use gateway_requests::registration::handshake::SharedKeys;
-use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::addressing::nodes::NodeIdentity;
-use rand::rngs::OsRng;
-use rand::seq::SliceRandom;
-use rand::thread_rng;
-use tap::TapFallible;
 use topology::{filter::VersionFilterable, gateway};
-use url::Url;
 
 use crate::{
     client::key_manager::KeyManager,
@@ -24,14 +16,14 @@ use crate::{
     error::ClientCoreError,
 };
 
-pub async fn query_gateway_details(
+pub(super) async fn query_gateway_details(
     validator_servers: Vec<Url>,
-    chosen_gateway_id: Option<&str>,
+    chosen_gateway_id: Option<String>,
 ) -> Result<gateway::Node, ClientCoreError> {
     let validator_api = validator_servers
         .choose(&mut thread_rng())
         .ok_or(ClientCoreError::ListOfValidatorApisIsEmpty)?;
-    let validator_client = validator_client::ApiClient::new(validator_api.clone());
+    let validator_client = validator_client::client::ApiClient::new(validator_api.clone());
 
     log::trace!("Fetching list of gateways from: {}", validator_api);
     let gateways = validator_client.get_cached_gateways().await?;
@@ -59,7 +51,30 @@ pub async fn query_gateway_details(
     }
 }
 
-pub async fn register_with_gateway_and_store_keys<T>(
+async fn register_with_gateway(
+    gateway: &gateway::Node,
+    our_identity: Arc<identity::KeyPair>,
+) -> Result<Arc<SharedKeys>, ClientCoreError> {
+    let timeout = Duration::from_millis(1500);
+    let mut gateway_client = GatewayClient::new_init(
+        gateway.clients_address(),
+        gateway.identity_key,
+        gateway.owner.clone(),
+        our_identity.clone(),
+        timeout,
+    );
+    gateway_client
+        .establish_connection()
+        .await
+        .tap_err(|_| log::warn!("Failed to establish connection with gateway!"))?;
+    let shared_keys = gateway_client
+        .perform_initial_authentication()
+        .await
+        .tap_err(|_| log::warn!("Failed to register with the gateway!"))?;
+    Ok(shared_keys)
+}
+
+pub(super) async fn register_with_gateway_and_store_keys<T>(
     gateway_details: gateway::Node,
     config: &Config<T>,
 ) -> Result<(), ClientCoreError>
@@ -78,71 +93,3 @@ where
         .store_keys(&pathfinder)
         .tap_err(|err| log::error!("Failed to generate keys: {err}"))?)
 }
-
-async fn register_with_gateway(
-    gateway: &gateway::Node,
-    our_identity: Arc<identity::KeyPair>,
-) -> Result<Arc<SharedKeys>, ClientCoreError> {
-    let timeout = Duration::from_millis(1500);
-    let mut gateway_client = GatewayClient::new_init(
-        gateway.clients_address(),
-        gateway.identity_key,
-        gateway.owner.clone(),
-        our_identity.clone(),
-        timeout,
-        None,
-    );
-    gateway_client
-        .establish_connection()
-        .await
-        .tap_err(|_| log::warn!("Failed to establish connection with gateway!"))?;
-    let shared_keys = gateway_client
-        .perform_initial_authentication()
-        .await
-        .tap_err(|_| log::warn!("Failed to register with the gateway!"))?;
-    Ok(shared_keys)
-}
-
-pub fn show_address<T>(config: &Config<T>) -> Result<(), ClientCoreError>
-where
-    T: config::NymConfig,
-{
-    fn load_identity_keys(
-        pathfinder: &ClientKeyPathfinder,
-    ) -> Result<identity::KeyPair, ClientCoreError> {
-        let identity_keypair: identity::KeyPair =
-            pemstore::load_keypair(&pemstore::KeyPairPath::new(
-                pathfinder.private_identity_key().to_owned(),
-                pathfinder.public_identity_key().to_owned(),
-            ))
-            .tap_err(|_| log::error!("Failed to read stored identity key files"))?;
-        Ok(identity_keypair)
-    }
-
-    fn load_sphinx_keys(
-        pathfinder: &ClientKeyPathfinder,
-    ) -> Result<encryption::KeyPair, ClientCoreError> {
-        let sphinx_keypair: encryption::KeyPair =
-            pemstore::load_keypair(&pemstore::KeyPairPath::new(
-                pathfinder.private_encryption_key().to_owned(),
-                pathfinder.public_encryption_key().to_owned(),
-            ))
-            .tap_err(|_| log::error!("Failed to read stored sphinx key files"))?;
-        Ok(sphinx_keypair)
-    }
-
-    let pathfinder = ClientKeyPathfinder::new_from_config(config);
-    let identity_keypair = load_identity_keys(&pathfinder)?;
-    let sphinx_keypair = load_sphinx_keys(&pathfinder)?;
-
-    let client_recipient = Recipient::new(
-        *identity_keypair.public_key(),
-        *sphinx_keypair.public_key(),
-        // TODO: below only works under assumption that gateway address == gateway id
-        // (which currently is true)
-        NodeIdentity::from_base58_string(config.get_gateway_id())?,
-    );
-
-    println!("\nThe address of this client is: {}", client_recipient);
-    Ok(())
-}

clients/client-core/src/init/mod.rs

@@ -0,0 +1,191 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+//! Collection of initialization steps used by client implementations
+
+use std::fmt::Display;
+
+use nymsphinx::addressing::{clients::Recipient, nodes::NodeIdentity};
+use serde::Serialize;
+use tap::TapFallible;
+
+use config::NymConfig;
+use crypto::asymmetric::{encryption, identity};
+
+use crate::{
+    config::{
+        persistence::key_pathfinder::ClientKeyPathfinder, ClientCoreConfigTrait, Config,
+        GatewayEndpointConfig,
+    },
+    error::ClientCoreError,
+    init::helpers::{query_gateway_details, register_with_gateway_and_store_keys},
+};
+
+mod helpers;
+
+#[derive(Debug, Serialize)]
+pub struct InitResults {
+    version: String,
+    id: String,
+    identity_key: String,
+    encryption_key: String,
+    gateway_id: String,
+    gateway_listener: String,
+}
+
+impl InitResults {
+    pub fn new<T>(config: &Config<T>, address: &Recipient) -> Self
+    where
+        T: NymConfig,
+    {
+        Self {
+            version: config.get_version().to_string(),
+            id: config.get_id(),
+            identity_key: address.identity().to_base58_string(),
+            encryption_key: address.encryption_key().to_base58_string(),
+            gateway_id: config.get_gateway_id(),
+            gateway_listener: config.get_gateway_listener(),
+        }
+    }
+}
+
+impl Display for InitResults {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "Version: {}", self.version)?;
+        writeln!(f, "ID: {}", self.id)?;
+        writeln!(f, "Identity key: {}", self.identity_key)?;
+        writeln!(f, "Encryption: {}", self.encryption_key)?;
+        writeln!(f, "Gateway ID: {}", self.gateway_id)?;
+        write!(f, "Gateway: {}", self.gateway_listener)
+    }
+}
+
+/// Convenience function for setting up the gateway for a client. Depending on the arguments given
+/// it will do the sensible thing.
+pub async fn setup_gateway<C: NymConfig + ClientCoreConfigTrait, T: NymConfig>(
+    register_gateway: bool,
+    user_chosen_gateway_id: Option<String>,
+    config: &Config<T>,
+) -> Result<GatewayEndpointConfig, ClientCoreError> {
+    let id = config.get_id();
+    if register_gateway {
+        register_with_gateway(user_chosen_gateway_id, config).await
+    } else if let Some(user_chosen_gateway_id) = user_chosen_gateway_id {
+        config_gateway_with_existing_keys(user_chosen_gateway_id, config).await
+    } else {
+        reuse_existing_gateway_config::<C>(&id)
+    }
+}
+
+/// Get the gateway details by querying the validator-api. Either pick one at random or use
+/// the chosen one if it's among the available ones.
+/// Saves keys to disk, specified by the paths in `config`.
+pub async fn register_with_gateway<T: NymConfig>(
+    user_chosen_gateway_id: Option<String>,
+    config: &Config<T>,
+) -> Result<GatewayEndpointConfig, ClientCoreError> {
+    println!("Configuring gateway");
+    let gateway =
+        query_gateway_details(config.get_validator_api_endpoints(), user_chosen_gateway_id).await?;
+    log::debug!("Querying gateway gives: {}", gateway);
+
+    // Registering with gateway by setting up and writing shared keys to disk
+    log::trace!("Registering gateway");
+    register_with_gateway_and_store_keys(gateway.clone(), config).await?;
+    println!("Saved all generated keys");
+
+    Ok(gateway.into())
+}
+
+/// Set the gateway using the usual procedue of querying the validator-api, but don't register or
+/// create any keys.
+/// This assumes that the user knows what they are doing, and that the existing keys are valid for
+/// the gateway being used
+pub async fn config_gateway_with_existing_keys<T: NymConfig>(
+    user_chosen_gateway_id: String,
+    config: &Config<T>,
+) -> Result<GatewayEndpointConfig, ClientCoreError> {
+    println!("Using gateway provided by user, keeping existing keys");
+    let gateway = query_gateway_details(
+        config.get_validator_api_endpoints(),
+        Some(user_chosen_gateway_id),
+    )
+    .await?;
+    log::debug!("Querying gateway gives: {}", gateway);
+    Ok(gateway.into())
+}
+
+/// Read and reuse the existing gateway configuration from a file that was generate earlier.
+pub fn reuse_existing_gateway_config<T: NymConfig + ClientCoreConfigTrait>(
+    id: &str,
+) -> Result<GatewayEndpointConfig, ClientCoreError> {
+    println!("Not registering gateway, will reuse existing config and keys");
+    T::load_from_file(Some(id))
+        .map(|existing_config| existing_config.get_gateway_endpoint().clone())
+        .map_err(|err| {
+            log::error!(
+                "Unable to configure gateway: {err}. \n
+                Seems like the client was already initialized but it was not possible to read \
+                the existing configuration file. \n
+                CAUTION: Consider backing up your gateway keys and try force gateway registration, or \
+                removing the existing configuration and starting over."
+            );
+            ClientCoreError::CouldNotLoadExistingGatewayConfiguration(err)
+        })
+}
+
+/// Get the client address by loading the keys from stored files.
+pub fn get_client_address_from_stored_keys<T>(
+    config: &Config<T>,
+) -> Result<Recipient, ClientCoreError>
+where
+    T: config::NymConfig,
+{
+    fn load_identity_keys(
+        pathfinder: &ClientKeyPathfinder,
+    ) -> Result<identity::KeyPair, ClientCoreError> {
+        let identity_keypair: identity::KeyPair =
+            pemstore::load_keypair(&pemstore::KeyPairPath::new(
+                pathfinder.private_identity_key().to_owned(),
+                pathfinder.public_identity_key().to_owned(),
+            ))
+            .tap_err(|_| log::error!("Failed to read stored identity key files"))?;
+        Ok(identity_keypair)
+    }
+
+    fn load_sphinx_keys(
+        pathfinder: &ClientKeyPathfinder,
+    ) -> Result<encryption::KeyPair, ClientCoreError> {
+        let sphinx_keypair: encryption::KeyPair =
+            pemstore::load_keypair(&pemstore::KeyPairPath::new(
+                pathfinder.private_encryption_key().to_owned(),
+                pathfinder.public_encryption_key().to_owned(),
+            ))
+            .tap_err(|_| log::error!("Failed to read stored sphinx key files"))?;
+        Ok(sphinx_keypair)
+    }
+
+    let pathfinder = ClientKeyPathfinder::new_from_config(config);
+    let identity_keypair = load_identity_keys(&pathfinder)?;
+    let sphinx_keypair = load_sphinx_keys(&pathfinder)?;
+
+    let client_recipient = Recipient::new(
+        *identity_keypair.public_key(),
+        *sphinx_keypair.public_key(),
+        // TODO: below only works under assumption that gateway address == gateway id
+        // (which currently is true)
+        NodeIdentity::from_base58_string(config.get_gateway_id())?,
+    );
+
+    Ok(client_recipient)
+}
+
+pub fn output_to_json<T: Serialize>(init_results: &T, output_file: &str) {
+    match std::fs::File::create(output_file) {
+        Ok(file) => match serde_json::to_writer_pretty(file, init_results) {
+            Ok(_) => println!("Saved: {}", output_file),
+            Err(err) => eprintln!("Could not save {}: {}", output_file, err),
+        },
+        Err(err) => eprintln!("Could not save {}: {}", output_file, err),
+    }
+}

clients/client-core/src/lib.rs

@@ -1,4 +1,23 @@
+use std::future::Future;
+
 pub mod client;
 pub mod config;
 pub mod error;
 pub mod init;
+
+#[cfg(target_arch = "wasm32")]
+pub(crate) fn spawn_future<F>(future: F)
+where
+    F: Future<Output = ()> + 'static,
+{
+    wasm_bindgen_futures::spawn_local(future);
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn spawn_future<F>(future: F)
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    tokio::spawn(future);
+}

clients/credential/Cargo.toml

@@ -6,16 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1.52"
 bip39 = "1.0.1"
 cfg-if = "0.1"
 clap = { version = "3.2", features = ["cargo", "derive"] }
-pickledb = "0.4.1"
 rand = "0.7.3"
 serde = { version = "1.0", features = ["derive"] }
 thiserror = "1.0"
 url = "2.2"
-tokio = { version = "1.19.1", features = ["rt-multi-thread", "net", "signal", "macros"] } # async runtime
+tokio = { version = "1.21.2", features = ["rt-multi-thread", "net", "signal", "macros"] } # async runtime
 
 coconut-interface = { path = "../../common/coconut-interface" }
 config = { path = "../../common/config" }

clients/credential/src/commands.rs

@@ -1,34 +1,29 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use async_trait::async_trait;
 use clap::{Args, Subcommand};
 use completions::ArgShell;
-use pickledb::PickleDb;
 use rand::rngs::OsRng;
 use std::str::FromStr;
 
-use coconut_interface::{Attribute, Base58, BlindSignRequest, Bytable, Parameters};
+use coconut_interface::{Base58, Parameters};
 use credential_storage::storage::Storage;
 use credential_storage::PersistentStorage;
 use credentials::coconut::bandwidth::{BandwidthVoucher, TOTAL_ATTRIBUTES};
 use credentials::coconut::utils::obtain_aggregate_signature;
 use crypto::asymmetric::{encryption, identity};
-use network_defaults::VOUCHER_INFO;
+use network_defaults::{NymNetworkDetails, VOUCHER_INFO};
 use validator_client::nymd::tx::Hash;
+use validator_client::{CoconutApiClient, Config};
 
 use crate::client::Client;
 use crate::error::{CredentialClientError, Result};
-use crate::state::{KeyPair, RequestData, State};
+use crate::state::{KeyPair, State};
 
 #[derive(Subcommand)]
-pub(crate) enum Commands {
-    /// Deposit funds for buying coconut credential
-    Deposit(Deposit),
-    /// Lists the tx hashes of previous deposits
-    ListDeposits(ListDeposits),
-    /// Get a credential for a given deposit
-    GetCredential(GetCredential),
+pub(crate) enum Command {
+    /// Run the binary
+    Run(Run),
 
     /// Generate shell completions
     Completions(ArgShell),
@@ -37,163 +32,82 @@ pub(crate) enum Commands {
     GenerateFigSpec,
 }
 
-#[async_trait]
-pub(crate) trait Execute {
-    async fn execute(&self, db: &mut PickleDb, shared_storage: PersistentStorage) -> Result<()>;
-}
+#[derive(Args)]
+pub(crate) struct Run {
+    /// Home directory of the client that is supposed to use the credential.
+    #[clap(long)]
+    pub(crate) client_home_directory: std::path::PathBuf,
 
-#[derive(Args, Clone)]
-pub(crate) struct Deposit {
     /// The nymd URL that should be used
     #[clap(long)]
-    nymd_url: String,
-    /// A mnemonic for the account that does the deposit
+    pub(crate) nymd_url: String,
+
+    /// A mnemonic for the account that buys the credential
     #[clap(long)]
-    mnemonic: String,
-    /// The amount that needs to be deposited
+    pub(crate) mnemonic: String,
+
+    /// The amount of utokens the credential will hold
     #[clap(long)]
-    amount: u64,
+    pub(crate) amount: u64,
 }
 
-#[async_trait]
-impl Execute for Deposit {
-    async fn execute(&self, db: &mut PickleDb, _shared_storage: PersistentStorage) -> Result<()> {
-        let mut rng = OsRng;
-        let signing_keypair = KeyPair::from(identity::KeyPair::new(&mut rng));
-        let encryption_keypair = KeyPair::from(encryption::KeyPair::new(&mut rng));
+pub(crate) async fn deposit(nymd_url: &str, mnemonic: &str, amount: u64) -> Result<State> {
+    let mut rng = OsRng;
+    let signing_keypair = KeyPair::from(identity::KeyPair::new(&mut rng));
+    let encryption_keypair = KeyPair::from(encryption::KeyPair::new(&mut rng));
 
-        let client = Client::new(&self.nymd_url, &self.mnemonic);
-        let tx_hash = client
-            .deposit(
-                self.amount,
-                signing_keypair.public_key.clone(),
-                encryption_keypair.public_key.clone(),
-                None,
-            )
-            .await?;
+    let client = Client::new(nymd_url, mnemonic);
+    let tx_hash = client
+        .deposit(
+            amount,
+            signing_keypair.public_key.clone(),
+            encryption_keypair.public_key.clone(),
+            None,
+        )
+        .await?;
 
-        let state = State {
-            amount: self.amount,
-            tx_hash: tx_hash.clone(),
-            signing_keypair,
-            encryption_keypair,
-            blind_request_data: None,
-            signature: None,
-        };
-        db.set(&tx_hash, &state).unwrap();
+    let state = State {
+        amount,
+        tx_hash,
+        signing_keypair,
+        encryption_keypair,
+    };
 
-        println!("{:?}", state);
-
-        Ok(())
-    }
+    Ok(state)
 }
 
-#[derive(Args, Clone)]
-pub(crate) struct ListDeposits {}
+pub(crate) async fn get_credential(state: &State, shared_storage: PersistentStorage) -> Result<()> {
+    let network_details = NymNetworkDetails::new_from_env();
+    let config = Config::try_from_nym_network_details(&network_details)?;
+    let client = validator_client::Client::new_query(config)?;
+    let coconut_api_clients = CoconutApiClient::all_coconut_api_clients(&client).await?;
 
-#[async_trait]
-impl Execute for ListDeposits {
-    async fn execute(&self, db: &mut PickleDb, _shared_storage: PersistentStorage) -> Result<()> {
-        for kv in db.iter() {
-            println!("{:?}", kv.get_value::<State>());
-        }
+    let params = Parameters::new(TOTAL_ATTRIBUTES).unwrap();
+    let bandwidth_credential_attributes = BandwidthVoucher::new(
+        &params,
+        state.amount.to_string(),
+        VOUCHER_INFO.to_string(),
+        Hash::from_str(&state.tx_hash).map_err(|_| CredentialClientError::InvalidTxHash)?,
+        identity::PrivateKey::from_base58_string(&state.signing_keypair.private_key)?,
+        encryption::PrivateKey::from_base58_string(&state.encryption_keypair.private_key)?,
+    );
 
-        Ok(())
-    }
-}
-
-#[derive(Args, Clone)]
-pub(crate) struct GetCredential {
-    /// The hash of a successful deposit transaction
-    #[clap(long)]
-    tx_hash: String,
-    /// The URLs to the validator-api endpoints the are run as coconut signer authorities, separated
-    /// by comma (,)
-    #[clap(long)]
-    signer_authorities: String,
-    /// If we want to get the signature without attaching a blind sign request; it is expected that
-    /// there is already a signature stored on the signer
-    #[clap(long, parse(from_flag))]
-    __no_request: bool,
-}
-
-#[async_trait]
-impl Execute for GetCredential {
-    async fn execute(&self, db: &mut PickleDb, shared_storage: PersistentStorage) -> Result<()> {
-        let mut state = db
-            .get::<State>(&self.tx_hash)
-            .ok_or(CredentialClientError::NoDeposit)?;
-
-        let urls = config::parse_validators(&self.signer_authorities);
-
-        let params = Parameters::new(TOTAL_ATTRIBUTES).unwrap();
-        let bandwidth_credential_attributes = if self.__no_request {
-            if let Some(blind_request_data) = state.blind_request_data {
-                let serial_number =
-                    Attribute::try_from_byte_slice(&blind_request_data.serial_number)
-                        .map_err(|_| CredentialClientError::CorruptedBlindSignRequest)?;
-                let binding_number =
-                    Attribute::try_from_byte_slice(&blind_request_data.binding_number)
-                        .map_err(|_| CredentialClientError::CorruptedBlindSignRequest)?;
-                let pedersen_commitments_openings = vec![
-                    Attribute::try_from_byte_slice(&blind_request_data.first_attribute)
-                        .map_err(|_| CredentialClientError::CorruptedBlindSignRequest)?,
-                    Attribute::try_from_byte_slice(&blind_request_data.second_attribute)
-                        .map_err(|_| CredentialClientError::CorruptedBlindSignRequest)?,
-                ];
-                let blind_sign_request =
-                    BlindSignRequest::from_bytes(blind_request_data.blind_sign_req.as_slice())
-                        .map_err(|_| CredentialClientError::CorruptedBlindSignRequest)?;
-                BandwidthVoucher::new_with_blind_sign_req(
-                    [serial_number, binding_number],
-                    [&state.amount.to_string(), VOUCHER_INFO],
-                    Hash::from_str(&self.tx_hash)
-                        .map_err(|_| CredentialClientError::InvalidTxHash)?,
-                    identity::PrivateKey::from_base58_string(&state.signing_keypair.private_key)?,
-                    encryption::PrivateKey::from_base58_string(
-                        &state.encryption_keypair.private_key,
-                    )?,
-                    pedersen_commitments_openings,
-                    blind_sign_request,
-                )
-            } else {
-                return Err(CredentialClientError::NoLocalBlindSignRequest);
-            }
-        } else {
-            BandwidthVoucher::new(
-                &params,
-                state.amount.to_string(),
-                VOUCHER_INFO.to_string(),
-                Hash::from_str(&self.tx_hash).map_err(|_| CredentialClientError::InvalidTxHash)?,
-                identity::PrivateKey::from_base58_string(&state.signing_keypair.private_key)?,
-                encryption::PrivateKey::from_base58_string(&state.encryption_keypair.private_key)?,
-            )
-        };
-
-        // Back up the blind sign req data, in case of sporadic failures
-        state.blind_request_data = Some(RequestData::new(
-            bandwidth_credential_attributes.get_private_attributes(),
-            bandwidth_credential_attributes.pedersen_commitments_openings(),
-            bandwidth_credential_attributes.blind_sign_request(),
-        )?);
-        db.set(&self.tx_hash, &state).unwrap();
-
-        let signature =
-            obtain_aggregate_signature(&params, &bandwidth_credential_attributes, &urls).await?;
-        shared_storage
-            .insert_coconut_credential(
-                state.amount.to_string(),
-                VOUCHER_INFO.to_string(),
-                bandwidth_credential_attributes.get_private_attributes()[0].to_bs58(),
-                bandwidth_credential_attributes.get_private_attributes()[1].to_bs58(),
-                signature.to_bs58(),
-            )
-            .await?;
-        state.signature = Some(signature.to_bs58());
-        db.set(&self.tx_hash, &state).unwrap();
-
-        println!("Signature: {:?}", state.signature);
-
-        Ok(())
-    }
+    let signature = obtain_aggregate_signature(
+        &params,
+        &bandwidth_credential_attributes,
+        &coconut_api_clients,
+    )
+    .await?;
+    println!("Signature: {:?}", signature.to_bs58());
+    shared_storage
+        .insert_coconut_credential(
+            state.amount.to_string(),
+            VOUCHER_INFO.to_string(),
+            bandwidth_credential_attributes.get_private_attributes()[0].to_bs58(),
+            bandwidth_credential_attributes.get_private_attributes()[1].to_bs58(),
+            signature.to_bs58(),
+        )
+        .await?;
+
+    Ok(())
 }

clients/credential/src/error.rs

@@ -8,6 +8,7 @@ use credentials::error::Error as CredentialError;
 use crypto::asymmetric::encryption::KeyRecoveryError;
 use crypto::asymmetric::identity::Ed25519RecoveryError;
 use validator_client::nymd::error::NymdError;
+use validator_client::ValidatorClientError;
 
 pub type Result<T> = std::result::Result<T, CredentialClientError>;
 
@@ -16,21 +17,12 @@ pub enum CredentialClientError {
     #[error("Nymd error: {0}")]
     Nymd(#[from] NymdError),
 
+    #[error("Validator client error: {0}")]
+    ValidatorClientError(#[from] ValidatorClientError),
+
     #[error("Credential error: {0}")]
     Credential(#[from] CredentialError),
 
-    #[error("No previous deposit with that tx hash")]
-    NoDeposit,
-
-    #[error("Wrong number of attributes")]
-    WrongAttributeNumber,
-
-    #[error("Could not find any backed up blind sign request data")]
-    NoLocalBlindSignRequest,
-
-    #[error("The local blind sign request data is corrupted")]
-    CorruptedBlindSignRequest,
-
     #[error("The tx hash provided is not valid")]
     InvalidTxHash,
 

clients/credential/src/main.rs

@@ -9,59 +9,41 @@ cfg_if::cfg_if! {
         mod error;
         mod state;
 
-        use commands::{Commands, Execute};
         use error::Result;
         use network_defaults::setup_env;
-        use clap::CommandFactory;
         use completions::fig_generate;
+        use commands::*;
+        use config::{DATA_DIR, DB_FILE_NAME};
 
-        use clap::Parser;
-        use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
+        use clap::{CommandFactory, Parser};
 
         #[derive(Parser)]
         #[clap(author = "Nymtech", version, about)]
         struct Cli {
             /// Path pointing to an env file that configures the client.
-            #[clap(long)]
+            #[clap(short, long)]
             pub(crate) config_env_file: Option<std::path::PathBuf>,
 
-            /// Path where the sqlite credental database will be located.
-            /// It should point to a $HOME/$CLIENT_ID/data/db.sqlite file of
-            /// the client that is supposed to use the credential.
-            #[clap(long)]
-            pub(crate) credential_db_path: std::path::PathBuf,
-
             #[clap(subcommand)]
-            command: Commands,
+            pub(crate) command: Command,
         }
 
         #[tokio::main]
         async fn main() -> Result<()> {
             let args = Cli::parse();
             setup_env(args.config_env_file.clone());
-
-            let shared_storage = credential_storage::initialise_storage(args.credential_db_path.clone()).await;
-            let mut db = match PickleDb::load(
-                "credential.db",
-                PickleDbDumpPolicy::AutoDump,
-                SerializationMethod::Json,
-            ) {
-                Ok(db) => db,
-                Err(_) => PickleDb::new(
-                    "credential.db",
-                    PickleDbDumpPolicy::AutoDump,
-                    SerializationMethod::Json,
-                ),
-            };
-
             let bin_name = "nym-credential-client";
 
-            match &args.command {
-                Commands::Deposit(m) => m.execute(&mut db, shared_storage).await?,
-                Commands::ListDeposits(m) => m.execute(&mut db, shared_storage).await?,
-                Commands::GetCredential(m) => m.execute(&mut db, shared_storage).await?,
-                Commands::Completions(s) => s.generate(&mut crate::Cli::into_app(), bin_name),
-                Commands::GenerateFigSpec => fig_generate(&mut crate::Cli::into_app(), bin_name)
+            match args.command {
+                Command::Run(r) => {
+                    let db_path = r.client_home_directory.join(DATA_DIR).join(DB_FILE_NAME);
+                    let shared_storage = credential_storage::initialise_storage(db_path).await;
+
+                    let state = deposit(&r.nymd_url, &r.mnemonic, r.amount).await?;
+                    get_credential(&state, shared_storage).await?;
+                }
+                Command::Completions(c) => c.generate(&mut crate::Cli::into_app(), bin_name),
+                Command::GenerateFigSpec => fig_generate(&mut crate::Cli::into_app(), bin_name)
             }
 
             Ok(())

clients/credential/src/state.rs

@@ -1,13 +1,10 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use coconut_interface::{Attribute, BlindSignRequest, Bytable, PrivateAttribute};
 use serde::{Deserialize, Serialize};
 
 use crypto::asymmetric::{encryption, identity};
 
-use crate::error::{CredentialClientError, Result};
-
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub(crate) struct KeyPair {
     pub public_key: String,
@@ -38,35 +35,4 @@ pub(crate) struct State {
     pub tx_hash: String,
     pub signing_keypair: KeyPair,
     pub encryption_keypair: KeyPair,
-    pub blind_request_data: Option<RequestData>,
-    pub signature: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub(crate) struct RequestData {
-    pub serial_number: Vec<u8>,
-    pub binding_number: Vec<u8>,
-    pub first_attribute: Vec<u8>,
-    pub second_attribute: Vec<u8>,
-    pub blind_sign_req: Vec<u8>,
-}
-
-impl RequestData {
-    pub fn new(
-        private_attributes: Vec<PrivateAttribute>,
-        attributes: &[Attribute],
-        blind_sign_request: &BlindSignRequest,
-    ) -> Result<Self> {
-        if private_attributes.len() != 2 || attributes.len() != 2 {
-            Err(CredentialClientError::WrongAttributeNumber)
-        } else {
-            Ok(RequestData {
-                serial_number: private_attributes[0].to_byte_vec(),
-                binding_number: private_attributes[1].to_byte_vec(),
-                first_attribute: attributes[0].to_byte_vec(),
-                second_attribute: attributes[1].to_byte_vec(),
-                blind_sign_req: blind_sign_request.to_bytes(),
-            })
-        }
-    }
 }

clients/native/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "nym-client"
-version = "1.0.2"
+version = "1.1.3"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 description = "Implementation of the Nym Client"
 edition = "2021"
-rust-version = "1.56"
+rust-version = "1.65"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -26,18 +26,23 @@ log = "0.4" # self explanatory
 pretty_env_logger = "0.4" # for formatting log messages
 rand = { version = "0.7.3", features = ["wasm-bindgen"] } # rng-related traits + some rng implementation to use
 serde = { version = "1.0.104", features = ["derive"] } # for config serialization/deserialization
+serde_json = "1.0"
 sled = "0.34" # for storage of replySURB decryption keys
-tokio = { version = "1.19.1", features = ["rt-multi-thread", "net", "signal"] } # async runtime
+tap = "1.0.1"
+thiserror = "1.0.34"
+tokio = { version = "1.21.2", features = ["rt-multi-thread", "net", "signal"] } # async runtime
 tokio-tungstenite = "0.14" # websocket
 
 ## internal
 client-core = { path = "../client-core" }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 config = { path = "../../common/config" }
 completions = { path = "../../common/completions" }
 credential-storage = { path = "../../common/credential-storage" }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
+logging = { path = "../../common/logging"}
 gateway-client = { path = "../../common/client-libs/gateway-client" }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 network-defaults = { path = "../../common/network-defaults" }

clients/native/examples/websocket_binarysend.rs

@@ -43,6 +43,7 @@ async fn send_file_with_reply() {
         recipient,
         message: read_data,
         with_reply_surb: true,
+        connection_id: Some(0),
     };
 
     println!("sending content of 'dummy_file' over the mix network...");
@@ -91,6 +92,7 @@ async fn send_file_without_reply() {
         recipient,
         message: read_data,
         with_reply_surb: false,
+        connection_id: Some(0),
     };
 
     println!("sending content of 'dummy_file' over the mix network...");

clients/native/src/client/config/mod.rs

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::client::config::template::config_template;
-use client_core::config::Config as BaseConfig;
 pub use client_core::config::MISSING_VALUE;
+use client_core::config::{ClientCoreConfigTrait, Config as BaseConfig};
 use config::defaults::DEFAULT_WEBSOCKET_LISTENING_PORT;
 use config::NymConfig;
 use serde::{Deserialize, Serialize};
@@ -27,6 +27,10 @@ impl SocketType {
             _ => SocketType::None,
         }
     }
+
+    pub fn is_websocket(self) -> bool {
+        matches!(self, SocketType::WebSocket)
+    }
 }
 
 #[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
@@ -69,6 +73,12 @@ impl NymConfig for Config {
     }
 }
 
+impl ClientCoreConfigTrait for Config {
+    fn get_gateway_endpoint(&self) -> &client_core::config::GatewayEndpointConfig {
+        self.base.get_gateway_endpoint()
+    }
+}
+
 impl Config {
     pub fn new<S: Into<String>>(id: S) -> Self {
         Config {

clients/native/src/client/config/template.rs

@@ -23,6 +23,13 @@ id = '{{ client.id }}'
 # to claim bandwidth without presenting bandwidth credentials.
 disabled_credentials_mode = {{ client.disabled_credentials_mode }}
 
+# Addresses to nymd validators via which the client can communicate with the chain.
+validator_urls = [
+    {{#each client.validator_urls }}
+        '{{this}}',
+    {{/each}}
+]
+
 # Addresses to APIs running on validator from which the client gets the view of the network.
 validator_api_urls = [
     {{#each client.validator_api_urls }}

clients/native/src/client/mod.rs

@@ -1,326 +1,113 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use client_core::client::cover_traffic_stream::LoopCoverTrafficStream;
-use client_core::client::inbound_messages::{
-    InputMessage, InputMessageReceiver, InputMessageSender,
-};
+use crate::client::config::Config;
+use crate::error::ClientError;
+use crate::websocket;
+use client_connections::TransmissionLane;
+use client_core::client::base_client::{BaseClientBuilder, ClientInput, ClientOutput};
+use client_core::client::inbound_messages::InputMessage;
 use client_core::client::key_manager::KeyManager;
-use client_core::client::mix_traffic::{
-    BatchMixMessageReceiver, BatchMixMessageSender, MixTrafficController,
-};
-use client_core::client::real_messages_control;
-use client_core::client::real_messages_control::RealMessagesController;
-use client_core::client::received_buffer::{
-    ReceivedBufferMessage, ReceivedBufferRequestReceiver, ReceivedBufferRequestSender,
-    ReceivedMessagesBufferController, ReconstructedMessagesReceiver,
-};
-use client_core::client::reply_key_storage::ReplyKeyStorage;
-use client_core::client::topology_control::{
-    TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
-};
+use client_core::client::received_buffer::{ReceivedBufferMessage, ReconstructedMessagesReceiver};
 use client_core::config::persistence::key_pathfinder::ClientKeyPathfinder;
-use crypto::asymmetric::identity;
 use futures::channel::mpsc;
 use gateway_client::bandwidth::BandwidthController;
-use gateway_client::{
-    AcknowledgementReceiver, AcknowledgementSender, GatewayClient, MixnetMessageReceiver,
-    MixnetMessageSender,
-};
 use log::*;
 use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::addressing::nodes::NodeIdentity;
 use nymsphinx::anonymous_replies::ReplySurb;
 use nymsphinx::receiver::ReconstructedMessage;
-use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
-
-use crate::client::config::{Config, SocketType};
-use crate::websocket;
+use task::{wait_for_signal, ShutdownNotifier};
 
 pub(crate) mod config;
 
-pub struct NymClient {
+pub struct SocketClient {
     /// Client configuration options, including, among other things, packet sending rates,
     /// key filepaths, etc.
     config: Config,
 
     /// KeyManager object containing smart pointers to all relevant keys used by the client.
     key_manager: KeyManager,
-
-    /// Channel used for transforming 'raw' messages into sphinx packets and sending them
-    /// through the mix network.
-    /// It is only available if the client started with the websocket listener disabled.
-    input_tx: Option<InputMessageSender>,
-
-    /// Channel used for obtaining reconstructed messages received from the mix network.
-    /// It is only available if the client started with the websocket listener disabled.
-    receive_tx: Option<ReconstructedMessagesReceiver>,
 }
 
-impl NymClient {
+impl SocketClient {
     pub fn new(config: Config) -> Self {
         let pathfinder = ClientKeyPathfinder::new_from_config(config.get_base());
         let key_manager = KeyManager::load_keys(&pathfinder).expect("failed to load stored keys");
 
-        NymClient {
+        SocketClient {
             config,
             key_manager,
-            input_tx: None,
-            receive_tx: None,
         }
     }
 
-    pub fn as_mix_recipient(&self) -> Recipient {
-        Recipient::new(
-            *self.key_manager.identity_keypair().public_key(),
-            *self.key_manager.encryption_keypair().public_key(),
-            // TODO: below only works under assumption that gateway address == gateway id
-            // (which currently is true)
-            NodeIdentity::from_base58_string(self.config.get_base().get_gateway_id()).unwrap(),
-        )
-    }
-
-    // future constantly pumping loop cover traffic at some specified average rate
-    // the pumped traffic goes to the MixTrafficController
-    fn start_cover_traffic_stream(
-        &self,
-        topology_accessor: TopologyAccessor,
-        mix_tx: BatchMixMessageSender,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting loop cover traffic stream...");
-
-        LoopCoverTrafficStream::new(
-            self.key_manager.ack_key(),
-            self.config.get_base().get_average_ack_delay(),
-            self.config.get_base().get_average_packet_delay(),
-            self.config
-                .get_base()
-                .get_loop_cover_traffic_average_delay(),
-            mix_tx,
-            self.as_mix_recipient(),
-            topology_accessor,
-            shutdown,
-        )
-        .start();
-    }
-
-    fn start_real_traffic_controller(
-        &self,
-        topology_accessor: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
-        ack_receiver: AcknowledgementReceiver,
-        input_receiver: InputMessageReceiver,
-        mix_sender: BatchMixMessageSender,
-        shutdown: ShutdownListener,
-    ) {
-        let controller_config = real_messages_control::Config::new(
-            self.key_manager.ack_key(),
-            self.config.get_base().get_ack_wait_multiplier(),
-            self.config.get_base().get_ack_wait_addition(),
-            self.config.get_base().get_average_ack_delay(),
-            self.config.get_base().get_message_sending_average_delay(),
-            self.config.get_base().get_average_packet_delay(),
-            self.as_mix_recipient(),
-        );
-
-        info!("Starting real traffic stream...");
-
-        RealMessagesController::new(
-            controller_config,
-            ack_receiver,
-            input_receiver,
-            mix_sender,
-            topology_accessor,
-            reply_key_storage,
-            shutdown,
-        )
-        .start();
-    }
-
-    // buffer controlling all messages fetched from provider
-    // required so that other components would be able to use them (say the websocket)
-    fn start_received_messages_buffer_controller(
-        &self,
-        query_receiver: ReceivedBufferRequestReceiver,
-        mixnet_receiver: MixnetMessageReceiver,
-        reply_key_storage: ReplyKeyStorage,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting received messages buffer controller...");
-        ReceivedMessagesBufferController::new(
-            self.key_manager.encryption_keypair(),
-            query_receiver,
-            mixnet_receiver,
-            reply_key_storage,
-            shutdown,
-        )
-        .start()
-    }
-
-    async fn start_gateway_client(
-        &mut self,
-        mixnet_message_sender: MixnetMessageSender,
-        ack_sender: AcknowledgementSender,
-        shutdown: ShutdownListener,
-    ) -> GatewayClient {
-        let gateway_id = self.config.get_base().get_gateway_id();
-        if gateway_id.is_empty() {
-            panic!("The identity of the gateway is unknown - did you run `nym-client` init?")
-        }
-        let gateway_owner = self.config.get_base().get_gateway_owner();
-        if gateway_owner.is_empty() {
-            panic!("The owner of the gateway is unknown - did you run `nym-client` init?")
-        }
-        let gateway_address = self.config.get_base().get_gateway_listener();
-        if gateway_address.is_empty() {
-            panic!("The address of the gateway is unknown - did you run `nym-client` init?")
-        }
-
-        let gateway_identity = identity::PublicKey::from_base58_string(gateway_id)
-            .expect("provided gateway id is invalid!");
-
+    async fn create_bandwidth_controller(config: &Config) -> BandwidthController {
         #[cfg(feature = "coconut")]
-        let bandwidth_controller = BandwidthController::new(
-            credential_storage::initialise_storage(self.config.get_base().get_database_path())
-                .await,
-            self.config.get_base().get_validator_api_endpoints(),
-        );
+        let bandwidth_controller = {
+            let details = network_defaults::NymNetworkDetails::new_from_env();
+            let mut client_config =
+                validator_client::Config::try_from_nym_network_details(&details)
+                    .expect("failed to construct validator client config");
+            let nymd_url = config
+                .get_base()
+                .get_validator_endpoints()
+                .pop()
+                .expect("No nymd validator endpoint provided");
+            let api_url = config
+                .get_base()
+                .get_validator_api_endpoints()
+                .pop()
+                .expect("No validator api endpoint provided");
+            // overwrite env configuration with config URLs
+            client_config = client_config.with_urls(nymd_url, api_url);
+            let client = validator_client::Client::new_query(client_config)
+                .expect("Could not construct query client");
+            let coconut_api_clients =
+                validator_client::CoconutApiClient::all_coconut_api_clients(&client)
+                    .await
+                    .expect("Could not query api clients");
+            BandwidthController::new(
+                credential_storage::initialise_storage(config.get_base().get_database_path()).await,
+                coconut_api_clients,
+            )
+        };
         #[cfg(not(feature = "coconut"))]
         let bandwidth_controller = BandwidthController::new(
-            credential_storage::initialise_storage(self.config.get_base().get_database_path())
-                .await,
+            credential_storage::initialise_storage(config.get_base().get_database_path()).await,
         )
         .expect("Could not create bandwidth controller");
-
-        let mut gateway_client = GatewayClient::new(
-            gateway_address,
-            self.key_manager.identity_keypair(),
-            gateway_identity,
-            gateway_owner,
-            Some(self.key_manager.gateway_shared_key()),
-            mixnet_message_sender,
-            ack_sender,
-            self.config.get_base().get_gateway_response_timeout(),
-            Some(bandwidth_controller),
-            Some(shutdown),
-        );
-
-        gateway_client
-            .set_disabled_credentials_mode(self.config.get_base().get_disabled_credentials_mode());
-
-        gateway_client
-            .authenticate_and_start()
-            .await
-            .expect("could not authenticate and start up the gateway connection");
-
-        gateway_client
-    }
-
-    // future responsible for periodically polling directory server and updating
-    // the current global view of topology
-    async fn start_topology_refresher(
-        &mut self,
-        topology_accessor: TopologyAccessor,
-        shutdown: ShutdownListener,
-    ) {
-        let topology_refresher_config = TopologyRefresherConfig::new(
-            self.config.get_base().get_validator_api_endpoints(),
-            self.config.get_base().get_topology_refresh_rate(),
-            env!("CARGO_PKG_VERSION").to_string(),
-        );
-        let mut topology_refresher =
-            TopologyRefresher::new(topology_refresher_config, topology_accessor);
-        // before returning, block entire runtime to refresh the current network view so that any
-        // components depending on topology would see a non-empty view
-        info!("Obtaining initial network topology");
-        topology_refresher.refresh().await;
-
-        // TODO: a slightly more graceful termination here
-        if !topology_refresher.is_topology_routable().await {
-            panic!(
-                "The current network topology seem to be insufficient to route any packets through\
-                - check if enough nodes and a gateway are online"
-            );
-        }
-
-        info!("Starting topology refresher...");
-        topology_refresher.start(shutdown);
-    }
-
-    // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
-    // TODO: if we want to send control messages to gateway_client, this CAN'T take the ownership
-    // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
-    // requests?
-    fn start_mix_traffic_controller(
-        &mut self,
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting mix traffic controller...");
-        MixTrafficController::new(mix_rx, gateway_client, shutdown).start();
+        bandwidth_controller
     }
 
     fn start_websocket_listener(
-        &self,
-        buffer_requester: ReceivedBufferRequestSender,
-        msg_input: InputMessageSender,
+        config: &Config,
+        client_input: ClientInput,
+        client_output: ClientOutput,
+        self_address: Recipient,
     ) {
         info!("Starting websocket listener...");
 
-        let websocket_handler =
-            websocket::Handler::new(msg_input, buffer_requester, self.as_mix_recipient());
+        let ClientInput {
+            shared_lane_queue_lengths,
+            connection_command_sender,
+            input_sender,
+        } = client_input;
 
-        websocket::Listener::new(self.config.get_listening_port()).start(websocket_handler);
+        let received_buffer_request_sender = client_output.received_buffer_request_sender;
+
+        let websocket_handler = websocket::Handler::new(
+            input_sender,
+            connection_command_sender,
+            received_buffer_request_sender,
+            self_address,
+            shared_lane_queue_lengths,
+        );
+
+        websocket::Listener::new(config.get_listening_port()).start(websocket_handler);
     }
 
-    /// EXPERIMENTAL DIRECT RUST API
-    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
-    /// well enough in local tests)
-    pub fn send_message(&mut self, recipient: Recipient, message: Vec<u8>, with_reply_surb: bool) {
-        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb);
-
-        self.input_tx
-            .as_ref()
-            .expect("start method was not called before!")
-            .unbounded_send(input_msg)
-            .unwrap();
-    }
-
-    /// EXPERIMENTAL DIRECT RUST API
-    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
-    /// well enough in local tests)
-    pub fn send_reply(&mut self, reply_surb: ReplySurb, message: Vec<u8>) {
-        let input_msg = InputMessage::new_reply(reply_surb, message);
-
-        self.input_tx
-            .as_ref()
-            .expect("start method was not called before!")
-            .unbounded_send(input_msg)
-            .unwrap();
-    }
-
-    /// EXPERIMENTAL DIRECT RUST API
-    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
-    /// well enough in local tests)
-    /// Note: it waits for the first occurrence of messages being sent to ourselves. If you expect multiple
-    /// messages, you might have to call this function repeatedly.
-    // TODO: I guess this should really return something that `impl Stream<Item=ReconstructedMessage>`
-    pub async fn wait_for_messages(&mut self) -> Vec<ReconstructedMessage> {
-        use futures::StreamExt;
-
-        self.receive_tx
-            .as_mut()
-            .expect("start method was not called before!")
-            .next()
-            .await
-            .expect("buffer controller seems to have somehow died!")
-    }
-
-    /// blocking version of `start` method. Will run forever (or until SIGINT is sent)
-    pub async fn run_forever(&mut self) {
-        let shutdown = self.start().await;
+    /// blocking version of `start_socket` method. Will run forever (or until SIGINT is sent)
+    pub async fn run_socket_forever(self) -> Result<(), ClientError> {
+        let shutdown = self.start_socket().await?;
         wait_for_signal().await;
 
         println!(
@@ -337,100 +124,120 @@ impl NymClient {
         //shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-client");
+        Ok(())
     }
 
-    pub async fn start(&mut self) -> ShutdownNotifier {
-        info!("Starting nym client");
-        // channels for inter-component communication
-        // TODO: make the channels be internally created by the relevant components
-        // rather than creating them here, so say for example the buffer controller would create the request channels
-        // and would allow anyone to clone the sender channel
-
-        // sphinx_message_sender is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
-        // they are used by cover traffic stream and real traffic stream
-        // sphinx_message_receiver is the receiver used by MixTrafficController that sends the actual traffic
-        let (sphinx_message_sender, sphinx_message_receiver) = mpsc::unbounded();
-
-        // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
-        // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
-        let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
-
-        // used for announcing connection or disconnection of a channel for pushing re-assembled messages to
-        let (received_buffer_request_sender, received_buffer_request_receiver) = mpsc::unbounded();
-
-        // channels responsible for controlling real messages
-        let (input_sender, input_receiver) = mpsc::unbounded::<InputMessage>();
-
-        // channels responsible for controlling ack messages
-        let (ack_sender, ack_receiver) = mpsc::unbounded();
-        let shared_topology_accessor = TopologyAccessor::new();
-
-        let reply_key_storage =
-            ReplyKeyStorage::load(self.config.get_base().get_reply_encryption_key_store_path())
-                .expect("Failed to load reply key storage!");
-
-        // Shutdown notifier for signalling tasks to stop
-        let shutdown = ShutdownNotifier::default();
-
-        // the components are started in very specific order. Unless you know what you are doing,
-        // do not change that.
-        self.start_topology_refresher(shared_topology_accessor.clone(), shutdown.subscribe())
-            .await;
-        self.start_received_messages_buffer_controller(
-            received_buffer_request_receiver,
-            mixnet_messages_receiver,
-            reply_key_storage.clone(),
-            shutdown.subscribe(),
-        );
-
-        let gateway_client = self
-            .start_gateway_client(mixnet_messages_sender, ack_sender, shutdown.subscribe())
-            .await;
-
-        self.start_mix_traffic_controller(
-            sphinx_message_receiver,
-            gateway_client,
-            shutdown.subscribe(),
-        );
-        self.start_real_traffic_controller(
-            shared_topology_accessor.clone(),
-            reply_key_storage,
-            ack_receiver,
-            input_receiver,
-            sphinx_message_sender.clone(),
-            shutdown.subscribe(),
-        );
-
-        self.start_cover_traffic_stream(
-            shared_topology_accessor,
-            sphinx_message_sender,
-            shutdown.subscribe(),
-        );
-
-        match self.config.get_socket_type() {
-            SocketType::WebSocket => {
-                self.start_websocket_listener(received_buffer_request_sender, input_sender)
-            }
-            SocketType::None => {
-                // if we did not start the socket, it means we're running (supposedly) in the native mode
-                // and hence we should announce 'ourselves' to the buffer
-                let (reconstructed_sender, reconstructed_receiver) = mpsc::unbounded();
-
-                // tell the buffer to start sending stuff to us
-                received_buffer_request_sender
-                    .unbounded_send(ReceivedBufferMessage::ReceiverAnnounce(
-                        reconstructed_sender,
-                    ))
-                    .expect("the buffer request failed!");
-
-                self.receive_tx = Some(reconstructed_receiver);
-                self.input_tx = Some(input_sender);
-            }
+    pub async fn start_socket(self) -> Result<ShutdownNotifier, ClientError> {
+        if !self.config.get_socket_type().is_websocket() {
+            return Err(ClientError::InvalidSocketMode);
         }
 
-        info!("Client startup finished!");
-        info!("The address of this client is: {}", self.as_mix_recipient());
+        let base_builder = BaseClientBuilder::new_from_base_config(
+            self.config.get_base(),
+            self.key_manager,
+            Some(Self::create_bandwidth_controller(&self.config).await),
+        );
 
-        shutdown
+        let self_address = base_builder.as_mix_recipient();
+        let mut started_client = base_builder.start_base().await?;
+        let client_input = started_client.client_input.register_producer();
+        let client_output = started_client.client_output.register_consumer();
+
+        Self::start_websocket_listener(&self.config, client_input, client_output, self_address);
+
+        info!("Client startup finished!");
+        info!("The address of this client is: {}", self_address);
+
+        Ok(started_client.shutdown_notifier)
+    }
+
+    pub async fn start_direct(self) -> Result<DirectClient, ClientError> {
+        if self.config.get_socket_type().is_websocket() {
+            return Err(ClientError::InvalidSocketMode);
+        }
+
+        let base_client = BaseClientBuilder::new_from_base_config(
+            self.config.get_base(),
+            self.key_manager,
+            Some(Self::create_bandwidth_controller(&self.config).await),
+        );
+
+        let mut started_client = base_client.start_base().await?;
+        let client_input = started_client.client_input.register_producer();
+        let client_output = started_client.client_output.register_consumer();
+
+        // register our receiver
+        let (reconstructed_sender, reconstructed_receiver) = mpsc::unbounded();
+
+        // tell the buffer to start sending stuff to us
+        client_output
+            .received_buffer_request_sender
+            .unbounded_send(ReceivedBufferMessage::ReceiverAnnounce(
+                reconstructed_sender,
+            ))
+            .expect("the buffer request failed!");
+
+        Ok(DirectClient {
+            client_input,
+            reconstructed_receiver,
+            _shutdown_notifier: started_client.shutdown_notifier,
+        })
+    }
+}
+
+pub struct DirectClient {
+    client_input: ClientInput,
+    reconstructed_receiver: ReconstructedMessagesReceiver,
+
+    // we need to keep reference to this guy otherwise things will start dropping
+    _shutdown_notifier: ShutdownNotifier,
+}
+
+impl DirectClient {
+    /// EXPERIMENTAL DIRECT RUST API
+    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
+    /// well enough in local tests)
+    pub async fn send_message(
+        &mut self,
+        recipient: Recipient,
+        message: Vec<u8>,
+        with_reply_surb: bool,
+    ) {
+        let lane = TransmissionLane::General;
+        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb, lane);
+
+        self.client_input
+            .input_sender
+            .send(input_msg)
+            .await
+            .expect("InputMessageReceiver has stopped receiving!");
+    }
+
+    /// EXPERIMENTAL DIRECT RUST API
+    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
+    /// well enough in local tests)
+    pub async fn send_reply(&mut self, reply_surb: ReplySurb, message: Vec<u8>) {
+        let input_msg = InputMessage::new_reply(reply_surb, message);
+
+        self.client_input
+            .input_sender
+            .send(input_msg)
+            .await
+            .expect("InputMessageReceiver has stopped receiving!");
+    }
+
+    /// EXPERIMENTAL DIRECT RUST API
+    /// It's untested and there are absolutely no guarantees about it (but seems to have worked
+    /// well enough in local tests)
+    /// Note: it waits for the first occurrence of messages being sent to ourselves. If you expect multiple
+    /// messages, you might have to call this function repeatedly.
+    // TODO: I guess this should really return something that `impl Stream<Item=ReconstructedMessage>`
+    pub async fn wait_for_messages(&mut self) -> Vec<ReconstructedMessage> {
+        use futures::StreamExt;
+
+        self.reconstructed_receiver
+            .next()
+            .await
+            .expect("buffer controller seems to have somehow died!")
     }
 }

clients/native/src/commands/init.rs

@@ -1,13 +1,18 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::fmt::Display;
+
 use clap::Args;
-use client_core::{config::GatewayEndpoint, error::ClientCoreError};
 use config::NymConfig;
+use nymsphinx::addressing::clients::Recipient;
+use serde::Serialize;
+use tap::TapFallible;
 
 use crate::{
     client::config::Config,
     commands::{override_config, OverrideConfig},
+    error::ClientError,
 };
 
 #[derive(Args, Clone)]
@@ -25,9 +30,13 @@ pub(crate) struct Init {
     #[clap(long)]
     force_register_gateway: bool,
 
-    /// Comma separated list of rest endpoints of the validators
+    /// Comma separated list of rest endpoints of the nymd validators
     #[clap(long)]
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+
+    /// Comma separated list of rest endpoints of the API validators
+    #[clap(long)]
+    api_validators: Option<String>,
 
     /// Whether to not start the websocket
     #[clap(long)]
@@ -42,20 +51,30 @@ pub(crate) struct Init {
     #[clap(long, hidden = true)]
     fastmode: bool,
 
+    /// Disable loop cover traffic and the Poisson rate limiter (for debugging only)
+    #[clap(long, hidden = true)]
+    no_cover: bool,
+
     /// Set this client to work in a enabled credentials mode that would attempt to use gateway
     /// with bandwidth credential requirement.
     #[cfg(feature = "coconut")]
     #[clap(long)]
     enabled_credentials_mode: bool,
+
+    /// Save a summary of the initialization to a json file
+    #[clap(long)]
+    output_json: bool,
 }
 
 impl From<Init> for OverrideConfig {
     fn from(init_config: Init) -> Self {
         OverrideConfig {
-            validators: init_config.validators,
+            nymd_validators: init_config.nymd_validators,
+            api_validators: init_config.api_validators,
             disable_socket: init_config.disable_socket,
             port: init_config.port,
             fastmode: init_config.fastmode,
+            no_cover: init_config.no_cover,
 
             #[cfg(feature = "coconut")]
             enabled_credentials_mode: init_config.enabled_credentials_mode,
@@ -63,7 +82,30 @@ impl From<Init> for OverrideConfig {
     }
 }
 
-pub(crate) async fn execute(args: &Init) {
+#[derive(Debug, Serialize)]
+pub struct InitResults {
+    #[serde(flatten)]
+    client_core: client_core::init::InitResults,
+    client_listening_port: String,
+}
+
+impl InitResults {
+    fn new(config: &Config, address: &Recipient) -> Self {
+        Self {
+            client_core: client_core::init::InitResults::new(config.get_base(), address),
+            client_listening_port: config.get_listening_port().to_string(),
+        }
+    }
+}
+
+impl Display for InitResults {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "{}", self.client_core)?;
+        write!(f, "Client listening port: {}", self.client_listening_port)
+    }
+}
+
+pub(crate) async fn execute(args: &Init) -> Result<(), ClientError> {
     println!("Initialising client...");
 
     let id = &args.id;
@@ -87,25 +129,44 @@ pub(crate) async fn execute(args: &Init) {
     let register_gateway = !already_init || user_wants_force_register;
 
     // Attempt to use a user-provided gateway, if possible
-    let user_chosen_gateway_id = args.gateway.as_deref();
+    let user_chosen_gateway_id = args.gateway.clone();
 
-    let mut config = Config::new(id);
-    let override_config_fields = OverrideConfig::from(args.clone());
-    config = override_config(config, override_config_fields);
+    // Load and potentially override config
+    let mut config = override_config(Config::new(id), OverrideConfig::from(args.clone()));
+
+    // Setup gateway by either registering a new one, or creating a new config from the selected
+    // one but with keys kept, or reusing the gateway configuration.
+    let gateway = client_core::init::setup_gateway::<Config, _>(
+        register_gateway,
+        user_chosen_gateway_id,
+        config.get_base(),
+    )
+    .await
+    .tap_err(|err| eprintln!("Failed to setup gateway\nError: {err}"))?;
 
-    let gateway = setup_gateway(id, register_gateway, user_chosen_gateway_id, &config)
-        .await
-        .unwrap_or_else(|err| {
-            eprintln!("Failed to setup gateway\nError: {err}");
-            std::process::exit(1)
-        });
     config.get_base_mut().with_gateway_endpoint(gateway);
 
-    let config_save_location = config.get_config_file_save_location();
-    config
-        .save_to_file(None)
-        .expect("Failed to save the config file");
+    config.save_to_file(None).tap_err(|_| {
+        log::error!("Failed to save the config file");
+    })?;
 
+    print_saved_config(&config);
+
+    let address = client_core::init::get_client_address_from_stored_keys(config.get_base())?;
+    let init_results = InitResults::new(&config, &address);
+    println!("{}", init_results);
+
+    // Output summary to a json file, if specified
+    if args.output_json {
+        client_core::init::output_to_json(&init_results, "client_init_results.json");
+    }
+
+    println!("\nThe address of this client is: {}\n", address);
+    Ok(())
+}
+
+fn print_saved_config(config: &Config) {
+    let config_save_location = config.get_config_file_save_location();
     println!("Saved configuration file to {:?}", config_save_location);
     println!("Using gateway: {}", config.get_base().get_gateway_id());
     log::debug!("Gateway id: {}", config.get_base().get_gateway_id());
@@ -114,63 +175,5 @@ pub(crate) async fn execute(args: &Init) {
         "Gateway listener: {}",
         config.get_base().get_gateway_listener()
     );
-    println!("Client configuration completed.");
-
-    client_core::init::show_address(config.get_base()).unwrap_or_else(|err| {
-        eprintln!("Failed to show address\nError: {err}");
-        std::process::exit(1)
-    });
-}
-
-async fn setup_gateway(
-    id: &str,
-    register: bool,
-    user_chosen_gateway_id: Option<&str>,
-    config: &Config,
-) -> Result<GatewayEndpoint, ClientCoreError> {
-    if register {
-        // Get the gateway details by querying the validator-api. Either pick one at random or use
-        // the chosen one if it's among the available ones.
-        println!("Configuring gateway");
-        let gateway = client_core::init::query_gateway_details(
-            config.get_base().get_validator_api_endpoints(),
-            user_chosen_gateway_id,
-        )
-        .await?;
-        log::debug!("Querying gateway gives: {}", gateway);
-
-        // Registering with gateway by setting up and writing shared keys to disk
-        log::trace!("Registering gateway");
-        client_core::init::register_with_gateway_and_store_keys(gateway.clone(), config.get_base())
-            .await?;
-        println!("Saved all generated keys");
-
-        Ok(gateway.into())
-    } else if user_chosen_gateway_id.is_some() {
-        // Just set the config, don't register or create any keys
-        // This assumes that the user knows what they are doing, and that the existing keys are
-        // valid for the gateway being used
-        println!("Using gateway provided by user, keeping existing keys");
-        let gateway = client_core::init::query_gateway_details(
-            config.get_base().get_validator_api_endpoints(),
-            user_chosen_gateway_id,
-        )
-        .await?;
-        log::debug!("Querying gateway gives: {}", gateway);
-        Ok(gateway.into())
-    } else {
-        println!("Not registering gateway, will reuse existing config and keys");
-        let existing_config = Config::load_from_file(Some(id)).map_err(|err| {
-            log::error!(
-                "Unable to configure gateway: {err}. \n
-                Seems like the client was already initialized but it was not possible to read \
-                the existing configuration file. \n
-                CAUTION: Consider backing up your gateway keys and try force gateway registration, or \
-                removing the existing configuration and starting over."
-            );
-            ClientCoreError::CouldNotLoadExistingGatewayConfiguration(err)
-        })?;
-
-        Ok(existing_config.get_base().get_gateway_endpoint().clone())
-    }
+    println!("Client configuration completed.\n");
 }

clients/native/src/commands/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::client::config::{Config, SocketType};
+use crate::error::ClientError;
 use clap::CommandFactory;
 use clap::{Parser, Subcommand};
 use completions::{fig_generate, ArgShell};
@@ -49,7 +50,7 @@ fn long_version_static() -> &'static str {
 #[clap(author = "Nymtech", version, long_version = long_version_static(), about)]
 pub(crate) struct Cli {
     /// Path pointing to an env file that configures the client.
-    #[clap(long)]
+    #[clap(short, long)]
     pub(crate) config_env_file: Option<std::path::PathBuf>,
 
     #[clap(subcommand)]
@@ -74,29 +75,43 @@ pub(crate) enum Commands {
 
 // Configuration that can be overridden.
 pub(crate) struct OverrideConfig {
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+    api_validators: Option<String>,
     disable_socket: bool,
     port: Option<u16>,
     fastmode: bool,
+    no_cover: bool,
 
     #[cfg(feature = "coconut")]
     enabled_credentials_mode: bool,
 }
 
-pub(crate) async fn execute(args: &Cli) {
+pub(crate) async fn execute(args: &Cli) -> Result<(), ClientError> {
     let bin_name = "nym-native-client";
 
     match &args.command {
-        Commands::Init(m) => init::execute(m).await,
-        Commands::Run(m) => run::execute(m).await,
+        Commands::Init(m) => init::execute(m).await?,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::Upgrade(m) => upgrade::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::into_app(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::into_app(), bin_name),
     }
+    Ok(())
 }
 
 pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Config {
-    if let Some(raw_validators) = args.validators {
+    if let Some(raw_validators) = args.nymd_validators {
+        config
+            .get_base_mut()
+            .set_custom_validators(config::parse_validators(&raw_validators));
+    } else if std::env::var(network_defaults::var_names::CONFIGURED).is_ok() {
+        let raw_validators = std::env::var(network_defaults::var_names::NYMD_VALIDATOR)
+            .expect("nymd validator not set");
+        config
+            .get_base_mut()
+            .set_custom_validators(config::parse_validators(&raw_validators));
+    }
+    if let Some(raw_validators) = args.api_validators {
         config
             .get_base_mut()
             .set_custom_validator_apis(config::parse_validators(&raw_validators));
@@ -127,6 +142,10 @@ pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Confi
         config.get_base_mut().set_high_default_traffic_volume();
     }
 
+    if args.no_cover {
+        config.get_base_mut().set_no_cover_traffic();
+    }
+
     config
 }
 

clients/native/src/commands/run.rs

@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    client::{config::Config, NymClient},
+    client::{config::Config, SocketClient},
     commands::{override_config, OverrideConfig},
+    error::ClientError,
 };
 
 use clap::Args;
@@ -17,9 +18,13 @@ pub(crate) struct Run {
     #[clap(long)]
     id: String,
 
-    /// Comma separated list of rest endpoints of the validators
+    /// Comma separated list of rest endpoints of the nymd validators
     #[clap(long)]
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+
+    /// Comma separated list of rest endpoints of the API validators
+    #[clap(long)]
+    api_validators: Option<String>,
 
     /// Id of the gateway we want to connect to. If overridden, it is user's responsibility to
     /// ensure prior registration happened
@@ -34,6 +39,15 @@ pub(crate) struct Run {
     #[clap(short, long)]
     port: Option<u16>,
 
+    /// Mostly debug-related option to increase default traffic rate so that you would not need to
+    /// modify config post init
+    #[clap(long, hidden = true)]
+    fastmode: bool,
+
+    /// Disable loop cover traffic and the Poisson rate limiter (for debugging only)
+    #[clap(long, hidden = true)]
+    no_cover: bool,
+
     /// Set this client to work in a enabled credentials mode that would attempt to use gateway
     /// with bandwidth credential requirement.
     #[cfg(feature = "coconut")]
@@ -44,10 +58,12 @@ pub(crate) struct Run {
 impl From<Run> for OverrideConfig {
     fn from(run_config: Run) -> Self {
         OverrideConfig {
-            validators: run_config.validators,
+            nymd_validators: run_config.nymd_validators,
+            api_validators: run_config.api_validators,
             disable_socket: run_config.disable_socket,
             port: run_config.port,
-            fastmode: false,
+            fastmode: run_config.fastmode,
+            no_cover: run_config.no_cover,
             #[cfg(feature = "coconut")]
             enabled_credentials_mode: run_config.enabled_credentials_mode,
         }
@@ -73,14 +89,14 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) {
+pub(crate) async fn execute(args: &Run) -> Result<(), ClientError> {
     let id = &args.id;
 
     let mut config = match Config::load_from_file(Some(id)) {
         Ok(cfg) => cfg,
         Err(err) => {
             error!("Failed to load config for {}. Are you sure you have run `init` before? (Error was: {})", id, err);
-            return;
+            return Err(ClientError::FailedToLoadConfig(id.to_string()));
         }
     };
 
@@ -89,8 +105,8 @@ pub(crate) async fn execute(args: &Run) {
 
     if !version_check(&config) {
         error!("failed the local version check");
-        return;
+        return Err(ClientError::FailedLocalVersionCheck);
     }
 
-    NymClient::new(config).run_forever().await;
+    SocketClient::new(config).run_socket_forever().await
 }

clients/native/src/error.rs

@@ -0,0 +1,17 @@
+use client_core::error::ClientCoreError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum ClientError {
+    #[error("I/O error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("client-core error: {0}")]
+    ClientCoreError(#[from] ClientCoreError),
+
+    #[error("Failed to load config for: {0}")]
+    FailedToLoadConfig(String),
+    #[error("Failed local version check, client and config mismatch")]
+    FailedLocalVersionCheck,
+
+    #[error("Attempted to start the client in invalid socket mode")]
+    InvalidSocketMode,
+}

clients/native/src/lib.rs

@@ -2,4 +2,5 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod client;
+pub mod error;
 pub mod websocket;

clients/native/src/main.rs

@@ -2,20 +2,23 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use clap::{crate_version, Parser};
+use error::ClientError;
+use logging::setup_logging;
 use network_defaults::setup_env;
 
 pub mod client;
 pub mod commands;
+pub mod error;
 pub mod websocket;
 
 #[tokio::main]
-async fn main() {
+async fn main() -> Result<(), ClientError> {
     setup_logging();
     println!("{}", banner());
 
     let args = commands::Cli::parse();
     setup_env(args.config_env_file.clone());
-    commands::execute(&args).await;
+    commands::execute(&args).await
 }
 
 fn banner() -> String {
@@ -34,25 +37,3 @@ fn banner() -> String {
         crate_version!()
     )
 }
-
-fn setup_logging() {
-    let mut log_builder = pretty_env_logger::formatted_timed_builder();
-    if let Ok(s) = ::std::env::var("RUST_LOG") {
-        log_builder.parse_filters(&s);
-    } else {
-        // default to 'Info'
-        log_builder.filter(None, log::LevelFilter::Info);
-    }
-
-    log_builder
-        .filter_module("hyper", log::LevelFilter::Warn)
-        .filter_module("tokio_reactor", log::LevelFilter::Warn)
-        .filter_module("reqwest", log::LevelFilter::Warn)
-        .filter_module("mio", log::LevelFilter::Warn)
-        .filter_module("want", log::LevelFilter::Warn)
-        .filter_module("tungstenite", log::LevelFilter::Warn)
-        .filter_module("tokio_tungstenite", log::LevelFilter::Warn)
-        .filter_module("handlebars", log::LevelFilter::Warn)
-        .filter_module("sled", log::LevelFilter::Warn)
-        .init();
-}

clients/native/src/websocket/handler.rs

@@ -1,6 +1,9 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use client_connections::{
+    ConnectionCommand, ConnectionCommandSender, LaneQueueLengths, TransmissionLane,
+};
 use client_core::client::{
     inbound_messages::{InputMessage, InputMessageSender},
     received_buffer::{
@@ -34,10 +37,12 @@ impl Default for ReceivedResponseType {
 
 pub(crate) struct Handler {
     msg_input: InputMessageSender,
+    client_connection_tx: ConnectionCommandSender,
     buffer_requester: ReceivedBufferRequestSender,
     self_full_address: Recipient,
     socket: Option<WebSocketStream<TcpStream>>,
     received_response_type: ReceivedResponseType,
+    lane_queue_lengths: LaneQueueLengths,
 }
 
 // clone is used to use handler on a new connection, which initially is `None`
@@ -45,10 +50,12 @@ impl Clone for Handler {
     fn clone(&self) -> Self {
         Handler {
             msg_input: self.msg_input.clone(),
+            client_connection_tx: self.client_connection_tx.clone(),
             buffer_requester: self.buffer_requester.clone(),
             self_full_address: self.self_full_address,
             socket: None,
             received_response_type: Default::default(),
+            lane_queue_lengths: self.lane_queue_lengths.clone(),
         }
     }
 }
@@ -64,38 +71,85 @@ impl Drop for Handler {
 impl Handler {
     pub(crate) fn new(
         msg_input: InputMessageSender,
+        client_connection_tx: ConnectionCommandSender,
         buffer_requester: ReceivedBufferRequestSender,
         self_full_address: Recipient,
+        lane_queue_lengths: LaneQueueLengths,
     ) -> Self {
         Handler {
             msg_input,
+            client_connection_tx,
             buffer_requester,
             self_full_address,
             socket: None,
             received_response_type: Default::default(),
+            lane_queue_lengths,
         }
     }
 
-    fn handle_send(
+    async fn handle_send(
         &mut self,
-        recipient: Recipient,
+        recipient: &Recipient,
         message: Vec<u8>,
         with_reply_surb: bool,
+        connection_id: Option<u64>,
     ) -> Option<ServerResponse> {
-        // the ack control is now responsible for chunking, etc.
-        let input_msg = InputMessage::new_fresh(recipient, message, with_reply_surb);
-        self.msg_input.unbounded_send(input_msg).unwrap();
+        // We map the absence of a connection id as going into the general lane.
+        let lane = connection_id.map_or(TransmissionLane::General, |id| {
+            TransmissionLane::ConnectionId(id)
+        });
 
-        None
+        // the ack control is now responsible for chunking, etc.
+        let input_msg = InputMessage::new_fresh(*recipient, message, with_reply_surb, lane);
+        self.msg_input
+            .send(input_msg)
+            .await
+            .expect("InputMessageReceiver has stopped receiving!");
+
+        // Only reply back with a `LaneQueueLength` if the sender providided a connection id
+        let connection_id = match lane {
+            TransmissionLane::General
+            | TransmissionLane::Reply
+            | TransmissionLane::Retransmission
+            | TransmissionLane::Control => return None,
+            TransmissionLane::ConnectionId(id) => id,
+        };
+
+        // on receiving a send, we reply back the current lane queue length for that connection id.
+        // Note that this does _NOT_ take into account the packets that have been received but not
+        // yet reach `OutQueueControl`, so it might be a tad low.
+        let Ok(lane_queue_lengths) = self.lane_queue_lengths.lock() else {
+            log::warn!(
+                "Failed to get the lane queue length lock, \
+                not responding back with the current queue length"
+            );
+            return None;
+        };
+
+        let queue_length = lane_queue_lengths.get(&lane).unwrap_or(0);
+        Some(ServerResponse::LaneQueueLength(connection_id, queue_length))
     }
 
-    fn handle_reply(&mut self, reply_surb: ReplySurb, message: Vec<u8>) -> Option<ServerResponse> {
+    async fn handle_reply(
+        &mut self,
+        reply_surb: ReplySurb,
+        message: Vec<u8>,
+    ) -> Option<ServerResponse> {
         if message.len() > ReplySurb::max_msg_len(Default::default()) {
-            return Some(ServerResponse::new_error(format!("too long message to put inside a reply SURB. Received: {} bytes and maximum is {} bytes", message.len(), ReplySurb::max_msg_len(Default::default()))));
+            return Some(
+                ServerResponse::new_error(
+                    format!(
+                        "too long message to put inside a reply SURB. Received: {} bytes and maximum is {} bytes",
+                        message.len(), ReplySurb::max_msg_len(Default::default()))
+                    )
+                );
         }
 
         let input_msg = InputMessage::new_reply(reply_surb, message);
-        self.msg_input.unbounded_send(input_msg).unwrap();
+        self.msg_input
+            .send(input_msg)
+            .await
+            .expect("InputMessageReceiver has stopped receiving!");
 
         None
     }
@@ -104,22 +158,48 @@ impl Handler {
         ServerResponse::SelfAddress(self.self_full_address)
     }
 
-    fn handle_request(&mut self, request: ClientRequest) -> Option<ServerResponse> {
+    fn handle_closed_connection(&self, connection_id: u64) -> Option<ServerResponse> {
+        self.client_connection_tx
+            .unbounded_send(ConnectionCommand::Close(connection_id))
+            .unwrap();
+        None
+    }
+
+    fn handle_get_lane_queue_length(&self, connection_id: u64) -> Option<ServerResponse> {
+        let Ok(lane_queue_lengths) = self.lane_queue_lengths.lock() else {
+            log::warn!(
+                "Failed to get the lane queue length lock, not responding back with the current queue length"
+            );
+            return None;
+        };
+
+        let lane = TransmissionLane::ConnectionId(connection_id);
+        let queue_length = lane_queue_lengths.get(&lane).unwrap_or(0);
+        Some(ServerResponse::LaneQueueLength(connection_id, queue_length))
+    }
+
+    async fn handle_request(&mut self, request: ClientRequest) -> Option<ServerResponse> {
         match request {
             ClientRequest::Send {
                 recipient,
                 message,
                 with_reply_surb,
-            } => self.handle_send(recipient, message, with_reply_surb),
+                connection_id,
+            } => {
+                self.handle_send(&recipient, message, with_reply_surb, connection_id)
+                    .await
+            }
             ClientRequest::Reply {
                 message,
                 reply_surb,
-            } => self.handle_reply(reply_surb, message),
+            } => self.handle_reply(reply_surb, message).await,
             ClientRequest::SelfAddress => Some(self.handle_self_address()),
+            ClientRequest::ClosedConnection(id) => self.handle_closed_connection(id),
+            ClientRequest::GetLaneQueueLength(id) => self.handle_get_lane_queue_length(id),
         }
     }
 
-    fn handle_text_message(&mut self, msg: String) -> Option<WsMessage> {
+    async fn handle_text_message(&mut self, msg: String) -> Option<WsMessage> {
         debug!("Handling text message request");
         trace!("Content: {:?}", msg);
 
@@ -128,13 +208,13 @@ impl Handler {
 
         let response = match client_request {
             Err(err) => Some(ServerResponse::Error(err)),
-            Ok(req) => self.handle_request(req),
+            Ok(req) => self.handle_request(req).await,
         };
 
         response.map(|resp| WsMessage::text(resp.into_text()))
     }
 
-    fn handle_binary_message(&mut self, msg: Vec<u8>) -> Option<WsMessage> {
+    async fn handle_binary_message(&mut self, msg: &[u8]) -> Option<WsMessage> {
         debug!("Handling binary message request");
 
         self.received_response_type = ReceivedResponseType::Binary;
@@ -142,49 +222,23 @@ impl Handler {
 
         let response = match client_request {
             Err(err) => Some(ServerResponse::Error(err)),
-            Ok(req) => self.handle_request(req),
+            Ok(req) => self.handle_request(req).await,
         };
 
         response.map(|resp| WsMessage::Binary(resp.into_binary()))
     }
 
-    fn handle_ws_request(&mut self, raw_request: WsMessage) -> Option<WsMessage> {
+    async fn handle_ws_request(&mut self, raw_request: WsMessage) -> Option<WsMessage> {
         // apparently tungstenite auto-handles ping/pong/close messages so for now let's ignore
         // them and let's test that claim. If that's not the case, just copy code from
         // old version of this file.
         match raw_request {
-            WsMessage::Text(text_message) => self.handle_text_message(text_message),
-            WsMessage::Binary(binary_message) => self.handle_binary_message(binary_message),
+            WsMessage::Text(text_message) => self.handle_text_message(text_message).await,
+            WsMessage::Binary(binary_message) => self.handle_binary_message(&binary_message).await,
             _ => None,
         }
     }
 
-    // I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
-    // let's just play along for now
-    fn prepare_reconstructed_binary(
-        &self,
-        reconstructed_messages: Vec<ReconstructedMessage>,
-    ) -> Vec<Result<WsMessage, WsError>> {
-        reconstructed_messages
-            .into_iter()
-            .map(ServerResponse::Received)
-            .map(|resp| Ok(WsMessage::Binary(resp.into_binary())))
-            .collect()
-    }
-
-    // I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
-    // let's just play along for now
-    fn prepare_reconstructed_text(
-        &self,
-        reconstructed_messages: Vec<ReconstructedMessage>,
-    ) -> Vec<Result<WsMessage, WsError>> {
-        reconstructed_messages
-            .into_iter()
-            .map(ServerResponse::Received)
-            .map(|resp| Ok(WsMessage::Text(resp.into_text())))
-            .collect()
-    }
-
     async fn push_websocket_received_plaintexts(
         &mut self,
         reconstructed_messages: Vec<ReconstructedMessage>,
@@ -193,10 +247,8 @@ impl Handler {
         // if it's text or binary, but for time being we use the naive assumption that if
         // client is sending Message::Text it expects text back. Same for Message::Binary
         let response_messages = match self.received_response_type {
-            ReceivedResponseType::Binary => {
-                self.prepare_reconstructed_binary(reconstructed_messages)
-            }
-            ReceivedResponseType::Text => self.prepare_reconstructed_text(reconstructed_messages),
+            ReceivedResponseType::Binary => prepare_reconstructed_binary(reconstructed_messages),
+            ReceivedResponseType::Text => prepare_reconstructed_text(reconstructed_messages),
         };
 
         let mut send_stream = futures::stream::iter(response_messages);
@@ -244,7 +296,7 @@ impl Handler {
                         break;
                     }
 
-                    if let Some(response) = self.handle_ws_request(socket_msg) {
+                    if let Some(response) = self.handle_ws_request(socket_msg).await {
                         if let Err(err) = self.send_websocket_response(response).await {
                             warn!(
                                 "Failed to send message over websocket: {}. Assuming the connection is dead.",
@@ -291,3 +343,27 @@ impl Handler {
         self.listen_for_requests(reconstructed_receiver).await;
     }
 }
+
+// I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
+// let's just play along for now
+fn prepare_reconstructed_binary(
+    reconstructed_messages: Vec<ReconstructedMessage>,
+) -> Vec<Result<WsMessage, WsError>> {
+    reconstructed_messages
+        .into_iter()
+        .map(ServerResponse::Received)
+        .map(|resp| Ok(WsMessage::Binary(resp.into_binary())))
+        .collect()
+}
+
+// I'm still not entirely sure why `send_all` requires `TryStream` rather than `Stream`, but
+// let's just play along for now
+fn prepare_reconstructed_text(
+    reconstructed_messages: Vec<ReconstructedMessage>,
+) -> Vec<Result<WsMessage, WsError>> {
+    reconstructed_messages
+        .into_iter()
+        .map(ServerResponse::Received)
+        .map(|resp| Ok(WsMessage::Text(resp.into_text())))
+        .collect()
+}

clients/native/websocket-requests/src/requests.rs

@@ -20,6 +20,12 @@ pub const REPLY_REQUEST_TAG: u8 = 0x01;
 /// Value tag representing [`SelfAddress`] variant of the [`ClientRequest`]
 pub const SELF_ADDRESS_REQUEST_TAG: u8 = 0x02;
 
+/// Value tag representing [`ClosedConnection`] variant of the [`ClientRequest`]
+pub const CLOSED_CONNECTION_REQUEST_TAG: u8 = 0x03;
+
+/// Value tag representing [`GetLaneQueueLength`] variant of the [`ClientRequest`]
+pub const GET_LANE_QUEUE_LENGHT_TAG: u8 = 0x04;
+
 #[allow(non_snake_case)]
 #[derive(Debug)]
 pub enum ClientRequest {
@@ -28,32 +34,42 @@ pub enum ClientRequest {
         message: Vec<u8>,
         // Perhaps we could change it to a number to indicate how many reply_SURBs we want to include?
         with_reply_surb: bool,
+        connection_id: Option<u64>,
     },
     Reply {
         message: Vec<u8>,
         reply_surb: ReplySurb,
     },
     SelfAddress,
+    ClosedConnection(u64),
+    GetLaneQueueLength(u64),
 }
 
 // we could have been parsing it directly TryFrom<WsMessage>, but we want to retain
 // information about whether it came from binary or text to send appropriate response back
 impl ClientRequest {
-    // SEND_REQUEST_TAG || with_surb || recipient || data_len || data
-    fn serialize_send(recipient: Recipient, data: Vec<u8>, with_reply_surb: bool) -> Vec<u8> {
+    // SEND_REQUEST_TAG || with_surb || recipient || conn_id || data_len || data
+    fn serialize_send(
+        recipient: Recipient,
+        data: Vec<u8>,
+        with_reply_surb: bool,
+        connection_id: Option<u64>,
+    ) -> Vec<u8> {
         let data_len_bytes = (data.len() as u64).to_be_bytes();
+        let conn_id_bytes = connection_id.unwrap_or(0).to_be_bytes();
         std::iter::once(SEND_REQUEST_TAG)
             .chain(std::iter::once(with_reply_surb as u8))
             .chain(recipient.to_bytes().iter().cloned()) // will not be length prefixed because the length is constant
+            .chain(conn_id_bytes.iter().cloned())
             .chain(data_len_bytes.iter().cloned())
             .chain(data.into_iter())
             .collect()
     }
 
-    // SEND_REQUEST_TAG || with_reply || recipient || data_len || data
+    // SEND_REQUEST_TAG || with_reply || recipient || conn_id || data_len || data
     fn deserialize_send(b: &[u8]) -> Result<Self, error::Error> {
-        // we need to have at least 1 (tag) + 1 (reply flag) + Recipient::LEN + sizeof<u64> bytes
-        if b.len() < 2 + Recipient::LEN + size_of::<u64>() {
+        // we need to have at least 1 (tag) + 1 (reply flag) + Recipient::LEN + 2*sizeof<u64> bytes
+        if b.len() < 2 + Recipient::LEN + 2 * size_of::<u64>() {
             return Err(error::Error::new(
                 ErrorKind::TooShortRequest,
                 "not enough data provided to recover 'send'".to_string(),
@@ -86,9 +102,20 @@ impl ClientRequest {
             }
         };
 
-        let data_len_bytes = &b[2 + Recipient::LEN..2 + Recipient::LEN + size_of::<u64>()];
+        let mut connection_id_bytes = [0u8; size_of::<u64>()];
+        connection_id_bytes
+            .copy_from_slice(&b[2 + Recipient::LEN..2 + Recipient::LEN + size_of::<u64>()]);
+        let connection_id = u64::from_be_bytes(connection_id_bytes);
+        let connection_id = if connection_id == 0 {
+            None
+        } else {
+            Some(connection_id)
+        };
+
+        let data_len_bytes =
+            &b[2 + Recipient::LEN + size_of::<u64>()..2 + Recipient::LEN + 2 * size_of::<u64>()];
         let data_len = u64::from_be_bytes(data_len_bytes.try_into().unwrap());
-        let data = &b[2 + Recipient::LEN + size_of::<u64>()..];
+        let data = &b[2 + Recipient::LEN + 2 * size_of::<u64>()..];
         if data.len() as u64 != data_len {
             return Err(error::Error::new(
                 ErrorKind::MalformedRequest,
@@ -104,11 +131,12 @@ impl ClientRequest {
             with_reply_surb,
             recipient,
             message: data.to_vec(),
+            connection_id,
         })
     }
 
     // REPLY_REQUEST_TAG || surb_len || surb || message_len || message
-    fn serialize_reply(message: Vec<u8>, reply_surb: ReplySurb) -> Vec<u8> {
+    fn serialize_reply(message: Vec<u8>, reply_surb: &ReplySurb) -> Vec<u8> {
         let reply_surb_bytes = reply_surb.to_bytes();
         let surb_len_bytes = (reply_surb_bytes.len() as u64).to_be_bytes();
         let message_len_bytes = (message.len() as u64).to_be_bytes();
@@ -202,20 +230,79 @@ impl ClientRequest {
         ClientRequest::SelfAddress
     }
 
+    // CLOSED_CONNECTION_REQUEST_TAG
+    fn serialize_closed_connection(connection_id: u64) -> Vec<u8> {
+        let conn_id_bytes = connection_id.to_be_bytes();
+        std::iter::once(CLOSED_CONNECTION_REQUEST_TAG)
+            .chain(conn_id_bytes.iter().copied())
+            .collect()
+    }
+
+    // CLOSED_CONNECTION_REQUEST_TAG
+    fn deserialize_closed_connection(b: &[u8]) -> Result<Self, error::Error> {
+        if b.len() != 1 + size_of::<u64>() {
+            return Err(error::Error::new(
+                ErrorKind::MalformedRequest,
+                "the received closed connection has invalid length".to_string(),
+            ));
+        }
+
+        // this MUST match because it was called by 'deserialize'
+        debug_assert_eq!(b[0], CLOSED_CONNECTION_REQUEST_TAG);
+
+        let mut connection_id_bytes = [0u8; size_of::<u64>()];
+        connection_id_bytes.copy_from_slice(&b[1..=size_of::<u64>()]);
+        let connection_id = u64::from_be_bytes(connection_id_bytes);
+
+        Ok(ClientRequest::ClosedConnection(connection_id))
+    }
+
+    // GET_LANE_QUEUE_LENGHT_TAG
+    fn serialize_get_lane_queue_lengths(connection_id: u64) -> Vec<u8> {
+        let conn_id_bytes = connection_id.to_be_bytes();
+        std::iter::once(GET_LANE_QUEUE_LENGHT_TAG)
+            .chain(conn_id_bytes.iter().copied())
+            .collect()
+    }
+
+    // GET_LANE_QUEUE_LENGHT_TAG
+    fn deserialize_get_lane_queue_length(b: &[u8]) -> Result<Self, error::Error> {
+        if b.len() != 1 + size_of::<u64>() {
+            return Err(error::Error::new(
+                ErrorKind::MalformedRequest,
+                "the received get lane queue length has invalid length".to_string(),
+            ));
+        }
+
+        // this MUST match because it was called by 'deserialize'
+        debug_assert_eq!(b[0], GET_LANE_QUEUE_LENGHT_TAG);
+
+        let mut connection_id_bytes = [0u8; size_of::<u64>()];
+        connection_id_bytes.copy_from_slice(&b[1..=size_of::<u64>()]);
+        let connection_id = u64::from_be_bytes(connection_id_bytes);
+
+        Ok(ClientRequest::GetLaneQueueLength(connection_id))
+    }
+
     pub fn serialize(self) -> Vec<u8> {
         match self {
             ClientRequest::Send {
                 recipient,
                 message,
                 with_reply_surb,
-            } => Self::serialize_send(recipient, message, with_reply_surb),
+                connection_id,
+            } => Self::serialize_send(recipient, message, with_reply_surb, connection_id),
 
             ClientRequest::Reply {
                 message,
                 reply_surb,
-            } => Self::serialize_reply(message, reply_surb),
+            } => Self::serialize_reply(message, &reply_surb),
 
             ClientRequest::SelfAddress => Self::serialize_self_address(),
+
+            ClientRequest::ClosedConnection(id) => Self::serialize_closed_connection(id),
+
+            ClientRequest::GetLaneQueueLength(id) => Self::serialize_get_lane_queue_lengths(id),
         }
     }
 
@@ -245,15 +332,17 @@ impl ClientRequest {
             SEND_REQUEST_TAG => Self::deserialize_send(b),
             REPLY_REQUEST_TAG => Self::deserialize_reply(b),
             SELF_ADDRESS_REQUEST_TAG => Ok(Self::deserialize_self_address(b)),
+            CLOSED_CONNECTION_REQUEST_TAG => Self::deserialize_closed_connection(b),
+            GET_LANE_QUEUE_LENGHT_TAG => Self::deserialize_get_lane_queue_length(b),
             n => Err(error::Error::new(
                 ErrorKind::UnknownRequest,
-                format!("type {}", n),
+                format!("type {n}"),
             )),
         }
     }
 
-    pub fn try_from_binary(raw_req: Vec<u8>) -> Result<Self, error::Error> {
-        Self::deserialize(&raw_req)
+    pub fn try_from_binary(raw_req: &[u8]) -> Result<Self, error::Error> {
+        Self::deserialize(raw_req)
     }
 
     pub fn try_from_text(raw_req: String) -> Result<Self, error::Error> {
@@ -280,6 +369,7 @@ mod tests {
             recipient,
             message: b"foomp".to_vec(),
             with_reply_surb: false,
+            connection_id: Some(42),
         };
 
         let bytes = send_request_no_surb.serialize();
@@ -289,10 +379,12 @@ mod tests {
                 recipient,
                 message,
                 with_reply_surb,
+                connection_id,
             } => {
                 assert_eq!(recipient.to_string(), recipient_string);
                 assert_eq!(message, b"foomp".to_vec());
-                assert!(!with_reply_surb)
+                assert!(!with_reply_surb);
+                assert_eq!(connection_id, Some(42))
             }
             _ => unreachable!(),
         }
@@ -301,6 +393,7 @@ mod tests {
             recipient,
             message: b"foomp".to_vec(),
             with_reply_surb: true,
+            connection_id: None,
         };
 
         let bytes = send_request_surb.serialize();
@@ -310,10 +403,12 @@ mod tests {
                 recipient,
                 message,
                 with_reply_surb,
+                connection_id,
             } => {
                 assert_eq!(recipient.to_string(), recipient_string);
                 assert_eq!(message, b"foomp".to_vec());
-                assert!(with_reply_surb)
+                assert!(with_reply_surb);
+                assert_eq!(connection_id, None)
             }
             _ => unreachable!(),
         }
@@ -352,4 +447,15 @@ mod tests {
             _ => unreachable!(),
         }
     }
+
+    #[test]
+    fn close_connection_request_serialization_works() {
+        let close_connection_request = ClientRequest::ClosedConnection(42);
+        let bytes = close_connection_request.serialize();
+        let recovered = ClientRequest::deserialize(&bytes).unwrap();
+        match recovered {
+            ClientRequest::ClosedConnection(id) => assert_eq!(id, 42),
+            _ => unreachable!(),
+        }
+    }
 }

clients/native/websocket-requests/src/responses.rs

@@ -23,10 +23,14 @@ pub const RECEIVED_RESPONSE_TAG: u8 = 0x01;
 /// Value tag representing [`SelfAddress`] variant of the [`ServerResponse`]
 pub const SELF_ADDRESS_RESPONSE_TAG: u8 = 0x02;
 
+/// Value tag representing [`LaneQueueLength`] variant of the [`ServerResponse`]
+pub const LANE_QUEUE_LENGTH_RESPONSE_TAG: u8 = 0x03;
+
 #[derive(Debug)]
 pub enum ServerResponse {
     Received(ReconstructedMessage),
     SelfAddress(Recipient),
+    LaneQueueLength(u64, usize),
     Error(error::Error),
 }
 
@@ -193,6 +197,31 @@ impl ServerResponse {
         Ok(ServerResponse::SelfAddress(recipient))
     }
 
+    // LANE_QUEUE_LENGTH_RESPONSE_TAG || lane || queue_length
+    fn serialize_lane_queue_length(lane: u64, queue_length: usize) -> Vec<u8> {
+        std::iter::once(LANE_QUEUE_LENGTH_RESPONSE_TAG)
+            .chain(lane.to_be_bytes().iter().cloned())
+            .chain(queue_length.to_be_bytes().iter().cloned())
+            .collect()
+    }
+
+    // LANE_QUEUE_LENGTH_RESPONSE_TAG || lane || queue_length
+    fn deserialize_lane_queue_length(b: &[u8]) -> Result<Self, error::Error> {
+        // this MUST match because it was called by 'deserialize'
+        debug_assert_eq!(b[0], LANE_QUEUE_LENGTH_RESPONSE_TAG);
+
+        let mut lane_bytes = [0u8; size_of::<u64>()];
+        lane_bytes.copy_from_slice(&b[1..=size_of::<u64>()]);
+        let lane = u64::from_be_bytes(lane_bytes);
+
+        let mut queue_length_bytes = [0u8; size_of::<usize>()];
+        queue_length_bytes
+            .copy_from_slice(&b[1 + size_of::<u64>()..1 + size_of::<u64>() + size_of::<usize>()]);
+        let queue_length = usize::from_be_bytes(queue_length_bytes);
+
+        Ok(ServerResponse::LaneQueueLength(lane, queue_length))
+    }
+
     // ERROR_RESPONSE_TAG || err_code || msg_len || msg
     fn serialize_error(error: error::Error) -> Vec<u8> {
         let message_len_bytes = (error.message.len() as u64).to_be_bytes();
@@ -272,6 +301,9 @@ impl ServerResponse {
                 Self::serialize_received(reconstructed_message)
             }
             ServerResponse::SelfAddress(address) => Self::serialize_self_address(address),
+            ServerResponse::LaneQueueLength(lane, queue_length) => {
+                Self::serialize_lane_queue_length(lane, queue_length)
+            }
             ServerResponse::Error(err) => Self::serialize_error(err),
         }
     }
@@ -302,6 +334,7 @@ impl ServerResponse {
         match response_tag {
             RECEIVED_RESPONSE_TAG => Self::deserialize_received(b),
             SELF_ADDRESS_RESPONSE_TAG => Self::deserialize_self_address(b),
+            LANE_QUEUE_LENGTH_RESPONSE_TAG => Self::deserialize_lane_queue_length(b),
             ERROR_RESPONSE_TAG => Self::deserialize_error(b),
             n => Err(error::Error::new(
                 ErrorKind::UnknownResponse,
@@ -378,6 +411,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn lane_queue_length_response_serialization_works() {
+        let lane_queue_length_response = ServerResponse::LaneQueueLength(13, 42);
+        let bytes = lane_queue_length_response.serialize();
+        let recovered = ServerResponse::deserialize(&bytes).unwrap();
+        match recovered {
+            ServerResponse::LaneQueueLength(lane, queue_length) => {
+                assert_eq!(lane, 13);
+                assert_eq!(queue_length, 42)
+            }
+            _ => unreachable!(),
+        }
+    }
+
     #[test]
     fn error_response_serialization_works() {
         let dummy_error = error::Error::new(ErrorKind::UnknownRequest, "foomp message".to_string());

clients/native/websocket-requests/src/text.rs

@@ -20,6 +20,7 @@ pub(super) enum ClientRequestText {
         message: String,
         recipient: String,
         with_reply_surb: bool,
+        connection_id: Option<u64>,
     },
     SelfAddress,
     #[serde(rename_all = "camelCase")]
@@ -46,6 +47,7 @@ impl TryInto<ClientRequest> for ClientRequestText {
                 message,
                 recipient,
                 with_reply_surb,
+                connection_id,
             } => {
                 let message_bytes = message.into_bytes();
                 let recipient = Recipient::try_from_base58_string(recipient).map_err(|err| {
@@ -56,6 +58,7 @@ impl TryInto<ClientRequest> for ClientRequestText {
                     message: message_bytes,
                     recipient,
                     with_reply_surb,
+                    connection_id,
                 })
             }
             ClientRequestText::SelfAddress => Ok(ClientRequest::SelfAddress),
@@ -91,6 +94,10 @@ pub(super) enum ServerResponseText {
     SelfAddress {
         address: String,
     },
+    LaneQueueLength {
+        lane: u64,
+        queue_length: usize,
+    },
     Error {
         message: String,
     },
@@ -132,6 +139,9 @@ impl From<ServerResponse> for ServerResponseText {
             ServerResponse::SelfAddress(recipient) => ServerResponseText::SelfAddress {
                 address: recipient.to_string(),
             },
+            ServerResponse::LaneQueueLength(lane, queue_length) => {
+                ServerResponseText::LaneQueueLength { lane, queue_length }
+            }
             ServerResponse::Error(err) => ServerResponseText::Error {
                 message: err.to_string(),
             },

clients/socks5/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nym-socks5-client"
-version = "1.0.2"
+version = "1.1.3"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 description = "A SOCKS5 localhost proxy that converts incoming messages to Sphinx and sends them to a Nym address"
 edition = "2021"
@@ -19,18 +19,22 @@ pin-project = "1.0"
 pretty_env_logger = "0.4"
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
 serde = { version = "1.0", features = ["derive"] } # for config serialization/deserialization
-snafu = "0.6"
-tokio = { version = "1.19.1", features = ["rt-multi-thread", "net", "signal"] }
+serde_json = "1.0.89"
+tap = "1.0.1"
+thiserror = "1.0.34"
+tokio = { version = "1.21.2", features = ["rt-multi-thread", "net", "signal"] }
 url = "2.2"
 
 # internal
 client-core = { path = "../client-core" }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 config = { path = "../../common/config" }
 completions = { path = "../../common/completions" }
 credential-storage = { path = "../../common/credential-storage" }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
+logging = { path = "../../common/logging"}
 gateway-client = { path = "../../common/client-libs/gateway-client" }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 network-defaults = { path = "../../common/network-defaults" }

clients/socks5/src/client/config/mod.rs

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::client::config::template::config_template;
-use client_core::config::Config as BaseConfig;
 pub use client_core::config::MISSING_VALUE;
+use client_core::config::{ClientCoreConfigTrait, Config as BaseConfig};
 use config::defaults::DEFAULT_SOCKS5_LISTENING_PORT;
 use config::NymConfig;
 use nymsphinx::addressing::clients::Recipient;
@@ -52,6 +52,12 @@ impl NymConfig for Config {
     }
 }
 
+impl ClientCoreConfigTrait for Config {
+    fn get_gateway_endpoint(&self) -> &client_core::config::GatewayEndpointConfig {
+        self.base.get_gateway_endpoint()
+    }
+}
+
 impl Config {
     pub fn new<S: Into<String>>(id: S, provider_mix_address: S) -> Self {
         Config {
@@ -60,11 +66,13 @@ impl Config {
         }
     }
 
+    #[must_use]
     pub fn with_port(mut self, port: u16) -> Self {
         self.socks5.listening_port = port;
         self
     }
 
+    #[must_use]
     pub fn with_provider_mix_address(mut self, address: String) -> Self {
         self.socks5.provider_mix_address = address;
         self

clients/socks5/src/client/config/template.rs

@@ -23,6 +23,13 @@ id = '{{ client.id }}'
 # to claim bandwidth without presenting bandwidth credentials.
 disabled_credentials_mode = {{ client.disabled_credentials_mode }}
 
+# Addresses to nymd validators via which the client can communicate with the chain.
+validator_urls = [
+    {{#each client.validator_urls }}
+        '{{this}}',
+    {{/each}}
+]
+
 # Addresses to APIs running on validator from which the client gets the view of the network.
 validator_api_urls = [
     {{#each client.validator_api_urls }}

clients/socks5/src/client/mod.rs

@@ -1,43 +1,22 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use std::sync::atomic::Ordering;
-
-use client_core::client::cover_traffic_stream::LoopCoverTrafficStream;
-use client_core::client::inbound_messages::{
-    InputMessage, InputMessageReceiver, InputMessageSender,
-};
-use client_core::client::key_manager::KeyManager;
-use client_core::client::mix_traffic::{
-    BatchMixMessageReceiver, BatchMixMessageSender, MixTrafficController,
-};
-use client_core::client::real_messages_control::RealMessagesController;
-use client_core::client::received_buffer::{
-    ReceivedBufferRequestReceiver, ReceivedBufferRequestSender, ReceivedMessagesBufferController,
-};
-use client_core::client::reply_key_storage::ReplyKeyStorage;
-use client_core::client::topology_control::{
-    TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
-};
-use client_core::config::persistence::key_pathfinder::ClientKeyPathfinder;
-use crypto::asymmetric::identity;
-use futures::channel::mpsc;
-use futures::StreamExt;
-use gateway_client::bandwidth::BandwidthController;
-use gateway_client::{
-    AcknowledgementReceiver, AcknowledgementSender, GatewayClient, MixnetMessageReceiver,
-    MixnetMessageSender,
-};
-use log::*;
-use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::addressing::nodes::NodeIdentity;
-use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
-
 use crate::client::config::Config;
+use crate::error::Socks5ClientError;
 use crate::socks::{
     authentication::{AuthenticationMethods, Authenticator, User},
     server::SphinxSocksServer,
 };
+use client_core::client::base_client::{BaseClientBuilder, ClientInput, ClientOutput};
+use client_core::client::key_manager::KeyManager;
+use client_core::config::persistence::key_pathfinder::ClientKeyPathfinder;
+use futures::channel::mpsc;
+use futures::StreamExt;
+use gateway_client::bandwidth::BandwidthController;
+use log::*;
+use nymsphinx::addressing::clients::Recipient;
+use std::error::Error;
+use task::{wait_for_signal_and_error, ShutdownListener, ShutdownNotifier};
 
 pub mod config;
 
@@ -71,238 +50,112 @@ impl NymClient {
         }
     }
 
-    pub fn as_mix_recipient(&self) -> Recipient {
-        Recipient::new(
-            *self.key_manager.identity_keypair().public_key(),
-            *self.key_manager.encryption_keypair().public_key(),
-            // TODO: below only works under assumption that gateway address == gateway id
-            // (which currently is true)
-            NodeIdentity::from_base58_string(self.config.get_base().get_gateway_id()).unwrap(),
-        )
-    }
-
-    // future constantly pumping loop cover traffic at some specified average rate
-    // the pumped traffic goes to the MixTrafficController
-    fn start_cover_traffic_stream(
-        &self,
-        topology_accessor: TopologyAccessor,
-        mix_tx: BatchMixMessageSender,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting loop cover traffic stream...");
-
-        LoopCoverTrafficStream::new(
-            self.key_manager.ack_key(),
-            self.config.get_base().get_average_ack_delay(),
-            self.config.get_base().get_average_packet_delay(),
-            self.config
-                .get_base()
-                .get_loop_cover_traffic_average_delay(),
-            mix_tx,
-            self.as_mix_recipient(),
-            topology_accessor,
-            shutdown,
-        )
-        .start();
-    }
-
-    fn start_real_traffic_controller(
-        &self,
-        topology_accessor: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
-        ack_receiver: AcknowledgementReceiver,
-        input_receiver: InputMessageReceiver,
-        mix_sender: BatchMixMessageSender,
-        shutdown: ShutdownListener,
-    ) {
-        let controller_config = client_core::client::real_messages_control::Config::new(
-            self.key_manager.ack_key(),
-            self.config.get_base().get_ack_wait_multiplier(),
-            self.config.get_base().get_ack_wait_addition(),
-            self.config.get_base().get_average_ack_delay(),
-            self.config.get_base().get_message_sending_average_delay(),
-            self.config.get_base().get_average_packet_delay(),
-            self.as_mix_recipient(),
-        );
-
-        info!("Starting real traffic stream...");
-
-        RealMessagesController::new(
-            controller_config,
-            ack_receiver,
-            input_receiver,
-            mix_sender,
-            topology_accessor,
-            reply_key_storage,
-            shutdown,
-        )
-        .start();
-    }
-
-    // buffer controlling all messages fetched from provider
-    // required so that other components would be able to use them (say the websocket)
-    fn start_received_messages_buffer_controller(
-        &self,
-        query_receiver: ReceivedBufferRequestReceiver,
-        mixnet_receiver: MixnetMessageReceiver,
-        reply_key_storage: ReplyKeyStorage,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting received messages buffer controller...");
-        ReceivedMessagesBufferController::new(
-            self.key_manager.encryption_keypair(),
-            query_receiver,
-            mixnet_receiver,
-            reply_key_storage,
-            shutdown,
-        )
-        .start()
-    }
-
-    async fn start_gateway_client(
-        &mut self,
-        mixnet_message_sender: MixnetMessageSender,
-        ack_sender: AcknowledgementSender,
-        shutdown: ShutdownListener,
-    ) -> GatewayClient {
-        let gateway_id = self.config.get_base().get_gateway_id();
-        if gateway_id.is_empty() {
-            panic!("The identity of the gateway is unknown - did you run `nym-client` init?")
-        }
-        let gateway_owner = self.config.get_base().get_gateway_owner();
-        if gateway_owner.is_empty() {
-            panic!("The owner of the gateway is unknown - did you run `nym-client` init?")
-        }
-        let gateway_address = self.config.get_base().get_gateway_listener();
-        if gateway_address.is_empty() {
-            panic!("The address of the gateway is unknown - did you run `nym-client` init?")
-        }
-
-        let gateway_identity = identity::PublicKey::from_base58_string(gateway_id)
-            .expect("provided gateway id is invalid!");
-
+    async fn create_bandwidth_controller(config: &Config) -> BandwidthController {
         #[cfg(feature = "coconut")]
-        let bandwidth_controller = BandwidthController::new(
-            credential_storage::initialise_storage(self.config.get_base().get_database_path())
-                .await,
-            self.config.get_base().get_validator_api_endpoints(),
-        );
+        let bandwidth_controller = {
+            let details = network_defaults::NymNetworkDetails::new_from_env();
+            let mut client_config =
+                validator_client::Config::try_from_nym_network_details(&details)
+                    .expect("failed to construct validator client config");
+            let nymd_url = config
+                .get_base()
+                .get_validator_endpoints()
+                .pop()
+                .expect("No nymd validator endpoint provided");
+            let api_url = config
+                .get_base()
+                .get_validator_api_endpoints()
+                .pop()
+                .expect("No validator api endpoint provided");
+            // overwrite env configuration with config URLs
+            client_config = client_config.with_urls(nymd_url, api_url);
+            let client = validator_client::Client::new_query(client_config)
+                .expect("Could not construct query client");
+            let coconut_api_clients =
+                validator_client::CoconutApiClient::all_coconut_api_clients(&client)
+                    .await
+                    .expect("Could not query api clients");
+            BandwidthController::new(
+                credential_storage::initialise_storage(config.get_base().get_database_path()).await,
+                coconut_api_clients,
+            )
+        };
         #[cfg(not(feature = "coconut"))]
         let bandwidth_controller = BandwidthController::new(
-            credential_storage::initialise_storage(self.config.get_base().get_database_path())
-                .await,
+            credential_storage::initialise_storage(config.get_base().get_database_path()).await,
         )
         .expect("Could not create bandwidth controller");
-
-        let mut gateway_client = GatewayClient::new(
-            gateway_address,
-            self.key_manager.identity_keypair(),
-            gateway_identity,
-            gateway_owner,
-            Some(self.key_manager.gateway_shared_key()),
-            mixnet_message_sender,
-            ack_sender,
-            self.config.get_base().get_gateway_response_timeout(),
-            Some(bandwidth_controller),
-            Some(shutdown),
-        );
-
-        gateway_client
-            .set_disabled_credentials_mode(self.config.get_base().get_disabled_credentials_mode());
-
-        gateway_client
-            .authenticate_and_start()
-            .await
-            .expect("could not authenticate and start up the gateway connection");
-
-        gateway_client
-    }
-
-    // future responsible for periodically polling directory server and updating
-    // the current global view of topology
-    async fn start_topology_refresher(
-        &mut self,
-        topology_accessor: TopologyAccessor,
-        shutdown: ShutdownListener,
-    ) {
-        let topology_refresher_config = TopologyRefresherConfig::new(
-            self.config.get_base().get_validator_api_endpoints(),
-            self.config.get_base().get_topology_refresh_rate(),
-            env!("CARGO_PKG_VERSION").to_string(),
-        );
-        let mut topology_refresher =
-            TopologyRefresher::new(topology_refresher_config, topology_accessor);
-        // before returning, block entire runtime to refresh the current network view so that any
-        // components depending on topology would see a non-empty view
-        info!("Obtaining initial network topology");
-        topology_refresher.refresh().await;
-
-        // TODO: a slightly more graceful termination here
-        if !topology_refresher.is_topology_routable().await {
-            panic!(
-                "The current network topology seem to be insufficient to route any packets through\
-                - check if enough nodes and a gateway are online"
-            );
-        }
-
-        info!("Starting topology refresher...");
-        topology_refresher.start(shutdown);
-    }
-
-    // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
-    // TODO: if we want to send control messages to gateway_client, this CAN'T take the ownership
-    // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
-    // requests?
-    fn start_mix_traffic_controller(
-        &mut self,
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-        shutdown: ShutdownListener,
-    ) {
-        info!("Starting mix traffic controller...");
-        MixTrafficController::new(mix_rx, gateway_client, shutdown).start();
+        bandwidth_controller
     }
 
     fn start_socks5_listener(
-        &self,
-        buffer_requester: ReceivedBufferRequestSender,
-        msg_input: InputMessageSender,
+        config: &Config,
+        client_input: ClientInput,
+        client_output: ClientOutput,
+        self_address: Recipient,
         shutdown: ShutdownListener,
     ) {
         info!("Starting socks5 listener...");
         let auth_methods = vec![AuthenticationMethods::NoAuth as u8];
         let allowed_users: Vec<User> = Vec::new();
 
+        let ClientInput {
+            shared_lane_queue_lengths,
+            connection_command_sender,
+            input_sender,
+        } = client_input;
+
+        let received_buffer_request_sender = client_output.received_buffer_request_sender;
+
         let authenticator = Authenticator::new(auth_methods, allowed_users);
         let mut sphinx_socks = SphinxSocksServer::new(
-            self.config.get_listening_port(),
+            config.get_listening_port(),
             authenticator,
-            self.config.get_provider_mix_address(),
-            self.as_mix_recipient(),
+            config.get_provider_mix_address(),
+            self_address,
+            shared_lane_queue_lengths,
+            shutdown.clone(),
+        );
+        task::spawn_with_report_error(
+            async move {
+                sphinx_socks
+                    .serve(
+                        input_sender,
+                        received_buffer_request_sender,
+                        connection_command_sender,
+                    )
+                    .await
+            },
             shutdown,
         );
-        tokio::spawn(async move { sphinx_socks.serve(msg_input, buffer_requester).await });
     }
 
     /// blocking version of `start` method. Will run forever (or until SIGINT is sent)
-    pub async fn run_forever(&mut self) {
-        let mut shutdown = self.start().await;
-        wait_for_signal().await;
+    pub async fn run_forever(self) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let mut shutdown = self.start().await?;
+
+        let res = wait_for_signal_and_error(&mut shutdown).await;
 
         log::info!("Sending shutdown");
-        client_core::client::SHUTDOWN_HAS_BEEN_SIGNALLED.store(true, Ordering::Relaxed);
         shutdown.signal_shutdown().ok();
 
         log::info!("Waiting for tasks to finish... (Press ctrl-c to force)");
         shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-socks5-client");
+        res
     }
 
     // Variant of `run_forever` that listends for remote control messages
-    pub async fn run_and_listen(&mut self, mut receiver: Socks5ControlMessageReceiver) {
-        let mut shutdown = self.start().await;
-        tokio::select! {
+    pub async fn run_and_listen(
+        self,
+        mut receiver: Socks5ControlMessageReceiver,
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        // Start the main task
+        let mut shutdown = self.start().await?;
+
+        let res = tokio::select! {
+            biased;
             message = receiver.next() => {
                 log::debug!("Received message: {:?}", message);
                 match message {
@@ -313,98 +166,51 @@ impl NymClient {
                         log::info!("Channel closed, stopping");
                     }
                 }
+                Ok(())
+            }
+            Some(msg) = shutdown.wait_for_error() => {
+                log::info!("Task error: {:?}", msg);
+                Err(msg)
             }
             _ = tokio::signal::ctrl_c() => {
                 log::info!("Received SIGINT");
+                Ok(())
             },
-        }
+        };
 
         log::info!("Sending shutdown");
-        client_core::client::SHUTDOWN_HAS_BEEN_SIGNALLED.store(true, Ordering::Relaxed);
         shutdown.signal_shutdown().ok();
 
         log::info!("Waiting for tasks to finish... (Press ctrl-c to force)");
         shutdown.wait_for_shutdown().await;
 
         log::info!("Stopping nym-socks5-client");
+        res
     }
 
-    pub async fn start(&mut self) -> ShutdownNotifier {
-        info!("Starting nym client");
-        // channels for inter-component communication
-        // TODO: make the channels be internally created by the relevant components
-        // rather than creating them here, so say for example the buffer controller would create the request channels
-        // and would allow anyone to clone the sender channel
-
-        // sphinx_message_sender is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
-        // they are used by cover traffic stream and real traffic stream
-        // sphinx_message_receiver is the receiver used by MixTrafficController that sends the actual traffic
-        let (sphinx_message_sender, sphinx_message_receiver) = mpsc::unbounded();
-
-        // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
-        // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
-        let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
-
-        // used for announcing connection or disconnection of a channel for pushing re-assembled messages to
-        let (received_buffer_request_sender, received_buffer_request_receiver) = mpsc::unbounded();
-
-        // channels responsible for controlling real messages
-        let (input_sender, input_receiver) = mpsc::unbounded::<InputMessage>();
-
-        // channels responsible for controlling ack messages
-        let (ack_sender, ack_receiver) = mpsc::unbounded();
-        let shared_topology_accessor = TopologyAccessor::new();
-
-        let reply_key_storage =
-            ReplyKeyStorage::load(self.config.get_base().get_reply_encryption_key_store_path())
-                .expect("Failed to load reply key storage!");
-
-        // Shutdown notifier for signalling tasks to stop
-        let shutdown = ShutdownNotifier::default();
-
-        // the components are started in very specific order. Unless you know what you are doing,
-        // do not change that.
-        self.start_topology_refresher(shared_topology_accessor.clone(), shutdown.subscribe())
-            .await;
-        self.start_received_messages_buffer_controller(
-            received_buffer_request_receiver,
-            mixnet_messages_receiver,
-            reply_key_storage.clone(),
-            shutdown.subscribe(),
+    pub async fn start(self) -> Result<ShutdownNotifier, Socks5ClientError> {
+        let base_builder = BaseClientBuilder::new_from_base_config(
+            self.config.get_base(),
+            self.key_manager,
+            Some(Self::create_bandwidth_controller(&self.config).await),
         );
 
-        let gateway_client = self
-            .start_gateway_client(mixnet_messages_sender, ack_sender, shutdown.subscribe())
-            .await;
+        let self_address = base_builder.as_mix_recipient();
+        let mut started_client = base_builder.start_base().await?;
+        let client_input = started_client.client_input.register_producer();
+        let client_output = started_client.client_output.register_consumer();
 
-        self.start_mix_traffic_controller(
-            sphinx_message_receiver,
-            gateway_client,
-            shutdown.subscribe(),
-        );
-        self.start_real_traffic_controller(
-            shared_topology_accessor.clone(),
-            reply_key_storage,
-            ack_receiver,
-            input_receiver,
-            sphinx_message_sender.clone(),
-            shutdown.subscribe(),
-        );
-
-        self.start_cover_traffic_stream(
-            shared_topology_accessor,
-            sphinx_message_sender,
-            shutdown.subscribe(),
-        );
-        self.start_socks5_listener(
-            received_buffer_request_sender,
-            input_sender,
-            shutdown.subscribe(),
+        Self::start_socks5_listener(
+            &self.config,
+            client_input,
+            client_output,
+            self_address,
+            started_client.shutdown_notifier.subscribe(),
         );
 
         info!("Client startup finished!");
-        info!("The address of this client is: {}", self.as_mix_recipient());
+        info!("The address of this client is: {}", self_address);
 
-        shutdown
+        Ok(started_client.shutdown_notifier)
     }
 }

clients/socks5/src/commands/init.rs

@@ -1,13 +1,18 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::fmt::Display;
+
 use clap::Args;
-use client_core::{config::GatewayEndpoint, error::ClientCoreError};
 use config::NymConfig;
+use nymsphinx::addressing::clients::Recipient;
+use serde::Serialize;
+use tap::TapFallible;
 
 use crate::{
     client::config::Config,
     commands::{override_config, OverrideConfig},
+    error::Socks5ClientError,
 };
 
 #[derive(Args, Clone)]
@@ -29,9 +34,13 @@ pub(crate) struct Init {
     #[clap(long)]
     force_register_gateway: bool,
 
-    /// Comma separated list of rest endpoints of the validators
+    /// Comma separated list of rest endpoints of the nymd validators
     #[clap(long)]
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+
+    /// Comma separated list of rest endpoints of the API validators
+    #[clap(long)]
+    api_validators: Option<String>,
 
     /// Port for the socket to listen on in all subsequent runs
     #[clap(short, long)]
@@ -42,26 +51,59 @@ pub(crate) struct Init {
     #[clap(long, hidden = true)]
     fastmode: bool,
 
+    /// Disable loop cover traffic and the Poisson rate limiter (for debugging only)
+    #[clap(long, hidden = true)]
+    no_cover: bool,
+
     /// Set this client to work in a enabled credentials mode that would attempt to use gateway
     /// with bandwidth credential requirement.
     #[cfg(feature = "coconut")]
     #[clap(long)]
     enabled_credentials_mode: bool,
+
+    /// Save a summary of the initialization to a json file
+    #[clap(long)]
+    output_json: bool,
 }
 
 impl From<Init> for OverrideConfig {
     fn from(init_config: Init) -> Self {
         OverrideConfig {
-            validators: init_config.validators,
+            nymd_validators: init_config.nymd_validators,
+            api_validators: init_config.api_validators,
             port: init_config.port,
             fastmode: init_config.fastmode,
+            no_cover: init_config.no_cover,
             #[cfg(feature = "coconut")]
             enabled_credentials_mode: init_config.enabled_credentials_mode,
         }
     }
 }
 
-pub(crate) async fn execute(args: &Init) {
+#[derive(Debug, Serialize)]
+pub struct InitResults {
+    #[serde(flatten)]
+    client_core: client_core::init::InitResults,
+    socks5_listening_port: String,
+}
+
+impl InitResults {
+    fn new(config: &Config, address: &Recipient) -> Self {
+        Self {
+            client_core: client_core::init::InitResults::new(config.get_base(), address),
+            socks5_listening_port: config.get_listening_port().to_string(),
+        }
+    }
+}
+
+impl Display for InitResults {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "{}", self.client_core)?;
+        write!(f, "SOCKS5 listening port: {}", self.socks5_listening_port)
+    }
+}
+
+pub(crate) async fn execute(args: &Init) -> Result<(), Socks5ClientError> {
     println!("Initialising client...");
 
     let id = &args.id;
@@ -86,25 +128,47 @@ pub(crate) async fn execute(args: &Init) {
     let register_gateway = !already_init || user_wants_force_register;
 
     // Attempt to use a user-provided gateway, if possible
-    let user_chosen_gateway_id = args.gateway.as_deref();
+    let user_chosen_gateway_id = args.gateway.clone();
 
-    let mut config = Config::new(id, provider_address);
-    let override_config_fields = OverrideConfig::from(args.clone());
-    config = override_config(config, override_config_fields);
+    // Load and potentially override config
+    let mut config = override_config(
+        Config::new(id, provider_address),
+        OverrideConfig::from(args.clone()),
+    );
+
+    // Setup gateway by either registering a new one, or creating a new config from the selected
+    // one but with keys kept, or reusing the gateway configuration.
+    let gateway = client_core::init::setup_gateway::<Config, _>(
+        register_gateway,
+        user_chosen_gateway_id,
+        config.get_base(),
+    )
+    .await
+    .tap_err(|err| eprintln!("Failed to setup gateway\nError: {err}"))?;
 
-    let gateway = setup_gateway(id, register_gateway, user_chosen_gateway_id, &config)
-        .await
-        .unwrap_or_else(|err| {
-            eprintln!("Failed to setup gateway\nError: {err}");
-            std::process::exit(1)
-        });
     config.get_base_mut().with_gateway_endpoint(gateway);
 
-    let config_save_location = config.get_config_file_save_location();
-    config
-        .save_to_file(None)
-        .expect("Failed to save the config file");
+    config.save_to_file(None).tap_err(|_| {
+        log::error!("Failed to save the config file");
+    })?;
 
+    print_saved_config(&config);
+
+    let address = client_core::init::get_client_address_from_stored_keys(config.get_base())?;
+    let init_results = InitResults::new(&config, &address);
+    println!("{}", init_results);
+
+    // Output summary to a json file, if specified
+    if args.output_json {
+        client_core::init::output_to_json(&init_results, "socks5_client_init_results.json");
+    }
+
+    println!("\nThe address of this client is: {}\n", address);
+    Ok(())
+}
+
+fn print_saved_config(config: &Config) {
+    let config_save_location = config.get_config_file_save_location();
     println!("Saved configuration file to {:?}", config_save_location);
     println!("Using gateway: {}", config.get_base().get_gateway_id());
     log::debug!("Gateway id: {}", config.get_base().get_gateway_id());
@@ -113,62 +177,5 @@ pub(crate) async fn execute(args: &Init) {
         "Gateway listener: {}",
         config.get_base().get_gateway_listener()
     );
-    println!("Client configuration completed.");
-
-    client_core::init::show_address(config.get_base()).unwrap_or_else(|err| {
-        eprintln!("Failed to show address\nError: {err}");
-        std::process::exit(1)
-    });
-}
-
-async fn setup_gateway(
-    id: &str,
-    register: bool,
-    user_chosen_gateway_id: Option<&str>,
-    config: &Config,
-) -> Result<GatewayEndpoint, ClientCoreError> {
-    if register {
-        // Get the gateway details by querying the validator-api. Either pick one at random or use
-        // the chosen one if it's among the available ones.
-        println!("Configuring gateway");
-        let gateway = client_core::init::query_gateway_details(
-            config.get_base().get_validator_api_endpoints(),
-            user_chosen_gateway_id,
-        )
-        .await?;
-        log::debug!("Querying gateway gives: {}", gateway);
-
-        // Registering with gateway by setting up and writing shared keys to disk
-        log::trace!("Registering gateway");
-        client_core::init::register_with_gateway_and_store_keys(gateway.clone(), config.get_base())
-            .await?;
-        println!("Saved all generated keys");
-
-        Ok(gateway.into())
-    } else if user_chosen_gateway_id.is_some() {
-        // Just set the config, don't register or create any keys
-        // This assumes that the user knows what they are doing, and that the existing keys are
-        // valid for the gateway being used
-        println!("Using gateway provided by user, keeping existing keys");
-        let gateway = client_core::init::query_gateway_details(
-            config.get_base().get_validator_api_endpoints(),
-            user_chosen_gateway_id,
-        )
-        .await?;
-        log::debug!("Querying gateway gives: {}", gateway);
-        Ok(gateway.into())
-    } else {
-        println!("Not registering gateway, will reuse existing config and keys");
-        let existing_config = Config::load_from_file(Some(id)).map_err(|err| {
-            log::error!(
-                "Unable to configure gateway: {err}. \n
-                Seems like the client was already initialized but it was not possible to read \
-                the existing configuration file. \n
-                CAUTION: Consider backing up your gateway keys and try force gateway registration, or \
-                removing the existing configuration and starting over."
-            );
-            ClientCoreError::CouldNotLoadExistingGatewayConfiguration(err)
-        })?;
-        Ok(existing_config.get_base().get_gateway_endpoint().clone())
-    }
+    println!("Client configuration completed.\n");
 }

clients/socks5/src/commands/mod.rs

@@ -1,6 +1,8 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::error::Error;
+
 use crate::client::config::Config;
 use clap::CommandFactory;
 use clap::{Parser, Subcommand};
@@ -50,7 +52,7 @@ fn long_version_static() -> &'static str {
 #[clap(author = "Nymtech", version, long_version = long_version_static(), about)]
 pub(crate) struct Cli {
     /// Path pointing to an env file that configures the client.
-    #[clap(long)]
+    #[clap(short, long)]
     pub(crate) config_env_file: Option<std::path::PathBuf>,
 
     #[clap(subcommand)]
@@ -61,8 +63,10 @@ pub(crate) struct Cli {
 pub(crate) enum Commands {
     /// Initialise a Nym client. Do this first!
     Init(init::Init),
+
     /// Run the Nym client with provided configuration client optionally overriding set parameters
     Run(run::Run),
+
     /// Try to upgrade the client
     Upgrade(upgrade::Upgrade),
 
@@ -75,28 +79,40 @@ pub(crate) enum Commands {
 
 // Configuration that can be overridden.
 pub(crate) struct OverrideConfig {
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+    api_validators: Option<String>,
     port: Option<u16>,
     fastmode: bool,
+    no_cover: bool,
 
     #[cfg(feature = "coconut")]
     enabled_credentials_mode: bool,
 }
 
-pub(crate) async fn execute(args: &Cli) {
+pub(crate) async fn execute(args: &Cli) -> Result<(), Box<dyn Error + Send + Sync>> {
     let bin_name = "nym-socks5-client";
 
     match &args.command {
-        Commands::Init(m) => init::execute(m).await,
-        Commands::Run(m) => run::execute(m).await,
+        Commands::Init(m) => init::execute(m).await?,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::Upgrade(m) => upgrade::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::into_app(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::into_app(), bin_name),
     }
+    Ok(())
 }
 
 pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Config {
-    if let Some(raw_validators) = args.validators {
+    if let Some(raw_validators) = args.nymd_validators {
+        config
+            .get_base_mut()
+            .set_custom_validators(parse_validators(&raw_validators));
+    } else if let Ok(raw_validators) = std::env::var(network_defaults::var_names::NYMD_VALIDATOR) {
+        config
+            .get_base_mut()
+            .set_custom_validators(parse_validators(&raw_validators));
+    }
+    if let Some(raw_validators) = args.api_validators {
         config
             .get_base_mut()
             .set_custom_validator_apis(parse_validators(&raw_validators));
@@ -121,6 +137,10 @@ pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Confi
         config.get_base_mut().set_high_default_traffic_volume();
     }
 
+    if args.no_cover {
+        config.get_base_mut().set_no_cover_traffic();
+    }
+
     config
 }
 

clients/socks5/src/commands/run.rs

@@ -4,6 +4,7 @@
 use crate::{
     client::{config::Config, NymClient},
     commands::{override_config, OverrideConfig},
+    error::Socks5ClientError,
 };
 
 use clap::Args;
@@ -30,14 +31,27 @@ pub(crate) struct Run {
     #[clap(long)]
     gateway: Option<String>,
 
-    /// Comma separated list of rest endpoints of the validators
+    /// Comma separated list of rest endpoints of the nymd validators
     #[clap(long)]
-    validators: Option<String>,
+    nymd_validators: Option<String>,
+
+    /// Comma separated list of rest endpoints of the API validators
+    #[clap(long)]
+    api_validators: Option<String>,
 
     /// Port for the socket to listen on
     #[clap(short, long)]
     port: Option<u16>,
 
+    /// Mostly debug-related option to increase default traffic rate so that you would not need to
+    /// modify config post init
+    #[clap(long, hidden = true)]
+    fastmode: bool,
+
+    /// Disable loop cover traffic and the Poisson rate limiter (for debugging only)
+    #[clap(long, hidden = true)]
+    no_cover: bool,
+
     /// Set this client to work in a enabled credentials mode that would attempt to use gateway
     /// with bandwidth credential requirement.
     #[cfg(feature = "coconut")]
@@ -48,10 +62,11 @@ pub(crate) struct Run {
 impl From<Run> for OverrideConfig {
     fn from(run_config: Run) -> Self {
         OverrideConfig {
-            validators: run_config.validators,
+            nymd_validators: run_config.nymd_validators,
+            api_validators: run_config.api_validators,
             port: run_config.port,
-            fastmode: false,
-
+            fastmode: run_config.fastmode,
+            no_cover: run_config.no_cover,
             #[cfg(feature = "coconut")]
             enabled_credentials_mode: run_config.enabled_credentials_mode,
         }
@@ -80,14 +95,16 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) {
+pub(crate) async fn execute(args: &Run) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let id = &args.id;
 
     let mut config = match Config::load_from_file(Some(id)) {
         Ok(cfg) => cfg,
         Err(err) => {
             error!("Failed to load config for {}. Are you sure you have run `init` before? (Error was: {})", id, err);
-            return;
+            return Err(Box::new(Socks5ClientError::FailedToLoadConfig(
+                id.to_string(),
+            )));
         }
     };
 
@@ -96,8 +113,8 @@ pub(crate) async fn execute(args: &Run) {
 
     if !version_check(&config) {
         error!("failed the local version check");
-        return;
+        return Err(Box::new(Socks5ClientError::FailedLocalVersionCheck));
     }
 
-    NymClient::new(config).run_forever().await;
+    NymClient::new(config).run_forever().await
 }

clients/socks5/src/error.rs

@@ -0,0 +1,21 @@
+use client_core::error::ClientCoreError;
+
+use crate::socks::types::SocksProxyError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Socks5ClientError {
+    #[error("I/O error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("client-core error: {0}")]
+    ClientCoreError(#[from] ClientCoreError),
+
+    #[error("SOCKS proxy error")]
+    SocksProxyError(SocksProxyError),
+
+    #[error("Failed to load config for: {0}")]
+    FailedToLoadConfig(String),
+    #[error("Failed local version check, client and config mismatch")]
+    FailedLocalVersionCheck,
+    #[error("Fail to bind address")]
+    FailToBindAddress,
+}

clients/socks5/src/lib.rs

@@ -2,4 +2,5 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod client;
+pub mod error;
 pub mod socks;

clients/socks5/src/main.rs

@@ -1,21 +1,25 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use std::error::Error;
+
 use clap::{crate_version, Parser};
+use logging::setup_logging;
 use network_defaults::setup_env;
 
 pub mod client;
 mod commands;
+pub mod error;
 pub mod socks;
 
 #[tokio::main]
-async fn main() {
+async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
     setup_logging();
     println!("{}", banner());
 
     let args = commands::Cli::parse();
     setup_env(args.config_env_file.clone());
-    commands::execute(&args).await;
+    commands::execute(&args).await
 }
 
 fn banner() -> String {
@@ -34,23 +38,3 @@ fn banner() -> String {
         crate_version!()
     )
 }
-
-fn setup_logging() {
-    let mut log_builder = pretty_env_logger::formatted_timed_builder();
-    if let Ok(s) = ::std::env::var("RUST_LOG") {
-        log_builder.parse_filters(&s);
-    } else {
-        // default to 'Info'
-        log_builder.filter(None, log::LevelFilter::Info);
-    }
-
-    log_builder
-        .filter_module("hyper", log::LevelFilter::Warn)
-        .filter_module("tokio_reactor", log::LevelFilter::Warn)
-        .filter_module("reqwest", log::LevelFilter::Warn)
-        .filter_module("mio", log::LevelFilter::Warn)
-        .filter_module("want", log::LevelFilter::Warn)
-        .filter_module("tungstenite", log::LevelFilter::Warn)
-        .filter_module("tokio_tungstenite", log::LevelFilter::Warn)
-        .init();
-}

clients/socks5/src/socks/client.rs

@@ -2,10 +2,10 @@
 
 use super::authentication::{AuthenticationMethods, Authenticator, User};
 use super::request::{SocksCommand, SocksRequest};
-use super::types::{ResponseCode, SocksProxyError};
-use super::{RESERVED, SOCKS_VERSION};
-use client_core::client::inbound_messages::InputMessage;
-use client_core::client::inbound_messages::InputMessageSender;
+use super::types::{ResponseCodeV4, ResponseCodeV5, SocksProxyError};
+use super::{SocksVersion, RESERVED, SOCKS4_VERSION, SOCKS5_VERSION};
+use client_connections::{LaneQueueLengths, TransmissionLane};
+use client_core::client::inbound_messages::{InputMessage, InputMessageSender};
 use futures::channel::mpsc;
 use futures::task::{Context, Poll};
 use log::*;
@@ -129,18 +129,19 @@ impl AsyncWrite for StreamState {
 /// A client connecting to the Socks proxy server, because
 /// it wants to make a Nym-protected outbound request. Typically, this is
 /// something like e.g. a wallet app running on your laptop connecting to
-/// SphinxSocksServer.
+/// `SphinxSocksServer`.
 pub(crate) struct SocksClient {
     controller_sender: ControllerSender,
     stream: StreamState,
     auth_nmethods: u8,
     authenticator: Authenticator,
-    socks_version: u8,
+    socks_version: Option<SocksVersion>,
     input_sender: InputMessageSender,
     connection_id: ConnectionId,
     service_provider: Recipient,
     self_address: Recipient,
     started_proxy: bool,
+    lane_queue_lengths: LaneQueueLengths,
     shutdown_listener: ShutdownListener,
 }
 
@@ -157,28 +158,34 @@ impl Drop for SocksClient {
 }
 
 impl SocksClient {
-    /// Create a new SOCKClient
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         stream: TcpStream,
         authenticator: Authenticator,
         input_sender: InputMessageSender,
-        service_provider: Recipient,
+        service_provider: &Recipient,
         controller_sender: ControllerSender,
-        self_address: Recipient,
-        shutdown_listener: ShutdownListener,
+        self_address: &Recipient,
+        lane_queue_lengths: LaneQueueLengths,
+        mut shutdown_listener: ShutdownListener,
     ) -> Self {
+        // If this task fails and exits, we don't want to send shutdown signal
+        shutdown_listener.mark_as_success();
+
         let connection_id = Self::generate_random();
+
         SocksClient {
             controller_sender,
             connection_id,
             stream: StreamState::Available(stream),
             auth_nmethods: 0,
-            socks_version: 0,
+            socks_version: None,
             authenticator,
             input_sender,
-            service_provider,
-            self_address,
+            service_provider: *service_provider,
+            self_address: *self_address,
             started_proxy: false,
+            lane_queue_lengths,
             shutdown_listener,
         }
     }
@@ -188,16 +195,49 @@ impl SocksClient {
         rng.next_u64()
     }
 
+    pub async fn send_error(&mut self, err: SocksProxyError) -> Result<(), SocksProxyError> {
+        let error_text = format!("{}", err);
+        let Some(ref version) = self.socks_version else {
+            log::error!("Trying to send error without knowing the version");
+            return Ok(());
+        };
+
+        match version {
+            SocksVersion::V4 => {
+                let response = ResponseCodeV4::RequestRejected;
+                self.send_error_v4(response).await
+            }
+            SocksVersion::V5 => {
+                let response = if error_text.contains("Host") {
+                    ResponseCodeV5::HostUnreachable
+                } else if error_text.contains("Network") {
+                    ResponseCodeV5::NetworkUnreachable
+                } else if error_text.contains("ttl") {
+                    ResponseCodeV5::TtlExpired
+                } else {
+                    ResponseCodeV5::Failure
+                };
+                self.send_error_v5(response).await
+            }
+        }
+    }
+
     // Send an error back to the client
-    pub async fn error(&mut self, r: ResponseCode) -> Result<(), SocksProxyError> {
-        self.stream.write_all(&[5, r as u8]).await?;
+    pub async fn send_error_v4(&mut self, r: ResponseCodeV4) -> Result<(), SocksProxyError> {
+        self.stream.write_all(&[SOCKS4_VERSION, r as u8]).await?;
         Ok(())
     }
 
-    /// Shutdown the TcpStream to the client and end the session
+    pub async fn send_error_v5(&mut self, r: ResponseCodeV5) -> Result<(), SocksProxyError> {
+        self.stream.write_all(&[SOCKS5_VERSION, r as u8]).await?;
+        Ok(())
+    }
+
+    /// Shutdown the `TcpStream` to the client and end the session
     pub async fn shutdown(&mut self) -> Result<(), SocksProxyError> {
         info!("client is shutting down its TCP stream");
         self.stream.shutdown().await?;
+        self.shutdown_listener.mark_as_success();
         Ok(())
     }
 
@@ -205,33 +245,43 @@ impl SocksClient {
     /// is in use and that the client is authenticated, then runs the request.
     pub async fn run(&mut self) -> Result<(), SocksProxyError> {
         debug!("New connection from: {}", self.stream.peer_addr()?.ip());
-        let mut header = [0u8; 2];
+
         // Read a byte from the stream and determine the version being requested
+        let mut header = [0u8];
         self.stream.read_exact(&mut header).await?;
 
-        self.socks_version = header[0];
-        self.auth_nmethods = header[1];
+        self.socks_version = match SocksVersion::try_from(header[0]) {
+            Ok(version) => Some(version),
+            Err(_err) => {
+                warn!("Init: Unsupported version: SOCKS{}", header[0]);
+                return self.shutdown().await;
+            }
+        };
 
-        // Handle SOCKS4 requests
-        if header[0] != SOCKS_VERSION {
-            warn!("Init: Unsupported version: SOCKS{}", self.socks_version);
-            self.shutdown().await
-        }
-        // Valid SOCKS5
-        else {
-            // Authenticate w/ client
-            self.authenticate().await?;
-            // Handle requests
-            self.handle_request().await
+        if self.socks_version == Some(SocksVersion::V5) {
+            let mut auth = [0u8];
+            self.stream.read_exact(&mut auth).await?;
+            self.auth_nmethods = auth[0];
+            self.authenticate_socks5().await?;
         }
+
+        self.handle_request().await
     }
 
     async fn send_connect_to_mixnet(&mut self, remote_address: RemoteAddress) {
         let req = Request::new_connect(self.connection_id, remote_address, self.self_address);
         let msg = Message::Request(req);
 
-        let input_message = InputMessage::new_fresh(self.service_provider, msg.into_bytes(), false);
-        self.input_sender.unbounded_send(input_message).unwrap();
+        let input_message = InputMessage::new_fresh(
+            self.service_provider,
+            msg.into_bytes(),
+            false,
+            TransmissionLane::ConnectionId(self.connection_id),
+        );
+        self.input_sender
+            .send(input_message)
+            .await
+            .expect("InputMessageReceiver has stopped receiving!");
     }
 
     async fn run_proxy(&mut self, conn_receiver: ConnectionReceiver, remote_proxy_target: String) {
@@ -239,10 +289,15 @@ impl SocksClient {
             .await;
 
         let stream = self.stream.run_proxy();
-        let local_stream_remote = stream
-            .peer_addr()
-            .expect("failed to extract peer address")
-            .to_string();
+        let peer_addr = match stream.peer_addr() {
+            Ok(peer_addr) => peer_addr,
+            Err(err) => {
+                log::error!("Unable to extract the remote peer address: {err}");
+                return;
+            }
+        };
+        let local_stream_remote = peer_addr.to_string();
+
         let connection_id = self.connection_id;
         let input_sender = self.input_sender.clone();
 
@@ -254,12 +309,14 @@ impl SocksClient {
             conn_receiver,
             input_sender,
             connection_id,
+            Some(self.lane_queue_lengths.clone()),
             self.shutdown_listener.clone(),
         )
         .run(move |conn_id, read_data, socket_closed| {
             let provider_request = Request::new_send(conn_id, read_data, socket_closed);
             let provider_message = Message::Request(provider_request);
-            InputMessage::new_fresh(recipient, provider_message.into_bytes(), false)
+            let lane = TransmissionLane::ConnectionId(conn_id);
+            InputMessage::new_fresh(recipient, provider_message.into_bytes(), false, lane)
         })
         .await
         .into_inner();
@@ -271,8 +328,17 @@ impl SocksClient {
     async fn handle_request(&mut self) -> Result<(), SocksProxyError> {
         debug!("Handling CONNECT Command");
 
-        let request = SocksRequest::from_stream(&mut self.stream).await?;
-        let remote_address = request.to_string();
+        let version = self
+            .socks_version
+            .as_ref()
+            .expect("Must read version before parsing request");
+
+        let request = match version {
+            SocksVersion::V4 => SocksRequest::from_stream_socks4(&mut self.stream).await?,
+            SocksVersion::V5 => SocksRequest::from_stream_socks5(&mut self.stream).await?,
+        };
+
+        let remote_address = request.address_string();
 
         // setup for receiving from the mixnet
         let (mix_sender, mix_receiver) = mpsc::unbounded();
@@ -281,7 +347,10 @@ impl SocksClient {
             // Use the Proxy to connect to the specified addr/port
             SocksCommand::Connect => {
                 trace!("Connecting to: {:?}", remote_address.clone());
-                self.acknowledge_socks5().await;
+                match version {
+                    SocksVersion::V4 => self.acknowledge_socks4().await,
+                    SocksVersion::V5 => self.acknowledge_socks5().await,
+                }
 
                 self.started_proxy = true;
                 self.controller_sender
@@ -312,8 +381,8 @@ impl SocksClient {
     async fn acknowledge_socks5(&mut self) {
         self.stream
             .write_all(&[
-                SOCKS_VERSION,
-                ResponseCode::Success as u8,
+                SOCKS5_VERSION,
+                ResponseCodeV5::Success as u8,
                 RESERVED,
                 1,
                 127,
@@ -327,13 +396,30 @@ impl SocksClient {
             .unwrap();
     }
 
+    /// Writes a Socks4 header back to the requesting client's TCP stream,
+    async fn acknowledge_socks4(&mut self) {
+        self.stream
+            .write_all(&[
+                0, //SOCKS4_VERSION,
+                ResponseCodeV4::Granted as u8,
+                0,
+                0,
+                127,
+                0,
+                0,
+                1,
+            ])
+            .await
+            .unwrap();
+    }
+
     /// Authenticate the incoming request. Each request is checked for its
     /// authentication method. A user/password request will extract the
     /// username and password from the stream, then check with the Authenticator
     /// to see if the resulting user is allowed.
     ///
     /// A lot of this could probably be put into the `SocksRequest::from_stream()`
-    /// constructor, and/or cleaned up with tokio::codec. It's mostly just
+    /// constructor, and/or cleaned up with `tokio::codec`. It's mostly just
     /// read-a-byte-or-two. The bytes being extracted look like this:
     ///
     /// +----+------+----------+------+------------+
@@ -345,7 +431,7 @@ impl SocksClient {
     /// Pulling out the stream code into its own home, and moving the if/else logic
     /// into the Authenticator (where it'll be more easily testable)
     /// would be a good next step.
-    async fn authenticate(&mut self) -> Result<(), SocksProxyError> {
+    async fn authenticate_socks5(&mut self) -> Result<(), SocksProxyError> {
         debug!("Authenticating w/ {}", self.stream.peer_addr()?.ip());
         // Get valid auth methods
         let methods = self.get_available_methods().await?;
@@ -354,7 +440,7 @@ impl SocksClient {
         let mut response = [0u8; 2];
 
         // Set the version in the response
-        response[0] = SOCKS_VERSION;
+        response[0] = SOCKS5_VERSION;
         if methods.contains(&(AuthenticationMethods::UserPass as u8)) {
             // Set the default auth method (NO AUTH)
             response[1] = AuthenticationMethods::UserPass as u8;
@@ -390,11 +476,11 @@ impl SocksClient {
             // Authenticate passwords
             if self.authenticator.is_allowed(&user) {
                 debug!("Access Granted. User: {}", user.username);
-                let response = [1, ResponseCode::Success as u8];
+                let response = [1, ResponseCodeV5::Success as u8];
                 self.stream.write_all(&response).await?;
             } else {
                 debug!("Access Denied. User: {}", user.username);
-                let response = [1, ResponseCode::Failure as u8];
+                let response = [1, ResponseCodeV5::Failure as u8];
                 self.stream.write_all(&response).await?;
 
                 // Shutdown
@@ -413,7 +499,7 @@ impl SocksClient {
             response[1] = AuthenticationMethods::NoMethods as u8;
             self.stream.write_all(&response).await?;
             self.shutdown().await?;
-            Err(ResponseCode::Failure.into())
+            Err(ResponseCodeV5::Failure.into())
         }
     }
 

clients/socks5/src/socks/mixnet_responses.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use futures::channel::mpsc;
 use futures::StreamExt;
 use log::*;
@@ -18,9 +20,16 @@ pub(crate) struct MixnetResponseListener {
 
 impl Drop for MixnetResponseListener {
     fn drop(&mut self) {
-        self.buffer_requester
+        if let Err(err) = self
+            .buffer_requester
             .unbounded_send(ReceivedBufferMessage::ReceiverDisconnect)
-            .expect("the buffer request failed!")
+        {
+            if self.shutdown.is_shutdown_poll() {
+                log::debug!("The buffer request failed: {}", err);
+            } else {
+                log::error!("The buffer request failed: {}", err);
+            }
+        }
     }
 }
 
@@ -96,7 +105,10 @@ impl MixnetResponseListener {
                 }
             }
         }
-        assert!(self.shutdown.is_shutdown_poll());
+        #[cfg(not(target_arch = "wasm32"))]
+        tokio::time::timeout(Duration::from_secs(5), self.shutdown.recv())
+            .await
+            .expect("Task stopped without shutdown called");
         log::debug!("MixnetResponseListener: Exiting");
     }
 }

clients/socks5/src/socks/mod.rs

@@ -1,5 +1,9 @@
 #![forbid(unsafe_code)]
 
+use std::convert::TryFrom;
+
+use self::types::SocksProxyError;
+
 pub mod authentication;
 mod client;
 pub(crate) mod mixnet_responses;
@@ -9,6 +13,27 @@ pub mod types;
 pub mod utils;
 
 /// Version of socks
-const SOCKS_VERSION: u8 = 0x05;
+const SOCKS4_VERSION: u8 = 0x04;
+const SOCKS5_VERSION: u8 = 0x05;
 
 const RESERVED: u8 = 0x00;
+
+#[derive(Clone, PartialEq, Eq)]
+pub enum SocksVersion {
+    V4 = 0x04,
+    V5 = 0x05,
+}
+
+pub struct InvalidSocksVersion;
+
+impl TryFrom<u8> for SocksVersion {
+    type Error = SocksProxyError;
+
+    fn try_from(version: u8) -> Result<Self, Self::Error> {
+        match version {
+            SOCKS4_VERSION => Ok(Self::V4),
+            SOCKS5_VERSION => Ok(Self::V5),
+            _ => Err(SocksProxyError::UnsupportedProxyVersion(version)),
+        }
+    }
+}

clients/socks5/src/socks/request.rs

@@ -1,5 +1,7 @@
-use super::types::{AddrType, ResponseCode, SocksProxyError};
-use super::{utils as socks_utils, SOCKS_VERSION};
+use crate::socks::SOCKS4_VERSION;
+
+use super::types::{AddrType, ResponseCodeV5, SocksProxyError};
+use super::{utils as socks_utils, SOCKS5_VERSION};
 use log::*;
 use std::fmt::{self, Display};
 use tokio::io::{AsyncRead, AsyncReadExt};
@@ -15,80 +17,114 @@ pub(crate) struct SocksRequest {
 }
 
 impl SocksRequest {
-    /// Parse a SOCKS5 request from a TcpStream
-    pub async fn from_stream<R>(stream: &mut R) -> Result<Self, SocksProxyError>
+    /// Parse a SOCKS4 request from a `TcpStream`
+    /// From documents at:
+    ///  - SOCKS4: https://www.openssh.com/txt/socks4.protocol
+    ///  - SOCKS4a: https://www.openssh.com/txt/socks4a.protocol
+    pub async fn from_stream_socks4<R>(stream: &mut R) -> Result<Self, SocksProxyError>
     where
         R: AsyncRead + Unpin,
     {
+        log::trace!("read from stream socks4");
+
+        let mut packet = [0u8; 3];
+        stream.read_exact(&mut packet).await?;
+
+        // CD (command)
+        let Some(command) = SocksCommand::from(packet[0] as usize) else {
+            log::warn!("Invalid Command");
+            return Err(ResponseCodeV5::CommandNotSupported.into());
+        };
+
+        // DSTPORT
+        let mut port = [0u8; 2];
+        port.copy_from_slice(&packet[1..]);
+        let port = merge_u8_into_u16(port[0], port[1]);
+
+        // DSTIP
+        let mut ip = [0u8; 4];
+        stream.read_exact(&mut ip).await?;
+
+        // USERID
+        let _userid = read_until_zero(stream).await;
+
+        // SOCKS4a extension
+        // https://www.openssh.com/txt/socks4a.protocol
+        // If the IP is 0.0.0.x with x nonzero, read the domain name
+        let (addr, addr_type) = if ip[..3] == [0, 0, 0] && ip[3] != 0 {
+            (read_until_zero(stream).await?, AddrType::Domain)
+        } else {
+            (ip.to_vec(), AddrType::V4)
+        };
+
+        // Return parsed request
+        Ok(SocksRequest {
+            version: SOCKS4_VERSION,
+            command,
+            addr_type,
+            addr,
+            port,
+        })
+    }
+    /// Parse a SOCKS5 request from a `TcpStream`
+    /// From: https://www.rfc-editor.org/rfc/rfc1928
+    pub async fn from_stream_socks5<R>(stream: &mut R) -> Result<Self, SocksProxyError>
+    where
+        R: AsyncRead + Unpin,
+    {
+        log::info!("read from stream socks5");
+
         let mut packet = [0u8; 4];
         // Read a byte from the stream and determine the version being requested
         stream.read_exact(&mut packet).await?;
 
-        if packet[0] != SOCKS_VERSION {
-            warn!("from_stream Unsupported version: SOCKS{}", packet[0]);
+        // VER
+        if packet[0] != SOCKS5_VERSION {
+            warn!("Unsupported version: SOCKS{}", packet[0]);
             return Err(SocksProxyError::UnsupportedProxyVersion(packet[0]));
         }
 
-        // Get command
-        let mut command: SocksCommand = SocksCommand::Connect;
-        match SocksCommand::from(packet[1] as usize) {
-            Some(com) => {
-                command = com;
-                Ok(())
-            }
-            None => {
-                warn!("Invalid Command");
-                Err(ResponseCode::CommandNotSupported)
-            }
-        }?;
+        // CMD
+        let Some(command) = SocksCommand::from(packet[1] as usize) else {
+            warn!("Invalid Command");
+            return Err(ResponseCodeV5::CommandNotSupported.into());
+        };
 
-        // DST.address
+        // RSV
+        // packet[2] is reserved
 
-        let mut addr_type: AddrType = AddrType::V6;
-        match AddrType::from(packet[3] as usize) {
-            Some(addr) => {
-                addr_type = addr;
-                Ok(())
-            }
-            None => {
-                error!("No Addr");
-                Err(ResponseCode::AddrTypeNotSupported)
-            }
-        }?;
+        // ATYP
+        let Some(addr_type) = AddrType::from(packet[3] as usize) else {
+            error!("No Addr");
+            return Err(ResponseCodeV5::AddrTypeNotSupported.into())
+        };
 
-        trace!("Getting Addr");
-        // Get Addr from addr_type and stream
-        let addr: Result<Vec<u8>, SocksProxyError> = match addr_type {
+        // DST.ADDR
+        let addr = match addr_type {
             AddrType::Domain => {
-                let mut domain_length = [0u8; 1];
+                let mut domain_length = [0u8];
                 stream.read_exact(&mut domain_length).await?;
-
                 let mut domain = vec![0u8; domain_length[0] as usize];
                 stream.read_exact(&mut domain).await?;
-
-                Ok(domain)
+                domain
             }
             AddrType::V4 => {
                 let mut addr = [0u8; 4];
                 stream.read_exact(&mut addr).await?;
-                Ok(addr.to_vec())
+                addr.to_vec()
             }
             AddrType::V6 => {
                 let mut addr = [0u8; 16];
                 stream.read_exact(&mut addr).await?;
-                Ok(addr.to_vec())
+                addr.to_vec()
             }
         };
 
-        let addr = addr?;
-
-        // read DST.port
+        // DST.PORT
         let mut port = [0u8; 2];
         stream.read_exact(&mut port).await?;
-        // Merge two u8s into u16
-        let port = (u16::from(port[0]) << 8) | u16::from(port[1]);
+        let port = merge_u8_into_u16(port[0], port[1]);
 
-        // Return parsed request
         Ok(SocksRequest {
             version: packet[0],
             command,
@@ -97,14 +133,18 @@ impl SocksRequest {
             port,
         })
     }
+
+    /// Print out the address and port to a String.
+    /// This might return domain:port, ipv6:port, or ipv4:port.
+    pub fn address_string(&self) -> String {
+        let address = socks_utils::pretty_print_addr(&self.addr_type, &self.addr);
+        format!("{}:{}", address, self.port)
+    }
 }
 
 impl Display for SocksRequest {
-    /// Print out the address and port to a String.
-    /// This might return domain:port, ipv6:port, or ipv4:port.
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let address = socks_utils::pretty_print_addr(&self.addr_type, &self.addr);
-        write!(f, "{}:{}", address, self.port)
+        write!(f, "{}", self.address_string())
     }
 }
 
@@ -127,3 +167,23 @@ impl SocksCommand {
         }
     }
 }
+
+fn merge_u8_into_u16(a: u8, b: u8) -> u16 {
+    (u16::from(a) << 8) | u16::from(b)
+}
+
+async fn read_until_zero<R>(stream: &mut R) -> Result<Vec<u8>, SocksProxyError>
+where
+    R: AsyncRead + Unpin,
+{
+    let mut result = Vec::new();
+    let mut char = [0u8];
+    loop {
+        stream.read_exact(&mut char).await?;
+        if char[0] == 0 {
+            break;
+        }
+        result.push(char[0]);
+    }
+    Ok(result)
+}

clients/socks5/src/socks/server.rs

@@ -1,16 +1,17 @@
-use super::authentication::Authenticator;
-use super::client::SocksClient;
+use crate::error::Socks5ClientError;
+
 use super::{
-    mixnet_responses::MixnetResponseListener,
-    types::{ResponseCode, SocksProxyError},
+    authentication::Authenticator, client::SocksClient, mixnet_responses::MixnetResponseListener,
 };
+use client_connections::{ConnectionCommandSender, LaneQueueLengths};
 use client_core::client::{
     inbound_messages::InputMessageSender, received_buffer::ReceivedBufferRequestSender,
 };
 use log::*;
 use nymsphinx::addressing::clients::Recipient;
-use proxy_helpers::connection_controller::Controller;
+use proxy_helpers::connection_controller::{BroadcastActiveConnections, Controller};
 use std::net::SocketAddr;
+use tap::TapFallible;
 use task::ShutdownListener;
 use tokio::net::TcpListener;
 
@@ -20,6 +21,7 @@ pub struct SphinxSocksServer {
     listening_address: SocketAddr,
     service_provider: Recipient,
     self_address: Recipient,
+    lane_queue_lengths: LaneQueueLengths,
     shutdown: ShutdownListener,
 }
 
@@ -30,6 +32,7 @@ impl SphinxSocksServer {
         authenticator: Authenticator,
         service_provider: Recipient,
         self_address: Recipient,
+        lane_queue_lengths: LaneQueueLengths,
         shutdown: ShutdownListener,
     ) -> Self {
         // hardcode ip as we (presumably) ONLY want to listen locally. If we change it, we can
@@ -41,6 +44,7 @@ impl SphinxSocksServer {
             listening_address: format!("{}:{}", ip, port).parse().unwrap(),
             service_provider,
             self_address,
+            lane_queue_lengths,
             shutdown,
         }
     }
@@ -51,13 +55,19 @@ impl SphinxSocksServer {
         &mut self,
         input_sender: InputMessageSender,
         buffer_requester: ReceivedBufferRequestSender,
-    ) -> Result<(), SocksProxyError> {
-        let listener = TcpListener::bind(self.listening_address).await.unwrap();
+        client_connection_tx: ConnectionCommandSender,
+    ) -> Result<(), Socks5ClientError> {
+        let listener = TcpListener::bind(self.listening_address)
+            .await
+            .tap_err(|err| log::error!("Failed to bind to address: {err}"))?;
         info!("Serving Connections...");
 
         // controller for managing all active connections
-        let (mut active_streams_controller, controller_sender) =
-            Controller::new(self.shutdown.clone());
+        let (mut active_streams_controller, controller_sender) = Controller::new(
+            client_connection_tx,
+            BroadcastActiveConnections::Off,
+            self.shutdown.clone(),
+        );
         tokio::spawn(async move {
             active_streams_controller.run().await;
         });
@@ -75,46 +85,26 @@ impl SphinxSocksServer {
         loop {
             tokio::select! {
                 Ok((stream, _remote)) = listener.accept() => {
-                    // TODO Optimize this
                     let mut client = SocksClient::new(
                         stream,
                         self.authenticator.clone(),
                         input_sender.clone(),
-                        self.service_provider,
+                        &self.service_provider,
                         controller_sender.clone(),
-                        self.self_address,
+                        &self.self_address,
+                        self.lane_queue_lengths.clone(),
                         self.shutdown.clone(),
                     );
 
                     tokio::spawn(async move {
-                        {
-                            match client.run().await {
-                                Ok(_) => {}
-                                Err(error) => {
-                                    error!("Error! {}", error);
-                                    let error_text = format!("{}", error);
-
-                                    let response: ResponseCode;
-
-                                    if error_text.contains("Host") {
-                                        response = ResponseCode::HostUnreachable;
-                                    } else if error_text.contains("Network") {
-                                        response = ResponseCode::NetworkUnreachable;
-                                    } else if error_text.contains("ttl") {
-                                        response = ResponseCode::TtlExpired
-                                    } else {
-                                        response = ResponseCode::Failure
-                                    }
-
-                                    if client.error(response).await.is_err() {
-                                        warn!("Failed to send error code");
-                                    };
-                                    if client.shutdown().await.is_err() {
-                                        warn!("Failed to shutdown TcpStream");
-                                    };
-                                }
+                        if let Err(err) = client.run().await {
+                            error!("Error! {}", err);
+                            if client.send_error(err).await.is_err() {
+                                warn!("Failed to send error code");
+                            };
+                            if client.shutdown().await.is_err() {
+                                warn!("Failed to shutdown TcpStream");
                             };
-                            // client gets dropped here
                         }
                     });
                 },

clients/socks5/src/socks/types.rs

@@ -1,23 +1,33 @@
-use snafu::Snafu;
-#[derive(Debug, Snafu)]
+/// SOCKS4 Response codes
+#[allow(dead_code)]
+pub(crate) enum ResponseCodeV4 {
+    Granted = 0x5a,
+    RequestRejected = 0x5b,
+    CannotConnectToIdent = 0x5c,
+    DifferentUserId = 0x5d,
+}
+
 /// Possible SOCKS5 Response Codes
-pub(crate) enum ResponseCode {
+#[allow(dead_code)]
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum ResponseCodeV5 {
+    #[error("SOCKS5 Server Success")]
     Success = 0x00,
-    #[snafu(display("SOCKS5 Server Failure"))]
+    #[error("SOCKS5 Server Failure")]
     Failure = 0x01,
-    #[snafu(display("SOCKS5 Rule failure"))]
+    #[error("SOCKS5 Rule failure")]
     RuleFailure = 0x02,
-    #[snafu(display("network unreachable"))]
+    #[error("network unreachable")]
     NetworkUnreachable = 0x03,
-    #[snafu(display("host unreachable"))]
+    #[error("host unreachable")]
     HostUnreachable = 0x04,
-    #[snafu(display("connection refused"))]
+    #[error("connection refused")]
     ConnectionRefused = 0x05,
-    #[snafu(display("TTL expired"))]
+    #[error("TTL expired")]
     TtlExpired = 0x06,
-    #[snafu(display("Command not supported"))]
+    #[error("Command not supported")]
     CommandNotSupported = 0x07,
-    #[snafu(display("Addr Type not supported"))]
+    #[error("Addr Type not supported")]
     AddrTypeNotSupported = 0x08,
 }
 
@@ -48,7 +58,7 @@ where
 }
 
 /// DST.addr variant types
-#[derive(PartialEq)]
+#[derive(Debug, PartialEq)]
 pub(crate) enum AddrType {
     V4 = 0x01,
     Domain = 0x03,

clients/webassembly/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "nym-client-wasm"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jedrzej Stuczynski <andrew@nymtech.net>"]
-version = "1.0.0"
+version = "1.1.0"
 edition = "2021"
 keywords = ["nym", "sphinx", "wasm", "webassembly", "privacy", "client"]
 license = "Apache-2.0"
@@ -19,14 +19,18 @@ coconut = ["coconut-interface", "credentials", "gateway-client/coconut"]
 
 [dependencies]
 futures = "0.3"
-serde = { version = "1.0", features = ["derive"] }
-wasm-bindgen = { version = "=0.2.78", features = ["serde-serialize"]  }
-wasm-bindgen-futures = "0.4"
 js-sys = "0.3"
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
+serde = { version = "1.0", features = ["derive"] }
+serde-wasm-bindgen = "0.4"
+tokio = { version = "1.21.2", features = ["sync"] }
 url = "2.2"
+wasm-bindgen = { version = "=0.2.83", features = ["serde-serialize"]  }
+wasm-bindgen-futures = "0.4"
 
 # internal
+client-core = { path = "../client-core", default-features = false, features = ["wasm"] }
+client-connections = { path = "../../common/client-connections" }
 coconut-interface = { path = "../../common/coconut-interface", optional = true }
 credentials = { path = "../../common/credentials", optional = true }
 crypto = { path = "../../common/crypto" }
@@ -35,7 +39,7 @@ topology = { path = "../../common/topology" }
 gateway-client = { path = "../../common/client-libs/gateway-client", default-features = false, features = ["wasm", "coconut"] }
 validator-client = { path = "../../common/client-libs/validator-client", default-features = false }
 wasm-utils = { path = "../../common/wasm-utils" }
-
+task = { path = "../../common/task" }
 # The `console_error_panic_hook` crate provides better debugging of panics by
 # logging them with `console.error`. This is great for development, but requires
 # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for
@@ -53,6 +57,8 @@ wee_alloc = { version = "0.4", optional = true }
 wasm-bindgen-test = "0.3"
 
 [package.metadata.wasm-pack.profile.release]
-wasm-opt = false
-
+wasm-opt = true
 
+[profile.release]
+lto = true
+opt-level = 'z'

clients/webassembly/Makefile

@@ -0,0 +1,5 @@
+clippy:
+	cargo clippy --all-features --target wasm32-unknown-unknown -- -D warnings
+
+test:
+	wasm-pack test --node

clients/webassembly/js-example/README.md

@@ -4,6 +4,16 @@ This example application demonstrates how to use WebAssembly to create Sphinx pa
 
 ## 🚴 Usage
 
+Build the WASM package for bundling:
+
+```
+wasm-pack build --scope nymproject --target no-modules
+```
+
+in the `clients/webassembly` directory (one up).
+
+Start the webpack dev server:
+
 ```
 npm install         # set up dependencies
 npm run start       # starts a web server at http://localhost:8001
@@ -15,4 +25,4 @@ Check your dev console for output.
 
 Install `wasm-pack`. Instruction are at the [Rust WASM tutorial](https://rustwasm.github.io/docs/book/game-of-life/hello-world.html). 
 
-`wasm-pack build` in the `clients/webassembly` directory (one up) will rebuild the wasm package if you make changes to the Rust source. That will be automatically picked up (and reloaded, if need be) by the npm dev server.
+`wasm-pack build --scope nymproject --target no-modules` in the `clients/webassembly` directory (one up) will rebuild the wasm package if you make changes to the Rust source. That will be automatically picked up (and reloaded, if need be) by the npm dev server.

clients/webassembly/js-example/bootstrap.js

@@ -1,5 +1,5 @@
 // A dependency graph that contains any wasm must all be imported
 // asynchronously. This `bootstrap.js` file does the single async import, so
 // that no one else needs to worry about it again.
-import("./index.js")
-  .catch(e => console.error("Error importing `index.js`:", e));
+import('./index.js')
+  .catch(e => console.error('Error importing `index.js`:', e));

clients/webassembly/js-example/index.js

@@ -1,4 +1,4 @@
-// Copyright 2020 Nym Technologies SA
+// Copyright 2020-2022 Nym Technologies SA
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,55 +12,67 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import {
-    NymClient,
-    set_panic_hook
-} from "@nymproject/nym-client-wasm"
+class WebWorkerClient {
+  worker = null;
 
-// current limitation of rust-wasm for async stuff : (
-let client = null
+  constructor() {
+    this.worker = new Worker('./worker.js');
+
+    this.worker.onmessage = (ev) => {
+      if (ev.data && ev.data.kind) {
+        switch (ev.data.kind) {
+          case 'Ready':
+            const { selfAddress } = ev.data.args;
+            displaySenderAddress(selfAddress);
+            break;
+          case 'ReceiveMessage':
+            const { message } = ev.data.args;
+            displayReceived(message);
+            break;
+        }
+      }
+    };
+  }
+
+  sendMessage = (message, recipient) => {
+    if (!this.worker) {
+      console.error('Could not send message because worker does not exist');
+      return;
+    }
+
+    this.worker.postMessage({
+      kind: 'SendMessage',
+      args: {
+        message, recipient,
+      },
+    });
+  };
+}
+
+let client = null;
 
 async function main() {
-    // sets up better stack traces in case of in-rust panics
-    set_panic_hook();
+  client = new WebWorkerClient();
 
-    // validator server we will use to get topology from
-    const validator = "https://validator.nymtech.net/api";
-
-    client = new NymClient(validator);
-
-    const on_message = (msg) => displayReceived(msg);
-    const on_connect = () => console.log("Established (and authenticated) gateway connection!");
-
-    client.set_on_message(on_message);
-    client.set_on_gateway_connect(on_connect);
-
-    // this is current limitation of wasm in rust - for async methods you can't take self my reference...
-    // I'm trying to figure out if I can somehow hack my way around it, but for time being you have to re-assign
-    // the object (it's the same one)
-    client = await client.initial_setup();
-
-    const self_address = client.self_address();
-    displaySenderAddress(self_address);
-
-    const sendButton = document.querySelector('#send-button');
-    sendButton.onclick = function () {
-        sendMessageTo();
-    }
+  const sendButton = document.querySelector('#send-button');
+  sendButton.onclick = function() {
+    sendMessageTo();
+  };
 }
 
 /**
  * Create a Sphinx packet and send it to the mixnet through the gateway node.
- * 
+ *
  * Message and recipient are taken from the values in the user interface.
  *
  * @param {Client} nymClient the nym client to use for message sending
  */
 async function sendMessageTo() {
-    const message = document.getElementById("message").value;
-    const recipient = document.getElementById("recipient").value;
-    client = await client.send_message(message, recipient);
-    displaySend(message);
+  const message = document.getElementById('message').value;
+  const recipient = document.getElementById('recipient').value;
+
+  await client.sendMessage(message, recipient);
+  displaySend(message);
 }
 
 /**
@@ -69,16 +81,16 @@ async function sendMessageTo() {
  * @param {string} message
  */
 function displaySend(message) {
-    let timestamp = new Date().toISOString().slice(11, 21);
+  let timestamp = new Date().toISOString().substr(11, 12);
 
-    let sendDiv = document.createElement("div")
-    let paragraph = document.createElement("p")
-    paragraph.setAttribute('style', 'color: blue')
-    let paragraphContent = document.createTextNode(timestamp + " sent >>> " + message)
-    paragraph.appendChild(paragraphContent)
+  let sendDiv = document.createElement('div');
+  let paragraph = document.createElement('p');
+  paragraph.setAttribute('style', 'color: blue');
+  let paragraphContent = document.createTextNode(timestamp + ' sent >>> ' + message);
+  paragraph.appendChild(paragraphContent);
 
-    sendDiv.appendChild(paragraph)
-    document.getElementById("output").appendChild(sendDiv)
+  sendDiv.appendChild(paragraph);
+  document.getElementById('output').appendChild(sendDiv);
 }
 
 /**
@@ -87,17 +99,17 @@ function displaySend(message) {
  * @param {string} message
  */
 function displayReceived(message) {
-    const content = message.message
-    const replySurb = message.replySurb
+  const content = message;
 
-    let timestamp = new Date().toISOString().slice(11, 21);
-    let receivedDiv = document.createElement("div")
-    let paragraph = document.createElement("p")
-    paragraph.setAttribute('style', 'color: green')
-    let paragraphContent = document.createTextNode(timestamp + " received >>> " + content)
-    paragraph.appendChild(paragraphContent)
-    receivedDiv.appendChild(paragraph)
-    document.getElementById("output").appendChild(receivedDiv)
+  let timestamp = new Date().toISOString().substr(11, 12);
+  let receivedDiv = document.createElement('div');
+  let paragraph = document.createElement('p');
+  paragraph.setAttribute('style', 'color: green');
+  let paragraphContent = document.createTextNode(timestamp + ' received >>> ' + content);
+  // let paragraphContent = document.createTextNode(timestamp + " received >>> " + content + ((replySurb != null) ? "Reply SURB was attached here (but we can't do anything with it yet" : " (NO REPLY-SURB AVAILABLE)"))
+  paragraph.appendChild(paragraphContent);
+  receivedDiv.appendChild(paragraph);
+  document.getElementById('output').appendChild(receivedDiv);
 }
 
 
@@ -107,7 +119,7 @@ function displayReceived(message) {
  * @param {Client} nymClient
  */
 function displaySenderAddress(address) {
-    document.getElementById("sender").value = address;
+  document.getElementById('sender').value = address;
 }
 
 // Let's get started!

clients/webassembly/js-example/package.json

@@ -34,6 +34,6 @@
     "webpack-dev-server": "^4.7.4"
   },
   "dependencies": {
-    "@nymproject/nym-client-wasm": "1.0.0"
+    "@nymproject/nym-client-wasm": "file:../pkg"
   }
 }

clients/webassembly/js-example/webpack.config.js

@@ -1,15 +1,33 @@
-const CopyWebpackPlugin = require("copy-webpack-plugin");
+const CopyWebpackPlugin = require('copy-webpack-plugin');
 const path = require('path');
 
 module.exports = {
-  entry: "./bootstrap.js",
-  output: {
-    path: path.resolve(__dirname, "dist"),
-    filename: "bootstrap.js",
+  performance: {
+    hints: false,
+    maxEntrypointSize: 512000,
+    maxAssetSize: 512000
   },
-  mode: "development",
+  entry: {
+    bootstrap: './bootstrap.js',
+    worker: './worker.js',
+  },
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: '[name].js',
+  },
+  // mode: 'development',
+  mode: 'production',
   plugins: [
-    new CopyWebpackPlugin({ patterns: ['index.html'] })
+    new CopyWebpackPlugin({
+      patterns: [
+        'index.html',
+        {
+          from: 'node_modules/@nymproject/nym-client-wasm/*.(js|wasm)',
+          to: '[name][ext]',
+        },
+      ],
+    }),
+
   ],
-  experiments: { syncWebAssembly: true }
+  experiments: { syncWebAssembly: true },
 };

clients/webassembly/js-example/worker.js

@@ -0,0 +1,125 @@
+// Copyright 2020-2022 Nym Technologies SA
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+importScripts('nym_client_wasm.js');
+
+console.log('Initializing worker');
+
+// wasm_bindgen creates a global variable (with the exports attached) that is in scope after `importScripts`
+const { default_debug, get_gateway, NymClient, set_panic_hook, Config } = wasm_bindgen;
+
+class ClientWrapper {
+  constructor(config, onMessageHandler) {
+    this.rustClient = new NymClient(config);
+    this.rustClient.set_on_message(onMessageHandler);
+    this.rustClient.set_on_gateway_connect(this.onConnect);
+  }
+
+  selfAddress = () => {
+    return this.rustClient.self_address();
+  };
+
+  onConnect = () => {
+    console.log('Established (and authenticated) gateway connection!');
+  };
+
+  start = async () => {
+    // this is current limitation of wasm in rust - for async methods you can't take self by reference...
+    // I'm trying to figure out if I can somehow hack my way around it, but for time being you have to re-assign
+    // the object (it's the same one)
+    this.rustClient = await this.rustClient.start();
+  };
+
+  sendMessage = async (recipient, message) => {
+    this.rustClient = await this.rustClient.send_message(recipient, message);
+  };
+
+  sendBinaryMessage = async (recipient, message) => {
+    this.rustClient = await this.rustClient.send_binary_message(recipient, message);
+  };
+}
+
+let client = null;
+
+async function main() {
+  // load WASM package
+  await wasm_bindgen('nym_client_wasm_bg.wasm');
+
+  console.log('Loaded WASM');
+
+  // sets up better stack traces in case of in-rust panics
+  set_panic_hook();
+
+  // validator server we will use to get topology from
+  const validator = 'https://validator.nymtech.net/api'; //"http://localhost:8081";
+  const preferredGateway = 'E3mvZTHQCdBvhfr178Swx9g4QG3kkRUun7YnToLMcMbM';
+
+  const gatewayEndpoint = await get_gateway(validator, preferredGateway);
+  gatewayEndpoint.gateway_listener = "wss://gateway1.nymtech.net:443"; // this is needed if we want it to work on the web. However this gateway is a v1 gateway, we will need to change for v2 once we get there
+
+  // only really useful if you want to adjust some settings like traffic rate
+  // (if not needed you can just pass a null)
+  const debug = default_debug();
+
+  debug.disable_main_poisson_packet_distribution = true;
+  debug.disable_loop_cover_traffic_stream = true;
+  debug.use_extended_packet_size = true;
+  // debug.average_packet_delay_ms = BigInt(10);
+  // debug.average_ack_delay_ms = BigInt(10);
+  // debug.ack_wait_addition_ms = BigInt(3000);
+  // debug.ack_wait_multiplier = 10;
+
+  debug.topology_refresh_rate_ms = BigInt(60000)
+
+  const config = new Config('my-awesome-wasm-client', validator, gatewayEndpoint, debug);
+
+  const onMessageHandler = (message) => {
+    self.postMessage({
+      kind: 'ReceiveMessage',
+      args: {
+        message,
+      },
+    });
+  };
+
+  console.log('Instantiating WASM client...');
+  client = new ClientWrapper(config, onMessageHandler);
+  console.log('Web worker creating WASM client...');
+  await client.start();
+  console.log('WASM client running!');
+
+  const selfAddress = client.rustClient.self_address();
+  console.log(`Client address is ${selfAddress}`);
+  self.postMessage({
+    kind: 'Ready',
+    args: {
+      selfAddress,
+    },
+  });
+
+  // Set callback to handle messages passed to the worker.
+  self.onmessage = async event => {
+    if (event.data && event.data.kind) {
+      switch (event.data.kind) {
+        case 'SendMessage': {
+          const { message, recipient } = event.data.args;
+          await client.sendMessage(message, recipient);
+        }
+      }
+    }
+  };
+}
+
+// Let's get started!
+main();

clients/webassembly/package-lock.json

@@ -1,11 +0,0 @@
-{
-    "requires": true,
-    "lockfileVersion": 1,
-    "dependencies": {
-        "semver": {
-            "version": "7.3.2",
-            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
-            "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ=="
-        }
-    }
-}

clients/webassembly/src/binary_message_helper.rs

@@ -0,0 +1,217 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsCast;
+
+#[wasm_bindgen(typescript_custom_section)]
+const TS_DEFS: &'static str = r#"
+export interface BinaryMessage {
+    kind: number,
+    payload: Uint8Array;
+    headers: string,
+}
+
+export interface StringMessage {
+    kind: number,
+    payload: string;
+}
+"#;
+
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen(typescript_type = "BinaryMessage")]
+    pub type IBinaryMessage;
+    #[wasm_bindgen(typescript_type = "StringMessage")]
+    pub type IStringMessage;
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BinaryMessage {
+    pub kind: u8,
+    pub payload: Vec<u8>,
+    pub headers: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct StringMessage {
+    pub kind: u8,
+    pub payload: String,
+}
+
+/// Create a new binary message with a user-specified `kind`.
+#[wasm_bindgen]
+pub fn create_binary_message(kind: u8, payload: Vec<u8>) -> Vec<u8> {
+    create_binary_message_with_headers(kind, payload, "".to_string())
+}
+
+/// Create a new message with a UTF-8 encoded string `payload` and a user-specified `kind`.
+#[wasm_bindgen]
+pub fn create_binary_message_from_string(kind: u8, payload: String) -> Vec<u8> {
+    create_binary_message_with_headers(kind, payload.as_bytes().to_vec(), "".to_string())
+}
+
+/// Create a new binary message with a user-specified `kind`, and `headers` as a string.
+#[wasm_bindgen]
+pub fn create_binary_message_with_headers(kind: u8, payload: Vec<u8>, headers: String) -> Vec<u8> {
+    let headers = headers.as_bytes().to_vec();
+    let size = (headers.len() as u64).to_be_bytes().to_vec();
+    vec![vec![kind], size, headers, payload].concat()
+}
+
+/// Parse the `kind` and byte array `payload` from a byte array
+#[wasm_bindgen]
+pub async fn parse_binary_message(message: Vec<u8>) -> Result<IBinaryMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, _headers, payload) = parse_binary_payload(&message);
+
+    Ok(serde_wasm_bindgen::to_value(&BinaryMessage {
+        kind,
+        payload: payload.to_vec(),
+        headers: "".to_string(),
+    })
+    .unwrap()
+    .unchecked_into::<IBinaryMessage>())
+}
+
+/// Parse the `kind` and byte array `payload` from a byte array with headers
+#[wasm_bindgen]
+pub async fn parse_binary_message_with_headers(
+    message: Vec<u8>,
+) -> Result<IBinaryMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, headers, payload) = parse_binary_payload(&message);
+
+    Ok(serde_wasm_bindgen::to_value(&BinaryMessage {
+        kind,
+        payload: payload.to_vec(),
+        headers,
+    })
+    .unwrap()
+    .unchecked_into::<IBinaryMessage>())
+}
+
+/// Parse the `kind` and UTF-8 string `payload` from a byte array with headers
+#[wasm_bindgen]
+pub async fn parse_string_message_with_headers(
+    message: Vec<u8>,
+) -> Result<IStringMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let (kind, _headers, payload) = parse_binary_payload(&message);
+    let payload = String::from_utf8_lossy(payload).into_owned();
+
+    Ok(
+        serde_wasm_bindgen::to_value(&StringMessage { kind, payload })
+            .unwrap()
+            .unchecked_into::<IStringMessage>(),
+    )
+}
+pub(crate) fn parse_binary_payload(message: &[u8]) -> (u8, String, &[u8]) {
+    // 1st byte is the kind
+    let kind = message[0];
+
+    // then the size as u64 big endian
+    let mut size = [0u8; 8];
+    size.clone_from_slice(&message[1..9]);
+    let size = u64::from_be_bytes(size) as usize;
+
+    // then the headers
+    let headers = String::from_utf8_lossy(&message[9..9 + size]).into_owned();
+
+    // finally the payload
+    let payload = &message[9 + size..];
+
+    (kind, headers, payload)
+}
+
+/// Parse the `kind` and UTF-8 string `payload` from a byte array
+#[wasm_bindgen]
+pub async fn parse_string_message(message: Vec<u8>) -> Result<IStringMessage, JsError> {
+    if message.len() < 2 {
+        return Err(JsError::new(
+            "Could not parse message, as less than 2 bytes long",
+        ));
+    }
+
+    let kind = message[0];
+    let payload = String::from_utf8_lossy(&message[1..]).into_owned();
+
+    Ok(
+        serde_wasm_bindgen::to_value(&StringMessage { kind, payload })
+            .unwrap()
+            .unchecked_into::<IStringMessage>(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{create_binary_message_with_headers, parse_binary_payload};
+    use wasm_bindgen_test::*;
+
+    #[wasm_bindgen_test]
+    fn test_binary_with_headers() {
+        let message_as_bytes = create_binary_message_with_headers(
+            42u8,
+            vec![0u8, 1u8, 2u8],
+            "test headers".to_string(),
+        );
+
+        // calculate header size
+        let headers = "test headers".as_bytes().to_vec();
+        let size = headers.len();
+
+        // the expected size
+        let expected_size = 12;
+        assert_eq!(size, expected_size);
+
+        assert_eq!(message_as_bytes[0], 42u8);
+        assert_eq!(message_as_bytes[1..9], 12u64.to_be_bytes());
+        assert_eq!(
+            message_as_bytes[9 + expected_size..9 + expected_size + 3],
+            vec![0u8, 1u8, 2u8]
+        );
+
+        let res = parse_binary_payload(&message_as_bytes);
+
+        assert_eq!(res.0, 42u8);
+        assert_eq!(res.1, "test headers".to_string());
+        assert_eq!(res.2, vec![0u8, 1u8, 2u8]);
+    }
+
+    #[wasm_bindgen_test]
+    fn test_binary_with_empty_headers() {
+        let message_as_bytes =
+            create_binary_message_with_headers(42u8, vec![0u8, 1u8, 2u8], "".to_string());
+
+        let expected_size = 0;
+
+        assert_eq!(message_as_bytes[0], 42u8);
+        assert_eq!(message_as_bytes[1..9], 0u64.to_be_bytes());
+        assert_eq!(
+            message_as_bytes[9 + expected_size..9 + expected_size + 3],
+            vec![0u8, 1u8, 2u8]
+        );
+
+        let res = parse_binary_payload(&message_as_bytes);
+
+        assert_eq!(res.0, 42u8);
+        assert_eq!(res.1, "".to_string());
+        assert_eq!(res.2, vec![0u8, 1u8, 2u8]);
+    }
+}