Skip errors on blind sign within threshold (#2976)

* Add resharing parameter

* Fix equality of dealers and members

* Contract resharing handling

* Dealer verification unit test

* Dealing commit unit test

* Epoch state unit tests

* Fix clippy

* Fmt

* Query initial dealer data

* Resharing nym-api changes

* Implement the mockups for nym-api dkg tests

* Dealing test

* Vk unit test

* Fix skipping vk submission

* Fix clippy

* Missing dealing for noninitial resharing dealer

* Check master vk holds after resharing on nym-apis

* Update changelog

* Fix clippy

.env.sample-dev

@@ -3,3 +3,21 @@
 
 RUST_LOG=info
 RUST_BACKTRACE=1
+
+#########################################
+# geoipupdate (needed for explorer-api) #
+#########################################
+# MaxMind account ID (change it to a valid account ID)
+GEOIPUPDATE_ACCOUNT_ID=xxx
+# MaxMind license key (change it to a valid license key)
+GEOIPUPDATE_LICENSE_KEY=xxx
+# List of space-separated database edition IDs. Edition IDs may
+# consist of letters, digits, and dashes. For example, GeoIP2-City
+# would download the GeoIP2 City database (GeoIP2-City).
+GEOIPUPDATE_EDITION_IDS=GeoLite2-City
+# The number of hours between geoipupdate runs. If this is not set
+# or is set to 0, geoipupdate will run once and exit.
+GEOIPUPDATE_FREQUENCY=72
+# The path to the directory where geoipupdate will download the
+# database.
+GEOIP_DB_DIRECTORY=./explorer-api/geo_ip

.github/CODEOWNERS

@@ -19,10 +19,10 @@
 Cargo.* @durch @futurechimp @jstuczyn @neacsu @octol
 
 # JS rules:
-*.js @mmsinclair @fmtabbara @Aid19801
-*.ts @mmsinclair @fmtabbara @Aid19801
-*.tsx @mmsinclair @fmtabbara @Aid19801
-*.jsx @mmsinclair @fmtabbara @Aid19801
+*.js @mmsinclair @fmtabbara
+*.ts @mmsinclair @fmtabbara
+*.tsx @mmsinclair @fmtabbara
+*.jsx @mmsinclair @fmtabbara
 
 # Something looking like possible documentation rules:
 *.md @mfahampshire

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+# Description
+
+Closes: #XXXX
+
+<!-- If appropriate, insert relevant description here -->
+
+# Checklist:
+
+- [ ] added a changelog entry to `CHANGELOG.md`

.github/workflows/audit.yml

@@ -0,0 +1,61 @@
+name: Daily security audit
+
+on:
+  schedule:
+    - cron: '5 9 * * *'
+jobs:
+  cargo-deny:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v2
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Install cargo deny
+        run: cargo install --locked cargo-deny
+      - name: Run cargo deny
+        run: |
+          find . -name Cargo.toml -exec cargo deny --manifest-path {} check \
+            advisories -A advisory-not-detected  --hide-inclusion-graph \; &> \
+            >(uniq  &> .github/workflows/support-files/notifications/deny.message )
+      - uses: actions/upload-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications/deny.message
+  notification:
+    needs: cargo-deny
+    runs-on: custom-runner-linux
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Download report from previous job
+        uses: actions/download-artifact@v3
+        with:
+          name: report
+          path: .github/workflows/support-files/notifications
+      - name: install npm
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Keybase - Node Install
+        run: npm install
+        working-directory: .github/workflows/support-files
+      - name: Keybase - Send Notification
+        env:
+          NYM_NOTIFICATION_KIND: security
+          NYM_PROJECT_NAME: "Daily security report"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+          KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "security"
+          MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+          MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+          MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+          MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+          MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
+        uses: docker://keybaseio/client:stable-node
+        with:
+          args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/build-and-upload-binaries-ci.yml

@@ -0,0 +1,113 @@
+name: Build and upload binaries to CI
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'clients/**'
+      - 'common/**'
+      - 'contracts/**'
+      - 'explorer-api/**'
+      - 'gateway/**'
+      - 'integrations/**'
+      - 'mixnode/**'
+      - 'sdk/rust/nym-sdk/**'
+      - 'service-providers/**'
+      - 'nym-api/**'
+      - 'nym-outfox/**'
+      - 'tools/nym-cli/**'
+      - 'tools/ts-rs-cli/**'
+  pull_request:
+    paths:
+      - 'clients/**'
+      - 'common/**'
+      - 'contracts/**'
+      - 'explorer-api/**'
+      - 'gateway/**'
+      - 'integrations/**'
+      - 'mixnode/**'
+      - 'sdk/rust/nym-sdk/**'
+      - 'service-providers/**'
+      - 'nym-api/**'
+      - 'nym-outfox/**'
+      - 'tools/nym-cli/**'
+      - 'tools/ts-rs-cli/**'
+
+env:
+  NETWORK: mainnet    
+
+jobs:
+  publish-nym:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Prepare build output directory
+        shell: bash
+        env:
+          OUTPUT_DIR: ci-builds/${{ github.ref_name }}
+        run: |
+          rm -rf ci-builds || true
+          mkdir -p $OUTPUT_DIR
+          echo $OUTPUT_DIR
+
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --release
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build release contracts
+        run: make wasm
+
+      - name: Prepare build output
+        shell: bash
+        env:
+          OUTPUT_DIR: ci-builds/${{ github.ref_name }}
+        run: |
+          cp target/release/nym-client $OUTPUT_DIR
+          cp target/release/nym-gateway $OUTPUT_DIR
+          cp target/release/nym-mixnode $OUTPUT_DIR
+          cp target/release/nym-socks5-client $OUTPUT_DIR
+          cp target/release/nym-api $OUTPUT_DIR
+          cp target/release/nym-network-requester $OUTPUT_DIR
+          cp target/release/nym-network-statistics $OUTPUT_DIR
+          cp target/release/nym-cli $OUTPUT_DIR
+
+          cp contracts/target/wasm32-unknown-unknown/release/mixnet_contract.wasm $OUTPUT_DIR
+          cp contracts/target/wasm32-unknown-unknown/release/vesting_contract.wasm $OUTPUT_DIR
+
+      - name: Deploy branch to CI www
+        continue-on-error: true
+        uses: easingthemes/ssh-deploy@main
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.CI_WWW_SSH_PRIVATE_KEY }}
+          ARGS: "-avzr"
+          SOURCE: "ci-builds/"
+          REMOTE_HOST: ${{ secrets.CI_WWW_REMOTE_HOST }}
+          REMOTE_USER: ${{ secrets.CI_WWW_REMOTE_USER }}
+          TARGET: ${{ secrets.CI_WWW_REMOTE_TARGET }}/builds/
+          EXCLUDE: "/dist/, /node_modules/"
+

.github/workflows/build-and-upload-binaries.yml

@@ -0,0 +1,59 @@
+name: Build and upload binaries to artifact storage
+
+on:
+  workflow_dispatch:
+    inputs:  
+      add_tokio_unstable:
+        description: 'True to add RUSTFLAGS="--cfg tokio_unstable"'
+        required: true
+        default: false
+        type: boolean 
+    
+env:
+  NETWORK: mainnet    
+
+jobs:
+  publish-nym:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+
+      - name: Sets env vars for tokio if set in manual dispatch inputs
+        run: |
+          echo 'RUSTFLAGS="--cfg tokio_unstable"' >> $GITHUB_ENV
+        if: github.event_name == 'workflow_dispatch' && inputs.add_tokio_unstable == true
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --release
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nym-binaries-artifacts
+          path: |
+            target/release/nym-client
+            target/release/nym-gateway
+            target/release/nym-mixnode
+            target/release/nym-socks5-client
+            target/release/nym-api
+            target/release/nym-network-requester
+            target/release/nym-network-statistics
+            target/release/nym-cli
+          retention-days: 30

.github/workflows/build-ts-packages.yml

@@ -12,10 +12,11 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install rsync
       run: sudo apt-get install rsync
+      continue-on-error: true
     - uses: rlespinasse/github-slug-action@v3.x
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: 16
     - name: Setup yarn
       run: npm install -g yarn
     - name: Build
@@ -58,6 +59,11 @@ jobs:
         KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
         KEYBASE_NYM_CHANNEL: "ci-ts-packages"
         IS_SUCCESS: "${{ job.status == 'success' }}"
+        MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+        MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+        MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+        MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+        MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
       uses: docker://keybaseio/client:stable-node
       with:
         args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/build.yml

@@ -2,21 +2,44 @@ name: Continuous integration
 
 on:
   push:
-    paths-ignore:
-      - 'explorer/**'
+    paths:
+      - 'clients/**'
+      - 'common/**'
+      - 'explorer-api/**'
+      - 'gateway/**'
+      - 'integrations/**'
+      - 'mixnode/**'
+      - 'sdk/rust/nym-sdk/**'
+      - 'service-providers/**'
+      - 'nym-api/**'
+      - 'nym-outfox/**'
+      - 'tools/nym-cli/**'
+      - 'tools/ts-rs-cli/**'
   pull_request:
-    paths-ignore:
-      - 'explorer/**'
+    paths:
+      - 'clients/**'
+      - 'common/**'
+      - 'explorer-api/**'
+      - 'gateway/**'
+      - 'integrations/**'
+      - 'mixnode/**'
+      - 'sdk/rust/nym-sdk/**'
+      - 'service-providers/**'
+      - 'nym-api/**'
+      - 'nym-outfox/**'
+      - 'tools/nym-cli/**'
+      - 'tools/ts-rs-cli/**'
 
 jobs:
   build:
-    runs-on: [ self-hosted, custom-linux-exoscale ]
-    # Enable sccache via environment variable
+    runs-on: [ self-hosted, custom-linux ]
+    # Enable sccache via environment variable
     env:
       RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
     steps:
       - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
 
       - name: Check out repository code
         uses: actions/checkout@v2
@@ -29,50 +52,46 @@ jobs:
           override: true
           components: rustfmt, clippy
 
-      - name: Build all binaries
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace
-
-      - name: Run all tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --all-features
-
       - name: Check formatting
         uses: actions-rs/cargo@v1
         with:
           command: fmt
           args: --all -- --check
 
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace
+
+      - name: Build all examples
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --examples
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace
+
+      - name: Run expensive tests
+        if: github.ref == 'refs/heads/develop' || github.event.pull_request.base.ref == 'develop' || github.event.pull_request.base.ref == 'master'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace -- --ignored
+
       - uses: actions-rs/clippy-check@v1
         name: Clippy checks
+        continue-on-error: true
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
+          args: --workspace
 
       - name: Run clippy
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --workspace -- -D warnings
-
-      - name: Build all binaries with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --features=coconut
-
-      - name: Run all tests with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --features=coconut
-
-      - name: Run clippy with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --features=coconut -- -D warnings
+          args: --workspace --all-targets -- -D warnings

.github/workflows/build_matrix_includes.json

@@ -1,6 +1,6 @@
 [
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"stable",
     "runOnEvent":"always"
   },

.github/workflows/connect-android.yml

@@ -0,0 +1,70 @@
+name: Nym Connect for Android (rust)
+
+on:
+  push:
+    paths:
+      - "nym-connect-android/src-tauri/**"
+      - "clients/client-core/**"
+      - "clients/socks5/**"
+      - "common/**"
+      - "gateway/gateway-requests/**"
+      - "contracts/vesting/**"
+      - "nym-api/nym-api-requests/**"
+  pull_request:
+    paths:
+      - "nym-connect-android/src-tauri/**"
+      - "clients/client-core/**"
+      - "clients/socks5/**"
+      - "common/**"
+      - "gateway/gateway-requests/**"
+      - "contracts/vesting/**"
+      - "nym-api/nym-api-requests/**"
+
+jobs:
+  build:
+    #runs-on: [self-hosted, custom-linux]
+    runs-on: ubuntu-22.04
+    #env:
+      #RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
+    #defaults:
+      #run:
+        #working-directory: nym-connect-android/src-tauri/
+    steps:
+      - name: Install Dependencies (Linux)
+        if: ${{ !env.ACT }}
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install \
+          libwebkit2gtk-4.1-dev \
+          build-essential \
+          curl \
+          wget \
+          libssl-dev \
+          libgtk-3-dev \
+          squashfs-tools \
+          libayatana-appindicator3-dev \
+          librsvg2-dev \
+          libsoup-3.0-dev \
+          libjavascriptcoregtk-4.1-dev
+        #continue-on-error: true
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+
+      - name: Check formatting
+        run: cargo fmt --manifest-path nym-connect-android/src-tauri/Cargo.toml -- --check
+
+      - name: Build all binaries
+        run: cargo build --manifest-path nym-connect-android/src-tauri/Cargo.toml
+
+      - name: Run all tests
+        run: cargo test  --manifest-path nym-connect-android/src-tauri/Cargo.toml
+
+      - name: Clippy
+        run: cargo clippy --manifest-path nym-connect-android/src-tauri/Cargo.toml --all-targets -- -D warnings

.github/workflows/connect.yml

@@ -0,0 +1,73 @@
+name: Nym Connect (rust)
+
+on:
+  push:
+    paths:
+      - 'nym-connect/**'
+      - 'clients/client-core/**'
+      - 'clients/socks5/**'
+      - 'common/**'
+      - 'gateway/gateway-requests/**'
+      - 'contracts/vesting/**'
+      - 'nym-api/nym-api-requests/**'
+  pull_request:
+    paths:
+      - 'nym-connect/**'
+      - 'clients/client-core/**'
+      - 'clients/socks5/**'
+      - 'common/**'
+      - 'gateway/gateway-requests/**'
+      - 'contracts/vesting/**'
+      - 'nym-api/nym-api-requests/**'
+
+jobs:
+  build:
+    runs-on: [ self-hosted, custom-linux ]
+    env:
+      RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
+    steps:
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools libayatana-appindicator3-dev
+        continue-on-error: true
+
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-connect/Cargo.toml --workspace
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-connect/Cargo.toml --workspace
+
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-connect/Cargo.toml --all -- --check
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy checks
+        continue-on-error: true
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --manifest-path nym-connect/Cargo.toml --workspace --all-features
+
+      - name: Run clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --manifest-path nym-connect/Cargo.toml --workspace --all-features -- -D warnings

.github/workflows/contracts-build.yml

@@ -0,0 +1,45 @@
+name: Build release of Nym smart contracts
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-contracts-') && github.event_name == 'release' }}
+    runs-on: [self-hosted, custom-runner-linux]
+    steps:
+      - uses: actions/checkout@v2
+      
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build release contracts
+        run: make wasm
+
+      - name: Upload Mixnet Contract Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: mixnet_contract.wasm
+          path: contracts/target/wasm32-unknown-unknown/release/mixnet_contract.wasm
+          retention-days: 5
+
+      - name: Upload Vesting Contract Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: vesting_contract.wasm
+          path: contracts/target/wasm32-unknown-unknown/release/vesting_contract.wasm
+          retention-days: 5
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            contracts/target/wasm32-unknown-unknown/release/vesting_contract.wasm
+            contracts/target/wasm32-unknown-unknown/release/mixnet_contract.wasm

.github/workflows/contracts.yml

@@ -2,15 +2,17 @@ name: Contracts
 
 on:
   push:
-    paths-ignore:
-      - 'explorer/**'
+    paths:
+      - 'contracts/**'
+      - 'common/**'
   pull_request:
     paths-ignore:
-      - 'explorer/**'
+      - 'contracts/**'
+      - 'common/**'
 
 jobs:
   matrix_prep:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -24,7 +26,7 @@ jobs:
   contracts:
     # since it's going to be compiled into wasm, there's absolutely
     # no point in running CI on different OS-es
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.rust == 'nightly' }}
     needs: matrix_prep
     strategy:

.github/workflows/nc-android-apk.yml

@@ -0,0 +1,127 @@
+name: NC Android APK Release
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "release/nc-android-v[0-9].[0-9].[0-9]*"
+
+jobs:
+  build:
+    name: Build APK
+    runs-on: ubuntu-latest
+    env:
+      ANDROID_HOME: ${{ github.workspace }}/android-sdk
+      NDK_VERSION: 25.1.8937393
+      NDK_HOME: ${{ env.ANDROID_HOME }}/ndk/${{ env.NDK_VERSION }}
+      SDK_PLATFORM_VERSION: android-33
+      SDK_BUILDTOOLS_VERSION: 33.0.1
+
+    steps:
+      - name: Install Dependencies (Linux)
+        # https://next--tauri.netlify.app/next/guides/getting-started/prerequisites/linux/#1-system-dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install \
+          libwebkit2gtk-4.0-dev \
+          build-essential \
+          curl \
+          wget \
+          libssl-dev \
+          libgtk-3-dev \
+          squashfs-tools \
+          libayatana-appindicator3-dev \
+          librsvg2-dev
+
+      - name: Install Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Install Android SDK manager
+        # https://developer.android.com/studio/command-line/sdkmanager
+        run: |
+          curl -sS https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip -o cmdline-tools.zip
+          unzip cmdline-tools.zip
+          mkdir -p $ANDROID_HOME/cmdline-tools/latest
+          mv cmdline-tools/* $ANDROID_HOME/cmdline-tools/latest
+          rm -rf cmdline-tools
+
+      - name: Install Android S/NDK
+        run: |
+          echo y | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --licenses
+          echo y | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager \
+            "platforms;$SDK_PLATFORM_VERSION" \
+            "platform-tools" \
+            "ndk;$NDK_VERSION" \
+            "build-tools;$SDK_BUILDTOOLS_VERSION"
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install tauri cli
+        run: cargo install tauri-cli --version "^2.0.0-alpha.2"
+
+      - name: Install rust android targets
+        run: |
+          rustup target add aarch64-linux-android \
+            armv7-linux-androideabi \
+            i686-linux-android \
+            x86_64-linux-android
+
+      - name: Setup Nodejs
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Install yarn
+        run: |
+          npm i -g yarn
+          yarn --version
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build frontend code
+        run: |
+          yarn install --frozen-lockfile
+          yarn build
+          yarn workspace @nym/nym-connect-android webpack:prod
+
+      - name: Build APK
+        working-directory: nym-connect-android
+        env:
+          WRY_ANDROID_PACKAGE: net.nymtech.nym_connect_android
+          WRY_ANDROID_LIBRARY: nym_connect_android
+        # TODO build with release profile (--release), it will requires
+        # to sign the APK. For now build with debug profile to avoid that
+        run: cargo tauri android build --debug --apk
+
+      # TODO add the version number to APK name
+      - name: Rename APK artifact
+        run: |
+          mv nym-connect-android/src-tauri/gen/android/nym_connect_android/app/build/outputs/apk/universal/debug/app-universal-debug.apk \
+            nym-connect-debug.apk
+
+      - name: Upload APK artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nc-apk-debug
+          path: nym-connect-debug.apk
+
+  publish:
+    name: Publish APK
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Download binary artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: nc-apk-debug
+          path: apk
+      # TODO add a step to upload the APK somewhere
+      # - name: Publish
+      #   uses: ???

.github/workflows/network-explorer-api.yml

@@ -0,0 +1,57 @@
+name: CI for Network Explorer API
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+    
+env:
+  NETWORK: mainnet    
+
+jobs:
+  publish-nym:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-explorer-api-') && (github.event_name == 'release' || github.event_name == 'workflow_dispatch') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+
+      - name: Check the release tag starts with `nym-explorer-api-`
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Release tag did not start with nym-explorer-api-...')
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build all explorer-api
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path explorer-api/Cargo.toml --workspace --release
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: my-artifact
+          path: |
+            target/release/explorer-api
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            target/release/explorer-api

.github/workflows/network-explorer-lint.yml

@@ -14,9 +14,9 @@ jobs:
     runs-on: custom-runner-linux
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: 16
       - name: Setup yarn
         run: npm install -g yarn
       - name: Run ESLint

.github/workflows/network-explorer.yml

@@ -1,6 +1,7 @@
 name: CI for Network Explorer
 
 on:
+  workflow_dispatch:
   push:
     paths:
       - 'explorer/**'
@@ -16,10 +17,11 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install rsync
       run: sudo apt-get install rsync
+      continue-on-error: true
     - uses: rlespinasse/github-slug-action@v3.x
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: 16
     - name: Setup yarn
       run: npm install -g yarn
       continue-on-error: true
@@ -31,6 +33,8 @@ jobs:
 #      continue-on-error: true
     - run: yarn && yarn build
       continue-on-error: true
+    - run: yarn storybook:build
+      name: Build storybook
     - name: Deploy branch to CI www
       continue-on-error: true
       uses: easingthemes/ssh-deploy@main
@@ -42,6 +46,17 @@ jobs:
         REMOTE_USER: ${{ secrets.CI_WWW_REMOTE_USER }}
         TARGET: ${{ secrets.CI_WWW_REMOTE_TARGET }}/network-explorer-${{ env.GITHUB_REF_SLUG }}
         EXCLUDE: "/dist/, /node_modules/"
+    - name: Deploy storybook to CI www
+      continue-on-error: true
+      uses: easingthemes/ssh-deploy@main
+      env:
+        SSH_PRIVATE_KEY: ${{ secrets.CI_WWW_SSH_PRIVATE_KEY }}
+        ARGS: "-rltgoDzvO --delete"
+        SOURCE: "explorer/storybook-static/"
+        REMOTE_HOST: ${{ secrets.CI_WWW_REMOTE_HOST }}
+        REMOTE_USER: ${{ secrets.CI_WWW_REMOTE_USER }}
+        TARGET: ${{ secrets.CI_WWW_REMOTE_TARGET }}/ne-sb-${{ env.GITHUB_REF_SLUG }}
+        EXCLUDE: "/dist/, /node_modules/"
     - name: Keybase - Node Install
       run: npm install
       working-directory: .github/workflows/support-files
@@ -51,6 +66,7 @@ jobs:
         NYM_PROJECT_NAME: "Network Explorer"
         NYM_CI_WWW_BASE: "${{ secrets.NYM_CI_WWW_BASE }}"
         NYM_CI_WWW_LOCATION: "network-explorer-${{ env.GITHUB_REF_SLUG }}"
+        NYM_CI_WWW_LOCATION_STORYBOOK: "ne-sb-${{ env.GITHUB_REF_SLUG }}"
         GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
         GIT_BRANCH: "${GITHUB_REF##*/}"
         KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
@@ -58,6 +74,22 @@ jobs:
         KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
         KEYBASE_NYM_CHANNEL: "ci-network-explorer"
         IS_SUCCESS: "${{ job.status == 'success' }}"
+        MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+        MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+        MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+        MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+        MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
       uses: docker://keybaseio/client:stable-node
       with:
         args: .github/workflows/support-files/notifications/entry_point.sh
+    - name: Deploy
+      if: github.event_name == 'workflow_dispatch'
+      uses: easingthemes/ssh-deploy@main
+      env:
+        SSH_PRIVATE_KEY: ${{ secrets.CD_PROD_NE_SSH_PRIVATE_KEY }}
+        ARGS: "-rltgoDzvO --delete"
+        SOURCE: "explorer/dist/"
+        REMOTE_HOST: ${{ secrets.CD_PROD_NE_REMOTE_HOST }}
+        REMOTE_USER: ${{ secrets.CD_PROD_NE_REMOTE_USER }}
+        TARGET: ${{ secrets.CD_PROD_NE_REMOTE_TARGET }}
+        EXCLUDE: "/dist/, /node_modules/"

.github/workflows/nightly_build.yml

@@ -2,10 +2,10 @@ name: Nightly builds
 
 on:
   schedule:
-    - cron: '14 4 * * *'
+    - cron: '14 1 * * *'
 jobs:
   matrix_prep:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -24,8 +24,9 @@ jobs:
     continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
     steps:
       - name: Install Dependencies (Linux)
-        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
-        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+        if: matrix.os == 'ubuntu-20.04'
 
       - name: Check out repository code
         uses: actions/checkout@v2
@@ -38,23 +39,54 @@ jobs:
           override: true
           components: rustfmt, clippy
 
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
       - name: Build all binaries
         uses: actions-rs/cargo@v1
         with:
           command: build
           args: --workspace
 
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Build all examples
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --examples
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
       - name: Run all tests
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --workspace
 
-      - name: Check formatting
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
+        with:
+          command: clean
+
+      - name: Run expensive tests
+        if: github.ref == 'refs/heads/develop' || github.event.pull_request.base.ref == 'develop' || github.event.pull_request.base.ref == 'master'
         uses: actions-rs/cargo@v1
         with:
-          command: fmt
-          args: --all -- --check
+          command: test
+          args: --workspace -- --ignored
 
       - name: Reclaim some disk space (because Windows is being annoying)
         uses: actions-rs/cargo@v1
@@ -64,9 +96,10 @@ jobs:
 
       - uses: actions-rs/clippy-check@v1
         name: Clippy checks
+        continue-on-error: true
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
+          args: --workspace
 
       - name: Run clippy
         uses: actions-rs/cargo@v1
@@ -75,39 +108,51 @@ jobs:
           command: clippy
           args: --workspace --all-targets -- -D warnings
 
-      # COCONUT stuff
-      - name: Build all binaries with coconut enabled
+      - name: Reclaim some disk space
         uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --features=coconut
-
-      - name: Run all tests with coconut enabled
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --features=coconut
-
-      - name: Reclaim some disk space (because Windows is being annoying)
-        uses: actions-rs/cargo@v1
-        if: ${{ matrix.os == 'windows-latest' }}
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
         with:
           command: clean
 
-      - name: Run clippy with coconut enabled
+      # nym-wallet (the rust part)
+      - name: Build nym-wallet rust code
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Run nym-wallet tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Check nym-wallet formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-wallet/Cargo.toml --all -- --check
+
+      - name: Run clippy for nym-wallet
         uses: actions-rs/cargo@v1
         if: ${{ matrix.rust != 'nightly' }}
         with:
           command: clippy
-          args: --workspace --all-targets --features=coconut -- -D warnings
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-targets -- -D warnings
+
   notification:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: custom-runner-linux
     steps:
       - name: Collect jobs status
         uses: technote-space/workflow-conclusion-action@v2
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: install npm
+        uses: actions/setup-node@v3
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        with:
+          node-version: 16
       - name: Keybase - Node Install
         if: env.WORKFLOW_CONCLUSION == 'failure'
         run: npm install
@@ -125,6 +170,11 @@ jobs:
           KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
           KEYBASE_NYM_CHANNEL: "ci-nightly"
           IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
+          MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+          MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+          MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+          MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+          MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
         uses: docker://keybaseio/client:stable-node
         with:
           args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nightly_build_matrix_includes.json

@@ -1,6 +1,6 @@
 [
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"stable",
     "runOnEvent":"schedule"
   },
@@ -17,7 +17,7 @@
   },
 
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"beta",
     "runOnEvent":"schedule"
   },
@@ -33,7 +33,7 @@
   },
 
   {
-    "os":"ubuntu-latest",
+    "os":"ubuntu-20.04",
     "rust":"nightly",
     "runOnEvent":"schedule"
   },

.github/workflows/nightly_build_release.yml

@@ -0,0 +1,183 @@
+name: Nightly builds on latest release
+
+on:
+  schedule:
+    - cron: '14 2 * * *'
+jobs:
+  matrix_prep:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      # creates the matrix strategy from nightly_build_matrix_includes.json
+      - uses: actions/checkout@v3
+      - id: set-matrix
+        uses: JoshuaTheMiller/conditional-build-matrix@main
+        with:
+          inputFile: '.github/workflows/nightly_build_matrix_includes.json'
+          filter: '[?runOnEvent==`${{ github.event_name }}` || runOnEvent==`always`]'
+  get_release:
+    runs-on: ubuntu-20.04
+    needs: matrix_prep
+    outputs:
+      output1: ${{ steps.step2.outputs.latest_release }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Fetch all branches
+        run: git fetch --all
+      - name: Set output variable to latest release branch
+        id: step2
+        run: echo "latest_release=$(git branch -r | grep -E 'release/v[0-9]+\.[0-9]+\.[0-9]+' | tail -n 1 | sed 's/  origin\///')" >> $GITHUB_OUTPUT
+  build:
+    needs: [get_release,matrix_prep]
+    strategy:
+      matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
+    steps:
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+        if: matrix.os == 'ubuntu-20.04'
+
+      - name: Check out latest release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{needs.get_release.outputs.output1}}
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run expensive tests
+        if: github.ref == 'refs/heads/develop' || github.event.pull_request.base.ref == 'develop' || github.event.pull_request.base.ref == 'master'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --all-features -- --ignored
+
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy checks
+        continue-on-error: true
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
+
+      - name: Run clippy
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --workspace --all-targets -- -D warnings
+
+      - name: Reclaim some disk space
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
+        with:
+          command: clean
+
+      # nym-wallet (the rust part)
+      - name: Build nym-wallet rust code
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Run nym-wallet tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Check nym-wallet formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-wallet/Cargo.toml --all -- --check
+
+      - name: Run clippy for nym-wallet
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-targets -- -D warnings
+
+  notification:
+    needs: [build,get_release]
+    runs-on: custom-runner-linux
+    steps:
+      - name: Collect jobs status
+        uses: technote-space/workflow-conclusion-action@v2
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: install npm
+        uses: actions/setup-node@v3
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        with:
+          node-version: 16
+      - name: Keybase - Node Install
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: npm install
+        working-directory: .github/workflows/support-files
+      - name: Keybase - Send Notification
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        env:
+          NYM_NOTIFICATION_KIND: nightly
+          NYM_PROJECT_NAME: "Nym nightly build on latest release"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+          GIT_BRANCH_NAME: "${{needs.get_release.outputs.output1}}"
+          KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+          KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "ci-nightly-release"
+          IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
+          MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+          MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+          MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+          MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+          MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
+        uses: docker://keybaseio/client:stable-node
+        with:
+          args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nightly_build_release2.yml

@@ -0,0 +1,183 @@
+name: Nightly builds on second latest release
+
+on:
+  schedule:
+    - cron: '24 2 * * *'
+jobs:
+  matrix_prep:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      # creates the matrix strategy from nightly_build_matrix_includes.json
+      - uses: actions/checkout@v3
+      - id: set-matrix
+        uses: JoshuaTheMiller/conditional-build-matrix@main
+        with:
+          inputFile: '.github/workflows/nightly_build_matrix_includes.json'
+          filter: '[?runOnEvent==`${{ github.event_name }}` || runOnEvent==`always`]'
+  get_release:
+    runs-on: ubuntu-20.04
+    needs: matrix_prep
+    outputs:
+      output1: ${{ steps.step2.outputs.latest_release }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Fetch all branches
+        run: git fetch --all
+      - name: Set output variable to latest release branch
+        id: step2
+        run: echo "latest_release=$(git branch -r | grep -E 'release/v[0-9]+\.[0-9]+\.[0-9]+' | tail -n 2 | head -n 1 | sed 's/  origin\///')" >> $GITHUB_OUTPUT
+  build:
+    needs: [get_release,matrix_prep]
+    strategy:
+      matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.rust == 'nightly' || matrix.rust == 'beta' || matrix.rust == 'stable' }}
+    steps:
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+        if: matrix.os == 'ubuntu-20.04'
+
+      - name: Check out latest release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{needs.get_release.outputs.output1}}
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - name: Run expensive tests
+        if: github.ref == 'refs/heads/develop' || github.event.pull_request.base.ref == 'develop' || github.event.pull_request.base.ref == 'master'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --all-features -- --ignored
+
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - name: Reclaim some disk space (because Windows is being annoying)
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' }}
+        with:
+          command: clean
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy checks
+        continue-on-error: true
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
+
+      - name: Run clippy
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --workspace --all-targets -- -D warnings
+
+      - name: Reclaim some disk space
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.os == 'windows-latest' || matrix.os == 'ubuntu-20.04' }}
+        with:
+          command: clean
+
+      # nym-wallet (the rust part)
+      - name: Build nym-wallet rust code
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Run nym-wallet tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Check nym-wallet formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-wallet/Cargo.toml --all -- --check
+
+      - name: Run clippy for nym-wallet
+        uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust != 'nightly' }}
+        with:
+          command: clippy
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-targets -- -D warnings
+
+  notification:
+    needs: [build,get_release]
+    runs-on: custom-runner-linux
+    steps:
+      - name: Collect jobs status
+        uses: technote-space/workflow-conclusion-action@v2
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: install npm
+        uses: actions/setup-node@v3
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        with:
+          node-version: 16
+      - name: Keybase - Node Install
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: npm install
+        working-directory: .github/workflows/support-files
+      - name: Keybase - Send Notification
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        env:
+          NYM_NOTIFICATION_KIND: nightly
+          NYM_PROJECT_NAME: "Nym nightly build on latest release"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+          GIT_BRANCH_NAME: "${{needs.get_release.outputs.output1}}"
+          KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+          KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+          KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+          KEYBASE_NYM_CHANNEL: "ci-nightly-release"
+          IS_SUCCESS: "${{ env.WORKFLOW_CONCLUSION == 'success' }}"
+          MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+          MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+          MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+          MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+          MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
+        uses: docker://keybaseio/client:stable-node
+        with:
+          args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nym-cli-publish.yml

@@ -0,0 +1,50 @@
+name: Publish Nym CLI binaries
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+    
+env:
+  NETWORK: mainnet
+
+jobs:
+  publish-nym-cli:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-cli-') && (github.event_name == 'release' || github.event_name = 'workflow_dispatch') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04, windows-latest, macos-latest]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check the release tag starts with `nym-cli-`
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('Release tag did not start with nym-cli-...')
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build binary
+        run: make build-nym-cli
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nym-cli-${{ matrix.platform }}
+          path: |
+            target/release/nym-cli*
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            target/release/nym-cli

.github/workflows/nym-connect-publish-macos.yml

@@ -0,0 +1,90 @@
+name: Publish Nym Connect (MacOS)
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+
+defaults:
+  run:
+    working-directory: nym-connect
+
+jobs:
+  publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-connect-') && github.event_name == 'release' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [macos-latest]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Node v16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Install the Apple developer certificate for code signing
+        env:
+          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        run: |
+          # create variables
+          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
+          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
+
+          # import certificate and provisioning profile from secrets
+          echo -n "$APPLE_CERTIFICATE" | base64 --decode --output $CERTIFICATE_PATH
+
+          # create temporary keychain
+          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+
+          # import certificate to keychain
+          security import $CERTIFICATE_PATH -P "$APPLE_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
+          security list-keychain -d user -s $KEYCHAIN_PATH
+
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
+      - name: Install app dependencies and build it
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ENABLE_CODE_SIGNING: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
+          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
+          APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_IDENTITY_ID }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
+          TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }}
+          TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
+        run: yarn && yarn build
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nym-connect_1.0.0_x64.dmg
+          path: nym-connect/target/release/bundle/dmg/nym-connect_1.0.0_x64.dmg
+          retention-days: 30
+
+      - name: Clean up keychain
+        if: ${{ always() }}
+        run: |
+          security delete-keychain $RUNNER_TEMP/app-signing.keychain-db
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            nym-connect/target/release/bundle/dmg/*.dmg
+            nym-connect/target/release/bundle/macos/*.app.tar.gz*

.github/workflows/nym-connect-publish-ubuntu.yml

@@ -0,0 +1,65 @@
+name: Publish Nym Connect (Ubuntu)
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+
+defaults:
+  run:
+    working-directory: nym-connect
+
+jobs:
+  publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-connect-') && github.event_name == 'release' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [custom-runner-linux]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Tauri dependencies
+        run: >
+          sudo apt-get update &&
+          sudo apt-get install -y webkit2gtk-4.0 libayatana-appindicator3-dev
+        continue-on-error: true
+
+      - name: Node v16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - name: Install app dependencies
+        run: yarn
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
+      - name: Build app
+        run: yarn build
+        env:
+          TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }}
+          TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nym-connect.AppImage.tar.gz
+          path: nym-connect/target/release/bundle/appimage/nym-connect_1.0.0_amd64.AppImage
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            nym-connect/target/release/bundle/appimage/*.AppImage
+            nym-connect/target/release/bundle/appimage/*.AppImage.tar.gz*

.github/workflows/nym-connect-publish-windows10.yml

@@ -0,0 +1,84 @@
+name: Publish Nym Connect (Windows 10)
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+
+defaults:
+  run:
+    working-directory: nym-connect
+
+jobs:
+  publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-connect-') && github.event_name == 'release' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [windows10]
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Clean up first
+        continue-on-error: true
+        working-directory: .
+        run: |
+          cd ..
+          del /s /q /A:H nym
+          rmdir /s /q nym
+
+      - uses: actions/checkout@v3
+
+      - name: Import signing certificate
+        env:
+          WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
+          WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
+        run: |
+          New-Item -ItemType directory -Path certificate
+          Set-Content -Path certificate/tempCert.txt -Value $env:WINDOWS_CERTIFICATE
+          certutil -decode certificate/tempCert.txt certificate/certificate.pfx
+          Remove-Item -path certificate -include tempCert.txt
+          Import-PfxCertificate -FilePath certificate/certificate.pfx -CertStoreLocation Cert:\CurrentUser\My -Password (ConvertTo-SecureString -String $env:WINDOWS_CERTIFICATE_PASSWORD -Force -AsPlainText)
+
+      - name: Node v16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install Rust stable
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
+      - name: Install app dependencies
+        run: yarn
+
+      - name: Build and sign it
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ENABLE_CODE_SIGNING: ${{ secrets.WINDOWS_CERTIFICATE }}
+          WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
+          WINDOWS_CERTIFICATE_PASSWORD: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }}
+          TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }}
+          TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
+        run: yarn build
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: nym-connect_1.0.0_x64_en-US.msi
+          path: nym-connect/target/release/bundle/msi/nym-connect_1.0.0_x64_en-US.msi
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            nym-connect/target/release/bundle/msi/*.msi
+            nym-connect/target/release/bundle/msi/*.msi.zip*

.github/workflows/nym-connect.yml

@@ -0,0 +1,65 @@
+name: CI for nym-connect
+
+on:
+  push:
+    paths:
+      - 'nym-connect/**'
+
+defaults:
+  run:
+    working-directory: nym-connect
+
+jobs:
+  build:
+    runs-on: custom-runner-linux
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install rsync
+      run: sudo apt-get install rsync
+      continue-on-error: true
+    - uses: rlespinasse/github-slug-action@v3.x
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    - name: Install Yarn
+      run: npm install -g yarn
+    - run: yarn
+      continue-on-error: true
+    - name: Set environment from the example
+      run: cp .env.sample .env
+    - run: yarn storybook:build
+    - name: Deploy branch to CI www
+      continue-on-error: true
+      uses: easingthemes/ssh-deploy@main
+      env:
+        SSH_PRIVATE_KEY: ${{ secrets.CI_WWW_SSH_PRIVATE_KEY }}
+        ARGS: "-rltgoDzvO --delete"
+        SOURCE: "nym-connect/storybook-static/"
+        REMOTE_HOST: ${{ secrets.CI_WWW_REMOTE_HOST }}
+        REMOTE_USER: ${{ secrets.CI_WWW_REMOTE_USER }}
+        TARGET: ${{ secrets.CI_WWW_REMOTE_TARGET }}/nym-connect-${{ env.GITHUB_REF_SLUG }}
+        EXCLUDE: "/dist/, /node_modules/"
+    - name: Keybase - Node Install
+      run: npm install
+      working-directory: .github/workflows/support-files
+    - name: Keybase - Send Notification
+      env:
+        NYM_NOTIFICATION_KIND: nym-connect
+        NYM_PROJECT_NAME: "nym-connect"
+        NYM_CI_WWW_BASE: "${{ secrets.NYM_CI_WWW_BASE }}"
+        NYM_CI_WWW_LOCATION: "nym-connect-${{ env.GITHUB_REF_SLUG }}"
+        GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+        GIT_BRANCH: "${GITHUB_REF##*/}"
+        KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+        KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+        KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+        KEYBASE_NYM_CHANNEL: "ci-nym-connect"
+        IS_SUCCESS: "${{ job.status == 'success' }}"
+        MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+        MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+        MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+        MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+        MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
+      uses: docker://keybaseio/client:stable-node
+      with:
+        args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nym-release-publish.yml

@@ -1,5 +1,13 @@
 name: Publish Nym binaries
+
 on:
+  workflow_dispatch:
+    inputs:  
+      add_tokio_unstable:
+        description: 'True to add RUSTFLAGS="--cfg tokio_unstable"'
+        required: true
+        default: false
+        type: boolean 
   release:
     types: [created]
     
@@ -8,21 +16,24 @@ env:
 
 jobs:
   publish-nym:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-binaries-') && github.event_name == 'release' }}
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [custom-runner-linux]
 
     runs-on: ${{ matrix.platform }}
     steps:
       - uses: actions/checkout@v3
 
-      - name: Check the release tag starts with `nym-binaries-`
-        if: startsWith(github.ref, 'refs/tags/nym-binaries-') == false
-        uses: actions/github-script@v3
-        with:
-          script: |
-            core.setFailed('Release tag did not start with nym-binaries-...')
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev libudev-dev squashfs-tools
+        continue-on-error: true
+            
+      - name: Sets env vars for tokio if set in manual dispatch inputs
+        run: |
+          echo 'RUSTFLAGS="--cfg tokio_unstable"' >> $GITHUB_ENV
+        if: github.event_name == 'workflow_dispatch' && inputs.add_tokio_unstable == true
 
       - name: Install Rust stable
         uses: actions-rs/toolchain@v1
@@ -35,12 +46,33 @@ jobs:
           command: build
           args: --workspace --release
 
-      - name: Upload to release based on tag name
-        uses: softprops/action-gh-release@v1
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
         with:
-          files: |
+          name: my-artifact
+          path: |
+            target/release/explorer-api
             target/release/nym-client
             target/release/nym-gateway
             target/release/nym-mixnode
             target/release/nym-socks5-client
-            target/release/nym-validator-api
+            target/release/nym-api
+            target/release/nym-network-requester
+            target/release/nym-network-statistics
+            target/release/nym-cli
+          retention-days: 30
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            target/release/explorer-api
+            target/release/nym-client
+            target/release/nym-gateway
+            target/release/nym-mixnode
+            target/release/nym-socks5-client
+            target/release/nym-api
+            target/release/nym-network-requester
+            target/release/nym-network-statistics
+            target/release/nym-cli

.github/workflows/nym-wallet-publish-macos.yml

@@ -1,5 +1,6 @@
 name: Publish Nym Wallet (MacOS)
 on:
+  workflow_dispatch:
   release:
     types: [created]
 
@@ -9,6 +10,7 @@ defaults:
 
 jobs:
   publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-wallet-') && github.event_name == 'release' }}
     strategy:
       fail-fast: false
       matrix:
@@ -18,17 +20,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Check the release tag starts with `nym-wallet-`
-        if: startsWith(github.ref, 'refs/tags/nym-wallet-') == false
-        uses: actions/github-script@v3
-        with:
-          script: |
-            core.setFailed('Release tag did not start with nym-wallet-...')
-
       - name: Node v16
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 16
       - name: Install Rust stable
         uses: actions-rs/toolchain@v1
         with:
@@ -55,6 +50,12 @@ jobs:
           security import $CERTIFICATE_PATH -P "$APPLE_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
           security list-keychain -d user -s $KEYCHAIN_PATH
 
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
       - name: Install app dependencies and build it
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -68,14 +69,22 @@ jobs:
           TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
         run: yarn && yarn build
 
-      - name: Upload to release based on tag name
-        uses: softprops/action-gh-release@v1
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
         with:
-          files: |
-            nym-wallet/target/release/bundle/dmg/*.dmg
-            nym-wallet/target/release/bundle/macos/*.app.tar.gz*
+          name: nym-wallet.app.tar.gz
+          path: nym-wallet/target/release/bundle/macos/nym-wallet.app.tar.gz
+          retention-days: 5
 
       - name: Clean up keychain
         if: ${{ always() }}
         run: |
           security delete-keychain $RUNNER_TEMP/app-signing.keychain-db
+
+      - name: Upload to release based on tag name
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'release'
+        with:
+          files: |
+            nym-wallet/target/release/bundle/dmg/*.dmg
+            nym-wallet/target/release/bundle/macos/*.app.tar.gz*

.github/workflows/nym-wallet-publish-ubuntu.yml

@@ -9,10 +9,11 @@ defaults:
 
 jobs:
   publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-wallet-') && github.event_name == 'release' }}
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [custom-runner-linux]
 
     runs-on: ${{ matrix.platform }}
     steps:
@@ -22,23 +23,25 @@ jobs:
         run: >
           sudo apt-get update &&
           sudo apt-get install -y webkit2gtk-4.0
-      - name: Check the release tag starts with `nym-wallet-`
-        if: startsWith(github.ref, 'refs/tags/nym-wallet-') == false
-        uses: actions/github-script@v3
-        with:
-          script: |
-            core.setFailed('Release tag did not start with nym-wallet-...')
+        continue-on-error: true
 
       - name: Node v16
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 16
+
       - name: Install Rust stable
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
       - name: Install app dependencies
         run: yarn
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
       - name: Build app
         run: yarn build
         env:

.github/workflows/nym-wallet-publish-windows10.yml

@@ -9,6 +9,7 @@ defaults:
 
 jobs:
   publish-tauri:
+    if: ${{ startsWith(github.ref, 'refs/tags/nym-wallet-') && github.event_name == 'release' }}
     strategy:
       fail-fast: false
       matrix:
@@ -26,13 +27,6 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - name: Check the release tag starts with `nym-wallet-`
-        if: startsWith(github.ref, 'refs/tags/nym-wallet-') == false
-        uses: actions/github-script@v3
-        with:
-          script: |
-            core.setFailed('Release tag did not start with nym-wallet-...')
-
       - name: Import signing certificate
         env:
           WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
@@ -45,15 +39,21 @@ jobs:
           Import-PfxCertificate -FilePath certificate/certificate.pfx -CertStoreLocation Cert:\CurrentUser\My -Password (ConvertTo-SecureString -String $env:WINDOWS_CERTIFICATE_PASSWORD -Force -AsPlainText)
 
       - name: Node v16
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 16
 
       - name: Install Rust stable
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
 
+      - name: Create env file
+        uses: timheuer/base64-to-file@v1.1
+        with:
+          fileName: '.env'
+          encodedString: ${{ secrets.WALLET_ADMIN_ADDRESS }}
+
       - name: Install app dependencies
         run: yarn
 

.github/workflows/nym-wallet-release.yml

@@ -0,0 +1,32 @@
+name: Release Nym Wallet
+on:
+  workflow_dispatch:
+    inputs:
+      nym_wallet_version:
+        description: 'The version of the Nym Wallet to release'
+        default: '1.0.x'
+        required: true
+        type: string
+jobs:
+  create-release:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-20.04]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Create release
+        uses: softprops/action-gh-release@v1
+        with:
+          body: >-
+            This is a pre-release
+            
+            Download the wallet for your platform:
+            
+              - [Linux](https://github.com/nymtech/nym/releases/download/nym-wallet-v${{ inputs.nym_wallet_version}}/nym-wallet_v${{ inputs.nym_wallet_version}}_amd64_ubuntu20.04.AppImage)
+              - [MacOS](https://github.com/nymtech/nym/releases/download/nym-wallet-v${{ inputs.nym_wallet_version}}/nym-wallet_v${{ inputs.nym_wallet_version}}_x64_macos_11.dmg)
+              - [Windows](https://github.com/nymtech/nym/releases/download/nym-wallet-v${{ inputs.nym_wallet_version}}/nym-wallet_v${{ inputs.nym_wallet_version}}_x64_windows.msi)
+          prerelease: true
+          name: Nym Wallet v${{ inputs.nym_wallet_version}}
+          tag_name: nym-wallet-v${{ inputs.nym_wallet_version}}

.github/workflows/nym-wallet-storybook.yml

@@ -12,10 +12,11 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install rsync
       run: sudo apt-get install rsync
+      continue-on-error: true
     - uses: rlespinasse/github-slug-action@v3.x
-    - uses: actions/setup-node@v2
+    - uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: 16
     - name: Setup yarn
       run: npm install -g yarn
     - name: Build dependencies
@@ -50,6 +51,11 @@ jobs:
         KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
         KEYBASE_NYM_CHANNEL: "ci-nym-wallet"
         IS_SUCCESS: "${{ job.status == 'success' }}"
+        MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+        MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+        MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+        MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+        MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
       uses: docker://keybaseio/client:stable-node
       with:
         args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/nym_wallet.yml

@@ -12,7 +12,7 @@ defaults:
 jobs:
   test:
     name: wallet tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
 
@@ -26,6 +26,7 @@ jobs:
           libappindicator3-dev
           webkit2gtk-driver
           xvfb
+        continue-on-error: true
 
       - name: Install minimal stable
         uses: actions-rs/toolchain@v1
@@ -34,9 +35,9 @@ jobs:
           toolchain: stable
 
       - name: Node v16
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 16
 
       - name: Install yarn for building application
         run: yarn install

.github/workflows/support-files/.env.example

@@ -2,6 +2,13 @@ KEYBASE_NYM_CHANNEL=
 KEYBASE_NYMBOT_USERNAME=
 KEYBASE_NYMBOT_PAPERKEY=
 
+MATRIX_SERVER=
+MATRIX_ROOM=
+MATRIX_ROOM_OF_SHAME=
+MATRIX_USER_ID=
+MATRIX_TOKEN=
+MATRIX_DEVICE_ID=
+
 NYM_NOTIFICATION_KIND=nightly
 NYM_PROJECT_NAME=Nightly Build
 

.github/workflows/support-files/.gitignore

@@ -2,4 +2,6 @@ node_modules
 .idea
 
 # don't commit the lock file to avoid cross-platform issues
-package-lock.json
+package-lock.json
+
+scratch

.github/workflows/support-files/network-explorer/templates/success

@@ -1,5 +1,6 @@
 🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩
 > :rocket: {{ env.NYM_PROJECT_NAME }}  ➡️➡️➡️➡️➡️ **View output:** https://{{ env.NYM_CI_WWW_LOCATION }}.{{ env.NYM_CI_WWW_BASE }}/
+> `storybook`: https://{{ env.NYM_CI_WWW_LOCATION_STORYBOOK }}.{{ env.NYM_CI_WWW_BASE }}
 > ✅ **SUCCESS**
 > `branch` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/tree/{{ env.GIT_BRANCH_NAME }}
 > `commit` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/commit/{{ env.GITHUB_SHA }}

.github/workflows/support-files/nightly/index.js

@@ -85,7 +85,7 @@ async function getMessageBody(context) {
             ...
           ],
           check_run_url: 'https://api.github.com/repos/nymtech/nym/check-runs/5182940024',
-          labels: [ 'ubuntu-latest' ],
+          labels: [ 'ubuntu-20.04' ],
           runner_id: 1,
           runner_name: 'Hosted Agent',
           runner_group_id: 2,

.github/workflows/support-files/notifications/send_message.js

@@ -1,9 +1,10 @@
 require('dotenv').config();
 
 const Bot = require('keybase-bot');
+const { sendMatrixMessage } = require('./send_message_to_matrix');
 
 let context = {
-  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly'],
+  kinds: ['nym-wallet', 'ts-packages', 'network-explorer', 'nightly', 'nym-connect','security'],
 };
 
 /**
@@ -38,6 +39,28 @@ function validateContext() {
       'Paperkey is not defined. Please set env var KEYBASE_NYMBOT_PAPERKEY',
     );
   }
+  if (context.env.MATRIX_ROOM) {
+    if (!context.env.MATRIX_SERVER) {
+      throw new Error(
+        'Matrix server is not defined. Please set env var MATRIX_SERVER',
+      );
+    }
+    if (!context.env.MATRIX_USER_ID) {
+      throw new Error(
+        'Matrix user id is not defined. Please set env var MATRIX_USER_ID',
+      );
+    }
+    if (!context.env.MATRIX_TOKEN) {
+      throw new Error(
+        'Matrix token is not defined. Please set env var MATRIX_TOKEN',
+      );
+    }
+    if (!context.env.MATRIX_DEVICE_ID) {
+      throw new Error(
+        'Matrix device id is not defined. Please set env var MATRIX_DEVICE_ID',
+      );
+    }
+  }
 }
 
 /**
@@ -89,7 +112,7 @@ async function sendKeybaseMessage(messageBody) {
     });
 
     const channel = {
-      name: 'nymtech_bot',
+      name: context.env.KEYBASE_NYMBOT_TEAM || 'nymtech_bot',
       membersType: 'team',
       topicName: context.keybase.channel,
       topic_type: 'CHAT',
@@ -147,6 +170,13 @@ async function main() {
     console.log('-----------------------------------------');
   }
   await sendKeybaseMessage(messageBody);
+  if(context.env.MATRIX_ROOM) {
+    await sendMatrixMessage(context, messageBody, context.env.MATRIX_ROOM)
+  }
+  if(context.env.MATRIX_ROOM_OF_SHAME && context.env.IS_SUCCESS !== 'true') {
+    // when a job fails
+    await sendMatrixMessage(context, messageBody, context.env.MATRIX_ROOM_OF_SHAME)
+  }
 }
 
 // call main function and let NodeJS handle the promise

.github/workflows/support-files/notifications/send_message_to_matrix.js

@@ -0,0 +1,80 @@
+const sdk = require('matrix-js-sdk');
+global.Olm = require('olm');
+const { LocalStorage } = require('node-localstorage');
+const localStorage = new LocalStorage('./scratch');
+const {
+  LocalStorageCryptoStore,
+} = require('matrix-js-sdk/lib/crypto/store/localStorage-crypto-store');
+
+const vfile = require('to-vfile');
+const unified = require('unified');
+const remarkParse = require('remark-parse');
+const remarkHtml = require('remark-html');
+const emoji = require('remark-emoji');
+
+// hide all matrix client output
+console.error = (error) => console.log('❌ error: ', error);
+process.stderr.write = () => {};
+process.stdout.write = () => {};
+
+
+function createClient(context, room, message) {
+  const server = context.env.MATRIX_SERVER;
+  const token = context.env.MATRIX_TOKEN;
+  const deviceId = context.env.MATRIX_DEVICE_ID;
+  const userId = context.env.MATRIX_USER_ID;
+
+  const client = sdk.createClient({
+    baseUrl: server,
+    accessToken: token,
+    userId,
+    deviceId,
+    sessionStore: new sdk.WebStorageSessionStore(localStorage),
+    cryptoStore: new LocalStorageCryptoStore(localStorage),
+  });
+
+  client.on('sync', async function(state, prevState, res) {
+    if (state !== 'PREPARED') return;
+    client.setGlobalErrorOnUnknownDevices(false);
+    try {
+      await client.joinRoom(room);
+      await client.sendEvent(
+        room,
+        'm.room.message',
+        {
+          msgtype: 'm.text',
+          format: 'org.matrix.custom.html',
+          body: message,
+          formatted_body: message,
+        },
+        '',
+      );
+    } catch (error) {
+      console.error('Job failed: ' + error.message);
+    }
+    client.stopClient();
+    process.exit(0);
+  });
+
+  return client;
+}
+
+async function markdownToHtml(messageAsMarkdown) {
+  const file = await unified()
+    .use(emoji)
+    .use(remarkParse)
+    .use(remarkHtml)
+    .process(await vfile({ path: 'test.md', contents: messageAsMarkdown}));
+  return String(file);
+}
+
+async function sendMatrixMessage(contextArg, messageAsMarkdown, roomId) {
+  const messageAsHtml = await markdownToHtml(messageAsMarkdown);
+  const client = createClient(contextArg, roomId, messageAsHtml);
+  await client.initCrypto();
+  await client.startClient({ initialSyncLimit: 1 });
+}
+
+module.exports = {
+  sendMatrixMessage,
+};

.github/workflows/support-files/nym-connect/index.js

@@ -0,0 +1,29 @@
+const Handlebars = require('handlebars');
+const fs = require('fs');
+const path = require('path');
+
+async function addToContextAndValidate(context) {
+  if (!context.env.NYM_CI_WWW_LOCATION) {
+    throw new Error('Please ensure the env var NYM_CI_WWW_LOCATION is set');
+  }
+  if (!context.env.NYM_CI_WWW_BASE) {
+    throw new Error('Please ensure the env var NYM_CI_WWW_BASE is set');
+  }
+}
+
+async function getMessageBody(context) {
+  const source = fs
+    .readFileSync(
+      context.env.IS_SUCCESS === 'true'
+        ? path.resolve(__dirname, 'templates', 'success')
+        : path.resolve(__dirname, 'templates', 'failure'),
+    )
+    .toString();
+  const template = Handlebars.compile(source);
+  return template(context);
+}
+
+module.exports = {
+  addToContextAndValidate,
+  getMessageBody,
+};

.github/workflows/support-files/nym-connect/templates/failure

@@ -0,0 +1,11 @@
+🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥🟥
+> :rocket: {{ env.NYM_PROJECT_NAME }}
+> 🔴 **FAILURE** :cry:
+> `branch` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/tree/{{ env.GIT_BRANCH_NAME }}
+> `commit` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/commit/{{ env.GITHUB_SHA }}
+> `build ` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/actions/runs/{{ env.GITHUB_RUN_ID }}
+
+Commit message:
+```
+{{ env.GIT_COMMIT_MESSAGE }}
+```

.github/workflows/support-files/nym-connect/templates/success

@@ -0,0 +1,11 @@
+🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩
+> :rocket: {{ env.NYM_PROJECT_NAME }}  ➡️➡️➡️➡️➡️ **View storybook:** https://{{ env.NYM_CI_WWW_LOCATION }}.{{ env.NYM_CI_WWW_BASE }}/
+> ✅ **SUCCESS**
+> `branch` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/tree/{{ env.GIT_BRANCH_NAME }}
+> `commit` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/commit/{{ env.GITHUB_SHA }}
+> `build ` {{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/actions/runs/{{ env.GITHUB_RUN_ID }}
+
+Commit message by `{{ env.GITHUB_ACTOR }}` at {{ timestamp }}:
+```
+{{ env.GIT_COMMIT_MESSAGE }}
+```

.github/workflows/support-files/package.json

@@ -11,7 +11,15 @@
     "dotenv": "^16.0.0",
     "handlebars": "^4.7.7",
     "keybase-bot": "^3.6.1",
-    "octokit": "^1.7.1"
+    "matrix-js-sdk": "^9.3.0",
+    "node-localstorage": "^2.1.6",
+    "octokit": "^1.7.1",
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "remark-emoji": "^2.2.0",
+    "remark-html": "^13.0.2",
+    "remark-parse": "^9.0.0",
+    "to-vfile": "^6.1.0",
+    "unified": "^9.2.2"
   },
   "devDependencies": {
     "prettier": "2.3.2"

.github/workflows/support-files/security/index.js

@@ -0,0 +1,24 @@
+const Handlebars = require('handlebars');
+const fs = require('fs');
+const path = require('path');
+const { Octokit, App } = require('octokit');
+
+async function addToContextAndValidate(context) {
+  return
+}
+
+async function getMessageBody(context) {
+  try {
+    const source = fs
+    .readFileSync("deny.message").toString();
+    return source;
+} catch (error) {
+    console.error(error);
+}
+
+}
+
+module.exports = {
+  addToContextAndValidate,
+  getMessageBody,
+};

.github/workflows/typescript-lint.yml

@@ -0,0 +1,51 @@
+name: CI for linting Typescript
+
+on:
+  push:
+    paths:
+      - 'ts-packages/**'
+      - 'sdk/typescript/**'
+      - nym-connect
+      - nym-connect-android
+      - nym-wallet
+
+jobs:
+  build:
+    runs-on: custom-runner-linux
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install rsync
+      run: sudo apt-get install rsync
+      continue-on-error: true
+    - uses: rlespinasse/github-slug-action@v3.x
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    - name: Setup yarn
+      run: npm install -g yarn
+    - name: Lint
+      run: yarn && yarn lint && yarn tsc
+    - name: Keybase - Node Install
+      run: npm install
+      working-directory: .github/workflows/support-files
+    - name: Keybase - Send Notification
+      env:
+        NYM_NOTIFICATION_KIND: ts-packages
+        NYM_PROJECT_NAME: "ts-packages"
+        NYM_CI_WWW_BASE: "${{ secrets.NYM_CI_WWW_BASE }}"
+        NYM_CI_WWW_LOCATION: "ts-${{ env.GITHUB_REF_SLUG }}"
+        GIT_COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
+        GIT_BRANCH: "${GITHUB_REF##*/}"
+        KEYBASE_NYMBOT_USERNAME: "${{ secrets.KEYBASE_NYMBOT_USERNAME }}"
+        KEYBASE_NYMBOT_PAPERKEY: "${{ secrets.KEYBASE_NYMBOT_PAPERKEY }}"
+        KEYBASE_NYMBOT_TEAM: "${{ secrets.KEYBASE_NYMBOT_TEAM }}"
+        KEYBASE_NYM_CHANNEL: "ci-ts-packages"
+        IS_SUCCESS: "${{ job.status == 'success' }}"
+        MATRIX_SERVER: "${{ secrets.MATRIX_SERVER }}"
+        MATRIX_ROOM: "${{ secrets.MATRIX_ROOM }}"
+        MATRIX_USER_ID: "${{ secrets.MATRIX_USER_ID }}"
+        MATRIX_TOKEN: "${{ secrets.MATRIX_TOKEN }}"
+        MATRIX_DEVICE_ID: "${{ secrets.MATRIX_DEVICE_ID }}"
+      uses: docker://keybaseio/client:stable-node
+      with:
+        args: .github/workflows/support-files/notifications/entry_point.sh

.github/workflows/wallet.yml

@@ -0,0 +1,67 @@
+name: Nym Wallet (rust)
+
+on:
+  push:
+    paths:
+      - 'nym-wallet/**'
+      - 'common/**'
+      - 'contracts/vesting/**'
+      - 'nym-api/nym-api-requests/**'
+  pull_request:
+    paths:
+      - 'nym-wallet/**'
+      - 'common/**'
+      - 'contracts/vesting/**'
+      - 'nym-api/nym-api-requests/**'
+
+jobs:
+  build:
+    runs-on: [ self-hosted, custom-linux ]
+    env:
+      RUSTC_WRAPPER: /home/ubuntu/.cargo/bin/sccache
+    steps:
+      - name: Install Dependencies (Linux)
+        run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools
+        continue-on-error: true
+
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt, clippy
+
+      - name: Build all binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Run all tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path nym-wallet/Cargo.toml --workspace
+
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path nym-wallet/Cargo.toml --all -- --check
+
+      - uses: actions-rs/clippy-check@v1
+        name: Clippy checks
+        continue-on-error: true
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-features
+
+      - name: Run clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --manifest-path nym-wallet/Cargo.toml --workspace --all-features -- -D warnings

.github/workflows/wasm_client_build.yml

@@ -2,12 +2,17 @@ name: Wasm Client
 
 on:
   pull_request:
-    paths-ignore:
-      - 'explorer/**'
+    paths:
+      - 'clients/webassembly/**'
+      - 'clients/client-core/**'
+      - 'common/**'
+      - 'contracts/**'
+      - 'gateway/gateway-requests/**'
+      - 'nym-api/nym-api-requests/**'
 
 jobs:
   wasm:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
 
@@ -24,16 +29,6 @@ jobs:
           command: build
           args: --manifest-path clients/webassembly/Cargo.toml --target wasm32-unknown-unknown
 
-      - uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --manifest-path clients/webassembly/Cargo.toml --target wasm32-unknown-unknown --features=coconut
-
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --manifest-path clients/webassembly/Cargo.toml
-
       - uses: actions-rs/cargo@v1
         with:
           command: fmt

.gitignore

@@ -37,4 +37,9 @@ validator-config
 *.patch
 validator-api-config.toml
 dist
-storybook-static
+storybook-static
+envs/qwerty.env
+Cargo.lock
+nym-connect/Cargo.lock
+.parcel-cache
+**/.DS_Store

.wellknown/wallet/validators.json

@@ -1,6 +1,8 @@
 {
-  "mainnet":[{
-    "nymd_url":"https://rpc.nyx.nodes.guru/",
-    "api_url":"https://api.nyx.nodes.guru/"
-  }]
-}
+  "mainnet": [
+    {
+      "nyxd_url": "https://rpc.nyx.nodes.guru/",
+      "api_url": "https://api.nyx.nodes.guru/"
+    }
+  ]
+}

CHANGELOG.md

@@ -1,5 +1,356 @@
 # Changelog
 
+Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+# [Unreleased]
+
+### Added
+
+- remove coconut feature and unify builds ([#2890])
+- native-client: is now capable of listening for requests on sockets different than `127.0.0.1` ([#2939]). This can be specified via `--host` flag during `init` or `run`. Alternatively a custom `host` can be set in `config.toml` file under `socket` section.
+- dkg resharing mode ([#2936])
+
+[#2890]: https://github.com/nymtech/nym/pull/2890
+[#2939]: https://github.com/nymtech/nym/pull/2939
+[#2936]: https://github.com/nymtech/nym/pull/2936
+
+# [v1.1.8] (2023-01-31)
+
+### Added
+
+- Rust SDK - Support SURBS (anonymous send + storage) ([#2754])
+- dkg rerun from scratch and dkg-specific epochs ([#2810])
+- Rename `'initial_supply'` field to `'total_supply'` in the circulating supply endpoint ([#2931])
+- Circulating supply api endpoint (read the note inside before testing/deploying) ([#1902])
+
+### Changed
+
+- nym-api: an `--id` flag is now always explicitly required ([#2873])
+
+[#2754]: https://github.com/nymtech/nym/issues/2754
+[#2810]: https://github.com/nymtech/nym/issues/2810
+[#2931]: https://github.com/nymtech/nym/issues/2931
+[#1902]: https://github.com/nymtech/nym/issues/1902
+[#2873]: https://github.com/nymtech/nym/issues/2873
+
+
+# [v1.1.7] (2023-01-24)
+
+### Added
+
+- Gateways now shut down gracefully ([#2019]).
+- Rust SDK - Initial version for nym-client ([#2669]).
+- Introduce vesting contract query for addresses of all vesting accounts (required for the circulating supply calculation) ([#2778]).
+- Add threshold value to the contract storage ([#1893])
+
+### Changed
+
+- Refactor vesting account storage (and in particular, ACCOUNTS saving) ([#2795]).
+- Move from manual advancing DKG state to an automatic process ([#2670]).
+
+### Fixed
+
+- Gateways now shut down gracefully ([#2019]).
+
+[#2019]: https://github.com/nymtech/nym/issues/2019
+[#2669]: https://github.com/nymtech/nym/issues/2669
+[#2795]: https://github.com/nymtech/nym/issues/2795
+[#2778]: https://github.com/nymtech/nym/issues/2778
+[#2670]: https://github.com/nymtech/nym/issues/2670
+[#1893]: https://github.com/nymtech/nym/issues/1893
+
+## [v1.1.6] (2023-01-17)
+
+### Added
+
+- nym-sdk: added initial version of a Rust client sdk
+- nym-api: added `/circulating-supply` endpoint ([#2814])
+- nym-api: add endpoint listing detailed gateway info by @octol in https://github.com/nymtech/nym/pull/2833
+
+### Changed
+
+- streamline override_config functions -> there's a lot of duplicate if statements everywhere ([#2774])
+- clean-up nym-api startup arguments/flags to use clap 3 and its macro-derived arguments ([#2772])
+- renamed all references to validator_api to nym_api
+- renamed all references to nymd to nyxd ([#2696])
+- all-binaries: standarised argument names (note: old names should still be accepted) ([#2762]
+
+### Fixed
+
+- nym-api: should now correctly use `rewarding.enabled` config flag ([#2753])
+
+[#2696]: https://github.com/nymtech/nym/pull/2696
+[#2753]: https://github.com/nymtech/nym/pull/2753
+[#2762]: https://github.com/nymtech/nym/pull/2762
+[#2814]: https://github.com/nymtech/nym/pull/2814
+[#2772]: https://github.com/nymtech/nym/pull/2772
+[#2774]: https://github.com/nymtech/nym/pull/2774
+
+## [v1.1.5] (2023-01-10)
+
+### Added
+
+### Changed
+
+## [v1.1.5] (2023-01-10)
+
+### Added
+
+- socks5: send status message for service ready, and network-requester error response in https://github.com/nymtech/nym/pull/2715
+
+### Changed
+
+- all-binaries: improved error logging in https://github.com/nymtech/nym/pull/2686
+- native client: bring shutdown logic up to the same level as socks5-client in https://github.com/nymtech/nym/pull/2695
+- nym-api, coconut-dkg contract: automatic, time-based dkg epoch state advancement in https://github.com/nymtech/nym/pull/2670
+- DKG resharing unit test by @neacsu in https://github.com/nymtech/nym/pull/2668
+- Renaming validator-api to nym-api by @futurechimp in https://github.com/nymtech/nym/pull/1863
+- Modify wasm specific make targets by @neacsu in https://github.com/nymtech/nym/pull/2693
+- client: create websocket handler builder by @octol in https://github.com/nymtech/nym/pull/2700
+- Outfox and Lion by @durch in https://github.com/nymtech/nym/pull/2730
+- Feature/multi surb transmission lanes by @jstuczyn in https://github.com/nymtech/nym/pull/2723
+
+## [v1.1.4] (2022-12-20)
+
+This release adds multiple Single Use Reply Blocks (SURBs) to allow arbitrarily-sized anonymized replies.
+At the moment this is turned off by default, but available for use by application developers.
+We will need to wait for network-requesters to upgrade to this new release, after which multi-SURB anonymization will become the default setting for the SOCKS proxy clients.
+
+The release also include some additional work for distributed key generation in the Coconut signing authority nodes.
+
+### Changed
+
+- Feature/dkg contract threshold by @neacsu in https://github.com/nymtech/nym/pull/1885
+- Multi-surbs by @jstuczyn in https://github.com/nymtech/nym/pull/2667
+- Fix multi-surb backwards compatibility in pre 1.1.4 client config files by @jstuczyn in https://github.com/nymtech/nym/pull/2703
+- fix: ignore corrupted surb storage and instead create fresh one by @jstuczyn in https://github.com/nymtech/nym/pull/2711
+- socks5: rework waiting in inbound.rs by @octol in https://github.com/nymtech/nym/pull/1880
+
+## [v1.1.3] (2022-12-13)
+
+### Changed
+
+- validator-api: can recover from shutdown during DKG process ([#1872])
+- clients: deduplicate gateway initialization, part of work towards a rust-sdk
+- clients: keep all transmission lanes going at all times by making priority probabilistic
+- clients: ability to use multi-reply SURBs to send arbitrarily long messages fully anonymously whilst requesting additional reply blocks whenever they're about to run out ([#1796], [#1801], [#1804], [#1835], [#1858], [#1883]))
+
+### Fixed
+
+- network-requester: fix bug where websocket connection disconnect resulted in success error code
+- clients: fix a few panics handling the gateway-client
+- mixnode, gateway, validator-api: Use mainnet values as defaults for URLs and mixnet contract ([#1884])
+- socks5: fixed bug where connections sometimes where closed too early
+- clients: improve message logging when received message fails to get reconstructed ([#1803])
+
+[#1796]: https://github.com/nymtech/nym/pull/1796
+[#1801]: https://github.com/nymtech/nym/pull/1801
+[#1803]: https://github.com/nymtech/nym/pull/1803
+[#1804]: https://github.com/nymtech/nym/pull/1804
+[#1835]: https://github.com/nymtech/nym/pull/1835
+[#1858]: https://github.com/nymtech/nym/pull/1858
+[#1872]: https://github.com/nymtech/nym/pull/1872
+[#1883]: https://github.com/nymtech/nym/pull/1883
+[#1884]: https://github.com/nymtech/nym/pull/1884
+
+## [v1.1.2]
+
+### Changed
+
+- gateway: Renamed flag from `enabled/disabled_credentials_mode` to `only-coconut-credentials`
+- "Family" feature for node families + layers
+- Initial coconut functionality including credentials and distributed key generation
+
+## [v1.1.1](https://github.com/nymtech/nym/tree/v1.1.1) (2022-11-29)
+
+### Added
+
+- binaries: add `-c` shortform for `--config-env-file`
+- websocket-requests: add server response signalling current packet queue length in the client
+- contracts: DKG contract that handles coconut key generation ([#1678][#1708][#1747])
+- validator-api: generate coconut keys interactively, using DKG and multisig contracts ([#1678][#1708][#1747])
+
+### Changed
+
+- clients: add concept of transmission lanes to better handle multiple data streams ([#1720])
+- clients,validator-api: take coconut signers from the chain instead of specifying them via CLI ([#1747])
+- multisig contract: add DKG contract to the list of addresses that can create proposals ([#1747])
+- socks5-client: wait closing inbound connection until data is sent, and throttle incoming data in general ([#1783])
+- nym-cli: improve error reporting/handling and changed `vesting-schedule` queries to use query client instead of signing client
+
+### Fixed
+
+- gateway-client: fix decrypting stored messages on reconnect ([#1786])
+
+### Fixed
+
+- gateway-client: fix decrypting stored messages on reconnect ([#1786])
+- socks5-client: fix shutting down all tasks if anyone of them panics or errors out ([#1805])
+
+[#1678]: https://github.com/nymtech/nym/pull/1678
+[#1708]: https://github.com/nymtech/nym/pull/1708
+[#1720]: https://github.com/nymtech/nym/pull/1720
+[#1747]: https://github.com/nymtech/nym/pull/1747
+[#1783]: https://github.com/nymtech/nym/pull/1783
+[#1786]: https://github.com/nymtech/nym/pull/1786
+[#1805]: https://github.com/nymtech/nym/pull/1805
+
+## [v1.1.0](https://github.com/nymtech/nym/tree/v1.1.0) (2022-11-09)
+
+### Added
+
+- clients: add testing-only support for two more extended packet sizes (8kb and 16kb).
+- common/ledger: new library for communicating with a Ledger device ([#1640])
+- native-client/socks5-client/wasm-client: `disable_loop_cover_traffic_stream` Debug config option to disable the separate loop cover traffic stream ([#1666])
+- native-client/socks5-client/wasm-client: `disable_main_poisson_packet_distribution` Debug config option to make the client ignore poisson distribution in the main packet stream and ONLY send real message (and as fast as they come) ([#1664])
+- native-client/socks5-client/wasm-client: `use_extended_packet_size` Debug config option to make the client use 'ExtendedPacketSize' for its traffic (32kB as opposed to 2kB in 1.0.2) ([#1671])
+- network-requester: added additional Blockstream Green wallet endpoint to `example.allowed.list` ([#1611])
+- validator-api: add `interval_operating_cost` and `profit_margin_percent` to compute reward estimation endpoint
+- validator-client: added `query_contract_smart` and `query_contract_raw` on `NyxdClient` ([#1558])
+- wasm-client: uses updated wasm-compatible `client-core` so that it's now capable of packet retransmission, cover traffic and poisson delay (among other things!) ([#1673])
+
+### Fixed
+
+- socks5-client: fix bug where in some cases packet reordering could trigger a connection being closed too early ([#1702],[#1724])
+- validator-api: mixnode, gateway should now prefer values in config.toml over mainnet defaults ([#1645])
+- validator-api: should now correctly update historical uptimes for all mixnodes and gateways every 24h ([#1721])
+
+### Changed
+
+- clients: bound the sphinx packet channel and reduce sending rate if gateway can't keep up ([#1703],[#1725])
+- gateway-client: will attempt to read now as many as 8 websocket messages at once, assuming they're already available on the socket ([#1669])
+- moved `Percent` struct to `contracts-common`, change affects explorer-api
+- socks5 client: graceful shutdown should fix error on disconnect in nym-connect ([#1591])
+- validator-api: changed error serialization on `inclusion_probability`, `stake-saturation` and `reward-estimation` endpoints to provide more accurate information ([#1681])
+- validator-client: made `fee` argument optional for `execute` and `execute_multiple` ([#1541])
+- wasm-client: fixed build errors on MacOS and changed example JS code to use mainnet ([#1585])
+- validator-api: changes to internal SQL schema due to the mixnet contract revamp ([#1472])
+- validator-api: changes to internal data structures due to the mixnet contract revamp ([#1472])
+- validator-api: split epoch-operations into multiple separate transactions ([#1472])
+
+[#1472]: https://github.com/nymtech/nym/pull/1472
+[#1541]: https://github.com/nymtech/nym/pull/1541
+[#1558]: https://github.com/nymtech/nym/pull/1558
+[#1577]: https://github.com/nymtech/nym/pull/1577
+[#1585]: https://github.com/nymtech/nym/pull/1585
+[#1591]: https://github.com/nymtech/nym/pull/1591
+[#1640]: https://github.com/nymtech/nym/pull/1640
+[#1645]: https://github.com/nymtech/nym/pull/1645
+[#1611]: https://github.com/nymtech/nym/pull/1611
+[#1664]: https://github.com/nymtech/nym/pull/1664
+[#1666]: https://github.com/nymtech/nym/pull/1645
+[#1669]: https://github.com/nymtech/nym/pull/1669
+[#1671]: https://github.com/nymtech/nym/pull/1671
+[#1673]: https://github.com/nymtech/nym/pull/1673
+[#1681]: https://github.com/nymtech/nym/pull/1681
+[#1702]: https://github.com/nymtech/nym/pull/1702
+[#1703]: https://github.com/nymtech/nym/pull/1703
+[#1721]: https://github.com/nymtech/nym/pull/1721
+[#1724]: https://github.com/nymtech/nym/pull/1724
+[#1725]: https://github.com/nymtech/nym/pull/1725
+
+## [nym-binaries-1.0.2](https://github.com/nymtech/nym/tree/nym-binaries-1.0.2)
+
+### Added
+
+- socks5 client/websocket client: add `--force-register-gateway` flag, useful when rerunning init ([#1353])
+- all: added network compilation target to `--help` (or `--version`) commands ([#1256]).
+- explorer-api: learned how to sum the delegations by owner in a new endpoint.
+- explorer-api: add apy values to `mix_nodes` endpoint
+- gateway: Added gateway coconut verifications and validator-api communication for double spending protection ([#1261])
+- network-explorer-ui: Upgrade to React Router 6
+- rewarding: replace circulating supply with staking supply in reward calculations ([#1324])
+- validator-api: add `estimated_node_profit` and `estimated_operator_cost` to `reward-estimate` endpoint ([#1284])
+- validator-api: add detailed mixnode bond endpoints, and explorer-api makes use of that data to append stake saturation
+- validator-api: add Swagger to document the REST API ([#1249]).
+- validator-api: Added new endpoints for coconut spending flow and communications with coconut & multisig contracts ([#1261])
+- validator-api: add `uptime`, `estimated_operator_apy`, `estimated_delegators_apy` to `/mixnodes/detailed` endpoint ([#1393])
+- validator-api: add node info cache storing simulated active set inclusion probabilities
+- network-statistics: a new mixnet service that aggregates and exposes anonymized data about mixnet services ([#1328])
+- mixnode: Added basic mixnode hardware reporting to the HTTP API ([#1308]).
+- validator-api: endpoint, in coconut mode, for returning the validator-api cosmos address ([#1404]).
+- validator-client: add `denom` argument and add simple test for querying an account balance
+- gateway, validator-api: Checks for coconut credential double spending attempts, taking the coconut bandwidth contract as source of truth ([#1457])
+- coconut-bandwidth-contract: Record the state of a coconut credential; create specific proposal for releasing funds ([#1457])
+- inclusion-probability: add simulator for active set inclusion probability
+
+### Fixed
+
+- mixnode, gateway: attempting to determine reconnection backoff to persistently failing mixnode could result in a crash ([#1260])
+- mixnode: the mixnode learned how to shutdown gracefully
+- mixnode: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service.
+- native & socks5 clients: fail early when clients try to re-init with a different gateway, which is not supported yet ([#1322])
+- native & socks5 clients: rerun init will now reuse previous gateway configuration instead of failing ([#1353])
+- native & socks5 clients: deduplicate big chunks of init logic
+- validator: fixed local docker-compose setup to work on Apple M1 ([#1329])
+- explorer-api: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service ([#1482]).
+- network-requester: fix filter for suffix-only domains ([#1487])
+- validator-api: listen out for SIGTERM and SIGQUIT too, making it play nicely as a system service; cleaner shutdown, without panics ([#1496], [#1573]).
+
+### Changed
+
+- validator-client: created internal `Coin` type that replaces coins from `cosmrs` and `cosmwasm` for API entrypoints [[#1295]]
+- all: updated all `cosmwasm`-related dependencies to `1.0.0` and `cw-storage-plus` to `0.13.4` [[#1318]]
+- all: updated `rocket` to `0.5.0-rc.2`.
+- network-requester: allow to voluntarily store and send statistical data about the number of bytes the proxied server serves ([#1328])
+- gateway: allow to voluntarily send statistical data about the number of active inboxes served by a gateway ([#1376])
+- gateway & mixnode: move detailed build info back to `--version` from `--help`.
+- socks5 client/websocket client: upgrade to latest clap and switched to declarative commandline parsing.
+- validator-api: fee payment for multisig operations comes from the gateway account instead of the validator APIs' accounts ([#1419])
+- multisig-contract: Limit the proposal creating functionality to one address (coconut-bandwidth-contract address) ([#1457])
+- All binaries and cosmwasm blobs are configured at runtime now; binaries are configured using environment variables or .env files and contracts keep the configuration parameters in storage ([#1463])
+- gateway, network-statistics: include gateway id in the sent statistical data ([#1478])
+- network explorer: tweak how active set probability is shown ([#1503])
+- validator-api: rewarder set update fails without panicking on possible nyxd queries ([#1520])
+- network-requester, socks5 client (nym-connect): send and receive respectively a message error to be displayed about filter check failure ([#1576])
+
+[#1249]: https://github.com/nymtech/nym/pull/1249
+[#1256]: https://github.com/nymtech/nym/pull/1256
+[#1260]: https://github.com/nymtech/nym/pull/1260
+[#1261]: https://github.com/nymtech/nym/pull/1261
+[#1267]: https://github.com/nymtech/nym/pull/1267
+[#1278]: https://github.com/nymtech/nym/pull/1278
+[#1295]: https://github.com/nymtech/nym/pull/1295
+[#1302]: https://github.com/nymtech/nym/pull/1302
+[#1308]: https://github.com/nymtech/nym/pull/1308
+[#1318]: https://github.com/nymtech/nym/pull/1318
+[#1322]: https://github.com/nymtech/nym/pull/1322
+[#1324]: https://github.com/nymtech/nym/pull/1324
+[#1328]: https://github.com/nymtech/nym/pull/1328
+[#1329]: https://github.com/nymtech/nym/pull/1329
+[#1353]: https://github.com/nymtech/nym/pull/1353
+[#1376]: https://github.com/nymtech/nym/pull/1376
+[#1393]: https://github.com/nymtech/nym/pull/1393
+[#1404]: https://github.com/nymtech/nym/pull/1404
+[#1419]: https://github.com/nymtech/nym/pull/1419
+[#1457]: https://github.com/nymtech/nym/pull/1457
+[#1463]: https://github.com/nymtech/nym/pull/1463
+[#1478]: https://github.com/nymtech/nym/pull/1478
+[#1482]: https://github.com/nymtech/nym/pull/1482
+[#1487]: https://github.com/nymtech/nym/pull/1487
+[#1496]: https://github.com/nymtech/nym/pull/1496
+[#1503]: https://github.com/nymtech/nym/pull/1503
+[#1520]: https://github.com/nymtech/nym/pull/1520
+[#1573]: https://github.com/nymtech/nym/pull/1573
+[#1576]: https://github.com/nymtech/nym/pull/1576
+
+## [v1.0.1](https://github.com/nymtech/nym/tree/v1.0.1) (2022-05-04)
+
+### Added
+
+- validator-api: introduced endpoint for getting average mixnode uptime ([#1238])
+
+### Changed
+
+- all: the default behaviour of validator client is changed to use `broadcast_sync` and poll for transaction inclusion instead of using `broadcast_commit` to deal with timeouts ([#1246])
+
+### Fixed
+
+- nym-network-requester: is included in the Github Actions for building release binaries
+
+[#1238]: https://github.com/nymtech/nym/pull/1238
+[#1246]: https://github.com/nymtech/nym/pull/1246
+
 ## [v1.0.0](https://github.com/nymtech/nym/tree/v1.0.0) (2022-05-03)
 
 [Full Changelog](https://github.com/nymtech/nym/compare/v0.12.1...v1.0.0)
@@ -14,77 +365,10 @@
 
 [Full Changelog](https://github.com/nymtech/nym/compare/nym-wallet-v1.0.3...nym-binaries-1.0.0)
 
-## [nym-wallet-v1.0.3](https://github.com/nymtech/nym/tree/nym-wallet-v1.0.3) (2022-04-25)
-
-[Full Changelog](https://github.com/nymtech/nym/compare/nym-binaries-1.0.0-rc.2...nym-wallet-v1.0.3)
-
-**Fixed bugs:**
-
-- \[Issue\] Wallet 1.0.2 cannot send NYM tokens from a DelayedVestingAccount [\#1215](https://github.com/nymtech/nym/issues/1215)
-- Main README not showing properly with GitHub dark mode [\#1211](https://github.com/nymtech/nym/issues/1211)
-
-**Merged pull requests:**
-
-- Bugfix - wallet undelegation for vesting accounts [\#1220](https://github.com/nymtech/nym/pull/1220) ([mmsinclair](https://github.com/mmsinclair))
-- Bugfix/delegation reconcile [\#1219](https://github.com/nymtech/nym/pull/1219) ([jstuczyn](https://github.com/jstuczyn))
-- Bugfix/query proxied pending delegations [\#1218](https://github.com/nymtech/nym/pull/1218) ([jstuczyn](https://github.com/jstuczyn))
-- Using custom gas multiplier in the wallet [\#1217](https://github.com/nymtech/nym/pull/1217) ([jstuczyn](https://github.com/jstuczyn))
-- Feature/vesting accounts support [\#1216](https://github.com/nymtech/nym/pull/1216) ([jstuczyn](https://github.com/jstuczyn))
-- Release/1.0.0 rc.2 [\#1214](https://github.com/nymtech/nym/pull/1214) ([jstuczyn](https://github.com/jstuczyn))
-- chore: fix dark mode rendering [\#1212](https://github.com/nymtech/nym/pull/1212) ([pwnfoo](https://github.com/pwnfoo))
-- Feature/spend coconut [\#1210](https://github.com/nymtech/nym/pull/1210) ([neacsu](https://github.com/neacsu))
-- Bugfix/unique sphinx key [\#1207](https://github.com/nymtech/nym/pull/1207) ([jstuczyn](https://github.com/jstuczyn))
-- Add cache read and write timeouts [\#1206](https://github.com/nymtech/nym/pull/1206) ([durch](https://github.com/durch))
-- Additional, more informative routes [\#1204](https://github.com/nymtech/nym/pull/1204) ([durch](https://github.com/durch))
-- Feature/aggregated econ dynamics explorer endpoint [\#1203](https://github.com/nymtech/nym/pull/1203) ([jstuczyn](https://github.com/jstuczyn))
-- Debugging validator [\#1198](https://github.com/nymtech/nym/pull/1198) ([durch](https://github.com/durch))
-- wallet: expose additional validator configuration functionality to the frontend [\#1195](https://github.com/nymtech/nym/pull/1195) ([octol](https://github.com/octol))
-- Update rewarding validator address [\#1193](https://github.com/nymtech/nym/pull/1193) ([durch](https://github.com/durch))
-- Crypto part of the Groth's NIDKG [\#1182](https://github.com/nymtech/nym/pull/1182) ([jstuczyn](https://github.com/jstuczyn))
-- fix unbond page [\#1180](https://github.com/nymtech/nym/pull/1180) ([tommyv1987](https://github.com/tommyv1987))
-- Type safe bounds [\#1179](https://github.com/nymtech/nym/pull/1179) ([durch](https://github.com/durch))
-- Fix delegation paging [\#1174](https://github.com/nymtech/nym/pull/1174) ([durch](https://github.com/durch))
-- Update binaries to rc version [\#1172](https://github.com/nymtech/nym/pull/1172) ([tommyv1987](https://github.com/tommyv1987))
-- Bump ansi-regex from 4.1.0 to 4.1.1 in /docker/typescript\_client/upload\_contract [\#1171](https://github.com/nymtech/nym/pull/1171) ([dependabot[bot]](https://github.com/apps/dependabot))
-
 ## [nym-binaries-1.0.0-rc.2](https://github.com/nymtech/nym/tree/nym-binaries-1.0.0-rc.2) (2022-04-15)
 
 [Full Changelog](https://github.com/nymtech/nym/compare/nym-wallet-v1.0.2...nym-binaries-1.0.0-rc.2)
 
-## [nym-wallet-v1.0.2](https://github.com/nymtech/nym/tree/nym-wallet-v1.0.2) (2022-04-05)
-
-[Full Changelog](https://github.com/nymtech/nym/compare/nym-wallet-v1.0.1...nym-wallet-v1.0.2)
-
-**Merged pull requests:**
-
-- Wallet 1.0.2 visual tweaks [\#1197](https://github.com/nymtech/nym/pull/1197) ([mmsinclair](https://github.com/mmsinclair))
-- Password for wallet with routes [\#1196](https://github.com/nymtech/nym/pull/1196) ([fmtabbara](https://github.com/fmtabbara))
-- Add auto-updater to Nym Wallet [\#1194](https://github.com/nymtech/nym/pull/1194) ([mmsinclair](https://github.com/mmsinclair))
-- Fix clippy warnings for beta toolchain [\#1191](https://github.com/nymtech/nym/pull/1191) ([octol](https://github.com/octol))
-- wallet: expose validator urls to the frontend [\#1190](https://github.com/nymtech/nym/pull/1190) ([octol](https://github.com/octol))
-- wallet: add test for decrypting stored wallet file [\#1189](https://github.com/nymtech/nym/pull/1189) ([octol](https://github.com/octol))
-- Fix clippy warnings [\#1188](https://github.com/nymtech/nym/pull/1188) ([octol](https://github.com/octol))
-- Password for wallet with routes [\#1187](https://github.com/nymtech/nym/pull/1187) ([mmsinclair](https://github.com/mmsinclair))
-- wallet: add validate\_mnemonic [\#1186](https://github.com/nymtech/nym/pull/1186) ([octol](https://github.com/octol))
-- wallet: support removing accounts from the wallet file [\#1185](https://github.com/nymtech/nym/pull/1185) ([octol](https://github.com/octol))
-- Feature/adding discord [\#1184](https://github.com/nymtech/nym/pull/1184) ([gala1234](https://github.com/gala1234))
-- wallet: config backend for validator selection [\#1183](https://github.com/nymtech/nym/pull/1183) ([octol](https://github.com/octol))
-- Add storybook to wallet [\#1178](https://github.com/nymtech/nym/pull/1178) ([mmsinclair](https://github.com/mmsinclair))
-- wallet: connection test nymd and api urls independently [\#1170](https://github.com/nymtech/nym/pull/1170) ([octol](https://github.com/octol))
-- wallet: wire up account storage [\#1153](https://github.com/nymtech/nym/pull/1153) ([octol](https://github.com/octol))
-- Feature/signature on deposit [\#1151](https://github.com/nymtech/nym/pull/1151) ([neacsu](https://github.com/neacsu))
-
-## [nym-wallet-v1.0.1](https://github.com/nymtech/nym/tree/nym-wallet-v1.0.1) (2022-04-05)
-
-[Full Changelog](https://github.com/nymtech/nym/compare/nym-binaries-1.0.0-rc.1...nym-wallet-v1.0.1)
-
-**Closed issues:**
-
-- Check enabling bbbc simultaneously with open access. Estimate what it would take to make this the default compilation target. [\#1175](https://github.com/nymtech/nym/issues/1175)
-- Get coconut credential for deposited tokens [\#1138](https://github.com/nymtech/nym/issues/1138)
-- Make payments lazy [\#1135](https://github.com/nymtech/nym/issues/1135)
-- Uptime on node selection for sets [\#1049](https://github.com/nymtech/nym/issues/1049)
-
 ## [nym-binaries-1.0.0-rc.1](https://github.com/nymtech/nym/tree/nym-binaries-1.0.0-rc.1) (2022-03-28)
 
 [Full Changelog](https://github.com/nymtech/nym/compare/nym-wallet-v1.0.0...nym-binaries-1.0.0-rc.1)
@@ -112,9 +396,9 @@
 - Bump minimist from 1.2.5 to 1.2.6 in /clients/tauri-client [\#1163](https://github.com/nymtech/nym/pull/1163) ([dependabot[bot]](https://github.com/apps/dependabot))
 - Bump minimist from 1.2.5 to 1.2.6 in /clients/webassembly/js-example [\#1162](https://github.com/nymtech/nym/pull/1162) ([dependabot[bot]](https://github.com/apps/dependabot))
 - Bump minimist from 1.2.5 to 1.2.6 in /clients/native/examples/js-examples/websocket [\#1160](https://github.com/nymtech/nym/pull/1160) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump minimist from 1.2.5 to 1.2.6 in /docker/typescript\_client/upload\_contract [\#1159](https://github.com/nymtech/nym/pull/1159) ([dependabot[bot]](https://github.com/apps/dependabot))
+- Bump minimist from 1.2.5 to 1.2.6 in /docker/typescript_client/upload_contract [\#1159](https://github.com/nymtech/nym/pull/1159) ([dependabot[bot]](https://github.com/apps/dependabot))
 - Feature/vesting full [\#1158](https://github.com/nymtech/nym/pull/1158) ([fmtabbara](https://github.com/fmtabbara))
-- get\_current\_epoch tauri [\#1156](https://github.com/nymtech/nym/pull/1156) ([durch](https://github.com/durch))
+- get_current_epoch tauri [\#1156](https://github.com/nymtech/nym/pull/1156) ([durch](https://github.com/durch))
 - Cleanup [\#1155](https://github.com/nymtech/nym/pull/1155) ([durch](https://github.com/durch))
 - Feature flag reward payments [\#1154](https://github.com/nymtech/nym/pull/1154) ([durch](https://github.com/durch))
 - Add Query endpoints for calculating rewards [\#1152](https://github.com/nymtech/nym/pull/1152) ([durch](https://github.com/durch))
@@ -123,7 +407,7 @@
 - wallet: use Urls rather than Strings for validator urls [\#1148](https://github.com/nymtech/nym/pull/1148) ([octol](https://github.com/octol))
 - Change accumulated reward to Option, migrate delegations [\#1147](https://github.com/nymtech/nym/pull/1147) ([durch](https://github.com/durch))
 - wallet: fetch validators url remotely if available [\#1146](https://github.com/nymtech/nym/pull/1146) ([octol](https://github.com/octol))
-- Fix delegated\_free calculation [\#1145](https://github.com/nymtech/nym/pull/1145) ([durch](https://github.com/durch))
+- Fix delegated_free calculation [\#1145](https://github.com/nymtech/nym/pull/1145) ([durch](https://github.com/durch))
 - Update Nym wallet dependencies to use `ts-packages` [\#1144](https://github.com/nymtech/nym/pull/1144) ([mmsinclair](https://github.com/mmsinclair))
 - wallet: try validators one by one if available [\#1143](https://github.com/nymtech/nym/pull/1143) ([octol](https://github.com/octol))
 - Update Network Explorer Packages and add mix node identity key copy [\#1142](https://github.com/nymtech/nym/pull/1142) ([mmsinclair](https://github.com/mmsinclair))
@@ -163,116 +447,13 @@
 - feature/pedersen-commitments [\#1048](https://github.com/nymtech/nym/pull/1048) ([danielementary](https://github.com/danielementary))
 - Feature/reuse init owner [\#970](https://github.com/nymtech/nym/pull/970) ([neacsu](https://github.com/neacsu))
 
-## [nym-wallet-v1.0.0](https://github.com/nymtech/nym/tree/nym-wallet-v1.0.0) (2022-02-03)
-
-[Full Changelog](https://github.com/nymtech/nym/compare/v0.12.1...nym-wallet-v1.0.0)
-
-**Implemented enhancements:**
-
-- \[Feature Request\] Please enable registration without need for Telegram account [\#1016](https://github.com/nymtech/nym/issues/1016)
-- Fast mixnode launch with a pre-built ISO + VM software [\#1001](https://github.com/nymtech/nym/issues/1001)
-
-**Fixed bugs:**
-
-- \[Issue\] [\#1000](https://github.com/nymtech/nym/issues/1000)
-- \[Issue\] `nym-client` requires multiple attempts to run a server [\#869](https://github.com/nymtech/nym/issues/869)
-- De-'float'-ing `Interval` \(`Display` impl + `serde`\) [\#1065](https://github.com/nymtech/nym/pull/1065) ([jstuczyn](https://github.com/jstuczyn))
-- display client address on wallet creation [\#1058](https://github.com/nymtech/nym/pull/1058) ([fmtabbara](https://github.com/fmtabbara))
-
-**Closed issues:**
-
-- Rewarded set inclusion probability API endpoint [\#1037](https://github.com/nymtech/nym/issues/1037)
-- Update cw-storage-plus to 0.11  [\#1032](https://github.com/nymtech/nym/issues/1032)
-- Change `u128` fields in `RewardEstimationResponse` to `u64` [\#1029](https://github.com/nymtech/nym/issues/1029)
-- Test out the mainnet Gravity Bridge [\#1006](https://github.com/nymtech/nym/issues/1006)
-- Add vesting contract interface to nym-wallet [\#959](https://github.com/nymtech/nym/issues/959)
-- Mixnode crash [\#486](https://github.com/nymtech/nym/issues/486)
-
-**Merged pull requests:**
-
-- create custom urls for mainnet [\#1095](https://github.com/nymtech/nym/pull/1095) ([fmtabbara](https://github.com/fmtabbara))
-- Wallet signing on MacOS [\#1093](https://github.com/nymtech/nym/pull/1093) ([mmsinclair](https://github.com/mmsinclair))
-- Fix rust 2018 idioms warnings [\#1092](https://github.com/nymtech/nym/pull/1092) ([octol](https://github.com/octol))
-- Prevent contract overwriting [\#1090](https://github.com/nymtech/nym/pull/1090) ([durch](https://github.com/durch))
-- Logout operation [\#1087](https://github.com/nymtech/nym/pull/1087) ([jstuczyn](https://github.com/jstuczyn))
-- Update to rust edition 2021 everywhere [\#1086](https://github.com/nymtech/nym/pull/1086) ([octol](https://github.com/octol))
-- Tag contract errors, and print out lines for easier QA [\#1084](https://github.com/nymtech/nym/pull/1084) ([durch](https://github.com/durch))
-- Feature/flexible vesting + utility queries [\#1083](https://github.com/nymtech/nym/pull/1083) ([durch](https://github.com/durch))
-- Bump @openzeppelin/contracts from 4.3.1 to 4.4.2 in /contracts/basic-bandwidth-generation [\#1082](https://github.com/nymtech/nym/pull/1082) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump nth-check from 2.0.0 to 2.0.1 in /clients/native/examples/js-examples/websocket [\#1081](https://github.com/nymtech/nym/pull/1081) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump url-parse from 1.5.1 to 1.5.4 in /clients/native/examples/js-examples/websocket [\#1080](https://github.com/nymtech/nym/pull/1080) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump follow-redirects from 1.14.1 to 1.14.7 in /clients/native/examples/js-examples/websocket [\#1079](https://github.com/nymtech/nym/pull/1079) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump nanoid from 3.1.23 to 3.2.0 in /clients/native/examples/js-examples/websocket [\#1078](https://github.com/nymtech/nym/pull/1078) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Setup basic test for mixnode stats reporting [\#1077](https://github.com/nymtech/nym/pull/1077) ([octol](https://github.com/octol))
-- Make wallet\_address mandatory for mixnode init [\#1076](https://github.com/nymtech/nym/pull/1076) ([octol](https://github.com/octol))
-- Tidy nym-mixnode module visibility [\#1075](https://github.com/nymtech/nym/pull/1075) ([octol](https://github.com/octol))
-- Feature/wallet login with password [\#1074](https://github.com/nymtech/nym/pull/1074) ([fmtabbara](https://github.com/fmtabbara))
-- Add trait to mock client dependency in DelayForwarder [\#1073](https://github.com/nymtech/nym/pull/1073) ([octol](https://github.com/octol))
-- Bump rust-version to latest stable for nym-mixnode [\#1072](https://github.com/nymtech/nym/pull/1072) ([octol](https://github.com/octol))
-- Fixes CI for our wasm build [\#1069](https://github.com/nymtech/nym/pull/1069) ([jstuczyn](https://github.com/jstuczyn))
-- Add @octol as codeowner [\#1068](https://github.com/nymtech/nym/pull/1068) ([octol](https://github.com/octol))
-- set-up inclusion probability [\#1067](https://github.com/nymtech/nym/pull/1067) ([fmtabbara](https://github.com/fmtabbara))
-- Feature/wasm client [\#1066](https://github.com/nymtech/nym/pull/1066) ([neacsu](https://github.com/neacsu))
-- Changed bech32\_prefix from punk to nymt [\#1064](https://github.com/nymtech/nym/pull/1064) ([jstuczyn](https://github.com/jstuczyn))
-- Bump nanoid from 3.1.30 to 3.2.0 in /testnet-faucet [\#1063](https://github.com/nymtech/nym/pull/1063) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump nanoid from 3.1.30 to 3.2.0 in /nym-wallet [\#1062](https://github.com/nymtech/nym/pull/1062) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Rework vesting contract storage [\#1061](https://github.com/nymtech/nym/pull/1061) ([durch](https://github.com/durch))
-- Mixnet Contract constants extraction [\#1060](https://github.com/nymtech/nym/pull/1060) ([jstuczyn](https://github.com/jstuczyn))
-- fix: make explorer footer year dynamic [\#1059](https://github.com/nymtech/nym/pull/1059) ([martinyung](https://github.com/martinyung))
-- Add mnemonic just on creation, to display it [\#1057](https://github.com/nymtech/nym/pull/1057) ([neacsu](https://github.com/neacsu))
-- Network Explorer: updates to API and UI to show the active set [\#1056](https://github.com/nymtech/nym/pull/1056) ([mmsinclair](https://github.com/mmsinclair))
-- Made contract addresses for query NymdClient construction optional [\#1055](https://github.com/nymtech/nym/pull/1055) ([jstuczyn](https://github.com/jstuczyn))
-- Introduced RPC query for total token supply [\#1053](https://github.com/nymtech/nym/pull/1053) ([jstuczyn](https://github.com/jstuczyn))
-- Feature/tokio console [\#1052](https://github.com/nymtech/nym/pull/1052) ([durch](https://github.com/durch))
-- Implemented beta clippy lint recommendations [\#1051](https://github.com/nymtech/nym/pull/1051) ([jstuczyn](https://github.com/jstuczyn))
-- add new function to update profit percentage [\#1050](https://github.com/nymtech/nym/pull/1050) ([fmtabbara](https://github.com/fmtabbara))
-- Upgrade Clap and use declarative argument parsing for nym-mixnode [\#1047](https://github.com/nymtech/nym/pull/1047) ([octol](https://github.com/octol))
-- Feature/additional bond validation [\#1046](https://github.com/nymtech/nym/pull/1046) ([fmtabbara](https://github.com/fmtabbara))
-- Fix clippy on relevant lints [\#1044](https://github.com/nymtech/nym/pull/1044) ([neacsu](https://github.com/neacsu))
-- Bump shelljs from 0.8.4 to 0.8.5 in /contracts/basic-bandwidth-generation [\#1043](https://github.com/nymtech/nym/pull/1043) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Endpoint for rewarded set inclusion probabilities [\#1042](https://github.com/nymtech/nym/pull/1042) ([durch](https://github.com/durch))
-- Bump follow-redirects from 1.14.4 to 1.14.7 in /contracts/basic-bandwidth-generation [\#1041](https://github.com/nymtech/nym/pull/1041) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump follow-redirects from 1.14.5 to 1.14.7 in /testnet-faucet [\#1040](https://github.com/nymtech/nym/pull/1040) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Feature/node settings update [\#1036](https://github.com/nymtech/nym/pull/1036) ([fmtabbara](https://github.com/fmtabbara))
-- Migrate to cw-storage-plus 0.11.1 [\#1035](https://github.com/nymtech/nym/pull/1035) ([durch](https://github.com/durch))
-- Bump @openzeppelin/contracts from 4.4.1 to 4.4.2 in /contracts/basic-bandwidth-generation [\#1034](https://github.com/nymtech/nym/pull/1034) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Feature/configurable wallet [\#1033](https://github.com/nymtech/nym/pull/1033) ([neacsu](https://github.com/neacsu))
-- Feature/downcast reward estimation [\#1031](https://github.com/nymtech/nym/pull/1031) ([durch](https://github.com/durch))
-- Wallet UI updates [\#1028](https://github.com/nymtech/nym/pull/1028) ([fmtabbara](https://github.com/fmtabbara))
-- Remove migration code [\#1027](https://github.com/nymtech/nym/pull/1027) ([neacsu](https://github.com/neacsu))
-- Chore/stricter dependency requirements [\#1025](https://github.com/nymtech/nym/pull/1025) ([jstuczyn](https://github.com/jstuczyn))
-- Feature/validator api client endpoints [\#1024](https://github.com/nymtech/nym/pull/1024) ([jstuczyn](https://github.com/jstuczyn))
-- Updated cosmrs to 0.4.1 [\#1023](https://github.com/nymtech/nym/pull/1023) ([jstuczyn](https://github.com/jstuczyn))
-- Feature/testnet deploy scripts [\#1022](https://github.com/nymtech/nym/pull/1022) ([mfahampshire](https://github.com/mfahampshire))
-- Changed wallet's client to a full validator client [\#1021](https://github.com/nymtech/nym/pull/1021) ([jstuczyn](https://github.com/jstuczyn))
-- Fix 404 link [\#1020](https://github.com/nymtech/nym/pull/1020) ([RiccardoMasutti](https://github.com/RiccardoMasutti))
-- Feature/additional mixnode endpoints [\#1019](https://github.com/nymtech/nym/pull/1019) ([jstuczyn](https://github.com/jstuczyn))
-- Introduced denom check when trying to withdraw vested coins [\#1018](https://github.com/nymtech/nym/pull/1018) ([jstuczyn](https://github.com/jstuczyn))
-- Add network defaults for qa [\#1017](https://github.com/nymtech/nym/pull/1017) ([neacsu](https://github.com/neacsu))
-- Feature/expanded events [\#1015](https://github.com/nymtech/nym/pull/1015) ([jstuczyn](https://github.com/jstuczyn))
-- update frontend to use new profit update api [\#1014](https://github.com/nymtech/nym/pull/1014) ([fmtabbara](https://github.com/fmtabbara))
-- Feature/node state endpoint [\#1013](https://github.com/nymtech/nym/pull/1013) ([jstuczyn](https://github.com/jstuczyn))
-- Feature/hourly set updates [\#1012](https://github.com/nymtech/nym/pull/1012) ([durch](https://github.com/durch))
-- Feature/remove unused profit margin [\#1011](https://github.com/nymtech/nym/pull/1011) ([neacsu](https://github.com/neacsu))
-- Feature/explorer node status [\#1010](https://github.com/nymtech/nym/pull/1010) ([jstuczyn](https://github.com/jstuczyn))
-- Use serial integer instead of random [\#1009](https://github.com/nymtech/nym/pull/1009) ([durch](https://github.com/durch))
-- Feature/configure profit [\#1008](https://github.com/nymtech/nym/pull/1008) ([neacsu](https://github.com/neacsu))
-- Feature/fix gateway sign [\#1004](https://github.com/nymtech/nym/pull/1004) ([neacsu](https://github.com/neacsu))
-- Fix clippy [\#1003](https://github.com/nymtech/nym/pull/1003) ([neacsu](https://github.com/neacsu))
-- Update wallet version [\#998](https://github.com/nymtech/nym/pull/998) ([tommyv1987](https://github.com/tommyv1987))
-- Fix wallet build instructions [\#997](https://github.com/nymtech/nym/pull/997) ([tommyv1987](https://github.com/tommyv1987))
-- Make the separation between testnet-mode and erc20 bandwidth mode clearer [\#994](https://github.com/nymtech/nym/pull/994) ([neacsu](https://github.com/neacsu))
-- Bump @openzeppelin/contracts from 3.4.0 to 4.4.1 in /contracts/basic-bandwidth-generation [\#983](https://github.com/nymtech/nym/pull/983) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Feature/implicit runtime [\#973](https://github.com/nymtech/nym/pull/973) ([jstuczyn](https://github.com/jstuczyn))
-- Differentiate staking and ownership [\#961](https://github.com/nymtech/nym/pull/961) ([durch](https://github.com/durch))
-
 ## [v0.12.1](https://github.com/nymtech/nym/tree/v0.12.1) (2021-12-23)
 
 [Full Changelog](https://github.com/nymtech/nym/compare/v0.12.0...v0.12.1)
 
 **Implemented enhancements:**
 
-- Add version check to  binaries [\#967](https://github.com/nymtech/nym/issues/967)
+- Add version check to binaries [\#967](https://github.com/nymtech/nym/issues/967)
 
 **Fixed bugs:**
 
@@ -302,7 +483,7 @@
 - Bugfix/remove mixnode bonding overwrite [\#917](https://github.com/nymtech/nym/pull/917) ([jstuczyn](https://github.com/jstuczyn))
 - Fixes crash condition in validator API when calculating last day uptime [\#909](https://github.com/nymtech/nym/pull/909) ([jstuczyn](https://github.com/jstuczyn))
 - Bugfix/monitor initial values wait [\#907](https://github.com/nymtech/nym/pull/907) ([jstuczyn](https://github.com/jstuczyn))
-- Bug fix: Network Explorer: Add freegeoip API key and split out tasks for country distributions  [\#806](https://github.com/nymtech/nym/pull/806) ([mmsinclair](https://github.com/mmsinclair))
+- Bug fix: Network Explorer: Add freegeoip API key and split out tasks for country distributions [\#806](https://github.com/nymtech/nym/pull/806) ([mmsinclair](https://github.com/mmsinclair))
 - Explorer API: port test now split out address resolution and add units tests [\#755](https://github.com/nymtech/nym/pull/755) ([mmsinclair](https://github.com/mmsinclair))
 
 **Closed issues:**
@@ -317,7 +498,7 @@
 - help!!! [\#712](https://github.com/nymtech/nym/issues/712)
 - UX feature request: show all delegated nodes in wallet [\#711](https://github.com/nymtech/nym/issues/711)
 - UX feature request: add current balance on wallet pages [\#710](https://github.com/nymtech/nym/issues/710)
-- got sign issue from bot  [\#709](https://github.com/nymtech/nym/issues/709)
+- got sign issue from bot [\#709](https://github.com/nymtech/nym/issues/709)
 - As a wallet user, I would like to be able to log out of the wallet [\#706](https://github.com/nymtech/nym/issues/706)
 - As a wallet user, I would like to have a "receive" page where I can see my own wallet address [\#705](https://github.com/nymtech/nym/issues/705)
 - Update native client/socks client/mixnode/gateway `upgrade` command [\#689](https://github.com/nymtech/nym/issues/689)
@@ -327,7 +508,7 @@
 - nym-socks5-client crash after opening Keybase team "Browse all channels" [\#494](https://github.com/nymtech/nym/issues/494)
 - Mixed Content problem [\#400](https://github.com/nymtech/nym/issues/400)
 - Gateway disk quota [\#137](https://github.com/nymtech/nym/issues/137)
-- Simplify message encapsulation with regards to topology  [\#127](https://github.com/nymtech/nym/issues/127)
+- Simplify message encapsulation with regards to topology [\#127](https://github.com/nymtech/nym/issues/127)
 - Create constants for cli argument names [\#115](https://github.com/nymtech/nym/issues/115)
 - Using Blake3 as a hash function [\#103](https://github.com/nymtech/nym/issues/103)
 - Validator should decide which layer a node is in [\#86](https://github.com/nymtech/nym/issues/86)
@@ -383,10 +564,10 @@
 - Feature/pre cosmrs updates [\#935](https://github.com/nymtech/nym/pull/935) ([jstuczyn](https://github.com/jstuczyn))
 - Feature/client on behalf [\#934](https://github.com/nymtech/nym/pull/934) ([neacsu](https://github.com/neacsu))
 - Webpack wallet prod configuration [\#933](https://github.com/nymtech/nym/pull/933) ([tommyv1987](https://github.com/tommyv1987))
-- Adding tx\_hash to wallet response [\#932](https://github.com/nymtech/nym/pull/932) ([futurechimp](https://github.com/futurechimp))
+- Adding tx_hash to wallet response [\#932](https://github.com/nymtech/nym/pull/932) ([futurechimp](https://github.com/futurechimp))
 - Release/1.0.0 pre1 [\#931](https://github.com/nymtech/nym/pull/931) ([durch](https://github.com/durch))
 - Feature/identity verification [\#930](https://github.com/nymtech/nym/pull/930) ([jstuczyn](https://github.com/jstuczyn))
-- Move cleaned up smart contracts to main code repo  [\#929](https://github.com/nymtech/nym/pull/929) ([mfahampshire](https://github.com/mfahampshire))
+- Move cleaned up smart contracts to main code repo [\#929](https://github.com/nymtech/nym/pull/929) ([mfahampshire](https://github.com/mfahampshire))
 - Feature/mixnet contract further adjustments [\#928](https://github.com/nymtech/nym/pull/928) ([jstuczyn](https://github.com/jstuczyn))
 - typo copy change for nodemap [\#926](https://github.com/nymtech/nym/pull/926) ([Aid19801](https://github.com/Aid19801))
 - Feature/UI enhancements for Desktop Wallet [\#925](https://github.com/nymtech/nym/pull/925) ([fmtabbara](https://github.com/fmtabbara))
@@ -399,7 +580,7 @@
 - Feature/faucet page react [\#911](https://github.com/nymtech/nym/pull/911) ([fmtabbara](https://github.com/fmtabbara))
 - Feature/mixnet contract refactor [\#910](https://github.com/nymtech/nym/pull/910) ([futurechimp](https://github.com/futurechimp))
 - Update README.md [\#905](https://github.com/nymtech/nym/pull/905) ([tommyv1987](https://github.com/tommyv1987))
-- BUG: Bond cell denom  [\#904](https://github.com/nymtech/nym/pull/904) ([Aid19801](https://github.com/Aid19801))
+- BUG: Bond cell denom [\#904](https://github.com/nymtech/nym/pull/904) ([Aid19801](https://github.com/Aid19801))
 - Explorer UI tests missing data-testid [\#903](https://github.com/nymtech/nym/pull/903) ([tommyv1987](https://github.com/tommyv1987))
 - Fix up Nym-Wallet README.md [\#899](https://github.com/nymtech/nym/pull/899) ([tommyv1987](https://github.com/tommyv1987))
 - Feature/batch delegator rewarding [\#898](https://github.com/nymtech/nym/pull/898) ([jstuczyn](https://github.com/jstuczyn))
@@ -417,7 +598,7 @@
 - Reverted gateway registration handshake to its 0.11.0 version [\#882](https://github.com/nymtech/nym/pull/882) ([jstuczyn](https://github.com/jstuczyn))
 - Network Explorer [\#881](https://github.com/nymtech/nym/pull/881) ([mmsinclair](https://github.com/mmsinclair))
 - Feature/rewarding interval updates [\#880](https://github.com/nymtech/nym/pull/880) ([jstuczyn](https://github.com/jstuczyn))
-- Put client\_address and id in the correct order [\#875](https://github.com/nymtech/nym/pull/875) ([neacsu](https://github.com/neacsu))
+- Put client_address and id in the correct order [\#875](https://github.com/nymtech/nym/pull/875) ([neacsu](https://github.com/neacsu))
 - remove gateway selection on delegation and undelegation pages [\#873](https://github.com/nymtech/nym/pull/873) ([fmtabbara](https://github.com/fmtabbara))
 - Set MSRV on all binaries to 1.56 [\#872](https://github.com/nymtech/nym/pull/872) ([jstuczyn](https://github.com/jstuczyn))
 - add native window items \(copy/paste\) via tauri [\#871](https://github.com/nymtech/nym/pull/871) ([fmtabbara](https://github.com/fmtabbara))
@@ -433,7 +614,7 @@
 - Overflow checks in release [\#846](https://github.com/nymtech/nym/pull/846) ([jstuczyn](https://github.com/jstuczyn))
 - fix delegate success overflow [\#842](https://github.com/nymtech/nym/pull/842) ([fmtabbara](https://github.com/fmtabbara))
 - Feature NYM wallet webdriverio test [\#841](https://github.com/nymtech/nym/pull/841) ([tommyv1987](https://github.com/tommyv1987))
-- Update nym\_wallet.yml [\#840](https://github.com/nymtech/nym/pull/840) ([tommyv1987](https://github.com/tommyv1987))
+- Update nym_wallet.yml [\#840](https://github.com/nymtech/nym/pull/840) ([tommyv1987](https://github.com/tommyv1987))
 - Feature/vouchers [\#837](https://github.com/nymtech/nym/pull/837) ([aniampio](https://github.com/aniampio))
 - Apply readable ids to elements on Nym Wallet [\#836](https://github.com/nymtech/nym/pull/836) ([tommyv1987](https://github.com/tommyv1987))
 - Feature/removal of monitor good nodes [\#833](https://github.com/nymtech/nym/pull/833) ([jstuczyn](https://github.com/jstuczyn))
@@ -457,8 +638,8 @@
 - Created getters for AccountData [\#787](https://github.com/nymtech/nym/pull/787) ([jstuczyn](https://github.com/jstuczyn))
 - Feature/migrate hidden delegations [\#786](https://github.com/nymtech/nym/pull/786) ([neacsu](https://github.com/neacsu))
 - Feature/persistent gateway storage [\#784](https://github.com/nymtech/nym/pull/784) ([jstuczyn](https://github.com/jstuczyn))
-- Replaced unwrap\_or\_else with unwrap\_or\_default [\#780](https://github.com/nymtech/nym/pull/780) ([jstuczyn](https://github.com/jstuczyn))
-- Add block\_height method to Delegation [\#778](https://github.com/nymtech/nym/pull/778) ([durch](https://github.com/durch))
+- Replaced unwrap_or_else with unwrap_or_default [\#780](https://github.com/nymtech/nym/pull/780) ([jstuczyn](https://github.com/jstuczyn))
+- Add block_height method to Delegation [\#778](https://github.com/nymtech/nym/pull/778) ([durch](https://github.com/durch))
 - Make fee helpers public [\#777](https://github.com/nymtech/nym/pull/777) ([durch](https://github.com/durch))
 - re-enable bonding [\#776](https://github.com/nymtech/nym/pull/776) ([fmtabbara](https://github.com/fmtabbara))
 - Explorer-api: add API resource to show the delegations for each mix node [\#774](https://github.com/nymtech/nym/pull/774) ([mmsinclair](https://github.com/mmsinclair))
@@ -467,14 +648,14 @@
 - Adding deps for building the Tauri wallet under Ubuntu [\#770](https://github.com/nymtech/nym/pull/770) ([futurechimp](https://github.com/futurechimp))
 - remove alert [\#767](https://github.com/nymtech/nym/pull/767) ([fmtabbara](https://github.com/fmtabbara))
 - Feature/consumable bandwidth [\#766](https://github.com/nymtech/nym/pull/766) ([neacsu](https://github.com/neacsu))
-- Update coconut-rs and use hash\_to\_scalar from there [\#765](https://github.com/nymtech/nym/pull/765) ([neacsu](https://github.com/neacsu))
+- Update coconut-rs and use hash_to_scalar from there [\#765](https://github.com/nymtech/nym/pull/765) ([neacsu](https://github.com/neacsu))
 - Feature/active sets [\#764](https://github.com/nymtech/nym/pull/764) ([jstuczyn](https://github.com/jstuczyn))
 - add app alert banner [\#762](https://github.com/nymtech/nym/pull/762) ([fmtabbara](https://github.com/fmtabbara))
 - Updated cosmos-sdk [\#761](https://github.com/nymtech/nym/pull/761) ([jstuczyn](https://github.com/jstuczyn))
 - Feature/bond blockstamp [\#760](https://github.com/nymtech/nym/pull/760) ([neacsu](https://github.com/neacsu))
 - Feature/revert migration code [\#759](https://github.com/nymtech/nym/pull/759) ([neacsu](https://github.com/neacsu))
 - Bump next from 11.1.0 to 11.1.1 in /wallet-web [\#758](https://github.com/nymtech/nym/pull/758) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Add block\_height in the Delegation structure as well [\#757](https://github.com/nymtech/nym/pull/757) ([neacsu](https://github.com/neacsu))
+- Add block_height in the Delegation structure as well [\#757](https://github.com/nymtech/nym/pull/757) ([neacsu](https://github.com/neacsu))
 - Feature/add blockstamp [\#756](https://github.com/nymtech/nym/pull/756) ([neacsu](https://github.com/neacsu))
 - NetworkMonitorBuilder - starting the monitor after rocket has launched [\#754](https://github.com/nymtech/nym/pull/754) ([jstuczyn](https://github.com/jstuczyn))
 - Enabled validators api argument [\#753](https://github.com/nymtech/nym/pull/753) ([jstuczyn](https://github.com/jstuczyn))
@@ -486,21 +667,21 @@
 - Feature/more reliable uptime calculation [\#747](https://github.com/nymtech/nym/pull/747) ([jstuczyn](https://github.com/jstuczyn))
 - Update template toml key [\#746](https://github.com/nymtech/nym/pull/746) ([neacsu](https://github.com/neacsu))
 - Feature/cred after handshake [\#745](https://github.com/nymtech/nym/pull/745) ([neacsu](https://github.com/neacsu))
-- Reinstate the POST method blind\_sign [\#744](https://github.com/nymtech/nym/pull/744) ([neacsu](https://github.com/neacsu))
+- Reinstate the POST method blind_sign [\#744](https://github.com/nymtech/nym/pull/744) ([neacsu](https://github.com/neacsu))
 - explorer-api: add pending field to port check response [\#742](https://github.com/nymtech/nym/pull/742) ([mmsinclair](https://github.com/mmsinclair))
 - Feature/use delegation rates [\#741](https://github.com/nymtech/nym/pull/741) ([neacsu](https://github.com/neacsu))
 - Feature/copy to clipboard [\#740](https://github.com/nymtech/nym/pull/740) ([fmtabbara](https://github.com/fmtabbara))
 - Feature/update wallet with stake rates [\#739](https://github.com/nymtech/nym/pull/739) ([neacsu](https://github.com/neacsu))
 - Add stake reward rates and bump version of client [\#738](https://github.com/nymtech/nym/pull/738) ([neacsu](https://github.com/neacsu))
 - Bump next from 10.1.3 to 11.1.0 in /wallet-web [\#737](https://github.com/nymtech/nym/pull/737) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Feature/nymd client integration [\#736](https://github.com/nymtech/nym/pull/736) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/nyxd client integration [\#736](https://github.com/nymtech/nym/pull/736) ([jstuczyn](https://github.com/jstuczyn))
 - Bug/fix parking lot on wasm [\#735](https://github.com/nymtech/nym/pull/735) ([neacsu](https://github.com/neacsu))
 - Explorer API: add new HTTP resource to decorate mix nodes with geoip locations [\#734](https://github.com/nymtech/nym/pull/734) ([mmsinclair](https://github.com/mmsinclair))
-- Feature/completing nymd client api [\#732](https://github.com/nymtech/nym/pull/732) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/completing nyxd client api [\#732](https://github.com/nymtech/nym/pull/732) ([jstuczyn](https://github.com/jstuczyn))
 - Explorer API - add port check and node description/stats proxy [\#731](https://github.com/nymtech/nym/pull/731) ([mmsinclair](https://github.com/mmsinclair))
-- Feature/nymd client fee handling [\#730](https://github.com/nymtech/nym/pull/730) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/nyxd client fee handling [\#730](https://github.com/nymtech/nym/pull/730) ([jstuczyn](https://github.com/jstuczyn))
 - Update DelegationCheck.tsx [\#725](https://github.com/nymtech/nym/pull/725) ([jessgess](https://github.com/jessgess))
-- Rust nymd/cosmwasm client [\#724](https://github.com/nymtech/nym/pull/724) ([jstuczyn](https://github.com/jstuczyn))
+- Rust nyxd/cosmwasm client [\#724](https://github.com/nymtech/nym/pull/724) ([jstuczyn](https://github.com/jstuczyn))
 - Removed wasm feature bypassing cyclic dependencies [\#723](https://github.com/nymtech/nym/pull/723) ([jstuczyn](https://github.com/jstuczyn))
 - Updated used sphinx dependency to the most recent revision [\#722](https://github.com/nymtech/nym/pull/722) ([jstuczyn](https://github.com/jstuczyn))
 - update state management and validation [\#721](https://github.com/nymtech/nym/pull/721) ([fmtabbara](https://github.com/fmtabbara))
@@ -519,10 +700,8 @@
 - Bond and delegation alerts [\#698](https://github.com/nymtech/nym/pull/698) ([fmtabbara](https://github.com/fmtabbara))
 - Bugfix/network monitor version check [\#697](https://github.com/nymtech/nym/pull/697) ([jstuczyn](https://github.com/jstuczyn))
 - Feature/other containers [\#692](https://github.com/nymtech/nym/pull/692) ([neacsu](https://github.com/neacsu))
-- Using validator API instead of nymd [\#690](https://github.com/nymtech/nym/pull/690) ([futurechimp](https://github.com/futurechimp))
+- Using validator API instead of nyxd [\#690](https://github.com/nymtech/nym/pull/690) ([futurechimp](https://github.com/futurechimp))
 - Hang coconut issuance off the validator-api [\#679](https://github.com/nymtech/nym/pull/679) ([durch](https://github.com/durch))
 - Update hmac and blake3 [\#673](https://github.com/nymtech/nym/pull/673) ([durch](https://github.com/durch))
 
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
+\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_

Cargo.toml

@@ -9,6 +9,10 @@ overflow-checks = true
 [profile.dev]
 panic = "abort"
 
+[profile.test]
+# equivalent of running in `--release` (but since we're in test profile we're keeping overflow checks and all of those by default)
+opt-level = 3
+
 [workspace]
 
 resolver = "2"
@@ -18,20 +22,31 @@ members = [
     "clients/native",
     "clients/native/websocket-requests",
     "clients/socks5",
+    "common/bandwidth-claim-contract",
     "common/client-libs/gateway-client",
     "common/client-libs/mixnet-client",
     "common/client-libs/validator-client",
-    "common/credential-storage",
+    "common/client-connections",
     "common/coconut-interface",
+    "common/commands",
     "common/config",
+    "common/build-information",
+    "common/cosmwasm-smart-contracts/coconut-bandwidth-contract",
+    "common/cosmwasm-smart-contracts/coconut-dkg",
+    "common/cosmwasm-smart-contracts/contracts-common",
+    "common/cosmwasm-smart-contracts/group-contract",
+    "common/cosmwasm-smart-contracts/mixnet-contract",
+    "common/cosmwasm-smart-contracts/multisig-contract",
+    "common/cosmwasm-smart-contracts/vesting-contract",
+    "common/mobile-storage",
+    "common/credential-storage",
     "common/credentials",
     "common/crypto",
     "common/crypto/dkg",
-    "common/bandwidth-claim-contract",
-    "common/cosmwasm-smart-contracts/coconut-bandwidth-contract",
-    "common/cosmwasm-smart-contracts/contracts-common",
-    "common/cosmwasm-smart-contracts/mixnet-contract",
-    "common/cosmwasm-smart-contracts/vesting-contract",
+    "common/execute",
+    "common/inclusion-probability",
+    "common/ledger",
+    "common/logging",
     "common/mixnode-common",
     "common/network-defaults",
     "common/nonexhaustive-delayqueue",
@@ -49,15 +64,26 @@ members = [
     "common/pemstore",
     "common/socks5/proxy-helpers",
     "common/socks5/requests",
+    "common/statistics",
+    "common/task",
     "common/topology",
+    "common/types",
     "common/wasm-utils",
+    "common/completions",
     "explorer-api",
     "gateway",
     "gateway/gateway-requests",
+    "integrations/bity",
     "mixnode",
+    "sdk/rust/nym-sdk",
+    "service-providers/common",
     "service-providers/network-requester",
-    "validator-api",
-    "validator-api/validator-api-requests",
+    "service-providers/network-statistics",
+    "nym-api",
+    "nym-api/nym-api-requests",
+    "nym-outfox",
+    "tools/nym-cli",
+    "tools/ts-rs-cli"
 ]
 
 default-members = [
@@ -65,9 +91,24 @@ default-members = [
     "clients/socks5",
     "gateway",
     "service-providers/network-requester",
+    "service-providers/network-statistics",
     "mixnode",
-    "validator-api",
+    "nym-api",
     "explorer-api",
 ]
 
-exclude = ["explorer", "contracts", "tokenomics-py", "clients/webassembly"]
+exclude = ["explorer", "contracts", "clients/webassembly", "nym-wallet", "nym-connect", "nym-connect-android"]
+
+[workspace.package]
+authors = ["Nym Technologies SA"]
+repository = "https://github.com/nymtech/nym"
+homepage = "https://nymtech.net"
+edition = "2021"
+
+[workspace.dependencies]
+async-trait = "0.1.63"
+log = "0.4"
+thiserror = "1.0.38"
+serde = "1.0.152"
+serde_json = "1.0.91"
+tokio = "1.24.1"

Makefile

@@ -1,11 +1,13 @@
-test: build clippy-all cargo-test wasm fmt
+test: clippy-all cargo-test wasm fmt
+test-all: test cargo-test-expensive
 no-clippy: build cargo-test wasm fmt
 happy: fmt clippy-happy test
-clippy-all: clippy-all-main clippy-all-contracts clippy-all-wallet
-clippy-happy: clippy-happy-main clippy-happy-contracts clippy-happy-wallet
-cargo-test: test-main test-contracts test-wallet
-build: build-contracts build-wallet build-main
-fmt: fmt-main fmt-contracts fmt-wallet
+clippy-all: clippy-main clippy-all-contracts clippy-all-wallet clippy-all-connect clippy-all-wasm-client
+clippy-happy: clippy-happy-main clippy-happy-contracts clippy-happy-wallet clippy-happy-connect
+cargo-test: test-main test-contracts test-wallet test-connect
+cargo-test-expensive: test-main-expensive test-contracts-expensive test-wallet-expensive test-connect-expensive
+build: build-contracts build-wallet build-main build-connect build-wasm-client
+fmt: fmt-main fmt-contracts fmt-wallet fmt-connect fmt-wasm-client
 
 clippy-happy-main:
 	cargo clippy
@@ -16,8 +18,15 @@ clippy-happy-contracts:
 clippy-happy-wallet:
 	cargo clippy --manifest-path nym-wallet/Cargo.toml
 
-clippy-all-main:
-	cargo clippy --workspace --all-features -- -D warnings
+clippy-happy-connect:
+	cargo clippy --manifest-path nym-connect/Cargo.toml
+
+clippy-main:
+	cargo clippy --workspace -- -D warnings
+
+clippy-wasm:
+	cargo clippy --manifest-path clients/webassembly/Cargo.toml --target wasm32-unknown-unknown --workspace -- -D warnings
+
 
 clippy-all-contracts:
 	cargo clippy --workspace --manifest-path contracts/Cargo.toml --all-features --target wasm32-unknown-unknown -- -D warnings
@@ -25,15 +34,36 @@ clippy-all-contracts:
 clippy-all-wallet:
 	cargo clippy --workspace --manifest-path nym-wallet/Cargo.toml --all-features -- -D warnings
 
+clippy-all-connect:
+	cargo clippy --workspace --manifest-path nym-connect/Cargo.toml --all-features -- -D warnings
+
+clippy-all-wasm-client:
+	cargo clippy --workspace --manifest-path clients/webassembly/Cargo.toml --all-features --target wasm32-unknown-unknown -- -D warnings
+
 test-main:
-	cargo test --all-features --workspace --release
+	cargo test --workspace
+
+test-main-expensive:
+	cargo test --workspace -- --ignored
 
 test-contracts:
 	cargo test --manifest-path contracts/Cargo.toml --all-features
 
+test-contracts-expensive:
+	cargo test --manifest-path contracts/Cargo.toml --all-features -- --ignored
+
 test-wallet:
 	cargo test --manifest-path nym-wallet/Cargo.toml --all-features
 
+test-wallet-expensive:
+	cargo test --manifest-path nym-wallet/Cargo.toml --all-features -- --ignored
+
+test-connect:
+	cargo test --manifest-path nym-connect/Cargo.toml --all-features
+
+test-connect-expensive:
+	cargo test --manifest-path nym-connect/Cargo.toml --all-features -- --ignored
+
 build-main:
 	cargo build --workspace
 
@@ -43,6 +73,18 @@ build-contracts:
 build-wallet:
 	cargo build --manifest-path nym-wallet/Cargo.toml --workspace
 
+build-connect:
+	cargo build --manifest-path nym-connect/Cargo.toml --workspace
+
+build-explorer-api:
+	cargo build --manifest-path explorer-api/Cargo.toml --workspace
+
+build-wasm-client:
+	cargo build --manifest-path clients/webassembly/Cargo.toml --workspace --target wasm32-unknown-unknown
+
+build-nym-cli:
+	cargo build --release --manifest-path tools/nym-cli/Cargo.toml
+
 fmt-main:
 	cargo fmt --all
 
@@ -52,5 +94,21 @@ fmt-contracts:
 fmt-wallet:
 	cargo fmt --manifest-path nym-wallet/Cargo.toml --all
 
+fmt-connect:
+	cargo fmt --manifest-path nym-connect/Cargo.toml --all
+
+fmt-wasm-client:
+	cargo fmt --manifest-path clients/webassembly/Cargo.toml --all
+
 wasm:
 	RUSTFLAGS='-C link-arg=-s' cargo build --manifest-path contracts/Cargo.toml --release --target wasm32-unknown-unknown
+
+mixnet-opt: wasm
+	cd contracts/mixnet && make opt
+
+generate-typescript:
+	cd tools/ts-rs-cli && cargo run && cd ../..
+	yarn types:lint:fix
+
+run-validator-tests:
+	cd nym-api/tests/functional_test && yarn test:qa

README.md

@@ -9,8 +9,8 @@ The platform is composed of multiple Rust crates. Top-level executable binary cr
 
 * nym-mixnode - shuffles [Sphinx](https://github.com/nymtech/sphinx) packets together to provide privacy against network-level attackers.
 * nym-client - an executable which you can build into your own applications. Use it for interacting with Nym nodes.
-* nym-socks5-client - a Socks5 proxy you can run on your machine, and use with existing applications
-* nym-gateway - acts sort of like a mailbox for mixnet messages, removing the need for directly delivery to potentially offline or firewalled devices.
+* nym-socks5-client - a Socks5 proxy you can run on your machine and use with existing applications.
+* nym-gateway - acts sort of like a mailbox for mixnet messages, which removes the need for direct delivery to potentially offline or firewalled devices.
 * nym-network-monitor - sends packets through the full system to check that they are working as expected, and stores node uptime histories as the basis of a rewards system ("mixmining" or "proof-of-mixing").
 * nym-explorer - a (projected) block explorer and (existing) mixnet viewer.
 * nym-wallet - a desktop wallet implemented using the [Tauri](https://tauri.studio/en/docs/about/intro) framework. 

assets/fonts/non-variable/README.md

@@ -0,0 +1,8 @@
+Update fonts by doing the following:
+
+1. Go to https://fonts.google.com/specimen/Open+Sans
+2. Add all the styles you want and select `@import`
+3. Copy the url (e.g. curl https://fonts.googleapis.com/css2\?family\=Open+Sans:ital,wght@0,300\;0,400\;0,500\;0,600\;0,700\;0,800\;1,300\;1,400\;1,500\;1,600\;1,700\;1,800\&display\=swap)
+4. Run `curl curl https://fonts.googleapis.com/css2\?family\=Open+Sans:ital,wght@0,300\;0,400\;0,500\;0,600\;0,700\;0,800\;1,300\;1,400\;1,500\;1,600\;1,700\;1,800\&display\=swap`
+5. Use the response as the CSS import directives and download the font files for each font weight
+6. Remember to delete any old font files

assets/fonts/non-variable/fonts.css

@@ -0,0 +1,96 @@
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 300;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0Rk5hkaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 400;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0Rk8ZkaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 500;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0Rk_RkaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 600;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0RkxhjaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 700;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0RkyFjaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: italic;
+    font-weight: 800;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memQYaGs126MiZpBA-UFUIcVXSCEkx2cmqvXlWq8tWZ0Pw86hd0Rk0ZjaVc.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 300;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsiH0C4n.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 400;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsjZ0C4n.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 500;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsjr0C4n.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 600;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsgH1y4n.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 700;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsg-1y4n.ttf) format('truetype');
+}
+@font-face {
+    font-family: 'Open Sans';
+    font-style: normal;
+    font-weight: 800;
+    font-stretch: normal;
+    font-display: swap;
+    src: url(./memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgshZ1y4n.ttf) format('truetype');
+}

assets/token/token-dark-testnet.svg

@@ -0,0 +1,7 @@
+<svg viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M171.7,30.3001 C132.7,-8.7999 69.3001,-8.7999 30.3001,30.3001 C-8.7999,69.4001 -8.7999,132.7 30.3001,171.7 C69.4001,210.8 132.7,210.8 171.7,171.7 C210.8,132.7 210.8,69.3001 171.7,30.3001 Z M163.1,163.1 C128.8,197.4 73.1001,197.4 38.8001,163.1 C4.5001,128.8 4.5001,73.1001 38.8001,38.8001 C73.1001,4.5001 128.8,4.5001 163.1,38.8001 C197.5,73.2001 197.5,128.8 163.1,163.1 Z" id="Shape" fill="#fff"></path>
+    <path d="M163.1,38.9 C128.8,4.60005 73.1002,4.60005 38.8002,38.9 C4.50019,73.2 4.50019,128.9 38.8002,163.2 C73.1002,197.5 128.8,197.5 163.1,163.2 C197.5,128.8 197.5,73.2 163.1,38.9 Z" id="Shape" fill="#000"></path>
+    <g id="T" transform="translate(25, 25) scale(5,5)">
+        <path d="M18.4804688,24 C19.203125,24 19.7182617,23.8608398 20.0258789,23.5825195 C20.3334961,23.3041992 20.4873047,22.9453125 20.4873047,22.5058594 C20.4873047,22.0566406 20.3334961,21.6928711 20.0258789,21.4145508 C19.7182617,21.1362305 19.203125,20.9970703 18.4804688,20.9970703 L18.4804688,20.9970703 L16.4589844,20.9970703 L16.4589844,9.24902344 L19.7548828,9.24902344 L19.7548828,12.0908203 C19.7548828,12.8134766 19.894043,13.3286133 20.1723633,13.6362305 C20.4506836,13.9438477 20.8095703,14.0976562 21.2490234,14.0976562 C21.6982422,14.0976562 22.0620117,13.9438477 22.340332,13.6362305 C22.6186523,13.3286133 22.7578125,12.8134766 22.7578125,12.0908203 L22.7578125,12.0908203 L22.7578125,6.24609375 L7.20117188,6.23144531 L7.20117188,12.0908203 C7.20117188,12.8134766 7.34033203,13.3286133 7.61865234,13.6362305 C7.89697266,13.9438477 8.25585938,14.0976562 8.6953125,14.0976562 C9.14453125,14.0976562 9.50830078,13.9438477 9.78662109,13.6362305 C10.0649414,13.3286133 10.2041016,12.8134766 10.2041016,12.0908203 L10.2041016,12.0908203 L10.2041016,9.24902344 L13.4560547,9.24902344 L13.4560547,20.9970703 L11.4492188,20.9970703 C10.7265625,20.9970703 10.2114258,21.1362305 9.90380859,21.4145508 C9.59619141,21.6928711 9.44238281,22.0517578 9.44238281,22.4912109 C9.44238281,22.9404297 9.59619141,23.3041992 9.90380859,23.5825195 C10.2114258,23.8608398 10.7265625,24 11.4492188,24 L11.4492188,24 L18.4804688,24 Z" id="T" fill="#fff"></path>
+    </g>
+</svg>

assets/token/token-dark.svg

@@ -1,4 +1,4 @@
-<svg width="200" height="200" viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
+<svg viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
     <path d="M170.7 29.3001C131.7 -9.7999 68.3001 -9.7999 29.3001 29.3001C-9.7999 68.4001 -9.7999 131.7 29.3001 170.7C68.4001 209.8 131.7 209.8 170.7 170.7C209.8 131.7 209.8 68.3001 170.7 29.3001ZM162.1 162.1C127.8 196.4 72.1001 196.4 37.8001 162.1C3.5001 127.8 3.5001 72.1001 37.8001 37.8001C72.1001 3.5001 127.8 3.5001 162.1 37.8001C196.5 72.2001 196.5 127.8 162.1 162.1Z" fill="white"/>
     <path d="M162.1 37.9C127.8 3.60005 72.1002 3.60005 37.8002 37.9C3.50019 72.2 3.50019 127.9 37.8002 162.2C72.1002 196.5 127.8 196.5 162.1 162.2C196.5 127.8 196.5 72.2 162.1 37.9ZM63.0002 170.7C56.8002 167.4 51.1002 163.2 46.1002 158.4V41.7C51.3002 36.7 57.2002 32.5 63.6002 29.1L137 140.9V29.3C143.2 32.6 148.9 36.8 153.9 41.6V158.3C148.7 163.3 142.8 167.5 136.4 170.9L63.0002 59.1V170.7Z" fill="#070B15"/>
     <path d="M154 158.3V41.7C148.9 36.9 143.2 32.7 137.1 29.4V140.9L63.5 29C57.1 32.4 51.2 36.6 46 41.6V158.3C51.1 163.1 56.8 167.3 62.9 170.6V59.1L136.5 171C142.9 167.6 148.8 163.3 154 158.3Z" fill="white"/>

assets/token/token-light-testnet.svg

@@ -0,0 +1,7 @@
+<svg viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M171.7,30.3001 C132.7,-8.7999 69.3001,-8.7999 30.3001,30.3001 C-8.7999,69.4001 -8.7999,132.7 30.3001,171.7 C69.4001,210.8 132.7,210.8 171.7,171.7 C210.8,132.7 210.8,69.3001 171.7,30.3001 Z M163.1,163.1 C128.8,197.4 73.1001,197.4 38.8001,163.1 C4.5001,128.8 4.5001,73.1001 38.8001,38.8001 C73.1001,4.5001 128.8,4.5001 163.1,38.8001 C197.5,73.2001 197.5,128.8 163.1,163.1 Z" id="Shape" fill="#141521"></path>
+    <path d="M163.1,38.9 C128.8,4.60005 73.1002,4.60005 38.8002,38.9 C4.50019,73.2 4.50019,128.9 38.8002,163.2 C73.1002,197.5 128.8,197.5 163.1,163.2 C197.5,128.8 197.5,73.2 163.1,38.9 Z" id="Shape" fill="#FFFFFF"></path>
+    <g id="T" transform="translate(25, 25) scale(5,5)">
+        <path d="M18.4804688,24 C19.203125,24 19.7182617,23.8608398 20.0258789,23.5825195 C20.3334961,23.3041992 20.4873047,22.9453125 20.4873047,22.5058594 C20.4873047,22.0566406 20.3334961,21.6928711 20.0258789,21.4145508 C19.7182617,21.1362305 19.203125,20.9970703 18.4804688,20.9970703 L18.4804688,20.9970703 L16.4589844,20.9970703 L16.4589844,9.24902344 L19.7548828,9.24902344 L19.7548828,12.0908203 C19.7548828,12.8134766 19.894043,13.3286133 20.1723633,13.6362305 C20.4506836,13.9438477 20.8095703,14.0976562 21.2490234,14.0976562 C21.6982422,14.0976562 22.0620117,13.9438477 22.340332,13.6362305 C22.6186523,13.3286133 22.7578125,12.8134766 22.7578125,12.0908203 L22.7578125,12.0908203 L22.7578125,6.24609375 L7.20117188,6.23144531 L7.20117188,12.0908203 C7.20117188,12.8134766 7.34033203,13.3286133 7.61865234,13.6362305 C7.89697266,13.9438477 8.25585938,14.0976562 8.6953125,14.0976562 C9.14453125,14.0976562 9.50830078,13.9438477 9.78662109,13.6362305 C10.0649414,13.3286133 10.2041016,12.8134766 10.2041016,12.0908203 L10.2041016,12.0908203 L10.2041016,9.24902344 L13.4560547,9.24902344 L13.4560547,20.9970703 L11.4492188,20.9970703 C10.7265625,20.9970703 10.2114258,21.1362305 9.90380859,21.4145508 C9.59619141,21.6928711 9.44238281,22.0517578 9.44238281,22.4912109 C9.44238281,22.9404297 9.59619141,23.3041992 9.90380859,23.5825195 C10.2114258,23.8608398 10.7265625,24 11.4492188,24 L11.4492188,24 L18.4804688,24 Z" id="T" fill="#000" fill-rule="nonzero"></path>
+    </g>
+</svg>

assets/token/token-light.svg

@@ -1,4 +1,4 @@
-<svg width="200" height="200" viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
+<svg viewBox="0 0 200 200" fill="none" xmlns="http://www.w3.org/2000/svg">
     <path d="M170.7 29.3001C131.7 -9.7999 68.3001 -9.7999 29.3001 29.3001C-9.7999 68.4001 -9.7999 131.7 29.3001 170.7C68.4001 209.8 131.7 209.8 170.7 170.7C209.8 131.7 209.8 68.3001 170.7 29.3001ZM162.1 162.1C127.8 196.4 72.1001 196.4 37.8001 162.1C3.5001 127.8 3.5001 72.1001 37.8001 37.8001C72.1001 3.5001 127.8 3.5001 162.1 37.8001C196.5 72.2001 196.5 127.8 162.1 162.1Z" fill="#141521"/>
     <path d="M162.1 37.9C127.8 3.60005 72.1002 3.60005 37.8002 37.9C3.50019 72.2 3.50019 127.9 37.8002 162.2C72.1002 196.5 127.8 196.5 162.1 162.2C196.5 127.8 196.5 72.2 162.1 37.9ZM63.0002 170.7C56.8002 167.4 51.1002 163.2 46.1002 158.4V41.7C51.3002 36.7 57.2002 32.5 63.6002 29.1L137 140.9V29.3C143.2 32.6 148.9 36.8 153.9 41.6V158.3C148.7 163.3 142.8 167.5 136.4 170.9L63.0002 59.1V170.7Z" fill="white"/>
     <path d="M154 158.3V41.7C148.9 36.9 143.2 32.7 137.1 29.4V140.9L63.5 29C57.1 32.4 51.2 36.6 46 41.6V158.3C51.1 163.1 56.8 167.3 62.9 170.6V59.1L136.5 171C142.9 167.6 148.8 163.3 154 158.3Z" fill="#141521"/>

clients/client-core/Cargo.toml

@@ -1,35 +1,85 @@
 [package]
 name = "client-core"
-version = "1.0.0"
+version = "1.1.8"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 edition = "2021"
+rust-version = "1.66"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-dirs = "3.0"
+async-trait = { version = "0.1.58" }
+dirs = "4.0"
+dashmap = "5.4.0"
 futures = "0.3"
 humantime-serde = "1.0"
-log = "0.4"
+log = { workspace = true }
 rand = { version = "0.7.3", features = ["wasm-bindgen"] }
 serde = { version = "1.0", features = ["derive"] }
-sled = "0.34"
-tokio = { version = "1.4", features = ["macros"] }
+serde_json = "1.0.89"
+tap = "1.0.1"
+thiserror = "1.0.34"
 url = { version ="2.2", features = ["serde"] }
+tokio = { version = "1.24.1", features = ["macros"]}
+time = "0.3.17"
 
 # internal
 config = { path = "../../common/config" }
+client-connections = { path = "../../common/client-connections" }
 crypto = { path = "../../common/crypto" }
 gateway-client = { path = "../../common/client-libs/gateway-client" }
+#gateway-client = { path = "../../common/client-libs/gateway-client", default-features = false, features = ["wasm", "coconut"] }
 gateway-requests = { path = "../../gateway/gateway-requests" }
 nonexhaustive-delayqueue = { path = "../../common/nonexhaustive-delayqueue" }
 nymsphinx = { path = "../../common/nymsphinx" }
 pemstore = { path = "../../common/pemstore" }
 topology = { path = "../../common/topology" }
-validator-client = { path = "../../common/client-libs/validator-client" }
+validator-client = { path = "../../common/client-libs/validator-client", default-features = false }
+task = { path = "../../common/task" }
+
+[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio-stream]
+version = "0.1.11"
+features = ["time"]
+
+[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio]
+version = "1.24.1"
+features = ["time"]
+
+[target."cfg(not(target_arch = \"wasm32\"))".dependencies.sqlx]
+version = "0.6.2"
+features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"]
+optional = true
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen-futures]
+version = "0.4"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen]
+version = "0.2.83"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-timer]
+git = "https://github.com/mmsinclair/wasm-timer"
+rev = "b9d1a54ad514c2f230a026afe0dde341e98cd7b6"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.gloo-timers]
+version = "0.2.4"
+features = ["futures"]
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-utils]
+path = "../../common/wasm-utils"
+
+[target."cfg(target_arch = \"wasm32\")".dependencies.time]
+version = "0.3.17"
+features = ["wasm-bindgen"]
 
 [dev-dependencies]
 tempfile = "3.1.0"
 
+[build-dependencies]
+tokio = { version = "1.24.1", features = ["rt-multi-thread", "macros"] }
+sqlx = { version = "0.6.2", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
+
 [features]
-coconut = ["gateway-client/coconut", "gateway-requests/coconut"]
+default = []
+fs-surb-storage = ["sqlx"]
+wasm = ["gateway-client/wasm"]
+

clients/client-core/build.rs

@@ -0,0 +1,31 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#[tokio::main]
+async fn main() {
+    #[cfg(feature = "fs-surb-storage")]
+    {
+        use sqlx::{Connection, SqliteConnection};
+        use std::env;
+
+        let out_dir = env::var("OUT_DIR").unwrap();
+        let database_path = format!("{out_dir}/fs-surbs-example.sqlite");
+
+        let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc"))
+            .await
+            .expect("Failed to create SQLx database connection");
+
+        sqlx::migrate!("./fs_surbs_migrations")
+            .run(&mut conn)
+            .await
+            .expect("Failed to perform SQLx migrations");
+
+        #[cfg(target_family = "unix")]
+        println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
+
+        #[cfg(target_family = "windows")]
+        // for some strange reason we need to add a leading `/` to the windows path even though it's
+        // not a valid windows path... but hey, it works...
+        println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
+    }
+}

clients/client-core/fs_surbs_migrations/20221130120000_create_initial_tables.sql

@@ -0,0 +1,40 @@
+CREATE TABLE status
+(
+    flush_in_progress        INTEGER NOT NULL,
+    previous_flush_timestamp INTEGER NOT NULL,
+    client_in_use            INTEGER NOT NULL
+);
+
+CREATE TABLE reply_surb_storage_metadata
+(
+    min_reply_surb_threshold INTEGER NOT NULL,
+    max_reply_surb_threshold INTEGER NOT NULL
+);
+
+CREATE TABLE sender_tag
+(
+    recipient BLOB NOT NULL UNIQUE,
+    tag       BLOB NOT NULL UNIQUE
+);
+
+CREATE TABLE reply_key
+(
+    key_digest        BLOB    NOT NULL UNIQUE,
+    reply_key         BLOB    NOT NULL UNIQUE,
+    sent_at_timestamp INTEGER NOT NULL
+);
+
+CREATE TABLE reply_surb_sender
+(
+    id                  INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    last_sent_timestamp INTEGER NOT NULL,
+    tag                 BLOB    NOT NULL UNIQUE
+);
+
+CREATE TABLE reply_surb
+(
+    reply_surb_sender_id INTEGER NOT NULL,
+    reply_surb           BLOB    NOT NULL,
+
+    FOREIGN KEY (reply_surb_sender_id) REFERENCES reply_surb_sender (id)
+);

clients/client-core/src/client/base_client/helpers.rs

@@ -0,0 +1,11 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+//
+use crate::{client::replies::reply_storage, config::DebugConfig};
+
+pub fn setup_empty_reply_surb_backend(debug_config: &DebugConfig) -> reply_storage::Empty {
+    reply_storage::Empty {
+        min_surb_threshold: debug_config.minimum_reply_surb_storage_threshold,
+        max_surb_threshold: debug_config.maximum_reply_surb_storage_threshold,
+    }
+}

clients/client-core/src/client/base_client/mod.rs

@@ -0,0 +1,557 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::cover_traffic_stream::LoopCoverTrafficStream;
+use crate::client::inbound_messages::{InputMessage, InputMessageReceiver, InputMessageSender};
+use crate::client::key_manager::KeyManager;
+use crate::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
+use crate::client::real_messages_control;
+use crate::client::real_messages_control::RealMessagesController;
+use crate::client::received_buffer::{
+    ReceivedBufferRequestReceiver, ReceivedBufferRequestSender, ReceivedMessagesBufferController,
+};
+use crate::client::replies::reply_controller;
+use crate::client::replies::reply_controller::{ReplyControllerReceiver, ReplyControllerSender};
+use crate::client::replies::reply_storage::{
+    CombinedReplyStorage, PersistentReplyStorage, ReplyStorageBackend, SentReplyKeys,
+};
+use crate::client::topology_control::{
+    TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
+};
+use crate::config::{Config, DebugConfig, GatewayEndpointConfig};
+use crate::error::ClientCoreError;
+use crate::spawn_future;
+use client_connections::{ConnectionCommandReceiver, ConnectionCommandSender, LaneQueueLengths};
+use crypto::asymmetric::{encryption, identity};
+use futures::channel::mpsc;
+use gateway_client::bandwidth::BandwidthController;
+#[cfg(target_arch = "wasm32")]
+use gateway_client::wasm_mockups::CosmWasmClient;
+use gateway_client::{
+    AcknowledgementReceiver, AcknowledgementSender, GatewayClient, MixnetMessageReceiver,
+    MixnetMessageSender,
+};
+use log::{debug, info};
+use nymsphinx::acknowledgements::AckKey;
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::addressing::nodes::NodeIdentity;
+use nymsphinx::receiver::ReconstructedMessage;
+use std::sync::Arc;
+use std::time::Duration;
+use tap::TapFallible;
+use task::{TaskClient, TaskManager};
+use url::Url;
+#[cfg(not(target_arch = "wasm32"))]
+use validator_client::nyxd::CosmWasmClient;
+
+use super::received_buffer::ReceivedBufferMessage;
+
+#[cfg(all(not(target_arch = "wasm32"), feature = "fs-surb-storage"))]
+pub mod non_wasm_helpers;
+
+pub mod helpers;
+
+pub struct ClientInput {
+    pub connection_command_sender: ConnectionCommandSender,
+    pub input_sender: InputMessageSender,
+}
+
+pub struct ClientOutput {
+    pub received_buffer_request_sender: ReceivedBufferRequestSender,
+}
+
+impl ClientOutput {
+    pub fn register_receiver(
+        &mut self,
+    ) -> Result<mpsc::UnboundedReceiver<Vec<ReconstructedMessage>>, ClientCoreError> {
+        let (reconstructed_sender, reconstructed_receiver) = mpsc::unbounded();
+
+        self.received_buffer_request_sender
+            .unbounded_send(ReceivedBufferMessage::ReceiverAnnounce(
+                reconstructed_sender,
+            ))
+            .map_err(|_| ClientCoreError::FailedToRegisterReceiver)?;
+
+        Ok(reconstructed_receiver)
+    }
+}
+
+pub struct ClientState {
+    pub shared_lane_queue_lengths: LaneQueueLengths,
+    pub reply_controller_sender: ReplyControllerSender,
+}
+
+pub enum ClientInputStatus {
+    AwaitingProducer { client_input: ClientInput },
+    Connected,
+}
+
+impl ClientInputStatus {
+    pub fn register_producer(&mut self) -> ClientInput {
+        match std::mem::replace(self, ClientInputStatus::Connected) {
+            ClientInputStatus::AwaitingProducer { client_input } => client_input,
+            ClientInputStatus::Connected => panic!("producer was already registered before"),
+        }
+    }
+}
+
+pub enum ClientOutputStatus {
+    AwaitingConsumer { client_output: ClientOutput },
+    Connected,
+}
+
+impl ClientOutputStatus {
+    pub fn register_consumer(&mut self) -> ClientOutput {
+        match std::mem::replace(self, ClientOutputStatus::Connected) {
+            ClientOutputStatus::AwaitingConsumer { client_output } => client_output,
+            ClientOutputStatus::Connected => panic!("consumer was already registered before"),
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub enum CredentialsToggle {
+    Enabled,
+    Disabled,
+}
+
+impl CredentialsToggle {
+    pub fn is_enabled(&self) -> bool {
+        self == &CredentialsToggle::Enabled
+    }
+
+    pub fn is_disabled(&self) -> bool {
+        self == &CredentialsToggle::Disabled
+    }
+}
+
+impl From<bool> for CredentialsToggle {
+    fn from(value: bool) -> Self {
+        if value {
+            CredentialsToggle::Enabled
+        } else {
+            CredentialsToggle::Disabled
+        }
+    }
+}
+
+pub struct BaseClientBuilder<'a, B, C: Clone> {
+    // due to wasm limitations I had to split it like this : (
+    gateway_config: &'a GatewayEndpointConfig,
+    debug_config: &'a DebugConfig,
+    disabled_credentials: bool,
+    nym_api_endpoints: Vec<Url>,
+    reply_storage_backend: B,
+
+    bandwidth_controller: Option<BandwidthController<C>>,
+    key_manager: KeyManager,
+}
+
+impl<'a, B, C> BaseClientBuilder<'a, B, C>
+where
+    B: ReplyStorageBackend + Send + Sync + 'static,
+    C: CosmWasmClient + Sync + Send + Clone + 'static,
+{
+    pub fn new_from_base_config<T>(
+        base_config: &'a Config<T>,
+        key_manager: KeyManager,
+        bandwidth_controller: Option<BandwidthController<C>>,
+        reply_storage_backend: B,
+    ) -> BaseClientBuilder<'a, B, C> {
+        BaseClientBuilder {
+            gateway_config: base_config.get_gateway_endpoint_config(),
+            debug_config: base_config.get_debug_config(),
+            disabled_credentials: base_config.get_disabled_credentials_mode(),
+            nym_api_endpoints: base_config.get_nym_api_endpoints(),
+            bandwidth_controller,
+            reply_storage_backend,
+            key_manager,
+        }
+    }
+
+    pub fn new(
+        gateway_config: &'a GatewayEndpointConfig,
+        debug_config: &'a DebugConfig,
+        key_manager: KeyManager,
+        bandwidth_controller: Option<BandwidthController<C>>,
+        reply_storage_backend: B,
+        credentials_toggle: CredentialsToggle,
+        nym_api_endpoints: Vec<Url>,
+    ) -> BaseClientBuilder<'a, B, C> {
+        BaseClientBuilder {
+            gateway_config,
+            debug_config,
+            disabled_credentials: credentials_toggle.is_disabled(),
+            nym_api_endpoints,
+            reply_storage_backend,
+            bandwidth_controller,
+            key_manager,
+        }
+    }
+
+    pub fn as_mix_recipient(&self) -> Recipient {
+        Recipient::new(
+            *self.key_manager.identity_keypair().public_key(),
+            *self.key_manager.encryption_keypair().public_key(),
+            // TODO: below only works under assumption that gateway address == gateway id
+            // (which currently is true)
+            NodeIdentity::from_base58_string(&self.gateway_config.gateway_id).unwrap(),
+        )
+    }
+
+    // future constantly pumping loop cover traffic at some specified average rate
+    // the pumped traffic goes to the MixTrafficController
+    fn start_cover_traffic_stream(
+        debug_config: &DebugConfig,
+        ack_key: Arc<AckKey>,
+        self_address: Recipient,
+        topology_accessor: TopologyAccessor,
+        mix_tx: BatchMixMessageSender,
+        shutdown: TaskClient,
+    ) {
+        info!("Starting loop cover traffic stream...");
+
+        let mut stream = LoopCoverTrafficStream::new(
+            ack_key,
+            debug_config.average_ack_delay,
+            debug_config.average_packet_delay,
+            debug_config.loop_cover_traffic_average_delay,
+            mix_tx,
+            self_address,
+            topology_accessor,
+        );
+
+        if let Some(size) = debug_config.use_extended_packet_size {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
+        }
+
+        stream.start_with_shutdown(shutdown);
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn start_real_traffic_controller(
+        controller_config: real_messages_control::Config,
+        topology_accessor: TopologyAccessor,
+        ack_receiver: AcknowledgementReceiver,
+        input_receiver: InputMessageReceiver,
+        mix_sender: BatchMixMessageSender,
+        reply_storage: CombinedReplyStorage,
+        reply_controller_sender: ReplyControllerSender,
+        reply_controller_receiver: ReplyControllerReceiver,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
+        shutdown: TaskClient,
+    ) {
+        info!("Starting real traffic stream...");
+
+        RealMessagesController::new(
+            controller_config,
+            ack_receiver,
+            input_receiver,
+            mix_sender,
+            topology_accessor,
+            reply_storage,
+            reply_controller_sender,
+            reply_controller_receiver,
+            lane_queue_lengths,
+            client_connection_rx,
+        )
+        .start_with_shutdown(shutdown);
+    }
+
+    // buffer controlling all messages fetched from provider
+    // required so that other components would be able to use them (say the websocket)
+    fn start_received_messages_buffer_controller(
+        local_encryption_keypair: Arc<encryption::KeyPair>,
+        query_receiver: ReceivedBufferRequestReceiver,
+        mixnet_receiver: MixnetMessageReceiver,
+        reply_key_storage: SentReplyKeys,
+        reply_controller_sender: ReplyControllerSender,
+        shutdown: TaskClient,
+    ) {
+        info!("Starting received messages buffer controller...");
+        ReceivedMessagesBufferController::new(
+            local_encryption_keypair,
+            query_receiver,
+            mixnet_receiver,
+            reply_key_storage,
+            reply_controller_sender,
+        )
+        .start_with_shutdown(shutdown)
+    }
+
+    async fn start_gateway_client(
+        &mut self,
+        mixnet_message_sender: MixnetMessageSender,
+        ack_sender: AcknowledgementSender,
+        shutdown: TaskClient,
+    ) -> Result<GatewayClient<C>, ClientCoreError> {
+        let gateway_id = self.gateway_config.gateway_id.clone();
+        if gateway_id.is_empty() {
+            return Err(ClientCoreError::GatewayIdUnknown);
+        }
+        let gateway_address = self.gateway_config.gateway_listener.clone();
+        if gateway_address.is_empty() {
+            return Err(ClientCoreError::GatwayAddressUnknown);
+        }
+
+        let gateway_identity = identity::PublicKey::from_base58_string(gateway_id)
+            .map_err(ClientCoreError::UnableToCreatePublicKeyFromGatewayId)?;
+
+        // disgusting wasm workaround since there's no key persistence there (nor `client init`)
+        let shared_key = if self.key_manager.is_gateway_key_set() {
+            Some(self.key_manager.gateway_shared_key())
+        } else {
+            None
+        };
+
+        let mut gateway_client = GatewayClient::new(
+            gateway_address,
+            self.key_manager.identity_keypair(),
+            gateway_identity,
+            shared_key,
+            mixnet_message_sender,
+            ack_sender,
+            self.debug_config.gateway_response_timeout,
+            self.bandwidth_controller.take(),
+            shutdown,
+        );
+
+        gateway_client.set_disabled_credentials_mode(self.disabled_credentials);
+
+        gateway_client
+            .authenticate_and_start()
+            .await
+            .tap_err(|err| {
+                log::error!("Could not authenticate and start up the gateway connection - {err}")
+            })?;
+        Ok(gateway_client)
+    }
+
+    // future responsible for periodically polling directory server and updating
+    // the current global view of topology
+    async fn start_topology_refresher(
+        nym_api_urls: Vec<Url>,
+        refresh_rate: Duration,
+        topology_accessor: TopologyAccessor,
+        shutdown: TaskClient,
+    ) -> Result<(), ClientCoreError> {
+        let topology_refresher_config = TopologyRefresherConfig::new(
+            nym_api_urls,
+            refresh_rate,
+            env!("CARGO_PKG_VERSION").to_string(),
+        );
+        let mut topology_refresher =
+            TopologyRefresher::new(topology_refresher_config, topology_accessor);
+        // before returning, block entire runtime to refresh the current network view so that any
+        // components depending on topology would see a non-empty view
+        info!("Obtaining initial network topology");
+        topology_refresher.refresh().await;
+
+        if let Err(err) = topology_refresher.ensure_topology_is_routable().await {
+            log::error!(
+                "The current network topology seem to be insufficient to route any packets through \
+                - check if enough nodes and a gateway are online - source: {err}"
+            );
+            return Err(ClientCoreError::InsufficientNetworkTopology(err));
+        }
+
+        info!("Starting topology refresher...");
+        topology_refresher.start_with_shutdown(shutdown);
+        Ok(())
+    }
+
+    // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
+    // TODO: if we want to send control messages to gateway_client, this CAN'T take the ownership
+    // over it. Perhaps GatewayClient needs to be thread-shareable or have some channel for
+    // requests?
+    fn start_mix_traffic_controller(
+        gateway_client: GatewayClient<C>,
+        shutdown: TaskClient,
+    ) -> BatchMixMessageSender {
+        info!("Starting mix traffic controller...");
+        let (mix_traffic_controller, mix_tx) = MixTrafficController::new(gateway_client);
+        mix_traffic_controller.start_with_shutdown(shutdown);
+        mix_tx
+    }
+
+    async fn setup_persistent_reply_storage(
+        backend: B,
+        shutdown: TaskClient,
+    ) -> Result<CombinedReplyStorage, ClientCoreError>
+    where
+        <B as ReplyStorageBackend>::StorageError: Sync + Send,
+    {
+        if backend.is_active() {
+            log::trace!("Setup persistent reply storage");
+            let persistent_storage = PersistentReplyStorage::new(backend);
+            let mem_store = persistent_storage
+                .load_state_from_backend()
+                .await
+                .map_err(|err| ClientCoreError::SurbStorageError {
+                    source: Box::new(err),
+                })?;
+
+            let store_clone = mem_store.clone();
+            spawn_future(async move {
+                persistent_storage
+                    .flush_on_shutdown(store_clone, shutdown)
+                    .await
+            });
+
+            Ok(mem_store)
+        } else {
+            log::trace!("Setup inactive reply storage");
+            Ok(backend
+                .get_inactive_storage()
+                .map_err(|err| ClientCoreError::SurbStorageError {
+                    source: Box::new(err),
+                })?)
+        }
+    }
+
+    pub async fn start_base(mut self) -> Result<BaseClient, ClientCoreError>
+    where
+        <B as ReplyStorageBackend>::StorageError: Sync + Send,
+    {
+        info!("Starting nym client");
+        // channels for inter-component communication
+        // TODO: make the channels be internally created by the relevant components
+        // rather than creating them here, so say for example the buffer controller would create the request channels
+        // and would allow anyone to clone the sender channel
+
+        // unwrapped_sphinx_sender is the transmitter of mixnet messages received from the gateway
+        // unwrapped_sphinx_receiver is the receiver for said messages - used by ReceivedMessagesBuffer
+        let (mixnet_messages_sender, mixnet_messages_receiver) = mpsc::unbounded();
+
+        // used for announcing connection or disconnection of a channel for pushing re-assembled messages to
+        let (received_buffer_request_sender, received_buffer_request_receiver) = mpsc::unbounded();
+
+        // channels responsible for controlling real messages
+        let (input_sender, input_receiver) = tokio::sync::mpsc::channel::<InputMessage>(1);
+
+        // channels responsible for controlling ack messages
+        let (ack_sender, ack_receiver) = mpsc::unbounded();
+        let shared_topology_accessor = TopologyAccessor::new();
+
+        // Shutdown notifier for signalling tasks to stop
+        let task_manager = TaskManager::default();
+
+        // channels responsible for dealing with reply-related fun
+        let (reply_controller_sender, reply_controller_receiver) =
+            reply_controller::requests::new_control_channels();
+
+        let self_address = self.as_mix_recipient();
+
+        // the components are started in very specific order. Unless you know what you are doing,
+        // do not change that.
+        let gateway_client = self
+            .start_gateway_client(mixnet_messages_sender, ack_sender, task_manager.subscribe())
+            .await?;
+
+        let reply_storage = Self::setup_persistent_reply_storage(
+            self.reply_storage_backend,
+            task_manager.subscribe(),
+        )
+        .await?;
+
+        Self::start_topology_refresher(
+            self.nym_api_endpoints.clone(),
+            self.debug_config.topology_refresh_rate,
+            shared_topology_accessor.clone(),
+            task_manager.subscribe(),
+        )
+        .await?;
+
+        Self::start_received_messages_buffer_controller(
+            self.key_manager.encryption_keypair(),
+            received_buffer_request_receiver,
+            mixnet_messages_receiver,
+            reply_storage.key_storage(),
+            reply_controller_sender.clone(),
+            task_manager.subscribe(),
+        );
+
+        // The sphinx_message_sender is the transmitter for any component generating sphinx packets
+        // that are to be sent to the mixnet. They are used by cover traffic stream and real
+        // traffic stream.
+        // The MixTrafficController then sends the actual traffic
+        let sphinx_message_sender =
+            Self::start_mix_traffic_controller(gateway_client, task_manager.subscribe());
+
+        // Channels that the websocket listener can use to signal downstream to the real traffic
+        // controller that connections are closed.
+        let (client_connection_tx, client_connection_rx) = mpsc::unbounded();
+
+        // Shared queue length data. Published by the `OutQueueController` in the client, and used
+        // primarily to throttle incoming connections (e.g socks5 for attached network-requesters)
+        let shared_lane_queue_lengths = LaneQueueLengths::new();
+
+        let mut controller_config = real_messages_control::Config::new(
+            self.debug_config,
+            self.key_manager.ack_key(),
+            self_address,
+        );
+
+        if let Some(size) = self.debug_config.use_extended_packet_size {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
+        }
+
+        Self::start_real_traffic_controller(
+            controller_config,
+            shared_topology_accessor.clone(),
+            ack_receiver,
+            input_receiver,
+            sphinx_message_sender.clone(),
+            reply_storage,
+            reply_controller_sender.clone(),
+            reply_controller_receiver,
+            shared_lane_queue_lengths.clone(),
+            client_connection_rx,
+            task_manager.subscribe(),
+        );
+
+        if !self.debug_config.disable_loop_cover_traffic_stream {
+            Self::start_cover_traffic_stream(
+                self.debug_config,
+                self.key_manager.ack_key(),
+                self_address,
+                shared_topology_accessor,
+                sphinx_message_sender,
+                task_manager.subscribe(),
+            );
+        }
+
+        debug!("Core client startup finished!");
+        debug!("The address of this client is: {self_address}");
+
+        Ok(BaseClient {
+            client_input: ClientInputStatus::AwaitingProducer {
+                client_input: ClientInput {
+                    connection_command_sender: client_connection_tx,
+                    input_sender,
+                },
+            },
+            client_output: ClientOutputStatus::AwaitingConsumer {
+                client_output: ClientOutput {
+                    received_buffer_request_sender,
+                },
+            },
+            client_state: ClientState {
+                shared_lane_queue_lengths,
+                reply_controller_sender,
+            },
+            task_manager,
+        })
+    }
+}
+
+pub struct BaseClient {
+    pub client_input: ClientInputStatus,
+    pub client_output: ClientOutputStatus,
+    pub client_state: ClientState,
+
+    pub task_manager: TaskManager,
+}

clients/client-core/src/client/base_client/non_wasm_helpers.rs

@@ -0,0 +1,100 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::replies::reply_storage::{
+    fs_backend, CombinedReplyStorage, ReplyStorageBackend,
+};
+use crate::config::DebugConfig;
+use crate::error::ClientCoreError;
+use log::{error, info};
+use std::path::Path;
+use std::{fs, io};
+use time::OffsetDateTime;
+
+async fn setup_fresh_backend<P: AsRef<Path>>(
+    db_path: P,
+    debug_config: &DebugConfig,
+) -> Result<fs_backend::Backend, ClientCoreError> {
+    info!("creating fresh surb database");
+    let mut storage_backend = match fs_backend::Backend::init(db_path).await {
+        Ok(backend) => backend,
+        Err(err) => {
+            error!("failed to setup persistent storage backend for our reply needs: {err}");
+            return Err(ClientCoreError::SurbStorageError {
+                source: Box::new(err),
+            });
+        }
+    };
+
+    // while I kinda hate that we're going to be creating `CombinedReplyStorage` twice,
+    // it will only be happening on the very first run and in practice won't incur huge
+    // costs since the storage is going to be empty
+    let mem_store = CombinedReplyStorage::new(
+        debug_config.minimum_reply_surb_storage_threshold,
+        debug_config.maximum_reply_surb_storage_threshold,
+    );
+    storage_backend
+        .init_fresh(&mem_store)
+        .await
+        .map_err(|err| ClientCoreError::SurbStorageError {
+            source: Box::new(err),
+        })?;
+
+    Ok(storage_backend)
+}
+
+fn setup_inactive_backend(debug_config: &DebugConfig) -> fs_backend::Backend {
+    info!("creating inactive surb database");
+    fs_backend::Backend::new_inactive(
+        debug_config.minimum_reply_surb_storage_threshold,
+        debug_config.maximum_reply_surb_storage_threshold,
+    )
+}
+
+fn archive_corrupted_database<P: AsRef<Path>>(db_path: P) -> io::Result<()> {
+    let db_path = db_path.as_ref();
+    debug_assert!(db_path.exists());
+
+    let now = OffsetDateTime::now_utc().unix_timestamp();
+
+    let suffix = format!("_{now}.corrupted");
+
+    let new_extension =
+        if let Some(existing_extension) = db_path.extension().and_then(|ext| ext.to_str()) {
+            format!("{existing_extension}.{suffix}")
+        } else {
+            suffix
+        };
+
+    let mut renamed = db_path.to_owned();
+    renamed.set_extension(new_extension);
+
+    fs::rename(db_path, renamed)
+}
+
+pub async fn setup_fs_reply_surb_backend<P: AsRef<Path>>(
+    db_path: Option<P>,
+    debug_config: &DebugConfig,
+) -> Result<fs_backend::Backend, ClientCoreError> {
+    if let Some(db_path) = db_path {
+        // if the database file doesnt exist, initialise fresh storage, otherwise attempt to load
+        // the existing one
+        let db_path = db_path.as_ref();
+        if db_path.exists() {
+            info!("loading existing surb database");
+            match fs_backend::Backend::try_load(db_path).await {
+                Ok(backend) => Ok(backend),
+                Err(err) => {
+                    error!("failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future");
+
+                    archive_corrupted_database(db_path)?;
+                    setup_fresh_backend(db_path, debug_config).await
+                }
+            }
+        } else {
+            setup_fresh_backend(db_path, debug_config).await
+        }
+    } else {
+        Ok(setup_inactive_backend(debug_config))
+    }
+}

clients/client-core/src/client/cover_traffic_stream.rs

@@ -3,19 +3,27 @@
 
 use crate::client::mix_traffic::BatchMixMessageSender;
 use crate::client::topology_control::TopologyAccessor;
+use crate::spawn_future;
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
 use log::*;
 use nymsphinx::acknowledgements::AckKey;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::cover::generate_loop_cover_packet;
+use nymsphinx::params::PacketSize;
 use nymsphinx::utils::sample_poisson_duration;
 use rand::{rngs::OsRng, CryptoRng, Rng};
 use std::pin::Pin;
 use std::sync::Arc;
-use tokio::task::JoinHandle;
+use std::time::Duration;
+use tokio::sync::mpsc::error::TrySendError;
+
+#[cfg(not(target_arch = "wasm32"))]
 use tokio::time;
 
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
 pub struct LoopCoverTrafficStream<R>
 where
     R: CryptoRng + Rng,
@@ -24,18 +32,22 @@ where
     ack_key: Arc<AckKey>,
 
     /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
-    average_ack_delay: time::Duration,
+    average_ack_delay: Duration,
 
     /// Average delay a data packet is going to get delay at a single mixnode.
-    average_packet_delay: time::Duration,
+    average_packet_delay: Duration,
 
     /// Average delay between sending subsequent cover packets.
-    average_cover_message_sending_delay: time::Duration,
+    average_cover_message_sending_delay: Duration,
 
     /// Internal state, determined by `average_message_sending_delay`,
     /// used to keep track of when a next packet should be sent out.
+    #[cfg(not(target_arch = "wasm32"))]
     next_delay: Pin<Box<time::Sleep>>,
 
+    #[cfg(target_arch = "wasm32")]
+    next_delay: Pin<Box<wasm_timer::Delay>>,
+
     /// Channel used for sending prepared sphinx packets to `MixTrafficController` that sends them
     /// out to the network without any further delays.
     mix_tx: BatchMixMessageSender,
@@ -48,6 +60,9 @@ where
 
     /// Accessor to the common instance of network topology.
     topology_access: TopologyAccessor,
+
+    /// Predefined packet size used for the loop cover messages.
+    packet_size: PacketSize,
 }
 
 impl<R> Stream for LoopCoverTrafficStream<R>
@@ -69,13 +84,21 @@ where
         // we know it's time to send a message, so let's prepare delay for the next one
         // Get the `now` by looking at the current `delay` deadline
         let avg_delay = self.average_cover_message_sending_delay;
-        let now = self.next_delay.deadline();
         let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
 
         // The next interval value is `next_poisson_delay` after the one that just
         // yielded.
-        let next = now + next_poisson_delay;
-        self.next_delay.as_mut().reset(next);
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let now = self.next_delay.deadline();
+            let next = now + next_poisson_delay;
+            self.next_delay.as_mut().reset(next);
+        }
+
+        #[cfg(target_arch = "wasm32")]
+        {
+            self.next_delay.as_mut().reset(next_poisson_delay);
+        }
 
         Poll::Ready(Some(()))
     }
@@ -84,30 +107,52 @@ where
 // obviously when we finally make shared rng that is on 'higher' level, this should become
 // generic `R`
 impl LoopCoverTrafficStream<OsRng> {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         ack_key: Arc<AckKey>,
-        average_ack_delay: time::Duration,
-        average_packet_delay: time::Duration,
-        average_cover_message_sending_delay: time::Duration,
+        average_ack_delay: Duration,
+        average_packet_delay: Duration,
+        average_cover_message_sending_delay: Duration,
         mix_tx: BatchMixMessageSender,
         our_full_destination: Recipient,
         topology_access: TopologyAccessor,
     ) -> Self {
         let rng = OsRng;
 
+        #[cfg(not(target_arch = "wasm32"))]
+        let next_delay = Box::pin(time::sleep(Default::default()));
+
+        #[cfg(target_arch = "wasm32")]
+        let next_delay = Box::pin(wasm_timer::Delay::new(Default::default()));
+
         LoopCoverTrafficStream {
             ack_key,
             average_ack_delay,
             average_packet_delay,
             average_cover_message_sending_delay,
-            next_delay: Box::pin(time::sleep(Default::default())),
+            next_delay,
             mix_tx,
             our_full_destination,
             rng,
             topology_access,
+            packet_size: Default::default(),
         }
     }
 
+    pub fn set_custom_packet_size(&mut self, packet_size: PacketSize) {
+        self.packet_size = packet_size;
+    }
+
+    fn set_next_delay(&mut self, amount: Duration) {
+        #[cfg(not(target_arch = "wasm32"))]
+        let next_delay = Box::pin(time::sleep(amount));
+
+        #[cfg(target_arch = "wasm32")]
+        let next_delay = Box::pin(wasm_timer::Delay::new(amount));
+
+        self.next_delay = next_delay;
+    }
+
     async fn on_new_message(&mut self) {
         trace!("next cover message!");
 
@@ -116,31 +161,44 @@ impl LoopCoverTrafficStream<OsRng> {
         // poisson delay, but is it really a problem?
         let topology_permit = self.topology_access.get_read_permit().await;
         // the ack is sent back to ourselves (and then ignored)
-        let topology_ref_option = topology_permit.try_get_valid_topology_ref(
+        let topology_ref = match topology_permit.try_get_valid_topology_ref(
             &self.our_full_destination,
             Some(&self.our_full_destination),
-        );
-        if topology_ref_option.is_none() {
-            warn!("No valid topology detected - won't send any loop cover message this time");
-            return;
-        }
-        let topology_ref = topology_ref_option.unwrap();
+        ) {
+            Ok(topology) => topology,
+            Err(err) => {
+                warn!("We're not going to send any loop cover message this time, as the current topology seem to be invalid - {err}");
+                return;
+            }
+        };
 
         let cover_message = generate_loop_cover_packet(
             &mut self.rng,
             topology_ref,
-            &*self.ack_key,
+            &self.ack_key,
             &self.our_full_destination,
             self.average_ack_delay,
             self.average_packet_delay,
+            self.packet_size,
         )
         .expect("Somehow failed to generate a loop cover message with a valid topology");
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        self.mix_tx.unbounded_send(vec![cover_message]).unwrap();
+        if let Err(err) = self.mix_tx.try_send(vec![cover_message]) {
+            match err {
+                TrySendError::Full(_) => {
+                    // This isn't a problem, if the channel is full means we're already sending the
+                    // max amount of messages downstream can handle.
+                    log::debug!("Failed to send cover message - channel full");
+                    // However it's still useful to alert the user that the gateway or the link to
+                    // the gateway can't keep up. Either due to insufficient bandwidth on the
+                    // client side, or that the gateway is overloaded.
+                    log::warn!("Failed to send sphinx packet - gateway or connection to gatway can't keep up");
+                }
+                TrySendError::Closed(_) => {
+                    log::warn!("Failed to send cover message - channel closed");
+                }
+            }
+        }
 
         // TODO: I'm not entirely sure whether this is really required, because I'm not 100%
         // sure how `yield_now()` works - whether it just notifies the scheduler or whether it
@@ -149,24 +207,39 @@ impl LoopCoverTrafficStream<OsRng> {
 
         // JS: due to identical logical structure to OutQueueControl::on_message(), this is also
         // presumably required to prevent bugs in the future. Exact reason is still unknown to me.
+
+        // TODO: temporary and BAD workaround for wasm (we should find a way to yield here in wasm)
+        #[cfg(not(target_arch = "wasm32"))]
         tokio::task::yield_now().await;
     }
 
-    async fn run(&mut self) {
+    pub fn start_with_shutdown(mut self, mut shutdown: task::TaskClient) {
         // we should set initial delay only when we actually start the stream
-        self.next_delay = Box::pin(time::sleep(sample_poisson_duration(
-            &mut self.rng,
-            self.average_cover_message_sending_delay,
-        )));
+        let sampled =
+            sample_poisson_duration(&mut self.rng, self.average_cover_message_sending_delay);
+        self.set_next_delay(sampled);
 
-        while self.next().await.is_some() {
-            self.on_new_message().await;
-        }
-    }
+        spawn_future(async move {
+            debug!("Started LoopCoverTrafficStream with graceful shutdown support");
 
-    pub fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            self.run().await;
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv() => {
+                        log::trace!("LoopCoverTrafficStream: Received shutdown");
+                    }
+                    next = self.next() => {
+                        if next.is_some() {
+                            self.on_new_message().await;
+                        } else {
+                            log::trace!("LoopCoverTrafficStream: Stopping since channel closed");
+                            break;
+                        }
+                    }
+                }
+            }
+            shutdown.recv_timeout().await;
+            log::debug!("LoopCoverTrafficStream: Exiting");
         })
     }
 }

clients/client-core/src/client/helpers/mod.rs

@@ -0,0 +1,14 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#[cfg(not(target_arch = "wasm32"))]
+mod non_wasm;
+
+#[cfg(target_arch = "wasm32")]
+mod wasm;
+
+#[cfg(not(target_arch = "wasm32"))]
+pub use non_wasm::*;
+
+#[cfg(target_arch = "wasm32")]
+pub use wasm::*;

clients/client-core/src/client/helpers/non_wasm.rs

@@ -0,0 +1,13 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub use tokio::time::*;
+pub type IntervalStream = tokio_stream::wrappers::IntervalStream;
+
+pub(crate) fn get_time_now() -> Instant {
+    Instant::now()
+}
+
+pub(crate) fn new_interval_stream(polling_rate: Duration) -> IntervalStream {
+    tokio_stream::wrappers::IntervalStream::new(tokio::time::interval(polling_rate))
+}

clients/client-core/src/client/helpers/wasm.rs

@@ -0,0 +1,16 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use std::time::Duration;
+use wasm_timer;
+
+pub use wasm_timer::*;
+pub type IntervalStream = gloo_timers::future::IntervalStream;
+
+pub(crate) fn get_time_now() -> Instant {
+    wasm_timer::Instant::now()
+}
+
+pub(crate) fn new_interval_stream(polling_rate: Duration) -> IntervalStream {
+    gloo_timers::future::IntervalStream::new(polling_rate.as_millis() as u32)
+}

clients/client-core/src/client/inbound_messages.rs

@@ -1,33 +1,80 @@
-use futures::channel::mpsc;
+use client_connections::TransmissionLane;
 use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::anonymous_replies::ReplySurb;
+use nymsphinx::anonymous_replies::requests::AnonymousSenderTag;
 
-pub type InputMessageSender = mpsc::UnboundedSender<InputMessage>;
-pub type InputMessageReceiver = mpsc::UnboundedReceiver<InputMessage>;
+pub type InputMessageSender = tokio::sync::mpsc::Sender<InputMessage>;
+pub type InputMessageReceiver = tokio::sync::mpsc::Receiver<InputMessage>;
 
 #[derive(Debug)]
 pub enum InputMessage {
-    Fresh {
+    /// The simplest message variant where no additional information is attached.
+    /// You're simply sending your `data` to specified `recipient` without any tagging.
+    ///
+    /// Ends up with `NymMessage::Plain` variant
+    Regular {
         recipient: Recipient,
         data: Vec<u8>,
-        with_reply_surb: bool,
+        lane: TransmissionLane,
     },
-    Reply {
-        reply_surb: ReplySurb,
+
+    /// Creates a message used for a duplex anonymous communication where the recipient
+    /// will never learn of our true identity. This is achieved by carefully sending `reply_surbs`.
+    ///
+    /// Note that if reply_surbs is set to zero then
+    /// this variant requires the client having sent some reply_surbs in the past
+    /// (and thus the recipient also knowing our sender tag).
+    ///
+    /// Ends up with `NymMessage::Repliable` variant
+    Anonymous {
+        recipient: Recipient,
         data: Vec<u8>,
+        reply_surbs: u32,
+        lane: TransmissionLane,
+    },
+
+    /// Attempt to use our internally received and stored `ReplySurb` to send the message back
+    /// to specified recipient whilst not knowing its full identity (or even gateway).
+    ///
+    /// Ends up with `NymMessage::Reply` variant
+    Reply {
+        recipient_tag: AnonymousSenderTag,
+        data: Vec<u8>,
+        lane: TransmissionLane,
     },
 }
 
 impl InputMessage {
-    pub fn new_fresh(recipient: Recipient, data: Vec<u8>, with_reply_surb: bool) -> Self {
-        InputMessage::Fresh {
+    pub fn new_regular(recipient: Recipient, data: Vec<u8>, lane: TransmissionLane) -> Self {
+        InputMessage::Regular {
             recipient,
             data,
-            with_reply_surb,
+            lane,
         }
     }
 
-    pub fn new_reply(reply_surb: ReplySurb, data: Vec<u8>) -> Self {
-        InputMessage::Reply { reply_surb, data }
+    pub fn new_anonymous(
+        recipient: Recipient,
+        data: Vec<u8>,
+        reply_surbs: u32,
+        lane: TransmissionLane,
+    ) -> Self {
+        InputMessage::Anonymous {
+            recipient,
+            data,
+            reply_surbs,
+            lane,
+        }
+    }
+
+    pub fn new_reply(
+        recipient_tag: AnonymousSenderTag,
+        data: Vec<u8>,
+        lane: TransmissionLane,
+    ) -> Self {
+        InputMessage::Reply {
+            recipient_tag,
+            data,
+            lane,
+        }
     }
 }

clients/client-core/src/client/key_manager.rs

@@ -17,6 +17,7 @@ use std::sync::Arc;
 // use the old key after new one was issued.
 
 // Remember that Arc<T> has Deref implementation for T
+#[derive(Clone)]
 pub struct KeyManager {
     /// identity key associated with the client instance.
     identity_keypair: Arc<identity::KeyPair>,
@@ -41,9 +42,6 @@ pub struct KeyManager {
 */
 
 impl KeyManager {
-    // this is actually **NOT** dead code
-    // I have absolutely no idea why the compiler insists it's unused. The call happens during client::init::execute
-    #[allow(dead_code)]
     /// Creates new instance of a [`KeyManager`]
     pub fn new<R>(rng: &mut R) -> Self
     where
@@ -57,16 +55,22 @@ impl KeyManager {
         }
     }
 
-    // this is actually **NOT** dead code
-    // I have absolutely no idea why the compiler insists it's unused. The call happens during client::init::execute
-    #[allow(dead_code)]
-    /// After shared key with the gateway is derived, puts its ownership to this instance of a [`KeyManager`].
-    pub fn insert_gateway_shared_key(&mut self, gateway_shared_key: Arc<SharedKeys>) {
-        self.gateway_shared_key = Some(gateway_shared_key)
+    pub fn from_keys(
+        id_keypair: identity::KeyPair,
+        enc_keypair: encryption::KeyPair,
+        gateway_shared_key: SharedKeys,
+        ack_key: AckKey,
+    ) -> Self {
+        Self {
+            identity_keypair: Arc::new(id_keypair),
+            encryption_keypair: Arc::new(enc_keypair),
+            gateway_shared_key: Some(Arc::new(gateway_shared_key)),
+            ack_key: Arc::new(ack_key),
+        }
     }
 
-    /// Loads previously stored keys from the disk.
-    pub fn load_keys(client_pathfinder: &ClientKeyPathfinder) -> io::Result<Self> {
+    /// Loads previously stored client keys from the disk.
+    fn load_client_keys(client_pathfinder: &ClientKeyPathfinder) -> io::Result<Self> {
         let identity_keypair: identity::KeyPair =
             pemstore::load_keypair(&pemstore::KeyPairPath::new(
                 client_pathfinder.private_identity_key().to_owned(),
@@ -78,24 +82,51 @@ impl KeyManager {
                 client_pathfinder.public_encryption_key().to_owned(),
             ))?;
 
-        let gateway_shared_key: SharedKeys =
-            pemstore::load_key(client_pathfinder.gateway_shared_key())?;
-
         let ack_key: AckKey = pemstore::load_key(client_pathfinder.ack_key())?;
 
-        // TODO: ack key is never stored so it is generated now. But perhaps it should be stored
-        // after all for consistency sake?
         Ok(KeyManager {
             identity_keypair: Arc::new(identity_keypair),
             encryption_keypair: Arc::new(encryption_keypair),
-            gateway_shared_key: Some(Arc::new(gateway_shared_key)),
+            gateway_shared_key: None,
             ack_key: Arc::new(ack_key),
         })
     }
 
-    // this is actually **NOT** dead code
-    // I have absolutely no idea why the compiler insists it's unused. The call happens during client::init::execute
-    #[allow(dead_code)]
+    /// Loads previously stored keys from the disk. Fails if not all, including the shared gateway
+    /// key, is available.
+    pub fn load_keys(client_pathfinder: &ClientKeyPathfinder) -> io::Result<Self> {
+        let mut key_manager = Self::load_client_keys(client_pathfinder)?;
+
+        let gateway_shared_key: SharedKeys =
+            pemstore::load_key(client_pathfinder.gateway_shared_key())?;
+
+        key_manager.gateway_shared_key = Some(Arc::new(gateway_shared_key));
+
+        Ok(key_manager)
+    }
+
+    /// Loads previously stored keys from the disk. Fails if client keys are not availabe, but the
+    /// shared gateway key is optional.
+    pub fn load_keys_but_gateway_is_optional(
+        client_pathfinder: &ClientKeyPathfinder,
+    ) -> io::Result<Self> {
+        let mut key_manager = Self::load_client_keys(client_pathfinder)?;
+
+        let gateway_shared_key: Result<SharedKeys, io::Error> =
+            pemstore::load_key(client_pathfinder.gateway_shared_key());
+
+        // It's ok if the gateway key was not found
+        let gateway_shared_key = match gateway_shared_key {
+            Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
+            Err(err) => Err(err),
+            Ok(key) => Ok(Some(key)),
+        }?;
+
+        key_manager.gateway_shared_key = gateway_shared_key.map(Arc::new);
+
+        Ok(key_manager)
+    }
+
     /// Stores all available keys on the disk.
     // While perhaps there is no much point in storing the `AckKey` on the disk,
     // it is done so for the consistency sake so that you wouldn't require an rng instance
@@ -119,7 +150,7 @@ impl KeyManager {
         pemstore::store_key(self.ack_key.as_ref(), client_pathfinder.ack_key())?;
 
         match self.gateway_shared_key.as_ref() {
-            None => warn!("No gateway shared key available to store!"),
+            None => debug!("No gateway shared key available to store!"),
             Some(gate_key) => {
                 pemstore::store_key(gate_key.as_ref(), client_pathfinder.gateway_shared_key())?
             }
@@ -128,16 +159,57 @@ impl KeyManager {
         Ok(())
     }
 
+    pub fn store_gateway_key(&self, client_pathfinder: &ClientKeyPathfinder) -> io::Result<()> {
+        match self.gateway_shared_key.as_ref() {
+            None => {
+                return Err(io::Error::new(
+                    io::ErrorKind::Other,
+                    "trying to store a non-existing key",
+                ))
+            }
+            Some(gate_key) => {
+                pemstore::store_key(gate_key.as_ref(), client_pathfinder.gateway_shared_key())?
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Overwrite the existing identity keypair
+    pub fn set_identity_keypair(&mut self, id_keypair: identity::KeyPair) {
+        self.identity_keypair = Arc::new(id_keypair);
+    }
+
     /// Gets an atomically reference counted pointer to [`identity::KeyPair`].
     pub fn identity_keypair(&self) -> Arc<identity::KeyPair> {
         Arc::clone(&self.identity_keypair)
     }
 
+    /// Overwrite the existing encryption keypair
+    pub fn set_encryption_keypair(&mut self, enc_keypair: encryption::KeyPair) {
+        self.encryption_keypair = Arc::new(enc_keypair);
+    }
+
     /// Gets an atomically reference counted pointer to [`encryption::KeyPair`].
     pub fn encryption_keypair(&self) -> Arc<encryption::KeyPair> {
         Arc::clone(&self.encryption_keypair)
     }
 
+    /// Overwrite the existing ack key
+    pub fn set_ack_key(&mut self, ack_key: AckKey) {
+        self.ack_key = Arc::new(ack_key);
+    }
+
+    /// Gets an atomically reference counted pointer to [`AckKey`].
+    pub fn ack_key(&self) -> Arc<AckKey> {
+        Arc::clone(&self.ack_key)
+    }
+
+    /// After shared key with the gateway is derived, puts its ownership to this instance of a [`KeyManager`].
+    pub fn insert_gateway_shared_key(&mut self, gateway_shared_key: Arc<SharedKeys>) {
+        self.gateway_shared_key = Some(gateway_shared_key)
+    }
+
     /// Gets an atomically reference counted pointer to [`SharedKey`].
     // since this function is not fully public, it is not expected to be used externally and
     // hence it's up to us to ensure it's called in correct context
@@ -149,8 +221,7 @@ impl KeyManager {
         )
     }
 
-    /// Gets an atomically reference counted pointer to [`AckKey`].
-    pub fn ack_key(&self) -> Arc<AckKey> {
-        Arc::clone(&self.ack_key)
+    pub fn is_gateway_key_set(&self) -> bool {
+        self.gateway_shared_key.is_some()
     }
 }

clients/client-core/src/client/mix_traffic.rs

@@ -1,22 +1,26 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use futures::channel::mpsc;
-use futures::StreamExt;
+use crate::spawn_future;
+#[cfg(target_arch = "wasm32")]
+use gateway_client::wasm_mockups::CosmWasmClient;
 use gateway_client::GatewayClient;
 use log::*;
 use nymsphinx::forwarding::packet::MixPacket;
-use tokio::task::JoinHandle;
+#[cfg(not(target_arch = "wasm32"))]
+use validator_client::nyxd::CosmWasmClient;
 
-pub type BatchMixMessageSender = mpsc::UnboundedSender<Vec<MixPacket>>;
-pub type BatchMixMessageReceiver = mpsc::UnboundedReceiver<Vec<MixPacket>>;
+pub type BatchMixMessageSender = tokio::sync::mpsc::Sender<Vec<MixPacket>>;
+pub type BatchMixMessageReceiver = tokio::sync::mpsc::Receiver<Vec<MixPacket>>;
 
+// We remind ourselves that 32 x 32kb = 1024kb, a reasonable size for a network buffer.
+pub const MIX_MESSAGE_RECEIVER_BUFFER_SIZE: usize = 32;
 const MAX_FAILURE_COUNT: usize = 100;
 
-pub struct MixTrafficController {
+pub struct MixTrafficController<C: Clone> {
     // TODO: most likely to be replaced by some higher level construct as
     // later on gateway_client will need to be accessible by other entities
-    gateway_client: GatewayClient,
+    gateway_client: GatewayClient<C>,
     mix_rx: BatchMixMessageReceiver,
 
     // TODO: this is temporary work-around.
@@ -24,16 +28,23 @@ pub struct MixTrafficController {
     consecutive_gateway_failure_count: usize,
 }
 
-impl MixTrafficController {
+impl<C> MixTrafficController<C>
+where
+    C: CosmWasmClient + Sync + Send + Clone + 'static,
+{
     pub fn new(
-        mix_rx: BatchMixMessageReceiver,
-        gateway_client: GatewayClient,
-    ) -> MixTrafficController {
-        MixTrafficController {
-            gateway_client,
-            mix_rx,
-            consecutive_gateway_failure_count: 0,
-        }
+        gateway_client: GatewayClient<C>,
+    ) -> (MixTrafficController<C>, BatchMixMessageSender) {
+        let (sphinx_message_sender, sphinx_message_receiver) =
+            tokio::sync::mpsc::channel(MIX_MESSAGE_RECEIVER_BUFFER_SIZE);
+        (
+            MixTrafficController {
+                gateway_client,
+                mix_rx: sphinx_message_receiver,
+                consecutive_gateway_failure_count: 0,
+            },
+            sphinx_message_sender,
+        )
     }
 
     async fn on_messages(&mut self, mut mix_packets: Vec<MixPacket>) {
@@ -49,13 +60,13 @@ impl MixTrafficController {
         };
 
         match result {
-            Err(e) => {
-                error!("Failed to send sphinx packet(s) to the gateway! - {:?}", e);
+            Err(err) => {
+                error!("Failed to send sphinx packet(s) to the gateway! - {err}");
                 self.consecutive_gateway_failure_count += 1;
                 if self.consecutive_gateway_failure_count == MAX_FAILURE_COUNT {
                     // todo: in the future this should initiate a 'graceful' shutdown or try
                     // to reconnect?
-                    panic!("failed to send sphinx packet to the gateway {} times in a row - assuming the gateway is dead. Can't do anything about it yet :(", MAX_FAILURE_COUNT)
+                    panic!("failed to send sphinx packet to the gateway {MAX_FAILURE_COUNT} times in a row - assuming the gateway is dead. Can't do anything about it yet :(")
                 }
             }
             Ok(_) => {
@@ -65,15 +76,29 @@ impl MixTrafficController {
         }
     }
 
-    pub async fn run(&mut self) {
-        while let Some(mix_packets) = self.mix_rx.next().await {
-            self.on_messages(mix_packets).await;
-        }
-    }
+    pub fn start_with_shutdown(mut self, mut shutdown: task::TaskClient) {
+        spawn_future(async move {
+            debug!("Started MixTrafficController with graceful shutdown support");
 
-    pub fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            self.run().await;
+            loop {
+                tokio::select! {
+                    mix_packets = self.mix_rx.recv() => match mix_packets {
+                        Some(mix_packets) => {
+                            self.on_messages(mix_packets).await;
+                        },
+                        None => {
+                            log::trace!("MixTrafficController: Stopping since channel closed");
+                            break;
+                        }
+                    },
+                    _ = shutdown.recv_with_delay() => {
+                        log::trace!("MixTrafficController: Received shutdown");
+                        break;
+                    }
+                }
+            }
+            shutdown.recv_timeout().await;
+            log::debug!("MixTrafficController: Exiting");
         })
     }
 }

clients/client-core/src/client/mod.rs

@@ -1,8 +1,14 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod base_client;
 pub mod cover_traffic_stream;
+pub(crate) mod helpers;
 pub mod inbound_messages;
 pub mod key_manager;
 pub mod mix_traffic;
 pub mod real_messages_control;
 pub mod received_buffer;
-pub mod reply_key_storage;
+pub mod replies;
 pub mod topology_control;
+pub(crate) mod transmission_buffer;

clients/client-core/src/client/real_messages_control/acknowledgement_control/acknowledgement_listener.rs

@@ -1,7 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
+use super::action_controller::{AckActionSender, Action};
 use futures::StreamExt;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
@@ -16,14 +16,14 @@ use std::sync::Arc;
 pub(super) struct AcknowledgementListener {
     ack_key: Arc<AckKey>,
     ack_receiver: AcknowledgementReceiver,
-    action_sender: ActionSender,
+    action_sender: AckActionSender,
 }
 
 impl AcknowledgementListener {
     pub(super) fn new(
         ack_key: Arc<AckKey>,
         ack_receiver: AcknowledgementReceiver,
-        action_sender: ActionSender,
+        action_sender: AckActionSender,
     ) -> Self {
         AcknowledgementListener {
             ack_key,
@@ -33,7 +33,7 @@ impl AcknowledgementListener {
     }
 
     async fn on_ack(&mut self, ack_content: Vec<u8>) {
-        debug!("Received an ack");
+        trace!("Received an ack");
         let frag_id = match recover_identifier(&self.ack_key, &ack_content)
             .map(FragmentIdentifier::try_from_bytes)
         {
@@ -49,11 +49,6 @@ impl AcknowledgementListener {
         if frag_id == COVER_FRAG_ID {
             trace!("Received an ack for a cover message - no need to do anything");
             return;
-        } else if frag_id.is_reply() {
-            info!("Received an ack for a reply message - no need to do anything! (don't know what to do!)");
-            // TODO: probably there will need to be some extra procedure here, something to notify
-            // user that his reply reached the recipient (since we got an ack)
-            return;
         }
 
         trace!("Received {} from the mix network", frag_id);
@@ -63,14 +58,31 @@ impl AcknowledgementListener {
             .unwrap();
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started AcknowledgementListener");
-        while let Some(acks) = self.ack_receiver.next().await {
-            // realistically we would only be getting one ack at the time
-            for ack in acks {
-                self.on_ack(ack).await;
+    async fn handle_ack_receiver_item(&mut self, item: Vec<Vec<u8>>) {
+        // realistically we would only be getting one ack at the time
+        for ack in item {
+            self.on_ack(ack).await;
+        }
+    }
+
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started AcknowledgementListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                acks = self.ack_receiver.next() => match acks {
+                    Some(acks) => self.handle_ack_receiver_item(acks).await,
+                    None => {
+                        log::trace!("AcknowledgementListener: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("AcknowledgementListener: Received shutdown");
+                }
             }
         }
-        error!("TODO: error msg. Or maybe panic?")
+        shutdown.recv_timeout().await;
+        log::debug!("AcknowledgementListener: Exiting");
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/action_controller.rs

@@ -3,17 +3,18 @@
 
 use super::PendingAcknowledgement;
 use crate::client::real_messages_control::acknowledgement_control::RetransmissionRequestSender;
-use futures::channel::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use futures::channel::mpsc;
 use futures::StreamExt;
 use log::*;
-use nonexhaustive_delayqueue::{Expired, NonExhaustiveDelayQueue, QueueKey, TimerError};
+use nonexhaustive_delayqueue::{Expired, NonExhaustiveDelayQueue, QueueKey};
 use nymsphinx::chunking::fragment::FragmentIdentifier;
 use nymsphinx::Delay as SphinxDelay;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
-pub(crate) type ActionSender = UnboundedSender<Action>;
+pub(crate) type AckActionSender = mpsc::UnboundedSender<Action>;
+pub(crate) type AckActionReceiver = mpsc::UnboundedReceiver<Action>;
 
 // The actual data being sent off as well as potential key to the delay queue
 type PendingAckEntry = (Arc<PendingAcknowledgement>, Option<QueueKey>);
@@ -95,7 +96,7 @@ pub(super) struct ActionController {
     pending_acks_timers: NonExhaustiveDelayQueue<FragmentIdentifier>,
 
     /// Channel for receiving `Action`s from other modules.
-    incoming_actions: UnboundedReceiver<Action>,
+    incoming_actions: AckActionReceiver,
 
     /// Channel for notifying `RetransmissionRequestListener` about expired acknowledgements.
     retransmission_sender: RetransmissionRequestSender,
@@ -105,18 +106,15 @@ impl ActionController {
     pub(super) fn new(
         config: Config,
         retransmission_sender: RetransmissionRequestSender,
-    ) -> (Self, ActionSender) {
-        let (sender, receiver) = mpsc::unbounded();
-        (
-            ActionController {
-                config,
-                pending_acks_data: HashMap::new(),
-                pending_acks_timers: NonExhaustiveDelayQueue::new(),
-                incoming_actions: receiver,
-                retransmission_sender,
-            },
-            sender,
-        )
+        incoming_actions: AckActionReceiver,
+    ) -> Self {
+        ActionController {
+            config,
+            pending_acks_data: HashMap::new(),
+            pending_acks_timers: NonExhaustiveDelayQueue::new(),
+            incoming_actions,
+            retransmission_sender,
+        }
     }
 
     fn handle_insert(&mut self, pending_acks: Vec<PendingAcknowledgement>) {
@@ -138,13 +136,18 @@ impl ActionController {
         trace!("{} is starting its timer", frag_id);
 
         if let Some((pending_ack_data, queue_key)) = self.pending_acks_data.get_mut(&frag_id) {
-            if queue_key.is_some() {
-                // this branch should be IMPOSSIBLE under ANY condition. It would imply starting
-                // timer TWICE for the SAME PendingAcknowledgement
-                panic!("Tried to start an already started ack timer!")
-            }
-            let timeout = (pending_ack_data.delay.clone() * self.config.ack_wait_multiplier)
-                .to_duration()
+            // the fact that this branch is now POSSIBLE is a sign of a need to refactor this whole
+            // retransmission procedure
+            //
+            // (it can happen as timer is started when ack expires to make sure it's not stuck in memory
+            // and the second instance can be fired when we finally get reply surbs for data we failed to retransmit)
+
+            // if queue_key.is_some() {
+            //     // this branch should be IMPOSSIBLE under ANY condition. It would imply starting
+            //     // timer TWICE for the SAME PendingAcknowledgement
+            //     panic!("Tried to start an already started ack timer!")
+            // }
+            let timeout = (pending_ack_data.delay * self.config.ack_wait_multiplier).to_duration()
                 + self.config.ack_wait_addition;
 
             let new_queue_key = self.pending_acks_timers.insert(frag_id, timeout);
@@ -192,7 +195,8 @@ impl ActionController {
         trace!("{} is updating its delay", frag_id);
         // TODO: is it possible to solve this without either locking or temporarily removing the value?
         if let Some((pending_ack_data, queue_key)) = self.pending_acks_data.remove(&frag_id) {
-            // this Action is triggered by `RetransmissionRequestListener` which held the other potential
+            // this Action is triggered by `RetransmissionRequestListener` (for 'normal' packets)
+            // or `ReplyController` (for 'reply' packets) which held the other potential
             // reference to this Arc. HOWEVER, before the Action was pushed onto the queue, the reference
             // was dropped hence this unwrap is safe.
             let mut inner_data = Arc::try_unwrap(pending_ack_data).unwrap();
@@ -211,14 +215,13 @@ impl ActionController {
     // note: when the entry expires it's automatically removed from pending_acks_timers
     fn handle_expired_ack_timer(
         &mut self,
-        expired_ack: Result<Expired<FragmentIdentifier>, TimerError>,
+        expired_ack: Expired<FragmentIdentifier>,
+        task_client: &mut task::TaskClient,
     ) {
         // I'm honestly not sure how to handle it, because getting it means other things in our
         // system are already misbehaving. If we ever see this panic, then I guess we should worry
         // about it. Perhaps just reschedule it at later point?
-        let frag_id = expired_ack
-            .expect("Tokio timer returned an error!")
-            .into_inner();
+        let frag_id = expired_ack.into_inner();
 
         trace!("{} has expired", frag_id);
 
@@ -232,9 +235,16 @@ impl ActionController {
             // downgrading an arc and then upgrading vs cloning is difference of 30ns vs 15ns
             // so it's literally a NO difference while it might prevent us from unnecessarily
             // resending data (in maybe 1 in 1 million cases, but it's something)
-            self.retransmission_sender
+            if self
+                .retransmission_sender
                 .unbounded_send(Arc::downgrade(pending_ack_data))
-                .unwrap()
+                .is_err()
+            {
+                assert!(
+                    task_client.is_shutdown_poll(),
+                    "Failed to send pending ack for retransmission"
+                );
+            }
         } else {
             // this shouldn't cause any issues but shouldn't have happened to begin with!
             error!("An already removed pending ack has expired")
@@ -250,15 +260,33 @@ impl ActionController {
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        loop {
-            // at some point there will be a global shutdown signal here as the third option
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started ActionController with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
             tokio::select! {
-                // we NEVER expect for ANY sender to get dropped so unwrap here is fine
-                action = self.incoming_actions.next() => self.process_action(action.unwrap()),
-                // pending ack queue Stream CANNOT return a `None` so unwrap here is fine
-                expired_ack = self.pending_acks_timers.next() => self.handle_expired_ack_timer(expired_ack.unwrap())
+                action = self.incoming_actions.next() => match action {
+                    Some(action) => self.process_action(action),
+                    None => {
+                        log::trace!(
+                            "ActionController: Stopping since incoming actions channel closed"
+                        );
+                        break;
+                    }
+                },
+                expired_ack = self.pending_acks_timers.next() => match expired_ack {
+                    Some(expired_ack) => self.handle_expired_ack_timer(expired_ack, &mut shutdown),
+                    None => {
+                        log::trace!("ActionController: Stopping since ack channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("ActionController: Received shutdown");
+                }
             }
         }
+        shutdown.recv_timeout().await;
+        log::debug!("ActionController: Exiting");
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/input_message_listener.rs

@@ -1,21 +1,14 @@
-// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// Copyright 2021-2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
-use super::PendingAcknowledgement;
-use crate::client::reply_key_storage::ReplyKeyStorage;
-use crate::client::{
-    inbound_messages::{InputMessage, InputMessageReceiver},
-    real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
-    topology_control::TopologyAccessor,
-};
-use futures::StreamExt;
+use crate::client::inbound_messages::{InputMessage, InputMessageReceiver};
+use crate::client::real_messages_control::message_handler::MessageHandler;
+use crate::client::replies::reply_controller::ReplyControllerSender;
+use client_connections::TransmissionLane;
 use log::*;
-use nymsphinx::anonymous_replies::ReplySurb;
-use nymsphinx::preparer::MessagePreparer;
-use nymsphinx::{acknowledgements::AckKey, addressing::clients::Recipient};
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::anonymous_replies::requests::AnonymousSenderTag;
 use rand::{CryptoRng, Rng};
-use std::sync::Arc;
 
 /// Module responsible for dealing with the received messages: splitting them, creating acknowledgements,
 /// putting everything into sphinx packets, etc.
@@ -24,14 +17,9 @@ pub(super) struct InputMessageListener<R>
 where
     R: CryptoRng + Rng,
 {
-    ack_key: Arc<AckKey>,
-    ack_recipient: Recipient,
     input_receiver: InputMessageReceiver,
-    message_preparer: MessagePreparer<R>,
-    action_sender: ActionSender,
-    real_message_sender: BatchRealMessageSender,
-    topology_access: TopologyAccessor,
-    reply_key_storage: ReplyKeyStorage,
+    message_handler: MessageHandler<R>,
+    reply_controller_sender: ReplyControllerSender,
 }
 
 impl<R> InputMessageListener<R>
@@ -42,150 +30,105 @@ where
     // some considerable refactoring
     #[allow(clippy::too_many_arguments)]
     pub(super) fn new(
-        ack_key: Arc<AckKey>,
-        ack_recipient: Recipient,
         input_receiver: InputMessageReceiver,
-        message_preparer: MessagePreparer<R>,
-        action_sender: ActionSender,
-        real_message_sender: BatchRealMessageSender,
-        topology_access: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
+        message_handler: MessageHandler<R>,
+        reply_controller_sender: ReplyControllerSender,
     ) -> Self {
         InputMessageListener {
-            ack_key,
-            ack_recipient,
             input_receiver,
-            message_preparer,
-            action_sender,
-            real_message_sender,
-            topology_access,
-            reply_key_storage,
+            message_handler,
+            reply_controller_sender,
         }
     }
 
-    // we require topology for replies to generate surb_acks
-    async fn handle_reply(&mut self, reply_surb: ReplySurb, data: Vec<u8>) -> Option<RealMessage> {
-        let topology_permit = self.topology_access.get_read_permit().await;
-        let topology = match topology_permit.try_get_valid_topology_ref(&self.ack_recipient, None) {
-            Some(topology_ref) => topology_ref,
-            None => {
-                warn!("Could not process the message - the network topology is invalid");
-                return None;
-            }
-        };
-
-        match self
-            .message_preparer
-            .prepare_reply_for_use(data, reply_surb, topology, &self.ack_key)
-            .await
-        {
-            Ok((mix_packet, reply_id)) => {
-                // TODO: later probably write pending ack here
-                // and deal with them....
-                // ... somehow
-                Some(RealMessage::new(mix_packet, reply_id))
-            }
-            Err(err) => {
-                // TODO: should we have some mechanism to indicate to the user that the `reply_surb`
-                // could be reused since technically it wasn't used up here?
-                warn!("failed to deal with received reply surb - {:?}", err);
-                None
-            }
-        }
+    async fn handle_reply(
+        &mut self,
+        recipient_tag: AnonymousSenderTag,
+        data: Vec<u8>,
+        lane: TransmissionLane,
+    ) {
+        // offload reply handling to the dedicated task
+        self.reply_controller_sender
+            .send_reply(recipient_tag, data, lane)
     }
 
-    async fn handle_fresh_message(
+    async fn handle_plain_message(
         &mut self,
         recipient: Recipient,
         content: Vec<u8>,
-        with_reply_surb: bool,
-    ) -> Option<Vec<RealMessage>> {
-        let topology_permit = self.topology_access.get_read_permit().await;
-        let topology = match topology_permit
-            .try_get_valid_topology_ref(&self.ack_recipient, Some(&recipient))
+        lane: TransmissionLane,
+    ) {
+        if let Err(err) = self
+            .message_handler
+            .try_send_plain_message(recipient, content, lane)
+            .await
         {
-            Some(topology_ref) => topology_ref,
-            None => {
-                warn!("Could not process the message - the network topology is invalid");
-                return None;
-            }
-        };
-
-        // split the message, attach optional reply surb
-        let (split_message, reply_key) = self
-            .message_preparer
-            .prepare_and_split_message(content, with_reply_surb, topology)
-            .expect("somehow the topology was invalid after all!");
-
-        if let Some(reply_key) = reply_key {
-            self.reply_key_storage
-                .insert_encryption_key(reply_key)
-                .expect("Failed to insert surb reply key to the store!")
+            warn!("failed to send a plain message - {err}")
         }
+    }
 
-        // encrypt chunks, put them inside sphinx packets and generate acks
-        let mut pending_acks = Vec::with_capacity(split_message.len());
-        let mut real_messages = Vec::with_capacity(split_message.len());
-        for message_chunk in split_message {
-            // we need to clone it because we need to keep it in memory in case we had to retransmit
-            // it. And then we'd need to recreate entire ACK again.
-            let chunk_clone = message_chunk.clone();
-            let prepared_fragment = self
-                .message_preparer
-                .prepare_chunk_for_sending(chunk_clone, topology, &self.ack_key, &recipient)
-                .await
-                .unwrap();
-
-            real_messages.push(RealMessage::new(
-                prepared_fragment.mix_packet,
-                message_chunk.fragment_identifier(),
-            ));
-
-            pending_acks.push(PendingAcknowledgement::new(
-                message_chunk,
-                prepared_fragment.total_delay,
-                recipient,
-            ));
+    async fn handle_repliable_message(
+        &mut self,
+        recipient: Recipient,
+        content: Vec<u8>,
+        reply_surbs: u32,
+        lane: TransmissionLane,
+    ) {
+        if let Err(err) = self
+            .message_handler
+            .try_send_message_with_reply_surbs(recipient, content, reply_surbs, lane)
+            .await
+        {
+            warn!("failed to send a repliable message - {err}")
         }
-
-        // tells the controller to put this into the hashmap
-        self.action_sender
-            .unbounded_send(Action::new_insert(pending_acks))
-            .unwrap();
-
-        Some(real_messages)
     }
 
     async fn on_input_message(&mut self, msg: InputMessage) {
-        let real_messages = match msg {
-            InputMessage::Fresh {
+        match msg {
+            InputMessage::Regular {
                 recipient,
                 data,
-                with_reply_surb,
+                lane,
+            } => self.handle_plain_message(recipient, data, lane).await,
+            InputMessage::Anonymous {
+                recipient,
+                data,
+                reply_surbs,
+                lane,
             } => {
-                self.handle_fresh_message(recipient, data, with_reply_surb)
+                self.handle_repliable_message(recipient, data, reply_surbs, lane)
                     .await
             }
-            InputMessage::Reply { reply_surb, data } => self
-                .handle_reply(reply_surb, data)
-                .await
-                .map(|message| vec![message]),
+            InputMessage::Reply {
+                recipient_tag,
+                data,
+                lane,
+            } => {
+                self.handle_reply(recipient_tag, data, lane).await;
+            }
         };
-
-        // there's no point in trying to send nothing
-        if let Some(real_messages) = real_messages {
-            // tells real message sender (with the poisson timer) to send this to the mix network
-            self.real_message_sender
-                .unbounded_send(real_messages)
-                .unwrap();
-        }
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started InputMessageListener");
-        while let Some(input_msg) = self.input_receiver.next().await {
-            self.on_input_message(input_msg).await;
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started InputMessageListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                input_msg = self.input_receiver.recv() => match input_msg {
+                    Some(input_msg) => {
+                        self.on_input_message(input_msg).await;
+                    },
+                    None => {
+                        log::trace!("InputMessageListener: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("InputMessageListener: Received shutdown");
+                }
+            }
         }
-        error!("TODO: error msg. Or maybe panic?")
+        shutdown.recv_timeout().await;
+        log::debug!("InputMessageListener: Exiting");
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/mod.rs

@@ -7,17 +7,20 @@ use self::{
     retransmission_request_listener::RetransmissionRequestListener,
     sent_notification_listener::SentNotificationListener,
 };
-use super::real_traffic_stream::BatchRealMessageSender;
-use crate::client::reply_key_storage::ReplyKeyStorage;
-use crate::client::{inbound_messages::InputMessageReceiver, topology_control::TopologyAccessor};
+use crate::client::inbound_messages::InputMessageReceiver;
+use crate::client::real_messages_control::message_handler::MessageHandler;
+use crate::client::replies::reply_controller::ReplyControllerSender;
+use crate::spawn_future;
+use action_controller::AckActionReceiver;
 use futures::channel::mpsc;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
+use nymsphinx::anonymous_replies::requests::AnonymousSenderTag;
+use nymsphinx::params::PacketSize;
 use nymsphinx::{
     acknowledgements::AckKey,
     addressing::clients::Recipient,
     chunking::fragment::{Fragment, FragmentIdentifier},
-    preparer::MessagePreparer,
     Delay as SphinxDelay,
 };
 use rand::{CryptoRng, Rng};
@@ -25,7 +28,8 @@ use std::{
     sync::{Arc, Weak},
     time::Duration,
 };
-use tokio::task::JoinHandle;
+
+pub(crate) use action_controller::{AckActionSender, Action};
 
 mod acknowledgement_listener;
 mod action_controller;
@@ -47,24 +51,64 @@ pub(super) type SentPacketNotificationSender = mpsc::UnboundedSender<FragmentIde
 /// that it is about to be sent to the mix network and its timeout timer should be started.
 type SentPacketNotificationReceiver = mpsc::UnboundedReceiver<FragmentIdentifier>;
 
+#[derive(Debug)]
+pub(crate) enum PacketDestination {
+    Anonymous {
+        recipient_tag: AnonymousSenderTag,
+        // special flag to indicate whether this was an ack for requesting additional surbs,
+        // in that case we have to do everything we can to get it through, even if it means going
+        // below our stored reply surb threshold
+        extra_surb_request: bool,
+    },
+    KnownRecipient(Box<Recipient>),
+}
+
 /// Structure representing a data `Fragment` that is on-route to the specified `Recipient`
 #[derive(Debug)]
 pub(crate) struct PendingAcknowledgement {
     message_chunk: Fragment,
     delay: SphinxDelay,
-    recipient: Recipient,
+    destination: PacketDestination,
 }
 
 impl PendingAcknowledgement {
     /// Creates new instance of `PendingAcknowledgement` using the provided data.
-    fn new(message_chunk: Fragment, delay: SphinxDelay, recipient: Recipient) -> Self {
+    pub(crate) fn new_known(
+        message_chunk: Fragment,
+        delay: SphinxDelay,
+        recipient: Recipient,
+    ) -> Self {
         PendingAcknowledgement {
             message_chunk,
             delay,
-            recipient,
+            destination: PacketDestination::KnownRecipient(recipient.into()),
         }
     }
 
+    pub(crate) fn new_anonymous(
+        message_chunk: Fragment,
+        delay: SphinxDelay,
+        recipient_tag: AnonymousSenderTag,
+        extra_surb_request: bool,
+    ) -> Self {
+        PendingAcknowledgement {
+            message_chunk,
+            delay,
+            destination: PacketDestination::Anonymous {
+                recipient_tag,
+                extra_surb_request,
+            },
+        }
+    }
+
+    pub(crate) fn inner_fragment_identifier(&self) -> FragmentIdentifier {
+        self.message_chunk.fragment_identifier()
+    }
+
+    pub(crate) fn fragment_data(&self) -> Fragment {
+        self.message_chunk.clone()
+    }
+
     fn update_delay(&mut self, new_delay: SphinxDelay) {
         self.delay = new_delay;
     }
@@ -73,10 +117,6 @@ impl PendingAcknowledgement {
 /// AcknowledgementControllerConnectors represents set of channels for communication with
 /// other parts of the system in order to support acknowledgements and retransmission.
 pub(super) struct AcknowledgementControllerConnectors {
-    /// Channel used for forwarding prepared sphinx messages into the poisson sender
-    /// to be sent to the mix network.
-    real_message_sender: BatchRealMessageSender,
-
     /// Channel used for receiving raw messages from a client. The messages need to be put
     /// into sphinx packets first.
     input_receiver: InputMessageReceiver,
@@ -88,20 +128,28 @@ pub(super) struct AcknowledgementControllerConnectors {
 
     /// Channel used for receiving acknowledgements from the mix network.
     ack_receiver: AcknowledgementReceiver,
+
+    /// Channel used for sending request to `ActionController` to deal with anything ack-related,
+    ack_action_sender: AckActionSender,
+
+    /// Channel used for receiving request by `ActionController` to deal with anything ack-related,
+    ack_action_receiver: AckActionReceiver,
 }
 
 impl AcknowledgementControllerConnectors {
     pub(super) fn new(
-        real_message_sender: BatchRealMessageSender,
         input_receiver: InputMessageReceiver,
         sent_notifier: SentPacketNotificationReceiver,
         ack_receiver: AcknowledgementReceiver,
+        ack_action_sender: AckActionSender,
+        ack_action_receiver: AckActionReceiver,
     ) -> Self {
         AcknowledgementControllerConnectors {
-            real_message_sender,
             input_receiver,
             sent_notifier,
             ack_receiver,
+            ack_action_sender,
+            ack_action_receiver,
         }
     }
 }
@@ -114,163 +162,135 @@ pub(super) struct Config {
     /// Given ack timeout in the form a * BASE_DELAY + b, it specifies the multiplier `a`
     ack_wait_multiplier: f64,
 
-    /// Average delay an acknowledgement packet is going to get delayed at a single mixnode.
-    average_ack_delay: Duration,
-
-    /// Average delay a data packet is going to get delayed at a single mixnode.
-    average_packet_delay: Duration,
+    /// Predefined packet size used for the encapsulated messages.
+    packet_size: PacketSize,
 }
 
 impl Config {
-    pub(super) fn new(
-        ack_wait_addition: Duration,
-        ack_wait_multiplier: f64,
-        average_ack_delay: Duration,
-        average_packet_delay: Duration,
-    ) -> Self {
+    pub(super) fn new(ack_wait_addition: Duration, ack_wait_multiplier: f64) -> Self {
         Config {
             ack_wait_addition,
             ack_wait_multiplier,
-            average_ack_delay,
-            average_packet_delay,
+            packet_size: Default::default(),
         }
     }
+
+    pub fn with_custom_packet_size(mut self, packet_size: PacketSize) -> Self {
+        self.packet_size = packet_size;
+        self
+    }
 }
 
 pub(super) struct AcknowledgementController<R>
 where
     R: CryptoRng + Rng,
 {
-    acknowledgement_listener: Option<AcknowledgementListener>,
-    input_message_listener: Option<InputMessageListener<R>>,
-    retransmission_request_listener: Option<RetransmissionRequestListener<R>>,
-    sent_notification_listener: Option<SentNotificationListener>,
-    action_controller: Option<ActionController>,
+    acknowledgement_listener: AcknowledgementListener,
+    input_message_listener: InputMessageListener<R>,
+    retransmission_request_listener: RetransmissionRequestListener<R>,
+    sent_notification_listener: SentNotificationListener,
+    action_controller: ActionController,
 }
 
 impl<R> AcknowledgementController<R>
 where
-    R: 'static + CryptoRng + Rng + Clone + Send,
+    R: 'static + CryptoRng + Rng + Clone + Send + Sync,
 {
     pub(super) fn new(
         config: Config,
-        rng: R,
-        topology_access: TopologyAccessor,
         ack_key: Arc<AckKey>,
-        ack_recipient: Recipient,
-        reply_key_storage: ReplyKeyStorage,
         connectors: AcknowledgementControllerConnectors,
+        message_handler: MessageHandler<R>,
+        reply_controller_sender: ReplyControllerSender,
     ) -> Self {
         let (retransmission_tx, retransmission_rx) = mpsc::unbounded();
 
         let action_config =
             action_controller::Config::new(config.ack_wait_addition, config.ack_wait_multiplier);
-        let (action_controller, action_sender) =
-            ActionController::new(action_config, retransmission_tx);
-
-        let message_preparer = MessagePreparer::new(
-            rng,
-            ack_recipient,
-            config.average_packet_delay,
-            config.average_ack_delay,
+        let action_controller = ActionController::new(
+            action_config,
+            retransmission_tx,
+            connectors.ack_action_receiver,
         );
 
         // will listen for any acks coming from the network
         let acknowledgement_listener = AcknowledgementListener::new(
             Arc::clone(&ack_key),
             connectors.ack_receiver,
-            action_sender.clone(),
+            connectors.ack_action_sender.clone(),
         );
 
         // will listen for any new messages from the client
         let input_message_listener = InputMessageListener::new(
-            Arc::clone(&ack_key),
-            ack_recipient,
             connectors.input_receiver,
-            message_preparer.clone(),
-            action_sender.clone(),
-            connectors.real_message_sender.clone(),
-            topology_access.clone(),
-            reply_key_storage,
+            message_handler.clone(),
+            reply_controller_sender.clone(),
         );
 
         // will listen for any ack timeouts and trigger retransmission
         let retransmission_request_listener = RetransmissionRequestListener::new(
-            Arc::clone(&ack_key),
-            ack_recipient,
-            message_preparer,
-            action_sender.clone(),
-            connectors.real_message_sender,
+            connectors.ack_action_sender.clone(),
+            message_handler,
             retransmission_rx,
-            topology_access,
+            reply_controller_sender,
         );
 
         // will listen for events indicating the packet was sent through the network so that
         // the retransmission timer should be started.
         let sent_notification_listener =
-            SentNotificationListener::new(connectors.sent_notifier, action_sender);
+            SentNotificationListener::new(connectors.sent_notifier, connectors.ack_action_sender);
 
         AcknowledgementController {
-            acknowledgement_listener: Some(acknowledgement_listener),
-            input_message_listener: Some(input_message_listener),
-            retransmission_request_listener: Some(retransmission_request_listener),
-            sent_notification_listener: Some(sent_notification_listener),
-            action_controller: Some(action_controller),
+            acknowledgement_listener,
+            input_message_listener,
+            retransmission_request_listener,
+            sent_notification_listener,
+            action_controller,
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        let mut acknowledgement_listener = self.acknowledgement_listener.take().unwrap();
-        let mut input_message_listener = self.input_message_listener.take().unwrap();
-        let mut retransmission_request_listener =
-            self.retransmission_request_listener.take().unwrap();
-        let mut sent_notification_listener = self.sent_notification_listener.take().unwrap();
-        let mut action_controller = self.action_controller.take().unwrap();
+    pub(super) fn start_with_shutdown(self, shutdown: task::TaskClient) {
+        let mut acknowledgement_listener = self.acknowledgement_listener;
+        let mut input_message_listener = self.input_message_listener;
+        let mut retransmission_request_listener = self.retransmission_request_listener;
+        let mut sent_notification_listener = self.sent_notification_listener;
+        let mut action_controller = self.action_controller;
 
-        // the below are log messages are errors as at the current stage we do not expect any of
-        // the task to ever finish. This will of course change once we introduce
-        // graceful shutdowns.
-        let ack_listener_fut = tokio::spawn(async move {
-            acknowledgement_listener.run().await;
-            error!("The acknowledgement listener has finished execution!");
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
             acknowledgement_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The acknowledgement listener has finished execution!");
         });
-        let input_listener_fut = tokio::spawn(async move {
-            input_message_listener.run().await;
-            error!("The input listener has finished execution!");
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
             input_message_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The input listener has finished execution!");
         });
-        let retransmission_req_fut = tokio::spawn(async move {
-            retransmission_request_listener.run().await;
-            error!("The retransmission request listener has finished execution!");
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
             retransmission_request_listener
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The retransmission request listener has finished execution!");
         });
-        let sent_notification_fut = tokio::spawn(async move {
-            sent_notification_listener.run().await;
-            error!("The sent notification listener has finished execution!");
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
             sent_notification_listener
-        });
-        let action_controller_fut = tokio::spawn(async move {
-            action_controller.run().await;
-            error!("The controller has finished execution!");
-            action_controller
+                .run_with_shutdown(shutdown_handle)
+                .await;
+            debug!("The sent notification listener has finished execution!");
         });
 
-        // technically we don't have to bring `AcknowledgementController` back to a valid state
-        // but we can do it, so why not? Perhaps it might be useful if we wanted to allow
-        // for restarts of certain modules without killing the entire process.
-        self.acknowledgement_listener = Some(ack_listener_fut.await.unwrap());
-        self.input_message_listener = Some(input_listener_fut.await.unwrap());
-        self.retransmission_request_listener = Some(retransmission_req_fut.await.unwrap());
-        self.sent_notification_listener = Some(sent_notification_fut.await.unwrap());
-        self.action_controller = Some(action_controller_fut.await.unwrap());
-    }
-
-    #[allow(dead_code)]
-    pub(super) fn start(mut self) -> JoinHandle<Self> {
-        tokio::spawn(async move {
-            self.run().await;
-            self
-        })
+        spawn_future(async move {
+            action_controller.run_with_shutdown(shutdown).await;
+            debug!("The controller has finished execution!");
+        });
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/retransmission_request_listener.rs

@@ -1,32 +1,29 @@
-// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// Copyright 2021-2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
-use super::PendingAcknowledgement;
-use super::RetransmissionRequestReceiver;
-use crate::client::{
-    real_messages_control::real_traffic_stream::{BatchRealMessageSender, RealMessage},
-    topology_control::TopologyAccessor,
+use super::{
+    action_controller::{AckActionSender, Action},
+    PendingAcknowledgement, RetransmissionRequestReceiver,
 };
+use crate::client::real_messages_control::acknowledgement_control::PacketDestination;
+use crate::client::real_messages_control::message_handler::{MessageHandler, PreparationError};
+use crate::client::real_messages_control::real_traffic_stream::RealMessage;
+use crate::client::replies::reply_controller::ReplyControllerSender;
+use client_connections::TransmissionLane;
 use futures::StreamExt;
 use log::*;
-use nymsphinx::preparer::MessagePreparer;
-use nymsphinx::{acknowledgements::AckKey, addressing::clients::Recipient};
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::chunking::fragment::Fragment;
+use nymsphinx::preparer::PreparedFragment;
 use rand::{CryptoRng, Rng};
 use std::sync::{Arc, Weak};
 
 // responsible for packet retransmission upon fired timer
-pub(super) struct RetransmissionRequestListener<R>
-where
-    R: CryptoRng + Rng,
-{
-    ack_key: Arc<AckKey>,
-    ack_recipient: Recipient,
-    message_preparer: MessagePreparer<R>,
-    action_sender: ActionSender,
-    real_message_sender: BatchRealMessageSender,
+pub(super) struct RetransmissionRequestListener<R> {
+    action_sender: AckActionSender,
+    message_handler: MessageHandler<R>,
     request_receiver: RetransmissionRequestReceiver,
-    topology_access: TopologyAccessor,
+    reply_controller_sender: ReplyControllerSender,
 }
 
 impl<R> RetransmissionRequestListener<R>
@@ -34,44 +31,71 @@ where
     R: CryptoRng + Rng,
 {
     pub(super) fn new(
-        ack_key: Arc<AckKey>,
-        ack_recipient: Recipient,
-        message_preparer: MessagePreparer<R>,
-        action_sender: ActionSender,
-        real_message_sender: BatchRealMessageSender,
+        action_sender: AckActionSender,
+        message_handler: MessageHandler<R>,
         request_receiver: RetransmissionRequestReceiver,
-        topology_access: TopologyAccessor,
+        reply_controller_sender: ReplyControllerSender,
     ) -> Self {
         RetransmissionRequestListener {
-            ack_key,
-            ack_recipient,
-            message_preparer,
             action_sender,
-            real_message_sender,
+            message_handler,
             request_receiver,
-            topology_access,
+            reply_controller_sender,
         }
     }
 
-    async fn on_retransmission_request(&mut self, timed_out_ack: Weak<PendingAcknowledgement>) {
-        let timed_out_ack = match timed_out_ack.upgrade() {
+    async fn prepare_normal_retransmission_chunk(
+        &mut self,
+        packet_recipient: Recipient,
+        chunk_data: Fragment,
+    ) -> Result<PreparedFragment, PreparationError> {
+        debug!("retransmitting normal packet...");
+
+        self.message_handler
+            .try_prepare_single_chunk_for_sending(packet_recipient, chunk_data)
+            .await
+    }
+
+    async fn on_retransmission_request(
+        &mut self,
+        weak_timed_out_ack: Weak<PendingAcknowledgement>,
+    ) {
+        let timed_out_ack = match weak_timed_out_ack.upgrade() {
             Some(timed_out_ack) => timed_out_ack,
             None => {
                 debug!("We received an ack JUST as we were about to retransmit [1]");
                 return;
             }
         };
-        let packet_recipient = &timed_out_ack.recipient;
-        let chunk_clone = timed_out_ack.message_chunk.clone();
-        let frag_id = chunk_clone.fragment_identifier();
 
-        let topology_permit = self.topology_access.get_read_permit().await;
-        let topology_ref = match topology_permit
-            .try_get_valid_topology_ref(&self.ack_recipient, Some(packet_recipient))
-        {
-            Some(topology_ref) => topology_ref,
-            None => {
-                warn!("Could not retransmit the packet - the network topology is invalid");
+        let maybe_prepared_fragment = match &timed_out_ack.destination {
+            PacketDestination::Anonymous {
+                recipient_tag,
+                extra_surb_request,
+            } => {
+                // if this is retransmission for reply, offload it to the dedicated task
+                // that deals with all the surbs
+                return self.reply_controller_sender.send_retransmission_data(
+                    *recipient_tag,
+                    weak_timed_out_ack,
+                    *extra_surb_request,
+                );
+            }
+            PacketDestination::KnownRecipient(recipient) => {
+                self.prepare_normal_retransmission_chunk(
+                    **recipient,
+                    timed_out_ack.message_chunk.clone(),
+                )
+                .await
+            }
+        };
+
+        let frag_id = timed_out_ack.message_chunk.fragment_identifier();
+
+        let prepared_fragment = match maybe_prepared_fragment {
+            Ok(prepared_fragment) => prepared_fragment,
+            Err(err) => {
+                warn!("Could not retransmit the packet - {err}");
                 // we NEED to start timer here otherwise we will have this guy permanently stuck in memory
                 self.action_sender
                     .unbounded_send(Action::new_start_timer(frag_id))
@@ -80,12 +104,6 @@ where
             }
         };
 
-        let prepared_fragment = self
-            .message_preparer
-            .prepare_chunk_for_sending(chunk_clone, topology_ref, &self.ack_key, packet_recipient)
-            .await
-            .unwrap();
-
         // if we have the ONLY strong reference to the ack data, it means it was removed from the
         // pending acks
         if Arc::strong_count(&timed_out_ack) == 1 {
@@ -97,7 +115,6 @@ where
         // we no longer need the reference - let's drop it so that if somehow `UpdateTimer` action
         // reached the controller before this function terminated, the controller would not panic.
         drop(timed_out_ack);
-
         let new_delay = prepared_fragment.total_delay;
 
         // We know this update will be reflected by the `StartTimer` Action performed when this
@@ -112,19 +129,32 @@ where
             .unwrap();
 
         // send to `OutQueueControl` to eventually send to the mix network
-        self.real_message_sender
-            .unbounded_send(vec![RealMessage::new(
-                prepared_fragment.mix_packet,
-                frag_id,
-            )])
-            .unwrap();
+        self.message_handler
+            .forward_messages(
+                vec![RealMessage::new(prepared_fragment.mix_packet, frag_id)],
+                TransmissionLane::Retransmission,
+            )
+            .await
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started RetransmissionRequestListener");
-        while let Some(timed_out_ack) = self.request_receiver.next().await {
-            self.on_retransmission_request(timed_out_ack).await;
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started RetransmissionRequestListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                timed_out_ack = self.request_receiver.next() => match timed_out_ack {
+                    Some(timed_out_ack) => self.on_retransmission_request(timed_out_ack).await,
+                    None => {
+                        log::trace!("RetransmissionRequestListener: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv() => {
+                    log::trace!("RetransmissionRequestListener: Received shutdown");
+                }
+            }
         }
-        error!("TODO: error msg. Or maybe panic?")
+        shutdown.recv_timeout().await;
+        log::debug!("RetransmissionRequestListener: Exiting");
     }
 }

clients/client-core/src/client/real_messages_control/acknowledgement_control/sent_notification_listener.rs

@@ -1,7 +1,7 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use super::action_controller::{Action, ActionSender};
+use super::action_controller::{AckActionSender, Action};
 use super::SentPacketNotificationReceiver;
 use futures::StreamExt;
 use log::*;
@@ -13,13 +13,13 @@ use nymsphinx::chunking::fragment::{FragmentIdentifier, COVER_FRAG_ID};
 /// accidentally fire retransmission way quicker than we should have.
 pub(super) struct SentNotificationListener {
     sent_notifier: SentPacketNotificationReceiver,
-    action_sender: ActionSender,
+    action_sender: AckActionSender,
 }
 
 impl SentNotificationListener {
     pub(super) fn new(
         sent_notifier: SentPacketNotificationReceiver,
-        action_sender: ActionSender,
+        action_sender: AckActionSender,
     ) -> Self {
         SentNotificationListener {
             sent_notifier,
@@ -31,22 +31,32 @@ impl SentNotificationListener {
         if frag_id == COVER_FRAG_ID {
             trace!("sent off a cover message - no need to start retransmission timer!");
             return;
-        } else if frag_id.is_reply() {
-            debug!("sent off a reply message - no need to start retransmission timer!");
-            // TODO: probably there will need to be some extra procedure here, like it would
-            // be nice to know that our reply actually reached the recipient (i.e. we got the ack)
-            return;
         }
         self.action_sender
             .unbounded_send(Action::new_start_timer(frag_id))
             .unwrap();
     }
 
-    pub(super) async fn run(&mut self) {
-        debug!("Started SentNotificationListener");
-        while let Some(frag_id) = self.sent_notifier.next().await {
-            self.on_sent_message(frag_id).await;
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started SentNotificationListener with graceful shutdown support");
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                frag_id = self.sent_notifier.next() => match frag_id {
+                    Some(frag_id) => {
+                        self.on_sent_message(frag_id).await;
+                    }
+                    None => {
+                        log::trace!("SentNotificationListener: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("SentNotificationListener: Received shutdown");
+                }
+            }
         }
-        error!("TODO: error msg. Or maybe panic?")
+        assert!(shutdown.is_shutdown_poll());
+        log::debug!("SentNotificationListener: Exiting");
     }
 }

clients/client-core/src/client/real_messages_control/message_handler.rs

@@ -0,0 +1,566 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::real_messages_control::acknowledgement_control::PendingAcknowledgement;
+use crate::client::real_messages_control::real_traffic_stream::{
+    BatchRealMessageSender, RealMessage,
+};
+use crate::client::real_messages_control::{AckActionSender, Action};
+use crate::client::replies::reply_storage::{ReceivedReplySurbsMap, SentReplyKeys, UsedSenderTags};
+use crate::client::topology_control::{TopologyAccessor, TopologyReadPermit};
+use client_connections::TransmissionLane;
+use log::{debug, error, info, trace, warn};
+use nymsphinx::acknowledgements::AckKey;
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::anonymous_replies::requests::{AnonymousSenderTag, RepliableMessage, ReplyMessage};
+use nymsphinx::anonymous_replies::{ReplySurb, SurbEncryptionKey};
+use nymsphinx::chunking::fragment::{Fragment, FragmentIdentifier};
+use nymsphinx::message::NymMessage;
+use nymsphinx::params::{PacketSize, DEFAULT_NUM_MIX_HOPS};
+use nymsphinx::preparer::{MessagePreparer, PreparedFragment};
+use nymsphinx::Delay;
+use rand::{CryptoRng, Rng};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use thiserror::Error;
+use topology::{NymTopology, NymTopologyError};
+
+// TODO: move that error elsewhere since it seems to be contaminating different files
+#[derive(Debug, Clone, Error)]
+pub enum PreparationError {
+    #[error(transparent)]
+    NymTopologyError(#[from] NymTopologyError),
+
+    #[error("The received message cannot be sent using a single reply surb. It ended up getting split into {fragments} fragments.")]
+    MessageTooLongForSingleSurb { fragments: usize },
+
+    #[error("Not enough reply SURBs to send the message. We have {available} available and require at least {required}.")]
+    NotEnoughSurbs { available: usize, required: usize },
+}
+
+impl PreparationError {
+    fn return_surbs(self, returned_surbs: Vec<ReplySurb>) -> SurbWrappedPreparationError {
+        SurbWrappedPreparationError {
+            source: self,
+            returned_surbs: Some(returned_surbs),
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+#[error("Failed to prepare packets - {source}. {} reply surbs will be returned", .returned_surbs.as_ref().map(|s| s.len()).unwrap_or_default())]
+pub struct SurbWrappedPreparationError {
+    #[source]
+    source: PreparationError,
+
+    returned_surbs: Option<Vec<ReplySurb>>,
+}
+
+impl<T> From<T> for SurbWrappedPreparationError
+where
+    T: Into<PreparationError>,
+{
+    fn from(err: T) -> Self {
+        SurbWrappedPreparationError {
+            source: err.into(),
+            returned_surbs: None,
+        }
+    }
+}
+
+impl SurbWrappedPreparationError {
+    pub(crate) fn return_unused_surbs(
+        self,
+        surb_storage: &ReceivedReplySurbsMap,
+        target: &AnonymousSenderTag,
+    ) -> PreparationError {
+        if let Some(reply_surbs) = self.returned_surbs {
+            surb_storage.insert_surbs(target, reply_surbs)
+        }
+        self.source
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct Config {
+    /// Key used to decrypt contents of received SURBAcks
+    ack_key: Arc<AckKey>,
+
+    /// Address of this client which also represent an address to which all acknowledgements
+    /// and surb-based are going to be sent.
+    sender_address: Recipient,
+
+    /// Average delay a data packet is going to get delay at a single mixnode.
+    average_packet_delay: Duration,
+
+    /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
+    average_ack_delay: Duration,
+
+    /// Number of mix hops each packet ('real' message, ack, reply) is expected to take.
+    /// Note that it does not include gateway hops.
+    num_mix_hops: u8,
+
+    /// Predefined packet size used for the encapsulated messages.
+    packet_size: PacketSize,
+}
+
+impl Config {
+    pub fn new(
+        ack_key: Arc<AckKey>,
+        sender_address: Recipient,
+        average_packet_delay: Duration,
+        average_ack_delay: Duration,
+    ) -> Self {
+        Config {
+            ack_key,
+            sender_address,
+            average_packet_delay,
+            average_ack_delay,
+            num_mix_hops: DEFAULT_NUM_MIX_HOPS,
+            packet_size: PacketSize::default(),
+        }
+    }
+
+    /// Allows setting non-default number of expected mix hops in the network.
+    #[allow(dead_code)]
+    pub fn with_mix_hops(mut self, hops: u8) -> Self {
+        self.num_mix_hops = hops;
+        self
+    }
+
+    /// Allows setting non-default size of the sphinx packets sent out.
+    pub fn with_custom_packet_size(mut self, packet_size: PacketSize) -> Self {
+        self.packet_size = packet_size;
+        self
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct MessageHandler<R> {
+    config: Config,
+    rng: R,
+    message_preparer: MessagePreparer<R>,
+    action_sender: AckActionSender,
+    real_message_sender: BatchRealMessageSender,
+    topology_access: TopologyAccessor,
+    reply_key_storage: SentReplyKeys,
+    tag_storage: UsedSenderTags,
+}
+
+impl<R> MessageHandler<R>
+where
+    R: CryptoRng + Rng,
+{
+    pub(crate) fn new(
+        config: Config,
+        rng: R,
+        action_sender: AckActionSender,
+        real_message_sender: BatchRealMessageSender,
+        topology_access: TopologyAccessor,
+        reply_key_storage: SentReplyKeys,
+        tag_storage: UsedSenderTags,
+    ) -> Self
+    where
+        R: Copy,
+    {
+        let message_preparer = MessagePreparer::new(
+            rng,
+            config.sender_address,
+            config.average_packet_delay,
+            config.average_ack_delay,
+        )
+        .with_custom_real_message_packet_size(config.packet_size)
+        .with_mix_hops(config.num_mix_hops);
+
+        MessageHandler {
+            config,
+            rng,
+            message_preparer,
+            action_sender,
+            real_message_sender,
+            topology_access,
+            reply_key_storage,
+            tag_storage,
+        }
+    }
+
+    fn get_or_create_sender_tag(&mut self, recipient: &Recipient) -> AnonymousSenderTag {
+        if let Some(existing) = self.tag_storage.try_get_existing(recipient) {
+            trace!("we already had sender tag for {recipient}");
+            existing
+        } else {
+            info!("creating new sender tag for {recipient}");
+            let new_tag = AnonymousSenderTag::new_random(&mut self.rng);
+            self.tag_storage.insert_new(recipient, new_tag);
+            info!("we'll be using {new_tag} for all anonymous messages sent to {recipient}");
+            new_tag
+        }
+    }
+
+    fn get_topology<'a>(
+        &self,
+        permit: &'a TopologyReadPermit<'a>,
+    ) -> Result<&'a NymTopology, PreparationError> {
+        match permit.try_get_valid_topology_ref(&self.config.sender_address, None) {
+            Ok(topology_ref) => Ok(topology_ref),
+            Err(err) => {
+                warn!("Could not process the packet - the network topology is invalid - {err}");
+                Err(err.into())
+            }
+        }
+    }
+
+    async fn generate_reply_surbs_with_keys(
+        &mut self,
+        amount: usize,
+    ) -> Result<(Vec<ReplySurb>, Vec<SurbEncryptionKey>), PreparationError> {
+        let topology_permit = self.topology_access.get_read_permit().await;
+        let topology = self.get_topology(&topology_permit)?;
+
+        let reply_surbs = self
+            .message_preparer
+            .generate_reply_surbs(amount, topology)?;
+
+        let reply_keys = reply_surbs
+            .iter()
+            .map(|s| *s.encryption_key())
+            .collect::<Vec<_>>();
+
+        Ok((reply_surbs, reply_keys))
+    }
+
+    pub(crate) async fn try_send_single_surb_message(
+        &mut self,
+        target: AnonymousSenderTag,
+        message: ReplyMessage,
+        reply_surb: ReplySurb,
+        is_extra_surb_request: bool,
+    ) -> Result<(), SurbWrappedPreparationError> {
+        let mut fragment = self
+            .message_preparer
+            .pad_and_split_message(NymMessage::new_reply(message));
+        if fragment.len() > 1 {
+            // well, it's not a single surb message
+            return Err(SurbWrappedPreparationError {
+                source: PreparationError::MessageTooLongForSingleSurb {
+                    fragments: fragment.len(),
+                },
+                returned_surbs: Some(vec![reply_surb]),
+            });
+        }
+
+        let chunk = fragment.pop().unwrap();
+        let chunk_clone = chunk.clone();
+        let prepared_fragment = self
+            .try_prepare_single_reply_chunk_for_sending(reply_surb, chunk_clone)
+            .await?;
+
+        let real_messages =
+            RealMessage::new(prepared_fragment.mix_packet, chunk.fragment_identifier());
+        let delay = prepared_fragment.total_delay;
+        let pending_ack =
+            PendingAcknowledgement::new_anonymous(chunk, delay, target, is_extra_surb_request);
+
+        let lane = if is_extra_surb_request {
+            TransmissionLane::ReplySurbRequest
+        } else {
+            TransmissionLane::General
+        };
+
+        self.forward_messages(vec![real_messages], lane).await;
+        self.insert_pending_acks(vec![pending_ack]);
+        Ok(())
+    }
+
+    pub(crate) async fn try_request_additional_reply_surbs(
+        &mut self,
+        from: AnonymousSenderTag,
+        reply_surb: ReplySurb,
+        amount: u32,
+    ) -> Result<(), SurbWrappedPreparationError> {
+        debug!("requesting {amount} reply SURBs from {from}");
+
+        let surbs_request =
+            ReplyMessage::new_surb_request_message(self.config.sender_address, amount);
+        self.try_send_single_surb_message(from, surbs_request, reply_surb, true)
+            .await
+    }
+
+    // // TODO: this will require additional argument to make it use different variant of `ReplyMessage`
+    pub(crate) fn split_reply_message(&mut self, message: Vec<u8>) -> Vec<Fragment> {
+        self.message_preparer
+            .pad_and_split_message(NymMessage::new_reply(ReplyMessage::new_data_message(
+                message,
+            )))
+    }
+
+    pub(crate) async fn send_retransmission_reply_chunks(
+        &mut self,
+        prepared_fragments: Vec<PreparedFragment>,
+        lane: TransmissionLane,
+    ) {
+        let mut real_messages = Vec::with_capacity(prepared_fragments.len());
+
+        for prepared in prepared_fragments {
+            self.update_ack_delay(prepared.fragment_identifier, prepared.total_delay);
+            real_messages.push(prepared.into())
+        }
+
+        self.forward_messages(real_messages, lane).await;
+    }
+
+    pub(crate) async fn try_send_reply_chunks_on_lane(
+        &mut self,
+        target: AnonymousSenderTag,
+        fragments: Vec<Fragment>,
+        reply_surbs: Vec<ReplySurb>,
+        lane: TransmissionLane,
+    ) -> Result<(), SurbWrappedPreparationError> {
+        // TODO: technically this is performing an unnecessary cloning, but in the grand scheme of things
+        // is it really that bad?
+        self.try_send_reply_chunks(
+            target,
+            fragments.into_iter().map(|f| (lane, f)).collect(),
+            reply_surbs,
+        )
+        .await
+    }
+
+    pub(crate) async fn try_send_reply_chunks(
+        &mut self,
+        target: AnonymousSenderTag,
+        fragments: Vec<(TransmissionLane, Fragment)>,
+        reply_surbs: Vec<ReplySurb>,
+    ) -> Result<(), SurbWrappedPreparationError> {
+        let prepared_fragments = self
+            .prepare_reply_chunks_for_sending(
+                fragments.iter().map(|(_, f)| f.clone()).collect(),
+                reply_surbs,
+            )
+            .await?;
+
+        let mut pending_acks = Vec::with_capacity(fragments.len());
+        let mut to_forward: HashMap<_, Vec<_>> = HashMap::new();
+
+        for (raw, prepared) in fragments.into_iter().zip(prepared_fragments.into_iter()) {
+            let lane = raw.0;
+            let fragment = raw.1;
+
+            let real_message = RealMessage::new(prepared.mix_packet, prepared.fragment_identifier);
+            let delay = prepared.total_delay;
+            let pending_ack = PendingAcknowledgement::new_anonymous(fragment, delay, target, false);
+
+            let entry = to_forward.entry(lane).or_default();
+            entry.push(real_message);
+            pending_acks.push(pending_ack);
+        }
+
+        for (lane, real_messages) in to_forward {
+            self.forward_messages(real_messages, lane).await;
+        }
+
+        self.insert_pending_acks(pending_acks);
+        Ok(())
+    }
+
+    pub(crate) async fn try_send_plain_message(
+        &mut self,
+        recipient: Recipient,
+        message: Vec<u8>,
+        lane: TransmissionLane,
+    ) -> Result<(), PreparationError> {
+        let message = NymMessage::new_plain(message);
+        self.try_split_and_send_non_reply_message(message, recipient, lane)
+            .await
+    }
+
+    pub(crate) async fn try_split_and_send_non_reply_message(
+        &mut self,
+        message: NymMessage,
+        recipient: Recipient,
+        lane: TransmissionLane,
+    ) -> Result<(), PreparationError> {
+        // TODO: I really dislike existence of this assertion, it implies code has to be re-organised
+        debug_assert!(!matches!(message, NymMessage::Reply(_)));
+
+        // TODO2: it's really annoying we have to get topology permit again here due to borrow-checker
+        let topology_permit = self.topology_access.get_read_permit().await;
+        let topology = self.get_topology(&topology_permit)?;
+
+        let fragments = self.message_preparer.pad_and_split_message(message);
+
+        let mut pending_acks = Vec::with_capacity(fragments.len());
+        let mut real_messages = Vec::with_capacity(fragments.len());
+        for fragment in fragments {
+            // we need to clone it because we need to keep it in memory in case we had to retransmit
+            // it. And then we'd need to recreate entire ACK again.
+            let chunk_clone = fragment.clone();
+            let prepared_fragment = self.message_preparer.prepare_chunk_for_sending(
+                chunk_clone,
+                topology,
+                &self.config.ack_key,
+                &recipient,
+            )?;
+
+            let real_message =
+                RealMessage::new(prepared_fragment.mix_packet, fragment.fragment_identifier());
+            let delay = prepared_fragment.total_delay;
+            let pending_ack = PendingAcknowledgement::new_known(fragment, delay, recipient);
+
+            real_messages.push(real_message);
+            pending_acks.push(pending_ack);
+        }
+
+        self.insert_pending_acks(pending_acks);
+        self.forward_messages(real_messages, lane).await;
+
+        Ok(())
+    }
+
+    pub(crate) async fn try_send_additional_reply_surbs(
+        &mut self,
+        recipient: Recipient,
+        amount: u32,
+    ) -> Result<(), PreparationError> {
+        let sender_tag = self.get_or_create_sender_tag(&recipient);
+        let (reply_surbs, reply_keys) =
+            self.generate_reply_surbs_with_keys(amount as usize).await?;
+
+        let message = NymMessage::new_repliable(RepliableMessage::new_additional_surbs(
+            sender_tag,
+            reply_surbs,
+        ));
+
+        self.try_split_and_send_non_reply_message(
+            message,
+            recipient,
+            TransmissionLane::AdditionalReplySurbs,
+        )
+        .await?;
+
+        log::trace!("storing {} reply keys", reply_keys.len());
+        self.reply_key_storage.insert_multiple(reply_keys);
+
+        Ok(())
+    }
+
+    pub(crate) async fn try_send_message_with_reply_surbs(
+        &mut self,
+        recipient: Recipient,
+        message: Vec<u8>,
+        num_reply_surbs: u32,
+        lane: TransmissionLane,
+    ) -> Result<(), SurbWrappedPreparationError> {
+        let sender_tag = self.get_or_create_sender_tag(&recipient);
+        let (reply_surbs, reply_keys) = self
+            .generate_reply_surbs_with_keys(num_reply_surbs as usize)
+            .await?;
+
+        let message =
+            NymMessage::new_repliable(RepliableMessage::new_data(message, sender_tag, reply_surbs));
+
+        self.try_split_and_send_non_reply_message(message, recipient, lane)
+            .await?;
+
+        log::trace!("storing {} reply keys", reply_keys.len());
+        self.reply_key_storage.insert_multiple(reply_keys);
+
+        Ok(())
+    }
+
+    pub(crate) async fn try_prepare_single_chunk_for_sending(
+        &mut self,
+        recipient: Recipient,
+        chunk: Fragment,
+    ) -> Result<PreparedFragment, PreparationError> {
+        let topology_permit = self.topology_access.get_read_permit().await;
+        let topology = self.get_topology(&topology_permit)?;
+
+        let prepared_fragment = self
+            .message_preparer
+            .prepare_chunk_for_sending(chunk, topology, &self.config.ack_key, &recipient)
+            .unwrap();
+
+        Ok(prepared_fragment)
+    }
+
+    pub(crate) async fn prepare_reply_chunks_for_sending(
+        &mut self,
+        fragments: Vec<Fragment>,
+        reply_surbs: Vec<ReplySurb>,
+    ) -> Result<Vec<PreparedFragment>, SurbWrappedPreparationError> {
+        debug_assert_ne!(
+            fragments.len(),
+            reply_surbs.len(),
+            "attempted to send {} fragments with {} reply surbs",
+            fragments.len(),
+            reply_surbs.len()
+        );
+
+        let topology_permit = self.topology_access.get_read_permit().await;
+        let topology = match self.get_topology(&topology_permit) {
+            Ok(topology) => topology,
+            Err(err) => return Err(err.return_surbs(reply_surbs)),
+        };
+
+        Ok(fragments
+            .into_iter()
+            .zip(reply_surbs.into_iter())
+            .map(|(fragment, reply_surb)| {
+                // unwrap here is fine as we know we have a valid topology
+                self.message_preparer
+                    .prepare_reply_chunk_for_sending(
+                        fragment,
+                        topology,
+                        &self.config.ack_key,
+                        reply_surb,
+                    )
+                    .unwrap()
+            })
+            .collect())
+    }
+
+    pub(crate) async fn try_prepare_single_reply_chunk_for_sending(
+        &mut self,
+        reply_surb: ReplySurb,
+        chunk: Fragment,
+    ) -> Result<PreparedFragment, SurbWrappedPreparationError> {
+        let topology_permit = self.topology_access.get_read_permit().await;
+        let topology = match self.get_topology(&topology_permit) {
+            Ok(topology) => topology,
+            Err(err) => return Err(err.return_surbs(vec![reply_surb])),
+        };
+
+        let prepared_fragment = self
+            .message_preparer
+            .prepare_reply_chunk_for_sending(chunk, topology, &self.config.ack_key, reply_surb)
+            .unwrap();
+
+        Ok(prepared_fragment)
+    }
+
+    pub(crate) fn update_ack_delay(&self, id: FragmentIdentifier, new_delay: Delay) {
+        self.action_sender
+            .unbounded_send(Action::UpdateDelay(id, new_delay))
+            .expect("action control task has died")
+    }
+
+    pub(crate) fn insert_pending_acks(&self, pending_acks: Vec<PendingAcknowledgement>) {
+        self.action_sender
+            .unbounded_send(Action::new_insert(pending_acks))
+            .expect("action control task has died")
+    }
+
+    // tells real message sender (with the poisson timer) to send this to the mix network
+    pub(crate) async fn forward_messages(
+        &self,
+        messages: Vec<RealMessage>,
+        transmission_lane: TransmissionLane,
+    ) {
+        self.real_message_sender
+            .send((messages, transmission_lane))
+            .await
+            .expect("real message receiver task (OutQueueControl) has died");
+    }
+}

clients/client-core/src/client/real_messages_control/mod.rs

@@ -8,24 +8,37 @@
 use self::{
     acknowledgement_control::AcknowledgementController, real_traffic_stream::OutQueueControl,
 };
-use crate::client::real_messages_control::acknowledgement_control::AcknowledgementControllerConnectors;
-use crate::client::reply_key_storage::ReplyKeyStorage;
-use crate::client::{
-    inbound_messages::InputMessageReceiver, mix_traffic::BatchMixMessageSender,
-    topology_control::TopologyAccessor,
+use crate::client::real_messages_control::message_handler::MessageHandler;
+use crate::client::replies::reply_controller::{
+    ReplyController, ReplyControllerReceiver, ReplyControllerSender,
 };
+use crate::client::replies::reply_storage::CombinedReplyStorage;
+use crate::{
+    client::{
+        inbound_messages::InputMessageReceiver, mix_traffic::BatchMixMessageSender,
+        real_messages_control::acknowledgement_control::AcknowledgementControllerConnectors,
+        topology_control::TopologyAccessor,
+    },
+    spawn_future,
+};
+use client_connections::{ConnectionCommandReceiver, LaneQueueLengths};
 use futures::channel::mpsc;
 use gateway_client::AcknowledgementReceiver;
 use log::*;
 use nymsphinx::acknowledgements::AckKey;
 use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::params::PacketSize;
 use rand::{rngs::OsRng, CryptoRng, Rng};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::task::JoinHandle;
 
-mod acknowledgement_control;
-mod real_traffic_stream;
+use crate::client::replies::reply_controller;
+use crate::config;
+pub(crate) use acknowledgement_control::{AckActionSender, Action};
+
+pub(crate) mod acknowledgement_control;
+pub(crate) mod message_handler;
+pub(crate) mod real_traffic_stream;
 
 // TODO: ack_key and self_recipient shouldn't really be part of this config
 pub struct Config {
@@ -49,130 +62,230 @@ pub struct Config {
 
     /// Average delay an acknowledgement packet is going to get delayed at a single mixnode.
     average_ack_delay_duration: Duration,
+
+    /// Controls whether the main packet stream constantly produces packets according to the predefined
+    /// poisson distribution.
+    disable_main_poisson_packet_distribution: bool,
+
+    /// Predefined packet size used for the encapsulated messages.
+    packet_size: PacketSize,
+
+    /// Defines the minimum number of reply surbs the client would request.
+    minimum_reply_surb_request_size: u32,
+
+    /// Defines the maximum number of reply surbs the client would request.
+    maximum_reply_surb_request_size: u32,
+
+    /// Defines the maximum number of reply surbs a remote party is allowed to request from this client at once.
+    maximum_allowed_reply_surb_request_size: u32,
+
+    /// Defines maximum amount of time the client is going to wait for reply surbs before explicitly asking
+    /// for more even though in theory they wouldn't need to.
+    maximum_reply_surb_rerequest_waiting_period: Duration,
+
+    /// Defines maximum amount of time the client is going to wait for reply surbs before
+    /// deciding it's never going to get them and would drop all pending messages
+    maximum_reply_surb_drop_waiting_period: Duration,
+
+    /// Defines maximum amount of time given reply surb is going to be valid for.
+    /// This is going to be superseded by key rotation once implemented.
+    maximum_reply_surb_age: Duration,
+
+    /// Defines maximum amount of time given reply key is going to be valid for.
+    /// This is going to be superseded by key rotation once implemented.
+    maximum_reply_key_age: Duration,
+}
+
+impl<'a> From<&'a Config> for acknowledgement_control::Config {
+    fn from(cfg: &'a Config) -> Self {
+        acknowledgement_control::Config::new(cfg.ack_wait_addition, cfg.ack_wait_multiplier)
+            .with_custom_packet_size(cfg.packet_size)
+    }
+}
+
+impl<'a> From<&'a Config> for real_traffic_stream::Config {
+    fn from(cfg: &'a Config) -> Self {
+        real_traffic_stream::Config::new(
+            Arc::clone(&cfg.ack_key),
+            cfg.self_recipient,
+            cfg.average_ack_delay_duration,
+            cfg.average_packet_delay_duration,
+            cfg.average_message_sending_delay,
+            cfg.disable_main_poisson_packet_distribution,
+        )
+        .with_custom_cover_packet_size(cfg.packet_size)
+    }
+}
+
+impl<'a> From<&'a Config> for reply_controller::Config {
+    fn from(cfg: &'a Config) -> Self {
+        reply_controller::Config::new(
+            cfg.minimum_reply_surb_request_size,
+            cfg.maximum_reply_surb_request_size,
+            cfg.maximum_allowed_reply_surb_request_size,
+            cfg.maximum_reply_surb_rerequest_waiting_period,
+            cfg.maximum_reply_surb_drop_waiting_period,
+            cfg.maximum_reply_surb_age,
+            cfg.maximum_reply_key_age,
+        )
+    }
+}
+
+impl<'a> From<&'a Config> for message_handler::Config {
+    fn from(cfg: &'a Config) -> Self {
+        message_handler::Config::new(
+            Arc::clone(&cfg.ack_key),
+            cfg.self_recipient,
+            cfg.average_packet_delay_duration,
+            cfg.average_ack_delay_duration,
+        )
+        .with_custom_packet_size(cfg.packet_size)
+    }
 }
 
 impl Config {
     pub fn new(
+        base_client_debug_config: &config::DebugConfig,
         ack_key: Arc<AckKey>,
-        ack_wait_multiplier: f64,
-        ack_wait_addition: Duration,
-        average_ack_delay_duration: Duration,
-        average_message_sending_delay: Duration,
-        average_packet_delay_duration: Duration,
         self_recipient: Recipient,
     ) -> Self {
         Config {
             ack_key,
-            ack_wait_addition,
-            ack_wait_multiplier,
             self_recipient,
-            average_message_sending_delay,
-            average_packet_delay_duration,
-            average_ack_delay_duration,
+            packet_size: Default::default(),
+            ack_wait_addition: base_client_debug_config.ack_wait_addition,
+            ack_wait_multiplier: base_client_debug_config.ack_wait_multiplier,
+            average_message_sending_delay: base_client_debug_config.message_sending_average_delay,
+            average_packet_delay_duration: base_client_debug_config.average_packet_delay,
+            average_ack_delay_duration: base_client_debug_config.average_ack_delay,
+            disable_main_poisson_packet_distribution: base_client_debug_config
+                .disable_main_poisson_packet_distribution,
+            minimum_reply_surb_request_size: base_client_debug_config
+                .minimum_reply_surb_request_size,
+            maximum_reply_surb_request_size: base_client_debug_config
+                .maximum_reply_surb_request_size,
+            maximum_allowed_reply_surb_request_size: base_client_debug_config
+                .maximum_allowed_reply_surb_request_size,
+            maximum_reply_surb_rerequest_waiting_period: base_client_debug_config
+                .maximum_reply_surb_rerequest_waiting_period,
+            maximum_reply_surb_drop_waiting_period: base_client_debug_config
+                .maximum_reply_surb_drop_waiting_period,
+            maximum_reply_surb_age: base_client_debug_config.maximum_reply_surb_age,
+            maximum_reply_key_age: base_client_debug_config.maximum_reply_key_age,
         }
     }
+
+    pub fn set_custom_packet_size(&mut self, packet_size: PacketSize) {
+        self.packet_size = packet_size;
+    }
 }
 
-pub struct RealMessagesController<R>
+pub(crate) struct RealMessagesController<R>
 where
     R: CryptoRng + Rng,
 {
-    out_queue_control: Option<OutQueueControl<R>>,
-    ack_control: Option<AcknowledgementController<R>>,
+    out_queue_control: OutQueueControl<R>,
+    ack_control: AcknowledgementController<R>,
+    reply_control: ReplyController<R>,
 }
 
 // obviously when we finally make shared rng that is on 'higher' level, this should become
 // generic `R`
 impl RealMessagesController<OsRng> {
-    pub fn new(
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
         config: Config,
         ack_receiver: AcknowledgementReceiver,
         input_receiver: InputMessageReceiver,
         mix_sender: BatchMixMessageSender,
         topology_access: TopologyAccessor,
-        reply_key_storage: ReplyKeyStorage,
+        reply_storage: CombinedReplyStorage,
+        // so much refactoring needed, but this is temporary just to test things out
+        reply_controller_sender: ReplyControllerSender,
+        reply_controller_receiver: ReplyControllerReceiver,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
     ) -> Self {
         let rng = OsRng;
 
-        let (real_message_sender, real_message_receiver) = mpsc::unbounded();
+        // create channels for inter-task communication
+        let (real_message_sender, real_message_receiver) = tokio::sync::mpsc::channel(1);
         let (sent_notifier_tx, sent_notifier_rx) = mpsc::unbounded();
-
+        let (ack_action_tx, ack_action_rx) = mpsc::unbounded();
         let ack_controller_connectors = AcknowledgementControllerConnectors::new(
-            real_message_sender,
             input_receiver,
             sent_notifier_rx,
             ack_receiver,
+            ack_action_tx.clone(),
+            ack_action_rx,
         );
 
-        let ack_control_config = acknowledgement_control::Config::new(
-            config.ack_wait_addition,
-            config.ack_wait_multiplier,
-            config.average_ack_delay_duration,
-            config.average_packet_delay_duration,
+        // create all configs for the components
+        let ack_control_config = (&config).into();
+        let out_queue_config = (&config).into();
+        let reply_controller_config = (&config).into();
+        let message_handler_config = (&config).into();
+
+        // create the actual components
+        let message_handler = MessageHandler::new(
+            message_handler_config,
+            rng,
+            ack_action_tx,
+            real_message_sender,
+            topology_access.clone(),
+            reply_storage.key_storage(),
+            reply_storage.tags_storage(),
         );
 
         let ack_control = AcknowledgementController::new(
             ack_control_config,
-            rng,
-            topology_access.clone(),
             Arc::clone(&config.ack_key),
-            config.self_recipient,
-            reply_key_storage,
             ack_controller_connectors,
+            message_handler.clone(),
+            reply_controller_sender,
         );
 
-        let out_queue_config = real_traffic_stream::Config::new(
-            config.average_ack_delay_duration,
-            config.average_packet_delay_duration,
-            config.average_message_sending_delay,
+        let reply_control = ReplyController::new(
+            reply_controller_config,
+            message_handler,
+            reply_storage,
+            reply_controller_receiver,
         );
 
         let out_queue_control = OutQueueControl::new(
             out_queue_config,
-            Arc::clone(&config.ack_key),
+            rng,
             sent_notifier_tx,
             mix_sender,
             real_message_receiver,
-            rng,
-            config.self_recipient,
             topology_access,
+            lane_queue_lengths,
+            client_connection_rx,
         );
 
         RealMessagesController {
-            out_queue_control: Some(out_queue_control),
-            ack_control: Some(ack_control),
+            out_queue_control,
+            ack_control,
+            reply_control,
         }
     }
 
-    pub(super) async fn run(&mut self) {
-        let mut out_queue_control = self.out_queue_control.take().unwrap();
-        let mut ack_control = self.ack_control.take().unwrap();
+    pub fn start_with_shutdown(self, shutdown: task::TaskClient) {
+        let mut out_queue_control = self.out_queue_control;
+        let ack_control = self.ack_control;
+        let mut reply_control = self.reply_control;
 
-        // the below are log messages are errors as at the current stage we do not expect any of
-        // the task to ever finish. This will of course change once we introduce
-        // graceful shutdowns.
-        let out_queue_control_fut = tokio::spawn(async move {
-            out_queue_control.run_out_queue_control().await;
-            error!("The out queue controller has finished execution!");
-            out_queue_control
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            out_queue_control.run_with_shutdown(shutdown_handle).await;
+            debug!("The out queue controller has finished execution!");
         });
-        let ack_control_fut = tokio::spawn(async move {
-            ack_control.run().await;
-            error!("The acknowledgement controller has finished execution!");
-            ack_control
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            reply_control.run_with_shutdown(shutdown_handle).await;
+            debug!("The reply controller has finished execution!");
         });
 
-        // technically we don't have to bring `RealMessagesController` back to a valid state
-        // but we can do it, so why not? Perhaps it might be useful if we wanted to allow
-        // for restarts of certain modules without killing the entire process.
-        self.out_queue_control = Some(out_queue_control_fut.await.unwrap());
-        self.ack_control = Some(ack_control_fut.await.unwrap());
-    }
-
-    pub fn start(mut self) -> JoinHandle<Self> {
-        tokio::spawn(async move {
-            self.run().await;
-            self
-        })
+        ack_control.start_with_shutdown(shutdown);
     }
 }

clients/client-core/src/client/real_messages_control/real_traffic_stream.rs

@@ -1,10 +1,14 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+use self::sending_delay_controller::SendingDelayController;
 use crate::client::mix_traffic::BatchMixMessageSender;
 use crate::client::real_messages_control::acknowledgement_control::SentPacketNotificationSender;
 use crate::client::topology_control::TopologyAccessor;
-use futures::channel::mpsc;
+use crate::client::transmission_buffer::TransmissionBuffer;
+use client_connections::{
+    ConnectionCommand, ConnectionCommandReceiver, ConnectionId, LaneQueueLengths, TransmissionLane,
+};
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
 use log::*;
@@ -13,16 +17,30 @@ use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::chunking::fragment::FragmentIdentifier;
 use nymsphinx::cover::generate_loop_cover_packet;
 use nymsphinx::forwarding::packet::MixPacket;
+use nymsphinx::params::PacketSize;
+use nymsphinx::preparer::PreparedFragment;
 use nymsphinx::utils::sample_poisson_duration;
 use rand::{CryptoRng, Rng};
-use std::collections::VecDeque;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
+
+#[cfg(not(target_arch = "wasm32"))]
 use tokio::time;
 
+#[cfg(target_arch = "wasm32")]
+use wasm_timer;
+
+mod sending_delay_controller;
+
 /// Configurable parameters of the `OutQueueControl`
 pub(crate) struct Config {
+    /// Key used to encrypt and decrypt content of an ACK packet.
+    ack_key: Arc<AckKey>,
+
+    /// Represents full address of this client.
+    our_full_destination: Recipient,
+
     /// Average delay an acknowledgement packet is going to get delay at a single mixnode.
     average_ack_delay: Duration,
 
@@ -31,20 +49,39 @@ pub(crate) struct Config {
 
     /// Average delay between sending subsequent packets.
     average_message_sending_delay: Duration,
+
+    /// Controls whether the stream constantly produces packets according to the predefined
+    /// poisson distribution.
+    disable_poisson_packet_distribution: bool,
+
+    /// Predefined packet size used for the loop cover messages.
+    cover_packet_size: PacketSize,
 }
 
 impl Config {
     pub(crate) fn new(
+        ack_key: Arc<AckKey>,
+        our_full_destination: Recipient,
         average_ack_delay: Duration,
         average_packet_delay: Duration,
         average_message_sending_delay: Duration,
+        disable_poisson_packet_distribution: bool,
     ) -> Self {
         Config {
+            ack_key,
+            our_full_destination,
             average_ack_delay,
             average_packet_delay,
             average_message_sending_delay,
+            disable_poisson_packet_distribution,
+            cover_packet_size: Default::default(),
         }
     }
+
+    pub fn with_custom_cover_packet_size(mut self, packet_size: PacketSize) -> Self {
+        self.cover_packet_size = packet_size;
+        self
+    }
 }
 
 pub(crate) struct OutQueueControl<R>
@@ -54,15 +91,20 @@ where
     /// Configurable parameters of the `ActionController`
     config: Config,
 
-    /// Key used to encrypt and decrypt content of an ACK packet.
-    ack_key: Arc<AckKey>,
-
     /// Channel used for notifying of a real packet being sent out. Used to start up retransmission timer.
     sent_notifier: SentPacketNotificationSender,
 
     /// Internal state, determined by `average_message_sending_delay`,
     /// used to keep track of when a next packet should be sent out.
-    next_delay: Pin<Box<time::Sleep>>,
+    #[cfg(not(target_arch = "wasm32"))]
+    next_delay: Option<Pin<Box<time::Sleep>>>,
+
+    #[cfg(target_arch = "wasm32")]
+    next_delay: Option<Pin<Box<wasm_timer::Delay>>>,
+
+    // To make sure we don't overload the mix_tx channel, we limit the rate we are pushing
+    // messages.
+    sending_delay_controller: SendingDelayController,
 
     /// Channel used for sending prepared sphinx packets to `MixTrafficController` that sends them
     /// out to the network without any further delays.
@@ -72,25 +114,45 @@ where
     /// before being sent out into the network.
     real_receiver: BatchRealMessageReceiver,
 
-    /// Represents full address of this client.
-    our_full_destination: Recipient,
-
     /// Instance of a cryptographically secure random number generator.
     rng: R,
 
     /// Accessor to the common instance of network topology.
     topology_access: TopologyAccessor,
 
-    /// Buffer containing all real messages received. It is first exhausted before more are pulled.
-    received_buffer: VecDeque<RealMessage>,
+    /// Buffer containing all incoming real messages keyed by transmission lane, that we will send
+    /// out to the mixnet.
+    transmission_buffer: TransmissionBuffer<RealMessage>,
+
+    /// Incoming channel for being notified of closed connections, so that we can close lanes
+    /// corresponding to connections. To avoid sending traffic unnecessary
+    client_connection_rx: ConnectionCommandReceiver,
+
+    /// Report queue lengths so that upstream can backoff sending data, and keep connections open.
+    lane_queue_lengths: LaneQueueLengths,
 }
 
+#[derive(Debug)]
 pub(crate) struct RealMessage {
     mix_packet: MixPacket,
     fragment_id: FragmentIdentifier,
+    // TODO: add info about it being constructed with reply-surb
+}
+
+impl From<PreparedFragment> for RealMessage {
+    fn from(fragment: PreparedFragment) -> Self {
+        RealMessage {
+            mix_packet: fragment.mix_packet,
+            fragment_id: fragment.fragment_identifier,
+        }
+    }
 }
 
 impl RealMessage {
+    pub(crate) fn packet_size(&self) -> usize {
+        self.mix_packet.sphinx_packet().len()
+    }
+
     pub(crate) fn new(mix_packet: MixPacket, fragment_id: FragmentIdentifier) -> Self {
         RealMessage {
             mix_packet,
@@ -101,63 +163,15 @@ impl RealMessage {
 
 // messages are already prepared, etc. the real point of it is to forward it to mix_traffic
 // after sufficient delay
-pub(crate) type BatchRealMessageSender = mpsc::UnboundedSender<Vec<RealMessage>>;
-type BatchRealMessageReceiver = mpsc::UnboundedReceiver<Vec<RealMessage>>;
+pub(crate) type BatchRealMessageSender =
+    tokio::sync::mpsc::Sender<(Vec<RealMessage>, TransmissionLane)>;
+type BatchRealMessageReceiver = tokio::sync::mpsc::Receiver<(Vec<RealMessage>, TransmissionLane)>;
 
 pub(crate) enum StreamMessage {
     Cover,
     Real(Box<RealMessage>),
 }
 
-impl<R> Stream for OutQueueControl<R>
-where
-    R: CryptoRng + Rng + Unpin,
-{
-    type Item = StreamMessage;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        // it is not yet time to return a message
-        if self.next_delay.as_mut().poll(cx).is_pending() {
-            return Poll::Pending;
-        };
-
-        // we know it's time to send a message, so let's prepare delay for the next one
-        // Get the `now` by looking at the current `delay` deadline
-        let avg_delay = self.config.average_message_sending_delay;
-        let now = self.next_delay.deadline();
-        let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
-
-        // The next interval value is `next_poisson_delay` after the one that just
-        // yielded.
-        let next = now + next_poisson_delay;
-        self.next_delay.as_mut().reset(next);
-
-        // check if we have anything immediately available
-        if let Some(real_available) = self.received_buffer.pop_front() {
-            return Poll::Ready(Some(StreamMessage::Real(Box::new(real_available))));
-        }
-
-        // decide what kind of message to send
-        match Pin::new(&mut self.real_receiver).poll_next(cx) {
-            // in the case our real message channel stream was closed, we should also indicate we are closed
-            // (and whoever is using the stream should panic)
-            Poll::Ready(None) => Poll::Ready(None),
-
-            // if there are more messages available, return first one and store the rest
-            Poll::Ready(Some(real_messages)) => {
-                self.received_buffer = real_messages.into();
-                // we MUST HAVE received at least ONE message
-                Poll::Ready(Some(StreamMessage::Real(Box::new(
-                    self.received_buffer.pop_front().unwrap(),
-                ))))
-            }
-
-            // otherwise construct a dummy one
-            Poll::Pending => Poll::Ready(Some(StreamMessage::Cover)),
-        }
-    }
-}
-
 impl<R> OutQueueControl<R>
 where
     R: CryptoRng + Rng + Unpin,
@@ -167,25 +181,26 @@ where
     #[allow(clippy::too_many_arguments)]
     pub(crate) fn new(
         config: Config,
-        ack_key: Arc<AckKey>,
+        rng: R,
         sent_notifier: SentPacketNotificationSender,
         mix_tx: BatchMixMessageSender,
         real_receiver: BatchRealMessageReceiver,
-        rng: R,
-        our_full_destination: Recipient,
         topology_access: TopologyAccessor,
+        lane_queue_lengths: LaneQueueLengths,
+        client_connection_rx: ConnectionCommandReceiver,
     ) -> Self {
         OutQueueControl {
             config,
-            ack_key,
             sent_notifier,
-            next_delay: Box::pin(time::sleep(Default::default())),
+            next_delay: None,
+            sending_delay_controller: Default::default(),
             mix_tx,
             real_receiver,
-            our_full_destination,
             rng,
             topology_access,
-            received_buffer: VecDeque::with_capacity(0), // we won't be putting any data into this guy directly
+            transmission_buffer: TransmissionBuffer::new(),
+            client_connection_rx,
+            lane_queue_lengths,
         }
     }
 
@@ -200,70 +215,353 @@ where
     async fn on_message(&mut self, next_message: StreamMessage) {
         trace!("created new message");
 
-        let next_message = match next_message {
+        let (next_message, fragment_id) = match next_message {
             StreamMessage::Cover => {
                 // TODO for way down the line: in very rare cases (during topology update) we might have
                 // to wait a really tiny bit before actually obtaining the permit hence messing with our
                 // poisson delay, but is it really a problem?
                 let topology_permit = self.topology_access.get_read_permit().await;
                 // the ack is sent back to ourselves (and then ignored)
-                let topology_ref_option = topology_permit.try_get_valid_topology_ref(
-                    &self.our_full_destination,
-                    Some(&self.our_full_destination),
-                );
-                if topology_ref_option.is_none() {
-                    warn!(
-                        "No valid topology detected - won't send any loop cover message this time"
-                    );
-                    return;
-                }
-                let topology_ref = topology_ref_option.unwrap();
+                let topology_ref = match topology_permit.try_get_valid_topology_ref(
+                    &self.config.our_full_destination,
+                    Some(&self.config.our_full_destination),
+                ) {
+                    Ok(topology) => topology,
+                    Err(err) => {
+                        warn!("We're not going to send any loop cover message this time, as the current topology seem to be invalid - {err}");
+                        return;
+                    }
+                };
 
-                generate_loop_cover_packet(
-                    &mut self.rng,
-                    topology_ref,
-                    &*self.ack_key,
-                    &self.our_full_destination,
-                    self.config.average_ack_delay,
-                    self.config.average_packet_delay,
+                (
+                    generate_loop_cover_packet(
+                        &mut self.rng,
+                        topology_ref,
+                        &self.config.ack_key,
+                        &self.config.our_full_destination,
+                        self.config.average_ack_delay,
+                        self.config.average_packet_delay,
+                        self.config.cover_packet_size,
+                    )
+                    .expect(
+                        "Somehow failed to generate a loop cover message with a valid topology",
+                    ),
+                    None,
                 )
-                .expect("Somehow failed to generate a loop cover message with a valid topology")
             }
             StreamMessage::Real(real_message) => {
-                self.sent_notify(real_message.fragment_id);
-                real_message.mix_packet
+                (real_message.mix_packet, Some(real_message.fragment_id))
             }
         };
 
-        // if this one fails, there's no retrying because it means that either:
-        // - we run out of memory
-        // - the receiver channel is closed
-        // in either case there's no recovery and we can only panic
-        self.mix_tx.unbounded_send(vec![next_message]).unwrap();
+        if let Err(err) = self.mix_tx.send(vec![next_message]).await {
+            log::error!("Failed to send: {err}");
+        }
+
+        // notify ack controller about sending our message only after we actually managed to push it
+        // through the channel
+        if let Some(fragment_id) = fragment_id {
+            self.sent_notify(fragment_id);
+        }
+
+        // In addition to closing connections on receiving messages throught client_connection_rx,
+        // also close connections when sufficiently stale.
+        self.transmission_buffer.prune_stale_connections();
 
         // JS: Not entirely sure why or how it fixes stuff, but without the yield call,
         // the UnboundedReceiver [of mix_rx] will not get a chance to read anything
         // JS2: Basically it was the case that with high enough rate, the stream had already a next value
         // ready and hence was immediately re-scheduled causing other tasks to be starved;
         // yield makes it go back the scheduling queue regardless of its value availability
+
+        // TODO: temporary and BAD workaround for wasm (we should find a way to yield here in wasm)
+        #[cfg(not(target_arch = "wasm32"))]
         tokio::task::yield_now().await;
     }
 
-    // Send messages at certain rate and if no real traffic is available, send cover message.
-    async fn run_normal_out_queue(&mut self) {
-        // we should set initial delay only when we actually start the stream
-        self.next_delay = Box::pin(time::sleep(sample_poisson_duration(
-            &mut self.rng,
-            self.config.average_message_sending_delay,
-        )));
+    fn on_close_connection(&mut self, connection_id: ConnectionId) {
+        log::debug!("Removing lane for connection: {connection_id}");
+        self.transmission_buffer
+            .remove(&TransmissionLane::ConnectionId(connection_id));
+    }
 
-        while let Some(next_message) = self.next().await {
-            self.on_message(next_message).await;
+    fn current_average_message_sending_delay(&self) -> Duration {
+        self.config.average_message_sending_delay
+            * self.sending_delay_controller.current_multiplier()
+    }
+
+    fn adjust_current_average_message_sending_delay(&mut self) {
+        let used_slots = self.mix_tx.max_capacity() - self.mix_tx.capacity();
+        log::trace!(
+            "used_slots: {used_slots}, current_multiplier: {}",
+            self.sending_delay_controller.current_multiplier()
+        );
+
+        // Even just a single used slot is enough to signal backpressure
+        if used_slots > 0 {
+            log::trace!("Backpressure detected");
+            self.sending_delay_controller.record_backpressure_detected();
+        }
+
+        // If the buffer is running out, slow down the sending rate
+        if self.mix_tx.capacity() == 0
+            && self.sending_delay_controller.not_increased_delay_recently()
+        {
+            self.sending_delay_controller.increase_delay_multiplier();
+        }
+
+        // Very carefully step up the sending rate in case it seems like we can solidly handle the
+        // current rate.
+        if self.sending_delay_controller.is_sending_reliable() {
+            self.sending_delay_controller.decrease_delay_multiplier();
         }
     }
 
-    pub(crate) async fn run_out_queue_control(&mut self) {
-        debug!("Starting out queue controller...");
-        self.run_normal_out_queue().await
+    fn pop_next_message(&mut self) -> Option<RealMessage> {
+        // Pop the next message from the transmission buffer
+        let (lane, real_next) = self
+            .transmission_buffer
+            .pop_next_message_at_random(&mut self.rng)?;
+
+        // Update the published queue length
+        let lane_length = self.transmission_buffer.lane_length(&lane);
+        self.lane_queue_lengths.set(&lane, lane_length);
+
+        Some(real_next)
+    }
+
+    fn poll_poisson(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
+        // The average delay could change depending on if backpressure in the downstream channel
+        // (mix_tx) was detected.
+        self.adjust_current_average_message_sending_delay();
+        let avg_delay = self.current_average_message_sending_delay();
+
+        // Start by checking if we have any incoming messages about closed connections
+        // NOTE: this feels a bit iffy, the `OutQueueControl` is getting ripe for a rewrite to
+        // something simpler.
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.client_connection_rx).poll_next(cx) {
+            match id {
+                ConnectionCommand::Close(id) => self.on_close_connection(id),
+                ConnectionCommand::ActiveConnections(_) => panic!(),
+            }
+        }
+
+        if let Some(ref mut next_delay) = &mut self.next_delay {
+            // it is not yet time to return a message
+            if next_delay.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
+            };
+
+            // we know it's time to send a message, so let's prepare delay for the next one
+            // Get the `now` by looking at the current `delay` deadline
+            let next_poisson_delay = sample_poisson_duration(&mut self.rng, avg_delay);
+
+            // The next interval value is `next_poisson_delay` after the one that just
+            // yielded.
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                let now = next_delay.deadline();
+                let next = now + next_poisson_delay;
+                next_delay.as_mut().reset(next);
+            }
+
+            #[cfg(target_arch = "wasm32")]
+            {
+                next_delay.as_mut().reset(next_poisson_delay);
+            }
+
+            // On every iteration we get new messages from upstream. Given that these come bunched
+            // in `Vec`, this ensures that on average we will fetch messages faster than we can
+            // send, which is a condition for being able to multiplex sphinx packets from multiple
+            // data streams.
+            match Pin::new(&mut self.real_receiver).poll_recv(cx) {
+                // in the case our real message channel stream was closed, we should also indicate we are closed
+                // (and whoever is using the stream should panic)
+                Poll::Ready(None) => Poll::Ready(None),
+
+                Poll::Ready(Some((real_messages, conn_id))) => {
+                    log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                    self.transmission_buffer.store(&conn_id, real_messages);
+                    let real_next = self.pop_next_message().expect("Just stored one");
+
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                }
+
+                Poll::Pending => {
+                    if let Some(real_next) = self.pop_next_message() {
+                        Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                    } else {
+                        // otherwise construct a dummy one
+                        Poll::Ready(Some(StreamMessage::Cover))
+                    }
+                }
+            }
+        } else {
+            // we never set an initial delay - let's do it now
+            cx.waker().wake_by_ref();
+
+            let sampled =
+                sample_poisson_duration(&mut self.rng, self.config.average_message_sending_delay);
+
+            #[cfg(not(target_arch = "wasm32"))]
+            let next_delay = Box::pin(time::sleep(sampled));
+
+            #[cfg(target_arch = "wasm32")]
+            let next_delay = Box::pin(wasm_timer::Delay::new(sampled));
+
+            self.next_delay = Some(next_delay);
+
+            Poll::Pending
+        }
+    }
+
+    fn poll_immediate(&mut self, cx: &mut Context<'_>) -> Poll<Option<StreamMessage>> {
+        // Start by checking if we have any incoming messages about closed connections
+        if let Poll::Ready(Some(id)) = Pin::new(&mut self.client_connection_rx).poll_next(cx) {
+            match id {
+                ConnectionCommand::Close(id) => self.on_close_connection(id),
+                ConnectionCommand::ActiveConnections(_) => panic!(),
+            }
+        }
+
+        match Pin::new(&mut self.real_receiver).poll_recv(cx) {
+            // in the case our real message channel stream was closed, we should also indicate we are closed
+            // (and whoever is using the stream should panic)
+            Poll::Ready(None) => Poll::Ready(None),
+
+            Poll::Ready(Some((real_messages, conn_id))) => {
+                log::trace!("handling real_messages: size: {}", real_messages.len());
+
+                // First store what we got for the given connection id
+                self.transmission_buffer.store(&conn_id, real_messages);
+                let real_next = self.pop_next_message().expect("we just added one");
+
+                Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+            }
+
+            Poll::Pending => {
+                if let Some(real_next) = self.pop_next_message() {
+                    Poll::Ready(Some(StreamMessage::Real(Box::new(real_next))))
+                } else {
+                    Poll::Pending
+                }
+            }
+        }
+    }
+
+    fn poll_next_message(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<StreamMessage>> {
+        if self.config.disable_poisson_packet_distribution {
+            self.poll_immediate(cx)
+        } else {
+            self.poll_poisson(cx)
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn log_status(&self, shutdown: &mut task::TaskClient) {
+        use crate::error::ClientCoreStatusMessage;
+
+        let packets = self.transmission_buffer.total_size();
+        let backlog = self.transmission_buffer.total_size_in_bytes() as f64 / 1024.0;
+        let lanes = self.transmission_buffer.num_lanes();
+        let mult = self.sending_delay_controller.current_multiplier();
+        let delay = self.current_average_message_sending_delay().as_millis();
+        let status_str = if self.config.disable_poisson_packet_distribution {
+            format!("Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), no delay")
+        } else {
+            format!(
+                "Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), avg delay: {delay}ms ({mult})"
+            )
+        };
+        if packets > 1000 {
+            log::warn!("{status_str}");
+        } else if packets > 0 {
+            log::info!("{status_str}");
+        } else {
+            log::debug!("{status_str}");
+        }
+
+        // Send status message to whoever is listening (possibly UI)
+        if mult == self.sending_delay_controller.max_multiplier() {
+            shutdown.send_status_msg(Box::new(ClientCoreStatusMessage::GatewayIsVerySlow));
+        } else if mult > self.sending_delay_controller.min_multiplier() {
+            shutdown.send_status_msg(Box::new(ClientCoreStatusMessage::GatewayIsSlow));
+        }
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    fn log_status_infrequent(&self) {
+        if self.sending_delay_controller.current_multiplier() > 1 {
+            log::warn!(
+                "Unable to send packets at the default rate - rate reduced by setting the delay multiplier set to: {}",
+                self.sending_delay_controller.current_multiplier()
+            );
+        }
+    }
+
+    pub(super) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started OutQueueControl with graceful shutdown support");
+
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let mut status_timer = tokio::time::interval(Duration::from_secs(5));
+            let mut infrequent_status_timer = tokio::time::interval(Duration::from_secs(60));
+
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv_with_delay() => {
+                        log::trace!("OutQueueControl: Received shutdown");
+                    }
+                    _ = status_timer.tick() => {
+                        self.log_status(&mut shutdown);
+                    }
+                    _ = infrequent_status_timer.tick() => {
+                        self.log_status_infrequent();
+                    }
+                    next_message = self.next() => if let Some(next_message) = next_message {
+                        self.on_message(next_message).await;
+                    } else {
+                        log::trace!("OutQueueControl: Stopping since channel closed");
+                        break;
+                    }
+                }
+            }
+            shutdown.recv_timeout().await;
+        }
+
+        #[cfg(target_arch = "wasm32")]
+        {
+            while !shutdown.is_shutdown() {
+                tokio::select! {
+                    biased;
+                    _ = shutdown.recv() => {
+                        log::trace!("OutQueueControl: Received shutdown");
+                    }
+                    next_message = self.next() => if let Some(next_message) = next_message {
+                        self.on_message(next_message).await;
+                    } else {
+                        log::trace!("OutQueueControl: Stopping since channel closed");
+                        break;
+                    }
+                }
+            }
+        }
+        log::debug!("OutQueueControl: Exiting");
+    }
+}
+
+impl<R> Stream for OutQueueControl<R>
+where
+    R: CryptoRng + Rng + Unpin,
+{
+    type Item = StreamMessage;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.poll_next_message(cx)
     }
 }

clients/client-core/src/client/real_messages_control/real_traffic_stream/sending_delay_controller.rs

@@ -0,0 +1,121 @@
+// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::helpers::{get_time_now, Instant};
+use std::time::Duration;
+
+// The minimum time between increasing the average delay between packets. If we hit the ceiling in
+// the available buffer space we want to take somewhat swift action, but we still need to give a
+// short time to give the channel a chance reduce pressure.
+const INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 1;
+// The minimum time between decreasing the average delay between packets. We don't want to change
+// to quickly to keep things somewhat stable. Also there are buffers downstreams meaning we need to
+// wait a little to see the effect before we decrease further.
+const DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS: u64 = 30;
+// If we enough time passes without any sign of backpressure in the channel, we can consider
+// lowering the average delay. The goal is to keep somewhat stable, rather than maxing out
+// bandwidth at all times.
+const ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS: u64 = 30;
+// The maximum multiplier we apply to the base average Poisson delay.
+const MAX_DELAY_MULTIPLIER: u32 = 6;
+// The minium multiplier we apply to the base average Poisson delay.
+const MIN_DELAY_MULTIPLIER: u32 = 1;
+
+pub(crate) struct SendingDelayController {
+    /// Multiply the average sending delay.
+    /// This is normally set to unity, but if we detect backpressure we increase this
+    /// multiplier. We use discrete steps.
+    current_multiplier: u32,
+
+    /// Maximum delay multiplier
+    upper_bound: u32,
+
+    /// Minimum delay multiplier
+    lower_bound: u32,
+
+    /// To make sure we don't change the multiplier to fast, we limit a change to some duration
+    time_when_changed: Instant,
+
+    /// If we have a long enough time without any backpressure detected we try reducing the sending
+    /// delay multiplier
+    time_when_backpressure_detected: Instant,
+}
+
+impl Default for SendingDelayController {
+    fn default() -> Self {
+        SendingDelayController::new(MIN_DELAY_MULTIPLIER, MAX_DELAY_MULTIPLIER)
+    }
+}
+
+impl SendingDelayController {
+    pub(crate) fn new(lower_bound: u32, upper_bound: u32) -> Self {
+        assert!(lower_bound <= upper_bound);
+        let now = get_time_now();
+        SendingDelayController {
+            current_multiplier: MIN_DELAY_MULTIPLIER,
+            upper_bound,
+            lower_bound,
+            time_when_changed: now,
+            time_when_backpressure_detected: now,
+        }
+    }
+
+    pub(crate) fn current_multiplier(&self) -> u32 {
+        self.current_multiplier
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn min_multiplier(&self) -> u32 {
+        self.lower_bound
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn max_multiplier(&self) -> u32 {
+        self.upper_bound
+    }
+
+    pub(crate) fn increase_delay_multiplier(&mut self) {
+        if self.current_multiplier < self.upper_bound {
+            self.current_multiplier =
+                (self.current_multiplier + 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::warn!(
+                "Increasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        } else {
+            log::warn!("Trying to increase delay multipler higher than allowed");
+        }
+    }
+
+    pub(crate) fn decrease_delay_multiplier(&mut self) {
+        if self.current_multiplier > self.lower_bound {
+            self.current_multiplier =
+                (self.current_multiplier - 1).clamp(self.lower_bound, self.upper_bound);
+            self.time_when_changed = get_time_now();
+            log::debug!(
+                "Decreasing sending delay multiplier to: {}",
+                self.current_multiplier
+            );
+        }
+    }
+
+    pub(crate) fn record_backpressure_detected(&mut self) {
+        self.time_when_backpressure_detected = get_time_now();
+    }
+
+    pub(crate) fn not_increased_delay_recently(&self) -> bool {
+        get_time_now()
+            > self.time_when_changed + Duration::from_secs(INCREASE_DELAY_MIN_CHANGE_INTERVAL_SECS)
+    }
+
+    pub(crate) fn is_sending_reliable(&self) -> bool {
+        let now = get_time_now();
+        let delay_change_interval = Duration::from_secs(DECREASE_DELAY_MIN_CHANGE_INTERVAL_SECS);
+        let acceptable_time_without_backpressure =
+            Duration::from_secs(ACCEPTABLE_TIME_WITHOUT_BACKPRESSURE_SECS);
+
+        now > self.time_when_backpressure_detected + acceptable_time_without_backpressure
+            && now > self.time_when_changed + delay_change_interval
+    }
+}

clients/client-core/src/client/received_buffer.rs

@@ -1,21 +1,25 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::client::reply_key_storage::ReplyKeyStorage;
+use crate::client::replies::reply_controller::ReplyControllerSender;
+use crate::client::replies::reply_storage::SentReplyKeys;
+use crate::spawn_future;
 use crypto::asymmetric::encryption;
-use crypto::symmetric::stream_cipher;
 use crypto::Digest;
 use futures::channel::mpsc;
 use futures::lock::Mutex;
 use futures::StreamExt;
 use gateway_client::MixnetMessageReceiver;
 use log::*;
+use nymsphinx::anonymous_replies::requests::{
+    RepliableMessage, RepliableMessageContent, ReplyMessage, ReplyMessageContent,
+};
 use nymsphinx::anonymous_replies::{encryption_key::EncryptionKeyDigest, SurbEncryptionKey};
-use nymsphinx::params::{ReplySurbEncryptionAlgorithm, ReplySurbKeyDigestAlgorithm};
+use nymsphinx::message::{NymMessage, PlainMessage};
+use nymsphinx::params::ReplySurbKeyDigestAlgorithm;
 use nymsphinx::receiver::{MessageReceiver, MessageRecoveryError, ReconstructedMessage};
 use std::collections::HashSet;
 use std::sync::Arc;
-use tokio::task::JoinHandle;
 
 // Buffer Requests to say "hey, send any reconstructed messages to this channel"
 // or to say "hey, I'm going offline, don't send anything more to me. Just buffer them instead"
@@ -42,26 +46,15 @@ struct ReceivedMessagesBufferInner {
 }
 
 impl ReceivedMessagesBufferInner {
-    fn process_received_fragment(&mut self, raw_fragment: Vec<u8>) -> Option<ReconstructedMessage> {
-        let fragment_data = match self
-            .message_receiver
-            .recover_plaintext(self.local_encryption_keypair.private_key(), raw_fragment)
-        {
-            Err(e) => {
-                warn!("failed to recover fragment data: {:?}. The whole underlying message might be corrupted and unrecoverable!", e);
-                return None;
-            }
-            Ok(frag_data) => frag_data,
-        };
-
-        if nymsphinx::cover::is_cover(&fragment_data) {
+    fn recover_from_fragment(&mut self, fragment_data: &[u8]) -> Option<NymMessage> {
+        if nymsphinx::cover::is_cover(fragment_data) {
             trace!("The message was a loop cover message! Skipping it");
             return None;
         }
 
-        let fragment = match self.message_receiver.recover_fragment(&fragment_data) {
-            Err(e) => {
-                warn!("failed to recover fragment from raw data: {:?}. The whole underlying message might be corrupted and unrecoverable!", e);
+        let fragment = match self.message_receiver.recover_fragment(fragment_data) {
+            Err(err) => {
+                warn!("failed to recover fragment from raw data: {err}. The whole underlying message might be corrupted and unrecoverable!");
                 return None;
             }
             Ok(frag) => frag,
@@ -75,9 +68,10 @@ impl ReceivedMessagesBufferInner {
         // if we returned an error the underlying message is malformed in some way
         match self.message_receiver.insert_new_fragment(fragment) {
             Err(err) => match err {
-                MessageRecoveryError::MalformedReconstructedMessage(message_sets) => {
+                MessageRecoveryError::MalformedReconstructedMessage { source, used_sets } => {
+                    error!("message reconstruction failed - {source}. Attempting to re-use the message sets...");
                     // TODO: should we really insert reconstructed sets? could this be abused for some attack?
-                    for set_id in message_sets {
+                    for set_id in used_sets {
                         if !self.recently_reconstructed.insert(set_id) {
                             // or perhaps we should even panic at this point?
                             error!("Reconstructed another message containing already used set id!")
@@ -103,6 +97,34 @@ impl ReceivedMessagesBufferInner {
             },
         }
     }
+
+    fn process_received_reply(
+        &mut self,
+        reply_ciphertext: &mut [u8],
+        reply_key: SurbEncryptionKey,
+    ) -> Option<NymMessage> {
+        // note: this performs decryption IN PLACE without extra allocation
+        self.message_receiver
+            .recover_plaintext_from_reply(reply_ciphertext, reply_key);
+        let fragment_data = reply_ciphertext;
+
+        self.recover_from_fragment(fragment_data)
+    }
+
+    fn process_received_regular_packet(&mut self, mut raw_fragment: Vec<u8>) -> Option<NymMessage> {
+        let fragment_data = match self.message_receiver.recover_plaintext_from_regular_packet(
+            self.local_encryption_keypair.private_key(),
+            &mut raw_fragment,
+        ) {
+            Err(err) => {
+                warn!("failed to recover fragment data: {err}. The whole underlying message might be corrupted and unrecoverable!");
+                return None;
+            }
+            Ok(frag_data) => frag_data,
+        };
+
+        self.recover_from_fragment(fragment_data)
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -110,16 +132,15 @@ impl ReceivedMessagesBufferInner {
 // You should always use .clone() to create additional instances
 struct ReceivedMessagesBuffer {
     inner: Arc<Mutex<ReceivedMessagesBufferInner>>,
-
-    /// Storage containing keys to all [`ReplySURB`]s ever sent out that we did not receive back.
-    // There's no need to put it behind a Mutex since it's already properly concurrent
-    reply_key_storage: ReplyKeyStorage,
+    reply_key_storage: SentReplyKeys,
+    reply_controller_sender: ReplyControllerSender,
 }
 
 impl ReceivedMessagesBuffer {
     fn new(
         local_encryption_keypair: Arc<encryption::KeyPair>,
-        reply_key_storage: ReplyKeyStorage,
+        reply_key_storage: SentReplyKeys,
+        reply_controller_sender: ReplyControllerSender,
     ) -> Self {
         ReceivedMessagesBuffer {
             inner: Arc::new(Mutex::new(ReceivedMessagesBufferInner {
@@ -130,6 +151,7 @@ impl ReceivedMessagesBuffer {
                 recently_reconstructed: HashSet::new(),
             })),
             reply_key_storage,
+            reply_controller_sender,
         }
     }
 
@@ -171,37 +193,143 @@ impl ReceivedMessagesBuffer {
         guard.message_sender = Some(sender);
     }
 
-    async fn add_reconstructed_messages(&mut self, msgs: Vec<ReconstructedMessage>) {
-        debug!("Adding {:?} new messages to the buffer!", msgs.len());
-        trace!("Adding new messages to the buffer! {:?}", msgs);
-        self.inner.lock().await.messages.extend(msgs)
+    fn handle_reconstructed_plain_messages(
+        &mut self,
+        msgs: Vec<PlainMessage>,
+    ) -> Vec<ReconstructedMessage> {
+        msgs.into_iter().map(Into::into).collect()
     }
 
-    fn process_received_reply(
-        reply_ciphertext: &[u8],
-        reply_key: SurbEncryptionKey,
-    ) -> Option<ReconstructedMessage> {
-        let zero_iv = stream_cipher::zero_iv::<ReplySurbEncryptionAlgorithm>();
+    fn handle_reconstructed_repliable_messages(
+        &mut self,
+        msgs: Vec<RepliableMessage>,
+    ) -> Vec<ReconstructedMessage> {
+        let mut reconstructed = Vec::new();
+        for msg in msgs {
+            let (reply_surbs, from_surb_request) = match msg.content {
+                RepliableMessageContent::Data {
+                    message,
+                    reply_surbs,
+                } => {
+                    trace!(
+                        "received message that also contained additional {} reply surbs from {:?}!",
+                        reply_surbs.len(),
+                        msg.sender_tag
+                    );
 
-        let mut reply_msg = stream_cipher::decrypt::<ReplySurbEncryptionAlgorithm>(
-            reply_key.inner(),
-            &zero_iv,
-            reply_ciphertext,
+                    reconstructed.push(ReconstructedMessage::new(message, msg.sender_tag));
+
+                    (reply_surbs, false)
+                }
+                RepliableMessageContent::AdditionalSurbs { reply_surbs } => {
+                    trace!(
+                        "received additional {} reply surbs from {:?}!",
+                        reply_surbs.len(),
+                        msg.sender_tag
+                    );
+                    (reply_surbs, true)
+                }
+                RepliableMessageContent::Heartbeat {
+                    additional_reply_surbs,
+                } => {
+                    error!("received a repliable heartbeat message - we don't know how to handle it yet (and we won't know until future PRs)");
+                    (additional_reply_surbs, false)
+                }
+            };
+
+            self.reply_controller_sender.send_additional_surbs(
+                msg.sender_tag,
+                reply_surbs,
+                from_surb_request,
+            )
+        }
+        reconstructed
+    }
+
+    fn handle_reconstructed_reply_messages(
+        &mut self,
+        msgs: Vec<ReplyMessage>,
+    ) -> Vec<ReconstructedMessage> {
+        let mut reconstructed = Vec::new();
+        for msg in msgs {
+            match msg.content {
+                ReplyMessageContent::Data { message } => reconstructed.push(message.into()),
+                ReplyMessageContent::SurbRequest { recipient, amount } => {
+                    debug!("received request for {amount} additional reply SURBs from {recipient}");
+                    self.reply_controller_sender
+                        .send_additional_surbs_request(*recipient, amount);
+                }
+            }
+        }
+        reconstructed
+    }
+
+    async fn handle_reconstructed_messages(&mut self, msgs: Vec<NymMessage>) {
+        if msgs.is_empty() {
+            return;
+        }
+
+        let mut plain_messages = Vec::new();
+        let mut repliable_messages = Vec::new();
+        let mut reply_messages = Vec::new();
+
+        for msg in msgs {
+            match msg {
+                NymMessage::Plain(plain) => plain_messages.push(plain),
+                NymMessage::Repliable(repliable) => repliable_messages.push(repliable),
+                NymMessage::Reply(reply) => reply_messages.push(reply),
+            }
+        }
+
+        let mut reconstructed_messages = self.handle_reconstructed_plain_messages(plain_messages);
+        reconstructed_messages
+            .append(&mut self.handle_reconstructed_repliable_messages(repliable_messages));
+        reconstructed_messages
+            .append(&mut self.handle_reconstructed_reply_messages(reply_messages));
+
+        let mut inner_guard = self.inner.lock().await;
+        debug!(
+            "Adding {:?} new messages to the buffer!",
+            reconstructed_messages.len()
         );
-        if let Err(err) = MessageReceiver::remove_padding(&mut reply_msg) {
-            warn!("Received reply had malformed padding! - {:?}", err);
-            None
+
+        if let Some(sender) = &inner_guard.message_sender {
+            trace!("Sending reconstructed messages to announced sender");
+            if let Err(err) = sender.unbounded_send(reconstructed_messages) {
+                warn!("The reconstructed message receiver went offline without explicit notification (relevant error: - {err})");
+                inner_guard.message_sender = None;
+                inner_guard.messages.extend(err.into_inner());
+            }
         } else {
-            // TODO: perhaps having to say it doesn't have a surb an indication the type should be changed?
-            Some(ReconstructedMessage {
-                message: reply_msg,
-                reply_surb: None,
-            })
+            trace!("No sender available - buffering reconstructed messages");
+            inner_guard.messages.extend(reconstructed_messages)
         }
     }
 
+    // this function doesn't really belong here...
+    fn get_reply_key<'a>(
+        &self,
+        raw_message: &'a mut [u8],
+    ) -> Option<(SurbEncryptionKey, &'a mut [u8])> {
+        let reply_surb_digest_size = ReplySurbKeyDigestAlgorithm::output_size();
+        if raw_message.len() < reply_surb_digest_size {
+            return None;
+        }
+
+        let possible_key_digest =
+            EncryptionKeyDigest::clone_from_slice(&raw_message[..reply_surb_digest_size]);
+        self.reply_key_storage
+            .try_pop(possible_key_digest)
+            .map(|reply_encryption_key| {
+                (
+                    *reply_encryption_key,
+                    &mut raw_message[reply_surb_digest_size..],
+                )
+            })
+    }
+
     async fn handle_new_received(&mut self, msgs: Vec<Vec<u8>>) {
-        debug!(
+        trace!(
             "Processing {:?} new message that might get added to the buffer!",
             msgs.len()
         );
@@ -209,58 +337,28 @@ impl ReceivedMessagesBuffer {
         let mut completed_messages = Vec::new();
         let mut inner_guard = self.inner.lock().await;
 
-        let reply_surb_digest_size = ReplySurbKeyDigestAlgorithm::output_size();
-
         // first check if this is a reply or a chunked message
-        // TODO: verify with @AP if this way of doing it is safe or whether it could
-        // cause some attacks due to, I don't know, stupid edge case collisions?
-        // Update: this DOES introduce a possible leakage: https://github.com/nymtech/nym/issues/296
-        for msg in msgs {
-            let possible_key_digest =
-                EncryptionKeyDigest::clone_from_slice(&msg[..reply_surb_digest_size]);
-
+        // note: there's a possible information leakage associated with this check https://github.com/nymtech/nym/issues/296
+        for mut msg in msgs {
             // check first `HasherOutputSize` bytes if they correspond to known encryption key
             // if yes - this is a reply message
+            let completed_message =
+                if let Some((reply_key, reply_message)) = self.get_reply_key(&mut msg) {
+                    inner_guard.process_received_reply(reply_message, reply_key)
+                } else {
+                    inner_guard.process_received_regular_packet(msg)
+                };
 
-            // TODO: this might be a bottleneck - since the keys are stored on disk we, presumably,
-            // are doing a disk operation every single received fragment
-            if let Some(reply_encryption_key) = self
-                .reply_key_storage
-                .get_and_remove_encryption_key(possible_key_digest)
-                .expect("storage operation failed!")
-            {
-                if let Some(completed_message) = Self::process_received_reply(
-                    &msg[reply_surb_digest_size..],
-                    reply_encryption_key,
-                ) {
-                    completed_messages.push(completed_message)
-                }
-            } else {
-                // otherwise - it's a 'normal' message
-                if let Some(completed_message) = inner_guard.process_received_fragment(msg) {
-                    completed_messages.push(completed_message)
-                }
+            if let Some(completed) = completed_message {
+                info!("received {completed}");
+                completed_messages.push(completed)
             }
         }
 
+        drop(inner_guard);
+
         if !completed_messages.is_empty() {
-            if let Some(sender) = &inner_guard.message_sender {
-                trace!("Sending reconstructed messages to announced sender");
-                if let Err(err) = sender.unbounded_send(completed_messages) {
-                    warn!("The reconstructed message receiver went offline without explicit notification (relevant error: - {:?})", err);
-                    // make sure to drop the lock to not deadlock
-                    // (it is required by `add_reconstructed_messages`)
-                    inner_guard.message_sender = None;
-                    drop(inner_guard);
-                    self.add_reconstructed_messages(err.into_inner()).await;
-                }
-            } else {
-                // make sure to drop the lock to not deadlock
-                // (it is required by `add_reconstructed_messages`)
-                drop(inner_guard);
-                trace!("No sender available - buffering reconstructed messages");
-                self.add_reconstructed_messages(completed_messages).await;
-            }
+            self.handle_reconstructed_messages(completed_messages).await
         }
     }
 }
@@ -290,19 +388,37 @@ impl RequestReceiver {
         }
     }
 
-    fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            while let Some(request) = self.query_receiver.next().await {
-                match request {
-                    ReceivedBufferMessage::ReceiverAnnounce(sender) => {
-                        self.received_buffer.connect_sender(sender).await;
-                    }
-                    ReceivedBufferMessage::ReceiverDisconnect => {
-                        self.received_buffer.disconnect_sender().await
-                    }
-                }
+    async fn handle_message(&mut self, message: ReceivedBufferMessage) {
+        match message {
+            ReceivedBufferMessage::ReceiverAnnounce(sender) => {
+                self.received_buffer.connect_sender(sender).await;
             }
-        })
+            ReceivedBufferMessage::ReceiverDisconnect => {
+                self.received_buffer.disconnect_sender().await
+            }
+        }
+    }
+
+    async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started RequestReceiver with graceful shutdown support");
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                biased;
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("RequestReceiver: Received shutdown");
+                }
+                request = self.query_receiver.next() => {
+                    if let Some(message) = request {
+                        self.handle_message(message).await
+                    } else {
+                        log::trace!("RequestReceiver: Stopping since channel closed");
+                        break;
+                    }
+                },
+            }
+        }
+        shutdown.recv_timeout().await;
+        log::debug!("RequestReceiver: Exiting");
     }
 }
 
@@ -321,29 +437,47 @@ impl FragmentedMessageReceiver {
             mixnet_packet_receiver,
         }
     }
-    fn start(mut self) -> JoinHandle<()> {
-        tokio::spawn(async move {
-            while let Some(new_messages) = self.mixnet_packet_receiver.next().await {
-                self.received_buffer.handle_new_received(new_messages).await;
+
+    async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started FragmentedMessageReceiver with graceful shutdown support");
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                new_messages = self.mixnet_packet_receiver.next() => {
+                    if let Some(new_messages) = new_messages {
+                        self.received_buffer.handle_new_received(new_messages).await;
+                    } else {
+                        log::trace!("FragmentedMessageReceiver: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("FragmentedMessageReceiver: Received shutdown");
+                }
             }
-        })
+        }
+        shutdown.recv_timeout().await;
+        log::debug!("FragmentedMessageReceiver: Exiting");
     }
 }
 
-pub struct ReceivedMessagesBufferController {
+pub(crate) struct ReceivedMessagesBufferController {
     fragmented_message_receiver: FragmentedMessageReceiver,
     request_receiver: RequestReceiver,
 }
 
 impl ReceivedMessagesBufferController {
-    pub fn new(
+    pub(crate) fn new(
         local_encryption_keypair: Arc<encryption::KeyPair>,
         query_receiver: ReceivedBufferRequestReceiver,
         mixnet_packet_receiver: MixnetMessageReceiver,
-        reply_key_storage: ReplyKeyStorage,
+        reply_key_storage: SentReplyKeys,
+        reply_controller_sender: ReplyControllerSender,
     ) -> Self {
-        let received_buffer =
-            ReceivedMessagesBuffer::new(local_encryption_keypair, reply_key_storage);
+        let received_buffer = ReceivedMessagesBuffer::new(
+            local_encryption_keypair,
+            reply_key_storage,
+            reply_controller_sender,
+        );
 
         ReceivedMessagesBufferController {
             fragmented_message_receiver: FragmentedMessageReceiver::new(
@@ -354,9 +488,18 @@ impl ReceivedMessagesBufferController {
         }
     }
 
-    pub fn start(self) {
-        // TODO: should we do anything with JoinHandle(s) returned by start methods?
-        self.fragmented_message_receiver.start();
-        self.request_receiver.start();
+    pub fn start_with_shutdown(self, shutdown: task::TaskClient) {
+        let mut fragmented_message_receiver = self.fragmented_message_receiver;
+        let mut request_receiver = self.request_receiver;
+
+        let shutdown_handle = shutdown.clone();
+        spawn_future(async move {
+            fragmented_message_receiver
+                .run_with_shutdown(shutdown_handle)
+                .await;
+        });
+        spawn_future(async move {
+            request_receiver.run_with_shutdown(shutdown).await;
+        });
     }
 }

clients/client-core/src/client/replies/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod reply_controller;
+pub mod reply_storage;

clients/client-core/src/client/replies/reply_controller/mod.rs

@@ -0,0 +1,872 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::client::real_messages_control::acknowledgement_control::PendingAcknowledgement;
+use crate::client::real_messages_control::message_handler::{MessageHandler, PreparationError};
+use crate::client::replies::reply_storage::CombinedReplyStorage;
+use client_connections::{ConnectionId, TransmissionLane};
+use futures::channel::oneshot;
+use futures::StreamExt;
+use log::{debug, error, info, trace, warn};
+use nymsphinx::addressing::clients::Recipient;
+use nymsphinx::anonymous_replies::requests::AnonymousSenderTag;
+use nymsphinx::anonymous_replies::ReplySurb;
+use nymsphinx::chunking::fragment::{Fragment, FragmentIdentifier};
+use rand::{CryptoRng, Rng};
+use std::cmp::{max, min};
+use std::collections::btree_map::Entry;
+use std::collections::{BTreeMap, HashMap};
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+use time::OffsetDateTime;
+
+use crate::client::helpers::new_interval_stream;
+use crate::client::transmission_buffer::TransmissionBuffer;
+pub(crate) use requests::{ReplyControllerMessage, ReplyControllerReceiver, ReplyControllerSender};
+
+pub mod requests;
+
+pub struct Config {
+    min_surb_request_size: u32,
+    max_surb_request_size: u32,
+    maximum_allowed_reply_surb_request_size: u32,
+    max_surb_rerequest_waiting_period: Duration,
+    max_surb_drop_waiting_period: Duration,
+    max_reply_surb_age: Duration,
+    max_reply_key_age: Duration,
+}
+
+impl Config {
+    pub(crate) fn new(
+        min_surb_request_size: u32,
+        max_surb_request_size: u32,
+        maximum_allowed_reply_surb_request_size: u32,
+        max_surb_rerequest_waiting_period: Duration,
+        max_surb_drop_waiting_period: Duration,
+        max_reply_surb_age: Duration,
+        max_reply_key_age: Duration,
+    ) -> Self {
+        Self {
+            min_surb_request_size,
+            max_surb_request_size,
+            maximum_allowed_reply_surb_request_size,
+            max_surb_rerequest_waiting_period,
+            max_surb_drop_waiting_period,
+            max_reply_surb_age,
+            max_reply_key_age,
+        }
+    }
+}
+
+// the purpose of this task:
+// - buffers split messages from input message listener if there were insufficient surbs to send them
+// - upon getting extra surbs, resends them
+// - so I guess it will handle all 'RepliableMessage' and requests from 'ReplyMessage'
+// - replies to "give additional surbs" requests
+// - will reply to future heartbeats
+
+// TODO: this should be split into ingress and egress controllers
+// because currently its trying to perform two distinct jobs
+pub struct ReplyController<R> {
+    config: Config,
+
+    // TODO: incorporate that field at some point
+    // and use binomial distribution to determine the expected required number
+    // of surbs required to send the message through
+    // expected_reliability: f32,
+    request_receiver: ReplyControllerReceiver,
+    pending_replies: HashMap<AnonymousSenderTag, TransmissionBuffer<Fragment>>,
+
+    /// Retransmission packets that have already timed out and are waiting for additional reply SURBs
+    /// so that they could be sent back to the network. Once we receive more SURBs, we should send them ASAP.
+    // TODO: when purging stale entries, we must take extra care to also purge all pending ACK data!!
+    pending_retransmissions:
+        HashMap<AnonymousSenderTag, BTreeMap<FragmentIdentifier, Weak<PendingAcknowledgement>>>,
+
+    message_handler: MessageHandler<R>,
+    full_reply_storage: CombinedReplyStorage,
+}
+
+impl<R> ReplyController<R>
+where
+    R: CryptoRng + Rng,
+{
+    pub(crate) fn new(
+        config: Config,
+        message_handler: MessageHandler<R>,
+        full_reply_storage: CombinedReplyStorage,
+        request_receiver: ReplyControllerReceiver,
+    ) -> Self {
+        ReplyController {
+            config,
+            request_receiver,
+            pending_replies: HashMap::new(),
+            pending_retransmissions: HashMap::new(),
+            message_handler,
+            full_reply_storage,
+        }
+    }
+
+    fn insert_pending_replies<I: IntoIterator<Item = Fragment>>(
+        &mut self,
+        recipient: &AnonymousSenderTag,
+        fragments: I,
+        lane: TransmissionLane,
+    ) {
+        self.pending_replies
+            .entry(*recipient)
+            .or_insert_with(TransmissionBuffer::new)
+            .store(&lane, fragments)
+    }
+
+    fn re_insert_pending_replies(
+        &mut self,
+        recipient: &AnonymousSenderTag,
+        fragments: Vec<(TransmissionLane, Fragment)>,
+    ) {
+        // the buffer should ALWAYS exist at this point, if it doesn't, it's a bug...
+        self.pending_replies
+            .entry(*recipient)
+            .or_insert_with(TransmissionBuffer::new)
+            .store_multiple(fragments)
+    }
+
+    fn re_insert_pending_retransmission(
+        &mut self,
+        recipient: &AnonymousSenderTag,
+        data: Vec<Arc<PendingAcknowledgement>>,
+    ) {
+        // the underlying entry MUST exist as we've just got data from there
+        let map_entry = self
+            .pending_retransmissions
+            .get_mut(recipient)
+            .expect("our pending retransmission entry is somehow gone!");
+
+        for pending in data {
+            // if it's 0, we don't need to do anything - we just got that ack!
+            if Arc::strong_count(&pending) > 1 {
+                let id = pending.inner_fragment_identifier();
+                let downgraded = Arc::downgrade(&pending);
+                map_entry.insert(id, downgraded);
+            }
+        }
+    }
+
+    fn should_request_more_surbs(&self, target: &AnonymousSenderTag) -> bool {
+        trace!("checking if we should request more surbs from {:?}", target);
+
+        let pending_queue_size = self
+            .pending_replies
+            .get(target)
+            .map(|pending_queue| pending_queue.total_size())
+            .unwrap_or_default();
+
+        let retransmission_queue = self
+            .pending_retransmissions
+            .get(target)
+            .map(|pending_queue| pending_queue.len())
+            .unwrap_or_default();
+
+        let total_queue = pending_queue_size + retransmission_queue;
+
+        // simple as that - there's absolutely nothing to retransmit
+        if total_queue == 0 {
+            return false;
+        }
+
+        let available_surbs = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .available_surbs(target);
+        let pending_surbs = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .pending_reception(target) as usize;
+        let min_surbs_threshold = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .min_surb_threshold();
+        let max_surbs_threshold = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .max_surb_threshold();
+
+        debug!("total queue size: {total_queue} = pending data {pending_queue_size} + pending retransmission {retransmission_queue}, available surbs: {available_surbs} pending surbs: {pending_surbs} threshold range: {min_surbs_threshold}..{max_surbs_threshold}");
+
+        (pending_surbs + available_surbs) < max_surbs_threshold
+            && (pending_surbs + available_surbs) < (total_queue + min_surbs_threshold)
+    }
+
+    async fn handle_send_reply(
+        &mut self,
+        recipient_tag: AnonymousSenderTag,
+        data: Vec<u8>,
+        lane: TransmissionLane,
+    ) {
+        if !self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .contains_surbs_for(&recipient_tag)
+        {
+            warn!("received reply request for {:?} but we don't have any surbs stored for that recipient!", recipient_tag);
+            return;
+        }
+
+        trace!("handling reply to {:?}", recipient_tag);
+        let mut fragments = self.message_handler.split_reply_message(data);
+        let total_size = fragments.len();
+        trace!("This reply requires {:?} SURBs", total_size);
+
+        let available_surbs = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .available_surbs(&recipient_tag);
+        let min_surbs_threshold = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .min_surb_threshold();
+
+        let max_to_send = if available_surbs > min_surbs_threshold {
+            min(fragments.len(), available_surbs - min_surbs_threshold)
+        } else {
+            0
+        };
+
+        if max_to_send > 0 {
+            let (surbs, _surbs_left) = self
+                .full_reply_storage
+                .surbs_storage_ref()
+                .get_reply_surbs(&recipient_tag, max_to_send);
+
+            if let Some(reply_surbs) = surbs {
+                let to_send = fragments.drain(..max_to_send).collect::<Vec<_>>();
+                if let Err(err) = self
+                    .message_handler
+                    .try_send_reply_chunks_on_lane(
+                        recipient_tag,
+                        to_send.clone(),
+                        reply_surbs,
+                        lane,
+                    )
+                    .await
+                {
+                    let err = err.return_unused_surbs(
+                        self.full_reply_storage.surbs_storage_ref(),
+                        &recipient_tag,
+                    );
+                    warn!("failed to send reply to {recipient_tag}: {err}");
+                    self.insert_pending_replies(&recipient_tag, to_send, lane);
+                }
+            }
+        }
+
+        // if there's leftover data we didn't send because we didn't have enough (or any) surbs - buffer it
+        if !fragments.is_empty() {
+            self.insert_pending_replies(&recipient_tag, fragments, lane);
+        }
+
+        if self.should_request_more_surbs(&recipient_tag) {
+            self.request_reply_surbs_for_queue_clearing(recipient_tag)
+                .await;
+        }
+    }
+
+    async fn request_additional_reply_surbs(
+        &mut self,
+        target: AnonymousSenderTag,
+        amount: u32,
+    ) -> Result<(), PreparationError> {
+        let reply_surb = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .get_reply_surb_ignoring_threshold(&target)
+            .and_then(|(reply_surb, _)| reply_surb)
+            .ok_or(PreparationError::NotEnoughSurbs {
+                available: 0,
+                required: 1,
+            })?;
+
+        if let Err(err) = self
+            .message_handler
+            .try_request_additional_reply_surbs(target, reply_surb, amount)
+            .await
+        {
+            let err = err.return_unused_surbs(self.full_reply_storage.surbs_storage_ref(), &target);
+            warn!(
+                "failed to request additional surbs from {:?} - {err}",
+                target
+            );
+            return Err(err);
+        } else {
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .increment_pending_reception(&target, amount);
+        }
+
+        Ok(())
+    }
+
+    async fn try_clear_pending_retransmission(&mut self, target: AnonymousSenderTag) {
+        trace!("trying to clear pending retransmission queue");
+        let available_surbs = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .available_surbs(&target);
+        let min_surbs_threshold = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .min_surb_threshold();
+
+        let max_to_clear = if available_surbs > min_surbs_threshold {
+            available_surbs - min_surbs_threshold
+        } else {
+            trace!("we don't have enough surbs for retransmission queue clearing...");
+            return;
+        };
+        trace!("we can clear up to {max_to_clear} entries");
+
+        let Some(pending) = self.pending_retransmissions.get_mut(&target) else {
+            trace!("there are no pending retransmissions for {target}!");
+            return;
+        };
+
+        let mut to_take = Vec::new();
+
+        while to_take.len() < max_to_clear {
+            if let Some((_, data)) = pending.pop_first() {
+                // no need to do anything if we failed to upgrade the reference,
+                // it means we got the ack while the data was waiting in the queue
+                if let Some(upgraded) = data.upgrade() {
+                    to_take.push(upgraded)
+                }
+            } else {
+                // our map is empty!
+                break;
+            }
+        }
+
+        if to_take.is_empty() {
+            // no need to do anything
+            return;
+        }
+
+        let (surbs_for_reply, _) = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .get_reply_surbs(&target, to_take.len());
+
+        let Some(surbs_for_reply) = surbs_for_reply else {
+            error!("somehow different task has stolen our reply surbs! - this should have been impossible");
+            self.re_insert_pending_retransmission(&target, to_take);
+            return;
+        };
+
+        let to_send_vec = to_take.iter().map(|ack| ack.fragment_data()).collect();
+
+        let prepared_fragments = match self
+            .message_handler
+            .prepare_reply_chunks_for_sending(to_send_vec, surbs_for_reply)
+            .await
+        {
+            Ok(prepared) => prepared,
+            Err(err) => {
+                let err =
+                    err.return_unused_surbs(self.full_reply_storage.surbs_storage_ref(), &target);
+                self.re_insert_pending_retransmission(&target, to_take);
+
+                warn!(
+                    "failed to clear pending retransmission queue for {:?} - {err}",
+                    target
+                );
+                return;
+            }
+        };
+
+        // we can't fail at this point, so drop all references to acks so that timer updates wouldn't blow up
+        drop(to_take);
+
+        self.message_handler
+            .send_retransmission_reply_chunks(prepared_fragments, TransmissionLane::Retransmission)
+            .await;
+    }
+
+    fn pop_at_most_pending_replies(
+        &mut self,
+        from: &AnonymousSenderTag,
+        amount: usize,
+    ) -> Option<Vec<(TransmissionLane, Fragment)>> {
+        // if possible, pop all pending replies, if not, pop only entries for which we'd have a reply surb
+        let total = self.pending_replies.get(from)?.total_size();
+        trace!("pending queue has {total} elements");
+        if total == 0 {
+            return None;
+        }
+        self.pending_replies
+            .get_mut(from)?
+            .pop_at_most_n_next_messages_at_random(amount)
+    }
+
+    async fn try_clear_pending_queue(&mut self, target: AnonymousSenderTag) {
+        trace!("trying to clear pending queue");
+        let available_surbs = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .available_surbs(&target);
+        let min_surbs_threshold = self
+            .full_reply_storage
+            .surbs_storage_ref()
+            .min_surb_threshold();
+
+        let max_to_clear = if available_surbs > min_surbs_threshold {
+            available_surbs - min_surbs_threshold
+        } else {
+            trace!("we don't have enough surbs for queue clearing...");
+            return;
+        };
+        trace!("we can clear up to {max_to_clear} entries");
+
+        // we're guaranteed to not get more entries than we have reply surbs for
+        if let Some(to_send) = self.pop_at_most_pending_replies(&target, max_to_clear) {
+            let to_send_clone = to_send.clone();
+
+            if to_send_clone.is_empty() {
+                panic!(
+                    "please let the devs know if you ever see this message (reply_controller.rs)"
+                );
+            }
+
+            let (surbs_for_reply, _) = self
+                .full_reply_storage
+                .surbs_storage_ref()
+                .get_reply_surbs(&target, to_send_clone.len());
+
+            let Some(surbs_for_reply) = surbs_for_reply else {
+                error!("somehow different task has stolen our reply surbs! - this should have been impossible");
+                self.re_insert_pending_replies(&target, to_send);
+                return;
+            };
+
+            if let Err(err) = self
+                .message_handler
+                .try_send_reply_chunks(target, to_send_clone, surbs_for_reply)
+                .await
+            {
+                let err =
+                    err.return_unused_surbs(self.full_reply_storage.surbs_storage_ref(), &target);
+                self.re_insert_pending_replies(&target, to_send);
+                warn!("failed to clear pending queue for {:?} - {err}", target);
+            }
+        } else {
+            trace!("the pending queue is empty");
+        }
+    }
+
+    async fn handle_received_surbs(
+        &mut self,
+        from: AnonymousSenderTag,
+        reply_surbs: Vec<ReplySurb>,
+        from_surb_request: bool,
+    ) {
+        trace!("handling received surbs");
+
+        // clear the requesting flag since we should have been asking for surbs
+        self.full_reply_storage
+            .surbs_storage_ref()
+            .reset_surbs_last_received_at(&from);
+        if from_surb_request {
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .decrement_pending_reception(&from, reply_surbs.len() as u32);
+        }
+
+        // store received surbs
+        self.full_reply_storage
+            .surbs_storage_ref()
+            .insert_surbs(&from, reply_surbs);
+
+        // use as many as we can for clearing pending retransmission queue
+        self.try_clear_pending_retransmission(from).await;
+
+        // use as many as we can for clearing pending 'normal' queue
+        self.try_clear_pending_queue(from).await;
+
+        // if we have to, request more
+        if self.should_request_more_surbs(&from) {
+            self.request_reply_surbs_for_queue_clearing(from).await;
+        }
+    }
+
+    async fn handle_surb_request(&mut self, recipient: Recipient, mut amount: u32) {
+        // 1. check whether we sent any surbs in the past to this recipient, otherwise
+        // they have no business in asking for more
+        if !self
+            .full_reply_storage
+            .tags_storage_ref()
+            .exists(&recipient)
+        {
+            warn!("{recipient} asked us for reply SURBs even though we never sent them any anonymous messages before!");
+            return;
+        }
+
+        // 2. check whether the requested amount is within sane range
+        if amount > self.config.maximum_allowed_reply_surb_request_size {
+            warn!("The requested reply surb amount is larger than our maximum allowed ({amount} > {}). Lowering it to a more sane value...", self.config.maximum_allowed_reply_surb_request_size);
+            amount = self.config.maximum_allowed_reply_surb_request_size;
+        }
+
+        // 3. construct and send the surbs away
+        // (send them in smaller batches to make the experience a bit smoother
+        let mut remaining = amount;
+        while remaining > 0 {
+            let to_send = min(remaining, 100);
+            if let Err(err) = self
+                .message_handler
+                .try_send_additional_reply_surbs(recipient, to_send)
+                .await
+            {
+                warn!("failed to send additional surbs to {recipient} - {err}");
+            } else {
+                trace!("sent {to_send} reply SURBs to {recipient}");
+            }
+
+            remaining -= to_send;
+        }
+    }
+
+    fn buffer_pending_ack(
+        &mut self,
+        recipient: AnonymousSenderTag,
+        ack_ref: Arc<PendingAcknowledgement>,
+        weak_ack_ref: Weak<PendingAcknowledgement>,
+    ) {
+        let frag_id = ack_ref.inner_fragment_identifier();
+        if let Some(existing) = self.pending_retransmissions.get_mut(&recipient) {
+            if let Entry::Vacant(e) = existing.entry(frag_id) {
+                e.insert(weak_ack_ref);
+            } else {
+                warn!("we're already trying to retransmit {frag_id}. We must be really behind in surbs!");
+            }
+        } else {
+            let mut inner = BTreeMap::new();
+            inner.insert(frag_id, weak_ack_ref);
+            self.pending_retransmissions.insert(recipient, inner);
+        }
+    }
+
+    async fn handle_reply_retransmission(
+        &mut self,
+        recipient_tag: AnonymousSenderTag,
+        timed_out_ack: Weak<PendingAcknowledgement>,
+        extra_surbs_request: bool,
+    ) {
+        // seems we got the ack in the end
+        let ack_ref = match timed_out_ack.upgrade() {
+            Some(ack) => ack,
+            None => {
+                debug!("we received the ack for one of the reply packets as we were putting it in the retransmission queue");
+                return;
+            }
+        };
+
+        // if this is retransmission for obtaining additional reply surbs,
+        // we can dip below the storage threshold
+        let (maybe_reply_surb, _) = if extra_surbs_request {
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .get_reply_surb_ignoring_threshold(&recipient_tag)
+        } else {
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .get_reply_surb(&recipient_tag)
+        }
+        .expect("attempted to retransmit a packet to an unknown recipient - we shouldn't have sent the original packet in the first place!");
+
+        if let Some(reply_surb) = maybe_reply_surb {
+            match self
+                .message_handler
+                .try_prepare_single_reply_chunk_for_sending(reply_surb, ack_ref.fragment_data())
+                .await
+            {
+                Ok(prepared) => {
+                    // drop the ack ref so that controller would not panic on `UpdateTimer` if that task
+                    // got to handle the action before this function terminated (which is very much
+                    // possible if `forward_messages` takes a while)
+                    drop(ack_ref);
+
+                    self.message_handler
+                        .update_ack_delay(prepared.fragment_identifier, prepared.total_delay);
+                    self.message_handler
+                        .forward_messages(vec![prepared.into()], TransmissionLane::Retransmission)
+                        .await;
+                }
+                Err(err) => {
+                    let err = err.return_unused_surbs(
+                        self.full_reply_storage.surbs_storage_ref(),
+                        &recipient_tag,
+                    );
+                    warn!("failed to prepare message for retransmission - {err}");
+                    // we buffer that packet and to try another day
+                    self.buffer_pending_ack(recipient_tag, ack_ref, timed_out_ack);
+
+                    if self.should_request_more_surbs(&recipient_tag) {
+                        self.request_reply_surbs_for_queue_clearing(recipient_tag)
+                            .await;
+                    }
+                }
+            };
+        } else {
+            self.buffer_pending_ack(recipient_tag, ack_ref, timed_out_ack);
+
+            if self.should_request_more_surbs(&recipient_tag) {
+                self.request_reply_surbs_for_queue_clearing(recipient_tag)
+                    .await;
+            }
+        }
+    }
+
+    // to be honest this doesn't make a lot of sense in the context of `connection_id`,
+    // it should really be asked per tag
+    fn handle_lane_queue_length(
+        &self,
+        connection_id: ConnectionId,
+        response_channel: oneshot::Sender<usize>,
+    ) {
+        // TODO: if we ever have duplicate ids for different senders, it means our rng is super weak
+        // thus I don't think we have to worry about it?
+        let lane = TransmissionLane::ConnectionId(connection_id);
+        for buf in self.pending_replies.values() {
+            if let Some(length) = buf.lane_length(&lane) {
+                if response_channel.send(length).is_err() {
+                    error!("the requester for lane queue length has dropped the response channel!")
+                }
+                return;
+            }
+        }
+        // make sure that if we didn't find that lane, we reply with 0
+        if response_channel.send(0).is_err() {
+            error!("the requester for lane queue length has dropped the response channel!")
+        }
+    }
+
+    async fn handle_request(&mut self, request: ReplyControllerMessage) {
+        match request {
+            ReplyControllerMessage::RetransmitReply {
+                recipient,
+                timed_out_ack,
+                extra_surb_request,
+            } => {
+                self.handle_reply_retransmission(recipient, timed_out_ack, extra_surb_request)
+                    .await
+            }
+            ReplyControllerMessage::SendReply {
+                recipient,
+                message,
+                lane,
+            } => self.handle_send_reply(recipient, message, lane).await,
+            ReplyControllerMessage::AdditionalSurbs {
+                sender_tag,
+                reply_surbs,
+                from_surb_request,
+            } => {
+                self.handle_received_surbs(sender_tag, reply_surbs, from_surb_request)
+                    .await
+            }
+            ReplyControllerMessage::LaneQueueLength {
+                connection_id,
+                response_channel,
+            } => self.handle_lane_queue_length(connection_id, response_channel),
+            ReplyControllerMessage::AdditionalSurbsRequest { recipient, amount } => {
+                self.handle_surb_request(*recipient, amount).await
+            }
+        }
+    }
+
+    // TODO: modify this method to more accurately determine the amount of surbs it needs to request
+    // it should take into consideration the average latency, sending rate and queue size.
+    // it should request as many surbs as it takes to saturate its sending rate before next batch arrives
+    async fn request_reply_surbs_for_queue_clearing(&mut self, target: AnonymousSenderTag) {
+        trace!("requesting surbs for queues clearing");
+
+        let pending_queue_size = self
+            .pending_replies
+            .get(&target)
+            .map(|pending_queue| pending_queue.total_size())
+            .unwrap_or_default();
+
+        let retransmission_queue = self
+            .pending_retransmissions
+            .get(&target)
+            .map(|pending_queue| pending_queue.len())
+            .unwrap_or_default();
+
+        let total_queue = (pending_queue_size + retransmission_queue) as u32;
+
+        if total_queue == 0 {
+            trace!("the pending queues for {:?} are already empty", target);
+            return;
+        }
+
+        let request_size = min(
+            self.config.max_surb_request_size,
+            max(total_queue, self.config.min_surb_request_size),
+        );
+
+        if let Err(err) = self
+            .request_additional_reply_surbs(target, request_size)
+            .await
+        {
+            warn!("failed to request additional surbs... - {err}")
+        }
+    }
+
+    async fn inspect_stale_entries(&mut self) {
+        let mut to_request = Vec::new();
+        let mut to_remove = Vec::new();
+
+        let now = OffsetDateTime::now_utc();
+        for (pending_reply_target, vals) in &self.pending_replies {
+            if vals.is_empty() {
+                continue;
+            }
+
+            let Some(last_received) = self.full_reply_storage.surbs_storage_ref().surbs_last_received_at(pending_reply_target) else {
+                error!("we have {} pending replies for {pending_reply_target}, but we somehow never received any reply surbs from them!", vals.total_size());
+                to_remove.push(*pending_reply_target);
+                continue;
+            };
+
+            // this should never ever happen (famous last words, eh?), but in case it DOES happen eventually
+            // purge that malformed data
+            let Ok(last_received_time) = OffsetDateTime::from_unix_timestamp(last_received) else {
+                error!("somehow our stored timestamp ({last_received}) for surbs from {pending_reply_target} is corrupted!. Going to remove all the associated entries");
+                to_remove.push(*pending_reply_target);
+                continue;
+            };
+
+            let diff = now - last_received_time;
+
+            if diff > self.config.max_surb_rerequest_waiting_period {
+                if diff > self.config.max_surb_drop_waiting_period {
+                    to_remove.push(*pending_reply_target)
+                } else {
+                    debug!("We haven't received any surbs in {:?} from {pending_reply_target}. Going to explicitly ask for more", diff);
+                    to_request.push(*pending_reply_target);
+                }
+            }
+        }
+
+        for pending_reply_target in to_request {
+            self.request_reply_surbs_for_queue_clearing(pending_reply_target)
+                .await;
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .reset_pending_reception(&pending_reply_target)
+        }
+        for to_remove in to_remove {
+            self.pending_replies.remove(&to_remove);
+        }
+    }
+
+    async fn invalidate_old_data(&self) {
+        let now = OffsetDateTime::now_utc();
+
+        let mut to_remove_surbs = Vec::new();
+        let mut to_remove_keys = Vec::new();
+        for map_ref in self.full_reply_storage.surbs_storage_ref().as_raw_iter() {
+            let (sender, received) = map_ref.pair();
+            // TODO: handle the following edge case:
+            // there's a malicious client sending us exactly one reply surb just before we should have invalidated
+            // the data thus making us keep everything in memory
+            // possible solution: keep timestamp PER reply surb (but that seems like an overkill)
+            // but I doubt this is ever going to be a problem...
+            // ...
+            // However, if you're reading this message, it probably became a legit problem,
+            // so I guess add timestamp per surb then? chop-chop.
+
+            let last_received = received.surbs_last_received_at();
+            // this should never ever happen (famous last words, eh?), but in case it DOES happen eventually
+            // purge that malformed data
+            let Ok(last_received_time) = OffsetDateTime::from_unix_timestamp(last_received) else {
+                error!("somehow our stored timestamp ({last_received}) for surbs from {sender} is corrupted!. Going to remove all the associated entries");
+                to_remove_surbs.push(*sender);
+                continue;
+            };
+            let diff = now - last_received_time;
+
+            if diff > self.config.max_reply_surb_age {
+                info!("it's been {diff:?} since we last received any reply surb from {sender}. Going to remove all stored entries...");
+
+                to_remove_surbs.push(*sender);
+            }
+        }
+
+        for map_ref in self.full_reply_storage.key_storage_ref().as_raw_iter() {
+            let (digest, reply_key) = map_ref.pair();
+
+            // this should never ever happen (famous last words, eh?), but in case it DOES happen eventually
+            // purge that malformed data
+            let Ok(sent_at) = OffsetDateTime::from_unix_timestamp(reply_key.sent_at_timestamp) else {
+                error!("somehow our stored timestamp ({}) for one of our reply key is corrupted!. Going to remove all the entry", reply_key.sent_at_timestamp);
+                to_remove_keys.push(*digest);
+                continue;
+            };
+
+            let diff = now - sent_at;
+
+            if diff > self.config.max_reply_key_age {
+                debug!("it's been {diff:?} since we created this reply key. it's probably never going to get used, so we're going to purge it...");
+                to_remove_keys.push(*digest);
+            }
+        }
+
+        for to_remove in to_remove_surbs {
+            self.full_reply_storage
+                .surbs_storage_ref()
+                .remove(&to_remove);
+        }
+
+        for to_remove in to_remove_keys {
+            self.full_reply_storage.key_storage().remove(to_remove)
+        }
+    }
+
+    // #[cfg(not(target_arch = "wasm32"))]
+    // async fn log_status(&self) {
+    //     todo!()
+    // }
+
+    pub(crate) async fn run_with_shutdown(&mut self, mut shutdown: task::TaskClient) {
+        debug!("Started ReplyController with graceful shutdown support");
+
+        let polling_rate = Duration::from_secs(5);
+        let mut stale_inspection = new_interval_stream(polling_rate);
+
+        // this is in the order of hours/days so we don't have to poll it that often
+        let polling_rate = Duration::from_secs(self.config.max_reply_surb_age.as_secs() / 10);
+        let mut invalidation_inspection = new_interval_stream(polling_rate);
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                biased;
+                _ = shutdown.recv_with_delay() => {
+                    log::trace!("ReplyController: Received shutdown");
+                },
+                req = self.request_receiver.next() => match req {
+                    Some(req) => self.handle_request(req).await,
+                    None => {
+                        log::trace!("ReplyController: Stopping since channel closed");
+                        break;
+                    }
+                },
+                _ = stale_inspection.next() => {
+                    self.inspect_stale_entries().await
+                },
+                _ = invalidation_inspection.next() => {
+                    self.invalidate_old_data().await
+                }
+            }
+        }
+        assert!(shutdown.is_shutdown_poll());
+        log::debug!("ReplyController: Exiting");
+    }
+}