Compare commits

..

10 Commits

Author SHA1 Message Date
Rokas Ambrazevicius f194fb9cca Add appearance setting 2024-02-01 22:34:37 +02:00
Rokas Ambrazevicius 36a4f93127 Add settings 2024-02-01 17:04:49 +02:00
Rokas Ambrazevicius aae5adef38 Add app icon 2024-01-18 14:51:25 +02:00
Rokas Ambrazevicius 51754f737c Add country selection view 2024-01-17 16:45:13 +02:00
Rokas Ambrazevicius 85ccd835ab Add hop list view 2024-01-12 16:36:16 +02:00
Rokas Ambrazevicius 8e302bfdde Add Settings package 2024-01-08 14:06:08 +02:00
Rokas Ambrazevicius b609331b8e Add Home View 2024-01-08 13:39:07 +02:00
Rokas Ambrazevicius 59845174e7 Add localizations and status button 2023-12-28 16:40:02 +02:00
Rokas Ambrazevicius 0bdb6a626d Setup packages 2023-12-27 16:13:20 +02:00
Rokas Ambrazevicius 7f0256cc33 Setup iOS project 2023-12-22 13:17:21 +02:00
889 changed files with 29562 additions and 19929 deletions
+12 -23
View File
@@ -4,18 +4,18 @@ on:
workflow_dispatch:
pull_request:
paths:
- "clients/**"
- "common/**"
- "explorer-api/**"
- "gateway/**"
- "integrations/**"
- "mixnode/**"
- "sdk/rust/nym-sdk/**"
- "service-providers/**"
- "nym-api/**"
- "nym-outfox/**"
- "tools/nym-cli/**"
- "tools/ts-rs-cli/**"
- 'clients/**'
- 'common/**'
- 'explorer-api/**'
- 'gateway/**'
- 'integrations/**'
- 'mixnode/**'
- 'sdk/rust/nym-sdk/**'
- 'service-providers/**'
- 'nym-api/**'
- 'nym-outfox/**'
- 'tools/nym-cli/**'
- 'tools/ts-rs-cli/**'
jobs:
publish-nym:
@@ -53,16 +53,6 @@ jobs:
command: build
args: --workspace --release
- name: Install cargo-deb
uses: actions-rs/cargo@v1
with:
command: install
args: cargo-deb
- name: Build deb packages
shell: bash
run: make deb
- name: Prepare build output
shell: bash
env:
@@ -77,7 +67,6 @@ jobs:
cp target/release/nym-network-statistics $OUTPUT_DIR
cp target/release/nym-cli $OUTPUT_DIR
cp target/release/explorer-api $OUTPUT_DIR
cp target/debian/*.deb $OUTPUT_DIR
- name: Deploy branch to CI www
continue-on-error: true
+5 -5
View File
@@ -1,8 +1,5 @@
name: ci-cargo-deny
on:
workflow_dispatch:
pull_request:
on: [workflow_dispatch]
jobs:
cargo-deny:
runs-on: ubuntu-22.04
@@ -10,7 +7,10 @@ jobs:
matrix:
checks:
# - advisories
- licenses bans sources
- licenses
- bans sources
continue-on-error: ${{ matrix.checks == 'licenses' }}
steps:
- uses: actions/checkout@v3
-3
View File
@@ -17,9 +17,6 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: install yarn in root
run: cd ../.. && yarn install
- name: Install npm
run: npm install
+40
View File
@@ -0,0 +1,40 @@
name: ci-nym-vpn-ui-js
on:
workflow_dispatch:
pull_request:
paths:
- 'nym-vpn/ui/src/**'
- 'nym-vpn/ui/package.json'
- 'nym-vpn/ui/index.html'
jobs:
check:
runs-on: custom-linux
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Node
uses: actions/setup-node@v3
with:
node-version: 18
- name: Install Yarn
run: npm install -g yarn
- name: Install dependencies
working-directory: nym-vpn/ui
run: yarn
- name: Type-check
working-directory: nym-vpn/ui
run: yarn typecheck
- name: Check lint
working-directory: nym-vpn/ui
run: yarn lint
- name: Check formatting
working-directory: nym-vpn/ui
run: yarn fmt:check
# - name: Run tests
# working-directory: nym-vpn/ui
# run: yarn test
- name: Check build
working-directory: nym-vpn/ui
run: yarn build
+63
View File
@@ -0,0 +1,63 @@
name: ci-nym-vpn-ui-rust
on:
workflow_dispatch:
pull_request:
paths:
- 'nym-vpn/ui/src-tauri/**'
jobs:
build:
runs-on: custom-linux
env:
CARGO_TERM_COLOR: always
CARGOTOML_PATH: ./nym-vpn/ui/src-tauri/Cargo.toml
steps:
- name: Install Dependencies (Linux)
run: sudo apt-get update && sudo apt-get -y install libwebkit2gtk-4.0-dev build-essential curl wget libssl-dev libgtk-3-dev squashfs-tools libayatana-appindicator3-dev
continue-on-error: true
- name: Checkout
uses: actions/checkout@v4
- name: Install rust toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
components: rustfmt, clippy
- name: Prepare build
run: mkdir nym-vpn/ui/dist
- name: Build
uses: actions-rs/cargo@v1
with:
command: build
args: --manifest-path ${{ env.CARGOTOML_PATH }} --features custom-protocol
# - name: Run all tests
# uses: actions-rs/cargo@v1
# with:
# command: test
# args: --manifest-path ${{ env.CARGOTOML_PATH }}
- name: Check formatting
uses: actions-rs/cargo@v1
with:
command: fmt
args: --manifest-path ${{ env.CARGOTOML_PATH }} --all -- --check
- name: Annotate with clippy checks
uses: actions-rs/clippy-check@v1
continue-on-error: true
with:
token: ${{ secrets.GITHUB_TOKEN }}
args: --manifest-path ${{ env.CARGOTOML_PATH }} --all-features
- name: Clippy
uses: actions-rs/cargo@v1
with:
command: clippy
args: --manifest-path ${{ env.CARGOTOML_PATH }} --all-features --all-targets -- -D warnings
Generated
+250 -330
View File
File diff suppressed because it is too large Load Diff
+3 -10
View File
@@ -67,7 +67,6 @@ members = [
"common/nymsphinx/params",
"common/nymsphinx/routing",
"common/nymsphinx/types",
"common/nyxd-scraper",
"common/pemstore",
"common/socks5-client-core",
"common/socks5/proxy-helpers",
@@ -102,7 +101,6 @@ members = [
"nym-node",
"nym-node/nym-node-requests",
"nym-outfox",
"nym-validator-rewarder",
"tools/internal/ssl-inject",
"tools/internal/sdk-version-bump",
"tools/nym-cli",
@@ -125,10 +123,9 @@ default-members = [
"nym-api",
"tools/nymvisor",
"explorer-api",
"nym-validator-rewarder",
]
exclude = ["explorer", "contracts", "nym-wallet", "nym-connect/mobile/src-tauri", "nym-connect/desktop", "nym-vpn/ui/src-tauri", "cpu-cycles", "sdk/ffi/cpp"]
exclude = ["explorer", "contracts", "nym-wallet", "nym-connect/mobile/src-tauri", "nym-connect/desktop", "nym-vpn/ui/src-tauri", "cpu-cycles"]
[workspace.package]
authors = ["Nym Technologies SA"]
@@ -162,7 +159,6 @@ reqwest = "0.11.22"
schemars = "0.8.1"
serde = "1.0.152"
serde_json = "1.0.91"
sqlx = "0.6.3"
tap = "1.0.1"
time = "0.3.30"
thiserror = "1.0.48"
@@ -203,12 +199,9 @@ cw-controllers = { version = "=1.1.0" }
# cosmrs-related
bip32 = "0.5.1"
# temporarily using a fork again (yay.) because we need staking and slashing support
cosmrs = { git = "https://github.com/jstuczyn/cosmos-rust", branch ="nym-temp/all-validator-features" }
#cosmrs = { git = "https://github.com/jstuczyn/cosmos-rust", branch = "nym-temp/all-validator-features" } # unfortuntely we need a fork by yours truly to get the staking support
tendermint = "0.34" # same version as used by cosmrs
cosmrs = "=0.15.0"
tendermint-rpc = "0.34" # same version as used by cosmrs
tendermint = "0.34" # same version as used by cosmrs
prost = "0.12"
# wasm-related dependencies
+3 -17
View File
@@ -12,7 +12,6 @@ help:
@echo " clippy: run clippy for all workspaces"
@echo " test: run clippy, unit tests, and formatting."
@echo " test-all: like test, but also includes the expensive tests"
@echo " deb: build debian packages
# -----------------------------------------------------------------------------
# Meta targets
@@ -158,12 +157,6 @@ build-explorer-api:
build-nym-cli:
cargo build -p nym-cli --release
build-nym-gateway:
cargo build -p nym-gateway --release
build-nym-mixnode:
cargo build -p nym-mixnode --release
# -----------------------------------------------------------------------------
# Misc
# -----------------------------------------------------------------------------
@@ -176,13 +169,6 @@ run-api-tests:
cd nym-api/tests/functional_test && yarn test:qa
# Build debian package, and update PPA
deb-mixnode: build-nym-mixnode
cargo deb -p nym-mixnode
deb-gateway: build-nym-gateway
cargo deb -p nym-gateway
deb-cli: build-nym-cli
cargo deb -p nym-cli
deb: deb-mixnode deb-gateway deb-cli
# Requires base64 encode GPG key to be set up in environment PPA_SIGNING_KEY
deb:
scripts/ppa.sh
-70
View File
@@ -1,70 +0,0 @@
<html>
<head>
<style>
@media (prefers-color-scheme: dark) {
body {
background: #333;
color: white;
}
a {
color: skyblue;
}
}
.container {
font-family: sans-serif;
max-width: 800px;
margin: 0 auto;
}
.intro {
text-align: center;
}
.licenses-list {
list-style-type: none;
margin: 0;
padding: 0;
}
.license-used-by {
margin-top: -10px;
}
.license-text {
max-height: 200px;
overflow-y: scroll;
white-space: pre-wrap;
}
</style>
</head>
<body>
<main class="container">
<div class="intro">
<h1>Third Party Licenses</h1>
<p>This page lists the licenses of the projects used in cargo-about.</p>
</div>
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
{{#each overview}}
<li><a href="#{{id}}">{{name}}</a> ({{count}})</li>
{{/each}}
</ul>
<h2>All license text:</h2>
<ul class="licenses-list">
{{#each licenses}}
<li class="license">
<h3 id="{{id}}">{{name}}</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
{{#each used_by}}
<li><a href="{{#if crate.repository}} {{crate.repository}} {{else}} https://crates.io/crates/{{crate.name}} {{/if}}">{{crate.name}} {{crate.version}}</a></li>
{{/each}}
</ul>
<pre class="license-text">{{text}}</pre>
</li>
{{/each}}
</ul>
</main>
</body>
</html>
-19
View File
@@ -1,19 +0,0 @@
private = { ignore = true }
accepted = [
"0BSD",
"Apache-2.0",
"BSD-2-Clause",
"BSD-3-Clause",
"CC0-1.0",
"ISC",
"MIT",
"MPL-2.0",
"Unicode-DFS-2016",
"OpenSSL",
]
workarounds = [
"ring",
"rustls",
]
@@ -1667,9 +1667,9 @@
}
},
"node_modules/follow-redirects": {
"version": "1.15.4",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
"integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
"dev": true,
"funding": [
{
@@ -5800,9 +5800,9 @@
}
},
"follow-redirects": {
"version": "1.15.4",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
"integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz",
"integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==",
"dev": true
},
"forwarded": {
+1 -1
View File
@@ -51,7 +51,7 @@ impl InitialisableClient for NativeClientInit {
}
}
#[derive(Args, Clone, Debug)]
#[derive(Args, Clone)]
pub(crate) struct Init {
#[command(flatten)]
common_args: CommonClientInitArgs,
+1 -1
View File
@@ -51,7 +51,7 @@ impl InitialisableClient for Socks5ClientInit {
}
}
#[derive(Args, Clone, Debug)]
#[derive(Args, Clone)]
pub(crate) struct Init {
#[command(flatten)]
common_args: CommonClientInitArgs,
-4
View File
@@ -1,4 +0,0 @@
module.exports = {
ConfigHandler: require('./config/configHandler.ts'),
RestClient: require('./restClient/RestClient.ts')
};
+1 -3
View File
@@ -9,7 +9,6 @@ repository = { workspace = true }
[dependencies]
atty = "0.2"
const-str = "0.5.6"
clap = { workspace = true, features = ["derive"] }
clap_complete = "4.0"
clap_complete_fig = "4.0"
@@ -36,10 +35,9 @@ opentelemetry = { version = "0.19.0", optional = true, features = ["rt-tokio"] }
[build-dependencies]
vergen = { version = "=8.2.6", default-features = false, features = [
vergen = { version = "=7.4.3", default-features = false, features = [
"build",
"git",
"gitcl",
"rustc",
"cargo",
] }
+7 -8
View File
@@ -1,14 +1,13 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use vergen::EmitBuilder;
use vergen::{vergen, Config};
fn main() {
EmitBuilder::builder()
.all_build()
.all_git()
.all_rustc()
.all_cargo()
.emit()
.expect("failed to extract build metadata");
let mut config = Config::default();
if std::env::var("DOCS_RS").is_ok() {
// If we don't have access to git information, such as in a docs.rs build, don't error
*config.git_mut().skip_if_error_mut() = true;
}
vergen(config).expect("failed to extract build metadata");
}
+5 -14
View File
@@ -40,22 +40,14 @@ pub struct BinaryBuildInformation {
/// Provides the rustc channel that was used for the build, for example `nightly`.
pub rustc_channel: &'static str,
// VERGEN_CARGO_DEBUG
/// Provides the cargo debug mode that was used for the build.
// NOTE: keep the old name cargo_profile instead of cargo_debug for backwards compatibility
// VERGEN_CARGO_PROFILE
/// Provides the cargo profile that was used for the build, for example `debug`.
pub cargo_profile: &'static str,
}
impl BinaryBuildInformation {
// explicitly require the build_version to be passed as it's binary specific
pub const fn new(binary_name: &'static str, build_version: &'static str) -> Self {
let cargo_debug = env!("VERGEN_CARGO_DEBUG");
let cargo_profile = if const_str::equal!(cargo_debug, "true") {
"debug"
} else {
"release"
};
BinaryBuildInformation {
binary_name,
build_timestamp: env!("VERGEN_BUILD_TIMESTAMP"),
@@ -65,7 +57,7 @@ impl BinaryBuildInformation {
commit_branch: env!("VERGEN_GIT_BRANCH"),
rustc_version: env!("VERGEN_RUSTC_SEMVER"),
rustc_channel: env!("VERGEN_RUSTC_CHANNEL"),
cargo_profile,
cargo_profile: env!("VERGEN_CARGO_PROFILE"),
}
}
@@ -123,9 +115,8 @@ pub struct BinaryBuildInformationOwned {
/// Provides the rustc channel that was used for the build, for example `nightly`.
pub rustc_channel: String,
// VERGEN_CARGO_DEBUG
/// Provides the cargo debug mode that was used for the build.
// NOTE: keep the old name cargo_profile instead of cargo_debug for backwards compatibility
// VERGEN_CARGO_PROFILE
/// Provides the cargo profile that was used for the build, for example `debug`.
pub cargo_profile: String,
}
+2 -3
View File
@@ -46,7 +46,6 @@ nym-validator-client = { path = "../client-libs/validator-client", default-featu
nym-task = { path = "../task" }
nym-credential-storage = { path = "../credential-storage" }
nym-network-defaults = { path = "../network-defaults" }
si-scale = "0.2.2"
[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio-stream]
version = "0.1.11"
@@ -60,7 +59,7 @@ features = ["time"]
version = "0.20.1"
[target."cfg(not(target_arch = \"wasm32\"))".dependencies.sqlx]
workspace = true
version = "0.6.2"
features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"]
optional = true
@@ -91,7 +90,7 @@ tempfile = "3.1.0"
[build-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
sqlx = { version = "0.6.2", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
[features]
default = []
@@ -109,8 +109,6 @@ pub async fn initialise_client<C>(
) -> Result<InitResultsWithConfig<C::Config>, C::Error>
where
C: InitialisableClient,
<C as InitialisableClient>::Config: std::fmt::Debug,
<C as InitialisableClient>::InitArgs: std::fmt::Debug,
{
info!("initialising {} client", C::NAME);
@@ -142,32 +140,17 @@ where
// Attempt to use a user-provided gateway, if possible
let user_chosen_gateway_id = common_args.gateway;
log::debug!("User chosen gateway id: {user_chosen_gateway_id:?}");
let selection_spec = GatewaySelectionSpecification::new(
user_chosen_gateway_id.map(|id| id.to_base58_string()),
Some(common_args.latency_based_selection),
false,
);
log::debug!("Gateway selection specification: {selection_spec:?}");
// Load and potentially override config
log::debug!("Init arguments: {init_args:#?}");
let config = C::construct_config(&init_args);
log::debug!("Constructed config: {config:#?}");
let paths = config.common_paths();
let core = config.core_config();
log::info!(
"Using nym-api: {}",
core.client
.nym_api_urls
.iter()
.map(|url| url.as_str())
.collect::<Vec<&str>>()
.join(",")
);
// Setup gateway by either registering a new one, or creating a new config from the selected
// one but with keys kept, or reusing the gateway configuration.
let key_store = OnDiskKeys::new(paths.keys.clone());
@@ -1,7 +1,6 @@
// Copyright 2022-2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use super::packet_statistics_control::PacketStatisticsReporter;
use super::received_buffer::ReceivedBufferMessage;
use super::topology_control::geo_aware_provider::GeoAwareTopologyProvider;
use crate::client::base_client::storage::gateway_details::GatewayDetailsStore;
@@ -11,7 +10,6 @@ use crate::client::inbound_messages::{InputMessage, InputMessageReceiver, InputM
use crate::client::key_manager::persistence::KeyStore;
use crate::client::mix_traffic::transceiver::{GatewayReceiver, GatewayTransceiver, RemoteGateway};
use crate::client::mix_traffic::{BatchMixMessageSender, MixTrafficController};
use crate::client::packet_statistics_control::PacketStatisticsControl;
use crate::client::real_messages_control;
use crate::client::real_messages_control::RealMessagesController;
use crate::client::received_buffer::{
@@ -256,7 +254,6 @@ where
self_address: Recipient,
topology_accessor: TopologyAccessor,
mix_tx: BatchMixMessageSender,
stats_tx: PacketStatisticsReporter,
shutdown: TaskClient,
) {
info!("Starting loop cover traffic stream...");
@@ -269,7 +266,6 @@ where
topology_accessor,
debug_config.traffic,
debug_config.cover_traffic,
stats_tx,
);
stream.start_with_shutdown(shutdown);
@@ -289,7 +285,6 @@ where
client_connection_rx: ConnectionCommandReceiver,
shutdown: TaskClient,
packet_type: PacketType,
stats_tx: PacketStatisticsReporter,
) {
info!("Starting real traffic stream...");
@@ -304,7 +299,6 @@ where
reply_controller_receiver,
lane_queue_lengths,
client_connection_rx,
stats_tx,
)
.start_with_shutdown(shutdown, packet_type);
}
@@ -318,7 +312,6 @@ where
reply_key_storage: SentReplyKeys,
reply_controller_sender: ReplyControllerSender,
shutdown: TaskClient,
packet_statistics_control: PacketStatisticsReporter,
) {
info!("Starting received messages buffer controller...");
let controller: ReceivedMessagesBufferController<SphinxMessageReceiver> =
@@ -328,7 +321,6 @@ where
mixnet_receiver,
reply_key_storage,
reply_controller_sender,
packet_statistics_control,
);
controller.start_with_shutdown(shutdown)
}
@@ -514,13 +506,6 @@ where
Ok(())
}
fn start_packet_statistics_control(shutdown: TaskClient) -> PacketStatisticsReporter {
info!("Starting packet statistics control...");
let (packet_statistics_control, packet_stats_reporter) = PacketStatisticsControl::new();
packet_statistics_control.start_with_shutdown(shutdown);
packet_stats_reporter
}
fn start_mix_traffic_controller(
gateway_transceiver: Box<dyn GatewayTransceiver + Send>,
shutdown: TaskClient,
@@ -648,9 +633,6 @@ where
)
.await?;
let packet_stats_reporter =
Self::start_packet_statistics_control(shutdown.fork("packet_statistics_control"));
let gateway_packet_router = PacketRouter::new(
ack_sender,
mixnet_messages_sender,
@@ -680,7 +662,6 @@ where
reply_storage.key_storage(),
reply_controller_sender.clone(),
shutdown.fork("received_messages_buffer"),
packet_stats_reporter.clone(),
);
// The message_sender is the transmitter for any component generating sphinx packets
@@ -719,7 +700,6 @@ where
client_connection_rx,
shutdown.fork("real_traffic_controller"),
self.config.debug.traffic.packet_type,
packet_stats_reporter.clone(),
);
if !self
@@ -734,7 +714,6 @@ where
self_address,
shared_topology_accessor.clone(),
message_sender,
packet_stats_reporter,
shutdown.fork("cover_traffic_stream"),
);
}
@@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache-2.0
use crate::client::mix_traffic::BatchMixMessageSender;
use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
use crate::client::topology_control::TopologyAccessor;
use crate::{config, spawn_future};
use futures::task::{Context, Poll};
@@ -62,8 +61,6 @@ where
secondary_packet_size: Option<PacketSize>,
packet_type: PacketType,
stats_tx: PacketStatisticsReporter,
}
impl<R> Stream for LoopCoverTrafficStream<R>
@@ -100,8 +97,7 @@ where
// obviously when we finally make shared rng that is on 'higher' level, this should become
// generic `R`
impl LoopCoverTrafficStream<OsRng> {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
pub fn new(
ack_key: Arc<AckKey>,
average_ack_delay: Duration,
mix_tx: BatchMixMessageSender,
@@ -109,7 +105,6 @@ impl LoopCoverTrafficStream<OsRng> {
topology_access: TopologyAccessor,
traffic_config: config::Traffic,
cover_config: config::CoverTraffic,
stats_tx: PacketStatisticsReporter,
) -> Self {
let rng = OsRng;
@@ -127,7 +122,6 @@ impl LoopCoverTrafficStream<OsRng> {
primary_packet_size: traffic_config.primary_packet_size,
secondary_packet_size: traffic_config.secondary_packet_size,
packet_type: traffic_config.packet_type,
stats_tx,
}
}
@@ -197,10 +191,6 @@ impl LoopCoverTrafficStream<OsRng> {
log::warn!("Failed to send cover message - channel closed");
}
}
} else {
self.stats_tx.report(PacketStatisticsEvent::CoverPacketSent(
cover_traffic_packet_size.size(),
));
}
// TODO: I'm not entirely sure whether this is really required, because I'm not 100%
@@ -1,8 +1,6 @@
// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
#![allow(unused_imports)]
use std::time::Duration;
pub use wasmtimer::{std::Instant, tokio::*};
-1
View File
@@ -7,7 +7,6 @@ pub(crate) mod helpers;
pub mod inbound_messages;
pub mod key_manager;
pub mod mix_traffic;
pub(crate) mod packet_statistics_control;
pub mod real_messages_control;
pub mod received_buffer;
pub mod replies;
@@ -1,498 +0,0 @@
use std::{
collections::VecDeque,
time::{Duration, Instant},
};
use si_scale::helpers::bibytes2;
use crate::spawn_future;
// Time interval between reporting packet statistics
const PACKET_REPORT_INTERVAL_SECS: u64 = 2;
// Interval for taking snapshots of the packet statistics
const SNAPSHOT_INTERVAL_MS: u64 = 500;
// When computing rates, we include snapshots that are up to this old. We set it to some odd number
// a tad larger than an integer number of snapshot intervals, so that we don't have to worry about
// threshold effects.
// Also, set it larger than the packet report interval so that we don't miss notable singular events
const RECORDING_WINDOW_MS: u64 = 2300;
#[derive(Default, Debug, Clone)]
struct PacketStatistics {
// Sent
real_packets_sent: u64,
real_packets_sent_size: usize,
cover_packets_sent: u64,
cover_packets_sent_size: usize,
// Received
real_packets_received: u64,
real_packets_received_size: usize,
cover_packets_received: u64,
cover_packets_received_size: usize,
// Acks
total_acks_received: u64,
total_acks_received_size: usize,
real_acks_received: u64,
real_acks_received_size: usize,
cover_acks_received: u64,
cover_acks_received_size: usize,
// Types of packets queued
// TODO: track the type sent instead
real_packets_queued: u64,
retransmissions_queued: u64,
reply_surbs_queued: u64,
additional_reply_surbs_queued: u64,
}
impl PacketStatistics {
fn handle_event(&mut self, event: PacketStatisticsEvent) {
match event {
PacketStatisticsEvent::RealPacketSent(packet_size) => {
self.real_packets_sent += 1;
self.real_packets_sent_size += packet_size;
}
PacketStatisticsEvent::CoverPacketSent(packet_size) => {
self.cover_packets_sent += 1;
self.cover_packets_sent_size += packet_size;
}
PacketStatisticsEvent::RealPacketReceived(packet_size) => {
self.real_packets_received += 1;
self.real_packets_received_size += packet_size;
}
PacketStatisticsEvent::CoverPacketReceived(packet_size) => {
self.cover_packets_received += 1;
self.cover_packets_received_size += packet_size;
}
PacketStatisticsEvent::AckReceived(packet_size) => {
self.total_acks_received += 1;
self.total_acks_received_size += packet_size;
}
PacketStatisticsEvent::RealAckReceived(packet_size) => {
self.real_acks_received += 1;
self.real_acks_received_size += packet_size;
}
PacketStatisticsEvent::CoverAckReceived(packet_size) => {
self.cover_acks_received += 1;
self.cover_acks_received_size += packet_size;
}
PacketStatisticsEvent::RealPacketQueued => {
self.real_packets_queued += 1;
}
PacketStatisticsEvent::RetransmissionQueued => {
self.retransmissions_queued += 1;
}
PacketStatisticsEvent::ReplySurbRequestQueued => {
self.reply_surbs_queued += 1;
}
PacketStatisticsEvent::AdditionalReplySurbRequestQueued => {
self.additional_reply_surbs_queued += 1;
}
}
}
fn summary(&self) -> (String, String) {
(
format!(
"packets sent: {} (real: {}, cover: {}, retransmissions: {})",
self.real_packets_sent + self.cover_packets_sent,
self.real_packets_sent,
self.cover_packets_sent,
self.retransmissions_queued,
),
format!(
"packets received: {}, (real: {}, cover: {}, acks: {}, acks for cover: {})",
self.real_packets_received + self.cover_packets_received,
self.real_packets_received,
self.cover_packets_received,
self.real_acks_received,
self.cover_acks_received,
),
)
}
}
impl std::ops::Sub for PacketStatistics {
type Output = Self;
fn sub(self, rhs: Self) -> Self::Output {
Self {
real_packets_sent: self.real_packets_sent - rhs.real_packets_sent,
real_packets_sent_size: self.real_packets_sent_size - rhs.real_packets_sent_size,
cover_packets_sent: self.cover_packets_sent - rhs.cover_packets_sent,
cover_packets_sent_size: self.cover_packets_sent_size - rhs.cover_packets_sent_size,
real_packets_received: self.real_packets_received - rhs.real_packets_received,
real_packets_received_size: self.real_packets_received_size
- rhs.real_packets_received_size,
cover_packets_received: self.cover_packets_received - rhs.cover_packets_received,
cover_packets_received_size: self.cover_packets_received_size
- rhs.cover_packets_received_size,
total_acks_received: self.total_acks_received - rhs.total_acks_received,
total_acks_received_size: self.total_acks_received_size - rhs.total_acks_received_size,
real_acks_received: self.real_acks_received - rhs.real_acks_received,
real_acks_received_size: self.real_acks_received_size - rhs.real_acks_received_size,
cover_acks_received: self.cover_acks_received - rhs.cover_acks_received,
cover_acks_received_size: self.cover_acks_received_size - rhs.cover_acks_received_size,
real_packets_queued: self.real_packets_queued - rhs.real_packets_queued,
retransmissions_queued: self.retransmissions_queued - rhs.retransmissions_queued,
reply_surbs_queued: self.reply_surbs_queued - rhs.reply_surbs_queued,
additional_reply_surbs_queued: self.additional_reply_surbs_queued
- rhs.additional_reply_surbs_queued,
}
}
}
#[derive(Debug, Clone)]
struct PacketRates {
real_packets_sent: f64,
real_packets_sent_size: f64,
cover_packets_sent: f64,
cover_packets_sent_size: f64,
real_packets_received: f64,
real_packets_received_size: f64,
cover_packets_received: f64,
cover_packets_received_size: f64,
total_acks_received: f64,
total_acks_received_size: f64,
real_acks_received: f64,
real_acks_received_size: f64,
cover_acks_received: f64,
cover_acks_received_size: f64,
real_packets_queued: f64,
retransmissions_queued: f64,
reply_surbs_queued: f64,
additional_reply_surbs_queued: f64,
}
impl From<PacketStatistics> for PacketRates {
fn from(stats: PacketStatistics) -> Self {
Self {
real_packets_sent: stats.real_packets_sent as f64,
real_packets_sent_size: stats.real_packets_sent_size as f64,
cover_packets_sent: stats.cover_packets_sent as f64,
cover_packets_sent_size: stats.cover_packets_sent_size as f64,
real_packets_received: stats.real_packets_received as f64,
real_packets_received_size: stats.real_packets_received_size as f64,
cover_packets_received: stats.cover_packets_received as f64,
cover_packets_received_size: stats.cover_packets_received_size as f64,
total_acks_received: stats.total_acks_received as f64,
total_acks_received_size: stats.total_acks_received_size as f64,
real_acks_received: stats.real_acks_received as f64,
real_acks_received_size: stats.real_acks_received_size as f64,
cover_acks_received: stats.cover_acks_received as f64,
cover_acks_received_size: stats.cover_acks_received_size as f64,
real_packets_queued: stats.real_packets_queued as f64,
retransmissions_queued: stats.retransmissions_queued as f64,
reply_surbs_queued: stats.reply_surbs_queued as f64,
additional_reply_surbs_queued: stats.additional_reply_surbs_queued as f64,
}
}
}
impl std::ops::Sub for PacketRates {
type Output = Self;
fn sub(self, rhs: Self) -> Self::Output {
Self {
real_packets_sent: self.real_packets_sent - rhs.real_packets_sent,
real_packets_sent_size: self.real_packets_sent_size - rhs.real_packets_sent_size,
cover_packets_sent: self.cover_packets_sent - rhs.cover_packets_sent,
cover_packets_sent_size: self.cover_packets_sent_size - rhs.cover_packets_sent_size,
real_packets_received: self.real_packets_received - rhs.real_packets_received,
real_packets_received_size: self.real_packets_received_size
- rhs.real_packets_received_size,
cover_packets_received: self.cover_packets_received - rhs.cover_packets_received,
cover_packets_received_size: self.cover_packets_received_size
- rhs.cover_packets_received_size,
total_acks_received: self.total_acks_received - rhs.total_acks_received,
total_acks_received_size: self.total_acks_received_size - rhs.total_acks_received_size,
real_acks_received: self.real_acks_received - rhs.real_acks_received,
real_acks_received_size: self.real_acks_received_size - rhs.real_acks_received_size,
cover_acks_received: self.cover_acks_received - rhs.cover_acks_received,
cover_acks_received_size: self.cover_acks_received_size - rhs.cover_acks_received_size,
real_packets_queued: self.real_packets_queued - rhs.real_packets_queued,
retransmissions_queued: self.retransmissions_queued - rhs.retransmissions_queued,
reply_surbs_queued: self.reply_surbs_queued - rhs.reply_surbs_queued,
additional_reply_surbs_queued: self.additional_reply_surbs_queued
- rhs.additional_reply_surbs_queued,
}
}
}
impl std::ops::Div<f64> for PacketRates {
type Output = Self;
fn div(self, rhs: f64) -> Self::Output {
Self {
real_packets_sent: self.real_packets_sent / rhs,
real_packets_sent_size: self.real_packets_sent_size / rhs,
cover_packets_sent: self.cover_packets_sent / rhs,
cover_packets_sent_size: self.cover_packets_sent_size / rhs,
real_packets_received: self.real_packets_received / rhs,
real_packets_received_size: self.real_packets_received_size / rhs,
cover_packets_received: self.cover_packets_received / rhs,
cover_packets_received_size: self.cover_packets_received_size / rhs,
total_acks_received: self.total_acks_received / rhs,
total_acks_received_size: self.total_acks_received_size / rhs,
real_acks_received: self.real_acks_received / rhs,
real_acks_received_size: self.real_acks_received_size / rhs,
cover_acks_received: self.cover_acks_received / rhs,
cover_acks_received_size: self.cover_acks_received_size / rhs,
real_packets_queued: self.real_packets_queued / rhs,
retransmissions_queued: self.retransmissions_queued / rhs,
reply_surbs_queued: self.reply_surbs_queued / rhs,
additional_reply_surbs_queued: self.additional_reply_surbs_queued / rhs,
}
}
}
impl PacketRates {
fn summary(&self) -> String {
format!(
"rx: {}/s (real: {}/s), tx: {}/s (real: {}/s)",
bibytes2(self.real_packets_received_size + self.cover_packets_received_size),
bibytes2(self.real_packets_received_size),
bibytes2(self.real_packets_sent_size + self.cover_packets_sent_size),
bibytes2(self.real_packets_sent_size),
)
}
fn detailed_summary(&self) -> String {
format!(
"RX: {:.1} mixpkt/s, {}/s (real: {}/s, acks: {}/s), TX: {:.1} mixpkt/s, {}/s (real: {}/s)",
self.real_packets_received + self.cover_packets_received,
bibytes2(self.real_packets_received_size + self.cover_packets_received_size),
bibytes2(self.real_packets_received_size),
bibytes2(self.total_acks_received_size),
self.real_packets_sent + self.cover_packets_sent,
bibytes2(self.real_packets_sent_size + self.cover_packets_sent_size),
bibytes2(self.real_packets_sent_size),
)
}
}
pub(crate) enum PacketStatisticsEvent {
// The real packets sent. Recall that acks are sent by the gateway, so it's not included here.
RealPacketSent(usize),
// The cover packets sent
CoverPacketSent(usize),
// Real packets received
RealPacketReceived(usize),
// Cover packets received
CoverPacketReceived(usize),
// Ack of any type received. This is mostly used as a consistency check, and should be the sum
// of real and cover acks received.
AckReceived(usize),
// Out of the total acks received, this is the subset of those that were real
RealAckReceived(usize),
// Out of the total acks received, this is the subset of those that were for cover traffic
CoverAckReceived(usize),
// Types of packets queued
RealPacketQueued,
RetransmissionQueued,
ReplySurbRequestQueued,
AdditionalReplySurbRequestQueued,
}
type PacketStatisticsReceiver = tokio::sync::mpsc::UnboundedReceiver<PacketStatisticsEvent>;
#[derive(Clone)]
pub(crate) struct PacketStatisticsReporter {
stats_tx: tokio::sync::mpsc::UnboundedSender<PacketStatisticsEvent>,
}
impl PacketStatisticsReporter {
pub(crate) fn new(stats_tx: tokio::sync::mpsc::UnboundedSender<PacketStatisticsEvent>) -> Self {
Self { stats_tx }
}
pub(crate) fn report(&self, event: PacketStatisticsEvent) {
self.stats_tx.send(event).unwrap_or_else(|err| {
log::error!("Failed to report packet stat: {:?}", err);
});
}
}
pub(crate) struct PacketStatisticsControl {
// Incoming packet stats events from other tasks
stats_rx: PacketStatisticsReceiver,
// Keep track of packet statistics over time
stats: PacketStatistics,
// We keep snapshots of the statistics over time so we can compute rates, and also keeping the
// full history allows for some more fancy averaging if we want to do that.
history: VecDeque<(Instant, PacketStatistics)>,
// Keep previous rates so that we can detect notable events
rates: VecDeque<(Instant, PacketRates)>,
}
impl PacketStatisticsControl {
pub(crate) fn new() -> (Self, PacketStatisticsReporter) {
let (stats_tx, stats_rx) = tokio::sync::mpsc::unbounded_channel();
(
Self {
stats_rx,
stats: PacketStatistics::default(),
history: VecDeque::new(),
rates: VecDeque::new(),
},
PacketStatisticsReporter::new(stats_tx),
)
}
// Add the current stats to the history, and remove old ones.
fn update_history(&mut self) {
// Update latest
self.history.push_back((Instant::now(), self.stats.clone()));
// Filter out old ones
let recording_window = Instant::now() - Duration::from_millis(RECORDING_WINDOW_MS);
while self
.history
.front()
.map_or(false, |&(t, _)| t < recording_window)
{
self.history.pop_front();
}
}
fn compute_rates(&self) -> Option<PacketRates> {
// NOTE: consider changing this to compute rates over the history instead of using current
// stats. Currently it should not make much of a difference since we call this just after
// updating the history, but it seems like it could be more internally consistent to do it
// that way.
// Do basic averaging over the entire history, which just uses the first and last
if let Some((start, start_stats)) = self.history.front() {
let duration_secs = Instant::now().duration_since(*start).as_secs_f64();
let delta = self.stats.clone() - start_stats.clone();
let rates = PacketRates::from(delta) / duration_secs;
Some(rates)
} else {
None
}
}
fn update_rates(&mut self) {
// Update latest
if let Some(rates) = self.compute_rates() {
self.rates.push_back((Instant::now(), rates));
}
// Filter out old ones
let recording_window = Instant::now() - Duration::from_millis(RECORDING_WINDOW_MS);
while self
.rates
.front()
.map_or(false, |&(t, _)| t < recording_window)
{
self.rates.pop_front();
}
}
fn report_rates(&self) {
if let Some((_, rates)) = self.rates.back() {
log::info!("{}", rates.summary());
log::debug!("{}", rates.detailed_summary());
}
}
fn report_counters(&self) {
log::trace!("packet statistics: {:?}", &self.stats);
let (summary_sent, summary_recv) = self.stats.summary();
log::debug!("{}", summary_sent);
log::debug!("{}", summary_recv);
}
fn check_for_notable_events(&self) {
let Some((_, latest_rates)) = self.rates.back() else {
return;
};
// If we get a burst of retransmissions
// TODO: consider making this the number of retransmissions since last report instead.
if latest_rates.retransmissions_queued > 0.0 {
log::debug!(
"retransmissions: {:.2} pkt/s",
latest_rates.retransmissions_queued
);
// Check what the number of retransmissions was during the recording window
if let Some((_, start_stats)) = self.history.front() {
let delta = self.stats.clone() - start_stats.clone();
log::info!("retransmissions: {}", delta.retransmissions_queued,);
} else {
log::warn!("Unable to check retransmissions during recording window");
}
}
// IDEA: if there is a burst of acks, that could indicate tokio task starvation.
}
pub(crate) async fn run_with_shutdown(&mut self, mut shutdown: nym_task::TaskClient) {
log::debug!("Started PacketStatisticsControl with graceful shutdown support");
let report_interval = Duration::from_secs(PACKET_REPORT_INTERVAL_SECS);
let mut report_interval = tokio::time::interval(report_interval);
let snapshot_interval = Duration::from_millis(SNAPSHOT_INTERVAL_MS);
let mut snapshot_interval = tokio::time::interval(snapshot_interval);
loop {
tokio::select! {
stats_event = self.stats_rx.recv() => match stats_event {
Some(stats_event) => {
log::trace!("PacketStatisticsControl: Received stats event");
self.stats.handle_event(stats_event);
},
None => {
log::trace!("PacketStatisticsControl: stopping since stats channel was closed");
break;
}
},
_ = snapshot_interval.tick() => {
self.update_history();
self.update_rates();
}
_ = report_interval.tick() => {
self.report_rates();
self.check_for_notable_events();
self.report_counters();
}
_ = shutdown.recv_with_delay() => {
log::trace!("PacketStatisticsControl: Received shutdown");
break;
},
}
}
log::debug!("PacketStatisticsControl: Exiting");
}
pub(crate) fn start_with_shutdown(mut self, task_client: nym_task::TaskClient) {
spawn_future(async move {
self.run_with_shutdown(task_client).await;
})
}
}
@@ -1,8 +1,6 @@
// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
use super::action_controller::{AckActionSender, Action};
use futures::StreamExt;
use log::*;
@@ -19,7 +17,6 @@ pub(super) struct AcknowledgementListener {
ack_key: Arc<AckKey>,
ack_receiver: AcknowledgementReceiver,
action_sender: AckActionSender,
stats_tx: PacketStatisticsReporter,
}
impl AcknowledgementListener {
@@ -27,21 +24,16 @@ impl AcknowledgementListener {
ack_key: Arc<AckKey>,
ack_receiver: AcknowledgementReceiver,
action_sender: AckActionSender,
stats_tx: PacketStatisticsReporter,
) -> Self {
AcknowledgementListener {
ack_key,
ack_receiver,
action_sender,
stats_tx,
}
}
async fn on_ack(&mut self, ack_content: Vec<u8>) {
trace!("Received an ack");
self.stats_tx
.report(PacketStatisticsEvent::AckReceived(ack_content.len()));
let frag_id = match recover_identifier(&self.ack_key, &ack_content)
.map(FragmentIdentifier::try_from_bytes)
{
@@ -56,14 +48,11 @@ impl AcknowledgementListener {
// because nothing was inserted in the first place
if frag_id == COVER_FRAG_ID {
trace!("Received an ack for a cover message - no need to do anything");
self.stats_tx
.report(PacketStatisticsEvent::CoverAckReceived(ack_content.len()));
return;
}
trace!("Received {} from the mix network", frag_id);
self.stats_tx
.report(PacketStatisticsEvent::RealAckReceived(ack_content.len()));
self.action_sender
.unbounded_send(Action::new_remove(frag_id))
.unwrap();
@@ -8,7 +8,6 @@ use self::{
sent_notification_listener::SentNotificationListener,
};
use crate::client::inbound_messages::InputMessageReceiver;
use crate::client::packet_statistics_control::PacketStatisticsReporter;
use crate::client::real_messages_control::message_handler::MessageHandler;
use crate::client::replies::reply_controller::ReplyControllerSender;
use crate::spawn_future;
@@ -209,7 +208,6 @@ where
connectors: AcknowledgementControllerConnectors,
message_handler: MessageHandler<R>,
reply_controller_sender: ReplyControllerSender,
stats_tx: PacketStatisticsReporter,
) -> Self {
let (retransmission_tx, retransmission_rx) = mpsc::unbounded();
@@ -226,7 +224,6 @@ where
Arc::clone(&ack_key),
connectors.ack_receiver,
connectors.ack_action_sender.clone(),
stats_tx,
);
// will listen for any new messages from the client
@@ -35,8 +35,6 @@ use crate::client::replies::reply_controller;
use crate::config;
pub(crate) use acknowledgement_control::{AckActionSender, Action};
use super::packet_statistics_control::PacketStatisticsReporter;
pub(crate) mod acknowledgement_control;
pub(crate) mod message_handler;
pub(crate) mod real_traffic_stream;
@@ -145,7 +143,6 @@ impl RealMessagesController<OsRng> {
reply_controller_receiver: ReplyControllerReceiver,
lane_queue_lengths: LaneQueueLengths,
client_connection_rx: ConnectionCommandReceiver,
stats_tx: PacketStatisticsReporter,
) -> Self {
let rng = OsRng;
@@ -184,7 +181,6 @@ impl RealMessagesController<OsRng> {
ack_controller_connectors,
message_handler.clone(),
reply_controller_sender,
stats_tx.clone(),
);
let reply_control = ReplyController::new(
@@ -203,7 +199,6 @@ impl RealMessagesController<OsRng> {
topology_access,
lane_queue_lengths,
client_connection_rx,
stats_tx,
);
RealMessagesController {
@@ -3,7 +3,6 @@
use self::sending_delay_controller::SendingDelayController;
use crate::client::mix_traffic::BatchMixMessageSender;
use crate::client::packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter};
use crate::client::real_messages_control::acknowledgement_control::SentPacketNotificationSender;
use crate::client::topology_control::TopologyAccessor;
use crate::client::transmission_buffer::TransmissionBuffer;
@@ -114,9 +113,6 @@ where
/// Report queue lengths so that upstream can backoff sending data, and keep connections open.
lane_queue_lengths: LaneQueueLengths,
/// Channel used for sending statistics events to `PacketStatisticsControl`.
stats_tx: PacketStatisticsReporter,
}
#[derive(Debug)]
@@ -175,7 +171,6 @@ where
topology_access: TopologyAccessor,
lane_queue_lengths: LaneQueueLengths,
client_connection_rx: ConnectionCommandReceiver,
stats_tx: PacketStatisticsReporter,
) -> Self {
OutQueueControl {
config,
@@ -189,7 +184,6 @@ where
transmission_buffer: TransmissionBuffer::new(),
client_connection_rx,
lane_queue_lengths,
stats_tx,
}
}
@@ -220,7 +214,7 @@ where
async fn on_message(&mut self, next_message: StreamMessage) {
trace!("created new message");
let (next_message, fragment_id, packet_size) = match next_message {
let (next_message, fragment_id) = match next_message {
StreamMessage::Cover => {
let cover_traffic_packet_size = self.loop_cover_message_size();
trace!("the next loop cover message will be put in a {cover_traffic_packet_size} packet");
@@ -256,28 +250,15 @@ where
"Somehow failed to generate a loop cover message with a valid topology",
),
None,
cover_traffic_packet_size.size(),
)
}
StreamMessage::Real(real_message) => {
let packet_size = real_message.packet_size();
(
real_message.mix_packet,
real_message.fragment_id,
packet_size,
)
(real_message.mix_packet, real_message.fragment_id)
}
};
if let Err(err) = self.mix_tx.send(vec![next_message]).await {
log::error!("Failed to send: {err}");
} else {
let event = if fragment_id.is_some() {
PacketStatisticsEvent::RealPacketSent(packet_size)
} else {
PacketStatisticsEvent::CoverPacketSent(packet_size)
};
self.stats_tx.report(event);
}
// notify ack controller about sending our message only after we actually managed to push it
@@ -359,28 +340,6 @@ where
let lane_length = self.transmission_buffer.lane_length(&lane);
self.lane_queue_lengths.set(&lane, lane_length);
// This is the last step in the pipeline where we know the type of the message, so
// lets count the number of retransmissions and reply surb messages sent here.
let stat_event = match lane {
TransmissionLane::General => None,
TransmissionLane::ConnectionId(_) => None,
TransmissionLane::ReplySurbRequest => {
Some(PacketStatisticsEvent::ReplySurbRequestQueued)
}
TransmissionLane::AdditionalReplySurbs => {
Some(PacketStatisticsEvent::AdditionalReplySurbRequestQueued)
}
TransmissionLane::Retransmission => Some(PacketStatisticsEvent::RetransmissionQueued),
};
if let Some(stat_event) = stat_event {
self.stats_tx.report(stat_event);
}
// To avoid comparing apples to oranges when presenting the fraction of packets that are
// retransmissions, we also need to keep track to the total number of real messages queued,
// even though we also track the actual number of messages sent later in the pipeline.
self.stats_tx
.report(PacketStatisticsEvent::RealPacketQueued);
Some(real_next)
}
@@ -474,13 +433,6 @@ where
Poll::Ready(Some((real_messages, conn_id))) => {
log::trace!("handling real_messages: size: {}", real_messages.len());
// This is the last step in the pipeline where we know the type of the message, so
// lets count the number of retransmissions here.
if conn_id == TransmissionLane::Retransmission {
self.stats_tx
.report(PacketStatisticsEvent::RetransmissionQueued);
}
// First store what we got for the given connection id
self.transmission_buffer.store(&conn_id, real_messages);
let real_next = self.pop_next_message().expect("we just added one");
@@ -519,10 +471,10 @@ where
let mult = self.sending_delay_controller.current_multiplier();
let delay = self.current_average_message_sending_delay().as_millis();
let status_str = if self.config.traffic.disable_main_poisson_packet_distribution {
format!("Packet backlog: {backlog:.2} kiB ({packets}), {lanes} lanes, no delay")
format!("Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), no delay")
} else {
format!(
"Packet backlog: {backlog:.2} kiB ({packets}), {lanes} lanes, avg delay: {delay}ms ({mult})"
"Status: {lanes} lanes, backlog: {backlog:.2} kiB ({packets}), avg delay: {delay}ms ({mult})"
)
};
if packets > 1000 {
@@ -1,10 +1,8 @@
// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::client::{
packet_statistics_control::{PacketStatisticsEvent, PacketStatisticsReporter},
replies::{reply_controller::ReplyControllerSender, reply_storage::SentReplyKeys},
};
use crate::client::replies::reply_controller::ReplyControllerSender;
use crate::client::replies::reply_storage::SentReplyKeys;
use crate::spawn_future;
use futures::channel::mpsc;
use futures::lock::Mutex;
@@ -45,33 +43,15 @@ struct ReceivedMessagesBufferInner<R: MessageReceiver> {
// but perhaps it should be changed to include timestamps of when the message was reconstructed
// and every now and then remove ids older than X
recently_reconstructed: HashSet<i32>,
stats_tx: PacketStatisticsReporter,
}
impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
fn recover_from_fragment(
&mut self,
fragment_data: &[u8],
fragment_data_size: usize,
) -> Option<NymMessage> {
fn recover_from_fragment(&mut self, fragment_data: &[u8]) -> Option<NymMessage> {
if nym_sphinx::cover::is_cover(fragment_data) {
trace!("The message was a loop cover message! Skipping it");
// NOTE: it's important to note that there is quite a bit of difference in size of
// received and sent packets due to the sphinx layers being removed by the exit gateway
// before it reaches the mixnet client.
self.stats_tx
.report(PacketStatisticsEvent::CoverPacketReceived(
fragment_data_size,
));
return None;
}
self.stats_tx
.report(PacketStatisticsEvent::RealPacketReceived(
fragment_data_size,
));
let fragment = match self.message_receiver.recover_fragment(fragment_data) {
Err(err) => {
warn!("failed to recover fragment from raw data: {err}. The whole underlying message might be corrupted and unrecoverable!");
@@ -123,17 +103,15 @@ impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
reply_ciphertext: &mut [u8],
reply_key: SurbEncryptionKey,
) -> Result<Option<NymMessage>, MessageRecoveryError> {
let reply_ciphertext_size = reply_ciphertext.len();
// note: this performs decryption IN PLACE without extra allocation
self.message_receiver
.recover_plaintext_from_reply(reply_ciphertext, reply_key)?;
let fragment_data = reply_ciphertext;
Ok(self.recover_from_fragment(fragment_data, reply_ciphertext_size))
Ok(self.recover_from_fragment(fragment_data))
}
fn process_received_regular_packet(&mut self, mut raw_fragment: Vec<u8>) -> Option<NymMessage> {
let raw_fragment_size = raw_fragment.len();
let fragment_data = match self.message_receiver.recover_plaintext_from_regular_packet(
self.local_encryption_keypair.private_key(),
&mut raw_fragment,
@@ -145,7 +123,7 @@ impl<R: MessageReceiver> ReceivedMessagesBufferInner<R> {
Ok(frag_data) => frag_data,
};
self.recover_from_fragment(fragment_data, raw_fragment_size)
self.recover_from_fragment(fragment_data)
}
}
@@ -163,7 +141,6 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
local_encryption_keypair: Arc<encryption::KeyPair>,
reply_key_storage: SentReplyKeys,
reply_controller_sender: ReplyControllerSender,
stats_tx: PacketStatisticsReporter,
) -> Self {
ReceivedMessagesBuffer {
inner: Arc::new(Mutex::new(ReceivedMessagesBufferInner {
@@ -172,7 +149,6 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
message_receiver: R::new(),
message_sender: None,
recently_reconstructed: HashSet::new(),
stats_tx,
})),
reply_key_storage,
reply_controller_sender,
@@ -377,7 +353,7 @@ impl<R: MessageReceiver> ReceivedMessagesBuffer<R> {
};
if let Some(completed) = completed_message {
debug!("received {completed}");
info!("received {completed}");
completed_messages.push(completed)
}
}
@@ -504,13 +480,11 @@ impl<R: MessageReceiver + Clone + Send + 'static> ReceivedMessagesBufferControll
mixnet_packet_receiver: MixnetMessageReceiver,
reply_key_storage: SentReplyKeys,
reply_controller_sender: ReplyControllerSender,
packet_statistics_reporter: PacketStatisticsReporter,
) -> Self {
let received_buffer = ReceivedMessagesBuffer::new(
local_encryption_keypair,
reply_key_storage,
reply_controller_sender,
packet_statistics_reporter,
);
ReceivedMessagesBufferController {
+1 -1
View File
@@ -65,7 +65,7 @@ pub async fn current_gateways<R: Rng>(
.ok_or(ClientCoreError::ListOfNymApisIsEmpty)?;
let client = nym_validator_client::client::NymApiClient::new(nym_api.clone());
log::debug!("Fetching list of gateways from: {nym_api}");
log::trace!("Fetching list of gateways from: {nym_api}");
let gateways = client.get_cached_described_gateways().await?;
log::debug!("Found {} gateways", gateways.len());
+1 -1
View File
@@ -178,7 +178,7 @@ impl<T> From<PersistedGatewayDetails<T>> for GatewayDetails<T> {
}
}
#[derive(Clone, Debug)]
#[derive(Clone)]
pub enum GatewaySelectionSpecification<T = EmptyCustomDetails> {
/// Uniformly choose a random remote gateway.
UniformRemote { must_use_tls: bool },
@@ -42,14 +42,6 @@ pub struct Config {
nyxd_config: nyxd::Config,
}
impl TryFrom<NymNetworkDetails> for Config {
type Error = ValidatorClientError;
fn try_from(value: NymNetworkDetails) -> Result<Self, Self::Error> {
Config::try_from_nym_network_details(&value)
}
}
impl Config {
pub fn try_from_nym_network_details(
details: &NymNetworkDetails,
@@ -5,24 +5,21 @@ use crate::nym_api::error::NymAPIError;
use crate::nym_api::routes::{CORE_STATUS_COUNT, SINCE_ARG};
use async_trait::async_trait;
use http_api_client::{ApiClient, NO_PARAMS};
pub use nym_api_requests::{
coconut::{
models::{
EpochCredentialsResponse, IssuedCredential, IssuedCredentialBody,
IssuedCredentialResponse, IssuedCredentialsResponse,
},
BlindSignRequestBody, BlindedSignatureResponse, CredentialsRequestBody,
VerifyCredentialBody, VerifyCredentialResponse,
},
models::{
ComputeRewardEstParam, DescribedGateway, GatewayBondAnnotated, GatewayCoreStatusResponse,
GatewayStatusReportResponse, GatewayUptimeHistoryResponse, InclusionProbabilityResponse,
MixNodeBondAnnotated, MixnodeCoreStatusResponse, MixnodeStatusReportResponse,
MixnodeStatusResponse, MixnodeUptimeHistoryResponse, RewardEstimationResponse,
StakeSaturationResponse, UptimeResponse,
},
use nym_api_requests::coconut::models::{
EpochCredentialsResponse, IssuedCredentialResponse, IssuedCredentialsResponse,
};
pub use nym_coconut_dkg_common::types::EpochId;
use nym_api_requests::coconut::{
BlindSignRequestBody, BlindedSignatureResponse, CredentialsRequestBody, VerifyCredentialBody,
VerifyCredentialResponse,
};
use nym_api_requests::models::{
ComputeRewardEstParam, DescribedGateway, GatewayBondAnnotated, GatewayCoreStatusResponse,
GatewayStatusReportResponse, GatewayUptimeHistoryResponse, InclusionProbabilityResponse,
MixNodeBondAnnotated, MixnodeCoreStatusResponse, MixnodeStatusReportResponse,
MixnodeStatusResponse, MixnodeUptimeHistoryResponse, RewardEstimationResponse,
StakeSaturationResponse, UptimeResponse,
};
use nym_coconut_dkg_common::types::EpochId;
use nym_mixnet_contract_common::mixnode::MixNodeDetails;
use nym_mixnet_contract_common::{GatewayBond, IdentityKeyRef, MixId};
use nym_name_service_common::response::NamesListResponse;
@@ -8,8 +8,6 @@ use cosmwasm_std::{Fraction, Uint128};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::ops::Div;
use std::str::FromStr;
use thiserror::Error;
#[derive(Serialize, Deserialize, Clone, Copy, Default, Debug, PartialEq, Eq)]
pub struct MismatchedDenoms;
@@ -128,37 +126,6 @@ impl From<CosmWasmCoin> for Coin {
}
}
// unfortunately cosmwasm didn't re-export this correct so we just redefine its
#[derive(Error, Debug, PartialEq, Eq)]
pub enum CoinFromStrError {
#[error("Missing denominator")]
MissingDenom,
#[error("Missing amount or non-digit characters in amount")]
MissingAmount,
#[error("Invalid amount: {0}")]
InvalidAmount(#[from] std::num::ParseIntError),
}
impl FromStr for Coin {
type Err = CoinFromStrError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let pos = s
.find(|c: char| !c.is_ascii_digit())
.ok_or(CoinFromStrError::MissingDenom)?;
let (amount, denom) = s.split_at(pos);
if amount.is_empty() {
return Err(CoinFromStrError::MissingAmount);
}
Ok(Coin {
amount: amount.parse::<u128>()?,
denom: denom.to_string(),
})
}
}
pub trait CoinConverter {
type Target;
@@ -7,12 +7,12 @@ use crate::nyxd::error::NyxdError;
use crate::nyxd::CosmWasmClient;
use async_trait::async_trait;
use cosmrs::AccountId;
use nym_coconut_dkg_common::{
dealer::{ContractDealing, DealerDetailsResponse, PagedDealerResponse, PagedDealingsResponse},
msg::QueryMsg as DkgQueryMsg,
types::{DealerDetails, Epoch, EpochId, InitialReplacementData},
verification_key::{ContractVKShare, PagedVKSharesResponse},
use nym_coconut_dkg_common::dealer::{
ContractDealing, DealerDetailsResponse, PagedDealerResponse, PagedDealingsResponse,
};
use nym_coconut_dkg_common::msg::QueryMsg as DkgQueryMsg;
use nym_coconut_dkg_common::types::{DealerDetails, Epoch, EpochId, InitialReplacementData};
use nym_coconut_dkg_common::verification_key::{ContractVKShare, PagedVKSharesResponse};
use serde::Deserialize;
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
@@ -52,6 +52,10 @@ use wasmtimer::tokio::sleep;
pub const DEFAULT_BROADCAST_POLLING_RATE: Duration = Duration::from_secs(4);
pub const DEFAULT_BROADCAST_TIMEOUT: Duration = Duration::from_secs(60);
#[cfg(feature = "http-client")]
#[async_trait]
impl CosmWasmClient for cosmrs::rpc::HttpClient {}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
pub trait CosmWasmClient: TendermintRpcClient {
@@ -518,7 +522,3 @@ pub trait CosmWasmClient: TendermintRpcClient {
res.try_into()
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl<T> CosmWasmClient for T where T: TendermintRpcClient {}
@@ -425,7 +425,7 @@ where
amount: amount.into_iter().map(Into::into).collect(),
}
.to_any()
.map_err(|_| NyxdError::SerializationError("MsgSend".to_owned()))
.map_err(|_| NyxdError::SerializationError("MsgExecuteContract".to_owned()))
})
.collect::<Result<_, _>>()?;
@@ -1,7 +1,7 @@
// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::nyxd::cosmwasm_client::client_traits::SigningCosmWasmClient;
use crate::nyxd::cosmwasm_client::client_traits::{CosmWasmClient, SigningCosmWasmClient};
use crate::nyxd::error::NyxdError;
use crate::nyxd::{Config, GasPrice, Hash, Height};
use crate::rpc::TendermintRpcClient;
@@ -26,7 +26,6 @@ use cosmrs::rpc::{HttpClient, HttpClientUrl};
pub mod client_traits;
mod helpers;
pub mod logs;
pub mod module_traits;
pub mod types;
#[derive(Debug)]
@@ -330,6 +329,14 @@ where
}
}
#[async_trait]
impl<C, S> CosmWasmClient for MaybeSigningClient<C, S>
where
C: TendermintRpcClient + Send + Sync,
S: Send + Sync,
{
}
#[async_trait]
impl<C, S> SigningCosmWasmClient for MaybeSigningClient<C, S>
where
@@ -1,8 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
pub mod slashing;
pub mod staking;
pub use staking::query::StakingQueryClient;
// pub use slashing::query
@@ -1,4 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
pub mod query;
@@ -1,2 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
@@ -1,8 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
pub mod query;
pub use cosmrs::staking::{
QueryHistoricalInfoResponse, QueryValidatorResponse, QueryValidatorsResponse, Validator,
};
@@ -1,78 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use super::{QueryHistoricalInfoResponse, QueryValidatorResponse, QueryValidatorsResponse};
use crate::nyxd::error::NyxdError;
use crate::nyxd::{CosmWasmClient, PageRequest};
use async_trait::async_trait;
use cosmrs::proto::cosmos::staking::v1beta1::{
QueryHistoricalInfoRequest as ProtoQueryHistoricalInfoRequest,
QueryHistoricalInfoResponse as ProtoQueryHistoricalInfoResponse,
QueryValidatorRequest as ProtoQueryValidatorRequest,
QueryValidatorResponse as ProtoQueryValidatorResponse,
QueryValidatorsRequest as ProtoQueryValidatorsRequest,
QueryValidatorsResponse as ProtoQueryValidatorsResponse,
};
use cosmrs::staking::{QueryHistoricalInfoRequest, QueryValidatorRequest, QueryValidatorsRequest};
use cosmrs::AccountId;
// TODO: change trait restriction from `CosmWasmClient` to `TendermintRpcClient`
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
pub trait StakingQueryClient: CosmWasmClient {
async fn historical_info(&self, height: i64) -> Result<QueryHistoricalInfoResponse, NyxdError> {
let path = Some("/cosmos.staking.v1beta1.Query/HistoricalInfo".to_owned());
let req = QueryHistoricalInfoRequest { height };
let res = self
.make_abci_query::<ProtoQueryHistoricalInfoRequest, ProtoQueryHistoricalInfoResponse>(
path,
req.into(),
)
.await?;
Ok(res.try_into()?)
}
async fn validator(
&self,
validator_addr: AccountId,
) -> Result<QueryValidatorResponse, NyxdError> {
let path = Some("/cosmos.staking.v1beta1.Query/Validator".to_owned());
let req = QueryValidatorRequest { validator_addr };
let res = self
.make_abci_query::<ProtoQueryValidatorRequest, ProtoQueryValidatorResponse>(
path,
req.into(),
)
.await?;
Ok(res.try_into()?)
}
async fn validators(
&self,
status: String,
pagination: Option<PageRequest>,
) -> Result<QueryValidatorsResponse, NyxdError> {
let path = Some("/cosmos.staking.v1beta1.Query/Validators".to_owned());
let req = QueryValidatorsRequest { status, pagination };
let res = self
.make_abci_query::<ProtoQueryValidatorsRequest, ProtoQueryValidatorsResponse>(
path,
req.into(),
)
.await?;
Ok(res.try_into()?)
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl<T> StakingQueryClient for T where T: CosmWasmClient {}
@@ -29,30 +29,23 @@ use tendermint_rpc::endpoint::*;
use tendermint_rpc::{Error as TendermintRpcError, Order};
use url::Url;
pub use crate::nyxd::{
cosmwasm_client::{
client_traits::{CosmWasmClient, SigningCosmWasmClient},
module_traits::{self, StakingQueryClient},
},
fee::Fee,
};
pub use crate::nyxd::cosmwasm_client::client_traits::{CosmWasmClient, SigningCosmWasmClient};
pub use crate::nyxd::fee::Fee;
pub use crate::rpc::TendermintRpcClient;
pub use coin::Coin;
pub use cosmrs::{
bank::MsgSend,
bip32,
crypto::PublicKey,
query::{PageRequest, PageResponse},
tendermint::{
abci::{response::DeliverTx, types::ExecTxResult, Event, EventAttribute},
block::Height,
hash::{self, Algorithm, Hash},
validator::Info as TendermintValidatorInfo,
Time as TendermintTime,
},
tx::{self, Msg},
AccountId, Any, Coin as CosmosCoin, Denom, Gas,
pub use cosmrs::bank::MsgSend;
pub use cosmrs::tendermint::abci::{
response::DeliverTx, types::ExecTxResult, Event, EventAttribute,
};
pub use cosmrs::tendermint::block::Height;
pub use cosmrs::tendermint::hash::{self, Algorithm, Hash};
pub use cosmrs::tendermint::validator::Info as TendermintValidatorInfo;
pub use cosmrs::tendermint::Time as TendermintTime;
pub use cosmrs::tx::Msg;
pub use cosmrs::tx::{self};
pub use cosmrs::Coin as CosmosCoin;
pub use cosmrs::Gas;
pub use cosmrs::{bip32, AccountId, Denom};
pub use cosmwasm_std::Coin as CosmWasmCoin;
pub use cw2;
pub use cw3;
@@ -62,8 +55,9 @@ pub use fee::{gas_price::GasPrice, GasAdjustable, GasAdjustment};
pub use tendermint_rpc::{
endpoint::{tx::Response as TxResponse, validators::Response as ValidatorResponse},
query::Query,
Paging, Request, Response, SimpleRequest,
Paging,
};
pub use tendermint_rpc::{Request, Response, SimpleRequest};
#[cfg(feature = "http-client")]
use crate::http_client;
@@ -103,14 +97,6 @@ impl Config {
}
}
impl TryFrom<NymNetworkDetails> for Config {
type Error = NyxdError;
fn try_from(value: NymNetworkDetails) -> Result<Self, Self::Error> {
Config::try_from_nym_network_details(&value)
}
}
#[derive(Debug)]
pub struct NyxdClient<C, S = NoSigner> {
client: MaybeSigningClient<C, S>,
@@ -742,7 +728,7 @@ where
where
H: Into<Height> + Send,
{
TendermintRpcClient::validators(&self.client, height, paging).await
self.client.validators(height, paging).await
}
async fn latest_consensus_params(
@@ -817,6 +803,14 @@ where
}
}
#[async_trait]
impl<C, S> CosmWasmClient for NyxdClient<C, S>
where
C: TendermintRpcClient + Send + Sync,
S: Send + Sync,
{
}
impl<C, S> OfflineSigner for NyxdClient<C, S>
where
S: OfflineSigner,
+1 -1
View File
@@ -73,7 +73,7 @@ where
P: AsRef<Path>,
{
let path = path.as_ref();
log::info!("saving config file to {}", path.display());
log::debug!("trying to save config file to {}", path.display());
if let Some(parent) = path.parent() {
create_dir_all(parent)?;
@@ -226,8 +226,4 @@ impl EpochState {
pub fn is_final(&self) -> bool {
*self == EpochState::InProgress
}
pub fn is_in_progress(&self) -> bool {
matches!(self, EpochState::InProgress)
}
}
+3 -3
View File
@@ -14,14 +14,14 @@ thiserror = { workspace = true }
tokio = { version = "1.24.1", features = ["sync"]}
[target."cfg(not(target_arch = \"wasm32\"))".dependencies.sqlx]
workspace = true
version = "0.5"
features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"]
[target."cfg(not(target_arch = \"wasm32\"))".dependencies.tokio]
workspace = true
version = "1.24.1"
features = [ "rt-multi-thread", "net", "signal", "fs" ]
[build-dependencies]
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
tokio = { version = "1.24.1", features = ["rt-multi-thread", "macros"] }
-63
View File
@@ -81,15 +81,6 @@ ExitPolicy accept6 *6:119
ExitPolicy accept *4:120
ExitPolicy reject6 [FC00::]/7:*
# Portless
ExitPolicy accept *:0
ExitPolicy accept *4:0
ExitPolicy accept *6:0
ExitPolicy reject *:0
ExitPolicy reject *4:0
ExitPolicy reject *6:0
#ExitPolicy accept *:8080 #and another comment here
ExitPolicy reject FE80:0000:0000:0000:0202:B3FF:FE1E:8329:*
@@ -193,60 +184,6 @@ ExitPolicy reject *:*
},
);
// ExitPolicy accept *:0
expected.push(
Accept,
AddressPortPattern {
ip_pattern: IpPattern::Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy accept *4:0
expected.push(
Accept,
AddressPortPattern {
ip_pattern: IpPattern::V4Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy accept *6:0
expected.push(
Accept,
AddressPortPattern {
ip_pattern: IpPattern::V6Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy reject *:0
expected.push(
Reject,
AddressPortPattern {
ip_pattern: IpPattern::Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy reject *4:0
expected.push(
Reject,
AddressPortPattern {
ip_pattern: IpPattern::V4Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy reject *6:0
expected.push(
Reject,
AddressPortPattern {
ip_pattern: IpPattern::V6Star,
ports: PortRange::new_zero(),
},
);
// ExitPolicy FE80:0000:0000:0000:0202:B3FF:FE1E:8329:*
expected.push(
Reject,
+14 -37
View File
@@ -264,13 +264,7 @@ mod stringified_ip_pattern {
impl AddressPortPattern {
/// Return true iff this pattern matches a given address and port.
pub fn matches(&self, addr: &IpAddr, port: u16) -> bool {
// For backward compatibility, we treat port 0 as a wildcard until all gateways have
// upgraded, at which point we can add *:0 to the policy list.
if port == 0 {
self.ip_pattern.matches(addr)
} else {
self.ip_pattern.matches(addr) && self.ports.contains(port)
}
self.ip_pattern.matches(addr) && self.ports.contains(port)
}
/// As matches, but accept a SocketAddr.
@@ -401,9 +395,19 @@ fn parse_addr(s: &str) -> Result<IpAddr, PolicyError> {
})
}
/// Helper: try to parse a port making sure it's non-zero
fn parse_port(s: &str) -> Result<u16, PolicyError> {
s.parse::<u16>()
.map_err(|_| PolicyError::InvalidPort { raw: s.to_string() })
let port = s
.parse::<u16>()
.map_err(|_| PolicyError::InvalidPort { raw: s.to_string() })?;
if port == 0 {
Err(PolicyError::InvalidPort {
raw: port.to_string(),
})
} else {
Ok(port)
}
}
impl FromStr for IpPattern {
@@ -490,10 +494,6 @@ impl PortRange {
PortRange::new_unchecked(1, 65535)
}
pub fn new_zero() -> Self {
PortRange { start: 0, end: 0 }
}
/// Create a new PortRange.
///
/// The Portrange contains all ports between `start` and `end` inclusive.
@@ -574,7 +574,6 @@ mod test {
check("marzipan:80");
check("1.2.3.4:90-80");
check("1.2.3.4:0-80");
check("1.2.3.4/100:8888");
check("[1.2.3.4]/16:80");
check("[::1]/130:8888");
@@ -613,22 +612,6 @@ mod test {
check("0.0.0.0/0:*", &["127.0.0.1:80"], &["[f00b::]:80"]);
check("[::]/0:*", &["[f00b::]:80"], &["127.0.0.1:80"]);
check(
"*:0",
&["1.2.3.4:0", "[::1]:0", "9.0.0.0:0"],
&["1.2.3.4:443", "[::1]:500", "9.0.0.0:80", "[::1]:80"],
);
check(
"*4:0",
&["1.2.3.4:0", "9.0.0.0:0"],
&["1.2.3.4:443", "9.0.0.0:80", "[::1]:0", "[::1]:80"],
);
check(
"*6:0",
&["[::1]:0"],
&["[::1]:80", "1.2.3.4:0", "1.2.3.4:443"],
);
}
#[test]
@@ -637,7 +620,6 @@ mod test {
policy.push(AddressPolicyAction::Accept, "*:443".parse()?);
policy.push(AddressPolicyAction::Accept, "[::1]:80".parse()?);
policy.push(AddressPolicyAction::Reject, "*:80".parse()?);
policy.push(AddressPolicyAction::Accept, "*:0".parse()?);
let policy = policy; // drop mut
assert!(policy
@@ -658,9 +640,6 @@ mod test {
assert!(policy
.allows_sockaddr(&"127.0.0.1:66".parse().unwrap())
.is_none());
assert!(policy
.allows_sockaddr(&"127.0.0.1:0".parse().unwrap())
.unwrap());
Ok(())
}
@@ -693,6 +672,7 @@ mod test {
assert_eq!("*".parse::<PortRange>().unwrap(), PortRange::new_all());
assert!("hello".parse::<PortRange>().is_err());
assert!("0".parse::<PortRange>().is_err());
assert!("65536".parse::<PortRange>().is_err());
assert!("65537".parse::<PortRange>().is_err());
assert!("1-2-3".parse::<PortRange>().is_err());
@@ -700,9 +680,6 @@ mod test {
assert!("1-".parse::<PortRange>().is_err());
assert!("-2".parse::<PortRange>().is_err());
assert!("-".parse::<PortRange>().is_err());
assert_eq!("0".parse::<PortRange>().unwrap(), PortRange::new_zero(),);
assert!("0-1".parse::<PortRange>().is_err());
}
#[test]
-1
View File
@@ -11,7 +11,6 @@ repository.workspace = true
cfg-if = { workspace = true }
dotenvy = { workspace = true }
hex-literal = "0.3.3"
log = { workspace = true }
once_cell = { workspace = true }
schemars = { workspace = true, features = ["preserve_order"] }
serde = { workspace = true, features = ["derive"]}
+4 -38
View File
@@ -111,7 +111,6 @@ impl NymNetworkDetails {
.with_additional_validator_endpoint(ValidatorDetails::new(
var(var_names::NYXD).expect("nyxd validator not set"),
Some(var(var_names::NYM_API).expect("nym api not set")),
get_optional_env(var_names::NYXD_WEBSOCKET),
))
.with_mixnet_contract(Some(
var(var_names::MIXNET_CONTRACT_ADDRESS).expect("mixnet contract not set"),
@@ -341,9 +340,6 @@ impl DenomDetailsOwned {
pub struct ValidatorDetails {
// it is assumed those values are always valid since they're being provided in our defaults file
pub nyxd_url: String,
//
pub websocket_url: Option<String>,
// Right now api_url is optional as we are not running the api reliably on all validators
// however, later on it should be a mandatory field
pub api_url: Option<String>,
@@ -351,10 +347,9 @@ pub struct ValidatorDetails {
}
impl ValidatorDetails {
pub fn new<S: Into<String>>(nyxd_url: S, api_url: Option<S>, websocket_url: Option<S>) -> Self {
pub fn new<S: Into<String>>(nyxd_url: S, api_url: Option<S>) -> Self {
ValidatorDetails {
nyxd_url: nyxd_url.into(),
websocket_url: websocket_url.map(Into::into),
api_url: api_url.map(Into::into),
}
}
@@ -362,7 +357,6 @@ impl ValidatorDetails {
pub fn new_nyxd_only<S: Into<String>>(nyxd_url: S) -> Self {
ValidatorDetails {
nyxd_url: nyxd_url.into(),
websocket_url: None,
api_url: None,
}
}
@@ -378,12 +372,6 @@ impl ValidatorDetails {
.as_ref()
.map(|url| url.parse().expect("the provided api url is invalid!"))
}
pub fn websocket_url(&self) -> Option<Url> {
self.websocket_url
.as_ref()
.map(|url| url.parse().expect("the provided websocket url is invalid!"))
}
}
fn fix_deprecated_environmental_variables() {
@@ -400,25 +388,11 @@ fn fix_deprecated_environmental_variables() {
}
}
// Read the variables from the file and log what the corresponding values in the environment are.
fn print_env_vars_with_keys_in_file<P: AsRef<Path> + Copy>(config_env_file: P) {
let items = dotenvy::from_path_iter(config_env_file)
.expect("Invalid path to environment configuration file");
for item in items {
let (key, val) = item.expect("Invalid item in environment configuration file");
log::debug!("{}: {}", key, val);
}
}
pub fn setup_env<P: AsRef<Path>>(config_env_file: Option<P>) {
match std::env::var(var_names::CONFIGURED) {
// if the configuration is not already set in the env vars
Err(std::env::VarError::NotPresent) => {
if let Some(config_env_file) = &config_env_file {
log::debug!(
"Loading environment variables from {:?}",
config_env_file.as_ref()
);
if let Some(config_env_file) = config_env_file {
dotenvy::from_path(config_env_file)
.expect("Invalid path to environment configuration file");
fix_deprecated_environmental_variables();
@@ -426,25 +400,17 @@ pub fn setup_env<P: AsRef<Path>>(config_env_file: Option<P>) {
// if nothing is set, the use mainnet defaults
// if the user has not set `CONFIGURED`, then even if they set any of the env variables,
// overwrite them
log::debug!("Loading mainnet defaults");
crate::mainnet::export_to_env();
}
}
Err(_) => {
log::debug!("Environment variables already set. Using them");
crate::mainnet::export_to_env()
}
Err(_) => crate::mainnet::export_to_env(),
_ => {
fix_deprecated_environmental_variables();
}
}
// if we haven't explicitly defined any of the constants, fallback to defaults
crate::mainnet::export_to_env_if_not_set();
if let Some(config_env_file) = &config_env_file {
print_env_vars_with_keys_in_file(config_env_file);
}
crate::mainnet::export_to_env_if_not_set()
}
// Name of the event triggered by the eth contract. If the event name is changed,
+1 -15
View File
@@ -3,7 +3,6 @@
use crate::var_names;
use crate::{DenomDetails, ValidatorDetails};
use std::str::FromStr;
pub const NETWORK_NAME: &str = "mainnet";
@@ -26,7 +25,6 @@ pub const REWARDING_VALIDATOR_ADDRESS: &str = "n10yyd98e2tuwu0f7ypz9dy3hhjw7v772
pub const STATISTICS_SERVICE_DOMAIN_ADDRESS: &str = "https://mainnet-stats.nymte.ch:8090/";
pub const NYXD_URL: &str = "https://rpc.nymtech.net";
pub const NYM_API: &str = "https://validator.nymtech.net/api/";
pub const NYXD_WS: &str = "wss://rpc.nymtech.net/websocket";
pub const EXPLORER_API: &str = "https://explorer.nymtech.net/api/";
// I'm making clippy mad on purpose, because that url HAS TO be updated and deployed before merging
@@ -34,11 +32,7 @@ pub const EXIT_POLICY_URL: &str =
"https://nymtech.net/.wellknown/network-requester/exit-policy.txt";
pub(crate) fn validators() -> Vec<ValidatorDetails> {
vec![ValidatorDetails::new(
NYXD_URL,
Some(NYM_API),
Some(NYXD_WS),
)]
vec![ValidatorDetails::new(NYXD_URL, Some(NYM_API))]
}
const DEFAULT_SUFFIX: &str = "_MAINNET_DEFAULT";
@@ -66,12 +60,6 @@ pub fn read_var_if_not_default(var: &str) -> Option<String> {
}
}
pub fn read_parsed_var_if_not_default<T: FromStr>(var: &str) -> Option<Result<T, T::Err>> {
read_var_if_not_default(var)
.as_deref()
.map(FromStr::from_str)
}
pub fn export_to_env() {
set_var_to_default(var_names::CONFIGURED, "true");
set_var_to_default(var_names::NETWORK_NAME, NETWORK_NAME);
@@ -116,7 +104,6 @@ pub fn export_to_env() {
);
set_var_to_default(var_names::NYXD, NYXD_URL);
set_var_to_default(var_names::NYM_API, NYM_API);
set_var_to_default(var_names::NYXD_WEBSOCKET, NYXD_WS);
set_var_to_default(var_names::EXPLORER_API, EXPLORER_API);
set_var_to_default(var_names::EXIT_POLICY_URL, EXIT_POLICY_URL);
}
@@ -165,7 +152,6 @@ pub fn export_to_env_if_not_set() {
);
set_var_conditionally_to_default(var_names::NYXD, NYXD_URL);
set_var_conditionally_to_default(var_names::NYM_API, NYM_API);
set_var_conditionally_to_default(var_names::NYXD_WEBSOCKET, NYXD_WS);
set_var_conditionally_to_default(var_names::EXPLORER_API, EXPLORER_API);
set_var_conditionally_to_default(var_names::EXIT_POLICY_URL, EXIT_POLICY_URL);
}
-1
View File
@@ -26,7 +26,6 @@ pub const SERVICE_PROVIDER_DIRECTORY_CONTRACT_ADDRESS: &str =
pub const NAME_SERVICE_CONTRACT_ADDRESS: &str = "NAME_SERVICE_CONTRACT_ADDRESS";
pub const NYXD: &str = "NYXD";
pub const NYM_API: &str = "NYM_API";
pub const NYXD_WEBSOCKET: &str = "NYXD_WS";
pub const EXPLORER_API: &str = "EXPLORER_API";
pub const EXIT_POLICY_URL: &str = "EXIT_POLICY";
+1 -1
View File
@@ -255,7 +255,7 @@ fn bench_coconut(c: &mut Criterion) {
b.iter(|| {
verify_partial_blind_signature(
&params,
blind_sign_request.get_private_attributes_pedersen_commitments(),
&blind_sign_request,
&public_attributes,
random_blind_signature,
partial_verification_key,
-2
View File
@@ -43,5 +43,3 @@ mod utils;
pub type Attribute = bls12_381::Scalar;
pub type PrivateAttribute = Attribute;
pub type PublicAttribute = Attribute;
pub use bls12_381::G1Projective;
+8 -7
View File
@@ -362,12 +362,12 @@ pub fn blind_sign(
/// The function returns `true` if the partial blind signature is valid, and `false` otherwise.
pub fn verify_partial_blind_signature(
params: &Parameters,
private_attribute_commitments: &[G1Projective],
blind_sign_request: &BlindSignRequest,
public_attributes: &[&Attribute],
blind_sig: &BlindedSignature,
partial_verification_key: &VerificationKey,
) -> bool {
let num_private_attributes = private_attribute_commitments.len();
let num_private_attributes = blind_sign_request.private_attributes_commitments.len();
if num_private_attributes + public_attributes.len() > partial_verification_key.beta_g2.len() {
return false;
}
@@ -388,7 +388,8 @@ pub fn verify_partial_blind_signature(
];
// for each private attribute, add (cm_i, beta_i) to the miller terms
for (private_attr_commit, beta_g2) in private_attribute_commitments
for (private_attr_commit, beta_g2) in blind_sign_request
.private_attributes_commitments
.iter()
.zip(&partial_verification_key.beta_g2)
{
@@ -421,7 +422,7 @@ pub fn verify_partial_blind_signature(
// is equivalent to checking e(a, b) • e(c, d)^{-1} == id
// and thus to e(a, b) • e(c^{-1}, d) == id
//
// compute e(c^{-1}, g2) • e(s, alpha) • e(cm_0, beta_0) • e(cm_i, beta_i) • (s^pub_0, beta_{i+1}) (s^pub_j, beta_{i + j})
// compute e(c^1, g2) • e(s, alpha) • e(cm_0, beta_0) • e(cm_i, beta_i) • (s^pub_0, beta_{i+1}) (s^pub_j, beta_{i + j})
multi_miller_loop(&terms_refs)
.final_exponentiation()
.is_identity()
@@ -518,7 +519,7 @@ mod tests {
assert!(verify_partial_blind_signature(
&params,
&request.private_attributes_commitments,
&request,
&public_attributes,
&blind_sig,
validator_keypair.verification_key()
@@ -538,7 +539,7 @@ mod tests {
assert!(verify_partial_blind_signature(
&params,
&request.private_attributes_commitments,
&request,
&[],
&blind_sig,
validator_keypair.verification_key()
@@ -567,7 +568,7 @@ mod tests {
// this assertion should fail, as we try to verify with a wrong validator key
assert!(!verify_partial_blind_signature(
&params,
&request.private_attributes_commitments,
&request,
&public_attributes,
&blind_sig,
validator2_keypair.verification_key()
+11 -11
View File
@@ -108,17 +108,6 @@ pub fn transpose_matrix<T: Debug>(matrix: Vec<Vec<T>>) -> Vec<Vec<T>> {
.collect::<Vec<_>>()
}
#[macro_export]
macro_rules! random_scalars_refs {
( $x: ident, $params: expr, $n: expr ) => {
let _vec = $params.n_random_scalars($n);
#[allow(clippy::map_identity)]
let $x = _vec.iter().collect::<Vec<_>>();
};
}
pub use random_scalars_refs;
#[cfg(test)]
pub mod tests {
use super::*;
@@ -181,3 +170,14 @@ pub mod tests {
.collect()
}
}
#[macro_export]
macro_rules! random_scalars_refs {
( $x: ident, $params: expr, $n: expr ) => {
let _vec = $params.n_random_scalars($n);
#[allow(clippy::map_identity)]
let $x = _vec.iter().collect::<Vec<_>>();
};
}
pub use random_scalars_refs;
+2 -9
View File
@@ -239,15 +239,8 @@ impl NymMessage {
let (packets_used, space_left) =
chunking::number_of_required_fragments(total_required_bytes, plaintext_per_packet);
let wasted_space_percentage =
(space_left as f32 / (bytes.len() + 1 + space_left) as f32) * 100.0;
log::trace!(
"Padding {self_display}: {} of raw plaintext bytes are required. \
They're going to be put into {packets_used} sphinx packets with {space_left} bytes \
of leftover space. {wasted_space_percentage:.1}% of packet capacity is going to \
be wasted.",
bytes.len() + 1
);
let wasted_space = space_left as f32 / (bytes.len() + 1 + space_left) as f32;
log::trace!("Padding {self_display}: {} of raw plaintext bytes are required. They're going to be put into {packets_used} sphinx packets with {space_left} bytes of leftover space. {wasted_space}% of packet capacity is going to be wasted.", bytes.len() + 1);
bytes
.into_iter()
-37
View File
@@ -1,37 +0,0 @@
[package]
name = "nyxd-scraper"
version = "0.1.0"
authors.workspace = true
repository.workspace = true
homepage.workspace = true
documentation.workspace = true
edition.workspace = true
license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait.workspace = true
const_format = "0.2.32"
cosmrs.workspace = true
eyre = "0.6.9"
futures.workspace = true
sha2 = "0.10.8"
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "time"] }
tendermint.workspace = true
tendermint-rpc = { workspace = true, features = ["websocket-client", "http-client"] }
thiserror.workspace = true
tokio = { workspace = true, features = ["full"] }
tokio-stream = "0.1.14"
tokio-util = { version = "0.7.10", features = ["rt"]}
tracing.workspace = true
url.workspace = true
# TEMP
nym-bin-common = { path = "../bin-common", features = ["basic_tracing"]}
[build-dependencies]
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
-28
View File
@@ -1,28 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
#[tokio::main]
async fn main() {
use sqlx::{Connection, SqliteConnection};
use std::env;
let out_dir = env::var("OUT_DIR").unwrap();
let database_path = format!("{out_dir}/scraper-example.sqlite");
let mut conn = SqliteConnection::connect(&format!("sqlite://{database_path}?mode=rwc"))
.await
.expect("Failed to create SQLx database connection");
sqlx::migrate!("./sql_migrations")
.run(&mut conn)
.await
.expect("Failed to perform SQLx migrations");
#[cfg(target_family = "unix")]
println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
#[cfg(target_family = "windows")]
// for some strange reason we need to add a leading `/` to the windows path even though it's
// not a valid windows path... but hey, it works...
println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
}
@@ -1,10 +0,0 @@
/*
* Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
* SPDX-License-Identifier: Apache-2.0
*/
CREATE TABLE METADATA
(
id INTEGER PRIMARY KEY CHECK (id = 0),
last_processed_height INTEGER NOT NULL
);
@@ -1,76 +0,0 @@
CREATE TABLE validator
(
consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */
consensus_pubkey TEXT NOT NULL UNIQUE /* Validator consensus public key */
);
CREATE TABLE pre_commit
(
validator_address TEXT NOT NULL REFERENCES validator (consensus_address),
height BIGINT NOT NULL,
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
voting_power BIGINT NOT NULL,
proposer_priority BIGINT NOT NULL,
UNIQUE (validator_address, timestamp)
);
CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address);
CREATE INDEX pre_commit_height_index ON pre_commit (height);
CREATE TABLE block
(
height BIGINT UNIQUE PRIMARY KEY,
hash TEXT NOT NULL UNIQUE,
num_txs INTEGER DEFAULT 0,
total_gas BIGINT DEFAULT 0,
proposer_address TEXT REFERENCES validator (consensus_address),
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
CREATE INDEX block_height_index ON block (height);
CREATE INDEX block_hash_index ON block (hash);
CREATE INDEX block_proposer_address_index ON block (proposer_address);
-- no JSONB in sqlite (yet) : )
CREATE TABLE "transaction"
(
hash TEXT UNIQUE NOT NULL,
height BIGINT NOT NULL REFERENCES block (height),
"index" INTEGER NOT NULL,
success BOOLEAN NOT NULL,
/* Body */
num_messages INTEGER NOT NULL,
-- messages JSONB NOT NULL,-- DEFAULT '[]'::JSONB,
memo TEXT,
-- signatures TEXT[] NOT NULL,
/* AuthInfo */
-- signer_infos JSONB NOT NULL,-- DEFAULT '[]'::JSONB,
-- fee JSONB NOT NULL,-- DEFAULT '{}'::JSONB,
/* Tx response */
gas_wanted BIGINT DEFAULT 0,
gas_used BIGINT DEFAULT 0,
raw_log TEXT
-- logs JSONB
);
CREATE INDEX transaction_hash_index ON "transaction" (hash);
CREATE INDEX transaction_height_index ON "transaction" (height);
CREATE TABLE message
(
transaction_hash TEXT NOT NULL REFERENCES "transaction" (hash),
"index" BIGINT NOT NULL,
type TEXT NOT NULL,
-- value JSONB NOT NULL,
-- involved_accounts_addresses TEXT[] NOT NULL,
height BIGINT NOT NULL,
CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, "index")
);
CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
CREATE INDEX message_type_index ON message (type);
CREATE TABLE pruning
(
last_pruned_height BIGINT NOT NULL
);
@@ -1,65 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::MAX_RANGE_SIZE;
use std::cmp::min;
use std::collections::VecDeque;
use std::ops::Range;
pub(crate) fn split_request_range(request_range: Range<u32>) -> VecDeque<Range<u32>> {
let mut requests = VecDeque::new();
let mut start = request_range.start;
let mut end = min(request_range.end, start + MAX_RANGE_SIZE as u32);
loop {
requests.push_back(start..end);
start = min(start + MAX_RANGE_SIZE as u32, request_range.end);
end = min(end + MAX_RANGE_SIZE as u32, request_range.end);
if start == end {
break;
}
}
requests
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn splitting_request_range() {
let range = 0..100;
let mut expected = VecDeque::new();
expected.push_back(0..30);
expected.push_back(30..60);
expected.push_back(60..90);
expected.push_back(90..100);
assert_eq!(expected, split_request_range(range));
let range = 0..30;
let mut expected = VecDeque::new();
expected.push_back(0..30);
assert_eq!(expected, split_request_range(range));
let range = 0..60;
let mut expected = VecDeque::new();
expected.push_back(0..30);
expected.push_back(30..60);
assert_eq!(expected, split_request_range(range));
let range = 0..5;
let mut expected = VecDeque::new();
expected.push_back(0..5);
assert_eq!(expected, split_request_range(range));
let range = 123..200;
let mut expected = VecDeque::new();
expected.push_back(123..153);
expected.push_back(153..183);
expected.push_back(183..200);
assert_eq!(expected, split_request_range(range));
}
}
@@ -1,332 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::helpers::split_request_range;
use crate::block_processor::types::BlockToProcess;
use crate::block_requester::BlockRequest;
use crate::error::ScraperError;
use crate::modules::{BlockModule, MsgModule, TxModule};
use crate::rpc_client::RpcClient;
use crate::storage::{persist_block, ScraperStorage};
use futures::StreamExt;
use std::collections::{BTreeMap, HashSet, VecDeque};
use std::ops::{Add, Range};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc::{Sender, UnboundedReceiver};
use tokio::sync::Notify;
use tokio::time::{interval_at, Instant};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
mod helpers;
pub(crate) mod types;
const MISSING_BLOCKS_CHECK_INTERVAL: Duration = Duration::from_secs(30);
const MAX_MISSING_BLOCKS_DELAY: Duration = Duration::from_secs(15);
const MAX_RANGE_SIZE: usize = 30;
#[derive(Debug, Default)]
struct PendingSync {
request_in_flight: HashSet<u32>,
queued_requests: VecDeque<Range<u32>>,
}
impl PendingSync {
fn is_empty(&self) -> bool {
self.request_in_flight.is_empty() && self.queued_requests.is_empty()
}
}
pub struct BlockProcessor {
cancel: CancellationToken,
synced: Arc<Notify>,
last_processed_height: u32,
last_processed_at: Instant,
pending_sync: PendingSync,
queued_blocks: BTreeMap<u32, BlockToProcess>,
rpc_client: RpcClient,
incoming: UnboundedReceiverStream<BlockToProcess>,
block_requester: Sender<BlockRequest>,
storage: ScraperStorage,
// future work: rather than sending each msg to every msg module,
// let them subscribe based on `type_url` inside the message itself
// (like "/cosmwasm.wasm.v1.MsgExecuteContract")
block_modules: Vec<Box<dyn BlockModule + Send>>,
tx_modules: Vec<Box<dyn TxModule + Send>>,
msg_modules: Vec<Box<dyn MsgModule + Send>>,
}
impl BlockProcessor {
pub async fn new(
cancel: CancellationToken,
synced: Arc<Notify>,
incoming: UnboundedReceiver<BlockToProcess>,
block_requester: Sender<BlockRequest>,
storage: ScraperStorage,
rpc_client: RpcClient,
) -> Result<Self, ScraperError> {
let last_processed = storage.get_last_processed_height().await?;
Ok(BlockProcessor {
cancel,
synced,
last_processed_height: last_processed.try_into().unwrap_or_default(),
last_processed_at: Instant::now(),
pending_sync: Default::default(),
queued_blocks: Default::default(),
rpc_client,
incoming: incoming.into(),
block_requester,
storage,
block_modules: vec![],
tx_modules: vec![],
msg_modules: vec![],
})
}
async fn process_block(&mut self, block: BlockToProcess) -> Result<(), ScraperError> {
info!("processing block at height {}", block.height);
let full_info = self.rpc_client.try_get_full_details(block).await?;
debug!(
"this block has {} transaction(s)",
full_info.transactions.len()
);
for tx in &full_info.transactions {
debug!("{} has {} message(s)", tx.hash, tx.tx.body.messages.len());
for (index, msg) in tx.tx.body.messages.iter().enumerate() {
debug!("{index}: {:?}", msg.type_url)
}
}
// process the entire block as a transaction so that if anything fails,
// we won't end up with a corrupted storage.
let mut tx = self.storage.begin_processing_tx().await?;
persist_block(&full_info, &mut tx).await?;
// let the modules do whatever they want
// the ones wanting the full block:
for block_module in &mut self.block_modules {
block_module.handle_block(&full_info, &mut tx).await?;
}
// the ones wanting transactions:
for block_tx in full_info.transactions {
for tx_module in &mut self.tx_modules {
tx_module.handle_tx(&block_tx, &mut tx).await?;
}
// the ones concerned with individual messages
for (index, msg) in block_tx.tx.body.messages.iter().enumerate() {
for msg_module in &mut self.msg_modules {
msg_module
.handle_msg(index, msg, &block_tx, &mut tx)
.await?
}
}
}
tx.commit()
.await
.map_err(|source| ScraperError::StorageTxCommitFailure { source })?;
self.last_processed_height = full_info.block.header.height.value() as u32;
self.last_processed_at = Instant::now();
Ok(())
}
pub fn set_block_modules(&mut self, modules: Vec<Box<dyn BlockModule + Send>>) {
self.block_modules = modules;
}
pub fn set_tx_modules(&mut self, modules: Vec<Box<dyn TxModule + Send>>) {
self.tx_modules = modules;
}
pub fn set_msg_modules(&mut self, modules: Vec<Box<dyn MsgModule + Send>>) {
self.msg_modules = modules;
}
async fn maybe_request_missing_blocks(&mut self) -> Result<(), ScraperError> {
// we're still processing, so we're good
if self.last_processed_at.elapsed() < MAX_MISSING_BLOCKS_DELAY {
debug!("no need to request missing blocks");
return Ok(());
}
if self.try_request_pending().await {
return Ok(());
}
// TODO: properly fill in the gaps later with BlockRequest::Specific,
let request_range = if let Some((next_available, _)) = self.queued_blocks.first_key_value()
{
self.last_processed_height + 1..*next_available
} else {
let current_height = self.rpc_client.current_block_height().await? as u32;
self.last_processed_height + 1..current_height + 1
};
self.request_missing_blocks(request_range).await?;
Ok(())
}
async fn request_missing_blocks(
&mut self,
request_range: Range<u32>,
) -> Result<(), ScraperError> {
let request_range = if request_range.len() > MAX_RANGE_SIZE {
let mut split = split_request_range(request_range);
// SAFETY: we know that after the split of a non-empty range we have AT LEAST one value
#[allow(clippy::unwrap_used)]
let first = split.pop_front().unwrap();
self.pending_sync.queued_requests = split;
self.pending_sync.request_in_flight = first.clone().collect();
first
} else {
request_range
};
self.send_blocks_request(request_range).await
}
// technically we're not mutating self here,
// but we need it to help the compiler figure out the future is `Send`
async fn send_blocks_request(&mut self, request_range: Range<u32>) -> Result<(), ScraperError> {
debug!("requesting missing blocks: {request_range:?}");
self.block_requester
.send(BlockRequest::Range(request_range))
.await?;
Ok(())
}
async fn next_incoming(&mut self, block: BlockToProcess) {
let height = block.height;
self.pending_sync.request_in_flight.remove(&height);
if self.last_processed_height == 0 {
// this is the first time we've started up the process
debug!("setting up initial processing height");
self.last_processed_height = height - 1
}
if height <= self.last_processed_height {
warn!("we have already processed block for height {height}");
return;
}
if self.last_processed_height + 1 != height {
if self.queued_blocks.insert(height, block).is_some() {
warn!("we have already queued up block for height {height}");
}
return;
}
if let Err(err) = self.process_block(block).await {
error!("failed to process block at height {height}: {err}");
return;
}
// process as much as we can from the queue
let mut next = height + 1;
while let Some(next_block) = self.queued_blocks.remove(&next) {
if let Err(err) = self.process_block(next_block).await {
error!("failed to process queued-up block at height {next}: {err}")
}
next += 1;
}
self.try_request_pending().await;
if self.pending_sync.is_empty() {
self.synced.notify_one();
}
}
async fn try_request_pending(&mut self) -> bool {
if self.pending_sync.request_in_flight.is_empty() {
if let Some(next_sync) = self.pending_sync.queued_requests.pop_front() {
debug!(
"current request range has been resolved. requesting another bunch of blocks"
);
if let Err(err) = self.send_blocks_request(next_sync.clone()).await {
error!("failed to request resync blocks: {err}");
self.pending_sync.queued_requests.push_front(next_sync);
} else {
self.pending_sync.request_in_flight = next_sync.collect()
}
return true;
}
}
false
}
// technically we're not mutating self here,
// but we need it to help the compiler figure out the future is `Send`
async fn startup_resync(&mut self) -> Result<(), ScraperError> {
assert!(self.pending_sync.is_empty());
let latest_block = self.rpc_client.current_block_height().await? as u32;
if latest_block > self.last_processed_height && self.last_processed_height != 0 {
let request_range = self.last_processed_height + 1..latest_block + 1;
info!("we need to request {request_range:?} to resync");
self.request_missing_blocks(request_range).await?;
}
Ok(())
}
pub(crate) async fn run(&mut self) {
info!("starting processing loop");
// sure, we could be more efficient and reset it on every processed block,
// but the overhead is so minimal that it doesn't matter
let mut missing_check_interval = interval_at(
Instant::now().add(MISSING_BLOCKS_CHECK_INTERVAL),
MISSING_BLOCKS_CHECK_INTERVAL,
);
if let Err(err) = self.startup_resync().await {
error!("failed to perform startup sync: {err}");
self.cancel.cancel();
return;
};
loop {
tokio::select! {
_ = self.cancel.cancelled() => {
info!("received cancellation token");
break
}
_ = missing_check_interval.tick() => {
if let Err(err) = self.maybe_request_missing_blocks().await {
error!("failed to request missing blocks: {err}")
}
}
block = self.incoming.next() => {
match block {
Some(block) => self.next_incoming(block).await,
None => {
warn!("stopped receiving new blocks");
self.cancel.cancel();
break
}
}
}
}
}
}
}
@@ -1,121 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::error::ScraperError;
use crate::helpers;
use tendermint::{abci, block, tx, Block, Hash};
use tendermint_rpc::endpoint::{block as block_endpoint, block_results, validators};
use tendermint_rpc::event::{Event, EventData};
// just get all everything out of tx::Response, but parse raw `tx` bytes
#[derive(Clone, Debug)]
pub struct ParsedTransactionResponse {
/// The hash of the transaction.
///
/// Deserialized from a hex-encoded string (there is a discrepancy between
/// the format used for the request and the format used for the response in
/// the Tendermint RPC).
pub hash: Hash,
pub height: block::Height,
pub index: u32,
pub tx_result: abci::types::ExecTxResult,
pub tx: cosmrs::tx::Tx,
pub proof: Option<tx::Proof>,
}
#[derive(Debug)]
pub struct FullBlockInformation {
/// Basic block information, including its signers.
pub block: Block,
/// All of the emitted events alongside any tx results.
pub results: block_results::Response,
/// Validator set for this particular block
pub validators: validators::Response,
/// Transaction results from this particular block
pub transactions: Vec<ParsedTransactionResponse>,
}
impl FullBlockInformation {
pub fn ensure_proposer(&self) -> Result<(), ScraperError> {
let block_proposer = self.block.header.proposer_address;
if !self
.validators
.validators
.iter()
.any(|v| v.address == block_proposer)
{
let proposer = helpers::validator_consensus_address(block_proposer)?;
return Err(ScraperError::BlockProposerNotInValidatorSet {
height: self.block.header.height.value() as u32,
proposer: proposer.to_string(),
});
}
Ok(())
}
}
pub(crate) struct BlockToProcess {
pub(crate) height: u32,
pub(crate) block: Block,
}
impl From<Block> for BlockToProcess {
fn from(block: Block) -> Self {
BlockToProcess {
height: block.header.height.value() as u32,
block,
}
}
}
impl TryFrom<Event> for BlockToProcess {
type Error = ScraperError;
fn try_from(event: Event) -> Result<Self, Self::Error> {
let query = event.query.clone();
// TODO: we're losing `result_begin_block` and `result_end_block` here but maybe that's fine?
let maybe_block = match event.data {
// we don't care about `NewBlock` until CometBFT 0.38, i.e. until we upgrade to wasmd 0.50
EventData::NewBlock { .. } => {
return Err(ScraperError::InvalidSubscriptionEvent {
query,
kind: "NewBlock".to_string(),
})
}
EventData::LegacyNewBlock { block, .. } => block,
EventData::Tx { .. } => {
return Err(ScraperError::InvalidSubscriptionEvent {
query,
kind: "Tx".to_string(),
})
}
EventData::GenericJsonEvent(_) => {
return Err(ScraperError::InvalidSubscriptionEvent {
query,
kind: "GenericJsonEvent".to_string(),
})
}
};
let Some(block) = maybe_block else {
return Err(ScraperError::EmptyBlockData { query });
};
Ok((*block).into())
}
}
impl From<block_endpoint::Response> for BlockToProcess {
fn from(value: block_endpoint::Response) -> Self {
value.block.into()
}
}
@@ -1,91 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::BlockToProcess;
use crate::error::ScraperError;
use crate::rpc_client::RpcClient;
use futures::StreamExt;
use std::ops::Range;
use tokio::sync::mpsc::{Receiver, UnboundedSender};
use tokio_stream::wrappers::ReceiverStream;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, instrument, warn};
#[derive(Debug)]
pub enum BlockRequest {
Range(Range<u32>),
// UNIMPLEMENTED:
#[allow(dead_code)]
Specific(Vec<u32>),
}
pub(crate) struct BlockRequester {
cancel: CancellationToken,
rpc_client: RpcClient,
requests: ReceiverStream<BlockRequest>,
blocks: UnboundedSender<BlockToProcess>,
}
impl BlockRequester {
pub(crate) fn new(
cancel: CancellationToken,
rpc_client: RpcClient,
requests: Receiver<BlockRequest>,
blocks: UnboundedSender<BlockToProcess>,
) -> Self {
BlockRequester {
cancel,
rpc_client,
requests: requests.into(),
blocks,
}
}
async fn request_and_send(&self, height: u32) -> Result<(), ScraperError> {
let block = self.rpc_client.get_basic_block_details(height).await?;
self.blocks.send(block.into())?;
Ok(())
}
async fn request_blocks<I: IntoIterator<Item = u32>>(&self, heights: I) {
futures::stream::iter(heights)
.for_each_concurrent(4, |height| async move {
if let Err(err) = self.request_and_send(height).await {
error!("failed to request block data: {err}")
}
})
.await
}
#[instrument(skip(self))]
async fn handle_blocks_request(&self, request: BlockRequest) {
info!("received request for missed blocks");
match request {
BlockRequest::Range(range) => self.request_blocks(range).await,
BlockRequest::Specific(heights) => self.request_blocks(heights).await,
}
}
pub(crate) async fn run(&mut self) {
loop {
tokio::select! {
_ = self.cancel.cancelled() => {
info!("received cancellation token");
break
}
maybe_request = self.requests.next() => {
match maybe_request {
Some(request) => self.handle_blocks_request(request).await,
None => {
warn!("stopped receiving new requests");
self.cancel.cancel();
break
}
}
}
}
}
}
}
-47
View File
@@ -1,47 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use const_format::concatcp;
// TODO: make those configurable via 'NymNetworkDetails'
// BECH32_PREFIX defines the main SDK Bech32 prefix of an account's address
pub const BECH32_PREFIX: &str = "n";
// ACCOUNT_PREFIX is the prefix for account keys
pub const ACCOUNT_PREFIX: &str = "acc";
// VALIDATOR_PREFIX is the prefix for validator keys
pub const VALIDATOR_PREFIX: &str = "val";
// CONSENSUS_PREFIX is the prefix for consensus keys
pub const CONSENSUS_PREFIX: &str = "cons";
// PUBKEY_PREFIX is the prefix for public keys
pub const PUBKEY_PREFIX: &str = "pub";
// OPERATOR_PREFIX is the prefix for operator keys
pub const OPERATOR_PREFIX: &str = "oper";
// ADDRESS_PREFIX is the prefix for addresses
pub const ADDRESS_PREFIX: &str = "addr";
// BECH32_ACCOUNT_ADDRESS_PREFIX defines the Bech32 prefix of an account's address
pub const BECH32_ACCOUNT_ADDRESS_PREFIX: &str = BECH32_PREFIX;
// BECH32_ACCOUNT_PUBKEY_PREFIX defines the Bech32 prefix of an account's public key
pub const BECH32_ACCOUNT_PUBKEY_PREFIX: &str = concatcp!(BECH32_PREFIX, PUBKEY_PREFIX);
// BECH32_VALIDATOR_ADDRESS_PREFIX defines the Bech32 prefix of a validator's operator address
pub const BECH32_VALIDATOR_ADDRESS_PREFIX: &str =
concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, OPERATOR_PREFIX);
// BECH32_VALIDATOR_PUBKEY_PREFIX defines the Bech32 prefix of a validator's operator public key
pub const BECH32_VALIDATOR_PUBKEY_PREFIX: &str = concatcp!(
BECH32_PREFIX,
VALIDATOR_PREFIX,
OPERATOR_PREFIX,
PUBKEY_PREFIX
);
// BECH32_CONSENSUS_ADDRESS_PREFIX defines the Bech32 prefix of a consensus node address
pub const BECH32_CONSENSUS_ADDRESS_PREFIX: &str =
concatcp!(BECH32_PREFIX, VALIDATOR_PREFIX, CONSENSUS_PREFIX);
// BECH32_CONESNSUS_PUBKEY_PREFIX defines the Bech32 prefix of a consensus node public key
pub const BECH32_CONESNSUS_PUBKEY_PREFIX: &str = concatcp!(
BECH32_PREFIX,
VALIDATOR_PREFIX,
CONSENSUS_PREFIX,
PUBKEY_PREFIX
);
-131
View File
@@ -1,131 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use tendermint::Hash;
use thiserror::Error;
use tokio::sync::mpsc::error::SendError;
#[derive(Debug, Error)]
pub enum ScraperError {
#[error("experienced internal database error: {0}")]
InternalDatabaseError(#[from] sqlx::Error),
#[error("failed to perform startup SQL migration: {0}")]
StartupMigrationFailure(#[from] sqlx::migrate::MigrateError),
#[error("can't add any modules to the scraper as it's already running")]
ScraperAlreadyRunning,
#[error("failed to establish websocket connection to {url}: {source}")]
WebSocketConnectionFailure {
url: String,
#[source]
source: tendermint_rpc::Error,
},
#[error("failed to establish rpc connection to {url}: {source}")]
HttpConnectionFailure {
url: String,
#[source]
source: tendermint_rpc::Error,
},
#[error("failed to create chain subscription: {source}")]
ChainSubscriptionFailure {
#[source]
source: tendermint_rpc::Error,
},
#[error("could not obtain basic block information at height: {height}: {source}")]
BlockQueryFailure {
height: u32,
#[source]
source: tendermint_rpc::Error,
},
#[error("could not obtain block results information at height: {height}: {source}")]
BlockResultsQueryFailure {
height: u32,
#[source]
source: tendermint_rpc::Error,
},
#[error("could not obtain validators information at height: {height}: {source}")]
ValidatorsQueryFailure {
height: u32,
#[source]
source: tendermint_rpc::Error,
},
#[error("could not obtain tx results for tx: {hash}: {source}")]
TxResultsQueryFailure {
hash: Hash,
#[source]
source: tendermint_rpc::Error,
},
#[error("could not obtain current abci info: {source}")]
AbciInfoQueryFailure {
#[source]
source: tendermint_rpc::Error,
},
#[error("could not parse tx {hash}: {source}")]
TxParseFailure {
hash: Hash,
#[source]
source: cosmrs::ErrorReport,
},
#[error("received an invalid chain subscription event of kind {kind} while we were waiting for new block data (query: '{query}')")]
InvalidSubscriptionEvent { query: String, kind: String },
#[error("received block data was empty (query: '{query}')")]
EmptyBlockData { query: String },
#[error("reached maximum number of allowed errors for subscription events")]
MaximumSubscriptionFailures,
#[error("failed to begin storage tx: {source}")]
StorageTxBeginFailure {
#[source]
source: sqlx::Error,
},
#[error("failed to commit storage tx: {source}")]
StorageTxCommitFailure {
#[source]
source: sqlx::Error,
},
#[error("failed to send on a closed channel")]
ClosedChannelError,
#[error("failed to parse validator's address: {source}")]
MalformedValidatorAddress {
#[source]
source: eyre::Report,
},
#[error("failed to parse validator's address: {source}")]
MalformedValidatorPubkey {
#[source]
source: eyre::Report,
},
#[error(
"could not find the block proposer ('{proposer}') for height {height} in the validator set"
)]
BlockProposerNotInValidatorSet { height: u32, proposer: String },
#[error(
"could not find validator information for {address}; the validator has signed a commit"
)]
MissingValidatorInfoCommitted { address: String },
}
impl<T> From<SendError<T>> for ScraperError {
fn from(_: SendError<T>) -> Self {
ScraperError::ClosedChannelError
}
}
-46
View File
@@ -1,46 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::ParsedTransactionResponse;
use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX};
use crate::error::ScraperError;
use cosmrs::AccountId;
use sha2::{Digest, Sha256};
use tendermint::{account, PublicKey};
use tendermint::{validator, Hash};
use tendermint_rpc::endpoint::validators;
pub(crate) fn tx_hash<M: AsRef<[u8]>>(raw_tx: M) -> Hash {
Hash::Sha256(Sha256::digest(raw_tx).into())
}
pub(crate) fn validator_pubkey_to_bech32(pubkey: PublicKey) -> Result<AccountId, ScraperError> {
// TODO: this one seem to attach additional prefix to they pubkeys, is that what we want instead maybe?
// Ok(pubkey.to_bech32(BECH32_CONESNSUS_PUBKEY_PREFIX))
AccountId::new(BECH32_CONESNSUS_PUBKEY_PREFIX, &pubkey.to_bytes())
.map_err(|source| ScraperError::MalformedValidatorPubkey { source })
}
pub(crate) fn validator_consensus_address(id: account::Id) -> Result<AccountId, ScraperError> {
AccountId::new(BECH32_CONSENSUS_ADDRESS_PREFIX, id.as_ref())
.map_err(|source| ScraperError::MalformedValidatorAddress { source })
}
pub(crate) fn tx_gas_sum(txs: &[ParsedTransactionResponse]) -> i64 {
txs.iter().map(|tx| tx.tx_result.gas_used).sum()
}
pub(crate) fn validator_info(
id: account::Id,
validators: &validators::Response,
) -> Result<&validator::Info, ScraperError> {
match validators.validators.iter().find(|v| v.address == id) {
Some(info) => Ok(info),
None => {
let addr = validator_consensus_address(id)?;
Err(ScraperError::MissingValidatorInfoCommitted {
address: addr.to_string(),
})
}
}
}
-19
View File
@@ -1,19 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
#![warn(clippy::expect_used)]
#![warn(clippy::unwrap_used)]
pub(crate) mod block_processor;
pub(crate) mod block_requester;
pub mod constants;
pub mod error;
pub(crate) mod helpers;
pub mod modules;
pub(crate) mod rpc_client;
pub(crate) mod scraper;
pub mod storage;
pub use modules::{BlockModule, MsgModule, TxModule};
pub use scraper::{Config, NyxdScraper};
pub use storage::models;
@@ -1,16 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::FullBlockInformation;
use crate::error::ScraperError;
use crate::storage::StorageTransaction;
use async_trait::async_trait;
#[async_trait]
pub trait BlockModule {
async fn handle_block(
&mut self,
block: &FullBlockInformation,
storage_tx: &mut StorageTransaction,
) -> Result<(), ScraperError>;
}
-10
View File
@@ -1,10 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
mod block_module;
mod msg_module;
mod tx_module;
pub use block_module::BlockModule;
pub use msg_module::MsgModule;
pub use tx_module::TxModule;
@@ -1,19 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::ParsedTransactionResponse;
use crate::error::ScraperError;
use crate::storage::StorageTransaction;
use async_trait::async_trait;
use cosmrs::Any;
#[async_trait]
pub trait MsgModule {
async fn handle_msg(
&mut self,
index: usize,
msg: &Any,
tx: &ParsedTransactionResponse,
storage_tx: &mut StorageTransaction,
) -> Result<(), ScraperError>;
}
@@ -1,16 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::ParsedTransactionResponse;
use crate::error::ScraperError;
use crate::storage::StorageTransaction;
use async_trait::async_trait;
#[async_trait]
pub trait TxModule {
async fn handle_tx(
&mut self,
tx: &ParsedTransactionResponse,
storage_tx: &mut StorageTransaction,
) -> Result<(), ScraperError>;
}
-175
View File
@@ -1,175 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::{
BlockToProcess, FullBlockInformation, ParsedTransactionResponse,
};
use crate::error::ScraperError;
use crate::helpers::tx_hash;
use futures::future::join3;
use futures::StreamExt;
use std::collections::BTreeMap;
use std::sync::Arc;
use tendermint::Hash;
use tendermint_rpc::endpoint::{block, block_results, tx, validators};
use tendermint_rpc::{Client, HttpClient, Paging};
use tokio::sync::Mutex;
use tracing::{debug, instrument};
use url::Url;
#[derive(Clone)]
pub struct RpcClient {
// right now I don't care about anything nym specific, so a simple http client is sufficient,
// once this is inadequate, we can switch to a NyxdClient
inner: Arc<HttpClient>,
}
impl RpcClient {
pub fn new(url: &Url) -> Result<Self, ScraperError> {
let http_client = HttpClient::new(url.as_str()).map_err(|source| {
ScraperError::HttpConnectionFailure {
url: url.to_string(),
source,
}
})?;
Ok(RpcClient {
inner: Arc::new(http_client),
})
}
#[instrument(skip(self, block), fields(height = block.height))]
pub async fn try_get_full_details(
&self,
block: BlockToProcess,
) -> Result<FullBlockInformation, ScraperError> {
debug!("getting complete block details");
let height = block.height;
// make all the http requests concurrently
let (results, validators, raw_transactions) = join3(
self.get_block_results(height),
self.get_validators_details(height),
self.get_transaction_results(&block.block.data),
)
.await;
let raw_transactions = raw_transactions?;
let mut transactions = Vec::with_capacity(raw_transactions.len());
for tx in raw_transactions {
transactions.push(ParsedTransactionResponse {
hash: tx.hash,
height: tx.height,
index: tx.index,
tx_result: tx.tx_result,
tx: cosmrs::Tx::from_bytes(&tx.tx).map_err(|source| {
ScraperError::TxParseFailure {
hash: tx.hash,
source,
}
})?,
proof: tx.proof,
})
}
Ok(FullBlockInformation {
block: block.block,
results: results?,
validators: validators?,
transactions,
})
}
#[instrument(skip(self), err(Display))]
pub async fn get_basic_block_details(
&self,
height: u32,
) -> Result<block::Response, ScraperError> {
debug!("getting basic block details");
self.inner
.block(height)
.await
.map_err(|source| ScraperError::BlockQueryFailure { height, source })
}
#[instrument(skip(self), err(Display))]
pub async fn get_block_results(
&self,
height: u32,
) -> Result<block_results::Response, ScraperError> {
debug!("getting block results");
self.inner
.block_results(height)
.await
.map_err(|source| ScraperError::BlockResultsQueryFailure { height, source })
}
pub(crate) async fn current_block_height(&self) -> Result<u64, ScraperError> {
debug!("getting current block height");
let info = self
.inner
.abci_info()
.await
.map_err(|source| ScraperError::AbciInfoQueryFailure { source })?;
Ok(info.last_block_height.value())
}
async fn get_transaction_results(
&self,
raw: &[Vec<u8>],
) -> Result<Vec<tx::Response>, ScraperError> {
let ordered_results = Arc::new(Mutex::new(BTreeMap::new()));
// "Data is just a wrapper for a list of transactions, where transactions are arbitrary byte arrays"
// source: https://github.com/tendermint/spec/blob/d46cd7f573a2c6a2399fcab2cde981330aa63f37/spec/core/data_structures.md#data
//
// I hate that zip as much as you, dear reader, but for some reason the compiler didn't let me remove the `move`
futures::stream::iter(
raw.iter()
.map(tx_hash)
.enumerate()
.zip(std::iter::repeat(ordered_results.clone())),
)
.for_each_concurrent(4, |((id, tx_hash), ordered_results)| async move {
let res = self.get_transaction_result(tx_hash).await;
ordered_results.lock().await.insert(id, res);
})
.await;
// safety the futures have completed so we MUST have the only arc reference
#[allow(clippy::unwrap_used)]
let inner = Arc::into_inner(ordered_results).unwrap().into_inner();
// BTreeMap is ordered by its keys so we're guaranteed to get txs in correct order
inner.into_values().collect()
}
#[instrument(skip(self, tx_hash), fields(tx_hash = %tx_hash), err(Display))]
async fn get_transaction_result(&self, tx_hash: Hash) -> Result<tx::Response, ScraperError> {
debug!("getting tx results");
self.inner
.tx(tx_hash, false)
.await
.map_err(|source| ScraperError::TxResultsQueryFailure {
hash: tx_hash,
source,
})
}
#[instrument(skip(self))]
pub async fn get_validators_details(
&self,
height: u32,
) -> Result<validators::Response, ScraperError> {
debug!("getting validators set");
self.inner
.validators(height, Paging::All)
.await
.map_err(|source| ScraperError::ValidatorsQueryFailure { height, source })
}
}
-210
View File
@@ -1,210 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::BlockProcessor;
use crate::block_requester::BlockRequester;
use crate::error::ScraperError;
use crate::modules::{BlockModule, MsgModule, TxModule};
use crate::rpc_client::RpcClient;
use crate::scraper::subscriber::{run_websocket_driver, ChainSubscriber};
use crate::storage::ScraperStorage;
use std::path::PathBuf;
use std::sync::Arc;
use tendermint_rpc::WebSocketClientDriver;
use tokio::sync::mpsc::{channel, unbounded_channel};
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
use tracing::info;
use url::Url;
mod subscriber;
pub struct Config {
/// Url to the websocket endpoint of a validator, for example `wss://rpc.nymtech.net/websocket`
pub websocket_url: Url,
/// Url to the rpc endpoint of a validator, for example `https://rpc.nymtech.net/`
pub rpc_url: Url,
pub database_path: PathBuf,
}
pub struct NyxdScraperBuilder {
config: Config,
block_modules: Vec<Box<dyn BlockModule + Send>>,
tx_modules: Vec<Box<dyn TxModule + Send>>,
msg_modules: Vec<Box<dyn MsgModule + Send>>,
}
impl NyxdScraperBuilder {
pub async fn build_and_start(self) -> Result<NyxdScraper, ScraperError> {
let scraper = NyxdScraper::new(self.config).await?;
let (processing_tx, processing_rx) = unbounded_channel();
let (req_tx, req_rx) = channel(5);
let rpc_client = RpcClient::new(&scraper.config.rpc_url)?;
// create the tasks
let block_requester = BlockRequester::new(
scraper.cancel_token.clone(),
rpc_client.clone(),
req_rx,
processing_tx.clone(),
);
let mut block_processor = BlockProcessor::new(
scraper.cancel_token.clone(),
scraper.startup_sync.clone(),
processing_rx,
req_tx,
scraper.storage.clone(),
rpc_client,
)
.await?;
block_processor.set_block_modules(self.block_modules);
block_processor.set_tx_modules(self.tx_modules);
block_processor.set_msg_modules(self.msg_modules);
let mut chain_subscriber = ChainSubscriber::new(
&scraper.config.websocket_url,
scraper.cancel_token.clone(),
processing_tx,
)
.await?;
let ws_driver = chain_subscriber.ws_driver();
scraper.start_tasks(
block_requester,
block_processor,
chain_subscriber,
ws_driver,
);
Ok(scraper)
}
pub fn new(config: Config) -> Self {
NyxdScraperBuilder {
config,
block_modules: vec![],
tx_modules: vec![],
msg_modules: vec![],
}
}
pub fn with_block_module<M: BlockModule + Send + 'static>(mut self, module: M) -> Self {
self.block_modules.push(Box::new(module));
self
}
pub fn with_tx_module<M: TxModule + Send + 'static>(mut self, module: M) -> Self {
self.tx_modules.push(Box::new(module));
self
}
pub fn with_msg_module<M: MsgModule + Send + 'static>(mut self, module: M) -> Self {
self.msg_modules.push(Box::new(module));
self
}
}
pub struct NyxdScraper {
config: Config,
task_tracker: TaskTracker,
cancel_token: CancellationToken,
startup_sync: Arc<Notify>,
pub storage: ScraperStorage,
}
impl NyxdScraper {
pub fn builder(config: Config) -> NyxdScraperBuilder {
NyxdScraperBuilder::new(config)
}
pub async fn new(config: Config) -> Result<Self, ScraperError> {
let storage = ScraperStorage::init(&config.database_path).await?;
Ok(NyxdScraper {
config,
task_tracker: TaskTracker::new(),
cancel_token: CancellationToken::new(),
startup_sync: Arc::new(Default::default()),
storage,
})
}
fn start_tasks(
&self,
mut block_requester: BlockRequester,
mut block_processor: BlockProcessor,
mut chain_subscriber: ChainSubscriber,
ws_driver: WebSocketClientDriver,
) {
self.task_tracker
.spawn(async move { block_requester.run().await });
self.task_tracker
.spawn(async move { block_processor.run().await });
self.task_tracker
.spawn(async move { chain_subscriber.run().await });
self.task_tracker
.spawn(run_websocket_driver(ws_driver, self.cancel_token.clone()));
self.task_tracker.close();
}
pub async fn start(&self) -> Result<(), ScraperError> {
let (processing_tx, processing_rx) = unbounded_channel();
let (req_tx, req_rx) = channel(5);
let rpc_client = RpcClient::new(&self.config.rpc_url)?;
// create the tasks
let block_requester = BlockRequester::new(
self.cancel_token.clone(),
rpc_client.clone(),
req_rx,
processing_tx.clone(),
);
let block_processor = BlockProcessor::new(
self.cancel_token.clone(),
self.startup_sync.clone(),
processing_rx,
req_tx,
self.storage.clone(),
rpc_client,
)
.await?;
let mut chain_subscriber = ChainSubscriber::new(
&self.config.websocket_url,
self.cancel_token.clone(),
processing_tx,
)
.await?;
let ws_driver = chain_subscriber.ws_driver();
// spawn them
self.start_tasks(
block_requester,
block_processor,
chain_subscriber,
ws_driver,
);
Ok(())
}
pub async fn wait_for_startup_sync(&self) {
info!("awaiting startup chain sync");
self.startup_sync.notified().await
}
pub async fn stop(self) {
info!("stopping the chain scrapper");
assert!(self.task_tracker.is_closed());
self.cancel_token.cancel();
self.task_tracker.wait().await
}
}
@@ -1,129 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::BlockToProcess;
use crate::error::ScraperError;
use tendermint_rpc::event::Event;
use tendermint_rpc::query::EventType;
use tendermint_rpc::{SubscriptionClient, WebSocketClient, WebSocketClientDriver};
use tokio::sync::mpsc::UnboundedSender;
use tokio_stream::StreamExt;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use url::Url;
const MAX_FAILURES: usize = 10;
pub struct ChainSubscriber {
cancel: CancellationToken,
block_sender: UnboundedSender<BlockToProcess>,
websocket_client: WebSocketClient,
websocket_driver: Option<WebSocketClientDriver>,
}
impl ChainSubscriber {
pub async fn new(
websocket_endpoint: &Url,
cancel: CancellationToken,
block_sender: UnboundedSender<BlockToProcess>,
) -> Result<Self, ScraperError> {
// sure, we could have just used websocket client entirely, but let's keep the logic for
// getting current blocks and historical blocks completely separate with the dual connection
let (client, driver) = WebSocketClient::new(websocket_endpoint.as_str())
.await
.map_err(|source| ScraperError::WebSocketConnectionFailure {
url: websocket_endpoint.to_string(),
source,
})?;
Ok(ChainSubscriber {
cancel,
block_sender,
websocket_client: client,
websocket_driver: Some(driver),
})
}
fn handle_new_event(&mut self, event: Event) -> Result<(), ScraperError> {
if let Err(err) = self.block_sender.send(event.try_into()?) {
// this error has nothing to do with the websocket or chain
error!("failed to send block for processing: {err} - are we shutting down?")
}
Ok(())
}
pub(crate) async fn run(&mut self) -> Result<(), ScraperError> {
let _drop_guard = self.cancel.clone().drop_guard();
info!("creating chain subscription");
let mut subs = self
.websocket_client
.subscribe(EventType::NewBlock.into())
.await
.map_err(|source| ScraperError::ChainSubscriptionFailure { source })?;
let mut failures = 0;
info!("starting processing loop");
loop {
tokio::select! {
_ = self.cancel.cancelled() => {
info!("received cancellation token");
break
}
maybe_event = subs.next() => {
let Some(maybe_event) = maybe_event else {
warn!("stopped receiving new events");
break;
};
match maybe_event {
Ok(event) => {
if let Err(err) = self.handle_new_event(event) {
error!("failed to process received block: {err}");
failures += 1
} else {
failures = 0;
}
}
Err(err) => {
error!("failed to receive a valid subscription event: {err}");
failures += 1
}
}
if failures >= MAX_FAILURES {
// note: the drop_guard will get dropped and thus cause a shutdown
return Err(ScraperError::MaximumSubscriptionFailures);
}
}
}
}
Ok(())
}
pub(crate) fn ws_driver(&mut self) -> WebSocketClientDriver {
#[allow(clippy::expect_used)]
self.websocket_driver
.take()
.expect("websocket driver has already been started!")
}
}
pub async fn run_websocket_driver(driver: WebSocketClientDriver, cancel: CancellationToken) {
info!("starting websocket driver");
tokio::select! {
_ = cancel.cancelled() => {
info!("received cancellation token")
}
res = driver.run() => {
match res {
Ok(_) => info!("our websocket driver has finished execution"),
Err(err) => {
// TODO: in the future just attempt to reconnect
error!("our websocket driver has errored out: {err}")
}
}
cancel.cancel()
}
}
}
@@ -1,2 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
-352
View File
@@ -1,352 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::storage::models::{CommitSignature, Validator};
use sqlx::types::time::OffsetDateTime;
use sqlx::{Executor, Sqlite};
use tracing::{instrument, trace};
#[derive(Clone)]
pub(crate) struct StorageManager {
pub(crate) connection_pool: sqlx::SqlitePool,
}
impl StorageManager {
pub(crate) async fn set_initial_metadata(&self) -> Result<(), sqlx::Error> {
if sqlx::query("SELECT * from metadata")
.fetch_optional(&self.connection_pool)
.await?
.is_none()
{
sqlx::query("INSERT INTO metadata (id, last_processed_height) VALUES (0, 0)")
.execute(&self.connection_pool)
.await?;
}
Ok(())
}
pub(crate) async fn get_first_block_height_after(
&self,
time: OffsetDateTime,
) -> Result<Option<i64>, sqlx::Error> {
let maybe_record = sqlx::query!(
r#"
SELECT height
FROM block
WHERE timestamp > ?
ORDER BY timestamp
LIMIT 1
"#,
time
)
.fetch_optional(&self.connection_pool)
.await?;
if let Some(row) = maybe_record {
Ok(row.height)
} else {
Ok(None)
}
}
pub(crate) async fn get_last_block_height_before(
&self,
time: OffsetDateTime,
) -> Result<Option<i64>, sqlx::Error> {
let maybe_record = sqlx::query!(
r#"
SELECT height
FROM block
WHERE timestamp < ?
ORDER BY timestamp DESC
LIMIT 1
"#,
time
)
.fetch_optional(&self.connection_pool)
.await?;
if let Some(row) = maybe_record {
Ok(row.height)
} else {
Ok(None)
}
}
pub(crate) async fn get_signed_between(
&self,
consensus_address: &str,
start_height: i64,
end_height: i64,
) -> Result<i32, sqlx::Error> {
let count = sqlx::query!(
r#"
SELECT COUNT(*) as count FROM pre_commit
WHERE
validator_address == ?
AND height >= ?
AND height <= ?
"#,
consensus_address,
start_height,
end_height
)
.fetch_one(&self.connection_pool)
.await?
.count;
Ok(count)
}
pub(crate) async fn get_precommit(
&self,
consensus_address: &str,
height: i64,
) -> Result<Option<CommitSignature>, sqlx::Error> {
sqlx::query_as(
r#"
SELECT * FROM pre_commit
WHERE validator_address = ?
AND height = ?
"#,
)
.bind(consensus_address)
.bind(height)
.fetch_optional(&self.connection_pool)
.await
}
pub(crate) async fn get_block_validators(
&self,
height: i64,
) -> Result<Vec<Validator>, sqlx::Error> {
sqlx::query_as!(
Validator,
r#"
SELECT * FROM validator
WHERE EXISTS (
SELECT 1 FROM pre_commit
WHERE height == ?
AND pre_commit.validator_address = validator.consensus_address
)
"#,
height
)
.fetch_all(&self.connection_pool)
.await
}
pub(crate) async fn get_validators(&self) -> Result<Vec<Validator>, sqlx::Error> {
sqlx::query_as("SELECT * FROM validator")
.fetch_all(&self.connection_pool)
.await
}
pub(crate) async fn get_last_processed_height(&self) -> Result<i64, sqlx::Error> {
let maybe_record = sqlx::query!(
r#"
SELECT last_processed_height FROM metadata
"#
)
.fetch_optional(&self.connection_pool)
.await?;
if let Some(row) = maybe_record {
Ok(row.last_processed_height)
} else {
Ok(-1)
}
}
}
// make those generic over executor so that they could be performed over connection pool and a tx
#[instrument(skip(executor))]
pub(crate) async fn insert_validator<'a, E>(
consensus_address: String,
consensus_pubkey: String,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("insert validator");
sqlx::query!(
r#"
INSERT INTO validator (consensus_address, consensus_pubkey)
VALUES (?, ?)
ON CONFLICT DO NOTHING
"#,
consensus_address,
consensus_pubkey
)
.execute(executor)
.await?;
Ok(())
}
#[instrument(skip(executor))]
pub(crate) async fn insert_block<'a, E>(
height: i64,
hash: String,
num_txs: u32,
total_gas: i64,
proposer_address: String,
timestamp: OffsetDateTime,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("insert block");
sqlx::query!(
r#"
INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT DO NOTHING
"#,
height,
hash,
num_txs,
total_gas,
proposer_address,
timestamp
)
.execute(executor)
.await?;
Ok(())
}
#[instrument(skip(executor))]
pub(crate) async fn insert_precommit<'a, E>(
validator_address: String,
height: i64,
timestamp: OffsetDateTime,
voting_power: i64,
proposer_priority: i64,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("insert precommit");
sqlx::query!(
r#"
INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (validator_address, timestamp) DO NOTHING
"#,
validator_address,
height,
timestamp,
voting_power,
proposer_priority
)
.execute(executor)
.await?;
Ok(())
}
#[instrument(skip(executor))]
#[allow(clippy::too_many_arguments)]
pub(crate) async fn insert_transaction<'a, E>(
hash: String,
height: i64,
index: i64,
success: bool,
messages: i64,
memo: String,
gas_wanted: i64,
gas_used: i64,
raw_log: String,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("insert transaction");
sqlx::query!(
r#"
INSERT INTO "transaction" (hash, height, "index", success, num_messages, memo, gas_wanted, gas_used, raw_log)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (hash) DO UPDATE
SET height = excluded.height,
"index" = excluded."index",
success = excluded.success,
num_messages = excluded.num_messages,
memo = excluded.memo,
gas_wanted = excluded.gas_wanted,
gas_used = excluded.gas_used,
raw_log = excluded.raw_log
"#,
hash,
height,
index,
success,
messages,
memo,
gas_wanted,
gas_used,
raw_log,
)
.execute(executor)
.await?;
Ok(())
}
#[instrument(skip(executor))]
pub(crate) async fn insert_message<'a, E>(
transaction_hash: String,
index: i64,
typ: String,
height: i64,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("insert message");
sqlx::query!(
r#"
INSERT INTO message (transaction_hash, "index", type, height)
VALUES (?, ?, ?, ?)
ON CONFLICT (transaction_hash, "index") DO UPDATE
SET height = excluded.height,
type = excluded.type
"#,
transaction_hash,
index,
typ,
height
)
.execute(executor)
.await?;
Ok(())
}
#[instrument(skip(executor))]
pub(crate) async fn update_last_processed<'a, E>(
height: i64,
executor: E,
) -> Result<(), sqlx::Error>
where
E: Executor<'a, Database = Sqlite>,
{
trace!("update_last_processed");
sqlx::query!("UPDATE metadata SET last_processed_height = ?", height)
.execute(executor)
.await?;
Ok(())
}
-327
View File
@@ -1,327 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use crate::block_processor::types::{FullBlockInformation, ParsedTransactionResponse};
use crate::error::ScraperError;
use crate::storage::manager::{
insert_block, insert_message, insert_precommit, insert_transaction, insert_validator,
update_last_processed, StorageManager,
};
use crate::storage::models::{CommitSignature, Validator};
use sqlx::types::time::OffsetDateTime;
use sqlx::{ConnectOptions, Sqlite, Transaction};
use std::fmt::Debug;
use std::path::Path;
use tendermint::block::{Commit, CommitSig};
use tendermint::Block;
use tendermint_rpc::endpoint::validators;
use tracing::{debug, error, info, instrument, trace, warn};
mod helpers;
mod manager;
pub mod models;
pub type StorageTransaction = Transaction<'static, Sqlite>;
#[derive(Clone)]
pub struct ScraperStorage {
pub(crate) manager: StorageManager,
}
impl ScraperStorage {
#[instrument]
pub async fn init<P: AsRef<Path> + Debug>(database_path: P) -> Result<Self, ScraperError> {
let mut opts = sqlx::sqlite::SqliteConnectOptions::new()
.filename(database_path)
.create_if_missing(true);
// TODO: do we want auto_vacuum ?
opts.disable_statement_logging();
let connection_pool = match sqlx::SqlitePool::connect_with(opts).await {
Ok(db) => db,
Err(err) => {
error!("Failed to connect to SQLx database: {err}");
return Err(err.into());
}
};
if let Err(err) = sqlx::migrate!("./sql_migrations")
.run(&connection_pool)
.await
{
error!("Failed to initialize SQLx database: {err}");
return Err(err.into());
}
info!("Database migration finished!");
let manager = StorageManager { connection_pool };
manager.set_initial_metadata().await?;
let storage = ScraperStorage { manager };
Ok(storage)
}
#[instrument(skip_all)]
pub async fn begin_processing_tx(&self) -> Result<StorageTransaction, ScraperError> {
debug!("starting storage tx");
self.manager
.connection_pool
.begin()
.await
.map_err(|source| ScraperError::StorageTxBeginFailure { source })
}
pub async fn get_first_block_height_after(
&self,
time: OffsetDateTime,
) -> Result<Option<i64>, ScraperError> {
Ok(self.manager.get_first_block_height_after(time).await?)
}
pub async fn get_last_block_height_before(
&self,
time: OffsetDateTime,
) -> Result<Option<i64>, ScraperError> {
Ok(self.manager.get_last_block_height_before(time).await?)
}
pub async fn get_blocks_between(
&self,
start_time: OffsetDateTime,
end_time: OffsetDateTime,
) -> Result<i64, ScraperError> {
let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
return Ok(0);
};
let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
return Ok(0);
};
Ok(block_end - block_start)
}
pub async fn get_signed_between(
&self,
consensus_address: &str,
start_height: i64,
end_height: i64,
) -> Result<i32, ScraperError> {
Ok(self
.manager
.get_signed_between(consensus_address, start_height, end_height)
.await?)
}
pub async fn get_signed_between_times(
&self,
consensus_address: &str,
start_time: OffsetDateTime,
end_time: OffsetDateTime,
) -> Result<i32, ScraperError> {
let Some(block_start) = self.get_first_block_height_after(start_time).await? else {
return Ok(0);
};
let Some(block_end) = self.get_last_block_height_before(end_time).await? else {
return Ok(0);
};
self.get_signed_between(consensus_address, block_start, block_end)
.await
}
pub async fn get_precommit(
&self,
consensus_address: &str,
height: i64,
) -> Result<Option<CommitSignature>, ScraperError> {
Ok(self
.manager
.get_precommit(consensus_address, height)
.await?)
}
pub async fn get_block_signers(&self, height: i64) -> Result<Vec<Validator>, ScraperError> {
Ok(self.manager.get_block_validators(height).await?)
}
pub async fn get_all_known_validators(&self) -> Result<Vec<Validator>, ScraperError> {
Ok(self.manager.get_validators().await?)
}
pub async fn get_last_processed_height(&self) -> Result<i64, ScraperError> {
Ok(self.manager.get_last_processed_height().await?)
}
}
pub async fn persist_block(
block: &FullBlockInformation,
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
let total_gas = crate::helpers::tx_gas_sum(&block.transactions);
// SANITY CHECK: make sure the block proposer is present in the validator set
block.ensure_proposer()?;
// persist validators
persist_validators(&block.validators, tx).await?;
// persist block data
persist_block_data(&block.block, total_gas, tx).await?;
// persist commits
if let Some(commit) = &block.block.last_commit {
persist_commits(commit, &block.validators, tx).await?;
} else {
warn!("no commits for block {}", block.block.header.height)
}
// persist txs
persist_txs(&block.transactions, tx).await?;
// persist messages (inside the transactions)
persist_messages(&block.transactions, tx).await?;
update_last_processed(block.block.header.height.into(), tx).await?;
Ok(())
}
async fn persist_validators(
validators: &validators::Response,
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
debug!("persisting {} validators", validators.total);
for validator in &validators.validators {
let consensus_address = crate::helpers::validator_consensus_address(validator.address)?;
let consensus_pubkey = crate::helpers::validator_pubkey_to_bech32(validator.pub_key)?;
insert_validator(
consensus_address.to_string(),
consensus_pubkey.to_string(),
&mut *tx,
)
.await?;
}
Ok(())
}
async fn persist_block_data(
block: &Block,
total_gas: i64,
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
let proposer_address =
crate::helpers::validator_consensus_address(block.header.proposer_address)?.to_string();
insert_block(
block.header.height.into(),
block.header.hash().to_string(),
block.data.len() as u32,
total_gas,
proposer_address,
block.header.time.into(),
tx,
)
.await?;
Ok(())
}
async fn persist_commits(
commits: &Commit,
validators: &validators::Response,
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
debug!("persisting up to {} commits", commits.signatures.len());
let height: i64 = commits.height.into();
for commit_sig in &commits.signatures {
let (validator_id, timestamp, signature) = match commit_sig {
CommitSig::BlockIdFlagAbsent => {
trace!("absent signature");
continue;
}
CommitSig::BlockIdFlagCommit {
validator_address,
timestamp,
signature,
} => (validator_address, timestamp, signature),
CommitSig::BlockIdFlagNil {
validator_address,
timestamp,
signature,
} => (validator_address, timestamp, signature),
};
let validator = crate::helpers::validator_info(*validator_id, validators)?;
let validator_address = crate::helpers::validator_consensus_address(*validator_id)?;
if signature.is_none() {
warn!("empty signature for {validator_address} at height {height}");
continue;
}
insert_precommit(
validator_address.to_string(),
height,
(*timestamp).into(),
validator.power.into(),
validator.proposer_priority.value(),
&mut *tx,
)
.await?;
}
Ok(())
}
async fn persist_txs(
txs: &[ParsedTransactionResponse],
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
debug!("persisting {} txs", txs.len());
for chain_tx in txs {
insert_transaction(
chain_tx.hash.to_string(),
chain_tx.height.into(),
chain_tx.index as i64,
chain_tx.tx_result.code.is_ok(),
chain_tx.tx.body.messages.len() as i64,
chain_tx.tx.body.memo.clone(),
chain_tx.tx_result.gas_wanted,
chain_tx.tx_result.gas_used,
chain_tx.tx_result.log.clone(),
&mut *tx,
)
.await?;
}
Ok(())
}
async fn persist_messages(
txs: &[ParsedTransactionResponse],
tx: &mut StorageTransaction,
) -> Result<(), ScraperError> {
debug!("persisting messages");
for chain_tx in txs {
for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() {
insert_message(
chain_tx.hash.to_string(),
index as i64,
msg.type_url.clone(),
chain_tx.height.into(),
&mut *tx,
)
.await?
}
}
Ok(())
}
-30
View File
@@ -1,30 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
use sqlx::types::time::OffsetDateTime;
use sqlx::FromRow;
#[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)]
pub struct Validator {
pub consensus_address: String,
pub consensus_pubkey: String,
}
#[derive(Debug, Clone, FromRow)]
pub struct Block {
pub height: i64,
pub hash: String,
pub num_txs: u32,
pub total_gas: i64,
pub proposer_address: String,
pub timestamp: OffsetDateTime,
}
#[derive(Debug, Clone, FromRow)]
pub struct CommitSignature {
pub height: i64,
pub validator_address: String,
pub voting_power: i64,
pub proposer_priority: i64,
pub timestamp: OffsetDateTime,
}
+1 -1
View File
@@ -15,6 +15,6 @@ log = { workspace = true }
reqwest = { workspace = true, features = ["json"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1"
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "chrono"]}
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "chrono"]}
thiserror = { workspace = true }
tokio = { version = "1.24.1", features = [ "time" ] }
+3 -3
View File
@@ -116,8 +116,8 @@ impl TaskManager {
}
#[cfg(not(target_arch = "wasm32"))]
pub async fn catch_interrupt(&mut self) -> Result<(), SentError> {
let res = crate::wait_for_signal_and_error(self).await;
pub async fn catch_interrupt(mut self) -> Result<(), SentError> {
let res = crate::wait_for_signal_and_error(&mut self).await;
log::info!("Sending shutdown");
self.signal_shutdown().ok();
@@ -629,7 +629,7 @@ impl TaskHandle {
#[cfg(not(target_arch = "wasm32"))]
pub async fn wait_for_shutdown(self) -> Result<(), SentError> {
match self {
TaskHandle::Internal(mut task_manager) => task_manager.catch_interrupt().await,
TaskHandle::Internal(task_manager) => task_manager.catch_interrupt().await,
TaskHandle::External(mut task_client) => {
task_client.recv().await;
Ok(())
+1 -3
View File
@@ -16,12 +16,10 @@ base64 = "0.21.3"
# version mismatch with x25519-dalek/curve25519-dalek that is resolved in the
# latest commit. So pick that for now.
x25519-dalek = "2.0.0"
defguard_wireguard_rs = { git = "https://github.com/neacsu/wireguard-rs.git", rev = "c2cd0c1119f699f4bc43f5e6ffd6fc242caa42ed" }
ip_network = "0.4.1"
log.workspace = true
nym-network-defaults = { path = "../network-defaults" }
nym-task = { path = "../task" }
nym-wireguard-types = { path = "../wireguard-types" }
tokio = { workspace = true, features = ["rt-multi-thread", "net", "io-util"] }
[target."cfg(target_os = \"linux\")".dependencies]
defguard_wireguard_rs = { git = "https://github.com/neacsu/wireguard-rs.git", rev = "c2cd0c1119f699f4bc43f5e6ffd6fc242caa42ed" }
+21 -13
View File
@@ -5,20 +5,26 @@
pub mod setup;
use nym_wireguard_types::registration::GatewayClientRegistry;
use std::sync::Arc;
// Currently the module related to setting up the virtual network device is platform specific.
#[cfg(target_os = "linux")]
use crate::setup::{peer_allowed_ips, peer_static_public_key, PRIVATE_KEY};
use defguard_wireguard_rs::WGApi;
#[cfg(target_os = "linux")]
use defguard_wireguard_rs::{
host::Peer, key::Key, net::IpAddrMask, InterfaceConfiguration, WireguardInterfaceApi,
};
#[cfg(target_os = "linux")]
use nym_network_defaults::{WG_PORT, WG_TUN_DEVICE_ADDRESS};
/// Start wireguard device
#[cfg(target_os = "linux")]
pub async fn start_wireguard(
mut task_client: nym_task::TaskClient,
_gateway_client_registry: std::sync::Arc<
nym_wireguard_types::registration::GatewayClientRegistry,
>,
) -> Result<defguard_wireguard_rs::WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
use crate::setup::{peer_allowed_ips, peer_static_public_key, PRIVATE_KEY};
use defguard_wireguard_rs::{
host::Peer, key::Key, net::IpAddrMask, InterfaceConfiguration, WGApi, WireguardInterfaceApi,
};
use nym_network_defaults::{WG_PORT, WG_TUN_DEVICE_ADDRESS};
_gateway_client_registry: Arc<GatewayClientRegistry>,
) -> Result<WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
let ifname = String::from("wg0");
let wgapi = WGApi::new(ifname.clone(), false)?;
wgapi.create_interface()?;
@@ -42,8 +48,10 @@ pub async fn start_wireguard(
Ok(wgapi)
}
#[cfg(not(target_os = "linux"))]
pub async fn start_wireguard() {
todo!("WireGuard is currently only supported on Linux");
pub async fn start_wireguard(
_task_client: nym_task::TaskClient,
_gateway_client_registry: Arc<GatewayClientRegistry>,
) -> Result<WGApi, Box<dyn std::error::Error + Send + Sync + 'static>> {
todo!("WireGuard is currently only supported on Linux")
}
+1 -2
View File
@@ -4,10 +4,9 @@ version = "0.1.0"
edition = "2021"
build = "build.rs"
links = "cpucycles"
license = "LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT"
[dependencies]
libc = "0.2.140"
[build-dependencies]
cfg-if = "1"
cfg-if = "1"
-1
View File
@@ -114,7 +114,6 @@ allow = [
"CC0-1.0",
"Unicode-DFS-2016",
"OpenSSL",
"Zlib",
]
# List of explicitly disallowed licenses
# See https://spdx.org/licenses/ for list of possible licenses
-9
View File
@@ -51,19 +51,10 @@ assets_version = "3.0.0" # do not edit: managed by `mdbook-admonish install`
minimum_rust_version = "1.66"
wallet_release_version = "1.2.8"
# nym-vpn related variables
nym_vpn_latest_binary_url = "https://github.com/nymtech/nym/releases/tag/nym-vpn-alpha-0.0.3"
nym_vpn_form_url = "https://opnform.com/forms/nymvpn-user-research-at-37c3-yccqko-2"
[preprocessor.last-changed]
command = "mdbook-last-changed"
renderer = ["html"]
# used for grabbing output of binary commands for automation
# https://github.com/FauconFan/mdbook-cmdrun
[preprocessor.cmdrun]
#########
# BUILD #
#########
+4 -14
View File
@@ -18,23 +18,13 @@
# User Manuals
- [NymVPN alpha](nymvpn/intro.md)
- [GUI](nymvpn/gui.md)
- [Linux](nymvpn/gui-linux.md)
- [MacOS](nymvpn/gui-mac.md)
- [CLI](nymvpn/cli.md)
- [Linux](nymvpn/cli-linux.md)
- [MacOS](nymvpn/cli-mac.md)
- [Testing](nymvpn/testing.md)
- [Troubleshooting](nymvpn/troubleshooting.md)
- [NymVPN FAQ](nymvpn/faq.md)
- [NymConnect X Monero](tutorials/monero.md)
- [NymConnect X Matrix](tutorials/matrix.md)
- [NymConnect X Telegram](tutorials/telegram.md)
- [NymConnect X Electrum](tutorials/electrum.md)
- [NymConnect X Firo wallet](tutorials/firo.md)
# Code Examples
# Code Examples
- [Custom Service Providers](examples/custom-services.md)
- [Apps Using Network Requesters](examples/using-nrs.md)
@@ -59,9 +49,9 @@
- [Preparing Your Service](tutorials/cosmos-service/service.md)
- [Preparing Your Service pt2](tutorials/cosmos-service/service-src.md)
- [Querying the Chain](tutorials/cosmos-service/querying.md)
- [Typescript](tutorials/typescript.md)
- [Simple Service Provider](tutorials/simple-service-provider/simple-service-provider.md)
- [Simple Service Provider](tutorials/simple-service-provider/simple-service-provider.md)
- [Tutorial Overview](tutorials/simple-service-provider/overview.md)
- [Preparing Your User Client Environment](tutorials/simple-service-provider/preparating-env.md)
- [Building Your User Client](tutorials/simple-service-provider/user-client.md)
@@ -93,7 +83,7 @@
- [Community Applications and Guides](community-resources/community-applications-and-guides.md)
- [Change Service Grantee Information](info-request.md)
---
---
# Misc.
- [Code of Conduct](coc.md)
- [Licensing](licensing.md)
@@ -1,116 +0,0 @@
# NymVPN alpha CLI: Guide for GNU/Linux
```admonish info
NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
```
## Installation
> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for Debian based Linux
2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
```sh
echo "<SHA_STRING>" | shasum -a 256 -c
# choose a correct one according to your binary, this is just an example
echo "0e4abb461e86b2c168577e0294112a3bacd3a24bf8565b49783bfebd9b530e23 nym-vpn-cli_0.1.0_ubuntu-22.04_amd64.zip" | shasum -a 256 -c
```
3. Extract files with `unzip` command or manually as you are used to
4. Make executable by running:
```sh
chmod u+x ./nym-vpn-cli
```
5. Create Sandbox environment config file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the same directory as your NymVPN binaries by running:
```sh
curl -o sandbox.env -L https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env
```
## Run NymVPN
**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
Make sure your terminal is open in the same directory as your `nym-vpn-cli` binary.
1. Go to [nymvpn.com/en/alpha](https://nymvpn.com/en/alpha) to get the entire command with all the needed arguments' values and your wireguard private key for testing purposes
2. Run it as root with `sudo` - the command will look like this with specified arguments:
```sh
sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WIREGUARD_IP>
```
3. To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways) and pick one
4. See all possibilities in [command explanation](#cli-commands-and-options) section below
In case of errors, see [troubleshooting section](troubleshooting.md).
### CLI Commands and Options
The basic syntax of `nym-vpn-cli` is:
```sh
sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WG_IP>
```
* To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways)
* To see all possibilities run with `--help` flag:
```sh
./nym-vpn-cli --help
```
~~~admonish example collapsible=true title="Console output"
```
Usage: nym-vpn-cli [OPTIONS]
Options:
-c, --config-env-file <CONFIG_ENV_FILE>
Path pointing to an env file describing the network
--mixnet-client-path <MIXNET_CLIENT_PATH>
Path to the data directory of a previously initialised mixnet client, where the keys reside
--entry-gateway-id <ENTRY_GATEWAY_ID>
Mixnet public ID of the entry gateway
--entry-gateway-country <ENTRY_GATEWAY_COUNTRY>
Auto-select entry gateway by country ISO
--exit-router-address <EXIT_ROUTER_ADDRESS>
Mixnet recipient address
--exit-gateway-id <EXIT_GATEWAY_ID>
--exit-router-country <EXIT_ROUTER_COUNTRY>
Mixnet recipient address
--enable-wireguard
Enable the wireguard traffic between the client and the entry gateway
--private-key <PRIVATE_KEY>
Associated private key
--wg-ip <WG_IP>
The IP address of the wireguard interface used for the first hop to the entry gateway
--nym-ip <NYM_IP>
The IP address of the nym TUN device that wraps IP packets in sphinx packets
--nym-mtu <NYM_MTU>
The MTU of the nym TUN device that wraps IP packets in sphinx packets
--disable-routing
Disable routing all traffic through the nym TUN device. When the flag is set, the nym TUN device will be created, but to route traffic through it you will need to do it manually, e.g. ping -Itun0
--enable-two-hop
Enable two-hop mixnet traffic. This means that traffic jumps directly from entry gateway to exit gateway
--enable-poisson-rate
Enable Poisson process rate limiting of outbound traffic
-h, --help
Print help
-V, --version
Print version
```
~~~
Here is a list of the options and their descriptions. Some are essential, some are more technical and not needed to be adjusted by users.
**Fundamental commands and arguments**
- `-c` is a path to the [Sandbox config](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) file saved as `sandbox.env`
- `--entry-gateway-id`: paste one of the values labeled with a key `"identityKey"` (without `" "`) from [here](https://nymvpn.com/en/alpha/api/gateways)
- `--exit-router-address`: paste one of the values labeled with a key `"address"` (without `" "`) from here [here](https://nymvpn.com/en/alpha/api/gateways)
- `--enable-wireguard`: Enable the wireguard traffic between the client and the entry gateway. NymVPN uses Mullvad libraries for wrapping `wireguard-go` and to setup local routing rules to route all traffic to the TUN virtual network device
- `--wg-ip`: The address of the wireguard interface, you can get it [here](https://nymvpn.com/en/alpha)
- `--private-key`: get your private key for testing purposes [here](https://nymvpn.com/en/alpha)
- `--enable-two-hop` is a faster setup where the traffic is routed from the client to Entry Gateway and directly to Exit Gateway (default is 5-hops)
**Advanced options**
- `--enable-poisson`: Enables process rate limiting of outbound traffic (disabled by default). It means that NymVPN client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
- `--ip` is the IP address of the TUN device. That is the IP address of the local private network that is set up between local client and the Exit Gateway.
- `--mtu`: The MTU of the TUN device. That is the max IP packet size of the local private network that is set up between local client and the Exit Gateway.
- `--disable-routing`: Disable routing all traffic through the VPN TUN device.
@@ -1,117 +0,0 @@
# NymVPN alpha CLI: Guide for MacOS
```admonish info
NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
```
## Preparation
> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for MacOS
2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
```sh
echo "<SHA_STRING>" | shasum -a 256 -c
# choose a correct one according to your binary, this is just an example
echo "96623ccc69bc4cc0e4e3e18528b6dae6be69f645d0a592d926a3158ce2d0c269 nym-vpn-cli_0.1.0_macos_x86_64.zip" | shasum -a 256 -c
```
5. Extract files with `unzip` command or manually as you are used to
6. Make executable by running:
```sh
chmod u+x ./nym-vpn-cli
```
7. Create Sandbox environment config file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the same directory as your NymVPN binaries by running:
```sh
curl -L "https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env" -o sandbox.env
```
## Run NymVPN
**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
Make sure your terminal is open in the same directory as your `nym-vpn-cli` binary.
1. Go to [nymvpn.com/en/alpha](https://nymvpn.com/en/alpha) to get the entire command with all the needed arguments' values and your wireguard private key for testing purposes
2. Run it as root with `sudo` - the command will look like this with specified arguments:
```sh
sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WIREGUARD_IP>
```
3. To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways) and pick one
4. See all possibilities in [command explanation](#cli-commands-and-options) section below
In case of errors, see [troubleshooting section](troubleshooting.md).
### CLI Commands and Options
The basic syntax of `nym-vpn-cli` is:
```sh
sudo ./nym-vpn-cli -c ./sandbox.env --entry-gateway-id <ENTRY_GATEWAY_ID> --exit-router-address <EXIT_ROUTER_ADDRESS> --enable-wireguard --private-key <PRIVATE_KEY> --wg-ip <WG_IP>
```
* To choose different Gateways, visit [nymvpn.com/en/alpha/api/gateways](https://nymvpn.com/en/alpha/api/gateways)
* To see all possibilities run with `--help` flag:
```sh
./nym-vpn-cli --help
```
~~~admonish example collapsible=true title="Console output"
```
Usage: nym-vpn-cli [OPTIONS]
Options:
-c, --config-env-file <CONFIG_ENV_FILE>
Path pointing to an env file describing the network
--mixnet-client-path <MIXNET_CLIENT_PATH>
Path to the data directory of a previously initialised mixnet client, where the keys reside
--entry-gateway-id <ENTRY_GATEWAY_ID>
Mixnet public ID of the entry gateway
--entry-gateway-country <ENTRY_GATEWAY_COUNTRY>
Auto-select entry gateway by country ISO
--exit-router-address <EXIT_ROUTER_ADDRESS>
Mixnet recipient address
--exit-gateway-id <EXIT_GATEWAY_ID>
--exit-router-country <EXIT_ROUTER_COUNTRY>
Mixnet recipient address
--enable-wireguard
Enable the wireguard traffic between the client and the entry gateway
--private-key <PRIVATE_KEY>
Associated private key
--wg-ip <WG_IP>
The IP address of the wireguard interface used for the first hop to the entry gateway
--nym-ip <NYM_IP>
The IP address of the nym TUN device that wraps IP packets in sphinx packets
--nym-mtu <NYM_MTU>
The MTU of the nym TUN device that wraps IP packets in sphinx packets
--disable-routing
Disable routing all traffic through the nym TUN device. When the flag is set, the nym TUN device will be created, but to route traffic through it you will need to do it manually, e.g. ping -Itun0
--enable-two-hop
Enable two-hop mixnet traffic. This means that traffic jumps directly from entry gateway to exit gateway
--enable-poisson-rate
Enable Poisson process rate limiting of outbound traffic
-h, --help
Print help
-V, --version
Print version
```
~~~
Here is a list of the options and their descriptions. Some are essential, some are more technical and not needed to be adjusted by users.
**Fundamental commands and arguments**
- `-c` is a path to the [Sandbox config](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) file saved as `sandbox.env`
- `--entry-gateway-id`: paste one of the values labeled with a key `"identityKey"` (without `" "`) from [here](https://nymvpn.com/en/alpha/api/gateways)
- `--exit-router-address`: paste one of the values labeled with a key `"address"` (without `" "`) from here [here](https://nymvpn.com/en/alpha/api/gateways)
- `--enable-wireguard`: Enable the wireguard traffic between the client and the entry gateway. NymVPN uses Mullvad libraries for wrapping `wireguard-go` and to setup local routing rules to route all traffic to the TUN virtual network device
- `--wg-ip`: The address of the wireguard interface, you can get it [here](https://nymvpn.com/en/alpha)
- `--private-key`: get your private key for testing purposes [here](https://nymvpn.com/en/alpha)
- `--enable-two-hop` is a faster setup where the traffic is routed from the client to Entry Gateway and directly to Exit Gateway (default is 5-hops)
**Advanced options**
- `--enable-poisson`: Enables process rate limiting of outbound traffic (disabled by default). It means that NymVPN client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
- `--ip` is the IP address of the TUN device. That is the IP address of the local private network that is set up between local client and the Exit Gateway.
- `--mtu`: The MTU of the TUN device. That is the max IP packet size of the local private network that is set up between local client and the Exit Gateway.
- `--disable-routing`: Disable routing all traffic through the VPN TUN device.
@@ -1,46 +0,0 @@
# NymVPN Command Line Interface (CLI)
```admonish info
Our alpha testing round is done with participants at live workshop events. This guide will not work for everyone, as the NymVPN source code is not yet publicly accessible. The alpha testing is done on Nym testnet Sandbox environment, this configuration is limited and will not work in the future.
**If you commit to test NymVPN alpha, please start with the [user research form]({{nym_vpn_form_url}}) where all the steps will be provided**. If you disagree with any of the conditions listed, please leave this page.
```
NymVPN CLI is a fundamental way to run the client for different purposes, currently it is a must for users who want to run the [testing scripts](testing.md).
Follow the simple [automated script](#automated-script-for-cli-installation) below to install and run NymVPN CLI. If you prefer to do a manual setup follow the steps in the guide for [Linux](cli-linux.md) or [MacOS](cli-mac.md).
## Automated Script for CLI Installation
We wrote a [script](https://gist.github.com/tommyv1987/87267ded27e1eb7651aa9cc745ddf4af) which does download of the CLI, sha256 verification, extraction, installation and configuration for Linux and MacOS users automatically following the steps below:
1. Open a terminal window in a directory where you want the script and NymVPN CLI binary be downloaded and run
```sh
curl -o execute-nym-vpn-cli-binary.sh -L https://gist.githubusercontent.com/tommyv1987/87267ded27e1eb7651aa9cc745ddf4af/raw/2a8b703655549f2e515ef1960b5f6dc54adc02fa/execute-nym-vpn-cli-binary.sh
```
2. Make the script executable
```sh
chmod u+x execute-nym-vpn-cli-binary.sh
```
3. Start the script as root, turn off any VPN and run
```sh
sudo ./execute-nym-vpn-cli-binary.sh
```
4. Follow the prompts in the program
5. The script will automatically start the client. Make sure to **turn off any other VPNs** and follow the prompts:
* It prints a JSON view of existing Gateways and prompt you to:
- *Make sure to use two different Gateways for entry and exit!*
- `enter a gateway ID:` paste one of the values labeled with a key `"identityKey"` printed above (without `" "`)
- `enter an exit address:` paste one of the values labeled with a key `"address"` printed above (without `" "`)
- `do you want five hop or two hop?`: type `five` or `two`
- `enable WireGuard? (yes/no):` if you chose yes, find your private key and wireguard IP [here](https://nymvpn.com/en/alpha)
To run `nym-vpn-cli` again, reconnect your wifi, move to the directory of your CLI binary `cd ~/nym-vpn-cli-dir` and follow the guide for [Linux](cli-linux.md#run-nymvpn) or [MacOS](cli-mac.md#run-nymvpn). If you find it too difficult, just run this script again - like in step \#3 above.
In case of errors check out the [troubleshooting](troubleshooting.md) section.
@@ -1,71 +0,0 @@
# Frequently Asked Questions
**NymVPN [*Support & FAQ page*](https://nymvpn.com/en/support)** contains all essential FAQs regarding the client. This page (below) is a source of additional information often seeked by users, operators and developers testing NymVPN.
If you interested to read more about Nym platform, you can have a look at [Nym general FAQ](https://nymtech.net/developers/faq/general-faq.html) and read through Nym's technical [documentation](https://nymtech.net/docs), [Developer Portal](https://nymtech.net/developers) and [Operators Guide](https://nymtech.net/operators).
## NymVPN
If this your first time hearing about NymVPN, make sure you visit [NymVPN webpage](https://nymvpn.com/en), the official NymVPN [support & FAQ page](https://nymvpn.com/en/support) and the proceed to the introduction and guide on how to [install, run and test](intro.md#nymvpn-guides) the client.
Below are some extra FAQs which came out during the previous alpha testing rounds.
### What's the difference between 2-hops and 5-hops
The default is 5-hops (including Entry and Exit Gateways), which means that the traffic goes from the local client to Entry Gateway -> through 3 layers of Mix Nodes -> to Exit Gateway -> internet. this option uses all the Nym Mixnet features for maximum privacy.
```
┌─►mix──┐ mix mix
│ │
Entry │ │ Exit
client ───► Gateway ──┘ mix │ mix ┌─►mix ───► Gateway ───► internet
│ │
│ │
mix └─►mix──┘ mix
```
The 2-hop option is going from the local client -> Entry Gateway -> directly to Exit Gateway -> internet. This option is good for operations demanding faster connection. Keep in mind that this setup by-passes the 3 layers of Mix Nodes. The anonymising features done by your local client like breaking data into same-size packets with inserting additional "dummy" ones to break the time and volume patterns is done in both options.
```
Entry Exit
client ───► Gateway ────► Gateway ───► internet
```
We highly recommend to read more about [Nym network overview](https://nymtech.net/docs/architecture/network-overview.html) and the [Mixnet traffic flow](https://nymtech.net/docs/architecture/traffic-flow.html).
### Why do I see different sizes of packets in my terminal log?
One of features of Nym Mixnet's clients is to break data into the same size packets called Sphinx, which is currently ~2kb. When running NymVPN, the data log shows payload sizes, which are the raw sizes of the IP packets, not Sphinx. The payload sizes will be capped by the configured MTU, which is set around 1500 bytes.
### What is 'poisson filter' about?
By default `--enable-poisson` is disabled and packets are sent from the local client to the Entry Gateway as quickly as possible. With the poisson process enabled the Nym client will send packets at a steady stream to the Entry Gateway. By default it's on average one sphinx packet per 20ms, but there is some randomness (poisson distribution). When there are no real data to fill the sphinx packets with, cover packets are generated instead.
Enabling the poisson filter is one of the key mechanisms to de-correlate input and output traffic to the Mixnet. The performance impact however is dramatic:
1 packer per 20ms is 50 packets / sec so ballpark 100kb/s.
For mobile clients that means constantly sending data eating up data allowance.
## Nym Mixnet Architecture and Rewards
We have a list of questions related to Nym Nodes and the incentives behind running them under [FAQ pages](https://nymtech.net/operators/faq/mixnodes-faq.html) in our [Operators Guide](https://nymtech.net/operators). For better knowledge about Nym architecture we recommend to read [Nym network overview](https://nymtech.net/docs/architecture/network-overview.html) and the [Mixnet traffic flow](https://nymtech.net/docs/architecture/traffic-flow.html) in our [technical documentation](https://nymtech.net/docs).
## Project Smoosh
Project Smoosh is a code name for a process in which different components of Nym Mixnet architecture get *smooshed* into one binary. Check out [Smoosh FAQ](https://nymtech.net/operators/faq/smoosh-faq.html) in Operators Guide to read more.
## Exit Gateway
Part of the the transition under code name [Project Smoosh](#project-smoosh) is a creation of [Nym Exit Gateway](https://nymtech.net/operators/legal/exit-gateway.html) functionality. The operators running Gateways would have to “open” their nodes to a wider range of online services, in a similar fashion to Tor exit relays. The main change will be to expand the original short [allowed.list](https://nymtech.net/.wellknown/network-requester/standard-allowed-list.txt) to a more permissive setup. An [exit policy](https://nymtech.net/.wellknown/network-requester/exit-policy.txt) will constrain the hosts that the users of the Nym VPN and Mixnet can connect to. This will be done in an effort to protect the operators, as Gateways will act both as SOCKS5 Network Requesters, and exit nodes for IP traffic from Nym VPN and Mixnet clients.
* Read more how the exit policy gets implemented [here](https://nymtech.net/operators/faq/smoosh-faq.html#how-will-the-exit-policy-be-implemented)
* Check out [Nym Operators Legal Forum](https://nymtech.net/operators/legal/exit-gateway.html)
* Do reach out to us with any experiences you may have running Tor Exit relays or legal findings and suggestions for Nym Exit Gateway operators
## Nym Integrations and SDKs
If you are a dev who is interested to integrate Nym, have a look on our SDK tutorials:
* [Rust SDKs](https://nymtech.net/developers/tutorials/cosmos-service/intro.html)
* [TypeScript SDKs](https://sdk.nymtech.net/)
* [Integration FAQ](https://nymtech.net/developers/faq/integrations-faq.html)
@@ -1,73 +0,0 @@
# NymVPN alpha GUI: Guide for GNU/Linux
<div style="padding:56.25% 0 0 0;position:relative;"><iframe src="https://player.vimeo.com/video/908221306?h=404b2bbdc8" style="position:absolute;top:0;left:0;width:100%;height:100%;" frameborder="0" allow="autoplay; fullscreen; picture-in-picture" allowfullscreen></iframe></div><script src="https://player.vimeo.com/api/player.js"></script>
```admonish info
NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
```
## Preparation
> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
### Installation
1. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for Debian based Linux
2. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
```sh
echo "<SHA_STRING>" | shasum -a 256 -c
# choose a correct one according to your binary, this is just an example
echo "a5f91f20d587975e30b6a75d3a9e195234cf1269eac278139a5b9c39b039e807 nym-vpn-desktop_0.0.3_ubuntu-22.04_x86_64.zip" | shasum -a 256 -c
```
3. Extract files with `unzip` command or manually as you are used to
4. If you prefer to run `.AppImage` make executable by running:
```sh
chmod u+x ./appimage/nym-vpn_0.0.2_amd64.AppImage
```
5. If you prefer to use the `.deb` version for installation (works on Debian based Linux only), open terminal in the same directory and run:
```sh
cd deb
sudo dpkg -i ./nym-vpn_0.0.3_amd64.deb
# or
sudo apt-get install -f ./nym-vpn_0.0.3_amd64.deb
```
### Configuration
6. Create a NymVPN config directory called `nym-vpn` in your `~/.config`, either manually or by a command:
```sh
mkdir $HOME/.config/nym-vpn/
```
7. Create the network config by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `sandbox.env` in the directory `~/.config/nym-vpn/` you just created by running:
```sh
curl -o $HOME/.config/nym-vpn/sandbox.env -L https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env
```
8. Create NymVPN main config file called `config.toml` in the same directory `~/.config/nym-vpn/` with this content:
```toml
# change <USER> to your username
env_config_file = "/home/<USER>/.config/nym-vpn/sandbox.env"
entry_node_location = "DE" # two letters country code
# You can choose different entry by entering one of the following two letter country codes:
# DE, UK, FR, IE
```
## Run NymVPN
**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
In case you used `.deb` package and installed the client, you may be able to have a NymVPN application icon in your app menu. However this may not work as the application needs root permission.
Open terminal and run:
```sh
# .AppImage must be run from the same directory as the binary
sudo -E ./nym-vpn_0.0.2_amd64.AppImage
# .deb installation shall be executable from anywhere as
sudo -E nym-vpn
```
In case of errors, see [troubleshooting section](troubleshooting.md).
@@ -1,59 +0,0 @@
# NymVPN alpha GUI: Guide for Mac OS
```admonish info
NymVPN is an experimental software and it's for [testing](./testing.md) purposes only. All users testing the client are expected to sign GDPR Information Sheet and Consent Form (shared at the workshop) so we use their results to improve the client, and submit the form [*NymVPN User research*]({{nym_vpn_form_url}}) with the testing results.
```
## Preparation
> Any syntax in `<>` brackets is a user's/version unique variable. Exchange with a corresponding name without the `<>` brackets.
### Installation
1. Create a directory `~/nym-vpn-latest`
```sh
mkdir -p "$HOME/nym-vpn-latest"
```
2. Open Github [releases page]({{nym_vpn_latest_binary_url}}) and download the binary for MacOS
3. Verify sha hash of your downloaded binary with the one listed on the [releases page]({{nym_vpn_latest_binary_url}}). You can use a simple `shasum` command and compare strings (ie with Python) or run in the same directory the following command, exchanging `<SHA_STRING>` with the one of your binary, like in the example:
```sh
echo "<SHA_STRING>" | shasum -a 256 -c
# choose a correct one according to your binary, this is just an example
echo "da4c0bf8e8b52658312d341fa3581954cfcb6efd516d9a448c76d042a454b5df nym-vpn-desktop_0.0.3_macos_x86_64.zip" | shasum -a 256 -c
```
4. Extract files with `unzip` command or manually as you are used to
5. Move to the application directory and make executable
```sh
cd "macos/nym-vpn.app/Contents/MacOS"
chmod u+x nym-vpn
```
6. Move `nym-vpn` to your `~/nym-vpn-latest` directory
```sh
mv nym-vpn "$HOME/nym-vpn-latest"
```
### Configuration
7. Create the configuration file by opening a text editor and saving the lines below as `config.toml` in the same directory `~/nym-vpn-latest`
```toml
env_config_file = ".env"
entry_node_location = "DE" # two letters country code
# You can choose different entry by entering one of the following two letter country codes:
# DE, UK, FR, IE
```
8. Create testnet configuration file by saving [this](https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env) as `.env` in the same directory `~/nym-vpn-latest`
```sh
curl -L "https://raw.githubusercontent.com/nymtech/nym/develop/envs/sandbox.env" -o "$HOME/nym-vpn-latest/.env"
```
## Run NymVPN
**For NymVPN to work, all other VPNs must be switched off!** At this alpha stage of NymVPN, the network connection (wifi) must be reconnected after or in between the testing rounds.
Open terminal in your `~/nym-vpn-latest` directory and run:
```sh
sudo ./nym-vpn
```
In case of errors check out the [troubleshooting](troubleshooting.html#installing-gui-on-macos-not-working) section.
@@ -1,43 +0,0 @@
# NymVPN Application (GUI)
```admonish info
Our alpha testing round is done with participants at live workshop events. This guide will not work for everyone, as the NymVPN source code is not yet publicly accessible. The alpha testing is done on Nym testnet Sandbox environment, this configuration is limited and will not work in the future.
**If you commit to test NymVPN alpha, please start with the [user research form]({{nym_vpn_form_url}}) where all the steps will be provided**. If you disagree with any of the conditions listed, please leave this page.
```
This is the alpha version of NymVPN application - the GUI. A demo of how the client will look like for majority of day-to-day users. For qualitative testing the [CLI](cli.md) is a necessity but to run the GUI holds the same importance as it provides the user with an experience of the actual app and the developers with a valuable feedback from the users.
Follow the simple [automated script](#automated-script-for-gui-installation) below to install and run NymVPN GUI. If you prefer to do a manual setup follow the steps in the guide for [Linux](gui-linux.md) or [MacOS](gui-mac.md).
## Automated Script for GUI Installation
We wrote a [script](https://gist.github.com/serinko/e0a9f7ff3d79e974ec6f6783caa1137e) which does download of dependencies and the application, sha256 verification, extraction, installation and configuration for Linux and MacOS users automatically following the steps below:
1. Open a terminal window in a directory where you want the script and NymVPN binary be downloaded and run
```sh
curl -o nym-vpn-desktop-install-run.sh -L https://gist.githubusercontent.com/serinko/e0a9f7ff3d79e974ec6f6783caa1137e/raw/064c0d42af6a187b4f62998a0c2e6a84c319eeef/nym-vpn-desktop-install-run.sh
```
2. Make the script executable
```sh
chmod u+x nym-vpn-desktop-install-run.sh
```
3. Start the script as root, turn off any VPN and run
```sh
sudo -E ./nym-vpn-desktop-install-run.sh
```
4. Follow the prompts in the program
To start the application again, reconnect your wifi and run
```sh
# Linux
sudo -E ~/nym-vpn-latest/nym-vpn_0.0.3_amd64.AppImage
# MacOS
sudo $nym_vpn_dir/nym-vpn
```
In case of errors check out the [troubleshooting](troubleshooting.md#installing-gui-on-macos-not-working) section.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 97 KiB

Some files were not shown because too many files have changed in this diff Show More