Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Fran Arbanas
|
160436b556 |
@@ -57,12 +57,6 @@ jobs:
|
||||
command: fmt
|
||||
args: --all -- --check
|
||||
|
||||
- name: Clippy
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: --workspace --all-targets -- -D warnings
|
||||
|
||||
- name: Build all binaries
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
@@ -88,3 +82,9 @@ jobs:
|
||||
with:
|
||||
command: test
|
||||
args: --workspace -- --ignored
|
||||
|
||||
- name: Clippy
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: --workspace --all-targets -- -D warnings
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Build and upload Data observatory container to harbor.nymte.ch
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
WORKING_DIRECTORY: "nym-data-observatory"
|
||||
CONTAINER_NAME: "data-observatory"
|
||||
|
||||
jobs:
|
||||
build-container:
|
||||
runs-on: arc-ubuntu-22.04-dind
|
||||
steps:
|
||||
- name: Login to Harbor
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: harbor.nymte.ch
|
||||
username: ${{ secrets.HARBOR_ROBOT_USERNAME }}
|
||||
password: ${{ secrets.HARBOR_ROBOT_SECRET }}
|
||||
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure git identity
|
||||
run: |
|
||||
git config --global user.email "lawrence@nymtech.net"
|
||||
git config --global user.name "Lawrence Stalder"
|
||||
|
||||
- name: Get version from cargo.toml
|
||||
uses: mikefarah/yq@v4.44.3
|
||||
id: get_version
|
||||
with:
|
||||
cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
|
||||
|
||||
- name: Check if tag exists
|
||||
run: |
|
||||
if git rev-parse ${{ steps.get_version.outputs.value }} >/dev/null 2>&1; then
|
||||
echo "Tag ${{ steps.get_version.outputs.value }} already exists"
|
||||
fi
|
||||
|
||||
- name: Remove existing tag if exists
|
||||
run: |
|
||||
if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
|
||||
git push --delete origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
git tag -d ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
fi
|
||||
|
||||
- name: Create tag
|
||||
run: |
|
||||
git tag -a ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} -m "Version ${{ steps.get_version.outputs.result }}"
|
||||
git push origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
|
||||
- name: BuildAndPushImageOnHarbor
|
||||
run: |
|
||||
docker build -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }} -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:latest
|
||||
docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags
|
||||
@@ -1,56 +0,0 @@
|
||||
name: Build and upload Node Status agent container to harbor.nymte.ch
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
WORKING_DIRECTORY: "nym-node-status-agent"
|
||||
CONTAINER_NAME: "node-status-agent"
|
||||
|
||||
jobs:
|
||||
build-container:
|
||||
runs-on: arc-ubuntu-22.04-dind
|
||||
steps:
|
||||
- name: Login to Harbor
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: harbor.nymte.ch
|
||||
username: ${{ secrets.HARBOR_ROBOT_USERNAME }}
|
||||
password: ${{ secrets.HARBOR_ROBOT_SECRET }}
|
||||
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure git identity
|
||||
run: |
|
||||
git config --global user.email "lawrence@nymtech.net"
|
||||
git config --global user.name "Lawrence Stalder"
|
||||
|
||||
- name: Get version from cargo.toml
|
||||
uses: mikefarah/yq@v4.44.3
|
||||
id: get_version
|
||||
with:
|
||||
cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
|
||||
|
||||
- name: Check if tag exists
|
||||
run: |
|
||||
if git rev-parse ${{ steps.get_version.outputs.value }} >/dev/null 2>&1; then
|
||||
echo "Tag ${{ steps.get_version.outputs.value }} already exists"
|
||||
fi
|
||||
|
||||
- name: Remove existing tag if exists
|
||||
run: |
|
||||
if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
|
||||
git push --delete origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
git tag -d ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
fi
|
||||
|
||||
- name: Create tag
|
||||
run: |
|
||||
git tag -a ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} -m "Version ${{ steps.get_version.outputs.result }}"
|
||||
git push origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
|
||||
- name: BuildAndPushImageOnHarbor
|
||||
run: |
|
||||
docker build -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }} -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:latest
|
||||
docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags
|
||||
@@ -1,55 +1,11 @@
|
||||
name: Build and upload Node Status API container to harbor.nymte.ch
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
WORKING_DIRECTORY: "nym-node-status-api"
|
||||
CONTAINER_NAME: "node-status-api"
|
||||
|
||||
jobs:
|
||||
build-container:
|
||||
runs-on: arc-ubuntu-22.04-dind
|
||||
my-job:
|
||||
runs-on: arc-ubuntu-22.04
|
||||
steps:
|
||||
- name: Login to Harbor
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: harbor.nymte.ch
|
||||
username: ${{ secrets.HARBOR_ROBOT_USERNAME }}
|
||||
password: ${{ secrets.HARBOR_ROBOT_SECRET }}
|
||||
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure git identity
|
||||
run: |
|
||||
git config --global user.email "lawrence@nymtech.net"
|
||||
git config --global user.name "Lawrence Stalder"
|
||||
|
||||
- name: Get version from cargo.toml
|
||||
uses: mikefarah/yq@v4.44.3
|
||||
id: get_version
|
||||
with:
|
||||
cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
|
||||
|
||||
- name: Check if tag exists
|
||||
run: |
|
||||
if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
|
||||
echo "Tag ${{ steps.get_version.outputs.result }} already exists"
|
||||
fi
|
||||
|
||||
- name: Remove existing tag if exists
|
||||
run: |
|
||||
if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
|
||||
git push --delete origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
git tag -d ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
fi
|
||||
|
||||
- name: Create tag
|
||||
run: |
|
||||
git tag -a ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} -m "Version ${{ steps.get_version.outputs.result }}"
|
||||
git push origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
|
||||
|
||||
- name: BuildAndPushImageOnHarbor
|
||||
run: |
|
||||
docker build -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }} -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:latest
|
||||
docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags
|
||||
- name: my-step
|
||||
run: echo "Hello World!"
|
||||
|
||||
@@ -4,80 +4,6 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2024.12-aero] (2024-10-17)
|
||||
|
||||
- nym-node: don't use bloomfilters for double spending checks ([#4960])
|
||||
- bugfix: replace unreachable macro with an error return ([#4958])
|
||||
- [DOCs:/operators]: Update FAQ sphinx size ([#4946])
|
||||
- [DOCs/operators]: Release notes v2024.11-wedel ([#4939])
|
||||
- Fix handle drop ([#4934])
|
||||
- Assume offline mode ([#4926])
|
||||
- Make ip-packet-request VERSION pub ([#4925])
|
||||
- Expose error type ([#4924])
|
||||
- Fix argument to cargo-deny action ([#4922])
|
||||
- Fix nymvpn.com url in mainnet defaults ([#4920])
|
||||
- Check both version and type in message header ([#4918])
|
||||
- Bump http-api-client default timeout to 30 sec ([#4917])
|
||||
- Max/proxy ffi ([#4906])
|
||||
- Data Observatory stub ([#4905])
|
||||
- Fix missing duplication of modified tables ([#4904])
|
||||
- Update cargo deny ([#4901])
|
||||
- docs: add hostname instructions for wss ([#4900])
|
||||
- build(deps): bump the patch-updates group across 1 directory with 9 updates ([#4898])
|
||||
- Fix clippy for beta toolchain ([#4897])
|
||||
- Remove clippy github PR annotations ([#4896])
|
||||
- Fix apt install in ci-build-upload-binaries.yml ([#4894])
|
||||
- Update network monitor entrypoint ([#4893])
|
||||
- Update nym-vpn metapackage and replace nymvpn-x with nym-vpn-app ([#4889])
|
||||
- Entry wireguard tickets ([#4888])
|
||||
- Build and Push CI ([#4887])
|
||||
- Feature/updated gateway registration ([#4885])
|
||||
- Few fixes to NNM pre deploy ([#4883])
|
||||
- Fix sql serde with enum ([#4875])
|
||||
- allow clients to send stateless gateway requests without prior registration ([#4873])
|
||||
- chore: remove queued migration for adding explicit admin ([#4871])
|
||||
- Gateway database modifications for different modes ([#4868])
|
||||
- build(deps): bump strum from 0.25.0 to 0.26.3 ([#4848])
|
||||
- Use serde from workspace ([#4833])
|
||||
- build(deps): bump toml from 0.5.11 to 0.8.14 ([#4805])
|
||||
- Max/rust sdk stream abstraction ([#4743])
|
||||
|
||||
[#4960]: https://github.com/nymtech/nym/pull/4960
|
||||
[#4958]: https://github.com/nymtech/nym/pull/4958
|
||||
[#4946]: https://github.com/nymtech/nym/pull/4946
|
||||
[#4939]: https://github.com/nymtech/nym/pull/4939
|
||||
[#4934]: https://github.com/nymtech/nym/pull/4934
|
||||
[#4926]: https://github.com/nymtech/nym/pull/4926
|
||||
[#4925]: https://github.com/nymtech/nym/pull/4925
|
||||
[#4924]: https://github.com/nymtech/nym/pull/4924
|
||||
[#4922]: https://github.com/nymtech/nym/pull/4922
|
||||
[#4920]: https://github.com/nymtech/nym/pull/4920
|
||||
[#4918]: https://github.com/nymtech/nym/pull/4918
|
||||
[#4917]: https://github.com/nymtech/nym/pull/4917
|
||||
[#4906]: https://github.com/nymtech/nym/pull/4906
|
||||
[#4905]: https://github.com/nymtech/nym/pull/4905
|
||||
[#4904]: https://github.com/nymtech/nym/pull/4904
|
||||
[#4901]: https://github.com/nymtech/nym/pull/4901
|
||||
[#4900]: https://github.com/nymtech/nym/pull/4900
|
||||
[#4898]: https://github.com/nymtech/nym/pull/4898
|
||||
[#4897]: https://github.com/nymtech/nym/pull/4897
|
||||
[#4896]: https://github.com/nymtech/nym/pull/4896
|
||||
[#4894]: https://github.com/nymtech/nym/pull/4894
|
||||
[#4893]: https://github.com/nymtech/nym/pull/4893
|
||||
[#4889]: https://github.com/nymtech/nym/pull/4889
|
||||
[#4888]: https://github.com/nymtech/nym/pull/4888
|
||||
[#4887]: https://github.com/nymtech/nym/pull/4887
|
||||
[#4885]: https://github.com/nymtech/nym/pull/4885
|
||||
[#4883]: https://github.com/nymtech/nym/pull/4883
|
||||
[#4875]: https://github.com/nymtech/nym/pull/4875
|
||||
[#4873]: https://github.com/nymtech/nym/pull/4873
|
||||
[#4871]: https://github.com/nymtech/nym/pull/4871
|
||||
[#4868]: https://github.com/nymtech/nym/pull/4868
|
||||
[#4848]: https://github.com/nymtech/nym/pull/4848
|
||||
[#4833]: https://github.com/nymtech/nym/pull/4833
|
||||
[#4805]: https://github.com/nymtech/nym/pull/4805
|
||||
[#4743]: https://github.com/nymtech/nym/pull/4743
|
||||
|
||||
## [2024.11-wedel] (2024-09-23)
|
||||
|
||||
- Backport #4894 to fix ci ([#4899])
|
||||
|
||||
Generated
+443
-1073
File diff suppressed because it is too large
Load Diff
+8
-21
@@ -54,14 +54,12 @@ members = [
|
||||
"common/exit-policy",
|
||||
"common/gateway-requests",
|
||||
"common/gateway-storage",
|
||||
"common/gateway-stats-storage",
|
||||
"common/http-api-client",
|
||||
"common/http-api-common",
|
||||
"common/inclusion-probability",
|
||||
"common/ip-packet-requests",
|
||||
"common/ledger",
|
||||
"common/mixnode-common",
|
||||
"common/models",
|
||||
"common/network-defaults",
|
||||
"common/node-tester-utils",
|
||||
"common/nonexhaustive-delayqueue",
|
||||
@@ -121,8 +119,6 @@ members = [
|
||||
"nym-node",
|
||||
"nym-node/nym-node-http-api",
|
||||
"nym-node/nym-node-requests",
|
||||
"nym-node-status-api",
|
||||
"nym-node-status-agent",
|
||||
"nym-outfox",
|
||||
"nym-validator-rewarder",
|
||||
"tools/echo-server",
|
||||
@@ -150,16 +146,13 @@ members = [
|
||||
default-members = [
|
||||
"clients/native",
|
||||
"clients/socks5",
|
||||
"common/models",
|
||||
"explorer-api",
|
||||
"gateway",
|
||||
"mixnode",
|
||||
"nym-api",
|
||||
"nym-data-observatory",
|
||||
"nym-node",
|
||||
"nym-node-status-api",
|
||||
"nym-validator-rewarder",
|
||||
"nym-node-status-api",
|
||||
"service-providers/authenticator",
|
||||
"service-providers/ip-packet-router",
|
||||
"service-providers/network-requester",
|
||||
@@ -190,7 +183,7 @@ aes = "0.8.1"
|
||||
aes-gcm = "0.10.1"
|
||||
aes-gcm-siv = "0.11.1"
|
||||
aead = "0.5.2"
|
||||
anyhow = "1.0.90"
|
||||
anyhow = "1.0.89"
|
||||
argon2 = "0.5.0"
|
||||
async-trait = "0.1.83"
|
||||
axum = "0.7.5"
|
||||
@@ -215,7 +208,7 @@ chacha20 = "0.9.0"
|
||||
chacha20poly1305 = "0.10.1"
|
||||
chrono = "0.4.31"
|
||||
cipher = "0.4.3"
|
||||
clap = "4.5.20"
|
||||
clap = "4.5.18"
|
||||
clap_complete = "4.5"
|
||||
clap_complete_fig = "4.5"
|
||||
colored = "2.0"
|
||||
@@ -240,12 +233,10 @@ dotenvy = "0.15.6"
|
||||
ecdsa = "0.16"
|
||||
ed25519-dalek = "2.1"
|
||||
etherparse = "0.13.0"
|
||||
envy = "0.4"
|
||||
eyre = "0.6.9"
|
||||
fastrand = "2.1.1"
|
||||
flate2 = "1.0.34"
|
||||
futures = "0.3.28"
|
||||
futures-util = "0.3"
|
||||
generic-array = "0.14.7"
|
||||
getrandom = "0.2.10"
|
||||
getset = "0.1.3"
|
||||
@@ -275,7 +266,6 @@ ledger-transport-hid = "0.10.0"
|
||||
log = "0.4"
|
||||
maxminddb = "0.23.0"
|
||||
mime = "0.3.17"
|
||||
moka = { version = "0.12", features = ["future"] }
|
||||
nix = "0.27.1"
|
||||
notify = "5.1.0"
|
||||
okapi = "0.7.0"
|
||||
@@ -285,7 +275,7 @@ opentelemetry-jaeger = "0.18.0"
|
||||
parking_lot = "0.12.3"
|
||||
pem = "0.8"
|
||||
petgraph = "0.6.5"
|
||||
pin-project = "1.1"
|
||||
pin-project = "1.0"
|
||||
pin-project-lite = "0.2.14"
|
||||
pretty_env_logger = "0.4.0"
|
||||
publicsuffix = "2.2.3"
|
||||
@@ -305,11 +295,10 @@ rocket_okapi = "0.8.0"
|
||||
safer-ffi = "0.1.13"
|
||||
schemars = "0.8.21"
|
||||
semver = "1.0.23"
|
||||
serde = "1.0.211"
|
||||
serde = "1.0.210"
|
||||
serde_bytes = "0.11.15"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0.132"
|
||||
serde_json_path = "0.6.7"
|
||||
serde_json = "1.0.128"
|
||||
serde_repr = "0.1"
|
||||
serde_with = "3.9.0"
|
||||
serde_yaml = "0.9.25"
|
||||
@@ -318,7 +307,6 @@ si-scale = "0.2.3"
|
||||
sphinx-packet = "0.1.1"
|
||||
sqlx = "0.7.4"
|
||||
strum = "0.26"
|
||||
strum_macros = "0.26"
|
||||
subtle-encoding = "0.5"
|
||||
syn = "1"
|
||||
sysinfo = "0.30.13"
|
||||
@@ -340,7 +328,6 @@ tracing = "0.1.37"
|
||||
tracing-opentelemetry = "0.19.0"
|
||||
tracing-subscriber = "0.3.16"
|
||||
tracing-tree = "0.2.2"
|
||||
tracing-log = "0.2"
|
||||
ts-rs = "10.0.0"
|
||||
tungstenite = { version = "0.20.1", default-features = false }
|
||||
url = "2.5"
|
||||
@@ -402,10 +389,10 @@ indexed_db_futures = { git = "https://github.com/TiemenSch/rust-indexed-db", bra
|
||||
js-sys = "0.3.70"
|
||||
serde-wasm-bindgen = "0.6.5"
|
||||
tsify = "0.4.5"
|
||||
wasm-bindgen = "0.2.95"
|
||||
wasm-bindgen-futures = "0.4.45"
|
||||
wasm-bindgen = "0.2.93"
|
||||
wasm-bindgen-futures = "0.4.43"
|
||||
wasmtimer = "0.2.0"
|
||||
web-sys = "0.3.72"
|
||||
web-sys = "0.3.70"
|
||||
|
||||
|
||||
# Profile settings for individual crates
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "nym-client"
|
||||
version = "1.1.42"
|
||||
version = "1.1.41"
|
||||
authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
|
||||
description = "Implementation of the Nym Client"
|
||||
edition = "2021"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "nym-socks5-client"
|
||||
version = "1.1.42"
|
||||
version = "1.1.41"
|
||||
authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
|
||||
description = "A SOCKS5 localhost proxy that converts incoming messages to Sphinx and sends them to a Nym address"
|
||||
edition = "2021"
|
||||
|
||||
@@ -45,4 +45,3 @@ tracing = [
|
||||
"opentelemetry",
|
||||
]
|
||||
clap = [ "dep:clap", "dep:clap_complete", "dep:clap_complete_fig" ]
|
||||
models = []
|
||||
|
||||
@@ -25,7 +25,7 @@ serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
nym-http-api-client = { path = "../../../common/http-api-client" }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
log = { workspace = true }
|
||||
url = { workspace = true, features = ["serde"] }
|
||||
tokio = { workspace = true, features = ["sync", "time"] }
|
||||
time = { workspace = true, features = ["formatting"] }
|
||||
|
||||
@@ -265,13 +265,6 @@ impl NymApiClient {
|
||||
NymApiClient { nym_api }
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn new_with_timeout(api_url: Url, timeout: std::time::Duration) -> Self {
|
||||
let nym_api = nym_api::Client::new(api_url, Some(timeout));
|
||||
|
||||
NymApiClient { nym_api }
|
||||
}
|
||||
|
||||
pub fn new_with_user_agent(api_url: Url, user_agent: UserAgent) -> Self {
|
||||
let nym_api = nym_api::Client::builder::<_, ValidatorClientError>(api_url)
|
||||
.expect("invalid api url")
|
||||
|
||||
@@ -121,36 +121,36 @@ async fn test_nyxd_connection(
|
||||
{
|
||||
Ok(Err(NyxdError::TendermintErrorRpc(e))) => {
|
||||
// If we get a tendermint-rpc error, we classify the node as not contactable
|
||||
tracing::warn!("Checking: nyxd url: {url}: {}: {}", "failed".red(), e);
|
||||
log::warn!("Checking: nyxd url: {url}: {}: {}", "failed".red(), e);
|
||||
false
|
||||
}
|
||||
Ok(Err(NyxdError::AbciError { code, log, .. })) => {
|
||||
// We accept the mixnet contract not found as ok from a connection standpoint. This happens
|
||||
// for example on a pre-launch network.
|
||||
tracing::debug!(
|
||||
log::debug!(
|
||||
"Checking: nyxd url: {url}: {}, but with abci error: {code}: {log}",
|
||||
"success".green()
|
||||
);
|
||||
code == 18
|
||||
}
|
||||
Ok(Err(error @ NyxdError::NoContractAddressAvailable(_))) => {
|
||||
tracing::warn!("Checking: nyxd url: {url}: {}: {error}", "failed".red());
|
||||
log::warn!("Checking: nyxd url: {url}: {}: {error}", "failed".red());
|
||||
false
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
// For any other error, we're optimistic and just try anyway.
|
||||
tracing::warn!(
|
||||
log::warn!(
|
||||
"Checking: nyxd_url: {url}: {}, but with error: {e}",
|
||||
"success".green()
|
||||
);
|
||||
true
|
||||
}
|
||||
Ok(Ok(_)) => {
|
||||
tracing::debug!("Checking: nyxd_url: {url}: {}", "success".green());
|
||||
log::debug!("Checking: nyxd_url: {url}: {}", "success".green());
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Checking: nyxd_url: {url}: {}: {e}", "failed".red());
|
||||
log::warn!("Checking: nyxd_url: {url}: {}: {e}", "failed".red());
|
||||
false
|
||||
}
|
||||
};
|
||||
@@ -169,15 +169,15 @@ async fn test_nym_api_connection(
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) => {
|
||||
tracing::debug!("Checking: api_url: {url}: {}", "success".green());
|
||||
log::debug!("Checking: api_url: {url}: {}", "success".green());
|
||||
true
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
|
||||
log::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
|
||||
log::debug!("Checking: api_url: {url}: {}: {e}", "failed".red());
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
@@ -41,7 +41,6 @@ use nym_mixnet_contract_common::mixnode::MixNodeDetails;
|
||||
use nym_mixnet_contract_common::{GatewayBond, IdentityKeyRef, NodeId};
|
||||
use time::format_description::BorrowedFormatItem;
|
||||
use time::Date;
|
||||
use tracing::instrument;
|
||||
|
||||
pub mod error;
|
||||
pub mod routes;
|
||||
@@ -53,13 +52,11 @@ pub fn rfc_3339_date() -> Vec<BorrowedFormatItem<'static>> {
|
||||
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
|
||||
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
|
||||
pub trait NymApiClientExt: ApiClient {
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
|
||||
self.get_json(&[routes::API_VERSION, routes::MIXNODES], NO_PARAMS)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnodes_detailed(&self) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[
|
||||
@@ -73,7 +70,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateways_detailed(&self) -> Result<Vec<GatewayBondAnnotated>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[
|
||||
@@ -87,7 +83,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnodes_detailed_unfiltered(
|
||||
&self,
|
||||
) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
|
||||
@@ -103,13 +98,11 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateways(&self) -> Result<Vec<GatewayBond>, NymAPIError> {
|
||||
self.get_json(&[routes::API_VERSION, routes::GATEWAYS], NO_PARAMS)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateways_described(&self) -> Result<Vec<LegacyDescribedGateway>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::GATEWAYS, routes::DESCRIBED],
|
||||
@@ -118,7 +111,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnodes_described(&self) -> Result<Vec<LegacyDescribedMixNode>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::MIXNODES, routes::DESCRIBED],
|
||||
@@ -127,7 +119,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
async fn get_basic_mixnodes(
|
||||
&self,
|
||||
semver_compatibility: Option<String>,
|
||||
@@ -151,7 +142,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_basic_gateways(
|
||||
&self,
|
||||
semver_compatibility: Option<String>,
|
||||
@@ -177,7 +167,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
|
||||
/// retrieve basic information for nodes are capable of operating as an entry gateway
|
||||
/// this includes legacy gateways and nym-nodes
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_all_basic_entry_assigned_nodes(
|
||||
&self,
|
||||
semver_compatibility: Option<String>,
|
||||
@@ -219,7 +208,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
|
||||
/// retrieve basic information for nodes that got assigned 'mixing' node in this epoch
|
||||
/// this includes legacy mixnodes and nym-nodes
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_basic_active_mixing_assigned_nodes(
|
||||
&self,
|
||||
semver_compatibility: Option<String>,
|
||||
@@ -259,7 +247,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_active_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::MIXNODES, routes::ACTIVE],
|
||||
@@ -268,7 +255,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_active_mixnodes_detailed(&self) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[
|
||||
@@ -283,7 +269,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_rewarded_mixnodes(&self) -> Result<Vec<MixNodeDetails>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::MIXNODES, routes::REWARDED],
|
||||
@@ -292,7 +277,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_report(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -310,7 +294,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateway_report(
|
||||
&self,
|
||||
identity: IdentityKeyRef<'_>,
|
||||
@@ -328,7 +311,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_history(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -346,7 +328,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateway_history(
|
||||
&self,
|
||||
identity: IdentityKeyRef<'_>,
|
||||
@@ -364,7 +345,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_rewarded_mixnodes_detailed(
|
||||
&self,
|
||||
) -> Result<Vec<MixNodeBondAnnotated>, NymAPIError> {
|
||||
@@ -381,7 +361,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateway_core_status_count(
|
||||
&self,
|
||||
identity: IdentityKeyRef<'_>,
|
||||
@@ -413,7 +392,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_core_status_count(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -446,7 +424,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_status(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -464,7 +441,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_reward_estimation(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -482,7 +458,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn compute_mixnode_reward_estimation(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -502,7 +477,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_stake_saturation(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -520,7 +494,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnode_inclusion_probability(
|
||||
&self,
|
||||
mix_id: NodeId,
|
||||
@@ -538,7 +511,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_current_node_performance(
|
||||
&self,
|
||||
node_id: NodeId,
|
||||
@@ -569,7 +541,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_mixnodes_blacklisted(&self) -> Result<Vec<NodeId>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::MIXNODES, routes::BLACKLISTED],
|
||||
@@ -578,7 +549,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_gateways_blacklisted(&self) -> Result<Vec<IdentityKey>, NymAPIError> {
|
||||
self.get_json(
|
||||
&[routes::API_VERSION, routes::GATEWAYS, routes::BLACKLISTED],
|
||||
@@ -587,7 +557,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, request_body))]
|
||||
async fn blind_sign(
|
||||
&self,
|
||||
request_body: &BlindSignRequestBody,
|
||||
@@ -604,7 +573,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, request_body))]
|
||||
async fn verify_ecash_ticket(
|
||||
&self,
|
||||
request_body: &VerifyEcashTicketBody,
|
||||
@@ -621,7 +589,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, request_body))]
|
||||
async fn batch_redeem_ecash_tickets(
|
||||
&self,
|
||||
request_body: &BatchRedeemTicketsBody,
|
||||
@@ -638,7 +605,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn double_spending_filter_v1(&self) -> Result<SpentCredentialsResponse, NymAPIError> {
|
||||
self.get_json(
|
||||
&[
|
||||
@@ -651,7 +617,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn partial_expiration_date_signatures(
|
||||
&self,
|
||||
expiration_date: Option<Date>,
|
||||
@@ -675,7 +640,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn partial_coin_indices_signatures(
|
||||
&self,
|
||||
epoch_id: Option<EpochId>,
|
||||
@@ -696,7 +660,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn global_expiration_date_signatures(
|
||||
&self,
|
||||
expiration_date: Option<Date>,
|
||||
@@ -720,7 +683,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn global_coin_indices_signatures(
|
||||
&self,
|
||||
epoch_id: Option<EpochId>,
|
||||
@@ -741,7 +703,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn master_verification_key(
|
||||
&self,
|
||||
epoch_id: Option<EpochId>,
|
||||
@@ -761,7 +722,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn epoch_credentials(
|
||||
&self,
|
||||
dkg_epoch: EpochId,
|
||||
@@ -778,7 +738,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn issued_credential(
|
||||
&self,
|
||||
credential_id: i64,
|
||||
@@ -795,7 +754,6 @@ pub trait NymApiClientExt: ApiClient {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn issued_credentials(
|
||||
&self,
|
||||
credential_ids: Vec<i64>,
|
||||
|
||||
@@ -8,9 +8,9 @@ use crate::nyxd::CosmWasmClient;
|
||||
use async_trait::async_trait;
|
||||
use cosmrs::AccountId;
|
||||
use cosmwasm_std::Addr;
|
||||
use log::trace;
|
||||
use nym_coconut_dkg_common::types::{ChunkIndex, NodeIndex, StateAdvanceResponse};
|
||||
use serde::Deserialize;
|
||||
use tracing::trace;
|
||||
|
||||
use nym_coconut_dkg_common::dealer::RegisteredDealerDetails;
|
||||
pub use nym_coconut_dkg_common::{
|
||||
|
||||
+4
-3
@@ -29,6 +29,7 @@ use cosmrs::proto::cosmwasm::wasm::v1::{
|
||||
};
|
||||
use cosmrs::tendermint::{block, chain, Hash};
|
||||
use cosmrs::{AccountId, Coin as CosmosCoin, Tx};
|
||||
use log::trace;
|
||||
use prost::Message;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -67,7 +68,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
|
||||
Res: Message + Default,
|
||||
{
|
||||
if let Some(ref abci_path) = path {
|
||||
tracing::trace!("performing query on abci path {abci_path}")
|
||||
trace!("performing query on abci path {abci_path}")
|
||||
}
|
||||
let mut buf = Vec::with_capacity(req.encoded_len());
|
||||
req.encode(&mut buf)?;
|
||||
@@ -296,7 +297,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
|
||||
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
tracing::debug!(
|
||||
log::debug!(
|
||||
"Polling for result of including {} in a block...",
|
||||
broadcasted.hash
|
||||
);
|
||||
@@ -521,7 +522,7 @@ pub trait CosmWasmClient: TendermintRpcClient {
|
||||
.make_abci_query::<_, QuerySmartContractStateResponse>(path, req)
|
||||
.await?;
|
||||
|
||||
tracing::trace!("raw query response: {}", String::from_utf8_lossy(&res.data));
|
||||
trace!("raw query response: {}", String::from_utf8_lossy(&res.data));
|
||||
Ok(serde_json::from_slice(&res.data)?)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -25,12 +25,12 @@ use cosmrs::proto::cosmos::tx::signing::v1beta1::SignMode;
|
||||
use cosmrs::staking::{MsgDelegate, MsgUndelegate};
|
||||
use cosmrs::tx::{self, Msg};
|
||||
use cosmrs::{cosmwasm, AccountId, Any, Tx};
|
||||
use log::debug;
|
||||
use serde::Serialize;
|
||||
use sha2::Digest;
|
||||
use sha2::Sha256;
|
||||
use std::time::SystemTime;
|
||||
use tendermint_rpc::endpoint::broadcast;
|
||||
use tracing::debug;
|
||||
|
||||
fn empty_fee() -> tx::Fee {
|
||||
tx::Fee {
|
||||
|
||||
@@ -7,9 +7,9 @@ use base64::Engine;
|
||||
use cosmrs::abci::TxMsgData;
|
||||
use cosmrs::cosmwasm::MsgExecuteContractResponse;
|
||||
use cosmrs::proto::cosmos::base::query::v1beta1::{PageRequest, PageResponse};
|
||||
use log::error;
|
||||
use prost::bytes::Bytes;
|
||||
use tendermint_rpc::endpoint::broadcast;
|
||||
use tracing::error;
|
||||
|
||||
pub use cosmrs::abci::MsgResponse;
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2022-2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use crate::ecash::error::EcashTicketError;
|
||||
use crate::ecash::state::SharedState;
|
||||
use nym_ecash_double_spending::DoubleSpendingFilter;
|
||||
use nym_gateway_storage::Storage;
|
||||
use nym_task::TaskClient;
|
||||
use nym_validator_client::client::NymApiClientExt;
|
||||
use nym_validator_client::EcashApiClient;
|
||||
use rand::prelude::SliceRandom;
|
||||
use rand::thread_rng;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{RwLock, RwLockReadGuard};
|
||||
use tokio::time::{interval, Duration};
|
||||
use tracing::{info, trace, warn};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DoubleSpendingDetector<S> {
|
||||
spent_serial_numbers: Arc<RwLock<DoubleSpendingFilter>>,
|
||||
shared_state: SharedState<S>,
|
||||
}
|
||||
|
||||
impl<S> DoubleSpendingDetector<S>
|
||||
where
|
||||
S: Storage + Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub(crate) fn new(shared_state: SharedState<S>) -> Self {
|
||||
DoubleSpendingDetector {
|
||||
spent_serial_numbers: Arc::new(RwLock::new(DoubleSpendingFilter::new_empty_ecash())),
|
||||
shared_state,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn check(&self, serial_number: &Vec<u8>) -> bool {
|
||||
self.spent_serial_numbers.read().await.check(serial_number)
|
||||
}
|
||||
|
||||
async fn latest_api_endpoints(
|
||||
&self,
|
||||
) -> Result<RwLockReadGuard<Vec<EcashApiClient>>, EcashTicketError> {
|
||||
let epoch_id = self.shared_state.current_epoch_id().await?;
|
||||
self.shared_state.api_clients(epoch_id).await
|
||||
}
|
||||
|
||||
async fn refresh_bloomfilter(&self) {
|
||||
let mut filter_builder = self.spent_serial_numbers.read().await.rebuild();
|
||||
|
||||
let api_clients = match self.latest_api_endpoints().await {
|
||||
Ok(clients) => clients,
|
||||
Err(err) => {
|
||||
warn!("failed to obtain current api clients: {err}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut clients = api_clients
|
||||
.iter()
|
||||
.map(|c| c.api_client.clone())
|
||||
.collect::<Vec<_>>();
|
||||
clients.shuffle(&mut thread_rng());
|
||||
|
||||
for client in clients {
|
||||
match client.nym_api.double_spending_filter_v1().await {
|
||||
Ok(response) => {
|
||||
// due to relative big size of the filter, query only one api since all of them should contain
|
||||
// roughly the same data anyway.
|
||||
filter_builder.add_bytes(&response.bitmap);
|
||||
*self.spent_serial_numbers.write().await = filter_builder.build();
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Validator @ {} could not be reached. There might be a problem with the ecash endpoint: {err}", client.api_url());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
warn!("none of the validators could be reached. the bloomfilter will remain unchanged.");
|
||||
}
|
||||
|
||||
async fn run(&self, mut shutdown: TaskClient) {
|
||||
info!("Starting Ecash DoubleSpendingDetector");
|
||||
let mut interval = interval(Duration::from_secs(600));
|
||||
|
||||
while !shutdown.is_shutdown() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.recv() => {
|
||||
trace!("ecash_verifier::DoubleSpendingDetector : received shutdown");
|
||||
},
|
||||
_ = interval.tick() => self.refresh_bloomfilter().await,
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start(self, shutdown: nym_task::TaskClient) {
|
||||
tokio::spawn(async move { self.run(shutdown).await });
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
use crate::Error;
|
||||
use credential_sender::CredentialHandler;
|
||||
use credential_sender::CredentialHandlerConfig;
|
||||
use double_spending::DoubleSpendingDetector;
|
||||
use error::EcashTicketError;
|
||||
use futures::channel::mpsc::{self, UnboundedSender};
|
||||
use nym_credentials::CredentialSpendingData;
|
||||
@@ -17,6 +18,7 @@ use tokio::sync::{Mutex, RwLockReadGuard};
|
||||
use tracing::error;
|
||||
|
||||
pub mod credential_sender;
|
||||
pub(crate) mod double_spending;
|
||||
pub mod error;
|
||||
mod helpers;
|
||||
mod state;
|
||||
@@ -29,6 +31,7 @@ pub struct EcashManager<S> {
|
||||
pk_bytes: [u8; 32], // bytes representation of a pub key representing the verifier
|
||||
pay_infos: Mutex<Vec<NymPayInfo>>,
|
||||
cred_sender: UnboundedSender<ClientTicket>,
|
||||
double_spend_detector: DoubleSpendingDetector<S>,
|
||||
}
|
||||
|
||||
impl<S> EcashManager<S>
|
||||
@@ -44,6 +47,9 @@ where
|
||||
) -> Result<Self, Error> {
|
||||
let shared_state = SharedState::new(nyxd_client, storage).await?;
|
||||
|
||||
let double_spend_detector = DoubleSpendingDetector::new(shared_state.clone());
|
||||
double_spend_detector.clone().start(shutdown.clone());
|
||||
|
||||
let (cred_sender, cred_receiver) = mpsc::unbounded();
|
||||
|
||||
let cs =
|
||||
@@ -56,6 +62,7 @@ where
|
||||
pk_bytes,
|
||||
pay_infos: Default::default(),
|
||||
cred_sender,
|
||||
double_spend_detector,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -156,6 +163,10 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn check_double_spend(&self, serial_number: &Vec<u8>) -> bool {
|
||||
self.double_spend_detector.check(serial_number).await
|
||||
}
|
||||
|
||||
pub fn async_verify(&self, ticket: ClientTicket) {
|
||||
// TODO: I guess do something for shutdowns
|
||||
let _ = self
|
||||
|
||||
@@ -53,6 +53,18 @@ impl<S: Storage + Clone + 'static> CredentialVerifier<S> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_bloomfilter(&self, serial_number: &Vec<u8>) -> Result<()> {
|
||||
trace!("checking the bloomfilter...");
|
||||
|
||||
let spent = self.ecash_verifier.check_double_spend(serial_number).await;
|
||||
|
||||
if spent {
|
||||
trace!("the credential has already been spent before at some gateway before (bloomfilter failure)");
|
||||
return Err(Error::BandwidthCredentialAlreadySpent);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_local_db_for_double_spending(&self, serial_number: &[u8]) -> Result<()> {
|
||||
trace!("checking local db for double spending...");
|
||||
|
||||
@@ -116,6 +128,7 @@ impl<S: Storage + Clone + 'static> CredentialVerifier<S> {
|
||||
}
|
||||
|
||||
self.check_credential_spending_date(spend_date.ecash_date())?;
|
||||
self.check_bloomfilter(&serial_number).await?;
|
||||
self.check_local_db_for_double_spending(&serial_number)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
[package]
|
||||
name = "nym-gateway-stats-storage"
|
||||
version = "0.1.0"
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
documentation.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
sqlx = { workspace = true, features = [
|
||||
"runtime-tokio-rustls",
|
||||
"sqlite",
|
||||
"macros",
|
||||
"migrate",
|
||||
"time",
|
||||
] }
|
||||
time = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
nym-sphinx = { path = "../nymsphinx" }
|
||||
nym-credentials-interface = { path = "../credentials-interface" }
|
||||
|
||||
|
||||
[build-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
||||
sqlx = { workspace = true, features = [
|
||||
"runtime-tokio-rustls",
|
||||
"sqlite",
|
||||
"macros",
|
||||
"migrate",
|
||||
] }
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use sqlx::{Connection, SqliteConnection};
|
||||
use std::env;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let out_dir = env::var("OUT_DIR").unwrap();
|
||||
let database_path = format!("{}/gateway-stats-example.sqlite", out_dir);
|
||||
|
||||
let mut conn = SqliteConnection::connect(&format!("sqlite://{}?mode=rwc", database_path))
|
||||
.await
|
||||
.expect("Failed to create SQLx database connection");
|
||||
|
||||
sqlx::migrate!("./migrations")
|
||||
.run(&mut conn)
|
||||
.await
|
||||
.expect("Failed to perform SQLx migrations");
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
// for some strange reason we need to add a leading `/` to the windows path even though it's
|
||||
// not a valid windows path... but hey, it works...
|
||||
println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
*/
|
||||
|
||||
CREATE TABLE sessions_active
|
||||
(
|
||||
client_address TEXT NOT NULL PRIMARY KEY UNIQUE,
|
||||
start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL,
|
||||
typ TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE sessions_finished
|
||||
(
|
||||
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
day DATE NOT NULL,
|
||||
duration_ms INTEGER NOT NULL,
|
||||
typ TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE sessions_unique_users
|
||||
(
|
||||
day DATE NOT NULL,
|
||||
client_address TEXT NOT NULL,
|
||||
PRIMARY KEY (day, client_address)
|
||||
);
|
||||
@@ -1,13 +0,0 @@
|
||||
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum StatsStorageError {
|
||||
#[error("Database experienced an internal error: {0}")]
|
||||
InternalDatabaseError(#[from] sqlx::Error),
|
||||
|
||||
#[error("Failed to perform database migration: {0}")]
|
||||
MigrationError(#[from] sqlx::migrate::MigrateError),
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use error::StatsStorageError;
|
||||
use models::{ActiveSession, FinishedSession, SessionType, StoredFinishedSession};
|
||||
use nym_sphinx::DestinationAddressBytes;
|
||||
use sessions::SessionManager;
|
||||
use sqlx::ConnectOptions;
|
||||
use std::path::Path;
|
||||
use time::Date;
|
||||
use tracing::{debug, error};
|
||||
|
||||
pub mod error;
|
||||
pub mod models;
|
||||
mod sessions;
|
||||
|
||||
// note that clone here is fine as upon cloning the same underlying pool will be used
|
||||
#[derive(Clone)]
|
||||
pub struct PersistentStatsStorage {
|
||||
session_manager: SessionManager,
|
||||
}
|
||||
|
||||
impl PersistentStatsStorage {
|
||||
/// Initialises `PersistentStatsStorage` using the provided path.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `database_path`: path to the database.
|
||||
pub async fn init<P: AsRef<Path> + Send>(database_path: P) -> Result<Self, StatsStorageError> {
|
||||
debug!(
|
||||
"Attempting to connect to database {:?}",
|
||||
database_path.as_ref().as_os_str()
|
||||
);
|
||||
|
||||
// TODO: we can inject here more stuff based on our gateway global config
|
||||
// struct. Maybe different pool size or timeout intervals?
|
||||
let opts = sqlx::sqlite::SqliteConnectOptions::new()
|
||||
.filename(database_path)
|
||||
.create_if_missing(true)
|
||||
.disable_statement_logging();
|
||||
|
||||
// TODO: do we want auto_vacuum ?
|
||||
|
||||
let connection_pool = match sqlx::SqlitePool::connect_with(opts).await {
|
||||
Ok(db) => db,
|
||||
Err(err) => {
|
||||
error!("Failed to connect to SQLx database: {err}");
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = sqlx::migrate!("./migrations").run(&connection_pool).await {
|
||||
error!("Failed to perform migration on the SQLx database: {err}");
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
// the cloning here are cheap as connection pool is stored behind an Arc
|
||||
Ok(PersistentStatsStorage {
|
||||
session_manager: sessions::SessionManager::new(connection_pool),
|
||||
})
|
||||
}
|
||||
|
||||
//Sessions fn
|
||||
pub async fn insert_finished_session(
|
||||
&self,
|
||||
date: Date,
|
||||
session: FinishedSession,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.insert_finished_session(
|
||||
date,
|
||||
session.duration.whole_milliseconds() as i64,
|
||||
session.typ.to_string().into(),
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn get_finished_sessions(
|
||||
&self,
|
||||
date: Date,
|
||||
) -> Result<Vec<StoredFinishedSession>, StatsStorageError> {
|
||||
Ok(self.session_manager.get_finished_sessions(date).await?)
|
||||
}
|
||||
|
||||
pub async fn delete_finished_sessions(
|
||||
&self,
|
||||
before_date: Date,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.delete_finished_sessions(before_date)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn insert_unique_user(
|
||||
&self,
|
||||
date: Date,
|
||||
client_address_bs58: String,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.insert_unique_user(date, client_address_bs58)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn get_unique_users_count(&self, date: Date) -> Result<i32, StatsStorageError> {
|
||||
Ok(self.session_manager.get_unique_users_count(date).await?)
|
||||
}
|
||||
|
||||
pub async fn delete_unique_users(&self, before_date: Date) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.delete_unique_users(before_date)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn insert_active_session(
|
||||
&self,
|
||||
client_address: DestinationAddressBytes,
|
||||
session: ActiveSession,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.insert_active_session(
|
||||
client_address.as_base58_string(),
|
||||
session.start,
|
||||
session.typ.to_string().into(),
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn update_active_session_type(
|
||||
&self,
|
||||
client_address: DestinationAddressBytes,
|
||||
session_type: SessionType,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.update_active_session_type(
|
||||
client_address.as_base58_string(),
|
||||
session_type.to_string().into(),
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn get_active_session(
|
||||
&self,
|
||||
client_address: DestinationAddressBytes,
|
||||
) -> Result<Option<ActiveSession>, StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.get_active_session(client_address.as_base58_string())
|
||||
.await?
|
||||
.map(Into::into))
|
||||
}
|
||||
|
||||
pub async fn get_all_active_sessions(&self) -> Result<Vec<ActiveSession>, StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.get_all_active_sessions()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(Into::into)
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn get_started_sessions_count(
|
||||
&self,
|
||||
start_date: Date,
|
||||
) -> Result<i32, StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.get_started_sessions_count(start_date)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn get_active_users(&self) -> Result<Vec<String>, StatsStorageError> {
|
||||
Ok(self.session_manager.get_active_users().await?)
|
||||
}
|
||||
|
||||
pub async fn delete_active_session(
|
||||
&self,
|
||||
client_address: DestinationAddressBytes,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
Ok(self
|
||||
.session_manager
|
||||
.delete_active_session(client_address.as_base58_string())
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn cleanup_active_sessions(&self) -> Result<(), StatsStorageError> {
|
||||
Ok(self.session_manager.cleanup_active_sessions().await?)
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use nym_credentials_interface::TicketType;
|
||||
use sqlx::prelude::FromRow;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
|
||||
#[derive(FromRow)]
|
||||
pub struct StoredFinishedSession {
|
||||
duration_ms: i64,
|
||||
typ: String,
|
||||
}
|
||||
|
||||
impl StoredFinishedSession {
|
||||
pub fn serialize(&self) -> (u64, String) {
|
||||
(
|
||||
self.duration_ms as u64, //we are sure that it fits in a u64, see `fn end_at`
|
||||
self.typ.clone(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FinishedSession {
|
||||
pub duration: Duration,
|
||||
pub typ: SessionType,
|
||||
}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
pub enum SessionType {
|
||||
Vpn,
|
||||
Mixnet,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl SessionType {
|
||||
pub fn to_string(&self) -> &str {
|
||||
match self {
|
||||
Self::Vpn => "vpn",
|
||||
Self::Mixnet => "mixnet",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_string(s: &str) -> Self {
|
||||
match s {
|
||||
"vpn" => Self::Vpn,
|
||||
"mixnet" => Self::Mixnet,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TicketType> for SessionType {
|
||||
fn from(value: TicketType) -> Self {
|
||||
match value {
|
||||
TicketType::V1MixnetEntry => Self::Mixnet,
|
||||
TicketType::V1MixnetExit => Self::Mixnet,
|
||||
TicketType::V1WireguardEntry => Self::Vpn,
|
||||
TicketType::V1WireguardExit => Self::Vpn,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(FromRow)]
|
||||
pub(crate) struct StoredActiveSession {
|
||||
start_time: OffsetDateTime,
|
||||
typ: String,
|
||||
}
|
||||
|
||||
pub struct ActiveSession {
|
||||
pub start: OffsetDateTime,
|
||||
pub typ: SessionType,
|
||||
}
|
||||
|
||||
impl ActiveSession {
|
||||
pub fn new(start_time: OffsetDateTime) -> Self {
|
||||
ActiveSession {
|
||||
start: start_time,
|
||||
typ: SessionType::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_type(&mut self, ticket_type: TicketType) {
|
||||
self.typ = ticket_type.into();
|
||||
}
|
||||
|
||||
pub fn end_at(self, stop_time: OffsetDateTime) -> Option<FinishedSession> {
|
||||
let session_duration = stop_time - self.start;
|
||||
//ensure duration is positive to fit in a u64
|
||||
//u64::max milliseconds is 500k millenia so no overflow issue
|
||||
if session_duration > Duration::ZERO {
|
||||
Some(FinishedSession {
|
||||
duration: session_duration,
|
||||
typ: self.typ,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredActiveSession> for ActiveSession {
|
||||
fn from(value: StoredActiveSession) -> Self {
|
||||
ActiveSession {
|
||||
start: value.start_time,
|
||||
typ: SessionType::from_string(&value.typ),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use time::{Date, OffsetDateTime};
|
||||
|
||||
use crate::models::{StoredActiveSession, StoredFinishedSession};
|
||||
|
||||
pub(crate) type Result<T> = std::result::Result<T, sqlx::Error>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SessionManager {
|
||||
connection_pool: sqlx::SqlitePool,
|
||||
}
|
||||
|
||||
impl SessionManager {
|
||||
/// Creates new instance of the `SessionsManager` with the provided sqlite connection pool.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `connection_pool`: database connection pool to use.
|
||||
pub(crate) fn new(connection_pool: sqlx::SqlitePool) -> Self {
|
||||
SessionManager { connection_pool }
|
||||
}
|
||||
|
||||
pub(crate) async fn insert_finished_session(
|
||||
&self,
|
||||
date: Date,
|
||||
duration_ms: i64,
|
||||
typ: String,
|
||||
) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO sessions_finished (day, duration_ms, typ) VALUES (?, ?, ?)",
|
||||
date,
|
||||
duration_ms,
|
||||
typ
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_finished_sessions(
|
||||
&self,
|
||||
date: Date,
|
||||
) -> Result<Vec<StoredFinishedSession>> {
|
||||
sqlx::query_as("SELECT duration_ms, typ FROM sessions_finished WHERE day = ?")
|
||||
.bind(date)
|
||||
.fetch_all(&self.connection_pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_finished_sessions(&self, before_date: Date) -> Result<()> {
|
||||
sqlx::query!("DELETE FROM sessions_finished WHERE day <= ? ", before_date)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn insert_unique_user(
|
||||
&self,
|
||||
date: Date,
|
||||
client_address_b58: String,
|
||||
) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"INSERT OR IGNORE INTO sessions_unique_users (day, client_address) VALUES (?, ?)",
|
||||
date,
|
||||
client_address_b58,
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_unique_users_count(&self, date: Date) -> Result<i32> {
|
||||
Ok(sqlx::query!(
|
||||
"SELECT COUNT(*) as count FROM sessions_unique_users WHERE day = ?",
|
||||
date
|
||||
)
|
||||
.fetch_one(&self.connection_pool)
|
||||
.await?
|
||||
.count)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_unique_users(&self, before_date: Date) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"DELETE FROM sessions_unique_users WHERE day <= ? ",
|
||||
before_date
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn insert_active_session(
|
||||
&self,
|
||||
client_address_b58: String,
|
||||
start_time: OffsetDateTime,
|
||||
typ: String,
|
||||
) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"INSERT INTO sessions_active (client_address, start_time, typ) VALUES (?, ?, ?)",
|
||||
client_address_b58,
|
||||
start_time,
|
||||
typ
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn update_active_session_type(
|
||||
&self,
|
||||
client_address_b58: String,
|
||||
typ: String,
|
||||
) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"UPDATE sessions_active SET typ = ? WHERE client_address = ?",
|
||||
typ,
|
||||
client_address_b58,
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_active_session(
|
||||
&self,
|
||||
client_address_b58: String,
|
||||
) -> Result<Option<StoredActiveSession>> {
|
||||
sqlx::query_as("SELECT start_time, typ FROM sessions_active WHERE client_address = ?")
|
||||
.bind(client_address_b58)
|
||||
.fetch_optional(&self.connection_pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_all_active_sessions(&self) -> Result<Vec<StoredActiveSession>> {
|
||||
sqlx::query_as("SELECT start_time, typ FROM sessions_active")
|
||||
.fetch_all(&self.connection_pool)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_started_sessions_count(&self, start_date: Date) -> Result<i32> {
|
||||
Ok(sqlx::query!(
|
||||
"SELECT COUNT(*) as count FROM sessions_active WHERE date(start_time) = ?",
|
||||
start_date
|
||||
)
|
||||
.fetch_one(&self.connection_pool)
|
||||
.await?
|
||||
.count)
|
||||
}
|
||||
|
||||
pub(crate) async fn get_active_users(&self) -> Result<Vec<String>> {
|
||||
Ok(sqlx::query!("SELECT client_address from sessions_active")
|
||||
.fetch_all(&self.connection_pool)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|record| record.client_address)
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_active_session(&self, client_address_b58: String) -> Result<()> {
|
||||
sqlx::query!(
|
||||
"DELETE FROM sessions_active WHERE client_address = ?",
|
||||
client_address_b58
|
||||
)
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_active_sessions(&self) -> Result<()> {
|
||||
sqlx::query!("DELETE FROM sessions_active")
|
||||
.execute(&self.connection_pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Display;
|
||||
use std::time::Duration;
|
||||
use thiserror::Error;
|
||||
use tracing::{instrument, warn};
|
||||
use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
pub use reqwest::IntoUrl;
|
||||
@@ -202,7 +202,6 @@ impl Client {
|
||||
self.reqwest_client.get(url)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip_all, fields(path=?path))]
|
||||
async fn send_get_request<K, V, E>(
|
||||
&self,
|
||||
path: PathSegments<'_>,
|
||||
@@ -213,7 +212,6 @@ impl Client {
|
||||
V: AsRef<str>,
|
||||
E: Display,
|
||||
{
|
||||
tracing::trace!("Sending GET request");
|
||||
let url = sanitize_url(&self.base_url, path, params);
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
@@ -279,7 +277,6 @@ impl Client {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip_all)]
|
||||
pub async fn get_json<T, K, V, E>(
|
||||
&self,
|
||||
path: PathSegments<'_>,
|
||||
@@ -515,14 +512,12 @@ pub fn sanitize_url<K: AsRef<str>, V: AsRef<str>>(
|
||||
url
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn parse_response<T, E>(res: Response, allow_empty: bool) -> Result<T, HttpClientError<E>>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
E: DeserializeOwned + Display,
|
||||
{
|
||||
let status = res.status();
|
||||
tracing::debug!("Status: {} (success: {})", &status, status.is_success());
|
||||
|
||||
if !allow_empty {
|
||||
if let Some(0) = res.content_length() {
|
||||
@@ -531,18 +526,6 @@ where
|
||||
}
|
||||
|
||||
if res.status().is_success() {
|
||||
#[cfg(debug_assertions)]
|
||||
{
|
||||
let text = res.text().await.inspect_err(|err| {
|
||||
tracing::error!("Couldn't even get response text: {err}");
|
||||
})?;
|
||||
tracing::trace!("Result:\n{:#?}", text);
|
||||
|
||||
serde_json::from_str(&text)
|
||||
.map_err(|err| HttpClientError::GenericRequestFailure(err.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(not(debug_assertions))]
|
||||
Ok(res.json().await?)
|
||||
} else if res.status() == StatusCode::NOT_FOUND {
|
||||
Err(HttpClientError::NotFound)
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
[package]
|
||||
name = "nym-common-models"
|
||||
version = "0.1.0"
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
documentation.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
readme.workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
@@ -1 +0,0 @@
|
||||
pub mod ns_api;
|
||||
@@ -1,8 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TestrunAssignment {
|
||||
/// has nothing to do with GW identity key. This is PK from `gateways` table
|
||||
pub testrun_id: i64,
|
||||
pub gateway_pk_id: i64,
|
||||
}
|
||||
@@ -19,10 +19,10 @@ pub enum TicketTypeRepr {
|
||||
}
|
||||
|
||||
impl TicketTypeRepr {
|
||||
pub const WIREGUARD_ENTRY_TICKET_SIZE: u64 = 500 * 1000 * 1000; // 500 MB
|
||||
pub const WIREGUARD_EXIT_TICKET_SIZE: u64 = 500 * 1000 * 1000; // 500 MB
|
||||
pub const MIXNET_ENTRY_TICKET_SIZE: u64 = 200 * 1000 * 1000; // 200 MB
|
||||
pub const MIXNET_EXIT_TICKET_SIZE: u64 = 100 * 1000 * 1000; // 100 MB
|
||||
pub const WIREGUARD_ENTRY_TICKET_SIZE: u64 = 512 * 1024 * 1024; // 512 MB
|
||||
pub const WIREGUARD_EXIT_TICKET_SIZE: u64 = 512 * 1024 * 1024; // 512 MB
|
||||
pub const MIXNET_ENTRY_TICKET_SIZE: u64 = 128 * 1024 * 1024; // 128 MB
|
||||
pub const MIXNET_EXIT_TICKET_SIZE: u64 = 128 * 1024 * 1024; // 128 MB
|
||||
|
||||
/// How much bandwidth (in bytes) one ticket can grant
|
||||
pub const fn bandwidth_value(&self) -> u64 {
|
||||
|
||||
@@ -21,7 +21,7 @@ nym-sphinx-types = { path = "../types" }
|
||||
nym-topology = { path = "../../topology" }
|
||||
|
||||
[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen]
|
||||
version = "0.2.95"
|
||||
version = "0.2.93"
|
||||
|
||||
[dev-dependencies]
|
||||
rand_chacha = { workspace = true }
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::fragment::{linked_fragment_payload_max_len, unlinked_fragment_payload_max_len};
|
||||
use fragment::FragmentHeader;
|
||||
use dashmap::DashMap;
|
||||
use fragment::{Fragment, FragmentHeader};
|
||||
use nym_crypto::asymmetric::ed25519::PublicKey;
|
||||
use serde::Serialize;
|
||||
pub use set::split_into_sets;
|
||||
@@ -26,59 +29,6 @@ pub mod fragment;
|
||||
pub mod reconstruction;
|
||||
pub mod set;
|
||||
|
||||
pub mod monitoring {
|
||||
use crate::fragment::Fragment;
|
||||
use crate::{ReceivedFragment, SentFragment};
|
||||
use dashmap::DashMap;
|
||||
use nym_crypto::asymmetric::ed25519::PublicKey;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub static FRAGMENTS_RECEIVED: LazyLock<DashMap<i32, Vec<ReceivedFragment>>> =
|
||||
LazyLock::new(DashMap::new);
|
||||
|
||||
pub static FRAGMENTS_SENT: LazyLock<DashMap<i32, Vec<SentFragment>>> =
|
||||
LazyLock::new(DashMap::new);
|
||||
|
||||
pub fn enable() {
|
||||
ENABLED.store(true, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn enabled() -> bool {
|
||||
ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! now {
|
||||
() => {
|
||||
match std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) {
|
||||
Ok(n) => n.as_secs(),
|
||||
Err(_) => 0,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn fragment_received(fragment: &Fragment) {
|
||||
if enabled() {
|
||||
let id = fragment.fragment_identifier().set_id();
|
||||
let mut entry = FRAGMENTS_RECEIVED.entry(id).or_default();
|
||||
let r = ReceivedFragment::new(fragment.header(), now!());
|
||||
entry.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fragment_sent(fragment: &Fragment, client_nonce: i32, destination: PublicKey, hops: u8) {
|
||||
if enabled() {
|
||||
let id = fragment.fragment_identifier().set_id();
|
||||
let mut entry = FRAGMENTS_SENT.entry(id).or_default();
|
||||
let s = SentFragment::new(fragment.header(), now!(), client_nonce, destination, hops);
|
||||
entry.push(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FragmentMixParams {
|
||||
destination: PublicKey,
|
||||
@@ -162,6 +112,35 @@ impl ReceivedFragment {
|
||||
}
|
||||
}
|
||||
|
||||
pub static FRAGMENTS_RECEIVED: LazyLock<DashMap<i32, Vec<ReceivedFragment>>> =
|
||||
LazyLock::new(DashMap::new);
|
||||
|
||||
pub static FRAGMENTS_SENT: LazyLock<DashMap<i32, Vec<SentFragment>>> = LazyLock::new(DashMap::new);
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! now {
|
||||
() => {
|
||||
match std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) {
|
||||
Ok(n) => n.as_secs(),
|
||||
Err(_) => 0,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn fragment_received(fragment: &Fragment) {
|
||||
let id = fragment.fragment_identifier().set_id();
|
||||
let mut entry = FRAGMENTS_RECEIVED.entry(id).or_default();
|
||||
let r = ReceivedFragment::new(fragment.header(), now!());
|
||||
entry.push(r);
|
||||
}
|
||||
|
||||
pub fn fragment_sent(fragment: &Fragment, client_nonce: i32, destination: PublicKey, hops: u8) {
|
||||
let id = fragment.fragment_identifier().set_id();
|
||||
let mut entry = FRAGMENTS_SENT.entry(id).or_default();
|
||||
let s = SentFragment::new(fragment.header(), now!(), client_nonce, destination, hops);
|
||||
entry.push(s);
|
||||
}
|
||||
|
||||
/// The idea behind the process of chunking is to incur as little data overhead as possible due
|
||||
/// to very computationally costly sphinx encapsulation procedure.
|
||||
///
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
use crate::fragment::Fragment;
|
||||
use crate::{monitoring, ChunkingError};
|
||||
use crate::{fragment_received, ChunkingError};
|
||||
use log::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -110,7 +110,7 @@ impl ReconstructionBuffer {
|
||||
}
|
||||
});
|
||||
|
||||
monitoring::fragment_received(&fragment);
|
||||
fragment_received(&fragment);
|
||||
|
||||
let fragment_index = fragment.current_fragment() as usize - 1;
|
||||
if self.fragments[fragment_index].is_some() {
|
||||
|
||||
@@ -12,6 +12,7 @@ use nym_sphinx_addressing::clients::Recipient;
|
||||
use nym_sphinx_addressing::nodes::NymNodeRoutingAddress;
|
||||
use nym_sphinx_anonymous_replies::reply_surb::ReplySurb;
|
||||
use nym_sphinx_chunking::fragment::{Fragment, FragmentIdentifier};
|
||||
use nym_sphinx_chunking::fragment_sent;
|
||||
use nym_sphinx_forwarding::packet::MixPacket;
|
||||
use nym_sphinx_params::packet_sizes::PacketSize;
|
||||
use nym_sphinx_params::{PacketType, ReplySurbKeyDigestAlgorithm, DEFAULT_NUM_MIX_HOPS};
|
||||
@@ -20,7 +21,6 @@ use nym_topology::{NymTopology, NymTopologyError};
|
||||
use rand::{CryptoRng, Rng, SeedableRng};
|
||||
use rand_chacha::ChaCha20Rng;
|
||||
|
||||
use nym_sphinx_chunking::monitoring;
|
||||
use std::time::Duration;
|
||||
|
||||
pub(crate) mod payload;
|
||||
@@ -206,7 +206,7 @@ pub trait FragmentPreparer {
|
||||
|
||||
let destination = packet_recipient.gateway();
|
||||
let hops = mix_hops.unwrap_or(self.num_mix_hops());
|
||||
monitoring::fragment_sent(&fragment, self.nonce(), *destination, hops);
|
||||
fragment_sent(&fragment, self.nonce(), *destination, hops);
|
||||
|
||||
let non_reply_overhead = encryption::PUBLIC_KEY_SIZE;
|
||||
let expected_plaintext = match packet_type {
|
||||
|
||||
@@ -42,32 +42,8 @@ impl PendingSync {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlockProcessorConfig {
|
||||
pub pruning_options: PruningOptions,
|
||||
pub store_precommits: bool,
|
||||
}
|
||||
|
||||
impl Default for BlockProcessorConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pruning_options: PruningOptions::nothing(),
|
||||
store_precommits: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BlockProcessorConfig {
|
||||
pub fn new(pruning_options: PruningOptions, store_precommits: bool) -> Self {
|
||||
Self {
|
||||
pruning_options,
|
||||
store_precommits,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BlockProcessor {
|
||||
config: BlockProcessorConfig,
|
||||
pruning_options: PruningOptions,
|
||||
cancel: CancellationToken,
|
||||
synced: Arc<Notify>,
|
||||
last_processed_height: u32,
|
||||
@@ -89,10 +65,9 @@ pub struct BlockProcessor {
|
||||
msg_modules: Vec<Box<dyn MsgModule + Send>>,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
impl BlockProcessor {
|
||||
pub async fn new(
|
||||
config: BlockProcessorConfig,
|
||||
pruning_options: PruningOptions,
|
||||
cancel: CancellationToken,
|
||||
synced: Arc<Notify>,
|
||||
incoming: UnboundedReceiver<BlockToProcess>,
|
||||
@@ -107,7 +82,7 @@ impl BlockProcessor {
|
||||
let last_pruned_height = last_pruned.try_into().unwrap_or_default();
|
||||
|
||||
Ok(BlockProcessor {
|
||||
config,
|
||||
pruning_options,
|
||||
cancel,
|
||||
synced,
|
||||
last_processed_height,
|
||||
@@ -126,7 +101,7 @@ impl BlockProcessor {
|
||||
}
|
||||
|
||||
pub fn with_pruning(mut self, pruning_options: PruningOptions) -> Self {
|
||||
self.config.pruning_options = pruning_options;
|
||||
self.pruning_options = pruning_options;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -153,7 +128,7 @@ impl BlockProcessor {
|
||||
// we won't end up with a corrupted storage.
|
||||
let mut tx = self.storage.begin_processing_tx().await?;
|
||||
|
||||
persist_block(&full_info, &mut tx, self.config.store_precommits).await?;
|
||||
persist_block(&full_info, &mut tx).await?;
|
||||
|
||||
// let the modules do whatever they want
|
||||
// the ones wanting the full block:
|
||||
@@ -266,7 +241,7 @@ impl BlockProcessor {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
async fn prune_storage(&mut self) -> Result<(), ScraperError> {
|
||||
let keep_recent = self.config.pruning_options.strategy_keep_recent();
|
||||
let keep_recent = self.pruning_options.strategy_keep_recent();
|
||||
let last_to_keep = self.last_processed_height - keep_recent;
|
||||
|
||||
info!(
|
||||
@@ -307,12 +282,12 @@ impl BlockProcessor {
|
||||
async fn maybe_prune_storage(&mut self) -> Result<(), ScraperError> {
|
||||
debug!("checking for storage pruning");
|
||||
|
||||
if self.config.pruning_options.strategy.is_nothing() {
|
||||
if self.pruning_options.strategy.is_nothing() {
|
||||
trace!("the current pruning strategy is 'nothing'");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let interval = self.config.pruning_options.strategy_interval();
|
||||
let interval = self.pruning_options.strategy_interval();
|
||||
if self.last_pruned_height + interval <= self.last_processed_height {
|
||||
self.prune_storage().await?;
|
||||
}
|
||||
@@ -396,7 +371,7 @@ impl BlockProcessor {
|
||||
if latest_block > self.last_processed_height && self.last_processed_height != 0 {
|
||||
// in case we were offline for a while,
|
||||
// make sure we don't request blocks we'd have to prune anyway
|
||||
let keep_recent = self.config.pruning_options.strategy_keep_recent();
|
||||
let keep_recent = self.pruning_options.strategy_keep_recent();
|
||||
let last_to_keep = latest_block - keep_recent;
|
||||
self.last_processed_height = max(self.last_processed_height, last_to_keep);
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use crate::block_processor::types::BlockToProcess;
|
||||
use crate::block_processor::{BlockProcessor, BlockProcessorConfig};
|
||||
use crate::block_processor::BlockProcessor;
|
||||
use crate::block_requester::{BlockRequest, BlockRequester};
|
||||
use crate::error::ScraperError;
|
||||
use crate::modules::{BlockModule, MsgModule, TxModule};
|
||||
@@ -34,8 +34,6 @@ pub struct Config {
|
||||
pub database_path: PathBuf,
|
||||
|
||||
pub pruning_options: PruningOptions,
|
||||
|
||||
pub store_precommits: bool,
|
||||
}
|
||||
|
||||
pub struct NyxdScraperBuilder {
|
||||
@@ -47,10 +45,7 @@ pub struct NyxdScraperBuilder {
|
||||
}
|
||||
|
||||
impl NyxdScraperBuilder {
|
||||
pub async fn build_and_start(
|
||||
self,
|
||||
start_block: Option<u32>,
|
||||
) -> Result<NyxdScraper, ScraperError> {
|
||||
pub async fn build_and_start(self) -> Result<NyxdScraper, ScraperError> {
|
||||
let scraper = NyxdScraper::new(self.config).await?;
|
||||
|
||||
let (processing_tx, processing_rx) = unbounded_channel();
|
||||
@@ -65,14 +60,8 @@ impl NyxdScraperBuilder {
|
||||
req_rx,
|
||||
processing_tx.clone(),
|
||||
);
|
||||
|
||||
let block_processor_config = BlockProcessorConfig::new(
|
||||
scraper.config.pruning_options,
|
||||
scraper.config.store_precommits,
|
||||
);
|
||||
|
||||
let mut block_processor = BlockProcessor::new(
|
||||
block_processor_config,
|
||||
scraper.config.pruning_options,
|
||||
scraper.cancel_token.clone(),
|
||||
scraper.startup_sync.clone(),
|
||||
processing_rx,
|
||||
@@ -93,10 +82,6 @@ impl NyxdScraperBuilder {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let Some(height) = start_block {
|
||||
scraper.process_block_range(Some(height), None).await?;
|
||||
}
|
||||
|
||||
scraper.start_tasks(block_requester, block_processor, chain_subscriber);
|
||||
|
||||
Ok(scraper)
|
||||
@@ -209,10 +194,10 @@ impl NyxdScraper {
|
||||
.await?
|
||||
.with_pruning(PruningOptions::nothing());
|
||||
|
||||
let mut current_height = self.rpc_client.current_block_height().await? as u32;
|
||||
let current_height = self.rpc_client.current_block_height().await? as u32;
|
||||
let last_processed = block_processor.last_process_height();
|
||||
|
||||
let mut starting_height = match starting_height {
|
||||
let starting_height = match starting_height {
|
||||
// always attempt to use whatever the user has provided
|
||||
Some(explicit) => explicit,
|
||||
None => {
|
||||
@@ -226,8 +211,7 @@ impl NyxdScraper {
|
||||
}
|
||||
};
|
||||
|
||||
let must_catch_up = end_height.is_none();
|
||||
let mut end_height = match end_height {
|
||||
let end_height = match end_height {
|
||||
// always attempt to use whatever the user has provided
|
||||
Some(explicit) => explicit,
|
||||
None => {
|
||||
@@ -242,62 +226,32 @@ impl NyxdScraper {
|
||||
}
|
||||
};
|
||||
|
||||
let mut last_processed = starting_height;
|
||||
info!(
|
||||
starting_height = starting_height,
|
||||
end_height = end_height,
|
||||
"attempting to process block range"
|
||||
);
|
||||
|
||||
while last_processed < current_height {
|
||||
info!(
|
||||
starting_height = starting_height,
|
||||
end_height = end_height,
|
||||
"attempting to process block range"
|
||||
);
|
||||
let range = (starting_height..=end_height).collect::<Vec<_>>();
|
||||
|
||||
let range = (starting_height..=end_height).collect::<Vec<_>>();
|
||||
|
||||
// the most likely bottleneck here are going to be the chain queries,
|
||||
// so batch multiple requests
|
||||
for batch in range.chunks(4) {
|
||||
let batch_result = join_all(
|
||||
batch
|
||||
.iter()
|
||||
.map(|height| self.rpc_client.get_basic_block_details(*height)),
|
||||
)
|
||||
.await;
|
||||
for result in batch_result {
|
||||
match result {
|
||||
Ok(block) => block_processor.process_block(block.into()).await?,
|
||||
Err(err) => {
|
||||
error!("failed to retrieve the block: {err}. stopping...");
|
||||
return Err(err);
|
||||
}
|
||||
// the most likely bottleneck here are going to be the chain queries,
|
||||
// so batch multiple requests
|
||||
for batch in range.chunks(4) {
|
||||
let batch_result = join_all(
|
||||
batch
|
||||
.iter()
|
||||
.map(|height| self.rpc_client.get_basic_block_details(*height)),
|
||||
)
|
||||
.await;
|
||||
for result in batch_result {
|
||||
match result {
|
||||
Ok(block) => block_processor.process_block(block.into()).await?,
|
||||
Err(err) => {
|
||||
error!("failed to retrieve the block: {err}. stopping...");
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we don't need to catch up, return early
|
||||
if !must_catch_up {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// check if we have caught up to the current block height
|
||||
last_processed = end_height;
|
||||
current_height = self.rpc_client.current_block_height().await? as u32;
|
||||
|
||||
info!(
|
||||
last_processed = last_processed,
|
||||
current_height = current_height,
|
||||
"🏃 still need to catch up..."
|
||||
);
|
||||
|
||||
starting_height = last_processed + 1;
|
||||
end_height = current_height;
|
||||
}
|
||||
|
||||
if must_catch_up {
|
||||
info!(
|
||||
last_processed = last_processed,
|
||||
current_height = current_height,
|
||||
"✅ block processing has caught up!"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -321,11 +275,8 @@ impl NyxdScraper {
|
||||
req_tx: Sender<BlockRequest>,
|
||||
processing_rx: UnboundedReceiver<BlockToProcess>,
|
||||
) -> Result<BlockProcessor, ScraperError> {
|
||||
let block_processor_config =
|
||||
BlockProcessorConfig::new(self.config.pruning_options, self.config.store_precommits);
|
||||
|
||||
BlockProcessor::new(
|
||||
block_processor_config,
|
||||
self.config.pruning_options,
|
||||
self.cancel_token.clone(),
|
||||
self.startup_sync.clone(),
|
||||
processing_rx,
|
||||
|
||||
@@ -237,58 +237,6 @@ impl StorageManager {
|
||||
Ok(-1)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_transactions_after_height(
|
||||
&self,
|
||||
min_height: i64,
|
||||
message_type: Option<&str>,
|
||||
) -> Result<Vec<TransactionWithBlock>, sqlx::Error> {
|
||||
match message_type {
|
||||
Some(msg_type) => {
|
||||
sqlx::query_as!(
|
||||
TransactionWithBlock,
|
||||
r#"
|
||||
SELECT t.hash, t.height, t.memo, t.raw_log
|
||||
FROM message m
|
||||
JOIN "transaction" t ON m.transaction_hash = t.hash
|
||||
JOIN block b ON t.height = b.height
|
||||
WHERE t.height > ?
|
||||
AND m.type = ?
|
||||
ORDER BY t.height ASC
|
||||
"#,
|
||||
min_height,
|
||||
msg_type
|
||||
)
|
||||
.fetch_all(&self.connection_pool)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
sqlx::query_as!(
|
||||
TransactionWithBlock,
|
||||
r#"
|
||||
SELECT t.hash, t.height, t.memo, t.raw_log
|
||||
FROM message m
|
||||
JOIN "transaction" t ON m.transaction_hash = t.hash
|
||||
JOIN block b ON t.height = b.height
|
||||
WHERE t.height > ?
|
||||
ORDER BY t.height ASC
|
||||
"#,
|
||||
min_height
|
||||
)
|
||||
.fetch_all(&self.connection_pool)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
pub struct TransactionWithBlock {
|
||||
pub hash: String,
|
||||
pub height: i64,
|
||||
pub memo: Option<String>,
|
||||
pub raw_log: Option<String>,
|
||||
}
|
||||
|
||||
// make those generic over executor so that they could be performed over connection pool and a tx
|
||||
|
||||
@@ -13,7 +13,6 @@ use crate::{
|
||||
models::{CommitSignature, Validator},
|
||||
},
|
||||
};
|
||||
use manager::TransactionWithBlock;
|
||||
use sqlx::{types::time::OffsetDateTime, ConnectOptions, Sqlite, Transaction};
|
||||
use std::{fmt::Debug, path::Path};
|
||||
use tendermint::{
|
||||
@@ -208,23 +207,11 @@ impl ScraperStorage {
|
||||
pub async fn get_pruned_height(&self) -> Result<i64, ScraperError> {
|
||||
Ok(self.manager.get_pruned_height().await?)
|
||||
}
|
||||
|
||||
pub async fn get_transactions_after_height(
|
||||
&self,
|
||||
min_height: i64,
|
||||
message_type: Option<&str>,
|
||||
) -> Result<Vec<TransactionWithBlock>, ScraperError> {
|
||||
Ok(self
|
||||
.manager
|
||||
.get_transactions_after_height(min_height, message_type)
|
||||
.await?)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn persist_block(
|
||||
block: &FullBlockInformation,
|
||||
tx: &mut StorageTransaction,
|
||||
store_precommits: bool,
|
||||
) -> Result<(), ScraperError> {
|
||||
let total_gas = crate::helpers::tx_gas_sum(&block.transactions);
|
||||
|
||||
@@ -237,12 +224,11 @@ pub async fn persist_block(
|
||||
// persist block data
|
||||
persist_block_data(&block.block, total_gas, tx).await?;
|
||||
|
||||
if store_precommits {
|
||||
if let Some(commit) = &block.block.last_commit {
|
||||
persist_commits(commit, &block.validators, tx).await?;
|
||||
} else {
|
||||
warn!("no commits for block {}", block.block.header.height)
|
||||
}
|
||||
// persist commits
|
||||
if let Some(commit) = &block.block.last_commit {
|
||||
persist_commits(commit, &block.validators, tx).await?;
|
||||
} else {
|
||||
warn!("no commits for block {}", block.block.header.height)
|
||||
}
|
||||
|
||||
// persist txs
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("peers in wireguard don't match with in-memory ")]
|
||||
PeerMismatch,
|
||||
|
||||
#[error("traffic byte data needs to be increasing")]
|
||||
InconsistentConsumedBytes,
|
||||
|
||||
@@ -17,7 +20,4 @@ pub enum Error {
|
||||
|
||||
#[error("{0}")]
|
||||
GatewayStorage(#[from] nym_gateway_storage::error::StorageError),
|
||||
|
||||
#[error("{0}")]
|
||||
SystemTime(#[from] std::time::SystemTimeError),
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::{error::Error, peer_handle::SharedBandwidthStorageManager};
|
||||
pub enum PeerControlRequest {
|
||||
AddPeer {
|
||||
peer: Peer,
|
||||
client_id: Option<i64>,
|
||||
ticket_validation: bool,
|
||||
response_tx: oneshot::Sender<AddPeerControlResponse>,
|
||||
},
|
||||
RemovePeer {
|
||||
@@ -46,6 +46,7 @@ pub enum PeerControlRequest {
|
||||
|
||||
pub struct AddPeerControlResponse {
|
||||
pub success: bool,
|
||||
pub client_id: Option<i64>,
|
||||
}
|
||||
|
||||
pub struct RemovePeerControlResponse {
|
||||
@@ -117,13 +118,13 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
}
|
||||
|
||||
// Function that should be used for peer insertion, to handle both storage and kernel interaction
|
||||
pub async fn add_peer(&self, peer: &Peer, client_id: Option<i64>) -> Result<(), Error> {
|
||||
if client_id.is_none() {
|
||||
self.storage.insert_wireguard_peer(peer, false).await?;
|
||||
}
|
||||
let ret: Result<(), defguard_wireguard_rs::error::WireguardInterfaceError> =
|
||||
self.wg_api.inner.configure_peer(peer);
|
||||
if client_id.is_none() && ret.is_err() {
|
||||
pub async fn add_peer(&self, peer: &Peer, with_client_id: bool) -> Result<Option<i64>, Error> {
|
||||
let client_id = self
|
||||
.storage
|
||||
.insert_wireguard_peer(peer, with_client_id)
|
||||
.await?;
|
||||
let ret = self.wg_api.inner.configure_peer(peer);
|
||||
if ret.is_err() {
|
||||
// Try to revert the insertion in storage
|
||||
if self
|
||||
.storage
|
||||
@@ -134,7 +135,8 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
log::error!("The storage has been corrupted. Wireguard peer {} will persist in storage indefinitely.", peer.public_key);
|
||||
}
|
||||
}
|
||||
Ok(ret?)
|
||||
ret?;
|
||||
Ok(client_id)
|
||||
}
|
||||
|
||||
// Function that should be used for peer removal, to handle both storage and kernel interaction
|
||||
@@ -158,10 +160,13 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
.ok_or(Error::MissingClientBandwidthEntry)?
|
||||
.client_id
|
||||
{
|
||||
storage.create_bandwidth_entry(client_id).await?;
|
||||
let bandwidth = storage
|
||||
.get_available_bandwidth(client_id)
|
||||
.await?
|
||||
.ok_or(Error::MissingClientBandwidthEntry)?;
|
||||
Ok(Some(BandwidthStorageManager::new(
|
||||
storage,
|
||||
ClientBandwidth::new(Default::default()),
|
||||
ClientBandwidth::new(bandwidth.into()),
|
||||
client_id,
|
||||
BandwidthFlushingBehaviourConfig::default(),
|
||||
true,
|
||||
@@ -174,9 +179,9 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
async fn handle_add_request(
|
||||
&mut self,
|
||||
peer: &Peer,
|
||||
client_id: Option<i64>,
|
||||
) -> Result<(), Error> {
|
||||
self.add_peer(peer, client_id).await?;
|
||||
with_client_id: bool,
|
||||
) -> Result<Option<i64>, Error> {
|
||||
let client_id = self.add_peer(peer, with_client_id).await?;
|
||||
let bandwidth_storage_manager =
|
||||
Self::generate_bandwidth_manager(self.storage.clone(), &peer.public_key)
|
||||
.await?
|
||||
@@ -196,7 +201,7 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
log::error!("Peer handle shut down ungracefully - {e}");
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
Ok(client_id)
|
||||
}
|
||||
|
||||
async fn handle_query_peer(&self, key: &Key) -> Result<Option<Peer>, Error> {
|
||||
@@ -223,10 +228,14 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
.available_bandwidth()
|
||||
.await
|
||||
} else {
|
||||
let Some(peer) = self.host_information.read().await.peers.get(key).cloned() else {
|
||||
// host information not updated yet
|
||||
return Ok(None);
|
||||
};
|
||||
let peer = self
|
||||
.host_information
|
||||
.read()
|
||||
.await
|
||||
.peers
|
||||
.get(key)
|
||||
.ok_or(Error::PeerMismatch)?
|
||||
.clone();
|
||||
BANDWIDTH_CAP_PER_DAY.saturating_sub((peer.rx_bytes + peer.tx_bytes) as i64)
|
||||
};
|
||||
|
||||
@@ -251,12 +260,12 @@ impl<St: Storage + Clone + 'static> PeerController<St> {
|
||||
}
|
||||
msg = self.request_rx.recv() => {
|
||||
match msg {
|
||||
Some(PeerControlRequest::AddPeer { peer, client_id, response_tx }) => {
|
||||
let ret = self.handle_add_request(&peer, client_id).await;
|
||||
if ret.is_ok() {
|
||||
response_tx.send(AddPeerControlResponse { success: true }).ok();
|
||||
Some(PeerControlRequest::AddPeer { peer, ticket_validation, response_tx }) => {
|
||||
let ret = self.handle_add_request(&peer, ticket_validation).await;
|
||||
if let Ok(client_id) = ret {
|
||||
response_tx.send(AddPeerControlResponse { success: true, client_id }).ok();
|
||||
} else {
|
||||
response_tx.send(AddPeerControlResponse { success: false }).ok();
|
||||
response_tx.send(AddPeerControlResponse { success: false, client_id: None }).ok();
|
||||
}
|
||||
}
|
||||
Some(PeerControlRequest::RemovePeer { key, response_tx }) => {
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::peer_controller::PeerControlRequest;
|
||||
use defguard_wireguard_rs::host::Peer;
|
||||
use defguard_wireguard_rs::{host::Host, key::Key};
|
||||
use futures::channel::oneshot;
|
||||
use nym_authenticator_requests::v2::registration::BANDWIDTH_CAP_PER_DAY;
|
||||
@@ -13,12 +12,10 @@ use nym_gateway_storage::Storage;
|
||||
use nym_task::TaskClient;
|
||||
use nym_wireguard_types::DEFAULT_PEER_TIMEOUT_CHECK;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::{mpsc, RwLock};
|
||||
use tokio_stream::{wrappers::IntervalStream, StreamExt};
|
||||
|
||||
pub(crate) type SharedBandwidthStorageManager<St> = Arc<RwLock<BandwidthStorageManager<St>>>;
|
||||
const AUTO_REMOVE_AFTER: Duration = Duration::from_secs(60 * 60 * 24); // 24 hours
|
||||
|
||||
pub struct PeerHandle<St> {
|
||||
storage: St,
|
||||
@@ -28,7 +25,6 @@ pub struct PeerHandle<St> {
|
||||
request_tx: mpsc::Sender<PeerControlRequest>,
|
||||
timeout_check_interval: IntervalStream,
|
||||
task_client: TaskClient,
|
||||
startup_timestamp: SystemTime,
|
||||
}
|
||||
|
||||
impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
@@ -43,8 +39,7 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
let timeout_check_interval = tokio_stream::wrappers::IntervalStream::new(
|
||||
tokio::time::interval(DEFAULT_PEER_TIMEOUT_CHECK),
|
||||
);
|
||||
let mut task_client = task_client.fork(format!("peer-{public_key}"));
|
||||
task_client.disarm();
|
||||
let task_client = task_client.fork(format!("peer{public_key}"));
|
||||
PeerHandle {
|
||||
storage,
|
||||
public_key,
|
||||
@@ -53,11 +48,14 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
request_tx,
|
||||
timeout_check_interval,
|
||||
task_client,
|
||||
startup_timestamp: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_peer(&self) -> Result<bool, Error> {
|
||||
async fn remove_depleted_peer(&self) -> Result<bool, Error> {
|
||||
log::debug!(
|
||||
"Peer {} doesn't have bandwidth anymore, removing it",
|
||||
self.public_key.to_string()
|
||||
);
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
self.request_tx
|
||||
.send(PeerControlRequest::RemovePeer {
|
||||
@@ -73,11 +71,15 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
Ok(success)
|
||||
}
|
||||
|
||||
async fn active_peer(
|
||||
&mut self,
|
||||
storage_peer: WireguardPeer,
|
||||
kernel_peer: Peer,
|
||||
) -> Result<bool, Error> {
|
||||
async fn active_peer(&mut self, storage_peer: WireguardPeer) -> Result<bool, Error> {
|
||||
let kernel_peer = self
|
||||
.host_information
|
||||
.read()
|
||||
.await
|
||||
.peers
|
||||
.get(&self.public_key)
|
||||
.ok_or(Error::PeerMismatch)?
|
||||
.clone();
|
||||
if let Some(bandwidth_manager) = &self.bandwidth_storage_manager {
|
||||
let spent_bandwidth = (kernel_peer.rx_bytes + kernel_peer.tx_bytes)
|
||||
.checked_sub(storage_peer.rx_bytes as u64 + storage_peer.tx_bytes as u64)
|
||||
@@ -91,25 +93,13 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
let success = self.remove_peer().await?;
|
||||
let success = self.remove_depleted_peer().await?;
|
||||
return Ok(!success);
|
||||
}
|
||||
} else {
|
||||
if SystemTime::now().duration_since(self.startup_timestamp)? >= AUTO_REMOVE_AFTER {
|
||||
log::debug!(
|
||||
"Peer {} has been present for 24 hours, removing it",
|
||||
self.public_key.to_string()
|
||||
);
|
||||
let success = self.remove_peer().await?;
|
||||
return Ok(!success);
|
||||
}
|
||||
let spent_bandwidth = kernel_peer.rx_bytes + kernel_peer.tx_bytes;
|
||||
if spent_bandwidth >= BANDWIDTH_CAP_PER_DAY {
|
||||
log::debug!(
|
||||
"Peer {} doesn't have bandwidth anymore, removing it",
|
||||
self.public_key.to_string()
|
||||
);
|
||||
let success = self.remove_peer().await?;
|
||||
let success = self.remove_depleted_peer().await?;
|
||||
return Ok(!success);
|
||||
}
|
||||
}
|
||||
@@ -121,21 +111,11 @@ impl<St: Storage + Clone + 'static> PeerHandle<St> {
|
||||
while !self.task_client.is_shutdown() {
|
||||
tokio::select! {
|
||||
_ = self.timeout_check_interval.next() => {
|
||||
let Some(kernel_peer) = self
|
||||
.host_information
|
||||
.read()
|
||||
.await
|
||||
.peers
|
||||
.get(&self.public_key)
|
||||
.cloned() else {
|
||||
// the host information hasn't beed updated yet
|
||||
continue;
|
||||
};
|
||||
let Some(storage_peer) = self.storage.get_wireguard_peer(&self.public_key.to_string()).await? else {
|
||||
let Some(peer) = self.storage.get_wireguard_peer(&self.public_key.to_string()).await? else {
|
||||
log::debug!("Peer {:?} not in storage anymore, shutting down handle", self.public_key);
|
||||
return Ok(());
|
||||
};
|
||||
if !self.active_peer(storage_peer, kernel_peer).await? {
|
||||
if !self.active_peer(peer).await? {
|
||||
log::debug!("Peer {:?} doesn't have bandwidth anymore, shutting down handle", self.public_key);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
+2
-2
@@ -19,5 +19,5 @@ MULTISIG_CONTRACT_ADDRESS=n1zwv6feuzhy6a9wekh96cd57lsarmqlwxdypdsplw6zhfncqw6ftq
|
||||
COCONUT_DKG_CONTRACT_ADDRESS=n1aakfpghcanxtc45gpqlx8j3rq0zcpyf49qmhm9mdjrfx036h4z5sy2vfh9
|
||||
|
||||
EXPLORER_API=https://canary-explorer.performance.nymte.ch/api
|
||||
NYXD=https://canary-validator.performance.nymte.ch
|
||||
NYM_API=https://canary-api.performance.nymte.ch/api
|
||||
NYXD="https://canary-validator.performance.nymte.ch"
|
||||
NYM_API="https://canary-api.performance.nymte.ch/api"
|
||||
|
||||
+3
-3
@@ -21,8 +21,8 @@ COCONUT_DKG_CONTRACT_ADDRESS=n19604yflqggs9mk2z26mqygq43q2kr3n932egxx630svywd5mp
|
||||
|
||||
REWARDING_VALIDATOR_ADDRESS=n10yyd98e2tuwu0f7ypz9dy3hhjw7v772q6287gy
|
||||
STATISTICS_SERVICE_DOMAIN_ADDRESS="https://mainnet-stats.nymte.ch:8090"
|
||||
NYXD=https://rpc.nymtech.net
|
||||
NYM_API=https://validator.nymtech.net/api/
|
||||
NYXD="https://rpc.nymtech.net"
|
||||
NYM_API="https://validator.nymtech.net/api/"
|
||||
NYXD_WS="wss://rpc.nymtech.net/websocket"
|
||||
EXPLORER_API=https://explorer.nymtech.net/api/
|
||||
EXPLORER_API="https://explorer.nymtech.net/api/"
|
||||
NYM_VPN_API="https://nymvpn.com/api"
|
||||
|
||||
+2
-2
@@ -19,5 +19,5 @@ VESTING_CONTRACT_ADDRESS=n1jlzdxnyces4hrhqz68dqk28mrw5jgwtcfq0c2funcwrmw0dx9l9s8
|
||||
REWARDING_VALIDATOR_ADDRESS=n1rfvpsynktze6wvn6ldskj8xgwfzzk5v6pnff39
|
||||
|
||||
EXPLORER_API=https://qa-network-explorer.qa.nymte.ch/api
|
||||
NYXD=https://qa-validator.qa.nymte.ch
|
||||
NYM_API=https://qa-nym-api.qa.nymte.ch/api
|
||||
NYXD="https://qa-validator.qa.nymte.ch"
|
||||
NYM_API="https://qa-nym-api.qa.nymte.ch/api"
|
||||
|
||||
+3
-3
@@ -20,6 +20,6 @@ ECASH_CONTRACT_ADDRESS=n1v3vydvs2ued84yv3khqwtgldmgwn0elljsdh08dr5s2j9x4rc5fs9jl
|
||||
|
||||
STATISTICS_SERVICE_DOMAIN_ADDRESS="http://0.0.0.0"
|
||||
EXPLORER_API=https://sandbox-explorer.nymtech.net/api
|
||||
NYXD=https://rpc.sandbox.nymtech.net
|
||||
NYXD_WS=wss://rpc.sandbox.nymtech.net/websocket
|
||||
NYM_API=https://sandbox-nym-api1.nymtech.net/api
|
||||
NYXD="https://rpc.sandbox.nymtech.net"
|
||||
NYXD_WS="wss://rpc.sandbox.nymtech.net/websocket"
|
||||
NYM_API="https://sandbox-nym-api1.nymtech.net/api"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "explorer-api"
|
||||
version = "1.1.41"
|
||||
version = "1.1.40"
|
||||
edition = "2021"
|
||||
license.workspace = true
|
||||
|
||||
|
||||
@@ -7,12 +7,12 @@ license.workspace = true
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
log.workspace = true
|
||||
nym-explorer-api-requests = { path = "../explorer-api-requests" }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
serde.workspace = true
|
||||
thiserror.workspace = true
|
||||
url.workspace = true
|
||||
tracing = {workspace = true, features = ["attributes"]}
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
|
||||
@@ -3,7 +3,6 @@ use std::time::Duration;
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use thiserror::Error;
|
||||
use tracing::instrument;
|
||||
use url::Url;
|
||||
|
||||
// Re-export request types
|
||||
@@ -48,12 +47,6 @@ impl ExplorerClient {
|
||||
Ok(Self { client, url })
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn new_with_timeout(url: url::Url, timeout: Duration) -> Result<Self, ExplorerApiError> {
|
||||
let client = reqwest::Client::builder().timeout(timeout).build()?;
|
||||
Ok(Self { client, url })
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn new(url: url::Url) -> Result<Self, ExplorerApiError> {
|
||||
let client = reqwest::Client::builder().build()?;
|
||||
@@ -65,11 +58,10 @@ impl ExplorerClient {
|
||||
paths: &[&str],
|
||||
) -> Result<reqwest::Response, ExplorerApiError> {
|
||||
let url = combine_url(self.url.clone(), paths)?;
|
||||
tracing::debug!("Sending GET request");
|
||||
log::trace!("Sending GET request {url:?}");
|
||||
Ok(self.client.get(url).send().await?)
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all, fields(paths=?paths))]
|
||||
async fn query_explorer_api<T>(&self, paths: &[&str]) -> Result<T, ExplorerApiError>
|
||||
where
|
||||
T: std::fmt::Debug,
|
||||
@@ -78,14 +70,12 @@ impl ExplorerClient {
|
||||
let response = self.send_get_request(paths).await?;
|
||||
if response.status().is_success() {
|
||||
let res = response.json::<T>().await?;
|
||||
tracing::trace!("Got response: {res:?}");
|
||||
log::trace!("Got response: {res:?}");
|
||||
Ok(res)
|
||||
} else if response.status() == StatusCode::NOT_FOUND {
|
||||
Err(ExplorerApiError::NotFound)
|
||||
} else {
|
||||
let status = response.status();
|
||||
let err_msg = format!("{}: {}", response.text().await?, status);
|
||||
Err(ExplorerApiError::RequestFailure(err_msg))
|
||||
Err(ExplorerApiError::RequestFailure(response.text().await?))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,6 @@ nym-credentials-interface = { path = "../common/credentials-interface" }
|
||||
nym-credential-verification = { path = "../common/credential-verification" }
|
||||
nym-crypto = { path = "../common/crypto" }
|
||||
nym-gateway-storage = { path = "../common/gateway-storage" }
|
||||
nym-gateway-stats-storage = { path = "../common/gateway-stats-storage" }
|
||||
nym-gateway-requests = { path = "../common/gateway-requests" }
|
||||
nym-mixnet-client = { path = "../common/client-libs/mixnet-client" }
|
||||
nym-mixnode-common = { path = "../common/mixnode-common" }
|
||||
|
||||
@@ -12,7 +12,6 @@ pub const DEFAULT_PRIVATE_SPHINX_KEY_FILENAME: &str = "private_sphinx.pem";
|
||||
pub const DEFAULT_PUBLIC_SPHINX_KEY_FILENAME: &str = "public_sphinx.pem";
|
||||
|
||||
pub const DEFAULT_CLIENTS_STORAGE_FILENAME: &str = "db.sqlite";
|
||||
pub const DEFAULT_STATS_STORAGE_FILENAME: &str = "stats.sqlite";
|
||||
|
||||
pub const DEFAULT_NETWORK_REQUESTER_CONFIG_FILENAME: &str = "network_requester_config.toml";
|
||||
pub const DEFAULT_NETWORK_REQUESTER_DATA_DIR: &str = "network-requester-data";
|
||||
@@ -40,9 +39,6 @@ pub struct GatewayPaths {
|
||||
#[serde(alias = "persistent_storage")]
|
||||
pub clients_storage: PathBuf,
|
||||
|
||||
/// Path to sqlite database containing all persistent stats data.
|
||||
pub stats_storage: PathBuf,
|
||||
|
||||
/// Path to the configuration of the embedded network requester.
|
||||
#[serde(deserialize_with = "de_maybe_stringified")]
|
||||
pub network_requester_config: Option<PathBuf>,
|
||||
@@ -58,9 +54,7 @@ impl GatewayPaths {
|
||||
pub fn new_default<P: AsRef<Path>>(id: P) -> Self {
|
||||
GatewayPaths {
|
||||
keys: KeysPaths::new_default(id.as_ref()),
|
||||
clients_storage: default_data_directory(id.as_ref())
|
||||
.join(DEFAULT_CLIENTS_STORAGE_FILENAME),
|
||||
stats_storage: default_data_directory(id).join(DEFAULT_STATS_STORAGE_FILENAME),
|
||||
clients_storage: default_data_directory(id).join(DEFAULT_CLIENTS_STORAGE_FILENAME),
|
||||
// node_description: default_config_filepath(id).join(DEFAULT_DESCRIPTION_FILENAME),
|
||||
network_requester_config: None,
|
||||
ip_packet_router_config: None,
|
||||
@@ -76,7 +70,6 @@ impl GatewayPaths {
|
||||
public_sphinx_key_file: Default::default(),
|
||||
},
|
||||
clients_storage: Default::default(),
|
||||
stats_storage: Default::default(),
|
||||
network_requester_config: None,
|
||||
ip_packet_router_config: None,
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use nym_authenticator::error::AuthenticatorError;
|
||||
use nym_gateway_stats_storage::error::StatsStorageError;
|
||||
use nym_gateway_storage::error::StorageError;
|
||||
use nym_ip_packet_router::error::IpPacketRouterError;
|
||||
use nym_network_requester::error::{ClientCoreError, NetworkRequesterError};
|
||||
@@ -116,12 +115,6 @@ pub enum GatewayError {
|
||||
source: StorageError,
|
||||
},
|
||||
|
||||
#[error("stats storage failure: {source}")]
|
||||
StatsStorageError {
|
||||
#[from]
|
||||
source: StatsStorageError,
|
||||
},
|
||||
|
||||
#[error("Path to network requester configuration file hasn't been specified. Perhaps try to run `setup-network-requester`?")]
|
||||
UnspecifiedNetworkRequesterConfig,
|
||||
|
||||
|
||||
@@ -73,9 +73,6 @@ pub(crate) enum InitialAuthenticationError {
|
||||
#[error("Only 'Register' or 'Authenticate' requests are allowed")]
|
||||
InvalidRequest,
|
||||
|
||||
#[error("received a Message of type {typ} which was not expected in this context")]
|
||||
UnexpectedMessageType { typ: String },
|
||||
|
||||
#[error("Experienced connection error: {0}")]
|
||||
ConnectionError(#[from] WsError),
|
||||
|
||||
@@ -864,27 +861,9 @@ where
|
||||
Message::Binary(_) => {
|
||||
return Err(InitialAuthenticationError::BinaryRequestWithoutAuthentication);
|
||||
}
|
||||
other => {
|
||||
if other.is_ping() {
|
||||
debug!("unexpected ping message!");
|
||||
return Err(InitialAuthenticationError::UnexpectedMessageType {
|
||||
typ: "ping".to_string(),
|
||||
});
|
||||
} else if other.is_pong() {
|
||||
debug!("unexpected pong message!");
|
||||
return Err(InitialAuthenticationError::UnexpectedMessageType {
|
||||
typ: "pong".to_string(),
|
||||
});
|
||||
} else if other.is_close() {
|
||||
debug!("unexpected close message!");
|
||||
return Err(InitialAuthenticationError::UnexpectedMessageType {
|
||||
typ: "close".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// at this point this is definitely unreachable, but just in case, let's not panic...
|
||||
return Err(InitialAuthenticationError::InvalidRequest);
|
||||
}
|
||||
_ => unreachable!(
|
||||
"the underlying tunsgenite stream should be handling other message types"
|
||||
),
|
||||
};
|
||||
|
||||
text.parse()
|
||||
|
||||
@@ -5,7 +5,6 @@ use crate::config::Config;
|
||||
use crate::error::GatewayError;
|
||||
|
||||
use nym_crypto::asymmetric::encryption;
|
||||
use nym_gateway_stats_storage::PersistentStatsStorage;
|
||||
use nym_gateway_storage::PersistentStorage;
|
||||
use nym_pemstore::traits::PemStorableKeyPair;
|
||||
use nym_pemstore::KeyPairPath;
|
||||
@@ -75,14 +74,6 @@ pub(crate) async fn initialise_main_storage(
|
||||
Ok(PersistentStorage::init(path, retrieval_limit).await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn initialise_stats_storage(
|
||||
config: &Config,
|
||||
) -> Result<PersistentStatsStorage, GatewayError> {
|
||||
let path = &config.storage_paths.stats_storage;
|
||||
|
||||
Ok(PersistentStatsStorage::init(path).await?)
|
||||
}
|
||||
|
||||
pub fn load_keypair<T: PemStorableKeyPair>(
|
||||
paths: KeyPairPath,
|
||||
name: impl Into<String>,
|
||||
|
||||
+14
-25
@@ -12,9 +12,7 @@ use crate::http::HttpApiBuilder;
|
||||
use crate::node::client_handling::active_clients::ActiveClientsStore;
|
||||
use crate::node::client_handling::embedded_clients::{LocalEmbeddedClientHandle, MessageRouter};
|
||||
use crate::node::client_handling::websocket;
|
||||
use crate::node::helpers::{
|
||||
initialise_main_storage, initialise_stats_storage, load_network_requester_config,
|
||||
};
|
||||
use crate::node::helpers::{initialise_main_storage, load_network_requester_config};
|
||||
use crate::node::mixnet_handling::receiver::connection_handler::ConnectionHandler;
|
||||
use futures::channel::{mpsc, oneshot};
|
||||
use nym_credential_verification::ecash::{
|
||||
@@ -43,7 +41,6 @@ pub(crate) mod helpers;
|
||||
pub(crate) mod mixnet_handling;
|
||||
pub(crate) mod statistics;
|
||||
|
||||
pub use nym_gateway_stats_storage::PersistentStatsStorage;
|
||||
pub use nym_gateway_storage::{PersistentStorage, Storage};
|
||||
|
||||
// TODO: should this struct live here?
|
||||
@@ -99,8 +96,6 @@ pub async fn create_gateway(
|
||||
|
||||
let storage = initialise_main_storage(&config).await?;
|
||||
|
||||
let stats_storage = initialise_stats_storage(&config).await?;
|
||||
|
||||
let nr_opts = network_requester_config.map(|config| LocalNetworkRequesterOpts {
|
||||
config: config.clone(),
|
||||
custom_mixnet_path: custom_mixnet.clone(),
|
||||
@@ -111,7 +106,7 @@ pub async fn create_gateway(
|
||||
custom_mixnet_path: custom_mixnet.clone(),
|
||||
});
|
||||
|
||||
Gateway::new(config, nr_opts, ip_opts, storage, stats_storage)
|
||||
Gateway::new(config, nr_opts, ip_opts, storage)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -152,9 +147,7 @@ pub struct Gateway<St = PersistentStorage> {
|
||||
/// x25519 keypair used for Diffie-Hellman. Currently only used for sphinx key derivation.
|
||||
sphinx_keypair: Arc<encryption::KeyPair>,
|
||||
|
||||
client_storage: St,
|
||||
|
||||
stats_storage: PersistentStatsStorage,
|
||||
storage: St,
|
||||
|
||||
wireguard_data: Option<nym_wireguard::WireguardData>,
|
||||
|
||||
@@ -170,12 +163,10 @@ impl<St> Gateway<St> {
|
||||
config: Config,
|
||||
network_requester_opts: Option<LocalNetworkRequesterOpts>,
|
||||
ip_packet_router_opts: Option<LocalIpPacketRouterOpts>,
|
||||
client_storage: St,
|
||||
stats_storage: PersistentStatsStorage,
|
||||
storage: St,
|
||||
) -> Result<Self, GatewayError> {
|
||||
Ok(Gateway {
|
||||
client_storage,
|
||||
stats_storage,
|
||||
storage,
|
||||
identity_keypair: Arc::new(load_identity_keys(&config)?),
|
||||
sphinx_keypair: Arc::new(helpers::load_sphinx_keys(&config)?),
|
||||
config,
|
||||
@@ -188,7 +179,7 @@ impl<St> Gateway<St> {
|
||||
task_client: None,
|
||||
})
|
||||
}
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
|
||||
pub fn new_loaded(
|
||||
config: Config,
|
||||
network_requester_opts: Option<LocalNetworkRequesterOpts>,
|
||||
@@ -196,8 +187,7 @@ impl<St> Gateway<St> {
|
||||
authenticator_opts: Option<LocalAuthenticatorOpts>,
|
||||
identity_keypair: Arc<identity::KeyPair>,
|
||||
sphinx_keypair: Arc<encryption::KeyPair>,
|
||||
client_storage: St,
|
||||
stats_storage: PersistentStatsStorage,
|
||||
storage: St,
|
||||
) -> Self {
|
||||
Gateway {
|
||||
config,
|
||||
@@ -206,8 +196,7 @@ impl<St> Gateway<St> {
|
||||
authenticator_opts,
|
||||
identity_keypair,
|
||||
sphinx_keypair,
|
||||
client_storage,
|
||||
stats_storage,
|
||||
storage,
|
||||
wireguard_data: None,
|
||||
session_stats: None,
|
||||
run_http_server: true,
|
||||
@@ -251,7 +240,7 @@ impl<St> Gateway<St> {
|
||||
|
||||
let connection_handler = ConnectionHandler::new(
|
||||
packet_processor,
|
||||
self.client_storage.clone(),
|
||||
self.storage.clone(),
|
||||
ack_sender,
|
||||
active_clients_store,
|
||||
);
|
||||
@@ -286,7 +275,7 @@ impl<St> Gateway<St> {
|
||||
forwarding_channel,
|
||||
router_tx,
|
||||
);
|
||||
let all_peers = self.client_storage.get_all_wireguard_peers().await?;
|
||||
let all_peers = self.storage.get_all_wireguard_peers().await?;
|
||||
let used_private_network_ips = all_peers
|
||||
.iter()
|
||||
.cloned()
|
||||
@@ -341,7 +330,7 @@ impl<St> Gateway<St> {
|
||||
.start_with_shutdown(router_shutdown);
|
||||
|
||||
let wg_api = nym_wireguard::start_wireguard(
|
||||
self.client_storage.clone(),
|
||||
self.storage.clone(),
|
||||
all_peers,
|
||||
shutdown,
|
||||
wireguard_data,
|
||||
@@ -388,7 +377,7 @@ impl<St> Gateway<St> {
|
||||
|
||||
let shared_state = websocket::CommonHandlerState {
|
||||
ecash_verifier,
|
||||
storage: self.client_storage.clone(),
|
||||
storage: self.storage.clone(),
|
||||
local_identity: Arc::clone(&self.identity_keypair),
|
||||
only_coconut_credentials: self.config.gateway.only_coconut_credentials,
|
||||
bandwidth_cfg: (&self.config).into(),
|
||||
@@ -426,7 +415,7 @@ impl<St> Gateway<St> {
|
||||
info!("Starting gateway stats collector...");
|
||||
|
||||
let (mut stats_collector, stats_event_sender) =
|
||||
GatewayStatisticsCollector::new(shared_session_stats, self.stats_storage.clone());
|
||||
GatewayStatisticsCollector::new(shared_session_stats);
|
||||
tokio::spawn(async move { stats_collector.run(shutdown).await });
|
||||
stats_event_sender
|
||||
}
|
||||
@@ -665,7 +654,7 @@ impl<St> Gateway<St> {
|
||||
nyxd_client,
|
||||
self.identity_keypair.public_key().to_bytes(),
|
||||
shutdown.fork("EcashVerifier"),
|
||||
self.client_storage.clone(),
|
||||
self.storage.clone(),
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
|
||||
@@ -2,14 +2,13 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use futures::{channel::mpsc, StreamExt};
|
||||
use nym_gateway_stats_storage::PersistentStatsStorage;
|
||||
use nym_node_http_api::state::metrics::SharedSessionStats;
|
||||
use nym_statistics_common::events::{StatsEvent, StatsEventReceiver, StatsEventSender};
|
||||
use nym_task::TaskClient;
|
||||
use sessions::SessionStatsHandler;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{error, trace, warn};
|
||||
use tracing::trace;
|
||||
|
||||
pub mod sessions;
|
||||
|
||||
@@ -24,38 +23,21 @@ pub(crate) struct GatewayStatisticsCollector {
|
||||
impl GatewayStatisticsCollector {
|
||||
pub fn new(
|
||||
shared_session_stats: SharedSessionStats,
|
||||
stats_storage: PersistentStatsStorage,
|
||||
) -> (GatewayStatisticsCollector, StatsEventSender) {
|
||||
let (stats_event_tx, stats_event_rx) = mpsc::unbounded();
|
||||
|
||||
let session_stats = SessionStatsHandler::new(shared_session_stats, stats_storage);
|
||||
let collector = GatewayStatisticsCollector {
|
||||
stats_event_rx,
|
||||
session_stats,
|
||||
session_stats: SessionStatsHandler::new(shared_session_stats),
|
||||
};
|
||||
(collector, stats_event_tx)
|
||||
}
|
||||
|
||||
async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
|
||||
if let Err(e) = self
|
||||
.session_stats
|
||||
.maybe_update_shared_state(update_time)
|
||||
.await
|
||||
{
|
||||
error!("Failed to update session stats - {e}");
|
||||
}
|
||||
self.session_stats.update_shared_state(update_time).await;
|
||||
//here goes additionnal stats handler update
|
||||
}
|
||||
|
||||
async fn on_start(&mut self) {
|
||||
if let Err(e) = self.session_stats.on_start().await {
|
||||
error!("Failed to cleanup session stats handler - {e}");
|
||||
}
|
||||
//here goes additionnal stats handler start cleanup
|
||||
}
|
||||
|
||||
pub async fn run(&mut self, mut shutdown: TaskClient) {
|
||||
self.on_start().await;
|
||||
let mut update_interval = tokio::time::interval(STATISTICS_UPDATE_TIMER_INTERVAL);
|
||||
while !shutdown.is_shutdown() {
|
||||
tokio::select! {
|
||||
@@ -71,10 +53,7 @@ impl GatewayStatisticsCollector {
|
||||
Some(stat_event) = self.stats_event_rx.next() => {
|
||||
//dispatching event to proper handler
|
||||
match stat_event {
|
||||
StatsEvent::SessionStatsEvent(event) => {
|
||||
if let Err(e) = self.session_stats.handle_event(event).await{
|
||||
warn!("Session event handling error - {e}");
|
||||
}},
|
||||
StatsEvent::SessionStatsEvent(event) => self.session_stats.handle_event(event),
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -2,158 +2,176 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
use nym_credentials_interface::TicketType;
|
||||
use nym_gateway_stats_storage::models::FinishedSession;
|
||||
use nym_gateway_stats_storage::PersistentStatsStorage;
|
||||
use nym_gateway_stats_storage::{error::StatsStorageError, models::ActiveSession};
|
||||
use nym_node_http_api::state::metrics::SharedSessionStats;
|
||||
use nym_sphinx::DestinationAddressBytes;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use time::{Date, Duration, OffsetDateTime};
|
||||
|
||||
use nym_statistics_common::events::SessionEvent;
|
||||
|
||||
const FINISHED_SESSIONS_CAP: usize = 1_000_000; //to be on the safe side of memory blowups until persistent storage
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum SessionType {
|
||||
Vpn,
|
||||
Mixnet,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl SessionType {
|
||||
fn to_string(&self) -> &str {
|
||||
match self {
|
||||
Self::Vpn => "vpn",
|
||||
Self::Mixnet => "mixnet",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TicketType> for SessionType {
|
||||
fn from(value: TicketType) -> Self {
|
||||
match value {
|
||||
TicketType::V1MixnetEntry => Self::Mixnet,
|
||||
TicketType::V1MixnetExit => Self::Mixnet,
|
||||
TicketType::V1WireguardEntry => Self::Vpn,
|
||||
TicketType::V1WireguardExit => Self::Vpn,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FinishedSession {
|
||||
duration: Duration,
|
||||
typ: SessionType,
|
||||
}
|
||||
|
||||
impl FinishedSession {
|
||||
fn serialize(&self) -> (u64, String) {
|
||||
(
|
||||
self.duration.whole_milliseconds() as u64, //we are sure that it fits in a u64, see `fn end_at`
|
||||
self.typ.to_string().into(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
struct ActiveSession {
|
||||
start: OffsetDateTime,
|
||||
typ: SessionType,
|
||||
}
|
||||
|
||||
impl ActiveSession {
|
||||
fn new(start_time: OffsetDateTime) -> Self {
|
||||
ActiveSession {
|
||||
start: start_time,
|
||||
typ: SessionType::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_type(&mut self, ticket_type: TicketType) {
|
||||
self.typ = ticket_type.into();
|
||||
}
|
||||
|
||||
fn end_at(self, stop_time: OffsetDateTime) -> Option<FinishedSession> {
|
||||
let session_duration = stop_time - self.start;
|
||||
//ensure duration is positive to fit in a u64
|
||||
//u64::max milliseconds is 500k millenia so no overflow issue
|
||||
if session_duration > Duration::ZERO {
|
||||
Some(FinishedSession {
|
||||
duration: session_duration,
|
||||
typ: self.typ,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct SessionStatsHandler {
|
||||
storage: PersistentStatsStorage,
|
||||
current_day: Date,
|
||||
last_update_day: Date,
|
||||
|
||||
shared_session_stats: SharedSessionStats,
|
||||
active_sessions: HashMap<DestinationAddressBytes, ActiveSession>,
|
||||
unique_users: HashSet<DestinationAddressBytes>,
|
||||
sessions_started: u32,
|
||||
finished_sessions: Vec<FinishedSession>,
|
||||
}
|
||||
|
||||
impl SessionStatsHandler {
|
||||
pub fn new(shared_session_stats: SharedSessionStats, storage: PersistentStatsStorage) -> Self {
|
||||
pub fn new(shared_session_stats: SharedSessionStats) -> Self {
|
||||
SessionStatsHandler {
|
||||
storage,
|
||||
current_day: OffsetDateTime::now_utc().date(),
|
||||
last_update_day: OffsetDateTime::now_utc().date(),
|
||||
shared_session_stats,
|
||||
active_sessions: Default::default(),
|
||||
unique_users: Default::default(),
|
||||
sessions_started: 0,
|
||||
finished_sessions: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn handle_event(
|
||||
&mut self,
|
||||
event: SessionEvent,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
pub(crate) fn handle_event(&mut self, event: SessionEvent) {
|
||||
match event {
|
||||
SessionEvent::SessionStart { start_time, client } => {
|
||||
self.handle_session_start(start_time, client).await
|
||||
self.handle_session_start(start_time, client);
|
||||
}
|
||||
|
||||
SessionEvent::SessionStop { stop_time, client } => {
|
||||
self.handle_session_stop(stop_time, client).await
|
||||
self.handle_session_stop(stop_time, client);
|
||||
}
|
||||
|
||||
SessionEvent::EcashTicket {
|
||||
ticket_type,
|
||||
client,
|
||||
} => self.handle_ecash_ticket(ticket_type, client).await,
|
||||
} => self.handle_ecash_ticket(ticket_type, client),
|
||||
}
|
||||
}
|
||||
async fn handle_session_start(
|
||||
fn handle_session_start(
|
||||
&mut self,
|
||||
start_time: OffsetDateTime,
|
||||
client: DestinationAddressBytes,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
self.storage
|
||||
.insert_unique_user(self.current_day, client.as_base58_string())
|
||||
.await?;
|
||||
self.storage
|
||||
.insert_active_session(client, ActiveSession::new(start_time))
|
||||
.await?;
|
||||
Ok(())
|
||||
) {
|
||||
self.sessions_started += 1;
|
||||
self.unique_users.insert(client);
|
||||
self.active_sessions
|
||||
.insert(client, ActiveSession::new(start_time));
|
||||
}
|
||||
|
||||
async fn handle_session_stop(
|
||||
&mut self,
|
||||
stop_time: OffsetDateTime,
|
||||
client: DestinationAddressBytes,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
if let Some(session) = self.storage.get_active_session(client).await? {
|
||||
fn handle_session_stop(&mut self, stop_time: OffsetDateTime, client: DestinationAddressBytes) {
|
||||
if let Some(session) = self.active_sessions.remove(&client) {
|
||||
if let Some(finished_session) = session.end_at(stop_time) {
|
||||
self.storage
|
||||
.insert_finished_session(self.current_day, finished_session)
|
||||
.await?;
|
||||
self.storage.delete_active_session(client).await?;
|
||||
if self.finished_sessions.len() < FINISHED_SESSIONS_CAP {
|
||||
self.finished_sessions.push(finished_session);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_ecash_ticket(
|
||||
&mut self,
|
||||
ticket_type: TicketType,
|
||||
client: DestinationAddressBytes,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
self.storage
|
||||
.update_active_session_type(client, ticket_type.into())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn on_start(&mut self) -> Result<(), StatsStorageError> {
|
||||
let yesterday = OffsetDateTime::now_utc().date() - Duration::DAY;
|
||||
//publish yesterday's data if any
|
||||
self.publish_stats(yesterday).await?;
|
||||
//store "active" sessions as duration 0
|
||||
for active_session in self.storage.get_all_active_sessions().await? {
|
||||
self.storage
|
||||
.insert_finished_session(
|
||||
self.current_day,
|
||||
FinishedSession {
|
||||
duration: Duration::ZERO,
|
||||
typ: active_session.typ,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
fn handle_ecash_ticket(&mut self, ticket_type: TicketType, client: DestinationAddressBytes) {
|
||||
if let Some(active_session) = self.active_sessions.get_mut(&client) {
|
||||
if active_session.typ == SessionType::Unknown {
|
||||
active_session.set_type(ticket_type);
|
||||
}
|
||||
}
|
||||
//cleanup active sessions
|
||||
self.storage.cleanup_active_sessions().await?;
|
||||
|
||||
//delete old entries
|
||||
self.delete_old_stats(yesterday - Duration::DAY).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
//update shared state once a day has passed, with data from the previous day
|
||||
async fn publish_stats(&mut self, stats_date: Date) -> Result<(), StatsStorageError> {
|
||||
let finished_sessions = self.storage.get_finished_sessions(stats_date).await?;
|
||||
let user_count = self.storage.get_unique_users_count(stats_date).await?;
|
||||
let session_started = self.storage.get_started_sessions_count(stats_date).await? as u32;
|
||||
{
|
||||
let mut shared_state = self.shared_session_stats.write().await;
|
||||
shared_state.update_time = stats_date;
|
||||
shared_state.unique_active_users = user_count as u32;
|
||||
shared_state.session_started = session_started;
|
||||
shared_state.sessions = finished_sessions.iter().map(|s| s.serialize()).collect();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) async fn maybe_update_shared_state(
|
||||
&mut self,
|
||||
update_time: OffsetDateTime,
|
||||
) -> Result<(), StatsStorageError> {
|
||||
pub(crate) async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
|
||||
let update_date = update_time.date();
|
||||
if update_date != self.current_day {
|
||||
self.publish_stats(self.current_day).await?;
|
||||
self.delete_old_stats(self.current_day - Duration::DAY)
|
||||
.await?;
|
||||
self.reset_stats(update_date).await?;
|
||||
self.current_day = update_date;
|
||||
if update_date != self.last_update_day {
|
||||
{
|
||||
let mut shared_state = self.shared_session_stats.write().await;
|
||||
shared_state.update_time = self.last_update_day;
|
||||
shared_state.unique_active_users = self.unique_users.len() as u32;
|
||||
shared_state.session_started = self.sessions_started;
|
||||
shared_state.sessions = self
|
||||
.finished_sessions
|
||||
.iter()
|
||||
.map(|s| s.serialize())
|
||||
.collect();
|
||||
}
|
||||
self.reset_stats(update_date);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reset_stats(&mut self, reset_day: Date) -> Result<(), StatsStorageError> {
|
||||
//active users reset
|
||||
let new_active_users = self.storage.get_active_users().await?;
|
||||
for user in new_active_users {
|
||||
self.storage.insert_unique_user(reset_day, user).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_old_stats(&mut self, delete_before: Date) -> Result<(), StatsStorageError> {
|
||||
self.storage.delete_finished_sessions(delete_before).await?;
|
||||
self.storage.delete_unique_users(delete_before).await?;
|
||||
Ok(())
|
||||
fn reset_stats(&mut self, reset_day: Date) {
|
||||
self.last_update_day = reset_day;
|
||||
self.unique_users = self.active_sessions.keys().copied().collect();
|
||||
self.finished_sessions = Default::default();
|
||||
self.sessions_started = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
FROM rust:latest AS builder
|
||||
|
||||
COPY ./ /usr/src/nym
|
||||
WORKDIR /usr/src/nym/nym-api
|
||||
RUN cargo build --release
|
||||
|
||||
ENTRYPOINT ["/usr/src/nym/nym-api/entrypoint.sh"]
|
||||
+2
-2
@@ -4,7 +4,7 @@
|
||||
[package]
|
||||
name = "nym-api"
|
||||
license = "GPL-3.0"
|
||||
version = "1.1.45"
|
||||
version = "1.1.44"
|
||||
authors.workspace = true
|
||||
edition = "2021"
|
||||
rust-version.workspace = true
|
||||
@@ -18,7 +18,7 @@ bip39 = { workspace = true }
|
||||
bincode.workspace = true
|
||||
bloomfilter = { workspace = true }
|
||||
cfg-if = { workspace = true }
|
||||
clap = { workspace = true, features = ["cargo", "derive", "env"] }
|
||||
clap = { workspace = true, features = ["cargo", "derive"] }
|
||||
console-subscriber = { workspace = true, optional = true } # validator-api needs to be built with RUSTFLAGS="--cfg tokio_unstable"
|
||||
dirs = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
/usr/src/nym/target/release/nym-api init && /usr/src/nym/target/release/nym-api run
|
||||
@@ -24,9 +24,6 @@ pub type Result<T, E = EcashError> = std::result::Result<T, E>;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum EcashError {
|
||||
#[error("permanently restricted")]
|
||||
Restricted,
|
||||
|
||||
#[error(transparent)]
|
||||
IOError(#[from] std::io::Error),
|
||||
|
||||
|
||||
@@ -10,33 +10,27 @@ use std::net::SocketAddr;
|
||||
pub(crate) struct Args {
|
||||
/// Id of the nym-api we want to initialise. if unspecified, a default value will be used.
|
||||
/// default: "default"
|
||||
#[clap(long, default_value = "default", env = "NYMAPI_ID_ARG")]
|
||||
#[clap(long, default_value = "default")]
|
||||
pub(crate) id: String,
|
||||
|
||||
/// Specifies whether network monitoring is enabled on this API
|
||||
/// default: false
|
||||
#[clap(short = 'm', long, env = "NYMAPI_ENABLE_MONITOR_ARG")]
|
||||
#[clap(short = 'm', long)]
|
||||
pub(crate) enable_monitor: bool,
|
||||
|
||||
/// Specifies whether network rewarding is enabled on this API
|
||||
/// default: false
|
||||
#[clap(
|
||||
short = 'r',
|
||||
long,
|
||||
requires = "enable_monitor",
|
||||
requires = "mnemonic",
|
||||
env = "NYMAPI_ENABLE_REWARDING_ARG"
|
||||
)]
|
||||
#[clap(short = 'r', long, requires = "enable_monitor", requires = "mnemonic")]
|
||||
pub(crate) enable_rewarding: bool,
|
||||
|
||||
/// Endpoint to nyxd instance used for contract information.
|
||||
/// default: http://localhost:26657
|
||||
#[clap(long, env = "NYMAPI_NYXD_VALIDATOR_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) nyxd_validator: Option<url::Url>,
|
||||
|
||||
/// Mnemonic of the network monitor used for sending rewarding and zk-nyms transactions
|
||||
/// default: None
|
||||
#[clap(long, env = "NYMAPI_MNEMONIC_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) mnemonic: Option<bip39::Mnemonic>,
|
||||
|
||||
/// Flag to indicate whether credential signer authority is enabled on this API
|
||||
@@ -45,23 +39,18 @@ pub(crate) struct Args {
|
||||
long,
|
||||
requires = "mnemonic",
|
||||
requires = "announce_address",
|
||||
alias = "enable_coconut",
|
||||
env = "NYMAPI_ENABLE_ZK_NYM_ARG"
|
||||
alias = "enable_coconut"
|
||||
)]
|
||||
pub(crate) enable_zk_nym: bool,
|
||||
|
||||
/// Announced address that is going to be put in the DKG contract where zk-nym clients will connect
|
||||
/// to obtain their credentials
|
||||
/// default: None
|
||||
#[clap(long, env = "NYMAPI_ANNOUNCE_ADDRESS_NYM_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) announce_address: Option<url::Url>,
|
||||
|
||||
/// Set this nym api to work in a enabled credentials that would attempt to use gateway with the bandwidth credential requirement
|
||||
#[clap(
|
||||
long,
|
||||
requires = "enable_monitor",
|
||||
env = "NYMAPI_MONITOR_CREDENTIALS_MODE_ARG"
|
||||
)]
|
||||
#[clap(long, requires = "enable_monitor")]
|
||||
pub(crate) monitor_credentials_mode: bool,
|
||||
|
||||
/// Socket address this api will use for binding its http API.
|
||||
|
||||
@@ -20,11 +20,11 @@ fn pretty_build_info_static() -> &'static str {
|
||||
#[clap(author = "Nymtech", version, long_version = pretty_build_info_static(), about)]
|
||||
pub(crate) struct Cli {
|
||||
/// Path pointing to an env file that configures the Nym API.
|
||||
#[clap(short, long, env = "NYMAPI_CONFIG_ENV_FILE_ARG")]
|
||||
#[clap(short, long)]
|
||||
pub(crate) config_env_file: Option<std::path::PathBuf>,
|
||||
|
||||
/// A no-op flag included for consistency with other binaries (and compatibility with nymvisor, oops)
|
||||
#[clap(long, env = "NYMAPI_NO_BANNER_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) no_banner: bool,
|
||||
|
||||
#[clap(subcommand)]
|
||||
|
||||
@@ -44,33 +44,27 @@ use tracing::{error, info};
|
||||
pub(crate) struct Args {
|
||||
/// Id of the nym-api we want to run.if unspecified, a default value will be used.
|
||||
/// default: "default"
|
||||
#[clap(long, default_value = "default", env = "NYMAPI_ID_ARG")]
|
||||
#[clap(long, default_value = "default")]
|
||||
pub(crate) id: String,
|
||||
|
||||
/// Specifies whether network monitoring is enabled on this API
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(short = 'm', long, env = "NYMAPI_ENABLE_MONITOR_ARG")]
|
||||
#[clap(short = 'm', long)]
|
||||
pub(crate) enable_monitor: Option<bool>,
|
||||
|
||||
/// Specifies whether network rewarding is enabled on this API
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(
|
||||
short = 'r',
|
||||
long,
|
||||
requires = "enable_monitor",
|
||||
requires = "mnemonic",
|
||||
env = "NYMAPI_ENABLE_REWARDING_ARG"
|
||||
)]
|
||||
#[clap(short = 'r', long, requires = "enable_monitor", requires = "mnemonic")]
|
||||
pub(crate) enable_rewarding: Option<bool>,
|
||||
|
||||
/// Endpoint to nyxd instance used for contract information.
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(long, env = "NYMAPI_NYXD_VALIDATOR_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) nyxd_validator: Option<url::Url>,
|
||||
|
||||
/// Mnemonic of the network monitor used for sending rewarding and zk-nyms transactions
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(long, env = "NYMAPI_MNEMONIC_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) mnemonic: Option<bip39::Mnemonic>,
|
||||
|
||||
/// Flag to indicate whether coconut signer authority is enabled on this API
|
||||
@@ -79,20 +73,19 @@ pub(crate) struct Args {
|
||||
long,
|
||||
requires = "mnemonic",
|
||||
requires = "announce_address",
|
||||
alias = "enable_coconut",
|
||||
env = "NYMAPI_ENABLE_ZK_NYM_ARG"
|
||||
alias = "enable_coconut"
|
||||
)]
|
||||
pub(crate) enable_zk_nym: Option<bool>,
|
||||
|
||||
/// Announced address that is going to be put in the DKG contract where zk-nym clients will connect
|
||||
/// to obtain their credentials
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(long, env = "NYMAPI_ANNOUNCE_ADDRESS_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) announce_address: Option<url::Url>,
|
||||
|
||||
/// Set this nym api to work in a enabled credentials that would attempt to use gateway with the bandwidth credential requirement
|
||||
/// default: None - config value will be used instead
|
||||
#[clap(long, env = "NYMAPI_MONITOR_CREDENTIALS_MODE_ARG")]
|
||||
#[clap(long)]
|
||||
pub(crate) monitor_credentials_mode: Option<bool>,
|
||||
|
||||
/// Socket address this api will use for binding its http API.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "nym-credential-proxy"
|
||||
version = "0.1.1"
|
||||
version = "0.1.0"
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
|
||||
@@ -26,8 +26,6 @@ RUN cargo build --release
|
||||
|
||||
FROM ubuntu:24.04
|
||||
|
||||
RUN apt update && apt install -yy curl ca-certificates
|
||||
|
||||
WORKDIR /nym
|
||||
|
||||
COPY --from=builder /usr/src/nym/nym-credential-proxy/target/release/nym-credential-proxy ./
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO responses\n (joke_id, joke, date_created)\n VALUES\n ($1, $2, $3)\n ON CONFLICT(joke_id) DO UPDATE SET\n joke=excluded.joke,\n date_created=excluded.date_created;",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "249faa11b88b749f50342bb5c9cc41d20896db543eed74a6f320c041bcbb723d"
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT joke_id, joke, date_created FROM responses WHERE joke_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "joke_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "joke",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "date_created",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "aff7fbd06728004d2f2226d20c32f1482df00de2dc1d2b4debbb2e12553d997b"
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT joke_id, joke, date_created FROM responses",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "joke_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "joke",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "date_created",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e53f479f8cead3dc8aa1875e5d450ad69686cf6a109e37d6c3f0623c3e9f91d0"
|
||||
}
|
||||
@@ -24,13 +24,9 @@ nym-task = { path = "../common/task" }
|
||||
nym-node-requests = { path = "../nym-node/nym-node-requests", features = [
|
||||
"openapi",
|
||||
] }
|
||||
nyxd-scraper = {path = "../common/nyxd-scraper"}
|
||||
reqwest = {workspace= true, features = ["rustls-tls"]}
|
||||
rocket = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "time"] }
|
||||
time = {version = "0.3.36"}
|
||||
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] }
|
||||
tokio = { workspace = true, features = ["process", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
@@ -44,4 +40,4 @@ utoipauto = { workspace = true }
|
||||
[build-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite"] }
|
||||
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] }
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM rust:latest AS builder
|
||||
|
||||
COPY ./ /usr/src/nym
|
||||
WORKDIR /usr/src/nym/nym-data-observatory
|
||||
|
||||
RUN cargo build --release
|
||||
|
||||
FROM ubuntu:24.04
|
||||
|
||||
RUN echo "Acquire::http::Pipeline-Depth 0;" > /etc/apt/apt.conf.d/99custom && \
|
||||
echo "Acquire::http::No-Cache true;" >> /etc/apt/apt.conf.d/99custom && \
|
||||
echo "Acquire::BrokenProxy true;" >> /etc/apt/apt.conf.d/99custom
|
||||
|
||||
RUN apt update && apt install -yy curl
|
||||
|
||||
WORKDIR /nym
|
||||
|
||||
COPY --from=builder /usr/src/nym/target/release/nym-data-observatory ./
|
||||
ENTRYPOINT [ "/nym/nym-data-observatory" ]
|
||||
@@ -68,7 +68,6 @@ warning: no queries found; do you have the `offline` feature enabled
|
||||
### Possible solutions
|
||||
|
||||
- does your `sqlx-cli` version match `sqlx` version from `Cargo.toml`?
|
||||
+ `cargo install -f sqlx-cli --version <specific version>`
|
||||
```
|
||||
cargo install sqlx-cli --version <exact semver version as sqlx> --force
|
||||
```
|
||||
|
||||
@@ -1,51 +1,58 @@
|
||||
use anyhow::Result;
|
||||
use sqlx::{sqlite::SqliteConnectOptions, Connection, SqliteConnection};
|
||||
use sqlx::{Connection, PgConnection};
|
||||
use std::io::Write;
|
||||
use std::{collections::HashMap, fs::File, path::PathBuf, str::FromStr};
|
||||
use std::{collections::HashMap, fs::File};
|
||||
|
||||
const POSTGRES_USER: &str = "nym";
|
||||
const POSTGRES_PASSWORD: &str = "password123";
|
||||
const POSTGRES_DB: &str = "data_obs_db";
|
||||
|
||||
/// if schema changes, rerun `cargo sqlx prepare` with a running DB
|
||||
/// https://github.com/launchbadge/sqlx/blob/main/sqlx-cli/README.md#enable-building-in-offline-mode-with-query
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let db_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data_observatory.sqlite");
|
||||
|
||||
// Create the database directory if it doesn't exist
|
||||
if let Some(parent) = db_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let db_url = format!("sqlite:{}", db_path.display());
|
||||
|
||||
// Ensure database file is created with proper permissions
|
||||
let connect_options = SqliteConnectOptions::from_str(&db_url)?
|
||||
.create_if_missing(true)
|
||||
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
|
||||
.foreign_keys(true);
|
||||
|
||||
// Create initial connection to ensure database exists
|
||||
let mut conn = SqliteConnection::connect_with(&connect_options).await?;
|
||||
let db_url =
|
||||
format!("postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:5432/{POSTGRES_DB}");
|
||||
|
||||
export_db_variables(&db_url)?;
|
||||
println!("cargo:rustc-env=SQLX_OFFLINE=false");
|
||||
// if a live DB is reachable, use that
|
||||
if PgConnection::connect(&db_url).await.is_ok() {
|
||||
println!("cargo::rustc-env=SQLX_OFFLINE=false");
|
||||
run_migrations(&db_url).await?;
|
||||
} else {
|
||||
// by default, run in offline mode
|
||||
println!("cargo::rustc-env=SQLX_OFFLINE=true");
|
||||
}
|
||||
|
||||
// Run migrations after ensuring database exists
|
||||
sqlx::migrate!("./migrations").run(&mut conn).await?;
|
||||
|
||||
// Add rerun-if-changed directives
|
||||
println!("cargo:rerun-if-changed=migrations");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
println!("cargo:rerun-if-changed=src");
|
||||
rerun_if_changed();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn export_db_variables(db_url: &str) -> Result<()> {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("POSTGRES_USER", POSTGRES_USER);
|
||||
map.insert("POSTGRES_PASSWORD", POSTGRES_PASSWORD);
|
||||
map.insert("POSTGRES_DB", POSTGRES_DB);
|
||||
map.insert("DATABASE_URL", db_url);
|
||||
|
||||
let mut file = File::create(".env")?;
|
||||
for (var, value) in map.iter() {
|
||||
println!("cargo:rustc-env={}={}", var, value);
|
||||
writeln!(file, "{}={}", var, value)?;
|
||||
println!("cargo::rustc-env={}={}", var, value);
|
||||
writeln!(file, "{}={}", var, value).expect("Failed to write to dotenv file");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_migrations(db_url: &str) -> Result<()> {
|
||||
let mut conn = PgConnection::connect(db_url).await?;
|
||||
sqlx::migrate!("./migrations").run(&mut conn).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rerun_if_changed() {
|
||||
println!("cargo::rerun-if-changed=migrations");
|
||||
println!("cargo::rerun-if-changed=src/db/queries");
|
||||
}
|
||||
|
||||
@@ -18,14 +18,8 @@ services:
|
||||
dockerfile: nym-data-observatory/Dockerfile
|
||||
container_name: nym-data-observatory
|
||||
environment:
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_USERNAME: "postgres"
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_PASSWORD: "password"
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_HOST: "postgres"
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_PORT: "5432"
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_DB: ""
|
||||
NYM_DATA_OBSERVATORY_CONNECTION_URL: "postgres://postgres:password@postgres:5432"
|
||||
NYM_DATA_OBSERVATORY_HTTP_PORT: 8000
|
||||
env_file:
|
||||
- ../envs/qa.env
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
CREATE TABLE price_history (
|
||||
timestamp INTEGER PRIMARY KEY,
|
||||
chf REAL NOT NULL,
|
||||
usd REAL NOT NULL,
|
||||
eur REAL NOT NULL,
|
||||
btc REAL NOT NULL
|
||||
);
|
||||
@@ -1,10 +0,0 @@
|
||||
CREATE TABLE payments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transaction_hash TEXT NOT NULL UNIQUE,
|
||||
sender_address TEXT NOT NULL,
|
||||
receiver_address TEXT NOT NULL,
|
||||
amount REAL NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
memo TEXT
|
||||
);
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
# .env is generated in build.rs
|
||||
source .env
|
||||
|
||||
# Launching a container in such a way that it's destroyed after you detach from the terminal:
|
||||
docker compose up
|
||||
|
||||
# docker exec -it nym-data-observatory-pg /bin/bash
|
||||
# psql -U youruser -d yourdb
|
||||
|
||||
echo "Tearing down containers to have a clean slate"
|
||||
docker compose down -v
|
||||
@@ -0,0 +1,61 @@
|
||||
use core::str;
|
||||
use serde::Deserialize;
|
||||
use tokio::process::Command;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::Duration;
|
||||
|
||||
use crate::db::{self, DbPool};
|
||||
|
||||
const REFRESH_DELAY: Duration = Duration::from_secs(15);
|
||||
const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60 * 2);
|
||||
|
||||
pub(crate) async fn spawn_in_background(db_pool: DbPool) -> JoinHandle<()> {
|
||||
loop {
|
||||
tracing::info!("Running in a loop 🏃");
|
||||
|
||||
if let Err(e) = some_network_action(&db_pool).await {
|
||||
tracing::error!(
|
||||
"❌ Run failed: {e}, retrying in {}s...",
|
||||
FAILURE_RETRY_DELAY.as_secs()
|
||||
);
|
||||
tokio::time::sleep(FAILURE_RETRY_DELAY).await;
|
||||
} else {
|
||||
tracing::info!(
|
||||
"✅ Run successful, sleeping for {}s...",
|
||||
REFRESH_DELAY.as_secs()
|
||||
);
|
||||
tokio::time::sleep(REFRESH_DELAY).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub(crate) struct Response {
|
||||
#[serde(rename(deserialize = "id"))]
|
||||
pub(crate) joke_id: String,
|
||||
pub(crate) joke: String,
|
||||
#[serde(rename(deserialize = "status"))]
|
||||
pub(crate) _status: u16,
|
||||
}
|
||||
|
||||
async fn some_network_action(pool: &DbPool) -> anyhow::Result<()> {
|
||||
// for demonstration purposes only. You should use reqwest if you need it
|
||||
let output = Command::new("curl")
|
||||
.arg("-H")
|
||||
.arg("Accept: application/json")
|
||||
.arg("https://icanhazdadjoke.com/")
|
||||
.output()
|
||||
.await?;
|
||||
|
||||
if !output.status.success() {
|
||||
anyhow::bail!("Curl command failed with status: {}", output.status);
|
||||
}
|
||||
|
||||
let response_str = str::from_utf8(&output.stdout)?;
|
||||
let joke_response: Response = serde_json::from_str(response_str)?;
|
||||
|
||||
tracing::info!("{:?}", joke_response.joke);
|
||||
db::queries::insert_joke(pool, joke_response.into()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
use nyxd_scraper::{storage::ScraperStorage, Config, NyxdScraper, PruningOptions};
|
||||
|
||||
pub(crate) async fn run_chain_scraper() -> anyhow::Result<ScraperStorage> {
|
||||
let websocket_url =
|
||||
std::env::var("NYXD_WEBSOCKET_URL").expect("NYXD_WEBSOCKET_URL not defined");
|
||||
|
||||
let rpc_url = std::env::var("NYXD_RPC_URL").expect("NYXD_RPC_URL not defined");
|
||||
let websocket_url = reqwest::Url::parse(&websocket_url)?;
|
||||
let rpc_url = reqwest::Url::parse(&rpc_url)?;
|
||||
|
||||
let start_block_height = std::env::var("NYXD_SCRAPER_START_HEIGHT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u32>().ok());
|
||||
|
||||
let scraper = NyxdScraper::builder(Config {
|
||||
websocket_url,
|
||||
rpc_url,
|
||||
database_path: "chain_history.sqlite".into(),
|
||||
pruning_options: PruningOptions::nothing(),
|
||||
store_precommits: false,
|
||||
});
|
||||
|
||||
let instance = scraper.build_and_start(start_block_height).await?;
|
||||
|
||||
Ok(instance.storage)
|
||||
}
|
||||
@@ -1,34 +1,30 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use sqlx::{migrate::Migrator, sqlite::SqliteConnectOptions, SqlitePool};
|
||||
use sqlx::{migrate::Migrator, postgres::PgConnectOptions, ConnectOptions, PgPool};
|
||||
use std::str::FromStr;
|
||||
|
||||
pub(crate) mod models;
|
||||
pub(crate) mod queries {
|
||||
pub mod payments;
|
||||
pub mod price;
|
||||
}
|
||||
pub(crate) mod queries;
|
||||
|
||||
static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
|
||||
|
||||
pub(crate) type DbPool = SqlitePool;
|
||||
pub(crate) type DbPool = PgPool;
|
||||
|
||||
pub(crate) struct Storage {
|
||||
pool: DbPool,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
pub async fn init(connection_url: String) -> Result<Self> {
|
||||
pub async fn init(connection_url: Option<String>) -> Result<Self> {
|
||||
let connection_url =
|
||||
connection_url.ok_or_else(|| anyhow!("Missing the connection url for database!"))?;
|
||||
let connect_options =
|
||||
SqliteConnectOptions::from_str(&connection_url)?.create_if_missing(true);
|
||||
PgConnectOptions::from_str(&connection_url)?.disable_statement_logging();
|
||||
|
||||
let pool = DbPool::connect_with(connect_options)
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to connect to {}: {}", &connection_url, err))?;
|
||||
|
||||
MIGRATOR
|
||||
.run(&pool)
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to run migrations: {}", err))?;
|
||||
MIGRATOR.run(&pool).await?;
|
||||
|
||||
Ok(Storage { pool })
|
||||
}
|
||||
|
||||
@@ -1,41 +1,22 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
#[derive(Clone, Deserialize, Debug)]
|
||||
pub(crate) struct CurrencyPrices {
|
||||
pub(crate) chf: f32,
|
||||
pub(crate) usd: f32,
|
||||
pub(crate) eur: f32,
|
||||
pub(crate) btc: f32,
|
||||
}
|
||||
|
||||
// Struct to hold Coingecko response
|
||||
#[derive(Clone, Deserialize, Debug, ToSchema)]
|
||||
pub(crate) struct CoingeckoPriceResponse {
|
||||
pub(crate) nym: CurrencyPrices,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, Debug, ToSchema)]
|
||||
pub(crate) struct PriceRecord {
|
||||
pub(crate) timestamp: i64,
|
||||
pub(crate) nym: CurrencyPrices,
|
||||
}
|
||||
use crate::background_task::Response;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, ToSchema)]
|
||||
pub(crate) struct PriceHistory {
|
||||
pub(crate) timestamp: i64,
|
||||
pub(crate) chf: f32,
|
||||
pub(crate) usd: f32,
|
||||
pub(crate) eur: f32,
|
||||
pub(crate) btc: f32,
|
||||
pub(crate) struct JokeDto {
|
||||
pub(crate) joke_id: String,
|
||||
pub(crate) joke: String,
|
||||
pub(crate) date_created: i32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, ToSchema)]
|
||||
pub(crate) struct PaymentRecord {
|
||||
pub(crate) transaction_hash: String,
|
||||
pub(crate) sender_address: String,
|
||||
pub(crate) receiver_address: String,
|
||||
pub(crate) amount: f64,
|
||||
pub(crate) timestamp: i64,
|
||||
pub(crate) height: i64,
|
||||
impl From<Response> for JokeDto {
|
||||
fn from(value: Response) -> Self {
|
||||
Self {
|
||||
joke_id: value.joke_id,
|
||||
joke: value.joke,
|
||||
// casting not smart, can implicitly panic, don't do this in prod
|
||||
date_created: chrono::offset::Utc::now().timestamp() as i32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use crate::db::{models::JokeDto, DbPool};
|
||||
|
||||
pub(crate) async fn insert_joke(pool: &DbPool, joke: JokeDto) -> anyhow::Result<()> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
sqlx::query!(
|
||||
"INSERT INTO responses
|
||||
(joke_id, joke, date_created)
|
||||
VALUES
|
||||
($1, $2, $3)
|
||||
ON CONFLICT(joke_id) DO UPDATE SET
|
||||
joke=excluded.joke,
|
||||
date_created=excluded.date_created;",
|
||||
joke.joke_id,
|
||||
joke.joke,
|
||||
joke.date_created as i32,
|
||||
)
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn select_joke_by_id(pool: &DbPool, joke_id: &str) -> anyhow::Result<JokeDto> {
|
||||
sqlx::query_as!(
|
||||
JokeDto,
|
||||
"SELECT joke_id, joke, date_created FROM responses WHERE joke_id = $1",
|
||||
joke_id
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
|
||||
pub(crate) async fn select_all(pool: &DbPool) -> anyhow::Result<Vec<JokeDto>> {
|
||||
sqlx::query_as!(JokeDto, "SELECT joke_id, joke, date_created FROM responses",)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
mod payments;
|
||||
mod price;
|
||||
// group queries in files by theme
|
||||
mod joke;
|
||||
|
||||
// re-exporting allows us to access all queries via `queries::bla``
|
||||
pub(crate) use payments::{get_last_checked_height, insert_payment};
|
||||
pub(crate) use price::{get_latest_price, insert_nym_prices};
|
||||
pub(crate) use joke::{insert_joke, select_all, select_joke_by_id};
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
use crate::db::DbPool;
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn get_last_checked_height(pool: &DbPool) -> Result<i64> {
|
||||
let result = sqlx::query_scalar!("SELECT MAX(height) FROM payments")
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(result.unwrap_or(0))
|
||||
}
|
||||
|
||||
pub async fn insert_payment(
|
||||
pool: &DbPool,
|
||||
transaction_hash: String,
|
||||
sender_address: String,
|
||||
receiver_address: String,
|
||||
amount: f64,
|
||||
height: i64,
|
||||
memo: Option<String>,
|
||||
) -> Result<()> {
|
||||
let timestamp = chrono::Utc::now().timestamp();
|
||||
|
||||
sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO payments (
|
||||
transaction_hash, sender_address, receiver_address,
|
||||
amount, height, timestamp, memo
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
"#,
|
||||
transaction_hash,
|
||||
sender_address,
|
||||
receiver_address,
|
||||
amount,
|
||||
height,
|
||||
timestamp,
|
||||
memo,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
use crate::db::models::{PriceHistory, PriceRecord};
|
||||
use crate::db::DbPool;
|
||||
|
||||
pub(crate) async fn insert_nym_prices(
|
||||
pool: &DbPool,
|
||||
price_data: PriceRecord,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
let timestamp = price_data.timestamp;
|
||||
sqlx::query!(
|
||||
"INSERT INTO price_history
|
||||
(timestamp, chf, usd, eur, btc)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5)
|
||||
ON CONFLICT(timestamp) DO UPDATE SET
|
||||
chf=excluded.chf,
|
||||
usd=excluded.usd,
|
||||
eur=excluded.eur,
|
||||
btc=excluded.btc;",
|
||||
timestamp,
|
||||
price_data.nym.chf,
|
||||
price_data.nym.usd,
|
||||
price_data.nym.eur,
|
||||
price_data.nym.btc,
|
||||
)
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_latest_price(pool: &DbPool) -> anyhow::Result<PriceHistory> {
|
||||
let result = sqlx::query!(
|
||||
"SELECT timestamp, chf, usd, eur, btc FROM price_history ORDER BY timestamp DESC LIMIT 1;"
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok(PriceHistory {
|
||||
timestamp: result.timestamp,
|
||||
chf: result.chf as f32,
|
||||
usd: result.usd as f32,
|
||||
eur: result.eur as f32,
|
||||
btc: result.btc as f32,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
Json, Router,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use utoipa::IntoParams;
|
||||
|
||||
use crate::{
|
||||
db::{
|
||||
models::JokeDto,
|
||||
queries::{self, select_joke_by_id},
|
||||
},
|
||||
http::{
|
||||
error::{Error, HttpResult},
|
||||
state::AppState,
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/", axum::routing::get(jokes))
|
||||
.route("/:joke_id", axum::routing::get(joke_by_id))
|
||||
.route("/fetch_another", axum::routing::get(fetch_another))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
tag = "Dad Jokes",
|
||||
get,
|
||||
path = "/v1/jokes",
|
||||
responses(
|
||||
(status = 200, body = Vec<JokeDto>)
|
||||
)
|
||||
)]
|
||||
async fn jokes(State(state): State<AppState>) -> HttpResult<Json<Vec<JokeDto>>> {
|
||||
queries::select_all(state.db_pool())
|
||||
.await
|
||||
.map(Json::from)
|
||||
.map_err(|_| Error::internal())
|
||||
}
|
||||
|
||||
#[derive(Deserialize, IntoParams)]
|
||||
#[into_params(parameter_in = Path)]
|
||||
struct JokeIdParam {
|
||||
joke_id: String,
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
tag = "Dad Jokes",
|
||||
get,
|
||||
params(
|
||||
JokeIdParam
|
||||
),
|
||||
path = "/v1/jokes/{joke_id}",
|
||||
responses(
|
||||
(status = 200, body = JokeDto)
|
||||
)
|
||||
)]
|
||||
async fn joke_by_id(
|
||||
Path(JokeIdParam { joke_id }): Path<JokeIdParam>,
|
||||
State(state): State<AppState>,
|
||||
) -> HttpResult<Json<JokeDto>> {
|
||||
select_joke_by_id(state.db_pool(), &joke_id)
|
||||
.await
|
||||
.map(Json::from)
|
||||
.map_err(|_| Error::not_found(joke_id))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
tag = "Dad Jokes",
|
||||
get,
|
||||
path = "/v1/jokes/fetch_another",
|
||||
responses(
|
||||
(status = 200, body = String)
|
||||
)
|
||||
)]
|
||||
async fn fetch_another(State(_state): State<AppState>) -> HttpResult<Json<String>> {
|
||||
Ok(Json(String::from("Done boss, check the DB")))
|
||||
}
|
||||
@@ -7,8 +7,8 @@ use utoipa_swagger_ui::SwaggerUi;
|
||||
|
||||
use crate::http::{api_docs, server::HttpServer, state::AppState};
|
||||
|
||||
pub(crate) mod jokes;
|
||||
pub(crate) mod mixnodes;
|
||||
pub(crate) mod price;
|
||||
|
||||
pub(crate) struct RouterBuilder {
|
||||
unfinished_router: Router<AppState>,
|
||||
@@ -28,9 +28,8 @@ impl RouterBuilder {
|
||||
.nest(
|
||||
"/v1",
|
||||
Router::new()
|
||||
//.nest("/jokes", jokes::routes())
|
||||
.nest("/mixnodes", mixnodes::routes())
|
||||
.nest("/price", price::routes()),
|
||||
.nest("/jokes", jokes::routes())
|
||||
.nest("/mixnodes", mixnodes::routes()),
|
||||
);
|
||||
|
||||
Self {
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
use crate::db::models::PriceHistory;
|
||||
use crate::db::queries::price::get_latest_price;
|
||||
use crate::http::error::Error;
|
||||
use crate::http::error::HttpResult;
|
||||
use crate::http::state::AppState;
|
||||
use axum::{extract::State, Json, Router};
|
||||
|
||||
pub(crate) fn routes() -> Router<AppState> {
|
||||
Router::new().route("/", axum::routing::get(price))
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
tag = "Nym Price",
|
||||
get,
|
||||
path = "/v1/price",
|
||||
responses(
|
||||
(status = 200, body = String)
|
||||
)
|
||||
)]
|
||||
|
||||
/// Fetch the latest price cached by the data observatory
|
||||
async fn price(State(state): State<AppState>) -> HttpResult<Json<PriceHistory>> {
|
||||
get_latest_price(state.db_pool())
|
||||
.await
|
||||
.map(Json::from)
|
||||
.map_err(|_| Error::internal())
|
||||
}
|
||||
@@ -6,9 +6,5 @@ use utoipauto::utoipauto;
|
||||
// https://github.com/ProbablyClem/utoipauto/issues/13#issuecomment-1974911829
|
||||
#[utoipauto(paths = "./nym-data-observatory/src")]
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
info(title = "Nym Data Observatory API"),
|
||||
tags(),
|
||||
components(schemas())
|
||||
)]
|
||||
#[openapi(info(title = "Nym API"), tags(), components(schemas()))]
|
||||
pub(super) struct ApiDoc;
|
||||
|
||||
@@ -6,6 +6,13 @@ pub(crate) struct Error {
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub(crate) fn not_found(message: String) -> Self {
|
||||
Self {
|
||||
message,
|
||||
status: axum::http::StatusCode::NOT_FOUND,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn internal() -> Self {
|
||||
Self {
|
||||
message: String::from("Internal server error"),
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
use chain_scraper::run_chain_scraper;
|
||||
use clap::Parser;
|
||||
use nym_network_defaults::setup_env;
|
||||
use nym_task::signal::wait_for_signal;
|
||||
use tokio::join;
|
||||
|
||||
mod chain_scraper;
|
||||
mod background_task;
|
||||
mod db;
|
||||
mod http;
|
||||
mod logging;
|
||||
mod payment_listener;
|
||||
mod price_scraper;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about = None)]
|
||||
@@ -22,13 +18,9 @@ struct Args {
|
||||
#[arg(short, long, default_value = None, env = "NYM_DATA_OBSERVATORY_ENV_FILE")]
|
||||
env_file: Option<String>,
|
||||
|
||||
/// SQLite database file path
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "data_observatory.sqlite",
|
||||
env = "NYM_DATA_OBSERVATORY_DB_PATH"
|
||||
)]
|
||||
db_path: String,
|
||||
/// DB connection url
|
||||
#[arg(short, long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_URL")]
|
||||
connection_url: Option<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
@@ -36,50 +28,21 @@ async fn main() -> anyhow::Result<()> {
|
||||
logging::setup_tracing_logger();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
setup_env(args.env_file); // Defaults to mainnet if empty
|
||||
|
||||
let db_path = args.db_path;
|
||||
// Ensure parent directory exists
|
||||
if let Some(parent) = std::path::Path::new(&db_path).parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let connection_url = format!("sqlite://{}?mode=rwc", db_path);
|
||||
let storage = db::Storage::init(connection_url).await?;
|
||||
let observatory_pool = storage.pool_owned().await;
|
||||
|
||||
// Spawn the chain scraper and get its storage
|
||||
|
||||
// Spawn the payment listener task
|
||||
let payment_listener_handle = tokio::spawn({
|
||||
let obs_pool = observatory_pool.clone();
|
||||
let chain_storage = run_chain_scraper().await?;
|
||||
|
||||
async move {
|
||||
if let Err(e) = payment_listener::run_payment_listener(obs_pool, chain_storage).await {
|
||||
tracing::error!("Payment listener error: {}", e);
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
}
|
||||
});
|
||||
|
||||
// Clone pool for each task that needs it
|
||||
//let background_pool = db_pool.clone();
|
||||
|
||||
let price_scraper_handle = tokio::spawn(async move {
|
||||
price_scraper::run_price_scraper(&observatory_pool).await;
|
||||
let storage = db::Storage::init(args.connection_url).await?;
|
||||
let db_pool = storage.pool_owned().await;
|
||||
tokio::spawn(async move {
|
||||
background_task::spawn_in_background(db_pool).await;
|
||||
tracing::info!("Started task");
|
||||
});
|
||||
|
||||
let shutdown_handles = http::server::start_http_api(storage.pool_owned().await, args.http_port)
|
||||
.await
|
||||
.expect("Failed to start server");
|
||||
|
||||
tracing::info!("Started HTTP server on port {}", args.http_port);
|
||||
|
||||
// Wait for the short-lived tasks to complete
|
||||
let _ = join!(price_scraper_handle, payment_listener_handle);
|
||||
|
||||
// Wait for a signal to terminate the long-running task
|
||||
wait_for_signal().await;
|
||||
|
||||
if let Err(err) = shutdown_handles.shutdown().await {
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
use crate::db::queries;
|
||||
use nyxd_scraper::storage::ScraperStorage;
|
||||
use reqwest::Client;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::SqlitePool;
|
||||
use std::env;
|
||||
use tokio::time::{self, Duration};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TransferEvent {
|
||||
recipient: String,
|
||||
sender: String,
|
||||
amount: String,
|
||||
}
|
||||
|
||||
pub(crate) async fn run_payment_listener(
|
||||
observatory_pool: SqlitePool,
|
||||
chain_storage: ScraperStorage,
|
||||
) -> anyhow::Result<()> {
|
||||
let payment_receive_address = env::var("PAYMENT_RECEIVE_ADDRESS").map_err(|_| {
|
||||
anyhow::anyhow!("Environment variable `PAYMENT_RECEIVE_ADDRESS` not defined")
|
||||
})?;
|
||||
let webhook_url = env::var("WEBHOOK_URL")
|
||||
.map_err(|_| anyhow::anyhow!("Environment variable `WEBHOOK_URL` not defined"))?;
|
||||
|
||||
let client = Client::new();
|
||||
loop {
|
||||
let last_checked_height =
|
||||
queries::payments::get_last_checked_height(&observatory_pool).await?;
|
||||
tracing::info!("Last checked height: {}", last_checked_height);
|
||||
|
||||
let transactions = chain_storage
|
||||
.get_transactions_after_height(
|
||||
last_checked_height,
|
||||
Some("/cosmos.bank.v1beta1.MsgSend"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for tx in transactions {
|
||||
tracing::info!("Processing transaction: {}", tx.hash);
|
||||
if let Some(raw_log) = tx.raw_log.as_deref() {
|
||||
if let Some(transfer) = parse_transfer_from_raw_log(raw_log)? {
|
||||
if transfer.recipient == payment_receive_address {
|
||||
let amount: f64 = parse_unym_amount(&transfer.amount)?;
|
||||
|
||||
queries::payments::insert_payment(
|
||||
&observatory_pool,
|
||||
tx.hash.clone(),
|
||||
transfer.sender.clone(),
|
||||
transfer.recipient.clone(),
|
||||
amount,
|
||||
tx.height,
|
||||
tx.memo.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let webhook_data = json!({
|
||||
"transaction_hash": tx.hash,
|
||||
"sender_address": transfer.sender,
|
||||
"receiver_address": transfer.recipient,
|
||||
"amount": amount,
|
||||
"height": tx.height,
|
||||
"memo": tx.memo,
|
||||
});
|
||||
let _ = client.post(&webhook_url).json(&webhook_data).send().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
time::sleep(Duration::from_secs(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_transfer_from_raw_log(raw_log: &str) -> anyhow::Result<Option<TransferEvent>> {
|
||||
let log_value: Value = serde_json::from_str(raw_log)?;
|
||||
|
||||
if let Some(events) = log_value[0]["events"].as_array() {
|
||||
if let Some(transfer_event) = events.iter().find(|e| e["type"] == "transfer") {
|
||||
if let Some(attrs) = transfer_event["attributes"].as_array() {
|
||||
let mut transfer = TransferEvent {
|
||||
recipient: String::new(),
|
||||
sender: String::new(),
|
||||
amount: String::new(),
|
||||
};
|
||||
|
||||
for attr in attrs {
|
||||
match attr["key"].as_str() {
|
||||
Some("recipient") => {
|
||||
transfer.recipient = attr["value"].as_str().unwrap_or("").to_string()
|
||||
}
|
||||
Some("sender") => {
|
||||
transfer.sender = attr["value"].as_str().unwrap_or("").to_string()
|
||||
}
|
||||
Some("amount") => {
|
||||
transfer.amount = attr["value"].as_str().unwrap_or("").to_string()
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(Some(transfer));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn parse_unym_amount(amount: &str) -> anyhow::Result<f64> {
|
||||
let amount = amount.trim_end_matches("unym");
|
||||
let parsed: f64 = amount.parse()?;
|
||||
Ok(parsed / 1_000_000.0)
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
use crate::db::{
|
||||
models::{CoingeckoPriceResponse, PriceRecord},
|
||||
queries::price::insert_nym_prices,
|
||||
};
|
||||
use core::str;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::Duration;
|
||||
|
||||
use crate::db::DbPool;
|
||||
|
||||
const REFRESH_DELAY: Duration = Duration::from_secs(300);
|
||||
const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60 * 2);
|
||||
const COINGECKO_API_URL: &str =
|
||||
"https://api.coingecko.com/api/v3/simple/price?ids=nym&vs_currencies=chf,usd,eur,btc";
|
||||
|
||||
pub(crate) async fn run_price_scraper(db_pool: &DbPool) -> JoinHandle<()> {
|
||||
loop {
|
||||
tracing::info!("Running in a loop 🏃");
|
||||
if let Err(e) = get_coingecko_prices(db_pool).await {
|
||||
tracing::error!("❌ Failed to get CoinGecko prices: {e}");
|
||||
tracing::info!("Retrying in {}s...", FAILURE_RETRY_DELAY.as_secs());
|
||||
tokio::time::sleep(FAILURE_RETRY_DELAY).await;
|
||||
} else {
|
||||
tracing::info!("✅ Successfully fetched CoinGecko prices");
|
||||
tokio::time::sleep(REFRESH_DELAY).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_coingecko_prices(pool: &DbPool) -> anyhow::Result<()> {
|
||||
tracing::info!("💰 Fetching CoinGecko prices from {}", COINGECKO_API_URL);
|
||||
|
||||
let response = reqwest::get(COINGECKO_API_URL)
|
||||
.await?
|
||||
.json::<CoingeckoPriceResponse>()
|
||||
.await;
|
||||
|
||||
tracing::info!("Got response {:?}", response);
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
let price_record = PriceRecord {
|
||||
timestamp: time::OffsetDateTime::now_utc().unix_timestamp(),
|
||||
nym: resp.nym,
|
||||
};
|
||||
|
||||
insert_nym_prices(pool, price_record).await?;
|
||||
}
|
||||
Err(e) => {
|
||||
//tracing::info!("💰 CoinGecko price response: {:?}", response);
|
||||
tracing::error!("Error sending request: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet};
|
||||
use anyhow::Result;
|
||||
use futures::{stream::FuturesUnordered, StreamExt};
|
||||
use log::{debug, info};
|
||||
use nym_sphinx::chunking::{monitoring, SentFragment};
|
||||
use nym_sphinx::chunking::{SentFragment, FRAGMENTS_RECEIVED, FRAGMENTS_SENT};
|
||||
use nym_topology::{gateway, mix, NymTopology};
|
||||
use nym_types::monitoring::{MonitorMessage, NodeResult};
|
||||
use nym_validator_client::nym_api::routes::{API_VERSION, STATUS, SUBMIT_GATEWAY, SUBMIT_NODE};
|
||||
@@ -115,8 +115,8 @@ impl NetworkAccount {
|
||||
}
|
||||
|
||||
pub fn empty_buffers() {
|
||||
monitoring::FRAGMENTS_SENT.clear();
|
||||
monitoring::FRAGMENTS_RECEIVED.clear();
|
||||
FRAGMENTS_SENT.clear();
|
||||
FRAGMENTS_RECEIVED.clear();
|
||||
}
|
||||
|
||||
fn new() -> Self {
|
||||
@@ -125,7 +125,7 @@ impl NetworkAccount {
|
||||
topology,
|
||||
..Default::default()
|
||||
};
|
||||
for fragment_set in monitoring::FRAGMENTS_SENT.iter() {
|
||||
for fragment_set in FRAGMENTS_SENT.iter() {
|
||||
let sent_fragments = fragment_set
|
||||
.value()
|
||||
.first()
|
||||
@@ -138,7 +138,7 @@ impl NetworkAccount {
|
||||
sent_fragments
|
||||
);
|
||||
|
||||
let recv = monitoring::FRAGMENTS_RECEIVED.get(fragment_set.key());
|
||||
let recv = FRAGMENTS_RECEIVED.get(fragment_set.key());
|
||||
let recv_fragments = recv.as_ref().map(|r| r.value().len()).unwrap_or(0);
|
||||
debug!(
|
||||
"RECV Fragment set {} has {} fragments",
|
||||
@@ -170,7 +170,7 @@ impl NetworkAccount {
|
||||
}
|
||||
|
||||
fn hydrate_all_fragments(&mut self) -> Result<()> {
|
||||
for fragment_set in monitoring::FRAGMENTS_SENT.iter() {
|
||||
for fragment_set in FRAGMENTS_SENT.iter() {
|
||||
let fragment_set_id = fragment_set.key();
|
||||
for fragment in fragment_set.value() {
|
||||
let route = self.hydrate_route(fragment.clone())?;
|
||||
@@ -205,7 +205,7 @@ impl NetworkAccount {
|
||||
fn find_missing_fragments(&mut self) {
|
||||
let mut missing_fragments_map = HashMap::new();
|
||||
for fragment_set_id in &self.incomplete_fragment_sets {
|
||||
if let Some(fragment_ref) = monitoring::FRAGMENTS_RECEIVED.get(fragment_set_id) {
|
||||
if let Some(fragment_ref) = FRAGMENTS_RECEIVED.get(fragment_set_id) {
|
||||
if let Some(ref_fragment) = fragment_ref.value().first() {
|
||||
let ref_header = ref_fragment.header();
|
||||
let ref_id_set = (0..ref_header.total_fragments()).collect::<HashSet<u8>>();
|
||||
|
||||
@@ -6,7 +6,7 @@ use axum::{
|
||||
use futures::StreamExt;
|
||||
use log::{debug, error, warn};
|
||||
use nym_sdk::mixnet::MixnetMessageSender;
|
||||
use nym_sphinx::chunking::{monitoring, ReceivedFragment, SentFragment};
|
||||
use nym_sphinx::chunking::{ReceivedFragment, SentFragment, FRAGMENTS_RECEIVED, FRAGMENTS_SENT};
|
||||
use petgraph::{dot::Dot, Graph};
|
||||
use rand::{distributions::Alphanumeric, seq::SliceRandom, Rng};
|
||||
use serde::Serialize;
|
||||
@@ -113,7 +113,7 @@ pub async fn graph_handler() -> Result<String, StatusCode> {
|
||||
)]
|
||||
pub async fn sent_handler() -> Json<FragmentsSent> {
|
||||
Json(FragmentsSent(
|
||||
(*monitoring::FRAGMENTS_SENT)
|
||||
(*FRAGMENTS_SENT)
|
||||
.clone()
|
||||
.into_iter()
|
||||
.collect::<HashMap<_, _>>(),
|
||||
@@ -129,7 +129,7 @@ pub async fn sent_handler() -> Json<FragmentsSent> {
|
||||
)]
|
||||
pub async fn recv_handler() -> Json<FragmentsReceived> {
|
||||
Json(FragmentsReceived(
|
||||
(*monitoring::FRAGMENTS_RECEIVED)
|
||||
(*FRAGMENTS_RECEIVED)
|
||||
.clone()
|
||||
.into_iter()
|
||||
.collect::<HashMap<_, _>>(),
|
||||
|
||||
@@ -7,7 +7,6 @@ use nym_crypto::asymmetric::ed25519::PrivateKey;
|
||||
use nym_network_defaults::setup_env;
|
||||
use nym_network_defaults::var_names::NYM_API;
|
||||
use nym_sdk::mixnet::{self, MixnetClient};
|
||||
use nym_sphinx::chunking::monitoring;
|
||||
use nym_topology::{HardcodedTopologyProvider, NymTopology};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
@@ -155,9 +154,6 @@ async fn main() -> Result<()> {
|
||||
|
||||
setup_env(args.env); // Defaults to mainnet if empty
|
||||
|
||||
// enable monitoring client-side
|
||||
monitoring::enable();
|
||||
|
||||
let cancel_token = CancellationToken::new();
|
||||
let server_cancel_token = cancel_token.clone();
|
||||
let clients = Arc::new(RwLock::new(VecDeque::with_capacity(args.n_clients)));
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
nym-gateway-probe
|
||||
@@ -1,27 +0,0 @@
|
||||
# Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
[package]
|
||||
name = "nym-node-status-agent"
|
||||
version = "0.1.0"
|
||||
authors.workspace = true
|
||||
repository.workspace = true
|
||||
homepage.workspace = true
|
||||
documentation.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
readme.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true}
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
nym-bin-common = { path = "../common/bin-common", features = ["models"]}
|
||||
nym-common-models = { path = "../common/models" }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
serde_json = { workspace = true }
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user