Compare commits

...

4 Commits

Author SHA1 Message Date
Simon Wicky ab11508235 [Product Data] Introduce data persistence on gateways (#5022)
* add stats storage to gateways

* config fix

* add stats storage model and logic

* adapt stats collection to new storage

* stats cleanup on start

* change to linux only code

* tweaks

* modified stats cleanup + change session started

* change wrong table name

* store crashed session as 0 duration

* adapt for sqlx 0.7

* remove unused dependencies

* revert changes from gateway config, as it is broken anyway

* copyright and misc stuff

---------

Co-authored-by: Simon Wicky <simon@linode2-2.net>
2024-10-28 09:25:37 +01:00
Fran Arbanas e65bfaeb31 Fix/nym data observatory dockerfile (#5021)
* fix: added needed env vars to dockerfile, updated db env for a bit

* feat: add github workflow for pushing data observatory

* feat: split the postgresql connection string into multiple variables

* fix docker compose

* fix workflow

* fix: short in clap
2024-10-24 18:10:34 +02:00
Jędrzej Stuczyński 5a6982fd10 Merge pull request #5011 from nymtech/dependabot/cargo/patch-updates-9a83837eff
Bump the patch-updates group across 1 directory with 10 updates
2024-10-23 10:51:55 +01:00
dependabot[bot] 7abe1f505c Bump the patch-updates group across 1 directory with 10 updates
Bumps the patch-updates group with 9 updates in the / directory:

| Package | From | To |
| --- | --- | --- |
| [anyhow](https://github.com/dtolnay/anyhow) | `1.0.89` | `1.0.90` |
| [clap](https://github.com/clap-rs/clap) | `4.5.18` | `4.5.20` |
| [clap_complete](https://github.com/clap-rs/clap) | `4.5.29` | `4.5.33` |
| [pin-project](https://github.com/taiki-e/pin-project) | `1.1.5` | `1.1.6` |
| [serde](https://github.com/serde-rs/serde) | `1.0.210` | `1.0.211` |
| [serde_json](https://github.com/serde-rs/json) | `1.0.128` | `1.0.132` |
| [wasm-bindgen](https://github.com/rustwasm/wasm-bindgen) | `0.2.93` | `0.2.95` |
| [wasm-bindgen-futures](https://github.com/rustwasm/wasm-bindgen) | `0.4.43` | `0.4.45` |
| [web-sys](https://github.com/rustwasm/wasm-bindgen) | `0.3.70` | `0.3.72` |



Updates `anyhow` from 1.0.89 to 1.0.90
- [Release notes](https://github.com/dtolnay/anyhow/releases)
- [Commits](https://github.com/dtolnay/anyhow/compare/1.0.89...1.0.90)

Updates `clap` from 4.5.18 to 4.5.20
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.18...clap_complete-v4.5.20)

Updates `clap_complete` from 4.5.29 to 4.5.33
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.29...clap_complete-v4.5.33)

Updates `pin-project` from 1.1.5 to 1.1.6
- [Release notes](https://github.com/taiki-e/pin-project/releases)
- [Changelog](https://github.com/taiki-e/pin-project/blob/main/CHANGELOG.md)
- [Commits](https://github.com/taiki-e/pin-project/compare/v1.1.5...v1.1.6)

Updates `serde` from 1.0.210 to 1.0.211
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.210...v1.0.211)

Updates `serde_derive` from 1.0.210 to 1.0.211
- [Release notes](https://github.com/serde-rs/serde/releases)
- [Commits](https://github.com/serde-rs/serde/compare/v1.0.210...v1.0.211)

Updates `serde_json` from 1.0.128 to 1.0.132
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/1.0.128...1.0.132)

Updates `wasm-bindgen` from 0.2.93 to 0.2.95
- [Release notes](https://github.com/rustwasm/wasm-bindgen/releases)
- [Changelog](https://github.com/rustwasm/wasm-bindgen/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rustwasm/wasm-bindgen/compare/0.2.93...0.2.95)

Updates `wasm-bindgen-futures` from 0.4.43 to 0.4.45
- [Release notes](https://github.com/rustwasm/wasm-bindgen/releases)
- [Changelog](https://github.com/rustwasm/wasm-bindgen/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rustwasm/wasm-bindgen/commits)

Updates `web-sys` from 0.3.70 to 0.3.72
- [Release notes](https://github.com/rustwasm/wasm-bindgen/releases)
- [Changelog](https://github.com/rustwasm/wasm-bindgen/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rustwasm/wasm-bindgen/commits)

---
updated-dependencies:
- dependency-name: anyhow
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: clap
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: clap_complete
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: pin-project
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: serde
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: serde_derive
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: serde_json
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: wasm-bindgen
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: wasm-bindgen-futures
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
- dependency-name: web-sys
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: patch-updates
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-10-22 14:56:39 +00:00
32 changed files with 2508 additions and 343 deletions
@@ -0,0 +1,55 @@
name: Build and upload Data observatory container to harbor.nymte.ch
on:
workflow_dispatch:
env:
WORKING_DIRECTORY: "nym-data-observatory"
CONTAINER_NAME: "data-observatory"
jobs:
build-container:
runs-on: arc-ubuntu-22.04-dind
steps:
- name: Login to Harbor
uses: docker/login-action@v3
with:
registry: harbor.nymte.ch
username: ${{ secrets.HARBOR_ROBOT_USERNAME }}
password: ${{ secrets.HARBOR_ROBOT_SECRET }}
- name: Checkout repo
uses: actions/checkout@v4
- name: Configure git identity
run: |
git config --global user.email "lawrence@nymtech.net"
git config --global user.name "Lawrence Stalder"
- name: Get version from cargo.toml
uses: mikefarah/yq@v4.44.3
id: get_version
with:
cmd: yq -oy '.package.version' ${{ env.WORKING_DIRECTORY }}/Cargo.toml
- name: Check if tag exists
run: |
if git rev-parse ${{ steps.get_version.outputs.value }} >/dev/null 2>&1; then
echo "Tag ${{ steps.get_version.outputs.value }} already exists"
fi
- name: Remove existing tag if exists
run: |
if git rev-parse ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} >/dev/null 2>&1; then
git push --delete origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
git tag -d ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
fi
- name: Create tag
run: |
git tag -a ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }} -m "Version ${{ steps.get_version.outputs.result }}"
git push origin ${{ env.WORKING_DIRECTORY }}-${{ steps.get_version.outputs.result }}
- name: BuildAndPushImageOnHarbor
run: |
docker build -f ${{ env.WORKING_DIRECTORY }}/Dockerfile . -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:${{ steps.get_version.outputs.result }} -t harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }}:latest
docker push harbor.nymte.ch/nym/${{ env.CONTAINER_NAME }} --all-tags
Generated
+129 -115
View File
@@ -11,7 +11,7 @@ dependencies = [
"macroific",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -223,9 +223,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.89"
version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95"
[[package]]
name = "arbitrary"
@@ -283,7 +283,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -341,7 +341,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -352,7 +352,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -1058,9 +1058,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.18"
version = "4.5.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1068,9 +1068,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.18"
version = "4.5.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
dependencies = [
"anstream",
"anstyle",
@@ -1080,11 +1080,11 @@ dependencies = [
[[package]]
name = "clap_complete"
version = "4.5.29"
version = "4.5.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e"
checksum = "9646e2e245bf62f45d39a0f3f36f1171ad1ea0d6967fd114bca72cb02a8fcdfb"
dependencies = [
"clap 4.5.18",
"clap 4.5.20",
]
[[package]]
@@ -1093,7 +1093,7 @@ version = "4.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d494102c8ff3951810c72baf96910b980fb065ca5d3101243e6a8dc19747c86b"
dependencies = [
"clap 4.5.18",
"clap 4.5.20",
"clap_complete",
]
@@ -1106,7 +1106,7 @@ dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -1501,7 +1501,7 @@ dependencies = [
"anes",
"cast",
"ciborium",
"clap 4.5.18",
"clap 4.5.20",
"criterion-plot",
"is-terminal",
"itertools 0.10.5",
@@ -1756,7 +1756,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -1914,7 +1914,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim 0.11.1",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -1936,7 +1936,7 @@ checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
dependencies = [
"darling_core 0.20.9",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -1987,7 +1987,7 @@ dependencies = [
"macroific",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2030,7 +2030,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2063,7 +2063,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2131,7 +2131,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2367,7 +2367,7 @@ name = "explorer-api"
version = "1.1.41"
dependencies = [
"chrono",
"clap 4.5.18",
"clap 4.5.20",
"dotenvy",
"humantime-serde",
"isocountry",
@@ -2467,7 +2467,7 @@ dependencies = [
"macroific",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2691,7 +2691,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -2786,7 +2786,7 @@ dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -3360,7 +3360,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bip39",
"clap 4.5.18",
"clap 4.5.20",
"dirs",
"importer-contract",
"nym-bin-common",
@@ -3653,9 +3653,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
[[package]]
name = "js-sys"
version = "0.3.70"
version = "0.3.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9"
dependencies = [
"wasm-bindgen",
]
@@ -3876,7 +3876,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -3887,7 +3887,7 @@ checksum = "13198c120864097a565ccb3ff947672d969932b7975ebd4085732c9f09435e55"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -3900,7 +3900,7 @@ dependencies = [
"macroific_core",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -4327,7 +4327,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -4359,7 +4359,7 @@ dependencies = [
"bloomfilter",
"bs58",
"cfg-if",
"clap 4.5.18",
"clap 4.5.20",
"console-subscriber",
"cosmwasm-std",
"cw-utils",
@@ -4474,7 +4474,7 @@ dependencies = [
"bincode",
"bs58",
"bytes",
"clap 4.5.18",
"clap 4.5.20",
"defguard_wireguard_rs",
"fastrand 2.1.1",
"futures",
@@ -4552,7 +4552,7 @@ dependencies = [
name = "nym-bin-common"
version = "0.6.0"
dependencies = [
"clap 4.5.18",
"clap 4.5.20",
"clap_complete",
"clap_complete_fig",
"const-str",
@@ -4594,7 +4594,7 @@ dependencies = [
"base64 0.22.1",
"bip39",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"clap_complete",
"clap_complete_fig",
"dotenvy",
@@ -4620,7 +4620,7 @@ dependencies = [
"bip39",
"bs58",
"cfg-if",
"clap 4.5.18",
"clap 4.5.20",
"colored",
"comfy-table",
"cosmrs 0.17.0-pre",
@@ -4672,7 +4672,7 @@ name = "nym-client"
version = "1.1.42"
dependencies = [
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"dirs",
"futures",
"log",
@@ -4712,7 +4712,7 @@ dependencies = [
"base64 0.22.1",
"bs58",
"cfg-if",
"clap 4.5.18",
"clap 4.5.20",
"comfy-table",
"futures",
"gloo-timers",
@@ -5100,7 +5100,7 @@ dependencies = [
"anyhow",
"axum 0.7.7",
"chrono",
"clap 4.5.18",
"clap 4.5.20",
"nym-bin-common",
"nym-network-defaults",
"nym-node-requests",
@@ -5240,7 +5240,7 @@ dependencies = [
"async-trait",
"bip39",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"colored",
"dashmap",
"defguard_wireguard_rs",
@@ -5258,6 +5258,7 @@ dependencies = [
"nym-credentials-interface",
"nym-crypto",
"nym-gateway-requests",
"nym-gateway-stats-storage",
"nym-gateway-storage",
"nym-ip-packet-router",
"nym-mixnet-client",
@@ -5353,6 +5354,19 @@ dependencies = [
"zeroize",
]
[[package]]
name = "nym-gateway-stats-storage"
version = "0.1.0"
dependencies = [
"nym-credentials-interface",
"nym-sphinx",
"sqlx",
"thiserror",
"time",
"tokio",
"tracing",
]
[[package]]
name = "nym-gateway-storage"
version = "0.1.0"
@@ -5447,7 +5461,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"nym-bin-common",
"nym-credential-storage",
"nym-id",
@@ -5489,7 +5503,7 @@ dependencies = [
"bincode",
"bs58",
"bytes",
"clap 4.5.18",
"clap 4.5.20",
"etherparse",
"futures",
"log",
@@ -5587,7 +5601,7 @@ dependencies = [
"anyhow",
"axum 0.7.7",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"colored",
"cupid",
"dirs",
@@ -5688,7 +5702,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"axum 0.7.7",
"clap 4.5.18",
"clap 4.5.20",
"dashmap",
"futures",
"log",
@@ -5720,7 +5734,7 @@ dependencies = [
"anyhow",
"async-trait",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"dirs",
"futures",
"humantime-serde",
@@ -5772,7 +5786,7 @@ dependencies = [
"bs58",
"cargo_metadata 0.18.1",
"celes",
"clap 4.5.18",
"clap 4.5.20",
"colored",
"cupid",
"humantime-serde",
@@ -5919,7 +5933,7 @@ name = "nym-nr-query"
version = "0.1.0"
dependencies = [
"anyhow",
"clap 4.5.18",
"clap 4.5.20",
"log",
"nym-bin-common",
"nym-network-defaults",
@@ -6058,7 +6072,7 @@ name = "nym-socks5-client"
version = "1.1.42"
dependencies = [
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"log",
"nym-bin-common",
"nym-client-core",
@@ -6494,7 +6508,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bip39",
"clap 4.5.18",
"clap 4.5.20",
"cosmwasm-std",
"futures",
"humantime 2.1.0",
@@ -6604,7 +6618,7 @@ version = "0.1.8"
dependencies = [
"anyhow",
"bytes",
"clap 4.5.18",
"clap 4.5.20",
"dotenvy",
"flate2",
"futures",
@@ -6913,7 +6927,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7000,7 +7014,7 @@ dependencies = [
"pest_meta",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7026,22 +7040,22 @@ dependencies = [
[[package]]
name = "pin-project"
version = "1.1.5"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.5"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7256,7 +7270,7 @@ dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7276,7 +7290,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
"version_check",
"yansi",
]
@@ -7339,7 +7353,7 @@ dependencies = [
"itertools 0.12.1",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7548,7 +7562,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -7775,7 +7789,7 @@ dependencies = [
"proc-macro2",
"quote",
"rocket_http",
"syn 2.0.66",
"syn 2.0.82",
"unicode-xid",
"version_check",
]
@@ -7892,7 +7906,7 @@ dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.66",
"syn 2.0.82",
"walkdir",
]
@@ -8140,7 +8154,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals 0.29.1",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8172,7 +8186,7 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8249,9 +8263,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.210"
version = "1.0.211"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
checksum = "1ac55e59090389fb9f0dd9e0f3c09615afed1d19094284d0b200441f13550793"
dependencies = [
"serde_derive",
]
@@ -8298,13 +8312,13 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.210"
version = "1.0.211"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
checksum = "54be4f245ce16bc58d57ef2716271d0d4519e0f6defa147f6e081005bcb278ff"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8315,7 +8329,7 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8326,14 +8340,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
name = "serde_json"
version = "1.0.128"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
dependencies = [
"itoa",
"memchr",
@@ -8359,7 +8373,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8410,7 +8424,7 @@ dependencies = [
"darling 0.20.9",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -8888,7 +8902,7 @@ name = "ssl-inject"
version = "0.1.0"
dependencies = [
"anyhow",
"clap 4.5.18",
"clap 4.5.20",
"hex",
"tokio",
]
@@ -9015,7 +9029,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -9058,9 +9072,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.66"
version = "2.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021"
dependencies = [
"proc-macro2",
"quote",
@@ -9304,7 +9318,7 @@ dependencies = [
"anyhow",
"bip39",
"bs58",
"clap 4.5.18",
"clap 4.5.20",
"console",
"cw-utils",
"dkg-bypass-contract",
@@ -9359,7 +9373,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -9490,7 +9504,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -9769,7 +9783,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -9927,7 +9941,7 @@ checksum = "0ea0b99e8ec44abd6f94a18f28f7934437809dd062820797c52401298116f70e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
"termcolor",
]
@@ -9954,7 +9968,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals 0.28.0",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -10108,7 +10122,7 @@ checksum = "21345172d31092fd48c47fd56c53d4ae9e41c4b1f559fb8c38c1ab1685fd919f"
dependencies = [
"anyhow",
"camino",
"clap 4.5.18",
"clap 4.5.20",
"uniffi_bindgen",
"uniffi_build",
"uniffi_core",
@@ -10125,7 +10139,7 @@ dependencies = [
"askama",
"camino",
"cargo_metadata 0.15.4",
"clap 4.5.18",
"clap 4.5.20",
"fs-err",
"glob",
"goblin",
@@ -10157,7 +10171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55137c122f712d9330fd985d66fa61bdc381752e89c35708c13ce63049a3002c"
dependencies = [
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -10189,7 +10203,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde",
"syn 2.0.66",
"syn 2.0.82",
"toml 0.5.11",
"uniffi_build",
"uniffi_meta",
@@ -10321,7 +10335,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -10359,7 +10373,7 @@ checksum = "17e82ab96c5a55263b5bed151b8426410d93aa909a453acdbd4b6792b5af7d64"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -10370,7 +10384,7 @@ checksum = "86b8338dc3c9526011ffaa2aa6bd60ddfda9d49d2123108690755c6e34844212"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
"utoipauto-core",
]
@@ -10457,9 +10471,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
[[package]]
name = "wasm-bindgen"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
dependencies = [
"cfg-if",
"once_cell",
@@ -10468,24 +10482,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.43"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b"
dependencies = [
"cfg-if",
"js-sys",
@@ -10495,9 +10509,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -10505,22 +10519,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
[[package]]
name = "wasm-bindgen-test"
@@ -10545,7 +10559,7 @@ checksum = "4b8220be1fa9e4c889b30fd207d4906657e7e90b12e0e6b0c8b8d8709f5de021"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -10641,9 +10655,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.70"
version = "0.3.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -11072,7 +11086,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
@@ -11092,7 +11106,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.82",
]
[[package]]
+9 -8
View File
@@ -54,6 +54,7 @@ members = [
"common/exit-policy",
"common/gateway-requests",
"common/gateway-storage",
"common/gateway-stats-storage",
"common/http-api-client",
"common/http-api-common",
"common/inclusion-probability",
@@ -183,7 +184,7 @@ aes = "0.8.1"
aes-gcm = "0.10.1"
aes-gcm-siv = "0.11.1"
aead = "0.5.2"
anyhow = "1.0.89"
anyhow = "1.0.90"
argon2 = "0.5.0"
async-trait = "0.1.83"
axum = "0.7.5"
@@ -208,7 +209,7 @@ chacha20 = "0.9.0"
chacha20poly1305 = "0.10.1"
chrono = "0.4.31"
cipher = "0.4.3"
clap = "4.5.18"
clap = "4.5.20"
clap_complete = "4.5"
clap_complete_fig = "4.5"
colored = "2.0"
@@ -275,7 +276,7 @@ opentelemetry-jaeger = "0.18.0"
parking_lot = "0.12.3"
pem = "0.8"
petgraph = "0.6.5"
pin-project = "1.0"
pin-project = "1.1"
pin-project-lite = "0.2.14"
pretty_env_logger = "0.4.0"
publicsuffix = "2.2.3"
@@ -295,10 +296,10 @@ rocket_okapi = "0.8.0"
safer-ffi = "0.1.13"
schemars = "0.8.21"
semver = "1.0.23"
serde = "1.0.210"
serde = "1.0.211"
serde_bytes = "0.11.15"
serde_derive = "1.0"
serde_json = "1.0.128"
serde_json = "1.0.132"
serde_repr = "0.1"
serde_with = "3.9.0"
serde_yaml = "0.9.25"
@@ -389,10 +390,10 @@ indexed_db_futures = { git = "https://github.com/TiemenSch/rust-indexed-db", bra
js-sys = "0.3.70"
serde-wasm-bindgen = "0.6.5"
tsify = "0.4.5"
wasm-bindgen = "0.2.93"
wasm-bindgen-futures = "0.4.43"
wasm-bindgen = "0.2.95"
wasm-bindgen-futures = "0.4.45"
wasmtimer = "0.2.0"
web-sys = "0.3.70"
web-sys = "0.3.72"
# Profile settings for individual crates
+34
View File
@@ -0,0 +1,34 @@
[package]
name = "nym-gateway-stats-storage"
version = "0.1.0"
authors.workspace = true
repository.workspace = true
homepage.workspace = true
documentation.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
sqlx = { workspace = true, features = [
"runtime-tokio-rustls",
"sqlite",
"macros",
"migrate",
"time",
] }
time = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
nym-sphinx = { path = "../nymsphinx" }
nym-credentials-interface = { path = "../credentials-interface" }
[build-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
sqlx = { workspace = true, features = [
"runtime-tokio-rustls",
"sqlite",
"macros",
"migrate",
] }
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use sqlx::{Connection, SqliteConnection};
use std::env;
#[tokio::main]
async fn main() {
let out_dir = env::var("OUT_DIR").unwrap();
let database_path = format!("{}/gateway-stats-example.sqlite", out_dir);
let mut conn = SqliteConnection::connect(&format!("sqlite://{}?mode=rwc", database_path))
.await
.expect("Failed to create SQLx database connection");
sqlx::migrate!("./migrations")
.run(&mut conn)
.await
.expect("Failed to perform SQLx migrations");
#[cfg(target_family = "unix")]
println!("cargo:rustc-env=DATABASE_URL=sqlite://{}", &database_path);
#[cfg(target_family = "windows")]
// for some strange reason we need to add a leading `/` to the windows path even though it's
// not a valid windows path... but hey, it works...
println!("cargo:rustc-env=DATABASE_URL=sqlite:///{}", &database_path);
}
@@ -0,0 +1,26 @@
/*
* Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
* SPDX-License-Identifier: GPL-3.0-only
*/
CREATE TABLE sessions_active
(
client_address TEXT NOT NULL PRIMARY KEY UNIQUE,
start_time TIMESTAMP WITHOUT TIME ZONE NOT NULL,
typ TEXT NOT NULL
);
CREATE TABLE sessions_finished
(
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
day DATE NOT NULL,
duration_ms INTEGER NOT NULL,
typ TEXT NOT NULL
);
CREATE TABLE sessions_unique_users
(
day DATE NOT NULL,
client_address TEXT NOT NULL,
PRIMARY KEY (day, client_address)
);
+13
View File
@@ -0,0 +1,13 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use thiserror::Error;
#[derive(Error, Debug)]
pub enum StatsStorageError {
#[error("Database experienced an internal error: {0}")]
InternalDatabaseError(#[from] sqlx::Error),
#[error("Failed to perform database migration: {0}")]
MigrationError(#[from] sqlx::migrate::MigrateError),
}
+195
View File
@@ -0,0 +1,195 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use error::StatsStorageError;
use models::{ActiveSession, FinishedSession, SessionType, StoredFinishedSession};
use nym_sphinx::DestinationAddressBytes;
use sessions::SessionManager;
use sqlx::ConnectOptions;
use std::path::Path;
use time::Date;
use tracing::{debug, error};
pub mod error;
pub mod models;
mod sessions;
// note that clone here is fine as upon cloning the same underlying pool will be used
#[derive(Clone)]
pub struct PersistentStatsStorage {
session_manager: SessionManager,
}
impl PersistentStatsStorage {
/// Initialises `PersistentStatsStorage` using the provided path.
///
/// # Arguments
///
/// * `database_path`: path to the database.
pub async fn init<P: AsRef<Path> + Send>(database_path: P) -> Result<Self, StatsStorageError> {
debug!(
"Attempting to connect to database {:?}",
database_path.as_ref().as_os_str()
);
// TODO: we can inject here more stuff based on our gateway global config
// struct. Maybe different pool size or timeout intervals?
let opts = sqlx::sqlite::SqliteConnectOptions::new()
.filename(database_path)
.create_if_missing(true)
.disable_statement_logging();
// TODO: do we want auto_vacuum ?
let connection_pool = match sqlx::SqlitePool::connect_with(opts).await {
Ok(db) => db,
Err(err) => {
error!("Failed to connect to SQLx database: {err}");
return Err(err.into());
}
};
if let Err(err) = sqlx::migrate!("./migrations").run(&connection_pool).await {
error!("Failed to perform migration on the SQLx database: {err}");
return Err(err.into());
}
// the cloning here are cheap as connection pool is stored behind an Arc
Ok(PersistentStatsStorage {
session_manager: sessions::SessionManager::new(connection_pool),
})
}
//Sessions fn
pub async fn insert_finished_session(
&self,
date: Date,
session: FinishedSession,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.insert_finished_session(
date,
session.duration.whole_milliseconds() as i64,
session.typ.to_string().into(),
)
.await?)
}
pub async fn get_finished_sessions(
&self,
date: Date,
) -> Result<Vec<StoredFinishedSession>, StatsStorageError> {
Ok(self.session_manager.get_finished_sessions(date).await?)
}
pub async fn delete_finished_sessions(
&self,
before_date: Date,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.delete_finished_sessions(before_date)
.await?)
}
pub async fn insert_unique_user(
&self,
date: Date,
client_address_bs58: String,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.insert_unique_user(date, client_address_bs58)
.await?)
}
pub async fn get_unique_users_count(&self, date: Date) -> Result<i32, StatsStorageError> {
Ok(self.session_manager.get_unique_users_count(date).await?)
}
pub async fn delete_unique_users(&self, before_date: Date) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.delete_unique_users(before_date)
.await?)
}
pub async fn insert_active_session(
&self,
client_address: DestinationAddressBytes,
session: ActiveSession,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.insert_active_session(
client_address.as_base58_string(),
session.start,
session.typ.to_string().into(),
)
.await?)
}
pub async fn update_active_session_type(
&self,
client_address: DestinationAddressBytes,
session_type: SessionType,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.update_active_session_type(
client_address.as_base58_string(),
session_type.to_string().into(),
)
.await?)
}
pub async fn get_active_session(
&self,
client_address: DestinationAddressBytes,
) -> Result<Option<ActiveSession>, StatsStorageError> {
Ok(self
.session_manager
.get_active_session(client_address.as_base58_string())
.await?
.map(Into::into))
}
pub async fn get_all_active_sessions(&self) -> Result<Vec<ActiveSession>, StatsStorageError> {
Ok(self
.session_manager
.get_all_active_sessions()
.await?
.into_iter()
.map(Into::into)
.collect())
}
pub async fn get_started_sessions_count(
&self,
start_date: Date,
) -> Result<i32, StatsStorageError> {
Ok(self
.session_manager
.get_started_sessions_count(start_date)
.await?)
}
pub async fn get_active_users(&self) -> Result<Vec<String>, StatsStorageError> {
Ok(self.session_manager.get_active_users().await?)
}
pub async fn delete_active_session(
&self,
client_address: DestinationAddressBytes,
) -> Result<(), StatsStorageError> {
Ok(self
.session_manager
.delete_active_session(client_address.as_base58_string())
.await?)
}
pub async fn cleanup_active_sessions(&self) -> Result<(), StatsStorageError> {
Ok(self.session_manager.cleanup_active_sessions().await?)
}
}
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use nym_credentials_interface::TicketType;
use sqlx::prelude::FromRow;
use time::{Duration, OffsetDateTime};
#[derive(FromRow)]
pub struct StoredFinishedSession {
duration_ms: i64,
typ: String,
}
impl StoredFinishedSession {
pub fn serialize(&self) -> (u64, String) {
(
self.duration_ms as u64, //we are sure that it fits in a u64, see `fn end_at`
self.typ.clone(),
)
}
}
pub struct FinishedSession {
pub duration: Duration,
pub typ: SessionType,
}
#[derive(PartialEq)]
pub enum SessionType {
Vpn,
Mixnet,
Unknown,
}
impl SessionType {
pub fn to_string(&self) -> &str {
match self {
Self::Vpn => "vpn",
Self::Mixnet => "mixnet",
Self::Unknown => "unknown",
}
}
pub fn from_string(s: &str) -> Self {
match s {
"vpn" => Self::Vpn,
"mixnet" => Self::Mixnet,
_ => Self::Unknown,
}
}
}
impl From<TicketType> for SessionType {
fn from(value: TicketType) -> Self {
match value {
TicketType::V1MixnetEntry => Self::Mixnet,
TicketType::V1MixnetExit => Self::Mixnet,
TicketType::V1WireguardEntry => Self::Vpn,
TicketType::V1WireguardExit => Self::Vpn,
}
}
}
#[derive(FromRow)]
pub(crate) struct StoredActiveSession {
start_time: OffsetDateTime,
typ: String,
}
pub struct ActiveSession {
pub start: OffsetDateTime,
pub typ: SessionType,
}
impl ActiveSession {
pub fn new(start_time: OffsetDateTime) -> Self {
ActiveSession {
start: start_time,
typ: SessionType::Unknown,
}
}
pub fn set_type(&mut self, ticket_type: TicketType) {
self.typ = ticket_type.into();
}
pub fn end_at(self, stop_time: OffsetDateTime) -> Option<FinishedSession> {
let session_duration = stop_time - self.start;
//ensure duration is positive to fit in a u64
//u64::max milliseconds is 500k millenia so no overflow issue
if session_duration > Duration::ZERO {
Some(FinishedSession {
duration: session_duration,
typ: self.typ,
})
} else {
None
}
}
}
impl From<StoredActiveSession> for ActiveSession {
fn from(value: StoredActiveSession) -> Self {
ActiveSession {
start: value.start_time,
typ: SessionType::from_string(&value.typ),
}
}
}
@@ -0,0 +1,177 @@
// Copyright 2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use time::{Date, OffsetDateTime};
use crate::models::{StoredActiveSession, StoredFinishedSession};
pub(crate) type Result<T> = std::result::Result<T, sqlx::Error>;
#[derive(Clone)]
pub(crate) struct SessionManager {
connection_pool: sqlx::SqlitePool,
}
impl SessionManager {
/// Creates new instance of the `SessionsManager` with the provided sqlite connection pool.
///
/// # Arguments
///
/// * `connection_pool`: database connection pool to use.
pub(crate) fn new(connection_pool: sqlx::SqlitePool) -> Self {
SessionManager { connection_pool }
}
pub(crate) async fn insert_finished_session(
&self,
date: Date,
duration_ms: i64,
typ: String,
) -> Result<()> {
sqlx::query!(
"INSERT INTO sessions_finished (day, duration_ms, typ) VALUES (?, ?, ?)",
date,
duration_ms,
typ
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn get_finished_sessions(
&self,
date: Date,
) -> Result<Vec<StoredFinishedSession>> {
sqlx::query_as("SELECT duration_ms, typ FROM sessions_finished WHERE day = ?")
.bind(date)
.fetch_all(&self.connection_pool)
.await
}
pub(crate) async fn delete_finished_sessions(&self, before_date: Date) -> Result<()> {
sqlx::query!("DELETE FROM sessions_finished WHERE day <= ? ", before_date)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn insert_unique_user(
&self,
date: Date,
client_address_b58: String,
) -> Result<()> {
sqlx::query!(
"INSERT OR IGNORE INTO sessions_unique_users (day, client_address) VALUES (?, ?)",
date,
client_address_b58,
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn get_unique_users_count(&self, date: Date) -> Result<i32> {
Ok(sqlx::query!(
"SELECT COUNT(*) as count FROM sessions_unique_users WHERE day = ?",
date
)
.fetch_one(&self.connection_pool)
.await?
.count)
}
pub(crate) async fn delete_unique_users(&self, before_date: Date) -> Result<()> {
sqlx::query!(
"DELETE FROM sessions_unique_users WHERE day <= ? ",
before_date
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn insert_active_session(
&self,
client_address_b58: String,
start_time: OffsetDateTime,
typ: String,
) -> Result<()> {
sqlx::query!(
"INSERT INTO sessions_active (client_address, start_time, typ) VALUES (?, ?, ?)",
client_address_b58,
start_time,
typ
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn update_active_session_type(
&self,
client_address_b58: String,
typ: String,
) -> Result<()> {
sqlx::query!(
"UPDATE sessions_active SET typ = ? WHERE client_address = ?",
typ,
client_address_b58,
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn get_active_session(
&self,
client_address_b58: String,
) -> Result<Option<StoredActiveSession>> {
sqlx::query_as("SELECT start_time, typ FROM sessions_active WHERE client_address = ?")
.bind(client_address_b58)
.fetch_optional(&self.connection_pool)
.await
}
pub(crate) async fn get_all_active_sessions(&self) -> Result<Vec<StoredActiveSession>> {
sqlx::query_as("SELECT start_time, typ FROM sessions_active")
.fetch_all(&self.connection_pool)
.await
}
pub(crate) async fn get_started_sessions_count(&self, start_date: Date) -> Result<i32> {
Ok(sqlx::query!(
"SELECT COUNT(*) as count FROM sessions_active WHERE date(start_time) = ?",
start_date
)
.fetch_one(&self.connection_pool)
.await?
.count)
}
pub(crate) async fn get_active_users(&self) -> Result<Vec<String>> {
Ok(sqlx::query!("SELECT client_address from sessions_active")
.fetch_all(&self.connection_pool)
.await?
.into_iter()
.map(|record| record.client_address)
.collect())
}
pub(crate) async fn delete_active_session(&self, client_address_b58: String) -> Result<()> {
sqlx::query!(
"DELETE FROM sessions_active WHERE client_address = ?",
client_address_b58
)
.execute(&self.connection_pool)
.await?;
Ok(())
}
pub(crate) async fn cleanup_active_sessions(&self) -> Result<()> {
sqlx::query!("DELETE FROM sessions_active")
.execute(&self.connection_pool)
.await?;
Ok(())
}
}
@@ -21,7 +21,7 @@ nym-sphinx-types = { path = "../types" }
nym-topology = { path = "../../topology" }
[target."cfg(target_arch = \"wasm32\")".dependencies.wasm-bindgen]
version = "0.2.93"
version = "0.2.95"
[dev-dependencies]
rand_chacha = { workspace = true }
+1
View File
@@ -69,6 +69,7 @@ nym-credentials-interface = { path = "../common/credentials-interface" }
nym-credential-verification = { path = "../common/credential-verification" }
nym-crypto = { path = "../common/crypto" }
nym-gateway-storage = { path = "../common/gateway-storage" }
nym-gateway-stats-storage = { path = "../common/gateway-stats-storage" }
nym-gateway-requests = { path = "../common/gateway-requests" }
nym-mixnet-client = { path = "../common/client-libs/mixnet-client" }
nym-mixnode-common = { path = "../common/mixnode-common" }
+8 -1
View File
@@ -12,6 +12,7 @@ pub const DEFAULT_PRIVATE_SPHINX_KEY_FILENAME: &str = "private_sphinx.pem";
pub const DEFAULT_PUBLIC_SPHINX_KEY_FILENAME: &str = "public_sphinx.pem";
pub const DEFAULT_CLIENTS_STORAGE_FILENAME: &str = "db.sqlite";
pub const DEFAULT_STATS_STORAGE_FILENAME: &str = "stats.sqlite";
pub const DEFAULT_NETWORK_REQUESTER_CONFIG_FILENAME: &str = "network_requester_config.toml";
pub const DEFAULT_NETWORK_REQUESTER_DATA_DIR: &str = "network-requester-data";
@@ -39,6 +40,9 @@ pub struct GatewayPaths {
#[serde(alias = "persistent_storage")]
pub clients_storage: PathBuf,
/// Path to sqlite database containing all persistent stats data.
pub stats_storage: PathBuf,
/// Path to the configuration of the embedded network requester.
#[serde(deserialize_with = "de_maybe_stringified")]
pub network_requester_config: Option<PathBuf>,
@@ -54,7 +58,9 @@ impl GatewayPaths {
pub fn new_default<P: AsRef<Path>>(id: P) -> Self {
GatewayPaths {
keys: KeysPaths::new_default(id.as_ref()),
clients_storage: default_data_directory(id).join(DEFAULT_CLIENTS_STORAGE_FILENAME),
clients_storage: default_data_directory(id.as_ref())
.join(DEFAULT_CLIENTS_STORAGE_FILENAME),
stats_storage: default_data_directory(id).join(DEFAULT_STATS_STORAGE_FILENAME),
// node_description: default_config_filepath(id).join(DEFAULT_DESCRIPTION_FILENAME),
network_requester_config: None,
ip_packet_router_config: None,
@@ -70,6 +76,7 @@ impl GatewayPaths {
public_sphinx_key_file: Default::default(),
},
clients_storage: Default::default(),
stats_storage: Default::default(),
network_requester_config: None,
ip_packet_router_config: None,
}
+7
View File
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: GPL-3.0-only
use nym_authenticator::error::AuthenticatorError;
use nym_gateway_stats_storage::error::StatsStorageError;
use nym_gateway_storage::error::StorageError;
use nym_ip_packet_router::error::IpPacketRouterError;
use nym_network_requester::error::{ClientCoreError, NetworkRequesterError};
@@ -115,6 +116,12 @@ pub enum GatewayError {
source: StorageError,
},
#[error("stats storage failure: {source}")]
StatsStorageError {
#[from]
source: StatsStorageError,
},
#[error("Path to network requester configuration file hasn't been specified. Perhaps try to run `setup-network-requester`?")]
UnspecifiedNetworkRequesterConfig,
+9
View File
@@ -5,6 +5,7 @@ use crate::config::Config;
use crate::error::GatewayError;
use nym_crypto::asymmetric::encryption;
use nym_gateway_stats_storage::PersistentStatsStorage;
use nym_gateway_storage::PersistentStorage;
use nym_pemstore::traits::PemStorableKeyPair;
use nym_pemstore::KeyPairPath;
@@ -74,6 +75,14 @@ pub(crate) async fn initialise_main_storage(
Ok(PersistentStorage::init(path, retrieval_limit).await?)
}
pub(crate) async fn initialise_stats_storage(
config: &Config,
) -> Result<PersistentStatsStorage, GatewayError> {
let path = &config.storage_paths.stats_storage;
Ok(PersistentStatsStorage::init(path).await?)
}
pub fn load_keypair<T: PemStorableKeyPair>(
paths: KeyPairPath,
name: impl Into<String>,
+25 -14
View File
@@ -12,7 +12,9 @@ use crate::http::HttpApiBuilder;
use crate::node::client_handling::active_clients::ActiveClientsStore;
use crate::node::client_handling::embedded_clients::{LocalEmbeddedClientHandle, MessageRouter};
use crate::node::client_handling::websocket;
use crate::node::helpers::{initialise_main_storage, load_network_requester_config};
use crate::node::helpers::{
initialise_main_storage, initialise_stats_storage, load_network_requester_config,
};
use crate::node::mixnet_handling::receiver::connection_handler::ConnectionHandler;
use futures::channel::{mpsc, oneshot};
use nym_credential_verification::ecash::{
@@ -41,6 +43,7 @@ pub(crate) mod helpers;
pub(crate) mod mixnet_handling;
pub(crate) mod statistics;
pub use nym_gateway_stats_storage::PersistentStatsStorage;
pub use nym_gateway_storage::{PersistentStorage, Storage};
// TODO: should this struct live here?
@@ -96,6 +99,8 @@ pub async fn create_gateway(
let storage = initialise_main_storage(&config).await?;
let stats_storage = initialise_stats_storage(&config).await?;
let nr_opts = network_requester_config.map(|config| LocalNetworkRequesterOpts {
config: config.clone(),
custom_mixnet_path: custom_mixnet.clone(),
@@ -106,7 +111,7 @@ pub async fn create_gateway(
custom_mixnet_path: custom_mixnet.clone(),
});
Gateway::new(config, nr_opts, ip_opts, storage)
Gateway::new(config, nr_opts, ip_opts, storage, stats_storage)
}
#[derive(Debug, Clone)]
@@ -147,7 +152,9 @@ pub struct Gateway<St = PersistentStorage> {
/// x25519 keypair used for Diffie-Hellman. Currently only used for sphinx key derivation.
sphinx_keypair: Arc<encryption::KeyPair>,
storage: St,
client_storage: St,
stats_storage: PersistentStatsStorage,
wireguard_data: Option<nym_wireguard::WireguardData>,
@@ -163,10 +170,12 @@ impl<St> Gateway<St> {
config: Config,
network_requester_opts: Option<LocalNetworkRequesterOpts>,
ip_packet_router_opts: Option<LocalIpPacketRouterOpts>,
storage: St,
client_storage: St,
stats_storage: PersistentStatsStorage,
) -> Result<Self, GatewayError> {
Ok(Gateway {
storage,
client_storage,
stats_storage,
identity_keypair: Arc::new(load_identity_keys(&config)?),
sphinx_keypair: Arc::new(helpers::load_sphinx_keys(&config)?),
config,
@@ -179,7 +188,7 @@ impl<St> Gateway<St> {
task_client: None,
})
}
#[allow(clippy::too_many_arguments)]
pub fn new_loaded(
config: Config,
network_requester_opts: Option<LocalNetworkRequesterOpts>,
@@ -187,7 +196,8 @@ impl<St> Gateway<St> {
authenticator_opts: Option<LocalAuthenticatorOpts>,
identity_keypair: Arc<identity::KeyPair>,
sphinx_keypair: Arc<encryption::KeyPair>,
storage: St,
client_storage: St,
stats_storage: PersistentStatsStorage,
) -> Self {
Gateway {
config,
@@ -196,7 +206,8 @@ impl<St> Gateway<St> {
authenticator_opts,
identity_keypair,
sphinx_keypair,
storage,
client_storage,
stats_storage,
wireguard_data: None,
session_stats: None,
run_http_server: true,
@@ -240,7 +251,7 @@ impl<St> Gateway<St> {
let connection_handler = ConnectionHandler::new(
packet_processor,
self.storage.clone(),
self.client_storage.clone(),
ack_sender,
active_clients_store,
);
@@ -275,7 +286,7 @@ impl<St> Gateway<St> {
forwarding_channel,
router_tx,
);
let all_peers = self.storage.get_all_wireguard_peers().await?;
let all_peers = self.client_storage.get_all_wireguard_peers().await?;
let used_private_network_ips = all_peers
.iter()
.cloned()
@@ -330,7 +341,7 @@ impl<St> Gateway<St> {
.start_with_shutdown(router_shutdown);
let wg_api = nym_wireguard::start_wireguard(
self.storage.clone(),
self.client_storage.clone(),
all_peers,
shutdown,
wireguard_data,
@@ -377,7 +388,7 @@ impl<St> Gateway<St> {
let shared_state = websocket::CommonHandlerState {
ecash_verifier,
storage: self.storage.clone(),
storage: self.client_storage.clone(),
local_identity: Arc::clone(&self.identity_keypair),
only_coconut_credentials: self.config.gateway.only_coconut_credentials,
bandwidth_cfg: (&self.config).into(),
@@ -415,7 +426,7 @@ impl<St> Gateway<St> {
info!("Starting gateway stats collector...");
let (mut stats_collector, stats_event_sender) =
GatewayStatisticsCollector::new(shared_session_stats);
GatewayStatisticsCollector::new(shared_session_stats, self.stats_storage.clone());
tokio::spawn(async move { stats_collector.run(shutdown).await });
stats_event_sender
}
@@ -654,7 +665,7 @@ impl<St> Gateway<St> {
nyxd_client,
self.identity_keypair.public_key().to_bytes(),
shutdown.fork("EcashVerifier"),
self.storage.clone(),
self.client_storage.clone(),
)
.await?,
);
+25 -4
View File
@@ -2,13 +2,14 @@
// SPDX-License-Identifier: GPL-3.0-only
use futures::{channel::mpsc, StreamExt};
use nym_gateway_stats_storage::PersistentStatsStorage;
use nym_node_http_api::state::metrics::SharedSessionStats;
use nym_statistics_common::events::{StatsEvent, StatsEventReceiver, StatsEventSender};
use nym_task::TaskClient;
use sessions::SessionStatsHandler;
use std::time::Duration;
use time::OffsetDateTime;
use tracing::trace;
use tracing::{error, trace, warn};
pub mod sessions;
@@ -23,21 +24,38 @@ pub(crate) struct GatewayStatisticsCollector {
impl GatewayStatisticsCollector {
pub fn new(
shared_session_stats: SharedSessionStats,
stats_storage: PersistentStatsStorage,
) -> (GatewayStatisticsCollector, StatsEventSender) {
let (stats_event_tx, stats_event_rx) = mpsc::unbounded();
let session_stats = SessionStatsHandler::new(shared_session_stats, stats_storage);
let collector = GatewayStatisticsCollector {
stats_event_rx,
session_stats: SessionStatsHandler::new(shared_session_stats),
session_stats,
};
(collector, stats_event_tx)
}
async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
self.session_stats.update_shared_state(update_time).await;
if let Err(e) = self
.session_stats
.maybe_update_shared_state(update_time)
.await
{
error!("Failed to update session stats - {e}");
}
//here goes additionnal stats handler update
}
async fn on_start(&mut self) {
if let Err(e) = self.session_stats.on_start().await {
error!("Failed to cleanup session stats handler - {e}");
}
//here goes additionnal stats handler start cleanup
}
pub async fn run(&mut self, mut shutdown: TaskClient) {
self.on_start().await;
let mut update_interval = tokio::time::interval(STATISTICS_UPDATE_TIMER_INTERVAL);
while !shutdown.is_shutdown() {
tokio::select! {
@@ -53,7 +71,10 @@ impl GatewayStatisticsCollector {
Some(stat_event) = self.stats_event_rx.next() => {
//dispatching event to proper handler
match stat_event {
StatsEvent::SessionStatsEvent(event) => self.session_stats.handle_event(event),
StatsEvent::SessionStatsEvent(event) => {
if let Err(e) = self.session_stats.handle_event(event).await{
warn!("Session event handling error - {e}");
}},
}
},
+113 -131
View File
@@ -2,176 +2,158 @@
// SPDX-License-Identifier: GPL-3.0-only
use nym_credentials_interface::TicketType;
use nym_gateway_stats_storage::models::FinishedSession;
use nym_gateway_stats_storage::PersistentStatsStorage;
use nym_gateway_stats_storage::{error::StatsStorageError, models::ActiveSession};
use nym_node_http_api::state::metrics::SharedSessionStats;
use nym_sphinx::DestinationAddressBytes;
use std::collections::{HashMap, HashSet};
use time::{Date, Duration, OffsetDateTime};
use nym_statistics_common::events::SessionEvent;
const FINISHED_SESSIONS_CAP: usize = 1_000_000; //to be on the safe side of memory blowups until persistent storage
#[derive(PartialEq)]
enum SessionType {
Vpn,
Mixnet,
Unknown,
}
impl SessionType {
fn to_string(&self) -> &str {
match self {
Self::Vpn => "vpn",
Self::Mixnet => "mixnet",
Self::Unknown => "unknown",
}
}
}
impl From<TicketType> for SessionType {
fn from(value: TicketType) -> Self {
match value {
TicketType::V1MixnetEntry => Self::Mixnet,
TicketType::V1MixnetExit => Self::Mixnet,
TicketType::V1WireguardEntry => Self::Vpn,
TicketType::V1WireguardExit => Self::Vpn,
}
}
}
struct FinishedSession {
duration: Duration,
typ: SessionType,
}
impl FinishedSession {
fn serialize(&self) -> (u64, String) {
(
self.duration.whole_milliseconds() as u64, //we are sure that it fits in a u64, see `fn end_at`
self.typ.to_string().into(),
)
}
}
struct ActiveSession {
start: OffsetDateTime,
typ: SessionType,
}
impl ActiveSession {
fn new(start_time: OffsetDateTime) -> Self {
ActiveSession {
start: start_time,
typ: SessionType::Unknown,
}
}
fn set_type(&mut self, ticket_type: TicketType) {
self.typ = ticket_type.into();
}
fn end_at(self, stop_time: OffsetDateTime) -> Option<FinishedSession> {
let session_duration = stop_time - self.start;
//ensure duration is positive to fit in a u64
//u64::max milliseconds is 500k millenia so no overflow issue
if session_duration > Duration::ZERO {
Some(FinishedSession {
duration: session_duration,
typ: self.typ,
})
} else {
None
}
}
}
pub(crate) struct SessionStatsHandler {
last_update_day: Date,
storage: PersistentStatsStorage,
current_day: Date,
shared_session_stats: SharedSessionStats,
active_sessions: HashMap<DestinationAddressBytes, ActiveSession>,
unique_users: HashSet<DestinationAddressBytes>,
sessions_started: u32,
finished_sessions: Vec<FinishedSession>,
}
impl SessionStatsHandler {
pub fn new(shared_session_stats: SharedSessionStats) -> Self {
pub fn new(shared_session_stats: SharedSessionStats, storage: PersistentStatsStorage) -> Self {
SessionStatsHandler {
last_update_day: OffsetDateTime::now_utc().date(),
storage,
current_day: OffsetDateTime::now_utc().date(),
shared_session_stats,
active_sessions: Default::default(),
unique_users: Default::default(),
sessions_started: 0,
finished_sessions: Default::default(),
}
}
pub(crate) fn handle_event(&mut self, event: SessionEvent) {
pub(crate) async fn handle_event(
&mut self,
event: SessionEvent,
) -> Result<(), StatsStorageError> {
match event {
SessionEvent::SessionStart { start_time, client } => {
self.handle_session_start(start_time, client);
self.handle_session_start(start_time, client).await
}
SessionEvent::SessionStop { stop_time, client } => {
self.handle_session_stop(stop_time, client);
self.handle_session_stop(stop_time, client).await
}
SessionEvent::EcashTicket {
ticket_type,
client,
} => self.handle_ecash_ticket(ticket_type, client),
} => self.handle_ecash_ticket(ticket_type, client).await,
}
}
fn handle_session_start(
async fn handle_session_start(
&mut self,
start_time: OffsetDateTime,
client: DestinationAddressBytes,
) {
self.sessions_started += 1;
self.unique_users.insert(client);
self.active_sessions
.insert(client, ActiveSession::new(start_time));
}
fn handle_session_stop(&mut self, stop_time: OffsetDateTime, client: DestinationAddressBytes) {
if let Some(session) = self.active_sessions.remove(&client) {
if let Some(finished_session) = session.end_at(stop_time) {
if self.finished_sessions.len() < FINISHED_SESSIONS_CAP {
self.finished_sessions.push(finished_session);
}
}
}
) -> Result<(), StatsStorageError> {
self.storage
.insert_unique_user(self.current_day, client.as_base58_string())
.await?;
self.storage
.insert_active_session(client, ActiveSession::new(start_time))
.await?;
Ok(())
}
fn handle_ecash_ticket(&mut self, ticket_type: TicketType, client: DestinationAddressBytes) {
if let Some(active_session) = self.active_sessions.get_mut(&client) {
if active_session.typ == SessionType::Unknown {
active_session.set_type(ticket_type);
async fn handle_session_stop(
&mut self,
stop_time: OffsetDateTime,
client: DestinationAddressBytes,
) -> Result<(), StatsStorageError> {
if let Some(session) = self.storage.get_active_session(client).await? {
if let Some(finished_session) = session.end_at(stop_time) {
self.storage
.insert_finished_session(self.current_day, finished_session)
.await?;
self.storage.delete_active_session(client).await?;
}
}
Ok(())
}
async fn handle_ecash_ticket(
&mut self,
ticket_type: TicketType,
client: DestinationAddressBytes,
) -> Result<(), StatsStorageError> {
self.storage
.update_active_session_type(client, ticket_type.into())
.await?;
Ok(())
}
pub(crate) async fn on_start(&mut self) -> Result<(), StatsStorageError> {
let yesterday = OffsetDateTime::now_utc().date() - Duration::DAY;
//publish yesterday's data if any
self.publish_stats(yesterday).await?;
//store "active" sessions as duration 0
for active_session in self.storage.get_all_active_sessions().await? {
self.storage
.insert_finished_session(
self.current_day,
FinishedSession {
duration: Duration::ZERO,
typ: active_session.typ,
},
)
.await?
}
//cleanup active sessions
self.storage.cleanup_active_sessions().await?;
//delete old entries
self.delete_old_stats(yesterday - Duration::DAY).await?;
Ok(())
}
//update shared state once a day has passed, with data from the previous day
pub(crate) async fn update_shared_state(&mut self, update_time: OffsetDateTime) {
let update_date = update_time.date();
if update_date != self.last_update_day {
{
let mut shared_state = self.shared_session_stats.write().await;
shared_state.update_time = self.last_update_day;
shared_state.unique_active_users = self.unique_users.len() as u32;
shared_state.session_started = self.sessions_started;
shared_state.sessions = self
.finished_sessions
.iter()
.map(|s| s.serialize())
.collect();
}
self.reset_stats(update_date);
async fn publish_stats(&mut self, stats_date: Date) -> Result<(), StatsStorageError> {
let finished_sessions = self.storage.get_finished_sessions(stats_date).await?;
let user_count = self.storage.get_unique_users_count(stats_date).await?;
let session_started = self.storage.get_started_sessions_count(stats_date).await? as u32;
{
let mut shared_state = self.shared_session_stats.write().await;
shared_state.update_time = stats_date;
shared_state.unique_active_users = user_count as u32;
shared_state.session_started = session_started;
shared_state.sessions = finished_sessions.iter().map(|s| s.serialize()).collect();
}
Ok(())
}
pub(crate) async fn maybe_update_shared_state(
&mut self,
update_time: OffsetDateTime,
) -> Result<(), StatsStorageError> {
let update_date = update_time.date();
if update_date != self.current_day {
self.publish_stats(self.current_day).await?;
self.delete_old_stats(self.current_day - Duration::DAY)
.await?;
self.reset_stats(update_date).await?;
self.current_day = update_date;
}
Ok(())
}
fn reset_stats(&mut self, reset_day: Date) {
self.last_update_day = reset_day;
self.unique_users = self.active_sessions.keys().copied().collect();
self.finished_sessions = Default::default();
self.sessions_started = 0;
async fn reset_stats(&mut self, reset_day: Date) -> Result<(), StatsStorageError> {
//active users reset
let new_active_users = self.storage.get_active_users().await?;
for user in new_active_users {
self.storage.insert_unique_user(reset_day, user).await?;
}
Ok(())
}
async fn delete_old_stats(&mut self, delete_before: Date) -> Result<(), StatsStorageError> {
self.storage.delete_finished_sessions(delete_before).await?;
self.storage.delete_unique_users(delete_before).await?;
Ok(())
}
}
+12
View File
@@ -5,6 +5,18 @@ WORKDIR /usr/src/nym/nym-data-observatory
RUN cargo build --release
#-------------------------------------------------------------------
# The following environment variables are required at runtime:
#
# NYM_DATA_OBSERVATORY_CONNECTION_URL
#
# And optionally:
#
# NYM_DATA_OBSERVATORY_HTTP_PORT
#
# see https://github.com/nymtech/nym/blob/develop/nym-data-observatory/src/main.rs for details
#-------------------------------------------------------------------
FROM ubuntu:24.04
RUN apt update && apt install -yy curl ca-certificates
+7 -1
View File
@@ -18,8 +18,14 @@ services:
dockerfile: nym-data-observatory/Dockerfile
container_name: nym-data-observatory
environment:
NYM_DATA_OBSERVATORY_CONNECTION_URL: "postgres://postgres:password@postgres:5432"
NYM_DATA_OBSERVATORY_CONNECTION_USERNAME: "postgres"
NYM_DATA_OBSERVATORY_CONNECTION_PASSWORD: "password"
NYM_DATA_OBSERVATORY_CONNECTION_HOST: "postgres"
NYM_DATA_OBSERVATORY_CONNECTION_PORT: "5432"
NYM_DATA_OBSERVATORY_CONNECTION_DB: ""
NYM_DATA_OBSERVATORY_HTTP_PORT: 8000
env_file:
- ../envs/qa.env
volumes:
pgdata:
+1 -3
View File
@@ -14,9 +14,7 @@ pub(crate) struct Storage {
}
impl Storage {
pub async fn init(connection_url: Option<String>) -> Result<Self> {
let connection_url =
connection_url.ok_or_else(|| anyhow!("Missing the connection url for database!"))?;
pub async fn init(connection_url: String) -> Result<Self> {
let connect_options =
PgConnectOptions::from_str(&connection_url)?.disable_statement_logging();
+29 -4
View File
@@ -18,9 +18,25 @@ struct Args {
#[arg(short, long, default_value = None, env = "NYM_DATA_OBSERVATORY_ENV_FILE")]
env_file: Option<String>,
/// DB connection url
#[arg(short, long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_URL")]
connection_url: Option<String>,
/// DB connection username
#[arg(long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_USERNAME")]
connection_username: String,
/// DB connection password
#[arg(long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_PASSWORD")]
connection_password: String,
/// DB connection host
#[arg(long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_HOST")]
connection_host: String,
/// DB connection port
#[arg(long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_PORT")]
connection_port: String,
/// DB connection database name
#[arg(long, default_value = None, env = "NYM_DATA_OBSERVATORY_CONNECTION_DB")]
connection_db: String,
}
#[tokio::main]
@@ -31,7 +47,16 @@ async fn main() -> anyhow::Result<()> {
setup_env(args.env_file); // Defaults to mainnet if empty
let storage = db::Storage::init(args.connection_url).await?;
let connection_url = format!(
"postgres://{}:{}@{}:{}/{}",
args.connection_username,
args.connection_password,
args.connection_host,
args.connection_port,
args.connection_db
);
let storage = db::Storage::init(connection_url).await?;
let db_pool = storage.pool_owned().await;
tokio::spawn(async move {
background_task::spawn_in_background(db_pool).await;
+2
View File
@@ -4,7 +4,9 @@
mod old_config_v1;
mod old_config_v2;
mod old_config_v3;
mod old_config_v4;
pub use old_config_v1::try_upgrade_config_v1;
pub use old_config_v2::try_upgrade_config_v2;
pub use old_config_v3::try_upgrade_config_v3;
pub use old_config_v4::try_upgrade_config_v4;
@@ -4,13 +4,7 @@
#![allow(dead_code)]
use crate::{config::*, error::KeyIOFailure};
use entry_gateway::Debug as EntryGatewayConfigDebug;
use exit_gateway::{IpPacketRouter, IpPacketRouterDebug, NetworkRequester, NetworkRequesterDebug};
use mixnode::{Verloc, VerlocDebug};
use nym_client_core_config_types::{
disk_persistence::{ClientKeysPaths, CommonClientPaths},
DebugConfig as ClientDebugConfig,
};
use nym_client_core_config_types::DebugConfig as ClientDebugConfig;
use nym_config::serde_helpers::de_maybe_port;
use nym_crypto::asymmetric::{ed25519, x25519};
use nym_network_requester::{
@@ -19,6 +13,7 @@ use nym_network_requester::{
};
use nym_pemstore::{store_key, store_keypair};
use nym_sphinx_acknowledgements::AckKey;
use old_configs::old_config_v4::*;
use persistence::*;
use rand::rngs::OsRng;
use serde::{Deserialize, Serialize};
@@ -90,12 +85,12 @@ pub enum NodeModeV3 {
ExitGateway,
}
impl From<NodeModeV3> for NodeMode {
impl From<NodeModeV3> for NodeModeV4 {
fn from(config: NodeModeV3) -> Self {
match config {
NodeModeV3::Mixnode => NodeMode::Mixnode,
NodeModeV3::EntryGateway => NodeMode::EntryGateway,
NodeModeV3::ExitGateway => NodeMode::ExitGateway,
NodeModeV3::Mixnode => NodeModeV4::Mixnode,
NodeModeV3::EntryGateway => NodeModeV4::EntryGateway,
NodeModeV3::ExitGateway => NodeModeV4::ExitGateway,
}
}
}
@@ -601,23 +596,6 @@ impl AuthenticatorPathsV3 {
}
}
pub fn to_common_client_paths(&self) -> CommonClientPaths {
CommonClientPaths {
keys: ClientKeysPaths {
private_identity_key_file: self.private_ed25519_identity_key_file.clone(),
public_identity_key_file: self.public_ed25519_identity_key_file.clone(),
private_encryption_key_file: self.private_x25519_diffie_hellman_key_file.clone(),
public_encryption_key_file: self.public_x25519_diffie_hellman_key_file.clone(),
ack_key_file: self.ack_key_file.clone(),
},
gateway_registrations: self.gateway_registrations.clone(),
// not needed for embedded providers
credentials_database: Default::default(),
reply_surb_database: self.reply_surb_database.clone(),
}
}
pub fn ed25519_identity_storage_paths(&self) -> nym_pemstore::KeyPairPath {
nym_pemstore::KeyPairPath::new(
&self.private_ed25519_identity_key_file,
@@ -963,7 +941,7 @@ pub async fn initialise(
pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
path: P,
prev_config: Option<ConfigV3>,
) -> Result<Config, NymNodeError> {
) -> Result<ConfigV4, NymNodeError> {
tracing::debug!("Updating from 1.1.4");
let old_cfg = if let Some(prev_config) = prev_config {
prev_config
@@ -981,21 +959,21 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
.ok_or(NymNodeError::DataDirDerivationFailure)?,
);
let cfg = Config {
let cfg = ConfigV4 {
save_path: old_cfg.save_path,
id: old_cfg.id,
mode: old_cfg.mode.into(),
host: Host {
host: HostV4 {
public_ips: old_cfg.host.public_ips,
hostname: old_cfg.host.hostname,
location: old_cfg.host.location,
},
mixnet: Mixnet {
mixnet: MixnetV4 {
bind_address: old_cfg.mixnet.bind_address,
announce_port: None,
nym_api_urls: old_cfg.mixnet.nym_api_urls,
nyxd_urls: old_cfg.mixnet.nyxd_urls,
debug: MixnetDebug {
debug: MixnetDebugV4 {
packet_forwarding_initial_backoff: old_cfg
.mixnet
.debug
@@ -1009,8 +987,8 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
unsafe_disable_noise: old_cfg.mixnet.debug.unsafe_disable_noise,
},
},
storage_paths: NymNodePaths {
keys: KeysPaths {
storage_paths: NymNodePathsV4 {
keys: KeysPathsV4 {
private_ed25519_identity_key_file: old_cfg
.storage_paths
.keys
@@ -1038,7 +1016,7 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
},
description: old_cfg.storage_paths.description,
},
http: Http {
http: HttpV4 {
bind_address: old_cfg.http.bind_address,
landing_page_assets_path: old_cfg.http.landing_page_assets_path,
access_token: old_cfg.http.access_token,
@@ -1046,13 +1024,13 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
expose_system_hardware: old_cfg.http.expose_system_hardware,
expose_crypto_hardware: old_cfg.http.expose_crypto_hardware,
},
wireguard: Wireguard {
wireguard: WireguardV4 {
enabled: old_cfg.wireguard.enabled,
bind_address: old_cfg.wireguard.bind_address,
private_ip: old_cfg.wireguard.private_ip,
announced_port: old_cfg.wireguard.announced_port,
private_network_prefix: old_cfg.wireguard.private_network_prefix,
storage_paths: WireguardPaths {
storage_paths: WireguardPathsV4 {
private_diffie_hellman_key_file: old_cfg
.wireguard
.storage_paths
@@ -1063,12 +1041,12 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
.public_diffie_hellman_key_file,
},
},
mixnode: MixnodeConfig {
storage_paths: MixnodePaths {},
verloc: Verloc {
mixnode: MixnodeConfigV4 {
storage_paths: MixnodePathsV4 {},
verloc: VerlocV4 {
bind_address: old_cfg.mixnode.verloc.bind_address,
announce_port: None,
debug: VerlocDebug {
debug: VerlocDebugV4 {
packets_per_node: old_cfg.mixnode.verloc.debug.packets_per_node,
connection_timeout: old_cfg.mixnode.verloc.debug.connection_timeout,
packet_timeout: old_cfg.mixnode.verloc.debug.packet_timeout,
@@ -1078,16 +1056,16 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
retry_timeout: old_cfg.mixnode.verloc.debug.retry_timeout,
},
},
debug: mixnode::Debug {
debug: DebugV4 {
node_stats_logging_delay: old_cfg.mixnode.debug.node_stats_logging_delay,
node_stats_updating_delay: old_cfg.mixnode.debug.node_stats_updating_delay,
},
},
entry_gateway: EntryGatewayConfig {
storage_paths: EntryGatewayPaths {
entry_gateway: EntryGatewayConfigV4 {
storage_paths: EntryGatewayPathsV4 {
clients_storage: old_cfg.entry_gateway.storage_paths.clients_storage,
cosmos_mnemonic: old_cfg.entry_gateway.storage_paths.cosmos_mnemonic,
authenticator: AuthenticatorPaths {
authenticator: AuthenticatorPathsV4 {
private_ed25519_identity_key_file: old_cfg
.entry_gateway
.storage_paths
@@ -1129,16 +1107,16 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
bind_address: old_cfg.entry_gateway.bind_address,
announce_ws_port: old_cfg.entry_gateway.announce_ws_port,
announce_wss_port: old_cfg.entry_gateway.announce_wss_port,
debug: EntryGatewayConfigDebug {
debug: EntryGatewayConfigDebugV4 {
message_retrieval_limit: old_cfg.entry_gateway.debug.message_retrieval_limit,
// \/ ADDED
zk_nym_tickets: Default::default(),
},
},
exit_gateway: ExitGatewayConfig {
storage_paths: ExitGatewayPaths {
exit_gateway: ExitGatewayConfigV4 {
storage_paths: ExitGatewayPathsV4 {
clients_storage: exit_gateway_paths.clients_storage,
network_requester: NetworkRequesterPaths {
network_requester: NetworkRequesterPathsV4 {
private_ed25519_identity_key_file: old_cfg
.exit_gateway
.storage_paths
@@ -1175,7 +1153,7 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
.network_requester
.gateway_registrations,
},
ip_packet_router: IpPacketRouterPaths {
ip_packet_router: IpPacketRouterPathsV4 {
private_ed25519_identity_key_file: old_cfg
.exit_gateway
.storage_paths
@@ -1212,7 +1190,7 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
.ip_packet_router
.gateway_registrations,
},
authenticator: AuthenticatorPaths {
authenticator: AuthenticatorPathsV4 {
private_ed25519_identity_key_file: old_cfg
.exit_gateway
.storage_paths
@@ -1252,8 +1230,8 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
},
open_proxy: old_cfg.exit_gateway.open_proxy,
upstream_exit_policy_url: old_cfg.exit_gateway.upstream_exit_policy_url,
network_requester: NetworkRequester {
debug: NetworkRequesterDebug {
network_requester: NetworkRequesterV4 {
debug: NetworkRequesterDebugV4 {
enabled: old_cfg.exit_gateway.network_requester.debug.enabled,
disable_poisson_rate: old_cfg
.exit_gateway
@@ -1263,8 +1241,8 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
client_debug: old_cfg.exit_gateway.network_requester.debug.client_debug,
},
},
ip_packet_router: IpPacketRouter {
debug: IpPacketRouterDebug {
ip_packet_router: IpPacketRouterV4 {
debug: IpPacketRouterDebugV4 {
enabled: old_cfg.exit_gateway.ip_packet_router.debug.enabled,
disable_poisson_rate: old_cfg
.exit_gateway
@@ -1277,7 +1255,7 @@ pub async fn try_upgrade_config_v3<P: AsRef<Path>>(
debug: Default::default(),
},
authenticator: Default::default(),
logging: LoggingSettings {},
logging: LoggingSettingsV4 {},
};
Ok(cfg)
File diff suppressed because it is too large Load Diff
+9
View File
@@ -24,6 +24,7 @@ pub const DEFAULT_NYMNODE_DESCRIPTION_FILENAME: &str = "description.toml";
// Entry Gateway:
pub const DEFAULT_CLIENTS_STORAGE_FILENAME: &str = "clients.sqlite";
pub const DEFAULT_STATS_STORAGE_FILENAME: &str = "stats.sqlite";
pub const DEFAULT_MNEMONIC_FILENAME: &str = "cosmos_mnemonic";
// Exit Gateway:
@@ -147,6 +148,9 @@ pub struct EntryGatewayPaths {
/// derived shared keys, available client bandwidths and wireguard peers.
pub clients_storage: PathBuf,
/// Path to sqlite database containing all persistent stats data.
pub stats_storage: PathBuf,
/// Path to file containing cosmos account mnemonic used for zk-nym redemption.
pub cosmos_mnemonic: PathBuf,
@@ -157,6 +161,7 @@ impl EntryGatewayPaths {
pub fn new<P: AsRef<Path>>(data_dir: P) -> Self {
EntryGatewayPaths {
clients_storage: data_dir.as_ref().join(DEFAULT_CLIENTS_STORAGE_FILENAME),
stats_storage: data_dir.as_ref().join(DEFAULT_STATS_STORAGE_FILENAME),
cosmos_mnemonic: data_dir.as_ref().join(DEFAULT_MNEMONIC_FILENAME),
authenticator: AuthenticatorPaths::new(data_dir),
}
@@ -207,6 +212,9 @@ pub struct ExitGatewayPaths {
/// derived shared keys, available client bandwidths and wireguard peers.
pub clients_storage: PathBuf,
/// Path to sqlite database containing all persistent stats data.
pub stats_storage: PathBuf,
pub network_requester: NetworkRequesterPaths,
pub ip_packet_router: IpPacketRouterPaths,
@@ -459,6 +467,7 @@ impl ExitGatewayPaths {
let data_dir = data_dir.as_ref();
ExitGatewayPaths {
clients_storage: data_dir.join(DEFAULT_CLIENTS_STORAGE_FILENAME),
stats_storage: data_dir.join(DEFAULT_STATS_STORAGE_FILENAME),
network_requester: NetworkRequesterPaths::new(data_dir),
ip_packet_router: IpPacketRouterPaths::new(data_dir),
authenticator: AuthenticatorPaths::new(data_dir),
+7
View File
@@ -185,6 +185,9 @@ announce_wss_port = {{#if entry_gateway.announce_wss_port }} {{ entry_gateway.an
# derived shared keys, available client bandwidths and wireguard peers.
clients_storage = '{{ entry_gateway.storage_paths.clients_storage }}'
# Path to sqlite database containing all persistent stats data.
stats_storage = '{{ entry_gateway.storage_paths.stats_storage }}'
# Path to file containing cosmos account mnemonic used for zk-nym redemption.
cosmos_mnemonic = '{{ entry_gateway.storage_paths.cosmos_mnemonic }}'
@@ -237,6 +240,10 @@ upstream_exit_policy_url = '{{ exit_gateway.upstream_exit_policy_url }}'
# derived shared keys, available client bandwidths and wireguard peers.
clients_storage = '{{ exit_gateway.storage_paths.clients_storage }}'
# Path to sqlite database containing all persistent stats data.
stats_storage = '{{ exit_gateway.storage_paths.stats_storage }}'
[exit_gateway.storage_paths.network_requester]
# Path to file containing network requester ed25519 identity private key.
private_ed25519_identity_key_file = '{{ exit_gateway.storage_paths.network_requester.private_ed25519_identity_key_file }}'
+2 -1
View File
@@ -10,7 +10,8 @@ use std::path::Path;
async fn try_upgrade_config(path: &Path) -> Result<(), NymNodeError> {
let cfg = try_upgrade_config_v1(path, None).await.ok();
let cfg = try_upgrade_config_v2(path, cfg).await.ok();
match try_upgrade_config_v3(path, cfg).await {
let cfg = try_upgrade_config_v3(path, cfg).await.ok();
match try_upgrade_config_v4(path, cfg).await {
Ok(cfg) => cfg.save(),
Err(e) => {
tracing::error!("Failed to finish upgrade - {e}");
+15
View File
@@ -67,6 +67,7 @@ impl MixnodeData {
pub struct EntryGatewayData {
mnemonic: Zeroizing<bip39::Mnemonic>,
client_storage: nym_gateway::node::PersistentStorage,
stats_storage: nym_gateway::node::PersistentStatsStorage,
sessions_stats: SharedSessionStats,
}
@@ -94,6 +95,11 @@ impl EntryGatewayData {
)
.await
.map_err(nym_gateway::GatewayError::from)?,
stats_storage: nym_gateway::node::PersistentStatsStorage::init(
&config.storage_paths.stats_storage,
)
.await
.map_err(nym_gateway::GatewayError::from)?,
sessions_stats: SharedSessionStats::new(),
})
}
@@ -114,6 +120,7 @@ pub struct ExitGatewayData {
auth_x25519: x25519::PublicKey,
client_storage: nym_gateway::node::PersistentStorage,
stats_storage: nym_gateway::node::PersistentStatsStorage,
}
impl ExitGatewayData {
@@ -262,6 +269,11 @@ impl ExitGatewayData {
.await
.map_err(nym_gateway::GatewayError::from)?;
let stats_storage =
nym_gateway::node::PersistentStatsStorage::init(&config.storage_paths.stats_storage)
.await
.map_err(nym_gateway::GatewayError::from)?;
Ok(ExitGatewayData {
nr_ed25519,
nr_x25519,
@@ -270,6 +282,7 @@ impl ExitGatewayData {
auth_ed25519,
auth_x25519,
client_storage,
stats_storage,
})
}
}
@@ -580,6 +593,7 @@ impl NymNode {
self.ed25519_identity_keys.clone(),
self.x25519_sphinx_keys.clone(),
self.entry_gateway.client_storage.clone(),
self.entry_gateway.stats_storage.clone(),
);
entry_gateway.disable_http_server();
entry_gateway.set_task_client(task_client);
@@ -610,6 +624,7 @@ impl NymNode {
self.ed25519_identity_keys.clone(),
self.x25519_sphinx_keys.clone(),
self.exit_gateway.client_storage.clone(),
self.exit_gateway.stats_storage.clone(),
);
exit_gateway.disable_http_server();
exit_gateway.set_task_client(task_client);
+1 -1
View File
@@ -18,6 +18,6 @@ nym-sphinx-anonymous-replies = { path = "../../../common/nymsphinx/anonymous-rep
nym-ffi-shared = { path = "../shared" }
lazy_static = "1.4.0"
# error handling
anyhow = "1.0.75"
anyhow = "1.0.90"
# base58 en/decoding
bs58 = "0.5.0"
+1 -1
View File
@@ -20,7 +20,7 @@ nym-ffi-shared = { path = "../shared" }
tokio = { version = "1", features = ["full"] }
lazy_static = "1.4.0"
# error handling
anyhow = "1.0.79"
anyhow = "1.0.90"
thiserror = "1.0.64"
[build-dependencies]
+1 -1
View File
@@ -14,7 +14,7 @@ nym-sphinx-anonymous-replies = { path = "../../../common/nymsphinx/anonymous-rep
# static var macro
lazy_static = "1.4.0"
# error handling
anyhow = "1.0.75"
anyhow = "1.0.90"
# base58 en/decoding
bs58 = "0.5.0"
# UDL test