From 0b51d98e2b289e37924a3a814223d09fde04bbd5 Mon Sep 17 00:00:00 2001 From: durch Date: Fri, 4 Jul 2025 00:25:41 +0200 Subject: [PATCH] feat(nym-node-status-api): add PostgreSQL database support via feature flags Implement dual database support for SQLite and PostgreSQL through Cargo feature flags. The implementation uses a query wrapper that automatically converts SQLite-style ? placeholders to PostgreSQL-style $1, $2, ... placeholders at runtime. Key changes: - Add query wrapper functions that handle placeholder conversion - Convert all sqlx::query\! macros to use wrapper functions - Handle type conversions between databases (i64 vs i32) - Add feature-gated implementations for database-specific SQL syntax - Update Makefile with clippy targets for both database features - Document database support in README --- CLAUDE.md | 686 ++++++++++++++++++ nym-node-status-api/README.md | 58 ++ ...906b42a7e4b280c8efb32db15d7c6a51d7a27.json | 32 - ...7dc549cf503409fd6dfedcdc02dbcd61d5454.json | 32 - ...5d1bcd636315ab23537cb5f6d2f82a2bcb7bf.json | 26 - ...2f49ba5db84d63e89440bb494e8327fe73686.json | 20 - ...c7ad6e68c81932d295ff18ed198c54706a57c.json | 86 --- ...3ac5ebe44a398b199ab0120207a5569f54d0b.json | 26 + ...236a2b964b981c8cbcc282117fe9bc38338dd.json | 80 ++ ...03a2752e28df775d1af8fd943a5fc8d7dc00d.json | 38 - ...69682c4ae96f774261f5c298264d3c12e5b67.json | 12 - ...64741aefc9f7a7d1630f6b863ebd8174b6684.json | 12 - ...36ced54e08785f77e9ef85f3157d004ec85dc.json | 86 --- ...68bd5438d6c8496ca9cbdcfb70bb5375b345e.json | 26 - ...bde9ec6159c9fa03b13652e1f620dcd92125e.json | 20 - ...29f8758ec922dd2e7ea064a1e537e580c9ee5.json | 12 - .../nym-node-status-api/Makefile | 13 + .../nym-node-status-api/src/db/models.rs | 17 +- .../src/db/queries/gateways.rs | 76 +- .../src/db/queries/gateways_stats.rs | 3 +- .../src/db/queries/misc.rs | 24 +- .../src/db/queries/mixnodes.rs | 35 +- .../src/db/queries/nym_nodes.rs | 122 ++-- .../src/db/queries/packet_stats.rs | 90 ++- .../src/db/queries/scraper.rs | 25 +- .../src/db/queries/summary.rs | 20 +- .../src/db/queries/testruns.rs | 114 ++- .../src/db/query_wrapper.rs | 56 +- .../src/http/api/testruns.rs | 4 +- .../src/mixnet_scraper/helpers.rs | 19 +- .../src/mixnet_scraper/mod.rs | 6 +- .../nym-node-status-api/src/monitor/mod.rs | 27 +- .../src/testruns/models.rs | 2 +- .../nym-node-status-api/src/testruns/queue.rs | 77 +- 34 files changed, 1239 insertions(+), 743 deletions(-) create mode 100644 CLAUDE.md delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-021c6c65d1ed806d8430bef7883906b42a7e4b280c8efb32db15d7c6a51d7a27.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-1327b5118f9144dddbcf8edb11f7dc549cf503409fd6dfedcdc02dbcd61d5454.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-227539374e7473f6f9642289c5b5d1bcd636315ab23537cb5f6d2f82a2bcb7bf.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-25300e435780101fa207c8e26ef2f49ba5db84d63e89440bb494e8327fe73686.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-6de15de62c0caa545910a17877a3ac5ebe44a398b199ab0120207a5569f54d0b.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-74b76cd0d94c1afc51c21c14c12236a2b964b981c8cbcc282117fe9bc38338dd.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-7600823da7ce80b8ffda933608603a2752e28df775d1af8fd943a5fc8d7dc00d.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c42917c9542c1d720d92035863064741aefc9f7a7d1630f6b863ebd8174b6684.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c7656b2b1b4328415772ce69d0568bd5438d6c8496ca9cbdcfb70bb5375b345e.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c910788edefe64bbb34379702bcbde9ec6159c9fa03b13652e1f620dcd92125e.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-f7e3fa31d68c028bf39cc95389f29f8758ec922dd2e7ea064a1e537e580c9ee5.json diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..573e3538ca --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,686 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Nym is a privacy platform that uses mixnet technology to protect against metadata surveillance. The platform consists of several key components: +- Mixnet nodes (mixnodes) for packet mixing +- Gateways (entry/exit points for the network) +- Clients for interacting with the network +- Network monitoring tools +- Validators for network consensus +- Various service providers and integrations + +## Build Commands + +### Rust Components + +```bash +# Default build (debug) +cargo build + +# Release build +cargo build --release + +# Build a specific package +cargo build -p + +# Build main components +make build + +# Build release versions of main binaries and contracts +make build-release + +# Build specific binaries +make build-nym-cli +cargo build -p nym-node --release +cargo build -p nym-api --release +``` + +### Testing + +```bash +# Run clippy, unit tests, and formatting +make test + +# Run all tests including slow tests +make test-all + +# Run clippy on all workspaces +make clippy + +# Run unit tests for a specific package +cargo test -p + +# Run only expensive/ignored tests +cargo test --workspace -- --ignored + +# Run API tests +dotenv -f envs/sandbox.env -- cargo test --test public-api-tests + +# Run tests with specific log level +RUST_LOG=debug cargo test -p + +# Run specific test scripts +./nym-node/tests/test_apis.sh +./scripts/wireguard-exit-policy/exit-policy-tests.sh +``` + +### Linting and Formatting + +```bash +# Run rustfmt on all code +make fmt + +# Check formatting without modifying +cargo fmt --all -- --check + +# Run clippy with all targets +cargo clippy --workspace --all-targets -- -D warnings + +# TypeScript linting +yarn lint +yarn lint:fix +yarn types:lint:fix + +# Check dependencies for security/licensing issues +cargo deny check +``` + +### WASM Components + +```bash +# Build all WASM components +make sdk-wasm-build + +# Build TypeScript SDK +yarn build:sdk +npx lerna run --scope @nymproject/sdk build --stream + +# Build and test WASM components +make sdk-wasm + +# Build specific WASM packages +cd wasm/client && make +cd wasm/mix-fetch && make +cd wasm/node-tester && make +``` + +### Contract Development + +```bash +# Build all contracts +make contracts + +# Build contracts in release mode +make build-release-contracts + +# Generate contract schemas +make contract-schema + +# Run wasm-opt on contracts +make wasm-opt-contracts + +# Check contracts with cosmwasm-check +make cosmwasm-check-contracts +``` + +### Running Components + +```bash +# Run nym-node as a mixnode +cargo run -p nym-node -- run --mode mixnode + +# Run nym-node as a gateway +cargo run -p nym-node -- run --mode gateway + +# Run the network monitor +cargo run -p nym-network-monitor + +# Run the API server +cargo run -p nym-api + +# Run with specific environment +dotenv -f envs/sandbox.env -- cargo run -p nym-api + +# Start a local network +./scripts/localnet_start.sh +``` + +## Architecture + +The Nym platform consists of various components organized as a monorepo: + +1. **Core Mixnet Infrastructure**: + - `nym-node`: Core binary supporting mixnode and gateway modes + - `common/nymsphinx`: Implementation of the Sphinx packet format + - `common/topology`: Network topology management + - `common/types`: Shared data types across components + +2. **Network Monitoring**: + - `nym-network-monitor`: Monitors the network's reliability and performance + - `nym-api`: API server for network stats and monitoring data + - Metrics tracking for nodes, routes, and overall network health + +3. **Client Implementations**: + - `clients/native`: Native Rust client implementation + - `clients/socks5`: SOCKS5 proxy client for standard applications + - `wasm`: WebAssembly client implementations (for browsers) + - `nym-connect`: Desktop and mobile clients + +4. **Blockchain & Smart Contracts**: + - `common/cosmwasm-smart-contracts`: Smart contract implementations + - `contracts`: CosmWasm contracts for the Nym network + - `common/ledger`: Blockchain integration + +5. **Utilities & Tools**: + - `tools`: Various CLI tools and utilities + - `sdk`: SDKs for different languages and platforms + - `documentation`: Documentation generation and management + +## Packet System + +Nym uses a modified Sphinx packet format for its mixnet: + +1. **Message Chunking**: + - Messages are divided into "sets" and "fragments" + - Each fragment fits in a single Sphinx packet + - The `common/nymsphinx/chunking` module handles message fragmentation + +2. **Routing**: + - Packets traverse through 3 layers of mixnodes + - Routing information is encrypted in layers (onion routing) + - The final gateway receives and processes the messages + +3. **Monitoring**: + - Monitoring system tracks packet delivery through the network + - Routes are analyzed for reliability statistics + - Node performance metrics are collected + +## Network Protocol + +Nym implements the Loopix mixnet design with several key privacy features: + +1. **Continuous-time Mixing**: + - Each mixnode delays messages independently with an exponential distribution + - This creates random reordering of packets, destroying timing correlations + - Offers better anonymity properties than batch mixing approaches + +2. **Cover Traffic**: + - Clients and nodes generate dummy "loop" packets that circulate through the network + - These packets are indistinguishable from real traffic + - Creates a baseline level of traffic that hides actual communication patterns + - Provides unobservability (hiding when and how much real traffic is being sent) + +3. **Stratified Network Architecture**: + - Traffic flows through Entry Gateway → 3 Mixnode Layers → Exit Gateway + - Path selection is independent per-message (unlike Tor) + - Each node connects only to adjacent layers + +4. **Anonymous Replies**: + - Single-Use Reply Blocks (SURBs) allow receiving messages without revealing identity + - Enables bidirectional communication while maintaining privacy + +## Network Monitoring Architecture + +The network monitoring system is a core component that measures mixnet reliability: + +1. The `nym-network-monitor` sends test packets through the network +2. These packets follow predefined routes through multiple mixnodes +3. Metrics are collected about: + - Successful and failed packet deliveries + - Node reliability (percentage of successful packet handling) + - Route reliability (which specific route combinations work best) +4. Results are stored in the database and used by `nym-api` to: + - Present node performance statistics + - Determine network rewards + - Provide route selection guidance to clients + +In the current branch, metrics collection is being enhanced with a fanout approach to submit to multiple API endpoints. + +## Development Environment + +### Required Dependencies + +- Rust toolchain (stable, 1.80+) +- Node.js (v20+) and yarn for TypeScript components +- SQLite for local database development +- PostgreSQL for API database (optional, for full API functionality) +- CosmWasm tools for contract development +- For building contracts: `wasm-opt` tool from `binaryen` +- Python 3.8+ for some scripts +- Docker (optional, for containerized development) +- protoc (Protocol Buffers compiler) for some components + +### Environment Configurations + +The `envs/` directory contains pre-configured environments: + +#### Available Environments + +- **`local.env`**: Local development environment + - Points to local services (localhost) + - Uses test mnemonics and keys + - Ideal for testing without external dependencies + +- **`sandbox.env`**: Sandbox test network + - Public test network with real nodes + - Test tokens available from faucet + - Contract addresses for sandbox deployment + - API: https://sandbox-nym-api1.nymtech.net + +- **`mainnet.env`**: Production mainnet + - Real network with real tokens + - Production contract addresses + - API: https://validator.nymtech.net + - Use with caution! + +- **`canary.env`**: Canary deployment + - Pre-release testing environment + - Tests new features before mainnet + +- **`mainnet-local-api.env`**: Hybrid environment + - Uses mainnet contracts but local API + - Useful for API development against mainnet data + +#### Key Environment Variables + +```bash +# Network configuration +NETWORK_NAME=sandbox # Network identifier +BECH32_PREFIX=n # Address prefix (n for sandbox, n for mainnet) +NYM_API=https://sandbox-nym-api1.nymtech.net/api +NYXD=https://rpc.sandbox.nymtech.net +NYM_API_NETWORK=sandbox + +# Contract addresses (network-specific) +MIXNET_CONTRACT_ADDRESS=n1xr3rq8yvd7qplsw5yx90ftsr2zdhg4e9z60h5duusgxpv72hud3sjkxkav +VESTING_CONTRACT_ADDRESS=n1unyuj8qnmygvzuex3dwmg9yzt9alhvyeat0uu0jedg2wj33efl5qackslz +# ... other contract addresses + +# Mnemonic for testing (NEVER use in production) +MNEMONIC="clutch captain shoe salt awake harvest setup primary inmate ugly among become" + +# API Keys and tokens +IPINFO_API_TOKEN=your_token_here +AUTHENTICATOR_PASSWORD=password_here + +# Logging +RUST_LOG=info # Options: error, warn, info, debug, trace +RUST_BACKTRACE=1 # Enable backtraces + +# Database +DATABASE_URL=postgresql://user:pass@localhost/nym_api +``` + +#### Using Environment Files + +```bash +# Load environment and run command +dotenv -f envs/sandbox.env -- cargo run -p nym-api + +# Export to shell +source envs/sandbox.env + +# Use with make targets +dotenv -f envs/sandbox.env -- make run-api-tests +``` + +## Initial Setup + +### First Time Setup + +1. **Install Prerequisites** + ```bash + # Install Rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + + # Install Node.js and yarn + # Via nvm (recommended): + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash + nvm install 20 + npm install -g yarn + + # Install build tools + # Ubuntu/Debian: + sudo apt-get install build-essential pkg-config libssl-dev protobuf-compiler libpq-dev + + # macOS: + brew install protobuf postgresql + + # Install wasm-opt for contract builds + npm install -g wasm-opt + + # Add wasm target for Rust + rustup target add wasm32-unknown-unknown + ``` + +2. **Clone and Setup Repository** + ```bash + git clone https://github.com/nymtech/nym.git + cd nym/nym + + # Install JavaScript dependencies + yarn install + + # Build the project + make build + ``` + +3. **Database Setup (Optional, for API development)** + ```bash + # Install PostgreSQL + # Create database + createdb nym_api + + # Run migrations (from nym-api directory) + cd nym-api + sqlx migrate run + ``` + +### Quick Start + +```bash +# Run a mixnode locally +dotenv -f envs/sandbox.env -- cargo run -p nym-node -- run --mode mixnode --id my-mixnode + +# Run a gateway locally +dotenv -f envs/sandbox.env -- cargo run -p nym-node -- run --mode gateway --id my-gateway + +# Run the API server +dotenv -f envs/sandbox.env -- cargo run -p nym-api + +# Run a client +cargo run -p nym-client -- init --id my-client +cargo run -p nym-client -- run --id my-client +``` + +## CI/CD Pipeline + +The project uses GitHub Actions for CI/CD with several key workflows: + +1. **Build and Test**: + - `ci-build.yml`: Main build workflow for Rust components + - Tests are run on multiple platforms (Linux, Windows, macOS) + - Includes formatting check (rustfmt) and linting (clippy) + +2. **Release Process**: + - Binary artifacts are published on release tags + - Multiple platform builds are created + +3. **Documentation**: + - Documentation is automatically built and deployed + +## Database Structure + +The system uses SQLite databases with tables like: +- `mixnode_status`: Status information about mixnodes +- `gateway_status`: Status information about gateways +- `routes`: Route performance information (success/failure of specific paths) +- `monitor_run`: Information about monitoring test runs + +## Development Workflows + +### Running a Node + +To run the mixnode or gateway: + +```bash +# Run nym-node as a mixnode with specified identity +cargo run -p nym-node -- run --mode mixnode --id my-mixnode + +# Run nym-node as a gateway +cargo run -p nym-node -- run --mode gateway --id my-gateway +``` + +### Configuration + +Nodes can be configured with files in various locations: +- Command-line arguments +- Environment variables +- `.env` files specified with `--config-env-file` + +### Monitoring + +To monitor the health of your node: +- View logs for real-time information +- Use the node's HTTP API for status information +- Check the explorer for public node statistics + +## Common Libraries + +- `common/types`: Shared data types across all components +- `common/crypto`: Cryptographic primitives and wrappers +- `common/client-core`: Core client functionality +- `common/gateway-client`: Client-gateway communication +- `common/task`: Task management and concurrency utilities +- `common/nymsphinx`: Sphinx packet implementation for mixnet +- `common/topology`: Network topology management +- `common/credentials`: Credential system for privacy-preserving authentication +- `common/bandwidth-controller`: Bandwidth management and accounting + +## Code Conventions + +- Error handling: Use anyhow/thiserror for structured error handling +- Logging: Use the tracing framework for logging and diagnostics +- State management: Generally use Tokio/futures for async code +- Configuration: Use the config crate and env vars with defaults +- Database: Use sqlx for type-safe database queries +- Follow clippy recommendations and rustfmt formatting +- Use semantic commit messages: feat, fix, docs, refactor, test, chore + +## When Making Changes + +- Run `make test` before submitting PRs +- Follow Rust naming conventions +- Use `clippy` to check for common issues +- Update SQLx query caches when modifying DB queries: `cargo sqlx prepare` +- Consider backward compatibility for protocol changes +- Use lefthook pre-commit hooks for TypeScript formatting +- Run `cargo deny check` to verify dependency compliance +- Test against both sandbox and local environments when possible +- Update relevant documentation and CHANGELOG.md + +## Development Tools + +### Useful Cargo Commands + +```bash +# Check for outdated dependencies +cargo outdated + +# Analyze binary size +cargo bloat --release -p nym-node + +# Generate dependency graph +cargo tree -p nym-api + +# Run with instrumentation +cargo run --features profiling -p nym-node + +# Check for security advisories +cargo audit +``` + +### Database Tools + +```bash +# SQLx CLI for migrations +cargo install sqlx-cli + +# Create new migration +cd nym-api && sqlx migrate add + +# Prepare query metadata for offline compilation +cargo sqlx prepare --workspace + +# View database schema +./nym-api/enter_db.sh +``` + +### Development Scripts + +- `scripts/build_topology.py`: Generate network topology files +- `scripts/node_api_check.py`: Verify node API endpoints +- `scripts/network_tunnel_manager.sh`: Manage network tunnels +- `scripts/localnet_start.sh`: Start a local test network +- Various deployment scripts in `deployment/` for different environments + +## Debugging + +- Enable more verbose logging with the RUST_LOG environment variable: + ``` + RUST_LOG=debug,nym_node=trace cargo run -p nym-node -- run --mode mixnode + ``` +- Use the HTTP API endpoints for status information +- Check monitoring data in the database for network performance metrics +- For complex issues, use tracing tools to follow packet flow +- Enable backtraces: `RUST_BACKTRACE=full` +- For WASM debugging: Use browser developer tools with source maps + +## Deployment and Advanced Configurations + +### Deployment Structure + +The `deployment/` directory contains Ansible playbooks and configurations for various deployment scenarios: + +- **`aws/`**: AWS-specific deployment configurations +- **`mixnode/`**: Mixnode deployment playbooks +- **`gateway/`**: Gateway deployment playbooks +- **`validator/`**: Validator node deployment +- **`sandbox-v2/`**: Complete sandbox environment setup +- **`big-dipper-2/`**: Block explorer deployment + +### Sandbox V2 Deployment + +The sandbox-v2 deployment (`deployment/sandbox-v2/`) provides a complete test environment: + +```bash +# Key playbooks: +- deploy.yaml # Main deployment orchestrator +- deploy-mixnodes.yaml # Deploy mixnodes +- deploy-gateways.yaml # Deploy gateways +- deploy-validators.yaml # Deploy validator nodes +- deploy-nym-api.yaml # Deploy API services +``` + +### Custom Environment Setup + +To create a custom environment: + +1. Copy an existing env file: `cp envs/sandbox.env envs/custom.env` +2. Modify the network endpoints and contract addresses +3. Update the `NETWORK_NAME` to your identifier +4. Set appropriate mnemonics and keys (use fresh ones for production!) + +### Contract Addresses + +Contract addresses are network-specific and defined in environment files: +- Mixnet contract: Manages mixnode/gateway registry +- Vesting contract: Handles token vesting schedules +- Coconut contracts: Privacy-preserving credentials +- Name service: Human-readable address mapping +- Ecash contract: Electronic cash functionality + +### Local Network Setup + +For a completely local network: +```bash +# Start local chain +./scripts/localnet_start.sh + +# Deploy contracts +cd contracts +make deploy-local + +# Start nodes with local config +dotenv -f envs/local.env -- cargo run -p nym-node -- run --mode mixnode +``` + +## Common Issues and Troubleshooting + +### Database Issues + +- When modifying database queries, you must update SQLx query caches: + ```bash + cargo sqlx prepare + ``` +- If you see SQLx errors about missing query files, this is likely the cause +- For "database is locked" errors with SQLite, ensure only one process accesses the DB +- For PostgreSQL connection issues, verify DATABASE_URL and that the server is running + +### API Connection Issues + +- Check the environment variables pointing to the APIs (NYM_API, NYXD) +- Verify network connectivity and API health endpoints +- For authentication issues, check node keys and credentials +- Common endpoints to verify: + - API health: `$NYM_API/health` + - Chain status: `$NYXD/status` + - Contract info: `$NYXD/cosmwasm/wasm/v1/contract/$CONTRACT_ADDRESS` + +### Build Problems + +- Clean dependencies with `cargo clean` for a fresh build +- Check for compatible Rust version (1.80+ recommended) +- For smart contract builds, ensure wasm-opt is installed: `npm install -g wasm-opt` +- For cross-compilation issues, check target-specific dependencies +- WASM build issues: Ensure wasm32-unknown-unknown target is installed: + ```bash + rustup target add wasm32-unknown-unknown + ``` +- For "cannot find -lpq" errors, install PostgreSQL development files: + ```bash + # Ubuntu/Debian + sudo apt-get install libpq-dev + # macOS + brew install postgresql + ``` + +### Environment Issues + +- Contract address mismatches: Ensure you're using the correct environment file +- "Account sequence mismatch": The account nonce is out of sync, wait and retry +- Token decimal issues: Sandbox uses different decimal places than mainnet +- API version mismatches: Ensure your local API version matches the network +- "Insufficient funds": Get test tokens from faucet (sandbox) or check balance +- Gateway/mixnode bonding issues: Verify minimum stake requirements + +## Working with Routes and Monitoring + +1. Route monitoring metrics are stored in a `routes` table with: + - Layer node IDs (layer1, layer2, layer3, gw) + - Success flag (boolean) + - Timestamp + +2. To analyze routes: + - Check `NetworkAccount` and `AccountingRoute` in `nym-network-monitor/src/accounting.rs` + - View monitoring logic in `common/nymsphinx/chunking/monitoring.rs` + - Observe how routes are submitted to the database in the `submit_accounting_routes_to_db` function + +## Performance Optimization + +### Profiling and Benchmarking + +```bash +# Run benchmarks +cargo bench -p nym-node + +# Profile with perf (Linux) +cargo build --release --features profiling +perf record --call-graph=dwarf ./target/release/nym-node run --mode mixnode +perf report + +# Generate flamegraph +cargo install flamegraph +cargo flamegraph --bin nym-node -- run --mode mixnode +``` + +### Common Performance Considerations + +- Use bounded channels for backpressure +- Batch database operations where possible +- Monitor memory usage with `RUST_LOG=nym_node::metrics=debug` +- Use connection pooling for database connections +- Consider using `jemalloc` for better memory allocation performance \ No newline at end of file diff --git a/nym-node-status-api/README.md b/nym-node-status-api/README.md index 03da6ca3cb..1ef6706c1d 100644 --- a/nym-node-status-api/README.md +++ b/nym-node-status-api/README.md @@ -4,3 +4,61 @@ The Node Status API serves information about individual `nym-nodes` in the Mixne We recommend that developers building applications such as explorers or analytics interfaces about the Mixnet run their own instance of the API, in order to promote a robust network of downstream services, and spread the load of API calls amongst as many endpoints as possible. You can find build and operation instructions in the [docs](https://nym.com/docs/apis/ns-api). + +## Database Support + +The Node Status API supports both SQLite and PostgreSQL databases through Cargo feature flags: + +- **SQLite** (default): Lightweight, file-based database suitable for development and small deployments +- **PostgreSQL**: Full-featured database recommended for production deployments + +### Building with Different Database Backends + +```bash +# Build with SQLite (default) +cargo build --features sqlite --no-default-features + +# Build with PostgreSQL +cargo build --features pg --no-default-features +``` + +### Running Tests + +```bash +# Test with SQLite +cargo test --features sqlite --no-default-features + +# Test with PostgreSQL +make test-db # This sets up a test PostgreSQL instance +``` + +### Development Commands + +The project includes a Makefile with helpful commands for both database backends: + +```bash +# Check code compilation +make check-sqlite # Check with SQLite +make check-pg # Check with PostgreSQL + +# Run clippy linter +make clippy-sqlite # Lint with SQLite +make clippy-pg # Lint with PostgreSQL +make clippy # Run both + +# PostgreSQL development +make dev-db # Start a PostgreSQL instance for development +make prepare-pg # Prepare SQLx offline cache for PostgreSQL +``` + +### Implementation Details + +The database abstraction is implemented using a query wrapper that automatically converts SQLite-style `?` placeholders to PostgreSQL-style `$1, $2, ...` placeholders at runtime. This allows writing queries once using SQLite syntax while maintaining compatibility with both databases. + +Key differences handled: +- Placeholder syntax (`?` vs `$1, $2, ...`) +- Type conversions (SQLite uses i64, PostgreSQL uses i32 for many fields) +- SQL dialect differences (e.g., `INSERT OR IGNORE` vs `ON CONFLICT DO NOTHING`) +- RETURNING clause behavior + +For more details on PostgreSQL setup, see [README_PG.md](nym-node-status-api/README_PG.md). diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-021c6c65d1ed806d8430bef7883906b42a7e4b280c8efb32db15d7c6a51d7a27.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-021c6c65d1ed806d8430bef7883906b42a7e4b280c8efb32db15d7c6a51d7a27.json deleted file mode 100644 index 8992ec849c..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-021c6c65d1ed806d8430bef7883906b42a7e4b280c8efb32db15d7c6a51d7a27.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT mix_id as node_id, host, http_api_port\n FROM mixnodes\n WHERE bonded = true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "node_id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "host", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "http_api_port", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "021c6c65d1ed806d8430bef7883906b42a7e4b280c8efb32db15d7c6a51d7a27" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-1327b5118f9144dddbcf8edb11f7dc549cf503409fd6dfedcdc02dbcd61d5454.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-1327b5118f9144dddbcf8edb11f7dc549cf503409fd6dfedcdc02dbcd61d5454.json deleted file mode 100644 index c0373a1f5c..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-1327b5118f9144dddbcf8edb11f7dc549cf503409fd6dfedcdc02dbcd61d5454.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n key as \"key!\",\n value_json as \"value_json!\",\n last_updated_utc as \"last_updated_utc!\"\n FROM summary", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "key!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "value_json!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "last_updated_utc!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true, - false - ] - }, - "hash": "1327b5118f9144dddbcf8edb11f7dc549cf503409fd6dfedcdc02dbcd61d5454" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-227539374e7473f6f9642289c5b5d1bcd636315ab23537cb5f6d2f82a2bcb7bf.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-227539374e7473f6f9642289c5b5d1bcd636315ab23537cb5f6d2f82a2bcb7bf.json deleted file mode 100644 index f459f7d5c7..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-227539374e7473f6f9642289c5b5d1bcd636315ab23537cb5f6d2f82a2bcb7bf.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n bond_info as \"bond_info: serde_json::Value\"\n FROM\n nym_nodes\n WHERE\n bond_info IS NOT NULL\n AND\n self_described IS NOT NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "node_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "bond_info: serde_json::Value", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true - ] - }, - "hash": "227539374e7473f6f9642289c5b5d1bcd636315ab23537cb5f6d2f82a2bcb7bf" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-25300e435780101fa207c8e26ef2f49ba5db84d63e89440bb494e8327fe73686.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-25300e435780101fa207c8e26ef2f49ba5db84d63e89440bb494e8327fe73686.json deleted file mode 100644 index 45ec922037..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-25300e435780101fa207c8e26ef2f49ba5db84d63e89440bb494e8327fe73686.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT gateway_identity_key\n FROM gateways\n WHERE bonded = true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "gateway_identity_key", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "25300e435780101fa207c8e26ef2f49ba5db84d63e89440bb494e8327fe73686" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json deleted file mode 100644 index e92d2f1edf..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\"\n FROM\n nym_nodes\n WHERE\n self_described IS NOT NULL\n AND\n bond_info IS NOT NULL\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "node_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "ed25519_identity_pubkey", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "total_stake", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "ip_addresses!: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "mix_port", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "x25519_sphinx_pubkey", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "node_role: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 7, - "name": "supported_roles: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "entry: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "performance", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "self_described: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "bond_info: serde_json::Value", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - true - ] - }, - "hash": "283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-6de15de62c0caa545910a17877a3ac5ebe44a398b199ab0120207a5569f54d0b.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-6de15de62c0caa545910a17877a3ac5ebe44a398b199ab0120207a5569f54d0b.json new file mode 100644 index 0000000000..b9690c68ee --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-6de15de62c0caa545910a17877a3ac5ebe44a398b199ab0120207a5569f54d0b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO testruns (gateway_id, status, ip_address, created_utc, log) VALUES ($1, $2, $3, $4, $5) RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "6de15de62c0caa545910a17877a3ac5ebe44a398b199ab0120207a5569f54d0b" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-74b76cd0d94c1afc51c21c14c12236a2b964b981c8cbcc282117fe9bc38338dd.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-74b76cd0d94c1afc51c21c14c12236a2b964b981c8cbcc282117fe9bc38338dd.json new file mode 100644 index 0000000000..7fa6221256 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-74b76cd0d94c1afc51c21c14c12236a2b964b981c8cbcc282117fe9bc38338dd.json @@ -0,0 +1,80 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n mn.mix_id as \"mix_id!\",\n mn.bonded as \"bonded: bool\",\n mn.is_dp_delegatee as \"is_dp_delegatee: bool\",\n mn.total_stake as \"total_stake!\",\n mn.full_details as \"full_details!\",\n mn.self_described as \"self_described\",\n mn.last_updated_utc as \"last_updated_utc!\",\n md.moniker as \"moniker!\",\n md.website as \"website!\",\n md.security_contact as \"security_contact!\",\n md.details as \"details!\"\n FROM mixnodes mn\n LEFT JOIN mixnode_description md ON mn.mix_id = md.mix_id\n ORDER BY mn.mix_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mix_id!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "bonded: bool", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "is_dp_delegatee: bool", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "total_stake!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "full_details!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "self_described", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "last_updated_utc!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "moniker!", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "website!", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "security_contact!", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "details!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + false, + true, + true, + true, + true + ] + }, + "hash": "74b76cd0d94c1afc51c21c14c12236a2b964b981c8cbcc282117fe9bc38338dd" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-7600823da7ce80b8ffda933608603a2752e28df775d1af8fd943a5fc8d7dc00d.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-7600823da7ce80b8ffda933608603a2752e28df775d1af8fd943a5fc8d7dc00d.json deleted file mode 100644 index bdea8047ad..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-7600823da7ce80b8ffda933608603a2752e28df775d1af8fd943a5fc8d7dc00d.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n id as \"id!\",\n date as \"date!\",\n timestamp_utc as \"timestamp_utc!\",\n value_json as \"value_json!\"\n FROM summary_history\n ORDER BY date DESC\n LIMIT 30", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "date!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "timestamp_utc!", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "value_json!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true - ] - }, - "hash": "7600823da7ce80b8ffda933608603a2752e28df775d1af8fd943a5fc8d7dc00d" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json deleted file mode 100644 index 141b3da977..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nym_nodes\n SET\n self_described = NULL,\n bond_info = NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "b68796d1d8d2384b30f1aace06269682c4ae96f774261f5c298264d3c12e5b67" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c42917c9542c1d720d92035863064741aefc9f7a7d1630f6b863ebd8174b6684.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c42917c9542c1d720d92035863064741aefc9f7a7d1630f6b863ebd8174b6684.json deleted file mode 100644 index 8fd572d965..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c42917c9542c1d720d92035863064741aefc9f7a7d1630f6b863ebd8174b6684.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n mixnodes\n SET\n bonded = false\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c42917c9542c1d720d92035863064741aefc9f7a7d1630f6b863ebd8174b6684" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json deleted file mode 100644 index d25be8f69a..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\"\n FROM\n nym_nodes\n ORDER BY\n node_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "node_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "ed25519_identity_pubkey", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "total_stake", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "ip_addresses!: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "mix_port", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "x25519_sphinx_pubkey", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "node_role: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 7, - "name": "supported_roles: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "entry: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 9, - "name": "performance", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "self_described: serde_json::Value", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "bond_info: serde_json::Value", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - true - ] - }, - "hash": "c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c7656b2b1b4328415772ce69d0568bd5438d6c8496ca9cbdcfb70bb5375b345e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c7656b2b1b4328415772ce69d0568bd5438d6c8496ca9cbdcfb70bb5375b345e.json deleted file mode 100644 index a3dec3dad6..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c7656b2b1b4328415772ce69d0568bd5438d6c8496ca9cbdcfb70bb5375b345e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n self_described as \"self_described: serde_json::Value\"\n FROM\n nym_nodes\n WHERE\n self_described IS NOT NULL\n ORDER BY\n node_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "node_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "self_described: serde_json::Value", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true - ] - }, - "hash": "c7656b2b1b4328415772ce69d0568bd5438d6c8496ca9cbdcfb70bb5375b345e" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c910788edefe64bbb34379702bcbde9ec6159c9fa03b13652e1f620dcd92125e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c910788edefe64bbb34379702bcbde9ec6159c9fa03b13652e1f620dcd92125e.json deleted file mode 100644 index 14fa1c9f12..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c910788edefe64bbb34379702bcbde9ec6159c9fa03b13652e1f620dcd92125e.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT mix_id\n FROM mixnodes\n WHERE bonded = true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mix_id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "c910788edefe64bbb34379702bcbde9ec6159c9fa03b13652e1f620dcd92125e" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-f7e3fa31d68c028bf39cc95389f29f8758ec922dd2e7ea064a1e537e580c9ee5.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-f7e3fa31d68c028bf39cc95389f29f8758ec922dd2e7ea064a1e537e580c9ee5.json deleted file mode 100644 index bfa4d81356..0000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-f7e3fa31d68c028bf39cc95389f29f8758ec922dd2e7ea064a1e537e580c9ee5.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE\n gateways\n SET\n bonded = false\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "f7e3fa31d68c028bf39cc95389f29f8758ec922dd2e7ea064a1e537e580c9ee5" -} diff --git a/nym-node-status-api/nym-node-status-api/Makefile b/nym-node-status-api/nym-node-status-api/Makefile index 1b2db88b23..5f2881c1b4 100644 --- a/nym-node-status-api/nym-node-status-api/Makefile +++ b/nym-node-status-api/nym-node-status-api/Makefile @@ -80,6 +80,19 @@ check-sqlite: ## Check code with SQLite feature @echo "Checking code with SQLite feature..." cargo check --features sqlite --no-default-features +.PHONY: clippy +clippy: clippy-pg clippy-sqlite + +.PHONY: clippy-pg +clippy-pg: ## Run clippy with PostgreSQL feature + @echo "Running clippy with PostgreSQL feature..." + cargo clippy --features pg --no-default-features -- -D warnings + +.PHONY: clippy-sqlite +clippy-sqlite: ## Run clippy with SQLite feature (default) + @echo "Running clippy with SQLite feature..." + cargo clippy --features sqlite --no-default-features -- -D warnings + # --- Cleanup Targets --- .PHONY: clean clean: ## Clean build artifacts and SQLx cache diff --git a/nym-node-status-api/nym-node-status-api/src/db/models.rs b/nym-node-status-api/nym-node-status-api/src/db/models.rs index 1d4c0f1e11..76d6b7d52a 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/models.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/models.rs @@ -38,7 +38,7 @@ pub(crate) struct GatewayInsertRecord { pub(crate) performance: u8, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromRow)] pub(crate) struct GatewayDto { pub(crate) gateway_identity_key: String, pub(crate) bonded: bool, @@ -121,7 +121,7 @@ pub(crate) struct MixnodeRecord { pub(crate) is_dp_delegatee: bool, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromRow)] pub(crate) struct MixnodeDto { pub(crate) mix_id: i64, pub(crate) bonded: bool, @@ -183,14 +183,14 @@ pub(crate) struct BondedStatusDto { } #[allow(unused)] -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, FromRow)] pub(crate) struct SummaryDto { pub(crate) key: String, pub(crate) value_json: String, pub(crate) last_updated_utc: i64, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, FromRow)] pub(crate) struct SummaryHistoryDto { #[allow(dead_code)] pub id: i64, @@ -287,7 +287,7 @@ pub(crate) mod gateway { } #[allow(dead_code)] // not dead code, this is SQL data model -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromRow)] pub struct TestRunDto { pub id: i64, pub gateway_id: i64, @@ -313,7 +313,7 @@ pub struct GatewayIdentityDto { } #[allow(dead_code)] // it's not dead code but clippy doesn't detect usage in sqlx macros -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromRow)] pub struct GatewayInfoDto { pub id: i64, pub gateway_identity_key: String, @@ -362,7 +362,7 @@ impl TryFrom for http::models::SessionStats { } } -#[derive(strum_macros::Display)] +#[derive(strum_macros::Display, Clone)] pub(crate) enum ScrapeNodeKind { LegacyMixnode { mix_id: i64 }, MixingNymNode { node_id: i64 }, @@ -379,6 +379,7 @@ impl ScrapeNodeKind { } } +#[derive(Clone)] pub(crate) struct ScraperNodeInfo { pub node_kind: ScrapeNodeKind, pub hosts: Vec, @@ -514,7 +515,7 @@ impl TryFrom for SkimmedNode { } } -#[derive(Debug, Serialize, Deserialize, sqlx::Decode)] +#[derive(Debug, Serialize, Deserialize, sqlx::Decode, FromRow)] pub struct NodeStats { pub packets_received: i64, pub packets_sent: i64, diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs index 63bd5da284..8699b03335 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs @@ -9,25 +9,26 @@ use crate::{ mixnet_scraper::helpers::NodeDescriptionResponse, }; use futures_util::TryStreamExt; +use sqlx::Row; use tracing::error; pub(crate) async fn select_gateway_identity( conn: &mut DbConnection, gateway_pk: i64, ) -> anyhow::Result { - let record = sqlx::query!( + let record = crate::db::query( r#"SELECT gateway_identity_key FROM gateways WHERE id = ?"#, - gateway_pk ) + .bind(gateway_pk) .fetch_one(conn.as_mut()) .await?; - Ok(record.gateway_identity_key) + Ok(record.try_get("gateway_identity_key")?) } pub(crate) async fn update_bonded_gateways( @@ -36,7 +37,7 @@ pub(crate) async fn update_bonded_gateways( ) -> anyhow::Result<()> { let mut tx = pool.begin().await?; - sqlx::query!( + crate::db::query( r#"UPDATE gateways SET @@ -47,7 +48,7 @@ pub(crate) async fn update_bonded_gateways( .await?; for record in gateways { - sqlx::query!( + crate::db::query( "INSERT INTO gateways (gateway_identity_key, bonded, self_described, explorer_pretty_bond, @@ -59,13 +60,13 @@ pub(crate) async fn update_bonded_gateways( explorer_pretty_bond=excluded.explorer_pretty_bond, last_updated_utc=excluded.last_updated_utc, performance = excluded.performance;", - record.identity_key, - record.bonded, - record.self_described, - record.explorer_pretty_bond, - record.last_updated_utc, - record.performance ) + .bind(record.identity_key) + .bind(record.bonded) + .bind(record.self_described) + .bind(record.explorer_pretty_bond) + .bind(record.last_updated_utc) + .bind(record.performance as i8) .execute(&mut *tx) .await?; } @@ -77,22 +78,21 @@ pub(crate) async fn update_bonded_gateways( pub(crate) async fn get_all_gateways(pool: &DbPool) -> anyhow::Result> { let mut conn = pool.acquire().await?; - let items = sqlx::query_as!( - GatewayDto, + let items = crate::db::query_as::( r#"SELECT - gw.gateway_identity_key as "gateway_identity_key!", - gw.bonded as "bonded: bool", - gw.performance as "performance!", - gw.self_described as "self_described?", - gw.explorer_pretty_bond as "explorer_pretty_bond?", - gw.last_probe_result as "last_probe_result?", - gw.last_probe_log as "last_probe_log?", - gw.last_testrun_utc as "last_testrun_utc?", - gw.last_updated_utc as "last_updated_utc!", - COALESCE(gd.moniker, "NA") as "moniker!", - COALESCE(gd.website, "NA") as "website!", - COALESCE(gd.security_contact, "NA") as "security_contact!", - COALESCE(gd.details, "NA") as "details!" + gw.gateway_identity_key, + gw.bonded, + gw.performance, + gw.self_described, + gw.explorer_pretty_bond, + gw.last_probe_result, + gw.last_probe_log, + gw.last_testrun_utc, + gw.last_updated_utc, + COALESCE(gd.moniker, 'NA') as moniker, + COALESCE(gd.website, 'NA') as website, + COALESCE(gd.security_contact, 'NA') as security_contact, + COALESCE(gd.details, 'NA') as details FROM gateways gw LEFT JOIN gateway_description gd ON gw.gateway_identity_key = gd.gateway_identity_key @@ -113,17 +113,17 @@ pub(crate) async fn get_all_gateways(pool: &DbPool) -> anyhow::Result anyhow::Result> { let mut conn = pool.acquire().await?; - let items = sqlx::query!( + let items = crate::db::query( r#" SELECT gateway_identity_key FROM gateways WHERE bonded = true - "# + "#, ) .fetch_all(&mut *conn) .await? .into_iter() - .map(|record| record.gateway_identity_key) + .map(|record| record.try_get::("gateway_identity_key").unwrap()) .collect::>(); Ok(items) @@ -131,11 +131,11 @@ pub(crate) async fn get_bonded_gateway_id_keys(pool: &DbPool) -> anyhow::Result< pub(crate) async fn insert_gateway_description( conn: &mut DbConnection, - identity_key: &str, - description: &NodeDescriptionResponse, + identity_key: String, + description: NodeDescriptionResponse, timestamp: i64, ) -> anyhow::Result<()> { - sqlx::query!( + crate::db::query( r#" INSERT INTO gateway_description ( gateway_identity_key, @@ -152,13 +152,13 @@ pub(crate) async fn insert_gateway_description( details = excluded.details, last_updated_utc = excluded.last_updated_utc "#, - identity_key, - description.moniker, - description.website, - description.security_contact, - description.details, - timestamp, ) + .bind(identity_key) + .bind(description.moniker) + .bind(description.website) + .bind(description.security_contact) + .bind(description.details) + .bind(timestamp) .execute(conn.as_mut()) .await .map(drop) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/gateways_stats.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/gateways_stats.rs index 41ddf8ee04..bf90bff06b 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/gateways_stats.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/gateways_stats.rs @@ -101,7 +101,8 @@ pub(crate) async fn get_sessions_stats(pool: &DbPool) -> anyhow::Result anyhow::Result<()> { let mut conn = pool.acquire().await?; - sqlx::query!("DELETE FROM gateway_session_stats WHERE day <= ?", cut_off) + crate::db::query("DELETE FROM gateway_session_stats WHERE day <= ?") + .bind(cut_off) .execute(&mut *conn) .await?; Ok(()) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/misc.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/misc.rs index fa56a9ef75..248b556c1d 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/misc.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/misc.rs @@ -6,8 +6,8 @@ use crate::db::{models::NetworkSummary, DbPool}; /// `daily_summary` pub(crate) async fn insert_summaries( pool: &DbPool, - summaries: &[(&str, usize)], - summary: &NetworkSummary, + summaries: Vec<(String, usize)>, + summary: NetworkSummary, last_updated: UtcDateTime, ) -> anyhow::Result<()> { insert_summary(pool, summaries, last_updated).await?; @@ -19,7 +19,7 @@ pub(crate) async fn insert_summaries( async fn insert_summary( pool: &DbPool, - summaries: &[(&str, usize)], + summaries: Vec<(String, usize)>, last_updated: UtcDateTime, ) -> anyhow::Result<()> { let timestamp = last_updated.unix_timestamp(); @@ -27,17 +27,17 @@ async fn insert_summary( for (kind, value) in summaries { let value = value.to_string(); - sqlx::query!( + crate::db::query( "INSERT INTO summary (key, value_json, last_updated_utc) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json=excluded.value_json, last_updated_utc=excluded.last_updated_utc;", - kind, - value, - timestamp ) + .bind(kind.clone()) + .bind(value) + .bind(timestamp) .execute(&mut *tx) .await .map_err(|err| { @@ -60,7 +60,7 @@ async fn insert_summary( /// This is not aggregate data, it's a set of latest data points async fn insert_summary_history( pool: &DbPool, - summary: &NetworkSummary, + summary: NetworkSummary, last_updated: UtcDateTime, ) -> anyhow::Result<()> { let mut conn = pool.acquire().await?; @@ -70,17 +70,17 @@ async fn insert_summary_history( let date = datetime_to_only_date_str(last_updated); - sqlx::query!( + crate::db::query( "INSERT INTO summary_history (date, timestamp_utc, value_json) VALUES (?, ?, ?) ON CONFLICT(date) DO UPDATE SET timestamp_utc=excluded.timestamp_utc, value_json=excluded.value_json;", - date, - timestamp, - value_json ) + .bind(date) + .bind(timestamp) + .bind(value_json) .execute(&mut *conn) .await?; diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs index 6bec0ba226..ce4f165c2f 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use futures_util::TryStreamExt; +use sqlx::Row; use tracing::error; use crate::{ @@ -19,7 +20,7 @@ pub(crate) async fn update_mixnodes( let mut tx = pool.begin().await?; // mark all as unbonded - sqlx::query!( + crate::db::query( r#"UPDATE mixnodes SET @@ -77,10 +78,10 @@ pub(crate) async fn get_all_mixnodes(pool: &DbPool) -> anyhow::Result anyhow::Result anyhow::Result> { let mut conn = pool.acquire().await?; - let items = sqlx::query!( + let items = crate::db::query( r#" SELECT mix_id FROM mixnodes WHERE bonded = true - "# + "#, ) .fetch_all(&mut *conn) .await? .into_iter() - .map(|record| record.mix_id) + .map(|record| record.try_get::("mix_id").unwrap()) .collect::>(); Ok(items) @@ -161,11 +162,11 @@ pub(crate) async fn get_bonded_mix_ids(pool: &DbPool) -> anyhow::Result anyhow::Result<()> { - sqlx::query!( + crate::db::query( r#" INSERT INTO mixnode_description ( mix_id, moniker, website, security_contact, details, last_updated_utc @@ -177,13 +178,13 @@ pub(crate) async fn insert_mixnode_description( details = excluded.details, last_updated_utc = excluded.last_updated_utc "#, - mix_id, - description.moniker, - description.website, - description.security_contact, - description.details, - timestamp, ) + .bind(mix_id) + .bind(description.moniker) + .bind(description.website) + .bind(description.security_contact) + .bind(description.details) + .bind(timestamp) .execute(conn.as_mut()) .await .map(drop) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs index ad1a6a9531..a62903da64 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs @@ -4,6 +4,7 @@ use nym_validator_client::{ client::{NodeId, NymNodeDetails}, models::NymNodeDescription, }; +use sqlx::Row; use std::collections::HashMap; use tracing::instrument; @@ -18,21 +19,20 @@ use crate::{ pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result> { let mut conn = pool.acquire().await?; - sqlx::query_as!( - NymNodeDto, + crate::db::query_as::( r#"SELECT node_id, ed25519_identity_pubkey, total_stake, - ip_addresses as "ip_addresses!: serde_json::Value", + ip_addresses, mix_port, x25519_sphinx_pubkey, - node_role as "node_role: serde_json::Value", - supported_roles as "supported_roles: serde_json::Value", - entry as "entry: serde_json::Value", + node_role, + supported_roles, + entry, performance, - self_described as "self_described: serde_json::Value", - bond_info as "bond_info: serde_json::Value" + self_described, + bond_info FROM nym_nodes ORDER BY @@ -55,21 +55,20 @@ pub(crate) async fn get_described_bonded_nym_nodes( ) -> anyhow::Result> { let mut conn = pool.acquire().await?; - sqlx::query_as!( - NymNodeDto, + crate::db::query_as::( r#"SELECT node_id, ed25519_identity_pubkey, total_stake, - ip_addresses as "ip_addresses!: serde_json::Value", + ip_addresses, mix_port, x25519_sphinx_pubkey, - node_role as "node_role: serde_json::Value", - supported_roles as "supported_roles: serde_json::Value", - entry as "entry: serde_json::Value", + node_role, + supported_roles, + entry, performance, - self_described as "self_described: serde_json::Value", - bond_info as "bond_info: serde_json::Value" + self_described, + bond_info FROM nym_nodes WHERE @@ -91,7 +90,7 @@ pub(crate) async fn update_nym_nodes( ) -> anyhow::Result { let mut tx = pool.begin().await?; - sqlx::query!( + crate::db::query( "UPDATE nym_nodes SET self_described = NULL, @@ -103,7 +102,7 @@ pub(crate) async fn update_nym_nodes( let inserted = node_records.len(); for record in node_records { // https://www.sqlite.org/lang_upsert.html - sqlx::query!( + crate::db::query( "INSERT INTO nym_nodes (node_id, ed25519_identity_pubkey, total_stake, @@ -128,20 +127,20 @@ pub(crate) async fn update_nym_nodes( performance=excluded.performance, last_updated_utc=excluded.last_updated_utc ;", - record.node_id, - record.ed25519_identity_pubkey, - record.total_stake, - record.ip_addresses, - record.mix_port, - record.x25519_sphinx_pubkey, - record.node_role, - record.supported_roles, - record.entry, - record.self_described, - record.bond_info, - record.performance, - record.last_updated_utc, ) + .bind(record.node_id) + .bind(record.ed25519_identity_pubkey) + .bind(record.total_stake) + .bind(record.ip_addresses) + .bind(record.mix_port as i32) + .bind(record.x25519_sphinx_pubkey) + .bind(record.node_role) + .bind(record.supported_roles) + .bind(record.entry) + .bind(record.self_described) + .bind(record.bond_info) + .bind(record.performance) + .bind(record.last_updated_utc) .execute(&mut *tx) .await .map_err(|e| anyhow::anyhow!("Failed to INSERT node_id={}: {}", record.node_id, e))?; @@ -157,10 +156,10 @@ pub(crate) async fn get_described_node_bond_info( ) -> anyhow::Result> { let mut conn = pool.acquire().await?; - sqlx::query!( + crate::db::query( r#"SELECT node_id, - bond_info as "bond_info: serde_json::Value" + bond_info FROM nym_nodes WHERE @@ -175,11 +174,11 @@ pub(crate) async fn get_described_node_bond_info( records .into_iter() .filter_map(|record| { - record - .bond_info - // only return details for nodes which have details stored - .and_then(|bond_info| serde_json::from_value::(bond_info).ok()) - .map(|res| (record.node_id as NodeId, res)) + let node_id: i64 = record.try_get("node_id").ok()?; + let bond_info: serde_json::Value = record.try_get("bond_info").ok()?; + serde_json::from_value::(bond_info) + .ok() + .map(|res| (node_id as NodeId, res)) }) .collect::>() }) @@ -191,10 +190,10 @@ pub(crate) async fn get_node_self_description( ) -> anyhow::Result> { let mut conn = pool.acquire().await?; - sqlx::query!( + crate::db::query( r#"SELECT node_id, - self_described as "self_described: serde_json::Value" + self_described FROM nym_nodes WHERE @@ -209,13 +208,11 @@ pub(crate) async fn get_node_self_description( records .into_iter() .filter_map(|record| { - record - .self_described - // only return details for nodes which have details stored - .and_then(|description| { - serde_json::from_value::(description).ok() - }) - .map(|res| (record.node_id as NodeId, res)) + let node_id: i64 = record.try_get("node_id").ok()?; + let self_described: serde_json::Value = record.try_get("self_described").ok()?; + serde_json::from_value::(self_described) + .ok() + .map(|res| (node_id as NodeId, res)) }) .collect::>() }) @@ -227,7 +224,7 @@ pub(crate) async fn get_bonded_node_description( ) -> anyhow::Result> { let mut conn = pool.acquire().await?; - sqlx::query!( + crate::db::query( r#"SELECT nd.node_id, moniker, @@ -248,14 +245,15 @@ pub(crate) async fn get_bonded_node_description( records .into_iter() .map(|elem| { - let node_id: NodeId = elem.node_id.try_into().unwrap_or_default(); + let node_id: i64 = elem.try_get("node_id").unwrap_or(0); + let node_id: NodeId = node_id.try_into().unwrap_or_default(); ( node_id, NodeDescription { - moniker: elem.moniker.unwrap_or_default(), - website: elem.website.unwrap_or_default(), - security_contact: elem.security_contact.unwrap_or_default(), - details: elem.details.unwrap_or_default(), + moniker: elem.try_get("moniker").unwrap_or_default(), + website: elem.try_get("website").unwrap_or_default(), + security_contact: elem.try_get("security_contact").unwrap_or_default(), + details: elem.try_get("details").unwrap_or_default(), }, ) }) @@ -266,11 +264,11 @@ pub(crate) async fn get_bonded_node_description( pub(crate) async fn insert_nym_node_description( conn: &mut DbConnection, - node_id: &i64, - description: &NodeDescriptionResponse, + node_id: i64, + description: NodeDescriptionResponse, timestamp: i64, ) -> anyhow::Result<()> { - sqlx::query!( + crate::db::query( r#" INSERT INTO nym_node_descriptions ( node_id, moniker, website, security_contact, details, last_updated_utc @@ -282,13 +280,13 @@ pub(crate) async fn insert_nym_node_description( details = excluded.details, last_updated_utc = excluded.last_updated_utc "#, - node_id, - description.moniker, - description.website, - description.security_contact, - description.details, - timestamp, ) + .bind(node_id) + .bind(description.moniker) + .bind(description.website) + .bind(description.security_contact) + .bind(description.details) + .bind(timestamp) .execute(conn.as_mut()) .await .map(drop) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/packet_stats.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/packet_stats.rs index cd189447ad..8f8980d428 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/packet_stats.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/packet_stats.rs @@ -6,7 +6,7 @@ use anyhow::Result; pub(crate) async fn insert_node_packet_stats( pool: &DbPool, - node_kind: &ScrapeNodeKind, + node_kind: ScrapeNodeKind, stats: &NodeStats, timestamp_utc: i64, ) -> Result<()> { @@ -14,35 +14,35 @@ pub(crate) async fn insert_node_packet_stats( match node_kind { ScrapeNodeKind::LegacyMixnode { mix_id } => { - sqlx::query!( + crate::db::query( r#" INSERT INTO mixnode_packet_stats_raw ( mix_id, timestamp_utc, packets_received, packets_sent, packets_dropped ) VALUES (?, ?, ?, ?, ?) "#, - mix_id, - timestamp_utc, - stats.packets_received, - stats.packets_sent, - stats.packets_dropped, ) + .bind(mix_id) + .bind(timestamp_utc) + .bind(stats.packets_received) + .bind(stats.packets_sent) + .bind(stats.packets_dropped) .execute(&mut *conn) .await?; } ScrapeNodeKind::MixingNymNode { node_id } | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => { - sqlx::query!( + crate::db::query( r#" INSERT INTO nym_nodes_packet_stats_raw ( node_id, timestamp_utc, packets_received, packets_sent, packets_dropped ) VALUES (?, ?, ?, ?, ?) "#, - node_id, - timestamp_utc, - stats.packets_received, - stats.packets_sent, - stats.packets_dropped, ) + .bind(node_id) + .bind(timestamp_utc) + .bind(stats.packets_received) + .bind(stats.packets_sent) + .bind(stats.packets_dropped) .execute(&mut *conn) .await?; } @@ -53,7 +53,7 @@ pub(crate) async fn insert_node_packet_stats( pub(crate) async fn get_raw_node_stats( pool: &DbPool, - node: &ScraperNodeInfo, + node: ScraperNodeInfo, ) -> Result> { let mut conn = pool.acquire().await?; @@ -61,39 +61,37 @@ pub(crate) async fn get_raw_node_stats( // if no packets are found, it's fine to assume 0 because that's also // SQL default value if none provided ScrapeNodeKind::LegacyMixnode { mix_id } => { - sqlx::query_as!( - NodeStats, + crate::db::query_as::( r#" SELECT - COALESCE(packets_received, 0) as "packets_received!: _", - COALESCE(packets_sent, 0) as "packets_sent!: _", - COALESCE(packets_dropped, 0) as "packets_dropped!: _" + COALESCE(packets_received, 0) as packets_received, + COALESCE(packets_sent, 0) as packets_sent, + COALESCE(packets_dropped, 0) as packets_dropped FROM mixnode_packet_stats_raw WHERE mix_id = ? ORDER BY timestamp_utc DESC LIMIT 1 OFFSET 1 "#, - mix_id ) + .bind(mix_id) .fetch_optional(&mut *conn) .await? } ScrapeNodeKind::MixingNymNode { node_id } | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => { - sqlx::query_as!( - NodeStats, + crate::db::query_as::( r#" SELECT - COALESCE(packets_received, 0) as "packets_received!: _", - COALESCE(packets_sent, 0) as "packets_sent!: _", - COALESCE(packets_dropped, 0) as "packets_dropped!: _" + COALESCE(packets_received, 0) as packets_received, + COALESCE(packets_sent, 0) as packets_sent, + COALESCE(packets_dropped, 0) as packets_dropped FROM nym_nodes_packet_stats_raw WHERE node_id = ? ORDER BY timestamp_utc DESC LIMIT 1 OFFSET 1 "#, - node_id ) + .bind(node_id) .fetch_optional(&mut *conn) .await? } @@ -104,27 +102,27 @@ pub(crate) async fn get_raw_node_stats( pub(crate) async fn insert_daily_node_stats( pool: &DbPool, - node: &ScraperNodeInfo, - date_utc: &str, + node: ScraperNodeInfo, + date_utc: String, packets: NodeStats, ) -> Result<()> { let mut conn = pool.acquire().await?; match node.node_kind { ScrapeNodeKind::LegacyMixnode { mix_id } => { - let total_stake = sqlx::query_scalar!( + let total_stake = crate::db::query_scalar::( r#" SELECT total_stake FROM mixnodes WHERE mix_id = ? "#, - mix_id ) + .bind(mix_id) .fetch_one(&mut *conn) .await?; - sqlx::query!( + crate::db::query( r#" INSERT INTO mixnode_daily_stats ( mix_id, date_utc, @@ -137,31 +135,31 @@ pub(crate) async fn insert_daily_node_stats( packets_sent = mixnode_daily_stats.packets_sent + excluded.packets_sent, packets_dropped = mixnode_daily_stats.packets_dropped + excluded.packets_dropped "#, - mix_id, - date_utc, - total_stake, - packets.packets_received, - packets.packets_sent, - packets.packets_dropped, ) + .bind(mix_id) + .bind(date_utc) + .bind(total_stake) + .bind(packets.packets_received) + .bind(packets.packets_sent) + .bind(packets.packets_dropped) .execute(&mut *conn) .await?; } ScrapeNodeKind::MixingNymNode { node_id } | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => { - let total_stake = sqlx::query_scalar!( + let total_stake = crate::db::query_scalar::( r#" SELECT total_stake FROM nym_nodes WHERE node_id = ? "#, - node_id ) + .bind(node_id) .fetch_one(&mut *conn) .await?; - sqlx::query!( + crate::db::query( r#" INSERT INTO nym_node_daily_mixing_stats ( node_id, date_utc, @@ -174,13 +172,13 @@ pub(crate) async fn insert_daily_node_stats( packets_sent = nym_node_daily_mixing_stats.packets_sent + excluded.packets_sent, packets_dropped = nym_node_daily_mixing_stats.packets_dropped + excluded.packets_dropped "#, - node_id, - date_utc, - total_stake, - packets.packets_received, - packets.packets_sent, - packets.packets_dropped, ) + .bind(node_id) + .bind(date_utc) + .bind(total_stake) + .bind(packets.packets_received) + .bind(packets.packets_sent) + .bind(packets.packets_dropped) .execute(&mut *conn) .await?; } diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs index 02113affc4..a0ec714a7b 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs @@ -12,6 +12,7 @@ use crate::{ }; use anyhow::Result; use nym_validator_client::nym_api::SkimmedNode; +use sqlx::Row; pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result> { let mut nodes_to_scrape = Vec::new(); @@ -68,12 +69,12 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result Result Result Result<()> { let timestamp = now_utc().unix_timestamp(); let mut conn = pool.acquire().await?; @@ -138,7 +141,7 @@ pub(crate) async fn insert_scraped_node_description( node_id, identity_key, } => { - insert_nym_node_description(&mut conn, node_id, description, timestamp).await?; + insert_nym_node_description(&mut conn, node_id, description.clone(), timestamp).await?; // for historic reasons (/gateways API), store this info into gateways table as well insert_gateway_description(&mut conn, identity_key, description, timestamp).await?; } diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/summary.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/summary.rs index 79a32ee5bb..5c5ea976bc 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/summary.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/summary.rs @@ -23,13 +23,12 @@ use crate::{ pub(crate) async fn get_summary_history(pool: &DbPool) -> anyhow::Result> { let mut conn = pool.acquire().await?; - let items = sqlx::query_as!( - SummaryHistoryDto, + let items = crate::db::query_as::( r#"SELECT - id as "id!", - date as "date!", - timestamp_utc as "timestamp_utc!", - value_json as "value_json!" + id, + date, + timestamp_utc, + value_json FROM summary_history ORDER BY date DESC LIMIT 30"#, @@ -51,12 +50,11 @@ pub(crate) async fn get_summary_history(pool: &DbPool) -> anyhow::Result anyhow::Result> { let mut conn = pool.acquire().await?; - Ok(sqlx::query_as!( - SummaryDto, + Ok(crate::db::query_as::( r#"SELECT - key as "key!", - value_json as "value_json!", - last_updated_utc as "last_updated_utc!" + key, + value_json, + last_updated_utc FROM summary"# ) .fetch(&mut *conn) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/testruns.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/testruns.rs index dfbe0d8122..18e03351b2 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/testruns.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/testruns.rs @@ -3,6 +3,7 @@ use crate::db::DbConnection; use crate::db::DbPool; use crate::http::models::TestrunAssignment; use crate::utils::now_utc; +use sqlx::Row; use time::Duration; pub(crate) async fn count_testruns_in_progress(conn: &mut DbConnection) -> anyhow::Result { @@ -24,15 +25,14 @@ pub(crate) async fn get_in_progress_testrun_by_id( conn: &mut DbConnection, testrun_id: i64, ) -> anyhow::Result { - sqlx::query_as!( - TestRunDto, + crate::db::query_as::( r#"SELECT - id as "id!", - gateway_id as "gateway_id!", - status as "status!", - created_utc as "created_utc!", - ip_address as "ip_address!", - log as "log!", + id, + gateway_id, + status, + created_utc, + ip_address, + log, last_assigned_utc FROM testruns WHERE @@ -41,9 +41,9 @@ pub(crate) async fn get_in_progress_testrun_by_id( status = ? ORDER BY created_utc LIMIT 1"#, - testrun_id, - TestRunStatus::InProgress as i64, ) + .bind(testrun_id) + .bind(TestRunStatus::InProgress as i64) .fetch_one(conn.as_mut()) .await .map_err(|e| anyhow::anyhow!("Couldn't retrieve testrun {testrun_id}: {e}")) @@ -57,7 +57,7 @@ pub(crate) async fn update_testruns_assigned_before( let previous_run = now_utc() - max_age; let cutoff_timestamp = previous_run.unix_timestamp(); - let res = sqlx::query!( + let res = crate::db::query( r#"UPDATE testruns SET @@ -67,10 +67,10 @@ pub(crate) async fn update_testruns_assigned_before( AND last_assigned_utc < ? "#, - TestRunStatus::Queued as i64, - TestRunStatus::InProgress as i64, - cutoff_timestamp ) + .bind(TestRunStatus::Queued as i64) + .bind(TestRunStatus::InProgress as i64) + .bind(cutoff_timestamp) .execute(conn.as_mut()) .await?; @@ -91,7 +91,7 @@ pub(crate) async fn assign_oldest_testrun( ) -> anyhow::Result> { let now = now_utc().unix_timestamp(); // find & mark as "In progress" in the same transaction to avoid race conditions - let returning = sqlx::query!( + let returning = crate::db::query( r#"UPDATE testruns SET status = ?, @@ -105,18 +105,18 @@ pub(crate) async fn assign_oldest_testrun( LIMIT 1 ) RETURNING - id as "id!", + id, gateway_id "#, - TestRunStatus::InProgress as i64, - now, - TestRunStatus::Queued as i64, ) + .bind(TestRunStatus::InProgress as i64) + .bind(now) + .bind(TestRunStatus::Queued as i64) .fetch_optional(conn.as_mut()) .await?; if let Some(testrun) = returning { - let gw_identity = sqlx::query!( + let gw_identity = crate::db::query( r#" SELECT id, @@ -124,14 +124,14 @@ pub(crate) async fn assign_oldest_testrun( FROM gateways WHERE id = ? LIMIT 1"#, - testrun.gateway_id ) + .bind(testrun.try_get::("gateway_id")?) .fetch_one(conn.as_mut()) .await?; Ok(Some(TestrunAssignment { - testrun_id: testrun.id, - gateway_identity_key: gw_identity.gateway_identity_key, + testrun_id: testrun.try_get("id")?, + gateway_identity_key: gw_identity.try_get("gateway_identity_key")?, assigned_at_utc: now, })) } else { @@ -144,14 +144,12 @@ pub(crate) async fn update_testrun_status( testrun_id: i64, status: TestRunStatus, ) -> anyhow::Result<()> { - let status = status as u32; - sqlx::query!( - "UPDATE testruns SET status = ? WHERE id = ?", - status, - testrun_id - ) - .execute(conn.as_mut()) - .await?; + let status = status as i32; + crate::db::query("UPDATE testruns SET status = ? WHERE id = ?") + .bind(status) + .bind(testrun_id) + .execute(conn.as_mut()) + .await?; Ok(()) } @@ -159,33 +157,29 @@ pub(crate) async fn update_testrun_status( pub(crate) async fn update_gateway_last_probe_log( conn: &mut DbConnection, gateway_pk: i64, - log: &str, + log: String, ) -> anyhow::Result<()> { - sqlx::query!( - "UPDATE gateways SET last_probe_log = ? WHERE id = ?", - log, - gateway_pk - ) - .execute(conn.as_mut()) - .await - .map(drop) - .map_err(From::from) + crate::db::query("UPDATE gateways SET last_probe_log = ? WHERE id = ?") + .bind(log) + .bind(gateway_pk) + .execute(conn.as_mut()) + .await + .map(drop) + .map_err(From::from) } pub(crate) async fn update_gateway_last_probe_result( conn: &mut DbConnection, gateway_pk: i64, - result: &str, + result: String, ) -> anyhow::Result<()> { - sqlx::query!( - "UPDATE gateways SET last_probe_result = ? WHERE id = ?", - result, - gateway_pk - ) - .execute(conn.as_mut()) - .await - .map(drop) - .map_err(From::from) + crate::db::query("UPDATE gateways SET last_probe_result = ? WHERE id = ?") + .bind(result) + .bind(gateway_pk) + .execute(conn.as_mut()) + .await + .map(drop) + .map_err(From::from) } pub(crate) async fn update_gateway_score( @@ -193,14 +187,12 @@ pub(crate) async fn update_gateway_score( gateway_pk: i64, ) -> anyhow::Result<()> { let now = now_utc().unix_timestamp(); - sqlx::query!( - "UPDATE gateways SET last_testrun_utc = ?, last_updated_utc = ? WHERE id = ?", - now, - now, - gateway_pk - ) - .execute(conn.as_mut()) - .await - .map(drop) - .map_err(From::from) + crate::db::query("UPDATE gateways SET last_testrun_utc = ?, last_updated_utc = ? WHERE id = ?") + .bind(now) + .bind(now) + .bind(gateway_pk) + .execute(conn.as_mut()) + .await + .map(drop) + .map_err(From::from) } diff --git a/nym-node-status-api/nym-node-status-api/src/db/query_wrapper.rs b/nym-node-status-api/nym-node-status-api/src/db/query_wrapper.rs index 1fac3874b1..730c215301 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/query_wrapper.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/query_wrapper.rs @@ -5,17 +5,17 @@ use sqlx::Database; fn convert_placeholders(query: &str) -> String { let mut result = String::with_capacity(query.len() + 10); let mut placeholder_count = 0; - let mut chars = query.chars().peekable(); + let chars = query.chars().peekable(); let mut in_string = false; let mut escape_next = false; - - while let Some(ch) = chars.next() { + + for ch in chars { if escape_next { result.push(ch); escape_next = false; continue; } - + match ch { '\\' => { result.push(ch); @@ -27,32 +27,38 @@ fn convert_placeholders(query: &str) -> String { } '?' if !in_string => { placeholder_count += 1; - result.push_str(&format!("${}", placeholder_count)); + result.push_str(&format!("${placeholder_count}")); } _ => { result.push(ch); } } } - + result } /// Creates a query that automatically handles placeholder conversion #[cfg(feature = "sqlite")] -pub fn query(sql: &str) -> sqlx::query::Query<'_, sqlx::Sqlite, ::Arguments<'_>> { +pub fn query( + sql: &str, +) -> sqlx::query::Query<'_, sqlx::Sqlite, ::Arguments<'_>> { sqlx::query(sql) } #[cfg(feature = "pg")] -pub fn query(sql: &str) -> sqlx::query::Query<'static, sqlx::Postgres, ::Arguments<'static>> { +pub fn query( + sql: &str, +) -> sqlx::query::Query<'static, sqlx::Postgres, ::Arguments<'static>> { let converted = convert_placeholders(sql); sqlx::query(Box::leak(converted.into_boxed_str())) } /// Creates a query_as that automatically handles placeholder conversion #[cfg(feature = "sqlite")] -pub fn query_as(sql: &str) -> sqlx::query::QueryAs<'_, sqlx::Sqlite, O, ::Arguments<'_>> +pub fn query_as( + sql: &str, +) -> sqlx::query::QueryAs<'_, sqlx::Sqlite, O, ::Arguments<'_>> where O: for<'r> sqlx::FromRow<'r, ::Row>, { @@ -60,7 +66,14 @@ where } #[cfg(feature = "pg")] -pub fn query_as(sql: &str) -> sqlx::query::QueryAs<'static, sqlx::Postgres, O, ::Arguments<'static>> +pub fn query_as( + sql: &str, +) -> sqlx::query::QueryAs< + 'static, + sqlx::Postgres, + O, + ::Arguments<'static>, +> where O: for<'r> sqlx::FromRow<'r, ::Row>, { @@ -70,7 +83,9 @@ where /// Creates a query_scalar that automatically handles placeholder conversion #[cfg(feature = "sqlite")] -pub fn query_scalar(sql: &str) -> sqlx::query::QueryScalar<'_, sqlx::Sqlite, O, ::Arguments<'_>> +pub fn query_scalar( + sql: &str, +) -> sqlx::query::QueryScalar<'_, sqlx::Sqlite, O, ::Arguments<'_>> where (O,): for<'r> sqlx::FromRow<'r, ::Row>, { @@ -78,7 +93,14 @@ where } #[cfg(feature = "pg")] -pub fn query_scalar(sql: &str) -> sqlx::query::QueryScalar<'static, sqlx::Postgres, O, ::Arguments<'static>> +pub fn query_scalar( + sql: &str, +) -> sqlx::query::QueryScalar< + 'static, + sqlx::Postgres, + O, + ::Arguments<'static>, +> where (O,): for<'r> sqlx::FromRow<'r, ::Row>, { @@ -97,26 +119,26 @@ mod tests { convert_placeholders("SELECT * FROM table WHERE id = ?"), "SELECT * FROM table WHERE id = $1" ); - + assert_eq!( convert_placeholders("INSERT INTO table (a, b, c) VALUES (?, ?, ?)"), "INSERT INTO table (a, b, c) VALUES ($1, $2, $3)" ); - + assert_eq!( convert_placeholders("SELECT * FROM table WHERE name = 'test?' AND id = ?"), "SELECT * FROM table WHERE name = 'test?' AND id = $1" ); - + assert_eq!( convert_placeholders("UPDATE table SET a = ?, b = ? WHERE id = ?"), "UPDATE table SET a = $1, b = $2 WHERE id = $3" ); - + // Test with 10 placeholders (like in update_mixnodes) assert_eq!( convert_placeholders("INSERT INTO t VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"), "INSERT INTO t VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" ); } -} \ No newline at end of file +} diff --git a/nym-node-status-api/nym-node-status-api/src/http/api/testruns.rs b/nym-node-status-api/nym-node-status-api/src/http/api/testruns.rs index 5093302e62..3f6e6fd150 100644 --- a/nym-node-status-api/nym-node-status-api/src/http/api/testruns.rs +++ b/nym-node-status-api/nym-node-status-api/src/http/api/testruns.rs @@ -138,7 +138,7 @@ async fn submit_testrun( queries::testruns::update_gateway_last_probe_log( &mut conn, assigned_testrun.gateway_id, - &submitted_result.payload.probe_result, + submitted_result.payload.probe_result.clone(), ) .await .map_err(HttpError::internal_with_logging)?; @@ -146,7 +146,7 @@ async fn submit_testrun( queries::testruns::update_gateway_last_probe_result( &mut conn, assigned_testrun.gateway_id, - &result, + result, ) .await .map_err(HttpError::internal_with_logging)?; diff --git a/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/helpers.rs b/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/helpers.rs index d62d911b9a..7fbde6aca4 100644 --- a/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/helpers.rs +++ b/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/helpers.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::time::Duration; use time::UtcDateTime; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct NodeDescriptionResponse { pub moniker: Option, pub website: Option, @@ -116,7 +116,7 @@ pub fn sanitize_description( } } -pub async fn scrape_and_store_description(pool: &DbPool, node: &ScraperNodeInfo) -> Result<()> { +pub async fn scrape_and_store_description(pool: &DbPool, node: ScraperNodeInfo) -> Result<()> { let client = build_client()?; let urls = node.contact_addresses(); @@ -151,15 +151,12 @@ pub async fn scrape_and_store_description(pool: &DbPool, node: &ScraperNodeInfo) })?; let sanitized_description = sanitize_description(description, *node.node_id()); - insert_scraped_node_description(pool, &node.node_kind, &sanitized_description).await?; + insert_scraped_node_description(pool, node.node_kind.clone(), sanitized_description).await?; Ok(()) } -pub async fn scrape_and_store_packet_stats( - pool: &DbPool, - node: &ScraperNodeInfo, -) -> Result<()> { +pub async fn scrape_and_store_packet_stats(pool: &DbPool, node: ScraperNodeInfo) -> Result<()> { let client = build_client()?; let urls = node.contact_addresses(); @@ -189,7 +186,7 @@ pub async fn scrape_and_store_packet_stats( let timestamp = now_utc(); let timestamp_utc = timestamp.unix_timestamp(); - insert_node_packet_stats(pool, &node.node_kind, &stats, timestamp_utc).await?; + insert_node_packet_stats(pool, node.node_kind.clone(), &stats, timestamp_utc).await?; // Update daily stats update_daily_stats(pool, node, timestamp, &stats).await?; @@ -199,7 +196,7 @@ pub async fn scrape_and_store_packet_stats( pub async fn update_daily_stats( pool: &DbPool, - node: &ScraperNodeInfo, + node: ScraperNodeInfo, timestamp: UtcDateTime, current_stats: &NodeStats, ) -> Result<()> { @@ -211,7 +208,7 @@ pub async fn update_daily_stats( ); // Get previous stats - let previous_stats = get_raw_node_stats(pool, node).await?; + let previous_stats = get_raw_node_stats(pool, node.clone()).await?; let (diff_received, diff_sent, diff_dropped) = if let Some(prev) = previous_stats { ( @@ -226,7 +223,7 @@ pub async fn update_daily_stats( insert_daily_node_stats( pool, node, - &date_utc, + date_utc, NodeStats { packets_received: diff_received, packets_sent: diff_sent, diff --git a/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/mod.rs b/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/mod.rs index af65cf2895..5863a41523 100644 --- a/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/mixnet_scraper/mod.rs @@ -2,9 +2,9 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; pub mod helpers; +use crate::db::DbPool; use anyhow::Result; use helpers::{scrape_and_store_description, scrape_and_store_packet_stats}; -use crate::db::DbPool; use tracing::{debug, error, instrument, warn}; use crate::db::models::ScraperNodeInfo; @@ -127,7 +127,7 @@ impl Scraper { let pool = pool.clone(); tokio::spawn(async move { - match scrape_and_store_description(&pool, &node).await { + match scrape_and_store_description(&pool, node.clone()).await { Ok(_) => debug!( "📝 ✅ Description task #{} for node {} complete", task_id, @@ -172,7 +172,7 @@ impl Scraper { let pool = pool.clone(); tokio::spawn(async move { - match scrape_and_store_packet_stats(&pool, &node).await { + match scrape_and_store_packet_stats(&pool, node.clone()).await { Ok(_) => debug!( "📊 ✅ Packet stats task #{} for node {} complete", task_id, diff --git a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs index 844d7afd41..2bd1745a68 100644 --- a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs @@ -252,17 +252,23 @@ impl Monitor { // let nodes_summary = vec![ - (NYMNODE_COUNT, nym_node_count), - (ASSIGNED_MIXING_COUNT, assigned_mixing_count), - (MIXNODES_LEGACY_COUNT, count_legacy_mixnodes), - (NYMNODES_DESCRIBED_COUNT, described_nodes.len()), - (GATEWAYS_BONDED_COUNT, count_bonded_gateways), - (ASSIGNED_ENTRY_COUNT, assigned_entry_count), - (ASSIGNED_EXIT_COUNT, assigned_exit_count), + (NYMNODE_COUNT.to_string(), nym_node_count), + (ASSIGNED_MIXING_COUNT.to_string(), assigned_mixing_count), + (MIXNODES_LEGACY_COUNT.to_string(), count_legacy_mixnodes), + (NYMNODES_DESCRIBED_COUNT.to_string(), described_nodes.len()), + (GATEWAYS_BONDED_COUNT.to_string(), count_bonded_gateways), + (ASSIGNED_ENTRY_COUNT.to_string(), assigned_entry_count), + (ASSIGNED_EXIT_COUNT.to_string(), assigned_exit_count), // TODO dz doesn't make sense, could make sense with historical Nym // Nodes if we really need this data - (MIXNODES_HISTORICAL_COUNT, all_historical_mixnodes), - (GATEWAYS_HISTORICAL_COUNT, all_historical_gateways), + ( + MIXNODES_HISTORICAL_COUNT.to_string(), + all_historical_mixnodes, + ), + ( + GATEWAYS_HISTORICAL_COUNT.to_string(), + all_historical_gateways, + ), ]; let last_updated = now_utc(); @@ -295,7 +301,8 @@ impl Monitor { }, }; - queries::insert_summaries(&pool, &nodes_summary, &network_summary, last_updated).await?; + queries::insert_summaries(&pool, nodes_summary.clone(), network_summary, last_updated) + .await?; let mut log_lines: Vec = vec![]; for (key, value) in nodes_summary.iter() { diff --git a/nym-node-status-api/nym-node-status-api/src/testruns/models.rs b/nym-node-status-api/nym-node-status-api/src/testruns/models.rs index fe4b33384c..9d61921dc4 100644 --- a/nym-node-status-api/nym-node-status-api/src/testruns/models.rs +++ b/nym-node-status-api/nym-node-status-api/src/testruns/models.rs @@ -9,7 +9,7 @@ pub struct GatewayIdentityDto { #[derive(Debug, Clone, Deserialize, Serialize, utoipa::ToSchema)] pub struct TestRun { - pub id: u32, + pub id: i32, pub identity_key: String, pub status: String, pub log: String, diff --git a/nym-node-status-api/nym-node-status-api/src/testruns/queue.rs b/nym-node-status-api/nym-node-status-api/src/testruns/queue.rs index 5d5b8f12bf..25b130e45d 100644 --- a/nym-node-status-api/nym-node-status-api/src/testruns/queue.rs +++ b/nym-node-status-api/nym-node-status-api/src/testruns/queue.rs @@ -14,20 +14,19 @@ pub(crate) async fn try_queue_testrun( let timestamp = now.unix_timestamp(); let timestamp_pretty = now.to_string(); - let items = sqlx::query_as!( - GatewayInfoDto, + let items = crate::db::query_as::( r#"SELECT - id as "id!", - gateway_identity_key as "gateway_identity_key!", - self_described as "self_described?", - explorer_pretty_bond as "explorer_pretty_bond?" + id, + gateway_identity_key, + self_described, + explorer_pretty_bond FROM gateways WHERE gateway_identity_key = ? AND bonded = true ORDER BY gateway_identity_key LIMIT 1"#, - identity_key, ) + .bind(identity_key.clone()) // TODO dz should call .fetch_one // TODO dz replace this in other queries as well .fetch(conn.as_mut()) @@ -47,22 +46,21 @@ pub(crate) async fn try_queue_testrun( // // check if there is already a test run for this gateway // - let items = sqlx::query_as!( - TestRunDto, + let items = crate::db::query_as::( r#"SELECT - id as "id!", - gateway_id as "gateway_id!", - status as "status!", - created_utc as "created_utc!", - ip_address as "ip_address!", - log as "log!", + id, + gateway_id, + status, + created_utc, + ip_address, + log, last_assigned_utc FROM testruns WHERE gateway_id = ? AND status != 2 ORDER BY id DESC LIMIT 1"#, - gateway_id, ) + .bind(gateway_id) .fetch(conn.as_mut()) .try_collect::>() .await?; @@ -70,8 +68,8 @@ pub(crate) async fn try_queue_testrun( if !items.is_empty() { let testrun = items.first().unwrap(); return Ok(TestRun { - id: testrun.id as u32, - identity_key, + id: testrun.id as i32, + identity_key: identity_key.clone(), status: format!( "{}", TestRunStatus::from_repr(testrun.status as u8).unwrap() @@ -83,24 +81,43 @@ pub(crate) async fn try_queue_testrun( // // save test run // - let status = TestRunStatus::Queued as u32; + let status = TestRunStatus::Queued as i32; let log = format!("Test for {identity_key} requested at {timestamp_pretty} UTC\n\n"); - let id = sqlx::query!( - "INSERT INTO testruns (gateway_id, status, ip_address, created_utc, log) VALUES ($1, $2, $3, $4, $5)", - gateway_id, - status, - ip_address, - timestamp, - log, - ) + #[cfg(feature = "sqlite")] + let id = { + sqlx::query!( + "INSERT INTO testruns (gateway_id, status, ip_address, created_utc, log) VALUES (?, ?, ?, ?, ?)", + gateway_id, + status, + ip_address, + timestamp, + log, + ) .execute(conn.as_mut()) .await? - .last_insert_rowid(); + .last_insert_rowid() + }; + + #[cfg(feature = "pg")] + let id = { + let record = sqlx::query!( + "INSERT INTO testruns (gateway_id, status, ip_address, created_utc, log) VALUES ($1, $2, $3, $4, $5) RETURNING id", + gateway_id as i32, + status, + ip_address, + timestamp, + log, + ) + .fetch_one(conn.as_mut()) + .await?; + record.id + }; Ok(TestRun { - id: id as u32, - identity_key, + #[allow(clippy::useless_conversion)] + id: id.try_into().unwrap(), + identity_key: identity_key.clone(), status: format!("{}", TestRunStatus::Queued), log, })