GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mark Sinclair
2025-07-06 17:11:10 +01:00
parent 5b39cb9d8c
commit e4ad1e2e1b
40 changed files with 331 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/nyxd-scraper-psql/Makefile
+1 -1
View File
@@ -86,7 +86,7 @@ clean: ## Clean build artifacts and SQLx cache
.PHONY: clean-db
clean-db: test-db-down ## Stop database and clean volumes
docker volume rm -f nym-node-status-api_postgres_test_data 2>/dev/null || true
docker volume rm -f nyxd_scraper_postgres_test_data 2>/dev/null || true
# --- Utility Targets ---
.PHONY: sqlx-cli
common/nyxd-scraper-psql/docker-compose.yml
+2 -2
View File
@@ -15,7 +15,7 @@ services:
retries: 5
# Optional: Add volume for persistent data during development
# volumes:
# - postgres_test_data:/var/lib/postgresql/data
# - nyxd_scraper_postgres_test_data:/var/lib/postgresql/data
# volumes:
# postgres_test_data:
# nyxd_scraper_postgres_test_data:
common/nyxd-scraper-psql/sql_migrations/01_metadata.sql → common/nyxd-scraper-psql/sql_migrations/0001_metadata.sql
View File
common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql → common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql
View File
nyx-chain-watcher/pgsql/README_SQLX.md → nyx-chain-watcher/README_SQLX.md
View File
nyx-chain-watcher/pgsql/.gitignore
+2
View File
@@ -0,0 +1,2 @@
0001_metadata.sql
0002_cosmos.sql
nyx-chain-watcher/pgsql/.sqlx/query-140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda.json
Generated
+9 -2
View File
@@ -1,10 +1,17 @@
{
"db_name": "SQLite",
"db_name": "PostgreSQL",
"query": "INSERT INTO price_history\n (timestamp, chf, usd, eur, gbp, btc)\n VALUES\n ($1, $2, $3, $4, $5, $6)\n ON CONFLICT(timestamp) DO UPDATE SET\n chf=excluded.chf,\n usd=excluded.usd,\n eur=excluded.eur,\n gbp=excluded.gbp,\n btc=excluded.btc;",
"describe": {
"columns": [],
"parameters": {
"Right": 6
"Left": [
"Int8",
"Float8",
"Float8",
"Float8",
"Float8",
"Float8"
]
},
"nullable": []
},
nyx-chain-watcher/pgsql/.sqlx/query-1aa7733ad4bbf3e6b8db909b8646bee247bc021b9534f1d4b0fcad32e2e56218.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO watcher_execution(start, end, error_message)\n VALUES (?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 3
},
"nullable": []
},
"hash": "1aa7733ad4bbf3e6b8db909b8646bee247bc021b9534f1d4b0fcad32e2e56218"
}
nyx-chain-watcher/pgsql/.sqlx/query-34d0109e12e76621181b846bff9f8701f275d416a8d41686148fdb6a876de4c5.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO transactions (tx_hash, height, message_index, sender, recipient, amount, memo)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8",
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "34d0109e12e76621181b846bff9f8701f275d416a8d41686148fdb6a876de4c5"
}
nyx-chain-watcher/pgsql/.sqlx/query-454925e9a2f21cb370d90b3e105925542f62a5e1fc1e2cdba3df0e47e47b8c9f.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO payments (\n transaction_hash, sender_address, receiver_address,\n amount, height, timestamp, memo\n ) VALUES ($1, $2, $3, $4, $5, $6, $7)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Float8",
"Int8",
"Int8",
"Text"
]
},
"nullable": []
},
"hash": "454925e9a2f21cb370d90b3e105925542f62a5e1fc1e2cdba3df0e47e47b8c9f"
}
nyx-chain-watcher/pgsql/.sqlx/query-5fcd0f525ab790a7ebe8659d6cf3f3707065e7e34e920c49f2d50bef7ab9a968.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO transactions (tx_hash, height, message_index, sender, recipient, amount, memo)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "5fcd0f525ab790a7ebe8659d6cf3f3707065e7e34e920c49f2d50bef7ab9a968"
}
nyx-chain-watcher/pgsql/.sqlx/query-83df8a7c5fb24ba4d89f1feb300a8f6d4ac14e7e9b7b482bd57fae568ebd96ba.json
Generated
+5 -5
View File
@@ -1,19 +1,19 @@
{
"db_name": "SQLite",
"db_name": "PostgreSQL",
"query": "SELECT MAX(height) FROM payments",
"describe": {
"columns": [
{
"name": "MAX(height)",
"ordinal": 0,
"type_info": "Integer"
"name": "max",
"type_info": "Int8"
}
],
"parameters": {
"Right": 0
"Left": []
},
"nullable": [
true
null
]
},
"hash": "83df8a7c5fb24ba4d89f1feb300a8f6d4ac14e7e9b7b482bd57fae568ebd96ba"
nyx-chain-watcher/pgsql/.sqlx/query-a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618.json
Generated
+16 -14
View File
@@ -1,41 +1,43 @@
{
"db_name": "SQLite",
"db_name": "PostgreSQL",
"query": "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history WHERE timestamp >= $1;",
"describe": {
"columns": [
{
"name": "timestamp",
"ordinal": 0,
"type_info": "Integer"
"name": "timestamp",
"type_info": "Int8"
},
{
"name": "chf",
"ordinal": 1,
"type_info": "Float"
"name": "chf",
"type_info": "Float8"
},
{
"name": "usd",
"ordinal": 2,
"type_info": "Float"
"name": "usd",
"type_info": "Float8"
},
{
"name": "eur",
"ordinal": 3,
"type_info": "Float"
"name": "eur",
"type_info": "Float8"
},
{
"name": "gbp",
"ordinal": 4,
"type_info": "Float"
"name": "gbp",
"type_info": "Float8"
},
{
"name": "btc",
"ordinal": 5,
"type_info": "Float"
"name": "btc",
"type_info": "Float8"
}
],
"parameters": {
"Right": 1
"Left": [
"Int8"
]
},
"nullable": [
false,
nyx-chain-watcher/pgsql/.sqlx/query-a9d6227ecfa9866096da1367518612f219ae969e98b331af04381429fef8716c.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "SQLite",
"query": "\n INSERT INTO payments (\n transaction_hash, sender_address, receiver_address,\n amount, height, timestamp, memo\n ) VALUES (?, ?, ?, ?, ?, ?, ?)\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "a9d6227ecfa9866096da1367518612f219ae969e98b331af04381429fef8716c"
}
nyx-chain-watcher/pgsql/.sqlx/query-f69907735e9b1e1572c4bf6fe8d44d4ea4e55c2a9c4d4f7e1c7e57bcb848ee08.json → nyx-chain-watcher/pgsql/.sqlx/query-e64101f92363902aae3fe2e70fdb0f041f4bca756a12abe300fe91637c33b4ed.json
Generated
+19 -17
View File
@@ -1,56 +1,58 @@
{
"db_name": "SQLite",
"query": "\n SELECT id, tx_hash, height, message_index, sender, recipient, amount, memo, created_at as \"created_at: ::time::OffsetDateTime\"\n FROM transactions\n WHERE height > ?\n ORDER BY height ASC, message_index ASC\n ",
"db_name": "PostgreSQL",
"query": "\n SELECT id, tx_hash, height, message_index, sender, recipient, amount, memo, created_at as \"created_at: ::time::OffsetDateTime\"\n FROM transactions\n WHERE height > $1\n ORDER BY height, message_index\n ",
"describe": {
"columns": [
{
"name": "id",
"ordinal": 0,
"type_info": "Integer"
"name": "id",
"type_info": "Int4"
},
{
"name": "tx_hash",
"ordinal": 1,
"name": "tx_hash",
"type_info": "Text"
},
{
"name": "height",
"ordinal": 2,
"type_info": "Integer"
"name": "height",
"type_info": "Int8"
},
{
"name": "message_index",
"ordinal": 3,
"type_info": "Integer"
"name": "message_index",
"type_info": "Int8"
},
{
"name": "sender",
"ordinal": 4,
"name": "sender",
"type_info": "Text"
},
{
"name": "recipient",
"ordinal": 5,
"name": "recipient",
"type_info": "Text"
},
{
"name": "amount",
"ordinal": 6,
"name": "amount",
"type_info": "Text"
},
{
"name": "memo",
"ordinal": 7,
"name": "memo",
"type_info": "Text"
},
{
"name": "created_at: ::time::OffsetDateTime",
"ordinal": 8,
"type_info": "Datetime"
"name": "created_at: ::time::OffsetDateTime",
"type_info": "Date"
}
],
"parameters": {
"Right": 1
"Left": [
"Int8"
]
},
"nullable": [
false,
@@ -64,5 +66,5 @@
true
]
},
"hash": "f69907735e9b1e1572c4bf6fe8d44d4ea4e55c2a9c4d4f7e1c7e57bcb848ee08"
"hash": "e64101f92363902aae3fe2e70fdb0f041f4bca756a12abe300fe91637c33b4ed"
}
nyx-chain-watcher/pgsql/.sqlx/query-f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8.json
Generated
+14 -14
View File
@@ -1,41 +1,41 @@
{
"db_name": "SQLite",
"db_name": "PostgreSQL",
"query": "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history ORDER BY timestamp DESC LIMIT 1;",
"describe": {
"columns": [
{
"name": "timestamp",
"ordinal": 0,
"type_info": "Integer"
"name": "timestamp",
"type_info": "Int8"
},
{
"name": "chf",
"ordinal": 1,
"type_info": "Float"
"name": "chf",
"type_info": "Float8"
},
{
"name": "usd",
"ordinal": 2,
"type_info": "Float"
"name": "usd",
"type_info": "Float8"
},
{
"name": "eur",
"ordinal": 3,
"type_info": "Float"
"name": "eur",
"type_info": "Float8"
},
{
"name": "gbp",
"ordinal": 4,
"type_info": "Float"
"name": "gbp",
"type_info": "Float8"
},
{
"name": "btc",
"ordinal": 5,
"type_info": "Float"
"name": "btc",
"type_info": "Float8"
}
],
"parameters": {
"Right": 0
"Left": []
},
"nullable": [
false,
nyx-chain-watcher/pgsql/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO watcher_execution(start_ts, end_ts, error_message)\n VALUES ($1, $2, $3)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Timestamptz",
"Timestamptz",
"Text"
]
},
"nullable": []
},
"hash": "fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63"
}
nyx-chain-watcher/pgsql/Cargo.toml
+2 -2
View File
@@ -28,7 +28,7 @@ nyxd-scraper-psql = { path = "../../common/nyxd-scraper-psql" }
reqwest = { workspace = true, features = ["rustls-tls"] }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "time"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "time"] }
thiserror = { workspace = true }
time = { workspace = true }
tokio = { workspace = true, features = ["process", "rt-multi-thread"] }
@@ -44,4 +44,4 @@ utoipauto = { workspace = true }
[build-dependencies]
anyhow = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite"] }
sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] }
nyx-chain-watcher/pgsql/Dockerfile
+2 -2
View File
@@ -2,7 +2,7 @@
FROM harbor.nymte.ch/dockerhub/rust:latest AS builder
COPY ./ /usr/src/nym
WORKDIR /usr/src/nym/nyx-chain-watcher
WORKDIR /usr/src/nym/nyx-chain-watcher/pgsql
RUN cargo build --release
@@ -28,6 +28,6 @@ RUN apt update && apt install -yy curl ca-certificates
WORKDIR /nym
COPY --from=builder /usr/src/nym/target/release/nyx-chain-watcher ./
COPY --from=builder /usr/src/nym/target/release/nyx-chain-watcher/pgsql ./
ENTRYPOINT [ "/nym/nyx-chain-watcher", "run" ]
nyx-chain-watcher/pgsql/Makefile
+104
View File
@@ -0,0 +1,104 @@
# Makefile for nyx_chain_watcher database management
# --- Configuration ---
TEST_DATABASE_URL := postgres://testuser:testpass@localhost:5433/nyx_chain_watcher_test
# Docker compose service names
DB_SERVICE_NAME := postgres-test
DB_CONTAINER_NAME := nyx_chain_watcher_postgres_test
# Default target
.PHONY: default
default: help
# --- Main Targets ---
.PHONY: prepare-pg
prepare-pg: test-db-up test-db-wait test-db-migrate test-db-prepare test-db-down ## Setup PostgreSQL and prepare SQLx offline cache
.PHONY: test-db
test-db: test-db-up test-db-wait test-db-migrate test-db-run test-db-down ## Run tests with PostgreSQL database
.PHONY: dev-db
dev-db: test-db-up test-db-wait test-db-migrate ## Start PostgreSQL for development (keeps running)
@echo "PostgreSQL is running on port 5433"
@echo "Connection string: $(TEST_DATABASE_URL)"
# --- Docker Compose Targets ---
.PHONY: test-db-up
test-db-up: ## Start the PostgreSQL test database in the background
@echo "Starting PostgreSQL test database..."
docker compose up -d $(DB_SERVICE_NAME)
.PHONY: test-db-wait
test-db-wait: ## Wait for the PostgreSQL database to be healthy
@echo "Waiting for PostgreSQL database..."
@while ! docker inspect --format='{{.State.Health.Status}}' $(DB_CONTAINER_NAME) 2>/dev/null | grep -q 'healthy'; do \
echo -n "."; \
sleep 1; \
done; \
echo " Database is healthy!"
.PHONY: test-db-down
test-db-down: ## Stop and remove the test database
@echo "Stopping PostgreSQL test database..."
docker compose down
# --- SQLx Targets ---
.PHONY: test-db-migrate
test-db-migrate: ## Run database migrations against PostgreSQL
@echo "Copying common PostgreSQL migrations..."
cp ../../common/nyxd-scraper-psql/sql_migrations/* migrations
@echo "Running watcher PostgreSQL migrations..."
RUST_LOG=debug DATABASE_URL="$(TEST_DATABASE_URL)" sqlx migrate run --source migrations
.PHONY: test-db-prepare
test-db-prepare: ## Run sqlx prepare for compile-time query verification
@echo "Running sqlx prepare for PostgreSQL..."
DATABASE_URL="$(TEST_DATABASE_URL)" cargo sqlx prepare --
# --- Build and Test Targets ---
.PHONY: test-db-run
test-db-run: ## Run tests with PostgreSQL feature
@echo "Running tests with PostgreSQL..."
DATABASE_URL="$(TEST_DATABASE_URL)" cargo test --no-default-features
.PHONY: build-pg
build-pg: ## Build with PostgreSQL feature
@echo "Building with PostgreSQL feature..."
cargo build --no-default-features
.PHONY: check-pg
check-pg: ## Check code with PostgreSQL feature
@echo "Checking code with PostgreSQL feature..."
cargo check --no-default-features
.PHONY: clippy
clippy: clippy-pg
.PHONY: clippy-pg
clippy-pg: ## Run clippy with PostgreSQL feature
@echo "Running clippy with PostgreSQL feature..."
cargo clippy --no-default-features -- -D warnings
# --- Cleanup Targets ---
.PHONY: clean
clean: ## Clean build artifacts and SQLx cache
cargo clean
rm -rf .sqlx
.PHONY: clean-db
clean-db: test-db-down ## Stop database and clean volumes
docker volume rm -f nyx_chain_watcher_postgres_test_data 2>/dev/null || true
# --- Utility Targets ---
.PHONY: sqlx-cli
sqlx-cli: ## Install sqlx-cli if not already installed
@command -v sqlx >/dev/null 2>&1 || cargo install sqlx-cli --features postgres
.PHONY: psql
psql: ## Connect to the running PostgreSQL database with psql
@docker exec -it $(DB_CONTAINER_NAME) psql -U testuser -d nyx_chain_watcher_test
.PHONY: help
help: ## Show help for Makefile targets
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
nyx-chain-watcher/pgsql/README.md
+20 -2
View File
@@ -6,9 +6,23 @@ Look in [env.rs](./src/env.rs) for the names of environment variables that can b
## Running locally
Connect with `psql` to your local database:
```sql
CREATE USER nyx_chain_scraper WITH PASSWORD 'scrapymcscrapeface';
CREATE DATABASE nyx_chain_scraper_data;
GRANT ALL ON DATABASE nyx_chain_scraper_data TO nyx_chain_scraper;
```
NYX_CHAIN_WATCHER_HISTORY_DATABASE_PATH=chain_history.sqlite \
NYX_CHAIN_WATCHER_DATABASE_PATH=nyx_chain_watcher.sqlite \
Then run:
```
cargo run -- init --chain-history-db-connection-string postgres://nyx_chain_scraper:scrapymcscrapeface@localhost/nyx_chain_scraper_data
```
```
NYX_CHAIN_WATCHER_HISTORY_DATABASE_PATH=postgres://nyx_chain_scraper:scrapymcscrapeface@localhost/nyx_chain_scraper_data \
NYX_CHAIN_WATCHER_WATCH_ACCOUNTS=n1...,n1...,n1... \
NYX_CHAIN_WATCHER_WATCH_CHAIN_MESSAGE_TYPES="/cosmos.bank.v1beta1.MsgSend,/ibc.applications.transfer.v1.MsgTransfer"
NYX_CHAIN_WATCHER_WEBHOOK_URL="https://webhook.site" \
@@ -16,4 +30,8 @@ NYX_CHAIN_WATCHER_WEBHOOK_AUTH=1234 \
cargo run -- run
```
## sqlx
If you have issues with `sqlx` please see [README_SQLX.md](../README_SQLX.md).
nyx-chain-watcher/pgsql/build.rs
+5 -50
View File
@@ -1,53 +1,8 @@
use anyhow::Result;
use sqlx::{sqlite::SqliteConnectOptions, Connection, SqliteConnection};
use std::env::var;
use std::io::Write;
use std::{collections::HashMap, fs::File, path::PathBuf, str::FromStr};
// Copyright 2025 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
#[tokio::main]
async fn main() -> Result<()> {
let db_path = PathBuf::from(var("OUT_DIR").unwrap()).join("nyx_chain_watcher.sqlite");
// Create the database directory if it doesn't exist
if let Some(parent) = db_path.parent() {
std::fs::create_dir_all(parent)?;
fn main() {
if let Ok(database_url) = std::env::var("DATABASE_URL") {
println!("cargo::rustc-env=DATABASE_URL={database_url}");
}
let db_path_str = db_path.display().to_string().replace('\\', "/");
let db_url = format!("sqlite:{db_path_str}");
// Ensure database file is created with proper permissions
let connect_options = SqliteConnectOptions::from_str(&db_url)?
.create_if_missing(true)
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.foreign_keys(true);
// Create initial connection to ensure database exists
let mut conn = SqliteConnection::connect_with(&connect_options).await?;
sqlx::migrate!("./migrations").run(&mut conn).await?;
export_db_variables(&db_url)?;
// Force SQLx to prepare all queries during build
println!("cargo:rustc-env=SQLX_OFFLINE=true");
println!("cargo:rustc-env=DATABASE_URL={db_url}");
// Add rerun-if-changed directives
println!("cargo:rerun-if-changed=migrations");
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=src");
Ok(())
}
fn export_db_variables(db_url: &str) -> Result<()> {
let mut map = HashMap::new();
map.insert("DATABASE_URL", db_url);
let mut file = File::create(".env")?;
for (var, value) in map.iter() {
writeln!(file, "{var}={value}")?;
}
Ok(())
}
nyx-chain-watcher/pgsql/docker-compose.yml
+21
View File
@@ -0,0 +1,21 @@
services:
postgres-test:
image: postgres:16-alpine
container_name: nyx_chain_watcher_postgres_test
environment:
POSTGRES_DB: nyx_chain_watcher_test
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
ports:
- '5433:5432' # Map to 5433 to avoid conflicts with default PostgreSQL
healthcheck:
test: ['CMD-SHELL', 'pg_isready -U testuser -d nyx_chain_watcher_test']
interval: 5s
timeout: 5s
retries: 5
# Optional: Add volume for persistent data during development
# volumes:
# - nyx_chain_watcher_postgres_test_data:/var/lib/postgresql/data
#volumes:
# nyx_chain_watcher_postgres_test_data:
nyx-chain-watcher/pgsql/migrations/001_price_data.sql
-7
View File
@@ -1,7 +0,0 @@
CREATE TABLE price_history (
timestamp INTEGER PRIMARY KEY,
chf REAL NOT NULL,
usd REAL NOT NULL,
eur REAL NOT NULL,
btc REAL NOT NULL
);
nyx-chain-watcher/pgsql/migrations/004_add_gbp_column.sql
-1
View File
@@ -1 +0,0 @@
ALTER TABLE price_history ADD COLUMN gbp REAL NOT NULL DEFAULT 0;
nyx-chain-watcher/pgsql/migrations/0101_price_data.sql
+8
View File
@@ -0,0 +1,8 @@
CREATE TABLE price_history (
timestamp bigint PRIMARY KEY,
chf double precision NOT NULL,
usd double precision NOT NULL,
eur double precision NOT NULL,
btc double precision NOT NULL,
gbp double precision NOT NULL
);
nyx-chain-watcher/pgsql/migrations/002_payment_transactions.sql → nyx-chain-watcher/pgsql/migrations/0102_payment_transactions.sql
+4 -4
View File
@@ -1,10 +1,10 @@
CREATE TABLE payments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER PRIMARY KEY,
transaction_hash TEXT NOT NULL UNIQUE,
sender_address TEXT NOT NULL,
receiver_address TEXT NOT NULL,
amount REAL NOT NULL,
timestamp INTEGER NOT NULL,
height INTEGER NOT NULL,
amount double precision NOT NULL,
timestamp bigint NOT NULL,
height bigint NOT NULL,
memo TEXT
);
nyx-chain-watcher/pgsql/migrations/003_create_transactions_table.sql → nyx-chain-watcher/pgsql/migrations/0103_create_transactions_table.sql
+4 -4
View File
@@ -1,12 +1,12 @@
CREATE TABLE IF NOT EXISTS transactions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER PRIMARY KEY,
tx_hash TEXT NOT NULL,
height INTEGER NOT NULL,
message_index INTEGER NOT NULL,
height BIGINT NOT NULL,
message_index BIGINT NOT NULL,
sender TEXT NOT NULL,
recipient TEXT NOT NULL,
amount TEXT NOT NULL,
memo TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
created_at DATE DEFAULT CURRENT_TIMESTAMP,
UNIQUE(tx_hash, message_index)
);
nyx-chain-watcher/pgsql/migrations/005_add_listener_failure_table.sql → nyx-chain-watcher/pgsql/migrations/0104_add_listener_failure_table.sql
+3 -3
View File
@@ -5,7 +5,7 @@
CREATE TABLE watcher_execution
(
start TIMESTAMP WITHOUT TIME ZONE NOT NULL,
end TIMESTAMP WITHOUT TIME ZONE NOT NULL,
start_ts TIMESTAMPTZ NOT NULL,
end_ts TIMESTAMPTZ NOT NULL,
error_message TEXT
)
);
nyx-chain-watcher/pgsql/src/chain_scraper/mod.rs
+8 -8
View File
@@ -1,4 +1,5 @@
use crate::config::PaymentWatchersConfig;
use crate::db::DbPool;
use crate::env::vars::{
NYXD_SCRAPER_START_HEIGHT, NYXD_SCRAPER_UNSAFE_NUKE_DB,
NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT,
@@ -10,13 +11,12 @@ use nyxd_scraper_psql::{
MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, PostgresNyxdScraper,
PruningOptions, ScraperError,
};
use sqlx::SqlitePool;
use std::fs;
use tracing::{info, warn};
pub(crate) async fn run_chain_scraper(
config: &crate::config::Config,
db_pool: SqlitePool,
connection_pool: DbPool,
shared_state: BankScraperModuleState,
) -> anyhow::Result<PostgresNyxdScraper> {
let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined");
@@ -62,7 +62,7 @@ pub(crate) async fn run_chain_scraper(
},
})
.with_msg_module(BankScraperModule::new(
db_pool,
connection_pool,
config.payment_watcher_config.clone(),
shared_state,
));
@@ -76,19 +76,19 @@ pub(crate) async fn run_chain_scraper(
}
pub struct BankScraperModule {
db_pool: SqlitePool,
connection_pool: DbPool,
payment_config: PaymentWatchersConfig,
shared_state: BankScraperModuleState,
}
impl BankScraperModule {
pub fn new(
db_pool: SqlitePool,
connection_pool: DbPool,
payment_config: PaymentWatchersConfig,
shared_state: BankScraperModuleState,
) -> Self {
Self {
db_pool,
connection_pool,
payment_config,
shared_state,
}
@@ -108,7 +108,7 @@ impl BankScraperModule {
sqlx::query!(
r#"
INSERT INTO transactions (tx_hash, height, message_index, sender, recipient, amount, memo)
VALUES (?, ?, ?, ?, ?, ?, ?)
VALUES ($1, $2, $3, $4, $5, $6, $7)
"#,
tx_hash,
height,
@@ -118,7 +118,7 @@ impl BankScraperModule {
amount,
memo
)
.execute(&self.db_pool)
.execute(&self.connection_pool)
.await?;
Ok(())
nyx-chain-watcher/pgsql/src/cli/commands/run/config.rs
-6
View File
@@ -12,7 +12,6 @@ pub(crate) fn get_run_config(args: Args) -> Result<Config, NyxChainWatcherError>
ref watch_for_transfer_recipient_accounts,
mut watch_for_chain_message_types,
webhook_auth,
ref chain_watcher_db_path,
webhook_url,
..
} = args;
@@ -48,11 +47,6 @@ pub(crate) fn get_run_config(args: Args) -> Result<Config, NyxChainWatcherError>
args.chain_history_db_connection_string,
);
if let Some(db_path) = chain_watcher_db_path {
info!("Overriding database url with '{db_path}'");
builder = builder.with_db_path(db_path.clone());
}
if let Some(webhook_url) = webhook_url {
let authentication =
webhook_auth.map(|token| HttpAuthenticationOptions::AuthorizationBearerToken { token });
nyx-chain-watcher/pgsql/src/cli/commands/run/mod.rs
+6 -18
View File
@@ -29,11 +29,11 @@ async fn try_insert_watcher_execution_information(
) {
let _ = sqlx::query!(
r#"
INSERT INTO watcher_execution(start, end, error_message)
VALUES (?, ?, ?)
INSERT INTO watcher_execution(start_ts, end_ts, error_message)
VALUES ($1, $2, $3)
"#,
start,
end,
start.into(),
end.into(),
error_message
)
.execute(&db_pool)
@@ -114,27 +114,15 @@ pub(crate) async fn execute(args: Args, http_port: u16) -> Result<(), NyxChainWa
let config = config::get_run_config(args)?;
let db_path = config.database_path();
let db_connection_string = config.chain_scraper_connection_string();
info!("Config is {config:#?}");
info!(
"Database path is {:?}",
std::path::Path::new(&db_path)
.canonicalize()
.unwrap_or_default()
);
info!(
"Chain History Database path is {:?}",
std::path::Path::new(&config.chain_scraper_connection_string()).canonicalize()
);
// Ensure parent directory exists
if let Some(parent) = std::path::Path::new(&db_path).parent() {
std::fs::create_dir_all(parent)?;
}
let connection_url = format!("sqlite://{db_path}?mode=rwc");
let storage = db::Storage::init(connection_url).await?;
let storage = db::Storage::init(db_connection_string).await?;
let watcher_pool = storage.pool_owned();
let mut tasks = JoinSet::new();
nyx-chain-watcher/pgsql/src/config/mod.rs
-24
View File
@@ -19,8 +19,6 @@ use crate::error::NyxChainWatcherError;
const DEFAULT_NYM_CHAIN_WATCHER_DIR: &str = "nym-chain-watcher";
pub(crate) const DEFAULT_NYM_CHAIN_WATCHER_DB_FILENAME: &str = "nyx_chain_watcher.sqlite";
/// Derive default path to nym-chain-watcher's config directory.
/// It should get resolved to `$HOME/.nym/nym-chain-watcher/config`
pub fn default_config_directory() -> PathBuf {
@@ -41,8 +39,6 @@ pub struct ConfigBuilder {
pub data_dir: PathBuf,
pub db_path: Option<String>,
pub chain_scraper_connection_string: String,
pub payment_watcher_config: Option<PaymentWatchersConfig>,
@@ -60,16 +56,10 @@ impl ConfigBuilder {
data_dir,
payment_watcher_config: None,
logging: None,
db_path: None,
chain_scraper_connection_string,
}
}
pub fn with_db_path(mut self, db_path: String) -> Self {
self.db_path = Some(db_path);
self
}
#[allow(dead_code)]
pub fn with_payment_watcher_config(
mut self,
@@ -91,7 +81,6 @@ impl ConfigBuilder {
save_path: Some(self.config_path),
payment_watcher_config: self.payment_watcher_config.unwrap_or_default(),
data_dir: self.data_dir,
db_path: self.db_path,
chain_scraper_connection_string: self.chain_scraper_connection_string,
}
}
@@ -107,9 +96,6 @@ pub struct Config {
#[serde(skip)]
pub(crate) data_dir: PathBuf,
#[serde(skip)]
db_path: Option<String>,
pub chain_scraper_connection_string: String,
#[serde(default)]
@@ -194,16 +180,6 @@ impl Config {
Ok(node_dir.join(DEFAULT_DATA_DIR))
}
pub fn database_path(&self) -> String {
self.db_path.clone().unwrap_or_else(|| {
let mut path = self.data_dir.clone().to_path_buf();
path.push(DEFAULT_NYM_CHAIN_WATCHER_DB_FILENAME);
path.to_str()
.unwrap_or(DEFAULT_NYM_CHAIN_WATCHER_DB_FILENAME)
.to_string()
})
}
pub fn chain_scraper_connection_string(&self) -> String {
self.chain_scraper_connection_string.clone()
}
nyx-chain-watcher/pgsql/src/db/mod.rs
+3 -4
View File
@@ -1,5 +1,5 @@
use anyhow::{anyhow, Result};
use sqlx::{migrate::Migrator, sqlite::SqliteConnectOptions, SqlitePool};
use sqlx::{migrate::Migrator, postgres::PgConnectOptions, Postgres};
use std::str::FromStr;
pub(crate) mod models;
@@ -10,7 +10,7 @@ pub(crate) mod queries {
static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
pub(crate) type DbPool = SqlitePool;
pub(crate) type DbPool = sqlx::Pool<Postgres>;
pub(crate) struct Storage {
pool: DbPool,
@@ -18,8 +18,7 @@ pub(crate) struct Storage {
impl Storage {
pub async fn init(connection_url: String) -> Result<Self> {
let connect_options =
SqliteConnectOptions::from_str(&connection_url)?.create_if_missing(true);
let connect_options = PgConnectOptions::from_str(&connection_url)?;
let pool = DbPool::connect_with(connect_options)
.await
nyx-chain-watcher/pgsql/src/db/models.rs
+5 -5
View File
@@ -7,11 +7,11 @@ use utoipa::ToSchema;
#[derive(Clone, Serialize, Deserialize, Debug, ToSchema)]
pub(crate) struct CurrencyPrices {
pub(crate) chf: f32,
pub(crate) usd: f32,
pub(crate) eur: f32,
pub(crate) gbp: f32,
pub(crate) btc: f32,
pub(crate) chf: f64,
pub(crate) usd: f64,
pub(crate) eur: f64,
pub(crate) gbp: f64,
pub(crate) btc: f64,
}
// Struct to hold Coingecko response
nyx-chain-watcher/pgsql/src/db/queries/payments.rs
+2 -2
View File
@@ -5,7 +5,7 @@ pub async fn get_last_checked_height(pool: &DbPool) -> Result<i64> {
let result = sqlx::query_scalar!("SELECT MAX(height) FROM payments")
.fetch_one(pool)
.await?;
Ok(result.unwrap_or(0))
Ok(result.unwrap_or(0) as i64)
}
pub async fn insert_payment(
@@ -24,7 +24,7 @@ pub async fn insert_payment(
INSERT INTO payments (
transaction_hash, sender_address, receiver_address,
amount, height, timestamp, memo
) VALUES (?, ?, ?, ?, ?, ?, ?)
) VALUES ($1, $2, $3, $4, $5, $6, $7)
"#,
transaction_hash,
sender_address,
nyx-chain-watcher/pgsql/src/payment_listener/mod.rs
+5 -6
View File
@@ -3,28 +3,27 @@
use crate::config::PaymentWatchersConfig;
use crate::db::models::Transaction;
use crate::db::queries;
use crate::db::{queries, DbPool};
use crate::http::state::{
PaymentListenerFailureDetails, PaymentListenerState, ProcessedPayment, WatcherFailureDetails,
};
use crate::models::WebhookPayload;
use crate::payment_listener::watcher::PaymentWatcher;
use anyhow::Context;
use sqlx::SqlitePool;
use tokio::time::{self, Duration};
use tracing::{debug, error, info};
pub(crate) mod watcher;
pub(crate) struct PaymentListener {
db_pool: SqlitePool,
db_pool: DbPool,
payment_watchers: Vec<PaymentWatcher>,
shared_state: PaymentListenerState,
}
impl PaymentListener {
pub(crate) fn new(
db_pool: SqlitePool,
db_pool: DbPool,
config: PaymentWatchersConfig,
shared_state: PaymentListenerState,
) -> anyhow::Result<Self> {
@@ -46,8 +45,8 @@ impl PaymentListener {
r#"
SELECT id, tx_hash, height, message_index, sender, recipient, amount, memo, created_at as "created_at: ::time::OffsetDateTime"
FROM transactions
WHERE height > ?
ORDER BY height ASC, message_index ASC
WHERE height > $1
ORDER BY height, message_index
"#,
last_checked_height
)
nyx-chain-watcher/sqlite/Dockerfile
+2 -2
View File
@@ -2,7 +2,7 @@
FROM harbor.nymte.ch/dockerhub/rust:latest AS builder
COPY ./ /usr/src/nym
WORKDIR /usr/src/nym/nyx-chain-watcher
WORKDIR /usr/src/nym/nyx-chain-watcher/sqlite
RUN cargo build --release
@@ -28,6 +28,6 @@ RUN apt update && apt install -yy curl ca-certificates
WORKDIR /nym
COPY --from=builder /usr/src/nym/target/release/nyx-chain-watcher ./
COPY --from=builder /usr/src/nym/target/release/nyx-chain-watcher/sqlite ./
ENTRYPOINT [ "/nym/nyx-chain-watcher", "run" ]
nyx-chain-watcher/sqlite/README.md
+3
View File
@@ -16,4 +16,7 @@ NYX_CHAIN_WATCHER_WEBHOOK_AUTH=1234 \
cargo run -- run
```
## sqlx
If you have issues with `sqlx` please see [README_SQLX.md](../README_SQLX.md).
nyx-chain-watcher/sqlite/README_SQLX.md
-95
View File
@@ -1,95 +0,0 @@
# Making `sqlx` work
Some of the errors encountered and possible solutions.
## `The cargo feature offline has to be enabled to use SQLX_OFFLINE`
Did you enable `offline` **cargo feature** of `sqlx` dependency in your
`Cargo.toml`?
Also, it may happen if you have a version mismatch beetween
- `sqlx` cargo dependency in `Cargo.toml`
- `sqlx-cli` installed by cargo
To ensure correct version, do
```
cargo uninstall sqlx-cli
cargo install --version <version> sqlx-cli
```
where `<version>` matches the version of sqlx in your `Cargo.toml`.
## `Error: failed to connect to database: password authentication failed`
If it's in-code, make sure you don't "double authenticate", i.e.
- if username and password are already specified in `DATABASE_URL`
- then, you don't have to use
```rust
ConnectOptions::from_str(&database_Url)
.username() // unnecessary
.password() // unnecessary
```
If it's outside of code (i.e. when running `cargo check`)
- make sure password doesn't have any special characters that could be
interpreted by the command line/shell weirdly, like `$#\` etc.
## Offline query data looks like this
```
.sqlx/
├─ new_file
├─ query-249faa11b88b749f50342bb5c9cc41d20896db543eed74a6f320c041bcbb723d.json
├─ query-aff7fbd06728004d2f2226d20c32f1482df00de2dc1d2b4debbb2e12553d997b.json
├─ ...
├─ query-e53f479f8cead3dc8aa1875e5d450ad69686cf6a109e37d6c3f0623c3e9f91d0.json
```
The offline mode for the queries uses a separate file per `query!()` invocation
Each workspace member that works with `sqlx` has `.sqlx` directory, containing
its own schema description. This allows compile-time checks without needing a
live DB connection (so called `OFFLINE_MODE`).
To initialize those files, you need to run `cargo sqlx prepare` with a live
connection to DB (to pull schema information).
### Similar to:
```
warning: no queries found; do you have the `offline` feature enabled
```
### Possible solutions
- does your `sqlx-cli` version match `sqlx` version from `Cargo.toml`?
+ `cargo install -f sqlx-cli --version <specific version>`
```
cargo install sqlx-cli --version <exact semver version as sqlx> --force
```
- is your crate a library?
```
cargo sqlx prepare -- --lib
```
- are your `query!` invocations hidden behind a feature?
```
cargo sqlx prepare -- --features <feature_name>
```
- do you have `offline` cargo feature enabled?
- make sure to `cargo clean` after these updates
## Any many, many more
- `EOF while parsing a value at line`
- `failed to find data for query`
### Possible solutions
- Usually a DB connection issue
- Retry everything
- Throw in a `cargo clean -p <your package>` for good measure