Files
Mark Sinclair f2091cc9d6 Data Observatory (#6172)
* rename nyxd-scraper to sqlite

wip: made storage mostly generic minus modules

changed error types to make modules dyn compatible

implemented traits for sqlite instance

using sqlite instance for rewarder and chain watcher

psql scaffolding

initial postgres support - missing some proto -> json parsing

use postgres in chain scraper

added message registry to block processor

message content parsing in psql

involved addresses

adding null value for logs

Revert "use postgres in chain scraper"

This reverts commit 83c84bfd2d.

using SignerInfo proto definitions for db serialisation

added ibc messages to MessageRegistry

* add the data observatory

* add the data observatory

* move message parsing and change webhook

* handle wasm messages in a module

* formatting and clippy

* copy shared migrations and add comments to ignore file to explain

* update offline queries

* change to clap args and use url::Url to parse args

* tidy up README, startup info, typos

* tidy up validator rewarder

* lock file

* change webhook module from msg to tx handler

* ignore profiler output

* add missing things and make clippy happy

* updated cosmrs version used by the nym wallet

* add glob to workspace dependencies

* rename migration files

* remove copying from shared migrations

* duplicate shared migrations to keep things simple

* add check for manual migration sync that will fail on `cargo build` in CI

* build.rs checks data observatory migrations have content of all shared scraper migrations and errors on changes or new files

* update runner

* add reset target to make file

* process events and logs

* migrations - remove unnecessary columns

* update offline queries

* chore: run cargo fmt

* fix up: inpsect_err instead of map_err

---------

Co-authored-by: Jędrzej Stuczyński <jedrzej.stuczynski@gmail.com>
Co-authored-by: Mark Sinclair <mmsinclair@users.noreply.github.com>
Co-authored-by: benedettadavico <benedetta.davico@gmail.com>
2025-12-02 21:27:22 +00:00

128 lines
3.9 KiB
PL/PgSQL

CREATE TABLE validator
(
consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */
consensus_pubkey TEXT NOT NULL UNIQUE /* Validator consensus public key */
);
CREATE TABLE pre_commit
(
validator_address TEXT NOT NULL REFERENCES validator (consensus_address),
height BIGINT NOT NULL,
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
voting_power BIGINT NOT NULL,
proposer_priority BIGINT NOT NULL,
UNIQUE (validator_address, timestamp)
);
CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address);
CREATE INDEX pre_commit_height_index ON pre_commit (height);
CREATE TABLE block
(
height BIGINT UNIQUE PRIMARY KEY,
hash TEXT NOT NULL UNIQUE,
num_txs INTEGER DEFAULT 0,
total_gas BIGINT DEFAULT 0,
proposer_address TEXT REFERENCES validator (consensus_address),
timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
CREATE INDEX block_height_index ON block (height);
CREATE INDEX block_hash_index ON block (hash);
CREATE INDEX block_proposer_address_index ON block (proposer_address);
ALTER TABLE block
SET (
autovacuum_vacuum_scale_factor = 0,
autovacuum_analyze_scale_factor = 0,
autovacuum_vacuum_threshold = 10000,
autovacuum_analyze_threshold = 10000
);
CREATE TABLE transaction
(
hash TEXT NOT NULL,
height BIGINT NOT NULL REFERENCES block (height),
"index" INTEGER NOT NULL, -- <<<=== not present in original bdjuno table, but it's quite useful
success BOOLEAN NOT NULL,
/* Body */
messages JSONB NOT NULL DEFAULT '[]'::JSONB,
memo TEXT,
signatures TEXT[] NOT NULL,
/* AuthInfo */
signer_infos JSONB NOT NULL DEFAULT '[]'::JSONB,
fee JSONB NOT NULL DEFAULT '{}'::JSONB,
/* Tx response */
gas_wanted BIGINT DEFAULT 0,
gas_used BIGINT DEFAULT 0,
raw_log TEXT,
logs JSONB,
events JSONB,
CONSTRAINT unique_tx UNIQUE (hash)
);
CREATE INDEX transaction_hash_index ON transaction (hash);
CREATE INDEX transaction_height_index ON transaction (height);
CREATE TYPE COIN AS
(
denom TEXT,
amount TEXT
);
CREATE TABLE message
(
transaction_hash TEXT NOT NULL,
index BIGINT NOT NULL,
type TEXT NOT NULL,
value JSONB NOT NULL,
involved_accounts_addresses TEXT[] NOT NULL,
height BIGINT NOT NULL,
funds COIN[] DEFAULT '{}',
FOREIGN KEY (transaction_hash) REFERENCES transaction (hash),
CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index)
);
CREATE INDEX message_transaction_hash_index ON message (transaction_hash);
CREATE INDEX message_type_index ON message (type);
CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses);
/**
* This function is used to find all the utils that involve any of the given addresses and have
* type that is one of the specified types.
*/
CREATE FUNCTION messages_by_address(
addresses TEXT[],
types TEXT[],
"limit" BIGINT = 100,
"offset" BIGINT = 0)
RETURNS SETOF message AS
$$
SELECT *
FROM message
WHERE (cardinality(types) = 0 OR type = ANY (types))
AND addresses && involved_accounts_addresses
ORDER BY height DESC
LIMIT "limit" OFFSET "offset"
$$ LANGUAGE sql STABLE;
CREATE FUNCTION messages_by_type(
types text[],
"limit" bigint DEFAULT 100,
"offset" bigint DEFAULT 0)
RETURNS SETOF message AS
$$
SELECT *
FROM message
WHERE (cardinality(types) = 0 OR type = ANY (types))
ORDER BY height DESC
LIMIT "limit" OFFSET "offset"
$$ LANGUAGE sql STABLE;
CREATE TABLE pruning
(
last_pruned_height BIGINT NOT NULL
);