From 51f58427e943ca7ffa597ace42c02165b999ffce Mon Sep 17 00:00:00 2001 From: wiesche <120273695+wiesche89@users.noreply.github.com> Date: Mon, 25 May 2026 20:34:08 +0200 Subject: [PATCH 01/12] docs: update contributing process (#3833) * docs: update contributing process * add pr example --- CONTRIBUTING.md | 48 +++++++++++++++++++++++++++++++++++++----------- README.md | 3 +-- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 732efac1..40cd94e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,36 @@ More documentation or updates/fixes to existing documentation are also very welc We generally prefer you to PR your work earlier rather than later. This ensures everyone else has a better idea of what's being worked on, and can help reduce wasted effort. If work on your PR has just begun, please feel free to create the PR with [WIP] (work in progress) in the PR title, and let us know when it's ready for review in the comments. -Since mainnet has been released, the bar for having PRs accepted has been raised. Before submitting your PR for approval, please be ensure it: +All changes should first be submitted against the `staging` branch and follow the PR rules below. The `staging` branch is tested by the community before changes are merged into `master` by an admin when appropriate. + +```text +Contributor branch ++-- Pull request + +-- staging branch + +-- testing (community) + +-- Admin merge + +-- master +``` + +## Creating a Pull Request + +Create your work on a dedicated branch based on the latest `staging` branch. + +```sh +git checkout staging +git pull +git checkout -b my-change +``` + +Fork the Grin repository and add your fork as a git remote. + +```sh +git remote add my-fork +``` + +Push your branch to your fork and open a GitHub pull request from your fork's branch into the Grin repository's `staging` branch. You can do this through the GitHub web interface. + +Since mainnet has been released, the bar for having PRs accepted has been raised. Before submitting your PR for approval, please ensure it: * Includes a proper description of what problems the PR addresses, as well as a detailed explanation as to what it changes * Explains whether/how the change is consensus breaking or breaks existing client functionality * Contains unit tests exercising new/changed functionality @@ -26,23 +55,20 @@ The development team will be happy to help and guide you with any of these point # Find Us -When you are starting to contribute to grin, we really would appreciate if you come by the gitter chat channels. +When you are starting to contribute to grin, we really would appreciate if you come by one of the community channels. -In case of problems with trying out grin, before starting to contribute, there's the [grincoin#support](https://keybase.io/team/grincoin) on Keybase. Write there about what you've done, what you want to do, and maybe paste logs through a text paste webservice. - -* Please [join the grincoin#general on Keybase](https://keybase.io/team/grincoin) to get a feeling for the community. -* And see the developers chat channel [grincoin#dev on Keybase](https://keybase.io/team/grincoin) if you have questions about source code files. - If you explain what you're looking at and what you want to do, we'll try to help you along the way. +* Join the development discussion on [Telegram](https://t.me/grindevelopment). +* Join the [grincoin team on Keybase](https://keybase.io/team/grincoin) for community and support channels. +* Join the discussion on the [Forum](https://forum.grin.mw). * See `docs/*.md` and the folder structure explanations, [the wiki](https://github.com/mimblewimble/docs/wiki) and the official [Grin documentation](https://docs.grin.mw/). -* Further information and discussions are in the [Forum](https://forum.grin.mw), the [website](https://grin.mw), the [mailing list](https://lists.launchpad.net/mimblewimble/) and news channels like the [Reddit/grincoin](https://www.reddit.com/r/grincoin/) and a (mostly unfiltered!) Twitter bot that collects headlines, mailing list posts, and reddit posts related to Mimblewimble/Grin: [@grinmw](https://twitter.com/grinmw) ## Testing -Run all tests with `cargo test --all` and please remember to test locally before creating a PR on github. +Run all tests with `cargo test --all` and please remember to test locally before creating a PR against `staging`. -### Check Travis output +### Check CI output -After creating a PR on github, the code will be tested automatically by Travis CI, and from the results you'll get a red or green light. The test can take a while, and you'll have a "yellow traffic light" on your PR until Travis CI is done testing. +After creating a PR, the code will be tested automatically by CI. The test can take a while, and your PR may remain pending until CI is done testing. ### Building quality diff --git a/README.md b/README.md index ab3de1d9..948916f3 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,8 @@ To get involved, read our [contributing docs](CONTRIBUTING.md). Find us: +* Telegram: [Grin Development](https://t.me/grindevelopment) * Chat: [Keybase](https://keybase.io/team/grincoin), more instructions on how to join [here](https://grin.mw/community). -* Mailing list: join the [~Mimblewimble team](https://launchpad.net/~mimblewimble) and subscribe on Launchpad. -* Twitter for the Grin council: [@grincouncil](https://twitter.com/grincouncil) ## Getting Started From a6615611a9e04bfe537640da770e08be27c65bc1 Mon Sep 17 00:00:00 2001 From: wiesche <120273695+wiesche89@users.noreply.github.com> Date: Tue, 26 May 2026 11:32:33 +0200 Subject: [PATCH 02/12] merge https://github.com/mimblewimble/grin/pull/3808 (#3839) --- CONTRIBUTING.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40cd94e9..c310ff37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,10 +57,16 @@ The development team will be happy to help and guide you with any of these point When you are starting to contribute to grin, we really would appreciate if you come by one of the community channels. +If you run into problems while trying out Grin before contributing, ask in [grincoin#support](https://keybase.io/team/grincoin) on Keybase. Describe what you've done, what you want to do, and include logs using a paste service if needed. + * Join the development discussion on [Telegram](https://t.me/grindevelopment). -* Join the [grincoin team on Keybase](https://keybase.io/team/grincoin) for community and support channels. -* Join the discussion on the [Forum](https://forum.grin.mw). +* [Grin Governance Council Public](https://t.me/Grin_Governance) channel where biweekly meetings are held for governance and development issues. +* [Grin General Telegram channel](https://t.me/grinprivacy/1) for general community discussion. +* Please [join grincoin#general on Keybase](https://keybase.io/team/grincoin) to get a feeling for the community. +* See the developers' chat channel [grincoin#dev on Keybase](https://keybase.io/team/grincoin) if you have questions about source code files. + If you explain what you're looking at and what you want to do, we'll try to help you along the way. * See `docs/*.md` and the folder structure explanations, [the wiki](https://github.com/mimblewimble/docs/wiki) and the official [Grin documentation](https://docs.grin.mw/). +* Further information and discussions are in the [Forum](https://forum.grin.mw), the [website](https://grin.mw), the [mailing list](https://lists.launchpad.net/mimblewimble/), and news channels like [Reddit/grincoin](https://www.reddit.com/r/grincoin/). ## Testing From 90b153fafcae4583a77cd50ce9e79b40e71aae02 Mon Sep 17 00:00:00 2001 From: wiesche <120273695+wiesche89@users.noreply.github.com> Date: Tue, 26 May 2026 13:01:02 +0200 Subject: [PATCH 03/12] Clarifies the blinding factor range proof explanation and includes the remaining typo fixes (#3840) --- api/src/foreign.rs | 2 +- api/src/owner.rs | 2 +- doc/intro.md | 4 +++- doc/pow/pow.md | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/src/foreign.rs b/api/src/foreign.rs index cddfe783..f1f83c00 100644 --- a/api/src/foreign.rs +++ b/api/src/foreign.rs @@ -32,7 +32,7 @@ use crate::{rest::*, BlockListing}; use std::sync::Weak; /// Main interface into all node API functions. -/// Node APIs are split into two seperate blocks of functionality +/// Node APIs are split into two separate blocks of functionality /// called the ['Owner'](struct.Owner.html) and ['Foreign'](struct.Foreign.html) APIs /// /// Methods in this API are intended to be 'single use'. diff --git a/api/src/owner.rs b/api/src/owner.rs index b63609c2..2ec02c12 100644 --- a/api/src/owner.rs +++ b/api/src/owner.rs @@ -27,7 +27,7 @@ use std::net::SocketAddr; use std::sync::Weak; /// Main interface into all node API functions. -/// Node APIs are split into two seperate blocks of functionality +/// Node APIs are split into two separate blocks of functionality /// called the ['Owner'](struct.Owner.html) and ['Foreign'](struct.Foreign.html) APIs /// /// Methods in this API are intended to be 'single use'. diff --git a/doc/intro.md b/doc/intro.md index 9a05f9cb..a57763a7 100644 --- a/doc/intro.md +++ b/doc/intro.md @@ -252,7 +252,9 @@ which can be signed by the attacker because Carol's blinding factor cancels out This output (`(113 + 99)*G + 2*H`) requires that both the numbers 113 and 99 are known in order to be spent; the attacker would thus have successfully locked Carol's UTXO. The requirement for a range proof for the blinding factor prevents this -because the attacker doesn't know the number 113 and thus neither (113 + 99). A more detailed description of range proofs is further detailed in the [range proof paper](https://eprint.iacr.org/2017/1066.pdf). +because the attacker doesn't know the number 113 and thus neither (113 + 99). In other words, without knowing the private +key (blinding factor), the attacker would not know the value in the output and would not be able to produce a valid range proof for it. +A more detailed description of range proofs is further detailed in the [range proof paper](https://eprint.iacr.org/2017/1066.pdf). #### Putting It All Together diff --git a/doc/pow/pow.md b/doc/pow/pow.md index 52ef3607..3de4728d 100644 --- a/doc/pow/pow.md +++ b/doc/pow/pow.md @@ -113,10 +113,10 @@ Now, (hopefully) armed with a basic understanding of what the Cuckoo Cycle algor ## Mining in Grin -The Cuckoo Cycle outlined above forms the basis of Grin's mining process, however Grin uses two variantion of Cuckoo Cycle in tandem with several other systems to create a Proof-of-Work. +The Cuckoo Cycle outlined above forms the basis of Grin's mining process, however Grin uses two variations of Cuckoo Cycle in tandem with several other systems to create a Proof-of-Work. 1. for GPUs: Cuckaroo on 2^29 edges - * Tweaked every 6 months to maitain ASIC resistance. + * Tweaked every 6 months to maintain ASIC resistance. * 90% of rewards at launch, linearly decreasing to 0% in 2 years. * Variant of Cuckoo that enforces so-called ``mean'' mining. * Takes 5.5GB of memory (perhaps 4GB with slowdown). From 8688a98e4f7c4209a67729497f683d78a91818bc Mon Sep 17 00:00:00 2001 From: Noobvie <85051719+noobvie@users.noreply.github.com> Date: Wed, 27 May 2026 15:34:48 -0400 Subject: [PATCH 04/12] Change default IP address for P2PConfig to IPv6 (#3843) * Change default IP address for P2PConfig to IPv6 Feature request: Change default P2P listen host from 0.0.0.0 to :: for dual-stack IPv4+IPv6 support. As tested on several windows/linux machines and not found issue. Worked with ipv4 or ipv6 or dual ipv4+ipv6. * docs+config add p2p listen host options --------- Co-authored-by: Joerg --- config/src/comments.rs | 7 +++++-- p2p/src/types.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/src/comments.rs b/config/src/comments.rs index be3a5863..3b93ac6e 100644 --- a/config/src/comments.rs +++ b/config/src/comments.rs @@ -262,8 +262,11 @@ fn comments() -> HashMap { "host".to_string(), " #The interface on which to listen. -#0.0.0.0 will listen on all interfaces, allowing others to interact -#127.0.0.1 will listen on the local machine only +#:: will listen on all IPv6 interfaces and may also accept IPv4 depending on OS socket settings +#0.0.0.0 will listen on all IPv4 interfaces, allowing others to interact +#Set host to 0.0.0.0 if only IPv4 listening is desired +#127.0.0.1 will listen on the local machine only over IPv4 +#::1 will listen on the local machine only over IPv6 " .to_string(), ); diff --git a/p2p/src/types.rs b/p2p/src/types.rs index 76cc3fe7..8d0bdd1a 100644 --- a/p2p/src/types.rs +++ b/p2p/src/types.rs @@ -293,7 +293,7 @@ pub struct P2PConfig { /// Default address for peer-to-peer connections. impl Default for P2PConfig { fn default() -> P2PConfig { - let ipaddr = "0.0.0.0".parse().unwrap(); + let ipaddr = "::".parse().unwrap(); P2PConfig { host: ipaddr, port: 3414, From f23c94cac5d54dadf5c0ca591a5c06dd517ceb59 Mon Sep 17 00:00:00 2001 From: wiesche <120273695+wiesche89@users.noreply.github.com> Date: Thu, 28 May 2026 07:15:33 +0200 Subject: [PATCH 05/12] Snap addition (#3844) * add workflow * branches: - snap_addition * Add snap package build workflow * Add snap package build master, staging * snap version from Cargo.toml * Seems like staging branch should have grade: devel quality level * Set snap grade based on branch * add multi arch * Update snap.yaml * snap: use arm platform for arm build * snap: multi platform ci build --------- Co-authored-by: ardocrat --- .devcontainer/Dockerfile | 39 +++++++++++++++++++++++----- .devcontainer/devcontainer.json | 3 ++- .github/workflows/snap.yaml | 45 +++++++++++++++++++++++++++++++++ .packaging/snaps/snapcraft.yaml | 40 +++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/snap.yaml create mode 100644 .packaging/snaps/snapcraft.yaml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8067909b..1d9c0a67 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,33 @@ -# Rust latest -FROM mcr.microsoft.com/devcontainers/rust:latest - -# Install Required Dependencies -RUN apt-get -qq update -RUN apt-get install -y build-essential cmake git libgit2-dev clang libncurses5-dev libncursesw5-dev zlib1g-dev pkg-config libssl-dev llvm +# Rust latest +FROM mcr.microsoft.com/devcontainers/rust:latest + +# Install Required Dependencies +RUN apt-get -qq update +RUN apt-get install -y \ + build-essential \ + clang \ + cmake \ + fuse \ + gh \ + git \ + libgit2-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libssl-dev \ + llvm \ + pkg-config \ + snapd \ + squashfuse \ + sudo \ + zlib1g-dev + +RUN echo '#!/bin/bash\n\ +/usr/lib/snapd/snapd &\n\ +sleep 5\n\ +exec "$@"' > /start-snapd.sh && \ + chmod +x /start-snapd.sh + +VOLUME ["/sys/fs/cgroup"] + +ENTRYPOINT ["/start-snapd.sh"] +CMD ["bash"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ba8003bd..fe05b995 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,5 +16,6 @@ "github.copilot-chat" ] } - } + }, + "runArgs": ["--privileged", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "-v", "/tmp:/tmp"] } \ No newline at end of file diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml new file mode 100644 index 00000000..273c12a7 --- /dev/null +++ b/.github/workflows/snap.yaml @@ -0,0 +1,45 @@ +name: Snap Package + +on: + workflow_dispatch: + push: + branches: + - master + - staging + +jobs: + build-snap: + name: Build snap + strategy: + matrix: + runs-on: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.runs-on }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Prepare snapcraft project + run: | + grade=devel + if [ "${{ github.ref_name }}" = "master" ]; then + grade=stable + fi + + mkdir -p snap + sed "s/SNAP_GRADE/$grade/" .packaging/snaps/snapcraft.yaml > snap/snapcraft.yaml + + - name: Build snap package + id: build + uses: snapcore/action-build@v1 + with: + path: . + + - name: Show snap package + run: find . -maxdepth 1 -name "*.snap" -ls + + - name: Upload snap artifact + uses: actions/upload-artifact@v6 + with: + name: grin-snap + path: "*.snap" + if-no-files-found: error diff --git a/.packaging/snaps/snapcraft.yaml b/.packaging/snaps/snapcraft.yaml new file mode 100644 index 00000000..ec1bfec4 --- /dev/null +++ b/.packaging/snaps/snapcraft.yaml @@ -0,0 +1,40 @@ +name: grin +adopt-info: grin +summary: Minimal implementation of the Mimblewimble protocol +description: | + https://grin.mw/ + +confinement: strict +grade: SNAP_GRADE +base: core22 +architectures: + - build-on: [arm64] + build-for: [arm64] + - build-on: [amd64] + build-for: [amd64] + +parts: + grin: + plugin: rust + source: . + override-pull: | + craftctl default + craftctl set version="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n 1)" + build-packages: + - build-essential + - clang + - cmake + - libncurses5-dev + - libncursesw5-dev + - libssl-dev + - llvm + - pkg-config + - zlib1g-dev + +apps: + grin: + environment: + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + command: bin/grin + plugs: [network, network-bind] From 7332c742d690484472c71bed5fd05bbb2f0c91f9 Mon Sep 17 00:00:00 2001 From: wiesche <120273695+wiesche89@users.noreply.github.com> Date: Thu, 28 May 2026 16:39:38 +0200 Subject: [PATCH 06/12] add (#3846) mainnet.fountainoffairfortune.it testnet.fountainoffairfortune.it Tested the newly added seed. Handshakes are succeeding, will merge --- servers/src/grin/seed.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/servers/src/grin/seed.rs b/servers/src/grin/seed.rs index 3d5fe8d8..aa1e8d44 100644 --- a/servers/src/grin/seed.rs +++ b/servers/src/grin/seed.rs @@ -35,18 +35,20 @@ use crate::util::StopState; /// DNS Seeds with contacts associated - Mainnet pub const MAINNET_DNS_SEEDS: &[&str] = &[ - "mainnet-seed.grinnode.live", // info@grinnode.live - "grincoin.org", // xmpp:aglkm@conversations.im - "main.gri.mw", // admin@gri.mw - "mainnet.grinffindor.org", // support@grinffindor.org - "main-seed.grin.money", // support@grinily.com + "mainnet-seed.grinnode.live", // info@grinnode.live + "grincoin.org", // xmpp:aglkm@conversations.im + "main.gri.mw", // admin@gri.mw + "mainnet.grinffindor.org", // support@grinffindor.org + "main-seed.grin.money", // support@grinily.com + "mainnet.fountainoffairfortune.it", // support@fountainoffairfortune.it ]; /// DNS Seeds with contacts associated - Testnet pub const TESTNET_DNS_SEEDS: &[&str] = &[ - "testnet.grincoin.org", // xmpp:aglkm@conversations.im - "test.gri.mw", // admin@gri.mw - "testnet.grinffindor.org", // support@grinffindor.org - "test-seed.grin.money", // support@grinily.com + "testnet.grincoin.org", // xmpp:aglkm@conversations.im + "test.gri.mw", // admin@gri.mw + "testnet.grinffindor.org", // support@grinffindor.org + "test-seed.grin.money", // support@grinily.com + "testnet.fountainoffairfortune.it", // support@fountainoffairfortune.it ]; pub fn connect_and_monitor( From 151e74c860dd875465066ff2a30bfc6fd5a347fe Mon Sep 17 00:00:00 2001 From: ardocrat Date: Mon, 1 Jun 2026 14:47:29 +0300 Subject: [PATCH 07/12] tui: show server initialization status and error (#3836) * tui: show server initialization status and error * fix: compilation issues * fix: add documenting to status, remove unused imports * fix: do not empty server value * fix: server ref * tui: do not quit on q when another dialog is showing (progress or error) * fix: server stop on tui shutdown * fix: stop server if tui was stopped after start * server: panic on error at non-tui mode like before with unwrap * fix: pop dialog * fix: do not return result on tx after server start * tui: close current dialog before quit * tui: pass stop state to server creation after tui quit * tui: exit code 1 after error, also for non-tui * tui: better exit code --- servers/src/common/types.rs | 16 ++++- servers/src/grin/server.rs | 31 ++++++---- src/bin/cmd/server.rs | 73 +++++++++++++--------- src/bin/tui/ui.rs | 119 ++++++++++++++++++++++++++++-------- 4 files changed, 174 insertions(+), 65 deletions(-) diff --git a/servers/src/common/types.rs b/servers/src/common/types.rs index ea83fecf..a428a5eb 100644 --- a/servers/src/common/types.rs +++ b/servers/src/common/types.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use chrono::prelude::Utc; use rand::prelude::*; -use crate::api; use crate::chain; use crate::core::global::{ChainTypes, DEFAULT_FUTURE_TIME_LIMIT}; use crate::core::{core, libtx, pow}; @@ -28,6 +27,7 @@ use crate::p2p; use crate::pool; use crate::pool::types::DandelionConfig; use crate::store; +use crate::{api, Server}; /// Error type wrapping underlying module errors. #[derive(Debug)] @@ -405,3 +405,17 @@ impl DandelionEpoch { self.relay_peer.clone() } } + +/// Server initialization status. +pub enum ServerInitStatus { + /// Database loading. + LoadDatabase, + /// P2P server initialization. + StartSync, + /// API server initialization. + StartAPI, + /// Server instance after successful initialization. + FinishedLoading(Server), + /// Error on initialization. + ErrorLoading(Error), +} diff --git a/servers/src/grin/server.rs b/servers/src/grin/server.rs index d3510fe2..f492ed17 100644 --- a/servers/src/grin/server.rs +++ b/servers/src/grin/server.rs @@ -39,7 +39,7 @@ use crate::common::hooks::{init_chain_hooks, init_net_hooks}; use crate::common::stats::{ ChainStats, DiffBlock, DiffStats, PeerStats, ServerStateInfo, ServerStats, TxStats, }; -use crate::common::types::{Error, ServerConfig, StratumServerConfig}; +use crate::common::types::{Error, ServerConfig, ServerInitStatus, StratumServerConfig}; use crate::core::core::hash::{Hashed, ZERO_HASH}; use crate::core::ser::ProtocolVersion; use crate::core::{consensus, genesis, global, pow}; @@ -52,7 +52,6 @@ use crate::pool; use crate::util::file::get_first_line; use crate::util::{RwLock, StopState}; use futures::channel::oneshot; -use grin_util::logger::LogEntry; /// Arcified thread-safe TransactionPool with type parameters used by server components pub type ServerTxPool = Arc>>; @@ -84,20 +83,16 @@ impl Server { /// Instantiates and starts a new server. Optionally takes a callback /// for the server to send an ARC copy of itself, to allow another process /// to poll info about the server status - pub fn start( + pub fn start( config: ServerConfig, - logs_rx: Option>, - mut info_callback: F, stop_state: Option>, + server_tx: Option>, api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), - ) -> Result<(), Error> - where - F: FnMut(Server, Option>), - { + ) -> Result { let mining_config = config.stratum_mining_config.clone(); let enable_test_miner = config.run_test_miner; let test_miner_wallet_url = config.test_miner_wallet_url.clone(); - let serv = Server::new(config, stop_state, api_chan)?; + let serv = Server::new(config, stop_state, server_tx, api_chan)?; if let Some(c) = mining_config { let enable_stratum_server = c.enable_stratum_server; @@ -118,8 +113,7 @@ impl Server { } } - info_callback(serv, logs_rx); - Ok(()) + Ok(serv) } // Exclusive (advisory) lock_file to ensure we do not run multiple @@ -151,6 +145,7 @@ impl Server { pub fn new( config: ServerConfig, stop_state: Option>, + server_tx: Option>, api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), ) -> Result { // Obtain our lock_file or fail immediately with an error. @@ -193,6 +188,10 @@ impl Server { info!("Starting server, genesis block: {}", genesis.hash()); + if let Some(ref server_tx) = server_tx { + let _ = server_tx.send(ServerInitStatus::LoadDatabase); + } + let shared_chain = Arc::new(chain::Chain::init( config.db_root.clone(), chain_adapter.clone(), @@ -220,6 +219,10 @@ impl Server { }; debug!("Capabilities: {:?}", capabilities); + if let Some(ref server_tx) = server_tx { + let _ = server_tx.send(ServerInitStatus::StartSync); + } + let p2p_server = Arc::new(p2p::Server::new( &config.db_root, capabilities, @@ -265,6 +268,10 @@ impl Server { } })?; + if let Some(ref server_tx) = server_tx { + let _ = server_tx.send(ServerInitStatus::StartAPI); + } + info!("Starting rest apis at: {}", &config.api_http_addr); let api_secret = get_first_line(config.api_secret_path.clone()); let foreign_api_secret = get_first_line(config.foreign_api_secret_path.clone()); diff --git a/src/bin/cmd/server.rs b/src/bin/cmd/server.rs index 6eb60f9f..fbca7d47 100644 --- a/src/bin/cmd/server.rs +++ b/src/bin/cmd/server.rs @@ -28,7 +28,10 @@ use crate::tui::ui; use futures::channel::oneshot; use grin_p2p::msg::PeerAddrs; use grin_p2p::PeerAddr; +use grin_servers::common::types::ServerInitStatus; +use grin_servers::Server; use grin_util::logger::LogEntry; +use grin_util::StopState; use std::sync::mpsc; /// wrap below to allow UI to clean up on stop @@ -37,38 +40,50 @@ pub fn start_server( logs_rx: Option>, api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), ) { - start_server_tui(config, logs_rx, api_chan); - exit(0); + exit(start_server_tui(config, logs_rx, api_chan)); } fn start_server_tui( config: servers::ServerConfig, logs_rx: Option>, api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), -) { - // Run the UI controller.. here for now for simplicity to access - // everything it might need +) -> i32 { if config.run_tui.unwrap_or(false) { warn!("Starting GRIN in UI mode..."); - servers::Server::start( - config, - logs_rx, - |serv: servers::Server, logs_rx: Option>| { - let mut controller = ui::Controller::new(logs_rx.unwrap()).unwrap_or_else(|e| { - panic!("Error loading UI controller: {}", e); - }); - controller.run(serv); - }, - None, - api_chan, - ) - .unwrap(); + // Run the UI controller. + let (serv_tx, serv_rx) = mpsc::channel::(); + let mut controller = ui::Controller::new(logs_rx, serv_rx).unwrap_or_else(|e| { + panic!("Error loading UI controller: {}", e); + }); + let serv_tx_clone = serv_tx.clone(); + let stop_state = Arc::new(StopState::new()); + let stop_state_clone = stop_state.clone(); + thread::spawn(move || { + match Server::start( + config, + Some(stop_state_clone.clone()), + Some(serv_tx_clone.clone()), + api_chan, + ) { + Ok(s) => { + if stop_state_clone.is_stopped() { + s.stop(); + return; + } + let _ = serv_tx_clone.send(ServerInitStatus::FinishedLoading(s)); + } + Err(e) => { + let _ = serv_tx_clone.send(ServerInitStatus::ErrorLoading(e)); + } + } + }); + let exit_code = controller.run(); + stop_state.stop(); + exit_code } else { warn!("Starting GRIN w/o UI..."); - servers::Server::start( - config, - logs_rx, - |serv: servers::Server, _: Option>| { + match Server::start(config, None, None, api_chan) { + Ok(s) => { let running = Arc::new(AtomicBool::new(true)); let r = running.clone(); ctrlc::set_handler(move || { @@ -79,12 +94,14 @@ fn start_server_tui( thread::sleep(Duration::from_secs(1)); } warn!("Received SIGINT (Ctrl+C) or SIGTERM (kill)."); - serv.stop(); - }, - None, - api_chan, - ) - .unwrap(); + s.stop(); + 0 + } + Err(e) => { + error!("Error starting GRIN: {:?}", e); + 1 + } + } } } diff --git a/src/bin/tui/ui.rs b/src/bin/tui/ui.rs index 3021ebdf..e28876dc 100644 --- a/src/bin/tui/ui.rs +++ b/src/bin/tui/ui.rs @@ -15,6 +15,12 @@ //! Basic TUI to better output the overall system status and status //! of various subsystems +use super::constants::MAIN_MENU; +use crate::built_info; +use crate::servers::Server; +use crate::tui::constants::{ROOT_STACK, VIEW_BASIC_STATUS, VIEW_MINING, VIEW_PEER_SYNC}; +use crate::tui::types::{TUIStatusListener, UIMessage}; +use crate::tui::{logs, menu, mining, peers, status, version}; use chrono::prelude::Utc; use cursive::direction::Orientation; use cursive::theme::BaseColor::{Black, Blue, Cyan, White}; @@ -29,24 +35,20 @@ use cursive::views::{ CircularFocus, Dialog, LinearLayout, Panel, SelectView, StackView, TextView, ViewRef, }; use cursive::{CursiveRunnable, CursiveRunner}; -use std::sync::mpsc; -use std::{thread, time}; - -use super::constants::MAIN_MENU; -use crate::built_info; -use crate::servers::Server; -use crate::tui::constants::{ROOT_STACK, VIEW_BASIC_STATUS, VIEW_MINING, VIEW_PEER_SYNC}; -use crate::tui::types::{TUIStatusListener, UIMessage}; -use crate::tui::{logs, menu, mining, peers, status, version}; use grin_core::global; +use grin_servers::common::types::{Error, ServerInitStatus}; use grin_util::logger::LogEntry; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::{thread, time}; pub struct UI { cursive: CursiveRunner, ui_rx: mpsc::Receiver, ui_tx: mpsc::Sender, controller_tx: mpsc::Sender, - logs_rx: mpsc::Receiver, + logs_rx: Option>, + show_dialog: Arc, } fn modify_theme(theme: &mut Theme) { @@ -65,7 +67,7 @@ impl UI { /// Create a new UI pub fn new( controller_tx: mpsc::Sender, - logs_rx: mpsc::Receiver, + logs_rx: Option>, ) -> UI { let (ui_tx, ui_rx) = mpsc::channel::(); @@ -75,6 +77,7 @@ impl UI { ui_rx, controller_tx, logs_rx, + show_dialog: Arc::new(AtomicBool::new(false)), }; // Create UI objects, etc @@ -102,7 +105,7 @@ impl UI { built_info::PKG_VERSION, global::get_chain_type() ), - Color::Dark(BaseColor::Green), + Dark(BaseColor::Green), )); let main_layer = LinearLayout::new(Orientation::Vertical) @@ -117,17 +120,21 @@ impl UI { let mut theme = grin_ui.cursive.current_theme().clone(); modify_theme(&mut theme); grin_ui.cursive.set_theme(theme); + grin_ui.cursive.add_fullscreen_layer(main_layer); // Configure a callback (shutdown, for the first test) let controller_tx_clone = grin_ui.controller_tx.clone(); + let show_dialog_clone = grin_ui.show_dialog.clone(); grin_ui.cursive.add_global_callback('q', move |c| { + if show_dialog_clone.load(Ordering::Relaxed) { + c.pop_layer(); + } let content = StyledString::styled("Shutting down...", Color::Light(BaseColor::Yellow)); c.add_layer(CircularFocus::new(Dialog::around(TextView::new(content))).wrap_tab()); - controller_tx_clone - .send(ControllerMessage::Shutdown) - .unwrap(); + let _ = controller_tx_clone.send(ControllerMessage::Shutdown); }); + grin_ui.cursive.set_fps(3); grin_ui } @@ -139,8 +146,10 @@ impl UI { return false; } - while let Some(message) = self.logs_rx.try_iter().next() { - logs::TUILogsView::update(&mut self.cursive, message); + if let Some(logs_rx) = &self.logs_rx { + while let Some(message) = logs_rx.try_iter().next() { + logs::TUILogsView::update(&mut self.cursive, message); + } } // Process any pending UI messages @@ -174,6 +183,8 @@ impl UI { pub struct Controller { rx: mpsc::Receiver, ui: UI, + serv_rx: mpsc::Receiver, + server: Option, } pub enum ControllerMessage { @@ -182,39 +193,99 @@ pub enum ControllerMessage { impl Controller { /// Create a new controller - pub fn new(logs_rx: mpsc::Receiver) -> Result { + pub fn new( + logs_rx: Option>, + serv_rx: mpsc::Receiver, + ) -> Result { let (tx, rx) = mpsc::channel::(); Ok(Controller { rx, ui: UI::new(tx, logs_rx), + serv_rx, + server: None, }) } + /// Server initialization status. + pub fn init_status(&mut self, text: &str, pop: bool) { + if pop { + self.ui.cursive.pop_layer(); + } + let content = StyledString::styled(text, Color::Light(BaseColor::Green)); + self.ui + .cursive + .add_layer(CircularFocus::new(Dialog::around(TextView::new(content))).wrap_tab()); + self.ui.show_dialog.store(true, Ordering::Relaxed); + } + + /// Server initialization error. + pub fn init_error(&mut self, e: Error) { + let content = StyledString::styled(format!("{:?}", e), Color::Light(BaseColor::Red)); + self.ui.cursive.add_layer( + CircularFocus::new(Dialog::around(TextView::new(content)).button("Exit", |s| { + s.quit(); + })) + .wrap_tab(), + ); + self.ui.show_dialog.store(true, Ordering::Relaxed); + } + + /// Server UI after initialization. + pub fn server(&mut self, server: &Server) { + if let Ok(stats) = server.get_server_stats() { + self.ui.ui_tx.send(UIMessage::UpdateStatus(stats)).unwrap(); + } + } + /// Run the controller - pub fn run(&mut self, server: Server) { + pub fn run(&mut self) -> i32 { + self.init_status("Starting server...", false); + let stat_update_interval = 1; let mut next_stat_update = Utc::now().timestamp() + stat_update_interval; let delay = time::Duration::from_millis(50); + let mut exit_code = 0; while self.ui.step() { if let Some(message) = self.rx.try_iter().next() { match message { ControllerMessage::Shutdown => { warn!("Shutdown in progress, please wait"); self.ui.stop(); - server.stop(); - return; + if let Some(s) = self.server.take() { + s.stop(); + } + return exit_code; + } + } + } + + if let Some(m) = self.serv_rx.try_iter().next() { + match m { + ServerInitStatus::LoadDatabase => self.init_status("Loading database...", true), + ServerInitStatus::StartSync => self.init_status("Start syncing...", true), + ServerInitStatus::StartAPI => self.init_status("Starting API...", true), + ServerInitStatus::FinishedLoading(s) => { + self.ui.cursive.pop_layer(); + self.ui.show_dialog.store(false, Ordering::Relaxed); + self.server = Some(s) + } + ServerInitStatus::ErrorLoading(e) => { + exit_code = 1; + self.init_error(e); } } } if Utc::now().timestamp() > next_stat_update { next_stat_update = Utc::now().timestamp() + stat_update_interval; - if let Ok(stats) = server.get_server_stats() { - self.ui.ui_tx.send(UIMessage::UpdateStatus(stats)).unwrap(); + if let Some(server) = &self.server { + if let Ok(stats) = server.get_server_stats() { + self.ui.ui_tx.send(UIMessage::UpdateStatus(stats)).unwrap(); + } } } thread::sleep(delay); } - server.stop(); + exit_code } } From ebcf7feb28dcd7db007c38c91ef9b2d3b40b0076 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Mon, 1 Jun 2026 17:29:53 +0300 Subject: [PATCH 08/12] lmdb: migration progress --- chain/src/chain.rs | 5 +++-- chain/src/store.rs | 8 ++++++-- p2p/src/store.rs | 1 + servers/src/common/types.rs | 2 ++ servers/src/grin/server.rs | 20 ++++++++++++++++++-- src/bin/cmd/server.rs | 15 ++++----------- src/bin/tui/ui.rs | 10 +++++++--- store/src/lmdb.rs | 28 +++++++++++++++++++++++++--- 8 files changed, 66 insertions(+), 23 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 11057398..7a55d226 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -43,7 +43,7 @@ use std::collections::HashMap; use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant}; /// Orphan pool size is limited by MAX_ORPHAN_SIZE @@ -171,8 +171,9 @@ impl Chain { genesis: Block, pow_verifier: fn(&BlockHeader) -> Result<(), pow::Error>, archive_mode: bool, + db_migration_prog_tx: Option>, ) -> Result { - let store = Arc::new(store::ChainStore::new(&db_root)?); + let store = Arc::new(store::ChainStore::new(&db_root, db_migration_prog_tx)?); // open the txhashset, creating a new one if necessary let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None)?; diff --git a/chain/src/store.rs b/chain/src/store.rs index d8df65f2..dff11dde 100644 --- a/chain/src/store.rs +++ b/chain/src/store.rs @@ -28,7 +28,7 @@ use grin_core::ser; use grin_store as store; use grin_store::{option_to_not_found, Error}; use std::convert::TryInto; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; const STORE_SUBPATH: &str = "chain"; @@ -67,13 +67,17 @@ pub struct ChainStore { impl ChainStore { /// Create new chain store - pub fn new(db_root: &str) -> Result { + pub fn new( + db_root: &str, + db_migration_prog_tx: Option>, + ) -> Result { let db = store::Store::new( db_root, None, Some(STORE_SUBPATH), DB_PREFIXES.to_vec(), None, + db_migration_prog_tx, )?; Ok(ChainStore { db }) } diff --git a/p2p/src/store.rs b/p2p/src/store.rs index 39018ad5..675d31b8 100644 --- a/p2p/src/store.rs +++ b/p2p/src/store.rs @@ -124,6 +124,7 @@ impl PeerStore { Some(STORE_SUBPATH), vec![PEER_PREFIX], None, + None, )?; Ok(PeerStore { db }) } diff --git a/servers/src/common/types.rs b/servers/src/common/types.rs index a428a5eb..508f2610 100644 --- a/servers/src/common/types.rs +++ b/servers/src/common/types.rs @@ -410,6 +410,8 @@ impl DandelionEpoch { pub enum ServerInitStatus { /// Database loading. LoadDatabase, + /// Database migration progress. + DBMigrationProgress(i8), /// P2P server initialization. StartSync, /// API server initialization. diff --git a/servers/src/grin/server.rs b/servers/src/grin/server.rs index f492ed17..d95ca66d 100644 --- a/servers/src/grin/server.rs +++ b/servers/src/grin/server.rs @@ -16,6 +16,7 @@ //! the peer-to-peer server, the blockchain and the transaction pool) and acts //! as a facade. +use fs2::FileExt; use std::fs::File; use std::io::prelude::*; use std::path::Path; @@ -25,8 +26,6 @@ use std::{ thread::{self, JoinHandle}, time::{self, Duration}, }; - -use fs2::FileExt; use walkdir::WalkDir; use crate::api; @@ -192,12 +191,29 @@ impl Server { let _ = server_tx.send(ServerInitStatus::LoadDatabase); } + let (db_migration_prog_tx, db_migration_prog_rx) = mpsc::channel::(); + if let Some(ref server_tx) = server_tx { + let server_tx = server_tx.clone(); + thread::spawn(move || loop { + match db_migration_prog_rx.recv() { + Ok(p) => { + if p == 100 { + break; + } + let _ = server_tx.send(ServerInitStatus::DBMigrationProgress(p)); + } + Err(_) => break, + } + }); + } + let shared_chain = Arc::new(chain::Chain::init( config.db_root.clone(), chain_adapter.clone(), genesis.clone(), pow::verify_size, archive_mode, + Some(db_migration_prog_tx), )?); pool_adapter.set_chain(shared_chain.clone()); diff --git a/src/bin/cmd/server.rs b/src/bin/cmd/server.rs index fbca7d47..a511d30c 100644 --- a/src/bin/cmd/server.rs +++ b/src/bin/cmd/server.rs @@ -34,21 +34,13 @@ use grin_util::logger::LogEntry; use grin_util::StopState; use std::sync::mpsc; -/// wrap below to allow UI to clean up on stop +/// Start node server at TUI or non-TUI mode. pub fn start_server( config: servers::ServerConfig, logs_rx: Option>, api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), ) { - exit(start_server_tui(config, logs_rx, api_chan)); -} - -fn start_server_tui( - config: servers::ServerConfig, - logs_rx: Option>, - api_chan: &'static mut (oneshot::Sender<()>, oneshot::Receiver<()>), -) -> i32 { - if config.run_tui.unwrap_or(false) { + let exit_code = if config.run_tui.unwrap_or(false) { warn!("Starting GRIN in UI mode..."); // Run the UI controller. let (serv_tx, serv_rx) = mpsc::channel::(); @@ -102,7 +94,8 @@ fn start_server_tui( 1 } } - } + }; + exit(exit_code); } /// Handles the server part of the command line, mostly running, starting and diff --git a/src/bin/tui/ui.rs b/src/bin/tui/ui.rs index e28876dc..5bdb9fec 100644 --- a/src/bin/tui/ui.rs +++ b/src/bin/tui/ui.rs @@ -247,16 +247,16 @@ impl Controller { let mut exit_code = 0; while self.ui.step() { if let Some(message) = self.rx.try_iter().next() { - match message { + return match message { ControllerMessage::Shutdown => { warn!("Shutdown in progress, please wait"); self.ui.stop(); if let Some(s) = self.server.take() { s.stop(); } - return exit_code; + exit_code } - } + }; } if let Some(m) = self.serv_rx.try_iter().next() { @@ -273,6 +273,10 @@ impl Controller { exit_code = 1; self.init_error(e); } + ServerInitStatus::DBMigrationProgress(p) => { + let status = format!("Migrating database: {}%, please wait...", p); + self.init_status(status.as_str(), true); + } } } diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 9e8fd4cc..b12a2191 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -19,7 +19,7 @@ use heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, WithoutTls}; use std::collections::HashMap; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{mpsc, Arc, OnceLock}; use std::time::Duration; use std::{fs, thread}; @@ -159,6 +159,7 @@ impl Store { db_name: Option<&str>, prefixes: Vec, max_readers: Option, + db_migration_prog_tx: Option>, ) -> Result { let full_path = Path::new(root_path) .join(DEFAULT_MULTI_DB_ENV_NAME) @@ -251,7 +252,8 @@ impl Store { if env_name != DEFAULT_MULTI_DB_ENV_NAME { let migrate_from = Path::new(root_path).join(env_name); if migrate_from.exists() { - match s.migrate_to_default_env(db_name, &migrate_from) { + let _ = s.clear(); + match s.migrate_to_default_env(db_name, &migrate_from, db_migration_prog_tx) { Ok(_) => match fs::remove_dir_all(&migrate_from) { Ok(_) => {} Err(e) => { @@ -283,8 +285,14 @@ impl Store { &self, from_name: Option<&str>, from_path: &Path, + db_migration_prog_tx: Option>, ) -> Result<(), Error> { info!("Migrating DB {:?}, please wait...", from_path); + + if let Some(migration_prog_tx) = &db_migration_prog_tx { + let _ = migration_prog_tx.send(0i8); + } + let from_env = unsafe { let mut options = EnvOpenOptions::new().read_txn_without_tls(); let env_options = options.map_size(self.alloc_chunk_size).max_dbs(24); @@ -307,7 +315,16 @@ impl Store { let mut write_to = self.env.write_txn()?; let read_from = from_env.read_txn()?; let mut count = 0; - for kv in db_from.iter(&read_from)? { + let total = db_from.iter(&read_from)?.count(); + let mut prev_prog = 0; + for (index, kv) in db_from.iter(&read_from)?.enumerate() { + if let Some(migration_prog_tx) = &db_migration_prog_tx { + let prog = 100 * index / total; + if prev_prog != prog && prog != 100 { + prev_prog = prog; + let _ = migration_prog_tx.send(prog as i8); + } + } let (k, v) = kv?; if k.len() > 1 && k[1] == PREFIX_KEY_SEPARATOR { let db_name = k.split_at(1).0; @@ -324,6 +341,11 @@ impl Store { } } write_to.commit()?; + + if let Some(migration_prog_tx) = &db_migration_prog_tx { + let _ = migration_prog_tx.send(100i8); + } + info!("Migrated {} records from {:?}", count, from_path); Ok(()) } From 650df0f3b7973c4460fc8a8d79fb650834a9ace5 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Mon, 1 Jun 2026 19:36:38 +0300 Subject: [PATCH 09/12] fix: tests --- chain/tests/chain_test_helper.rs | 1 + chain/tests/mine_simple_chain.rs | 2 ++ chain/tests/store_kernel_pos_index.rs | 10 +++++----- chain/tests/test_coinbase_maturity.rs | 1 + chain/tests/test_header_perf.rs | 2 ++ chain/tests/test_pibd_copy.rs | 2 ++ chain/tests/test_pibd_validation.rs | 1 + chain/tests/test_txhashset.rs | 2 +- pool/tests/common.rs | 1 + store/tests/lmdb.rs | 9 +++++---- 10 files changed, 21 insertions(+), 10 deletions(-) diff --git a/chain/tests/chain_test_helper.rs b/chain/tests/chain_test_helper.rs index 2776414f..0b82a56b 100644 --- a/chain/tests/chain_test_helper.rs +++ b/chain/tests/chain_test_helper.rs @@ -42,6 +42,7 @@ pub fn init_chain(dir_name: &str, genesis: Block) -> Chain { genesis, pow::verify_size, false, + None, ) .unwrap() } diff --git a/chain/tests/mine_simple_chain.rs b/chain/tests/mine_simple_chain.rs index 257a9dce..abf92baf 100644 --- a/chain/tests/mine_simple_chain.rs +++ b/chain/tests/mine_simple_chain.rs @@ -67,6 +67,7 @@ fn setup_with_status_adapter(dir_name: &str, genesis: Block, adapter: Arc Result<(), Error> { let commit = Commitment::from_vec(vec![]); let commit2 = Commitment::from_vec(vec![1]); - let store = ChainStore::new(chain_dir)?; + let store = ChainStore::new(chain_dir, None)?; let index = store::nrd_recent_kernel_index(); // Add a couple of single entries to the index and commit the batch. diff --git a/chain/tests/test_coinbase_maturity.rs b/chain/tests/test_coinbase_maturity.rs index 48d1f3a6..17e7dc64 100644 --- a/chain/tests/test_coinbase_maturity.rs +++ b/chain/tests/test_coinbase_maturity.rs @@ -47,6 +47,7 @@ fn test_coinbase_maturity() { genesis_block, pow::verify_size, false, + None, ) .unwrap(); diff --git a/chain/tests/test_header_perf.rs b/chain/tests/test_header_perf.rs index f948606b..ec34c256 100644 --- a/chain/tests/test_header_perf.rs +++ b/chain/tests/test_header_perf.rs @@ -50,6 +50,7 @@ fn test_header_perf_impl(is_test_chain: bool, src_root_dir: &str, dest_root_dir: genesis.clone(), pow::verify_size, false, + None, ) .unwrap(), ); @@ -62,6 +63,7 @@ fn test_header_perf_impl(is_test_chain: bool, src_root_dir: &str, dest_root_dir: genesis.clone(), pow::verify_size, false, + None, ) .unwrap(), ); diff --git a/chain/tests/test_pibd_copy.rs b/chain/tests/test_pibd_copy.rs index 9af63583..06b7965e 100644 --- a/chain/tests/test_pibd_copy.rs +++ b/chain/tests/test_pibd_copy.rs @@ -74,6 +74,7 @@ impl SegmenterResponder { genesis, pow::verify_size, false, + None, ) .unwrap(), ), @@ -134,6 +135,7 @@ impl DesegmenterRequestor { genesis, pow::verify_size, false, + None, ) .unwrap(), ), diff --git a/chain/tests/test_pibd_validation.rs b/chain/tests/test_pibd_validation.rs index 5f67b07a..7cc7deff 100644 --- a/chain/tests/test_pibd_validation.rs +++ b/chain/tests/test_pibd_validation.rs @@ -52,6 +52,7 @@ fn test_pibd_chain_validation_impl(is_test_chain: bool, src_root_dir: &str) { genesis.clone(), pow::verify_size, false, + None, ) .unwrap(), ); diff --git a/chain/tests/test_txhashset.rs b/chain/tests/test_txhashset.rs index 36b23e84..af045cda 100644 --- a/chain/tests/test_txhashset.rs +++ b/chain/tests/test_txhashset.rs @@ -38,7 +38,7 @@ fn test_unexpected_zip() { let db_root = format!(".grin_txhashset_zip"); clean_output_dir(&db_root); { - let chain_store = ChainStore::new(&db_root).unwrap(); + let chain_store = ChainStore::new(&db_root, None).unwrap(); let store = Arc::new(chain_store); txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap(); let head = BlockHeader::default(); diff --git a/pool/tests/common.rs b/pool/tests/common.rs index c3f3ab91..55a230d1 100644 --- a/pool/tests/common.rs +++ b/pool/tests/common.rs @@ -55,6 +55,7 @@ pub fn init_chain(dir_name: &str, genesis: Block) -> Chain { genesis, pow::verify_size, false, + None, ) .unwrap() } diff --git a/store/tests/lmdb.rs b/store/tests/lmdb.rs index 7aebb528..ea70d428 100644 --- a/store/tests/lmdb.rs +++ b/store/tests/lmdb.rs @@ -79,7 +79,7 @@ fn test_exists() -> Result<(), store::Error> { setup(test_dir); let prefix = b'P'; - let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None)?; + let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None, None)?; let key = [0, 0, 0, 1]; let value = [1, 1, 1, 1]; @@ -109,7 +109,7 @@ fn test_iter() -> Result<(), store::Error> { setup(test_dir); let prefix = b'P'; - let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None)?; + let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None, None)?; let key = [0, 0, 0, 1]; let value = [1, 1, 1, 1]; @@ -148,7 +148,7 @@ fn lmdb_allocate() -> Result<(), store::Error> { // Allocate more than the initial chunk, ensuring // the DB resizes underneath { - let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None)?; + let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None, None)?; for i in 0..WRITE_CHUNK_SIZE * 2 { println!("Allocating chunk: {}", i); @@ -164,7 +164,7 @@ fn lmdb_allocate() -> Result<(), store::Error> { println!("***********************************"); // Open env again and keep adding { - let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None)?; + let store = store::Store::new(test_dir, Some("test1"), None, vec![prefix], None, None)?; for i in 0..WRITE_CHUNK_SIZE * 2 { println!("Allocating chunk: {}", i); let chunk = PhatChunkStruct::new(); @@ -253,6 +253,7 @@ fn test_migration() -> Result<(), store::Error> { Some(DEFAULT_ENV_NAME), vec![test_prefix_1, test_prefix_2], None, + None, )?; // Check we can see key value. From d2ead628ee1ef0a51c0228d57e23529943042a71 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Tue, 2 Jun 2026 01:25:06 +0300 Subject: [PATCH 10/12] lmdb: immediately set resizing flag, ignore resizing flag while there are more than 0 opened txs to avoid stuck, optimize tx counter for some operations --- store/src/lmdb.rs | 139 +++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index b12a2191..6d9c9f52 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -389,7 +389,7 @@ impl Store { /// Wait while database is resizing. fn wait_for_resize(&self) { loop { - if !ENV_MAP + if ENV_MAP .get() .unwrap() .read() @@ -397,11 +397,13 @@ impl Store { .unwrap() .resizing .load(Ordering::Relaxed) + && self.open_txs_count() == 0 { - break; + debug!("Wait on resizing DB {}", self.env_path); + thread::sleep(Duration::from_millis(100)); + continue; } - trace!("Wait on resizing DB {}", self.env_path); - thread::sleep(Duration::from_millis(100)); + break; } } @@ -421,9 +423,16 @@ impl Store { let env_path = self.env_path.clone(); let env = self.env.clone(); + { + let mut w_env_map = ENV_MAP.get().unwrap().write(); + let env_state = w_env_map.get_mut(&env_path).unwrap(); + env_state.resizing.store(true, Ordering::Relaxed); + } + // Resize immediately or at another thread to not interrupt current // transaction waiting all open transactions to be closed. if self.open_txs_count() != 0 { + debug!("Waiting txs to be closed before DB {} resize", env_path); thread::spawn(move || { loop { let txs_count = ENV_MAP @@ -436,19 +445,9 @@ impl Store { .load(Ordering::Relaxed); if txs_count == 0 { debug!("Start resizing DB {}", env_path); - ENV_MAP - .get() - .unwrap() - .write() - .get_mut(&env_path) - .unwrap() - .resizing - .store(true, Ordering::Relaxed); - // Wait to make sure there are no more active txs left. - thread::sleep(Duration::from_millis(1000)); break; } - thread::sleep(Duration::from_millis(10)); + thread::sleep(Duration::from_millis(100)); } unsafe { @@ -465,17 +464,15 @@ impl Store { }); return; } else { - let mut w_env_map = ENV_MAP.get().unwrap().write(); - let env_state = w_env_map.get_mut(&env_path).unwrap(); - debug!("Start immediate resizing DB {}", env_path); - env_state.resizing.store(true, Ordering::Relaxed); unsafe { match env.resize(new_size) { Ok(_) => debug!("End resizing DB {}", env_path), Err(e) => error!("Resize DB {} error: {:?}", env_path, e), } } + let mut w_env_map = ENV_MAP.get().unwrap().write(); + let env_state = w_env_map.get_mut(&env_path).unwrap(); env_state.resizing.store(false, Ordering::Relaxed); } } @@ -599,27 +596,27 @@ impl Store { self.wait_for_resize(); TxCounter::on_change_tx_count(&self.env_path, true); - match self.env.clone().static_read_txn() { - Ok(read) => { - let db_res = self.get_db(db_key); - match db_res { - Ok(db) => Ok(DatabaseIterator::new( - self, - Arc::new(db.clone()), - read, - deserialize, - )), - Err(e) => { - TxCounter::on_change_tx_count(&self.env_path, false); - Err(Error::from(e)) + let res = { + match self.env.clone().static_read_txn() { + Ok(read) => { + let db_res = self.get_db(db_key); + match db_res { + Ok(db) => Ok(DatabaseIterator::new( + self, + Arc::new(db.clone()), + read, + deserialize, + )), + Err(e) => Err(Error::from(e)), } } + Err(e) => Err(Error::from(e)), } - Err(e) => { - TxCounter::on_change_tx_count(&self.env_path, false); - Err(Error::from(e)) - } + }; + if res.is_err() { + TxCounter::on_change_tx_count(&self.env_path, false); } + res } /// Builds a new batch to be used with this store. @@ -627,13 +624,11 @@ impl Store { self.maybe_resize(); TxCounter::on_change_tx_count(&self.env_path, true); - match Batch::new(self) { - Ok(batch) => Ok(batch), - Err(e) => { - TxCounter::on_change_tx_count(&self.env_path, false); - Err(e) - } + let res = { Batch::new(self) }; + if res.is_err() { + TxCounter::on_change_tx_count(&self.env_path, false); } + res } } @@ -763,28 +758,27 @@ impl<'a> Batch<'a> { self.store.wait_for_resize(); TxCounter::on_change_tx_count(&self.store.env_path, true); - let read = self.write.nested_read_txn(); - match read { - Ok(read) => { - let db_res = self.store.get_db(db_key); - match db_res { - Ok(db) => Ok(DatabaseIterator::new( - self.store, - Arc::new(db.clone()), - read, - deserialize, - )), - Err(e) => { - TxCounter::on_change_tx_count(&self.store.env_path, false); - Err(Error::from(e)) + let res = { + match self.write.nested_read_txn() { + Ok(read) => { + let db_res = self.store.get_db(db_key); + match db_res { + Ok(db) => Ok(DatabaseIterator::new( + self.store, + Arc::new(db.clone()), + read, + deserialize, + )), + Err(e) => Err(Error::from(e)), } } + Err(e) => Err(Error::from(e)), } - Err(e) => { - TxCounter::on_change_tx_count(&self.store.env_path, false); - Err(Error::from(e)) - } + }; + if res.is_err() { + TxCounter::on_change_tx_count(&self.store.env_path, false); } + res } /// Gets a `Readable` value from the database by provided key and deserialization strategy. @@ -823,19 +817,22 @@ impl<'a> Batch<'a> { /// commit, abandoned otherwise. pub fn child(&mut self) -> Result, Error> { TxCounter::on_change_tx_count(&self.store.env_path, true); - match self.store.env.nested_write_txn(&mut self.write) { - Ok(write) => Ok(Batch { - store: self.store, - write, - tx_counter: TxCounter { - env_path: self.store.env_path.clone(), - }, - }), - Err(e) => { - TxCounter::on_change_tx_count(&self.store.env_path, false); - Err(Error::from(e)) + let res = { + match self.store.env.nested_write_txn(&mut self.write) { + Ok(write) => Ok(Batch { + store: self.store, + write, + tx_counter: TxCounter { + env_path: self.store.env_path.clone(), + }, + }), + Err(e) => Err(Error::from(e)), } + }; + if res.is_err() { + TxCounter::on_change_tx_count(&self.store.env_path, false); } + res } } From 2c3a067629632f1377117116d2935ec4c41d02b5 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Tue, 2 Jun 2026 11:14:57 +0300 Subject: [PATCH 11/12] lmdb: key for successful migration --- store/src/lmdb.rs | 55 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 6d9c9f52..7496564e 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -87,6 +87,8 @@ const DEFAULT_DB_VERSION: ProtocolVersion = ProtocolVersion(3); pub const DEFAULT_ENV_NAME: &'static str = "lmdb"; /// Default multi-database environment without prefixes. const DEFAULT_MULTI_DB_ENV_NAME: &'static str = "multi_lmdb"; +/// Migration completion marker in the default database. +const MIGRATION_COMPLETE_KEY: &[u8] = b"__grin_migration_complete"; /// Prefix key separator. pub const PREFIX_KEY_SEPARATOR: u8 = b':'; @@ -252,26 +254,42 @@ impl Store { if env_name != DEFAULT_MULTI_DB_ENV_NAME { let migrate_from = Path::new(root_path).join(env_name); if migrate_from.exists() { - let _ = s.clear(); - match s.migrate_to_default_env(db_name, &migrate_from, db_migration_prog_tx) { - Ok(_) => match fs::remove_dir_all(&migrate_from) { - Ok(_) => {} + let delete_old_db_file = || -> Result<(), Error> { + match fs::remove_dir_all(&migrate_from) { + Ok(_) => Ok(()), Err(e) => { return Err(Error::FileErr(format!( "Can not remove old DB file: {:?}", e ))); } - }, - Err(e) => { - error!("DB {} migration error: {:?}", env_name, e); - match s.clear() { - Ok(_) => {} - Err(e) => { - error!("Can not clear new DB after unsuccessful migration: {:?}", e) + } + }; + if s.migration_complete()? { + if let Err(e) = delete_old_db_file() { + return Err(e); + } + } else { + let _ = s.clear(); + match s.migrate_to_default_env(db_name, &migrate_from, db_migration_prog_tx) { + Ok(_) => { + if let Err(e) = delete_old_db_file() { + return Err(e); } } - return Err(e); + Err(e) => { + error!("DB {} migration error: {:?}", env_name, e); + match s.clear() { + Ok(_) => {} + Err(e) => { + error!( + "Can not clear new DB after unsuccessful migration: {:?}", + e + ) + } + } + return Err(e); + } } } } @@ -280,6 +298,18 @@ impl Store { Ok(s) } + /// Check if migration has already completed successfully. + fn migration_complete(&self) -> Result { + let read = self.env.read_txn()?; + Ok(self.def_db.get(&read, MIGRATION_COMPLETE_KEY)?.is_some()) + } + + /// Mark migration as successfully completed. + fn set_migration_complete(&self, write: &mut RwTxn<'_>) -> Result<(), Error> { + self.def_db.put(write, MIGRATION_COMPLETE_KEY, b"1")?; + Ok(()) + } + /// Migrate database from provided path to default environment. fn migrate_to_default_env( &self, @@ -340,6 +370,7 @@ impl Store { count += 1; } } + self.set_migration_complete(&mut write_to)?; write_to.commit()?; if let Some(migration_prog_tx) = &db_migration_prog_tx { From 0dc4280b68101c1ad1add59c13471f94ac43e8f4 Mon Sep 17 00:00:00 2001 From: ardocrat Date: Tue, 2 Jun 2026 12:31:56 +0300 Subject: [PATCH 12/12] lmdb: fix put database creation at separate block to avoid lifetime issues when returning an error on migration --- store/src/lmdb.rs | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/store/src/lmdb.rs b/store/src/lmdb.rs index 7496564e..b31844c3 100644 --- a/store/src/lmdb.rs +++ b/store/src/lmdb.rs @@ -228,25 +228,28 @@ impl Store { } // Database setup. - let r_env_map = env_map.read(); - let env = r_env_map.get(&full_path).unwrap().env.clone(); - let mut write = env.write_txn()?; - let def_name = db_name.unwrap_or(DEFAULT_ENV_NAME); - let def_db = env.create_database(&mut write, Some(def_name))?; - let mut dbs_map = HashMap::>::new(); - for p in prefixes { - let db = env.create_database(&mut write, Some(p.to_string().as_str()))?; - dbs_map.insert(p, db); - } - write.commit()?; + let s = { + let r_env_map = env_map.read(); + let env = r_env_map.get(&full_path).unwrap().env.clone(); + let mut write = env.write_txn()?; + let def_name = db_name.unwrap_or(DEFAULT_ENV_NAME); + let def_db = env.create_database(&mut write, Some(def_name))?; + let mut dbs_map = HashMap::>::new(); + for p in prefixes { + let db = env.create_database(&mut write, Some(p.to_string().as_str()))?; + dbs_map.insert(p, db); + } + write.commit()?; - let s = Store { - env: env.clone(), - env_path: full_path.clone(), - pre_dbs: Arc::new(dbs_map), - def_db, - version: DEFAULT_DB_VERSION, - alloc_chunk_size, + let s = Store { + env: env.clone(), + env_path: full_path.clone(), + pre_dbs: Arc::new(dbs_map), + def_db, + version: DEFAULT_DB_VERSION, + alloc_chunk_size, + }; + s }; // Migrate to default environment if needed.