Add more data in ephemera blocks

* Commit local peer before querying contract

* Default to anyonline

* Remove string from template

* Fix avg computing

* Use updated qa env

* Fix clippy

* Add unit tests for ephemera contract

* Upload ephemera contract in CI

* Add group check for peer signup

* Peer registration unit test

.github/workflows/build-and-upload-binaries-ci.yml

@@ -109,6 +109,7 @@ jobs:
           cp contracts/target/wasm32-unknown-unknown/release/cw4_group.wasm $OUTPUT_DIR
           cp contracts/target/wasm32-unknown-unknown/release/nym_service_provider_directory.wasm $OUTPUT_DIR
           cp contracts/target/wasm32-unknown-unknown/release/nym_name_service.wasm $OUTPUT_DIR
+          cp contracts/target/wasm32-unknown-unknown/release/nym_ephemera.wasm $OUTPUT_DIR
 
       - name: Deploy branch to CI www
         continue-on-error: true

Cargo.toml

@@ -34,6 +34,7 @@ members = [
     "common/cosmwasm-smart-contracts/coconut-bandwidth-contract",
     "common/cosmwasm-smart-contracts/coconut-dkg",
     "common/cosmwasm-smart-contracts/contracts-common",
+    "common/cosmwasm-smart-contracts/ephemera",
     "common/cosmwasm-smart-contracts/group-contract",
     "common/cosmwasm-smart-contracts/mixnet-contract",
     "common/cosmwasm-smart-contracts/multisig-contract",

clients/webassembly/Cargo.lock

@@ -2562,6 +2562,17 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "nym-ephemera-common"
+version = "0.1.0"
+dependencies = [
+ "cosmwasm-std",
+ "cw-utils",
+ "nym-contracts-common",
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "nym-gateway-client"
 version = "0.1.0"
@@ -2965,6 +2976,7 @@ dependencies = [
  "nym-coconut-interface",
  "nym-config",
  "nym-contracts-common",
+ "nym-ephemera-common",
  "nym-group-contract-common",
  "nym-mixnet-contract-common",
  "nym-multisig-contract-common",

common/client-libs/validator-client/Cargo.toml

@@ -13,6 +13,7 @@ colored = "2.0"
 
 nym-coconut-dkg-common = { path = "../../cosmwasm-smart-contracts/coconut-dkg" }
 nym-contracts-common = { path = "../../cosmwasm-smart-contracts/contracts-common" }
+nym-ephemera-common = { path = "../../cosmwasm-smart-contracts/ephemera" }
 nym-mixnet-contract-common = { path = "../../cosmwasm-smart-contracts/mixnet-contract" }
 nym-vesting-contract-common = { path = "../../cosmwasm-smart-contracts/vesting-contract" }
 nym-coconut-bandwidth-contract-common = { path = "../../cosmwasm-smart-contracts/coconut-bandwidth-contract" }

common/client-libs/validator-client/src/client.rs

@@ -17,7 +17,7 @@ pub use nym_mixnet_contract_common::{
 use url::Url;
 
 #[cfg(feature = "nyxd-client")]
-use crate::nyxd::traits::{DkgQueryClient, MixnetQueryClient};
+use crate::nyxd::traits::{DkgQueryClient, EphemeraQueryClient, MixnetQueryClient};
 #[cfg(feature = "nyxd-client")]
 use crate::nyxd::{self, CosmWasmClient, NyxdClient, QueryNyxdClient, SigningNyxdClient};
 #[cfg(feature = "nyxd-client")]
@@ -29,6 +29,8 @@ use nym_coconut_dkg_common::{types::EpochId, verification_key::ContractVKShare};
 #[cfg(feature = "nyxd-client")]
 use nym_coconut_interface::Base58;
 #[cfg(feature = "nyxd-client")]
+use nym_ephemera_common::types::JsonPeerInfo;
+#[cfg(feature = "nyxd-client")]
 use nym_mixnet_contract_common::{
     families::{Family, FamilyHead},
     mixnode::MixNodeBond,
@@ -569,6 +571,26 @@ impl<C> Client<C> {
 
         Ok(events)
     }
+
+    pub async fn get_all_ephemera_peers(&self) -> Result<Vec<JsonPeerInfo>, ValidatorClientError>
+    where
+        C: CosmWasmClient + Sync + Send,
+    {
+        let mut peers = Vec::new();
+        let mut start_after = None;
+        loop {
+            let mut paged_response = self.get_peers_paged(start_after.take(), None).await?;
+            peers.append(&mut paged_response.peers);
+
+            if let Some(start_after_res) = paged_response.start_next_after {
+                start_after = Some(start_after_res.to_string())
+            } else {
+                break;
+            }
+        }
+
+        Ok(peers)
+    }
 }
 
 // validator-api wrappers

common/client-libs/validator-client/src/nyxd/mod.rs

@@ -67,6 +67,7 @@ pub struct Config {
     pub(crate) group_contract_address: Option<AccountId>,
     pub(crate) multisig_contract_address: Option<AccountId>,
     pub(crate) coconut_dkg_contract_address: Option<AccountId>,
+    pub(crate) ephemera_contract_address: Option<AccountId>,
     pub(crate) service_provider_contract_address: Option<AccountId>,
     pub(crate) name_service_contract_address: Option<AccountId>,
     // TODO: add this in later commits
@@ -133,6 +134,10 @@ impl Config {
                 details.contracts.coconut_dkg_contract_address.as_ref(),
                 prefix,
             )?,
+            ephemera_contract_address: Self::parse_optional_account(
+                details.contracts.ephemera_contract_address.as_ref(),
+                prefix,
+            )?,
             service_provider_contract_address: Self::parse_optional_account(
                 details
                     .contracts
@@ -263,6 +268,10 @@ impl<C> NyxdClient<C> {
         self.config.service_provider_contract_address = Some(address);
     }
 
+    pub fn set_ephemera_contract_address(&mut self, address: AccountId) {
+        self.config.ephemera_contract_address = Some(address);
+    }
+
     // TODO: this should get changed into Result<&AccountId, NyxdError> (or Option<&AccountId> in future commits
     // note: what unwrap is doing here is just moving a failure that would have normally
     // occurred in `connect` when attempting to parse an empty address,
@@ -321,6 +330,14 @@ impl<C> NyxdClient<C> {
         self.config.coconut_dkg_contract_address.as_ref().unwrap()
     }
 
+    // TODO: this should get changed into Result<&AccountId, NyxdError> (or Option<&AccountId> in future commits
+    // note: what unwrap is doing here is just moving a failure that would have normally
+    // occurred in `connect` when attempting to parse an empty address,
+    // so it's not introducing new source of failure (just moves it)
+    pub fn ephemera_contract_address(&self) -> &AccountId {
+        self.config.ephemera_contract_address.as_ref().unwrap()
+    }
+
     // The service provider directory contract is optional, so we return an Option not a Result
     pub fn service_provider_contract_address(&self) -> Option<&AccountId> {
         self.config.service_provider_contract_address.as_ref()

common/client-libs/validator-client/src/nyxd/traits/ephemera_query_client.rs

@@ -0,0 +1,53 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::nyxd::error::NyxdError;
+use crate::nyxd::{CosmWasmClient, NyxdClient};
+use async_trait::async_trait;
+use nym_ephemera_common::msg::QueryMsg as EphemeraQueryMsg;
+use nym_ephemera_common::peers::PagedPeerResponse;
+use serde::Deserialize;
+
+#[async_trait]
+pub trait EphemeraQueryClient {
+    async fn query_ephemera_contract<T>(&self, query: EphemeraQueryMsg) -> Result<T, NyxdError>
+    where
+        for<'a> T: Deserialize<'a>;
+
+    async fn get_peers_paged(
+        &self,
+        start_after: Option<String>,
+        limit: Option<u32>,
+    ) -> Result<PagedPeerResponse, NyxdError> {
+        let request = EphemeraQueryMsg::GetPeers { start_after, limit };
+        self.query_ephemera_contract(request).await
+    }
+}
+
+#[async_trait]
+impl<C> EphemeraQueryClient for NyxdClient<C>
+where
+    C: CosmWasmClient + Send + Sync,
+{
+    async fn query_ephemera_contract<T>(&self, query: EphemeraQueryMsg) -> Result<T, NyxdError>
+    where
+        for<'a> T: Deserialize<'a>,
+    {
+        self.client
+            .query_contract_smart(self.ephemera_contract_address(), &query)
+            .await
+    }
+}
+
+#[async_trait]
+impl<C> EphemeraQueryClient for crate::Client<C>
+where
+    C: CosmWasmClient + Sync + Send,
+{
+    async fn query_ephemera_contract<T>(&self, query: EphemeraQueryMsg) -> Result<T, NyxdError>
+    where
+        for<'a> T: Deserialize<'a>,
+    {
+        self.nyxd.query_ephemera_contract(query).await
+    }
+}

common/client-libs/validator-client/src/nyxd/traits/ephemera_signing_client.rs

@@ -0,0 +1,43 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::nyxd::cosmwasm_client::types::ExecuteResult;
+use crate::nyxd::error::NyxdError;
+use crate::nyxd::{Fee, NyxdClient, SigningCosmWasmClient};
+use async_trait::async_trait;
+use nym_ephemera_common::msg::ExecuteMsg as EphemeraExecuteMsg;
+use nym_ephemera_common::types::JsonPeerInfo;
+
+#[async_trait]
+pub trait EphemeraSigningClient {
+    async fn register_as_peer(
+        &self,
+        peer_info: JsonPeerInfo,
+        fee: Option<Fee>,
+    ) -> Result<ExecuteResult, NyxdError>;
+}
+
+#[async_trait]
+impl<C> EphemeraSigningClient for NyxdClient<C>
+where
+    C: SigningCosmWasmClient + Send + Sync,
+{
+    async fn register_as_peer(
+        &self,
+        peer_info: JsonPeerInfo,
+        fee: Option<Fee>,
+    ) -> Result<ExecuteResult, NyxdError> {
+        let req = EphemeraExecuteMsg::RegisterPeer { peer_info };
+
+        self.client
+            .execute(
+                self.address(),
+                self.ephemera_contract_address(),
+                &req,
+                fee.unwrap_or_default(),
+                format!("registering {} as an ephemera peer", self.address()),
+                vec![],
+            )
+            .await
+    }
+}

common/client-libs/validator-client/src/nyxd/traits/mod.rs

@@ -5,6 +5,8 @@
 
 mod coconut_bandwidth_query_client;
 mod dkg_query_client;
+mod ephemera_query_client;
+mod ephemera_signing_client;
 mod group_query_client;
 mod mixnet_query_client;
 mod multisig_query_client;
@@ -24,6 +26,7 @@ mod name_service_signing_client;
 
 pub use coconut_bandwidth_query_client::CoconutBandwidthQueryClient;
 pub use dkg_query_client::DkgQueryClient;
+pub use ephemera_query_client::EphemeraQueryClient;
 pub use group_query_client::GroupQueryClient;
 pub use mixnet_query_client::MixnetQueryClient;
 pub use multisig_query_client::MultisigQueryClient;
@@ -33,6 +36,7 @@ pub use vesting_query_client::VestingQueryClient;
 
 pub use coconut_bandwidth_signing_client::CoconutBandwidthSigningClient;
 pub use dkg_signing_client::DkgSigningClient;
+pub use ephemera_signing_client::EphemeraSigningClient;
 pub use mixnet_signing_client::MixnetSigningClient;
 pub use multisig_signing_client::MultisigSigningClient;
 pub use name_service_signing_client::NameServiceSigningClient;

common/config/src/lib.rs

@@ -17,6 +17,7 @@ pub mod helpers;
 pub mod legacy_helpers;
 
 pub const NYM_DIR: &str = ".nym";
+pub const DEFAULT_NYM_APIS_DIR: &str = "nym-api";
 pub const DEFAULT_CONFIG_DIR: &str = "config";
 pub const DEFAULT_DATA_DIR: &str = "data";
 pub const DEFAULT_CONFIG_FILENAME: &str = "config.toml";

common/cosmwasm-smart-contracts/ephemera/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "nym-ephemera-common"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+cosmwasm-std = { workspace = true }
+cw-utils = { workspace = true }
+schemars = "0.8"
+serde = { version = "1.0.103", default-features = false, features = ["derive"] }
+
+contracts-common = { path = "../contracts-common", package = "nym-contracts-common" }

common/cosmwasm-smart-contracts/ephemera/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod msg;
+pub mod peers;
+pub mod types;

common/cosmwasm-smart-contracts/ephemera/src/msg.rs

@@ -0,0 +1,31 @@
+// Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::types::JsonPeerInfo;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
+pub struct InstantiateMsg {
+    pub group_addr: String,
+    pub mix_denom: String,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum ExecuteMsg {
+    RegisterPeer { peer_info: JsonPeerInfo },
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum QueryMsg {
+    GetPeers {
+        limit: Option<u32>,
+        start_after: Option<String>,
+    },
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub struct MigrateMsg {}

common/cosmwasm-smart-contracts/ephemera/src/peers.rs

@@ -0,0 +1,24 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::types::JsonPeerInfo;
+use cosmwasm_std::Addr;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct PagedPeerResponse {
+    pub peers: Vec<JsonPeerInfo>,
+    pub per_page: usize,
+    pub start_next_after: Option<Addr>,
+}
+
+impl PagedPeerResponse {
+    pub fn new(peers: Vec<JsonPeerInfo>, per_page: usize, start_next_after: Option<Addr>) -> Self {
+        PagedPeerResponse {
+            peers,
+            per_page,
+            start_next_after,
+        }
+    }
+}

common/cosmwasm-smart-contracts/ephemera/src/types.rs

@@ -0,0 +1,30 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use cosmwasm_std::Addr;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
+pub struct JsonPeerInfo {
+    /// The cosmos address of the peer, used in interacting with the chain.
+    pub cosmos_address: Addr,
+    /// The TCP/IP address of the peer.
+    /// Expected formats:
+    /// 1. `<IP>:<PORT>`
+    /// 2. `/ip4/<IP>/tcp/<PORT>` - this is the format used by libp2p multiaddr
+    pub ip_address: String,
+    ///Serialized public key.
+    pub public_key: String,
+}
+
+impl JsonPeerInfo {
+    #[must_use]
+    pub fn new(cosmos_address: Addr, ip_address: String, public_key: String) -> Self {
+        Self {
+            cosmos_address,
+            ip_address,
+            public_key,
+        }
+    }
+}

common/crypto/Cargo.toml

@@ -27,7 +27,7 @@ thiserror = "1.0.37"
 zeroize = { workspace = true, optional = true, features = ["zeroize_derive"] }
 
 # internal
-nym-sphinx-types = { path = "../nymsphinx/types", version = "0.2.0" }
+nym-sphinx-types = { path = "../nymsphinx/types", version = "0.2.0", default-features = false }
 nym-pemstore = { path = "../../common/pemstore", version = "0.3.0" }
 
 [dev-dependencies]
@@ -38,3 +38,4 @@ serde = ["serde_crate", "serde_bytes", "ed25519-dalek/serde", "x25519-dalek/serd
 asymmetric = ["x25519-dalek", "ed25519-dalek", "zeroize"]
 hashing = ["blake3", "digest", "hkdf", "hmac", "generic-array"]
 symmetric = ["aes", "ctr", "cipher", "generic-array"]
+sphinx-only = ["nym-sphinx-types/sphinx-only"]

common/network-defaults/src/lib.rs

@@ -30,6 +30,7 @@ pub struct NymContracts {
     pub group_contract_address: Option<String>,
     pub multisig_contract_address: Option<String>,
     pub coconut_dkg_contract_address: Option<String>,
+    pub ephemera_contract_address: Option<String>,
     pub service_provider_directory_contract_address: Option<String>,
     pub name_service_contract_address: Option<String>,
 }
@@ -129,6 +130,9 @@ impl NymNetworkDetails {
             .with_coconut_dkg_contract(Some(
                 var(var_names::COCONUT_DKG_CONTRACT_ADDRESS).expect("coconut dkg contract not set"),
             ))
+            .with_ephemera_contract(Some(
+                var(var_names::EPHEMERA_CONTRACT_ADDRESS).expect("ephemera contract not set"),
+            ))
             .with_service_provider_directory_contract(get_optional_env(
                 var_names::SERVICE_PROVIDER_DIRECTORY_CONTRACT_ADDRESS,
             ))
@@ -162,6 +166,7 @@ impl NymNetworkDetails {
                 coconut_dkg_contract_address: parse_optional_str(
                     mainnet::COCONUT_DKG_CONTRACT_ADDRESS,
                 ),
+                ephemera_contract_address: parse_optional_str(mainnet::EPHEMERA_CONTRACT_ADDRESS),
                 service_provider_directory_contract_address: None,
                 name_service_contract_address: None,
             },
@@ -246,6 +251,12 @@ impl NymNetworkDetails {
         self
     }
 
+    #[must_use]
+    pub fn with_ephemera_contract<S: Into<String>>(mut self, contract: Option<S>) -> Self {
+        self.contracts.ephemera_contract_address = contract.map(Into::into);
+        self
+    }
+
     #[must_use]
     pub fn with_service_provider_directory_contract<S: Into<String>>(
         mut self,

common/network-defaults/src/mainnet.rs

@@ -20,6 +20,7 @@ pub(crate) const COCONUT_BANDWIDTH_CONTRACT_ADDRESS: &str =
 pub(crate) const GROUP_CONTRACT_ADDRESS: &str = "n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0";
 pub(crate) const MULTISIG_CONTRACT_ADDRESS: &str = "n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0";
 pub(crate) const COCONUT_DKG_CONTRACT_ADDRESS: &str = "n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0";
+pub(crate) const EPHEMERA_CONTRACT_ADDRESS: &str = "n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0";
 pub(crate) const REWARDING_VALIDATOR_ADDRESS: &str = "n10yyd98e2tuwu0f7ypz9dy3hhjw7v772q6287gy";
 
 pub const STATISTICS_SERVICE_DOMAIN_ADDRESS: &str = "https://mainnet-stats.nymte.ch:8090/";
@@ -87,6 +88,10 @@ pub fn export_to_env() {
         var_names::COCONUT_DKG_CONTRACT_ADDRESS,
         COCONUT_DKG_CONTRACT_ADDRESS,
     );
+    set_var_to_default(
+        var_names::EPHEMERA_CONTRACT_ADDRESS,
+        EPHEMERA_CONTRACT_ADDRESS,
+    );
     set_var_to_default(
         var_names::REWARDING_VALIDATOR_ADDRESS,
         REWARDING_VALIDATOR_ADDRESS,
@@ -132,6 +137,10 @@ pub fn export_to_env_if_not_set() {
         var_names::COCONUT_DKG_CONTRACT_ADDRESS,
         COCONUT_DKG_CONTRACT_ADDRESS,
     );
+    set_var_conditionally_to_default(
+        var_names::EPHEMERA_CONTRACT_ADDRESS,
+        EPHEMERA_CONTRACT_ADDRESS,
+    );
     set_var_conditionally_to_default(
         var_names::REWARDING_VALIDATOR_ADDRESS,
         REWARDING_VALIDATOR_ADDRESS,

common/network-defaults/src/var_names.rs

@@ -17,6 +17,7 @@ pub const COCONUT_BANDWIDTH_CONTRACT_ADDRESS: &str = "COCONUT_BANDWIDTH_CONTRACT
 pub const GROUP_CONTRACT_ADDRESS: &str = "GROUP_CONTRACT_ADDRESS";
 pub const MULTISIG_CONTRACT_ADDRESS: &str = "MULTISIG_CONTRACT_ADDRESS";
 pub const COCONUT_DKG_CONTRACT_ADDRESS: &str = "COCONUT_DKG_CONTRACT_ADDRESS";
+pub const EPHEMERA_CONTRACT_ADDRESS: &str = "EPHEMERA_CONTRACT_ADDRESS";
 pub const REWARDING_VALIDATOR_ADDRESS: &str = "REWARDING_VALIDATOR_ADDRESS";
 pub const STATISTICS_SERVICE_DOMAIN_ADDRESS: &str = "STATISTICS_SERVICE_DOMAIN_ADDRESS";
 pub const SERVICE_PROVIDER_DIRECTORY_CONTRACT_ADDRESS: &str =

common/nymsphinx/types/Cargo.toml

@@ -9,5 +9,9 @@ repository = { workspace = true }
 
 [dependencies]
 sphinx-packet = { version = "0.1.0" }
-nym-outfox = { path = "../../../nym-outfox" }
+nym-outfox = { path = "../../../nym-outfox", optional = true }
 thiserror = "1"
+
+[features]
+default = ["nym-outfox"]
+sphinx-only = []

common/nymsphinx/types/src/lib.rs

@@ -1,11 +1,13 @@
 // Copyright 2021 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0
 
+#[cfg(not(feature = "sphinx-only"))]
 pub use nym_outfox::{
     constants::MIN_PACKET_SIZE, constants::MIX_PARAMS_LEN, constants::OUTFOX_PACKET_OVERHEAD,
     error::OutfoxError,
 };
 // re-exporting types and constants available in sphinx
+#[cfg(not(feature = "sphinx-only"))]
 use nym_outfox::packet::{OutfoxPacket, OutfoxProcessedPacket};
 pub use sphinx_packet::{
     constants::{
@@ -30,6 +32,7 @@ pub enum NymPacketError {
     Sphinx(#[from] sphinx_packet::Error),
 
     #[error("Outfox error: {0}")]
+    #[cfg(not(feature = "sphinx-only"))]
     Outfox(#[from] nym_outfox::error::OutfoxError),
 
     #[error("{0}")]
@@ -39,11 +42,13 @@ pub enum NymPacketError {
 #[allow(clippy::large_enum_variant)]
 pub enum NymPacket {
     Sphinx(SphinxPacket),
+    #[cfg(not(feature = "sphinx-only"))]
     Outfox(OutfoxPacket),
 }
 
 pub enum NymProcessedPacket {
     Sphinx(ProcessedPacket),
+    #[cfg(not(feature = "sphinx-only"))]
     Outfox(OutfoxProcessedPacket),
 }
 
@@ -54,6 +59,7 @@ impl fmt::Debug for NymPacket {
                 .debug_struct("NymPacket::Sphinx")
                 .field("len", &packet.len())
                 .finish(),
+            #[cfg(not(feature = "sphinx-only"))]
             NymPacket::Outfox(packet) => f
                 .debug_struct("NymPacket::Outfox")
                 .field("len", &packet.len())
@@ -80,6 +86,7 @@ impl NymPacket {
         Ok(NymPacket::Sphinx(SphinxPacket::from_bytes(bytes)?))
     }
 
+    #[cfg(not(feature = "sphinx-only"))]
     pub fn outfox_build<M: AsRef<[u8]>>(
         payload: M,
         route: &[Node],
@@ -94,6 +101,7 @@ impl NymPacket {
         )?))
     }
 
+    #[cfg(not(feature = "sphinx-only"))]
     pub fn outfox_from_bytes(bytes: &[u8]) -> Result<NymPacket, NymPacketError> {
         Ok(NymPacket::Outfox(OutfoxPacket::try_from(bytes)?))
     }
@@ -101,6 +109,7 @@ impl NymPacket {
     pub fn len(&self) -> usize {
         match self {
             NymPacket::Sphinx(packet) => packet.len(),
+            #[cfg(not(feature = "sphinx-only"))]
             NymPacket::Outfox(packet) => packet.len(),
         }
     }
@@ -112,6 +121,7 @@ impl NymPacket {
     pub fn to_bytes(&self) -> Result<Vec<u8>, NymPacketError> {
         match self {
             NymPacket::Sphinx(packet) => Ok(packet.to_bytes()),
+            #[cfg(not(feature = "sphinx-only"))]
             NymPacket::Outfox(packet) => Ok(packet.to_bytes()?),
         }
     }
@@ -124,6 +134,7 @@ impl NymPacket {
             NymPacket::Sphinx(packet) => {
                 Ok(NymProcessedPacket::Sphinx(packet.process(node_secret_key)?))
             }
+            #[cfg(not(feature = "sphinx-only"))]
             NymPacket::Outfox(mut packet) => {
                 let next_address = packet.decode_next_layer(node_secret_key)?;
                 Ok(NymProcessedPacket::Outfox(OutfoxProcessedPacket::new(

contracts/Cargo.lock

@@ -2,16 +2,6 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "aead"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
-dependencies = [
- "crypto-common",
- "generic-array 0.14.6",
-]
-
 [[package]]
 name = "aes"
 version = "0.7.5"
@@ -19,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
 dependencies = [
  "cfg-if",
- "cipher 0.3.0",
+ "cipher",
  "cpufeatures",
  "ctr",
  "opaque-debug 0.3.0",
@@ -48,12 +38,6 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
 
-[[package]]
-name = "arrayvec"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -102,20 +86,6 @@ dependencies = [
  "opaque-debug 0.2.3",
 ]
 
-[[package]]
-name = "blake3"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5"
-dependencies = [
- "arrayref",
- "arrayvec",
- "cc",
- "cfg-if",
- "constant_time_eq",
- "digest 0.10.7",
-]
-
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -189,30 +159,6 @@ dependencies = [
  "keystream",
 ]
 
-[[package]]
-name = "chacha20"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
-dependencies = [
- "cfg-if",
- "cipher 0.4.4",
- "cpufeatures",
-]
-
-[[package]]
-name = "chacha20poly1305"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
-dependencies = [
- "aead",
- "chacha20",
- "cipher 0.4.4",
- "poly1305",
- "zeroize",
-]
-
 [[package]]
 name = "cipher"
 version = "0.3.0"
@@ -222,17 +168,6 @@ dependencies = [
  "generic-array 0.14.6",
 ]
 
-[[package]]
-name = "cipher"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
-dependencies = [
- "crypto-common",
- "inout",
- "zeroize",
-]
-
 [[package]]
 name = "coconut-test"
 version = "0.1.0"
@@ -264,12 +199,6 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913"
 
-[[package]]
-name = "constant_time_eq"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
-
 [[package]]
 name = "cosmwasm-crypto"
 version = "1.2.5"
@@ -355,49 +284,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
-dependencies = [
- "autocfg",
- "cfg-if",
- "crossbeam-utils",
- "memoffset",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
-
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -423,7 +309,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array 0.14.6",
- "rand_core 0.6.4",
  "typenum",
 ]
 
@@ -453,7 +338,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea"
 dependencies = [
- "cipher 0.3.0",
+ "cipher",
 ]
 
 [[package]]
@@ -1002,10 +887,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
- "js-sys",
  "libc",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "wasm-bindgen",
 ]
 
 [[package]]
@@ -1120,15 +1003,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "inout"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
-dependencies = [
- "generic-array 0.14.6",
-]
-
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -1275,15 +1149,6 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
-[[package]]
-name = "memoffset"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mixnet-vesting-integration-tests"
 version = "0.1.0"
@@ -1310,16 +1175,6 @@ dependencies = [
  "libm",
 ]
 
-[[package]]
-name = "num_cpus"
-version = "1.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
 [[package]]
 name = "nym-coconut-bandwidth"
 version = "0.1.0"
@@ -1403,6 +1258,37 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "nym-ephemera"
+version = "0.1.0"
+dependencies = [
+ "cosmwasm-std",
+ "cosmwasm-storage",
+ "cw-controllers",
+ "cw-multi-test",
+ "cw-storage-plus",
+ "cw4",
+ "cw4-group",
+ "lazy_static",
+ "nym-ephemera-common",
+ "nym-group-contract-common",
+ "rusty-fork",
+ "schemars",
+ "serde",
+ "thiserror",
+]
+
+[[package]]
+name = "nym-ephemera-common"
+version = "0.1.0"
+dependencies = [
+ "cosmwasm-std",
+ "cw-utils",
+ "nym-contracts-common",
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "nym-group-contract-common"
 version = "0.1.0"
@@ -1507,23 +1393,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "nym-outfox"
-version = "0.1.0"
-dependencies = [
- "blake3",
- "chacha20",
- "chacha20poly1305",
- "curve25519-dalek",
- "getrandom 0.2.10",
- "log",
- "rand 0.7.3",
- "rayon",
- "sphinx-packet",
- "thiserror",
- "zeroize",
-]
-
 [[package]]
 name = "nym-pemstore"
 version = "0.3.0"
@@ -1571,7 +1440,6 @@ dependencies = [
 name = "nym-sphinx-types"
 version = "0.2.0"
 dependencies = [
- "nym-outfox",
  "sphinx-packet",
  "thiserror",
 ]
@@ -1673,17 +1541,6 @@ version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
-[[package]]
-name = "poly1305"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
-dependencies = [
- "cpufeatures",
- "opaque-debug 0.3.0",
- "universal-hash",
-]
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1842,28 +1699,6 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
-[[package]]
-name = "rayon"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
-dependencies = [
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-utils",
- "num_cpus",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.3.5"
@@ -1996,12 +1831,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
 [[package]]
 name = "sec1"
 version = "0.3.0"
@@ -2333,16 +2162,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "universal-hash"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
-dependencies = [
- "crypto-common",
- "subtle 2.4.1",
-]
-
 [[package]]
 name = "url"
 version = "2.3.1"

contracts/Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "coconut-bandwidth",
     "coconut-dkg",
     "coconut-test",
+    "ephemera",
     "mixnet",
     "mixnet-vesting-integration-tests",
     "multisig/cw3-flex-multisig",

contracts/ephemera/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "nym-ephemera"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+nym-ephemera-common = { path = "../../common/cosmwasm-smart-contracts/ephemera" }
+
+cosmwasm-std = { workspace = true }
+cosmwasm-storage = { workspace = true }
+cw-storage-plus = { workspace = true }
+cw-controllers = { workspace = true }
+cw4 = { workspace = true }
+
+schemars = "0.8"
+serde = { version = "1.0.103", default-features = false, features = ["derive"] }
+thiserror = "1.0.23"
+
+[dev-dependencies]
+cw-multi-test = { workspace = true }
+cw4-group = { path = "../multisig/cw4-group" }
+nym-group-contract-common = { path = "../../common/cosmwasm-smart-contracts/group-contract" }
+lazy_static = "1.4"
+rusty-fork = "0.3"

contracts/ephemera/src/contract.rs

@@ -0,0 +1,97 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::ContractError;
+use crate::peers::queries::query_peers_paged;
+use crate::peers::transactions::try_register_peer;
+use crate::state::{State, STATE};
+use cosmwasm_std::{
+    entry_point, to_binary, Deps, DepsMut, Env, MessageInfo, QueryResponse, Response,
+};
+use cw4::Cw4Contract;
+use nym_ephemera_common::msg::{ExecuteMsg, InstantiateMsg, MigrateMsg, QueryMsg};
+
+/// Instantiate the contract.
+///
+/// `deps` contains Storage, API and Querier
+/// `env` contains block, message and contract info
+/// `msg` is the contract initialization message, sort of like a constructor call.
+#[entry_point]
+pub fn instantiate(
+    deps: DepsMut<'_>,
+    _env: Env,
+    _info: MessageInfo,
+    msg: InstantiateMsg,
+) -> Result<Response, ContractError> {
+    let group_addr = Cw4Contract(deps.api.addr_validate(&msg.group_addr).map_err(|_| {
+        ContractError::InvalidGroup {
+            addr: msg.group_addr.clone(),
+        }
+    })?);
+
+    let state = State {
+        group_addr,
+        mix_denom: msg.mix_denom,
+    };
+    STATE.save(deps.storage, &state)?;
+
+    Ok(Response::default())
+}
+
+/// Handle an incoming message
+#[entry_point]
+pub fn execute(
+    deps: DepsMut<'_>,
+    _env: Env,
+    info: MessageInfo,
+    msg: ExecuteMsg,
+) -> Result<Response, ContractError> {
+    match msg {
+        ExecuteMsg::RegisterPeer { peer_info } => try_register_peer(deps, info, peer_info),
+    }
+}
+
+#[entry_point]
+pub fn query(deps: Deps<'_>, _env: Env, msg: QueryMsg) -> Result<QueryResponse, ContractError> {
+    let response = match msg {
+        QueryMsg::GetPeers { limit, start_after } => {
+            to_binary(&query_peers_paged(deps, start_after, limit)?)?
+        }
+    };
+
+    Ok(response)
+}
+
+#[entry_point]
+pub fn migrate(_deps: DepsMut<'_>, _env: Env, _msg: MigrateMsg) -> Result<Response, ContractError> {
+    Ok(Default::default())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use cosmwasm_std::testing::{mock_dependencies, mock_env, mock_info};
+    use cosmwasm_std::Addr;
+
+    #[test]
+    fn initialize_contract() {
+        let mut deps = mock_dependencies();
+        let env = mock_env();
+
+        let init_msg = InstantiateMsg {
+            group_addr: "group_addr".to_string(),
+            mix_denom: "uatom".to_string(),
+        };
+
+        let sender = mock_info("sender", &[]);
+        let res = instantiate(deps.as_mut(), env, sender, init_msg);
+        assert!(res.is_ok());
+
+        let expected_state = State {
+            group_addr: Cw4Contract::new(Addr::unchecked("group_addr")),
+            mix_denom: "uatom".to_string(),
+        };
+        let state = STATE.load(deps.as_ref().storage).unwrap();
+        assert_eq!(state, expected_state);
+    }
+}

contracts/ephemera/src/error.rs

@@ -0,0 +1,21 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use cosmwasm_std::StdError;
+use thiserror::Error;
+
+/// Custom errors for contract failure conditions.
+#[derive(Error, Debug, PartialEq)]
+pub enum ContractError {
+    #[error(transparent)]
+    Std(#[from] StdError),
+
+    #[error("Group contract invalid address '{addr}'")]
+    InvalidGroup { addr: String },
+
+    #[error("This potential ephemera peer is not in the ephemera group")]
+    Unauthorized,
+
+    #[error("This sender is already registered")]
+    AlreadyRegistered,
+}

contracts/ephemera/src/lib.rs

@@ -0,0 +1,8 @@
+// Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod contract;
+pub mod error;
+mod peers;
+mod state;
+mod support;

contracts/ephemera/src/peers/mod.rs

@@ -0,0 +1,6 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod queries;
+pub mod storage;
+pub mod transactions;

contracts/ephemera/src/peers/queries.rs

@@ -0,0 +1,164 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::peers::storage::{PEERS, PEERS_PAGE_DEFAULT_LIMIT, PEERS_PAGE_MAX_LIMIT};
+use cosmwasm_std::{Deps, Order, StdResult};
+use cw_storage_plus::Bound;
+use nym_ephemera_common::peers::PagedPeerResponse;
+
+pub fn query_peers_paged(
+    deps: Deps<'_>,
+    start_after: Option<String>,
+    limit: Option<u32>,
+) -> StdResult<PagedPeerResponse> {
+    let limit = limit
+        .unwrap_or(PEERS_PAGE_DEFAULT_LIMIT)
+        .min(PEERS_PAGE_MAX_LIMIT) as usize;
+
+    let addr = start_after
+        .map(|addr| deps.api.addr_validate(&addr))
+        .transpose()?;
+
+    let start = addr.map(Bound::exclusive);
+
+    let peers = PEERS
+        .range(deps.storage, start, None, Order::Ascending)
+        .take(limit)
+        .map(|res| res.map(|item| item.1))
+        .collect::<StdResult<Vec<_>>>()?;
+
+    let start_next_after = peers
+        .last()
+        .map(|peer_info| peer_info.cosmos_address.clone());
+
+    Ok(PagedPeerResponse::new(peers, limit, start_next_after))
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use crate::peers::storage::{PEERS_PAGE_DEFAULT_LIMIT, PEERS_PAGE_MAX_LIMIT};
+    use crate::support::tests::fixtures::peer_fixture;
+    use crate::support::tests::helpers::init_contract;
+    use cosmwasm_std::DepsMut;
+
+    fn fill_peers(deps: DepsMut<'_>, size: usize) {
+        for n in 0..size {
+            let peer = peer_fixture(&format!("peer{}", n));
+            PEERS
+                .save(deps.storage, peer.cosmos_address.clone(), &peer)
+                .unwrap();
+        }
+    }
+
+    fn remove_peers(deps: DepsMut<'_>, size: usize) {
+        for n in 0..size {
+            let peer = peer_fixture(&format!("peer{}", n));
+            PEERS.remove(deps.storage, peer.cosmos_address);
+        }
+    }
+
+    #[test]
+    fn peers_empty_on_init() {
+        let deps = init_contract();
+
+        let page1 = query_peers_paged(deps.as_ref(), None, None).unwrap();
+        assert_eq!(0, page1.peers.len() as u32);
+    }
+
+    #[test]
+    fn peers_paged_retrieval_obeys_limits() {
+        let mut deps = init_contract();
+        let limit = 2;
+
+        fill_peers(deps.as_mut(), 1000);
+
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(limit)).unwrap();
+        assert_eq!(limit, page1.peers.len() as u32);
+
+        remove_peers(deps.as_mut(), 1000);
+    }
+
+    #[test]
+    fn peers_paged_retrieval_has_default_limit() {
+        let mut deps = init_contract();
+
+        fill_peers(deps.as_mut(), 1000);
+
+        // query without explicitly setting a limit
+        let page1 = query_peers_paged(deps.as_ref(), None, None).unwrap();
+
+        assert_eq!(PEERS_PAGE_DEFAULT_LIMIT, page1.peers.len() as u32);
+
+        remove_peers(deps.as_mut(), 1000);
+    }
+
+    #[test]
+    fn peers_paged_retrieval_has_max_limit() {
+        let mut deps = init_contract();
+
+        // query with a crazily high limit in an attempt to use too many resources
+        let crazy_limit = 1000 * PEERS_PAGE_MAX_LIMIT;
+
+        fill_peers(deps.as_mut(), 1000);
+
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(crazy_limit)).unwrap();
+
+        // we default to a decent sized upper bound instead
+        let expected_limit = PEERS_PAGE_MAX_LIMIT;
+        assert_eq!(expected_limit, page1.peers.len() as u32);
+
+        remove_peers(deps.as_mut(), 1000);
+    }
+
+    #[test]
+    fn peers_pagination_works() {
+        let mut deps = init_contract();
+
+        let per_page = 2;
+
+        fill_peers(deps.as_mut(), 1);
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(per_page)).unwrap();
+
+        // page should have 1 result on it
+        assert_eq!(1, page1.peers.len());
+        remove_peers(deps.as_mut(), 1);
+
+        fill_peers(deps.as_mut(), 2);
+        // page1 should have 2 results on it
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(per_page)).unwrap();
+        assert_eq!(2, page1.peers.len());
+        remove_peers(deps.as_mut(), 2);
+
+        fill_peers(deps.as_mut(), 3);
+        // page1 still has 2 results
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(per_page)).unwrap();
+        assert_eq!(2, page1.peers.len());
+
+        // retrieving the next page should start after the last key on this page
+        let start_after = page1.start_next_after.unwrap();
+        let page2 = query_peers_paged(
+            deps.as_ref(),
+            Option::from(start_after.to_string()),
+            Option::from(per_page),
+        )
+        .unwrap();
+
+        assert_eq!(1, page2.peers.len());
+        remove_peers(deps.as_mut(), 3);
+
+        fill_peers(deps.as_mut(), 4);
+        let page1 = query_peers_paged(deps.as_ref(), None, Option::from(per_page)).unwrap();
+        let start_after = page1.start_next_after.unwrap();
+        let page2 = query_peers_paged(
+            deps.as_ref(),
+            Option::from(start_after.to_string()),
+            Option::from(per_page),
+        )
+        .unwrap();
+
+        // now we have 2 pages, with 2 results on the second page
+        assert_eq!(2, page2.peers.len());
+        remove_peers(deps.as_mut(), 4);
+    }
+}

contracts/ephemera/src/peers/storage.rs

@@ -0,0 +1,11 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use cosmwasm_std::Addr;
+use cw_storage_plus::Map;
+use nym_ephemera_common::types::JsonPeerInfo;
+
+pub(crate) const PEERS_PAGE_MAX_LIMIT: u32 = 75;
+pub(crate) const PEERS_PAGE_DEFAULT_LIMIT: u32 = 50;
+
+pub(crate) const PEERS: Map<'_, Addr, JsonPeerInfo> = Map::new("prs");

contracts/ephemera/src/peers/transactions.rs

@@ -0,0 +1,63 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::error::ContractError;
+use crate::peers::storage::PEERS;
+use crate::state::STATE;
+use cosmwasm_std::{DepsMut, MessageInfo, Response};
+use nym_ephemera_common::types::JsonPeerInfo;
+
+pub fn try_register_peer(
+    deps: DepsMut<'_>,
+    info: MessageInfo,
+    peer_info: JsonPeerInfo,
+) -> Result<Response, ContractError> {
+    if PEERS.may_load(deps.storage, info.sender.clone())?.is_none() {
+        if STATE
+            .load(deps.storage)?
+            .group_addr
+            .is_voting_member(&deps.querier, &info.sender, None)?
+            .is_some()
+        {
+            PEERS.save(deps.storage, info.sender, &peer_info)?;
+            Ok(Default::default())
+        } else {
+            Err(ContractError::Unauthorized {})
+        }
+    } else {
+        Err(ContractError::AlreadyRegistered)
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use crate::support::tests::fixtures::peer_fixture;
+    use crate::support::tests::helpers;
+    use crate::support::tests::helpers::GROUP_MEMBERS;
+    use cosmwasm_std::testing::mock_info;
+    use cw4::Member;
+
+    #[test]
+    fn peer_registration() {
+        let mut deps = helpers::init_contract();
+        let peer_info = peer_fixture("owner");
+        let info = mock_info("owner", &[]);
+
+        let ret = try_register_peer(deps.as_mut(), info.clone(), peer_info.clone()).unwrap_err();
+        assert_eq!(ret, ContractError::Unauthorized);
+
+        GROUP_MEMBERS.lock().unwrap().push((
+            Member {
+                addr: "owner".to_string(),
+                weight: 10,
+            },
+            1,
+        ));
+
+        try_register_peer(deps.as_mut(), info.clone(), peer_info.clone()).unwrap();
+
+        let ret = try_register_peer(deps.as_mut(), info, peer_info).unwrap_err();
+        assert_eq!(ret, ContractError::AlreadyRegistered);
+    }
+}

contracts/ephemera/src/state/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use cw4::Cw4Contract;
+use cw_storage_plus::Item;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+// unique items
+pub const STATE: Item<State> = Item::new("state");
+
+#[derive(Serialize, Deserialize, Clone, PartialEq, JsonSchema, Debug)]
+pub struct State {
+    pub mix_denom: String,
+    pub group_addr: Cw4Contract,
+}

contracts/ephemera/src/support/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+#[cfg(test)]
+pub mod tests;

contracts/ephemera/src/support/tests/fixtures.rs

@@ -0,0 +1,15 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use cosmwasm_std::Addr;
+use nym_ephemera_common::types::JsonPeerInfo;
+
+pub const TEST_MIX_DENOM: &str = "unym";
+
+pub fn peer_fixture(cosmos_address: &str) -> JsonPeerInfo {
+    JsonPeerInfo {
+        cosmos_address: Addr::unchecked(cosmos_address),
+        ip_address: "127.0.0.1".to_string(),
+        public_key: "random_key".to_string(),
+    }
+}

contracts/ephemera/src/support/tests/helpers.rs

@@ -0,0 +1,74 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::contract::instantiate;
+use cosmwasm_std::testing::{mock_dependencies, mock_env, mock_info, MockApi, MockQuerier};
+use cosmwasm_std::{
+    from_binary, to_binary, ContractResult, Empty, MemoryStorage, OwnedDeps, QuerierResult,
+    SystemResult, WasmQuery,
+};
+use cw4::{Cw4QueryMsg, Member, MemberListResponse, MemberResponse};
+use lazy_static::lazy_static;
+use nym_ephemera_common::msg::InstantiateMsg;
+use std::sync::Mutex;
+
+use super::fixtures::TEST_MIX_DENOM;
+
+pub const ADMIN_ADDRESS: &str = "admin address";
+pub const GROUP_CONTRACT: &str = "group contract address";
+
+lazy_static! {
+    pub static ref GROUP_MEMBERS: Mutex<Vec<(Member, u64)>> = Mutex::new(vec![]);
+}
+
+fn querier_handler(query: &WasmQuery) -> QuerierResult {
+    let bin = match query {
+        WasmQuery::Smart { contract_addr, msg } => {
+            if contract_addr != GROUP_CONTRACT {
+                panic!("Not supported");
+            }
+            match from_binary(msg) {
+                Ok(Cw4QueryMsg::Member { addr, at_height }) => {
+                    let weight = GROUP_MEMBERS.lock().unwrap().iter().find_map(|(m, h)| {
+                        if m.addr == addr {
+                            if let Some(height) = at_height {
+                                if height != *h {
+                                    return None;
+                                }
+                            }
+                            Some(m.weight)
+                        } else {
+                            None
+                        }
+                    });
+                    to_binary(&MemberResponse { weight }).unwrap()
+                }
+                Ok(Cw4QueryMsg::ListMembers { .. }) => {
+                    let members = GROUP_MEMBERS
+                        .lock()
+                        .unwrap()
+                        .iter()
+                        .map(|m| m.0.clone())
+                        .collect();
+                    to_binary(&MemberListResponse { members }).unwrap()
+                }
+                _ => panic!("Not supported"),
+            }
+        }
+        _ => panic!("Not supported"),
+    };
+    SystemResult::Ok(ContractResult::Ok(bin))
+}
+
+pub fn init_contract() -> OwnedDeps<MemoryStorage, MockApi, MockQuerier<Empty>> {
+    let mut deps = mock_dependencies();
+    deps.querier.update_wasm(querier_handler);
+    let msg = InstantiateMsg {
+        group_addr: GROUP_CONTRACT.to_string(),
+        mix_denom: TEST_MIX_DENOM.to_string(),
+    };
+    let env = mock_env();
+    let info = mock_info(ADMIN_ADDRESS, &[]);
+    instantiate(deps.as_mut(), env, info, msg).unwrap();
+    deps
+}

contracts/ephemera/src/support/tests/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod fixtures;
+pub mod helpers;

contracts/mixnet-vesting-integration-tests/Cargo.toml

@@ -22,7 +22,7 @@ nym-mixnet-contract = { path = "../mixnet" }
 nym-vesting-contract = { path = "../vesting" }
 
 # other local dependencies
-nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand"] }
+nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand", "sphinx-only"] }
 
 # external dependencies
 rand_chacha = "0.2"

contracts/mixnet/Cargo.toml

@@ -42,7 +42,7 @@ semver = { version = "1.0.16", default-features = false }
 [dev-dependencies]
 cosmwasm-schema = { workspace = true }
 rand_chacha = "0.2"
-nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand"] }
+nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand", "sphinx-only"] }
 
 [build-dependencies]
 vergen = { version = "=7.4.3", default-features = false, features = ["build", "git", "rustc"] }

contracts/service-provider-directory/Cargo.toml

@@ -25,6 +25,6 @@ vergen = { version = "=7.4.3", default-features = false, features = ["build", "g
 [dev-dependencies]
 anyhow = "1.0.40"
 cw-multi-test = { workspace = true }
-nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand"] }
+nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "rand", "sphinx-only"] }
 rand_chacha = "0.2"
 rstest = "0.17.0"

envs/local.env

@@ -16,6 +16,7 @@ COCONUT_BANDWIDTH_CONTRACT_ADDRESS=n1lp5zex6685kd22agzskhqsylpnssxnweyuvsz4edr4p
 GROUP_CONTRACT_ADDRESS=n1rw8fw2mpcpzzq3jpa4e52ufawnmj5a4u68p35umvgskewuw0nlzsaa5w4m
 MULTISIG_CONTRACT_ADDRESS=n1rw8fw2mpcpzzq3jpa4e52ufawnmj5a4u68p35umvgskewuw0nlzsaa5w4m
 COCONUT_DKG_CONTRACT_ADDRESS=n1vwtgazgpancsfel04y7syc95ausmat47cjtldqzkdmx6phyrwx2qvkv32p
+EPHEMERA_CONTRACT_ADDRESS=n1vwtgazgpancsfel04y7syc95ausmat47cjtldqzkdmx6phyrwx2qvkv32p
 REWARDING_VALIDATOR_ADDRESS=n1tfzd4qz3a45u8p4mr5zmzv66457uwjgcl05jdq
 STATISTICS_SERVICE_DOMAIN_ADDRESS="http://0.0.0.0"
 NYXD="http://127.0.0.1:26657"

envs/mainnet.env

@@ -16,6 +16,7 @@ COCONUT_BANDWIDTH_CONTRACT_ADDRESS=n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0
 GROUP_CONTRACT_ADDRESS=n1rw8fw2mpcpzzq3jpa4e52ufawnmj5a4u68p35umvgskewuw0nlzsaa5w4m
 MULTISIG_CONTRACT_ADDRESS=n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0
 COCONUT_DKG_CONTRACT_ADDRESS=n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0
+EPHEMERA_CONTRACT_ADDRESS=n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0
 REWARDING_VALIDATOR_ADDRESS=n10yyd98e2tuwu0f7ypz9dy3hhjw7v772q6287gy
 STATISTICS_SERVICE_DOMAIN_ADDRESS="https://mainnet-stats.nymte.ch:8090"
 NYXD="https://rpc.nymtech.net"

envs/qa-qwerty.env

@@ -16,6 +16,7 @@ COCONUT_BANDWIDTH_CONTRACT_ADDRESS=n19d2nwj7fdhxqmyvgy8lf3ad49a6vmww4shryhrkj2mq
 GROUP_CONTRACT_ADDRESS=n1fqquzw4mk0pkamgr2ywt2v7h2j9nuyjjn4gvpy8zlpp6xn0uyuzqfm28l5
 MULTISIG_CONTRACT_ADDRESS=n1gaq3666chd5348apj8cka8t2mckv7azp9espyr7wgpxyuzur5d0sazpysy
 COCONUT_DKG_CONTRACT_ADDRESS=n18yadscxw8v35dds7ksv3j0svmjh3h6e7tmxpadk96mvgz27zygkshuf4vs
+EPHEMERA_CONTRACT_ADDRESS=n18yadscxw8v35dds7ksv3j0svmjh3h6e7tmxpadk96mvgz27zygkshuf4vs
 REWARDING_VALIDATOR_ADDRESS=n10yyd98e2tuwu0f7ypz9dy3hhjw7v772q6287gy
 SERVICE_PROVIDER_DIRECTORY_CONTRACT_ADDRESS=n1qsn2655eflc0nx2uwqtwyv5kad5dwm4c0gn72yr4q4de5r3jaz2slvqjgt
 NAME_SERVICE_CONTRACT_ADDRESS=n1cm2u5vfjd3zalfw0p65xyh4tcrw3hjlm0960gzhewga449h4mgas77mjkl

envs/qa.env

@@ -9,16 +9,19 @@ MIX_DENOM_DISPLAY=nym
 STAKE_DENOM=unyx
 STAKE_DENOM_DISPLAY=nyx
 DENOMS_EXPONENT=6
-MIXNET_CONTRACT_ADDRESS=n1rjzps6qrmdqmf0xz4cn4x4rcmqeqzq6hnzqg4wcvd0r2lyasdq5sepn5s8
-VESTING_CONTRACT_ADDRESS=n1xr3rq8yvd7qplsw5yx90ftsr2zdhg4e9z60h5duusgxpv72hud3sjkxkav
+MIXNET_CONTRACT_ADDRESS=n10qt8wg0n7z740ssvf3urmvgtjhxpyp74hxqvqt7z226gykuus7eq5u9pvq
+VESTING_CONTRACT_ADDRESS=n1vguuxez2h5ekltfj9gjd62fs5k4rl2zy5hfrncasykzw08rezpfstk9xtk
 BANDWIDTH_CLAIM_CONTRACT_ADDRESS=n19lc9u84cz0yz3fww5283nucc9yvr8gsjmgeul0
-COCONUT_BANDWIDTH_CONTRACT_ADDRESS=n1ghd753shjuwexxywmgs4xz7x2q732vcn7ty4yw
-GROUP_CONTRACT_ADDRESS=n1rw8fw2mpcpzzq3jpa4e52ufawnmj5a4u68p35umvgskewuw0nlzsaa5w4m
-MULTISIG_CONTRACT_ADDRESS=n17p9rzwnnfxcjp32un9ug7yhhzgtkhvl988qccs
-COCONUT_DKG_CONTRACT_ADDRESS=n17p9rzwnnfxcjp32un9ug7yhhzgtkhvl988qccs
-REWARDING_VALIDATOR_ADDRESS=n1tfzd4qz3a45u8p4mr5zmzv66457uwjgcl05jdq
-STATISTICS_SERVICE_DOMAIN_ADDRESS="http://0.0.0.0"
-NYXD="https://qa-validator.nymtech.net"
-NYM_API="https://qa-validator-api.nymtech.net/api"
+COCONUT_BANDWIDTH_CONTRACT_ADDRESS=n1x22q8lfhz7qcvtzs0dakhgx2th64l79kepjujhhxk5x804taeqlqcywn96
+GROUP_CONTRACT_ADDRESS=n1g4xlpqy29m50j5y69reguae328tc9y83l4299pf2wmjn0xczq5js3704ql
+MULTISIG_CONTRACT_ADDRESS=n1p54qvfde6mpnqvz3dnpa78x2qyyr5k4sgw9qr97mxjgklc5gze9sv6t964
+COCONUT_DKG_CONTRACT_ADDRESS=n1xqkp8x4gqwjnhemtemc5dqhwll6w6rrgpywvhka7sh8vz8swul9sp3lv3w
+EPHEMERA_CONTRACT_ADDRESS=n1tdxl4uv6fg3ke52n0umhv96648ttmwtg0u6uujmv5cuvjktg6qvq4mlzeg
+REWARDING_VALIDATOR_ADDRESS=n1547eraje66h9mf99fm50mea04n9mrzsc8c23r5
+NAME=n1uz24lsnwxvhep8m3gjec7ev86twlhlqrf5rphlgn3rda3zu048ssjqr5w9
+SERVICE_PROVICER=n1nhdr07kmjns2x8dnp53tdk4qxreze8zdxj6xucyvkdj9tta73rjqa96wps
+STATISTICS_SERVICE_DOMAIN_ADDRESS="https://mainnet-stats.nymte.ch:8090"
+NYXD="https://qa-validator.qa.nymte.ch/"
+NYM_API="https://qa-nym-api.qa.nymte.ch/api"
 
-DKG_TIME_CONFIGURATION="600,300,300,60,60,1209600"
+DKG_TIME_CONFIGURATION="600,300,300,60,60,1209600"

envs/sandbox.env

@@ -17,6 +17,7 @@ BANDWIDTH_CLAIM_CONTRACT_ADDRESS="nymt1rhmk9udessnv3r8f3eh2s03f45svnjaczpmcqz"
 MULTISIG_CONTRACT_ADDRESS="nymt142dkm8xe9f0ytyarp7ww4kvclva65705jphxsk9exn3nqdsm8jkqnp06ac"
 COCONUT_BANDWIDTH_CONTRACT_ADDRESS="nymt1ty0frysegskh6ndm3v96z5xdq66qzcu0aw7xcxlgp54jg0mjwlgqplc6v0"
 COCONUT_DKG_CONTRACT_ADDRESS="nymt1gwk6muhmzeuxje7df7rjvqwl2vex0kj4t2hwuzmyx5k62kfusu5qk4k5z4"
+EPHEMERA_CONTRACT_ADDRESS="nymt1gwk6muhmzeuxje7df7rjvqwl2vex0kj4t2hwuzmyx5k62kfusu5qk4k5z4"
 GROUP_CONTRACT_ADDRESS="nymt14ry36mwauycz08v8ndcujghxz4hmua5epxcn0mamlr3suqe0l2qsqx5ya2"
 
 STATISTICS_SERVICE_DOMAIN_ADDRESS="http://0.0.0.0"

ephemera/Cargo.toml

@@ -0,0 +1,65 @@
+[package]
+name = "ephemera"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "ephemera"
+path = "bin/main.rs"
+
+[dependencies]
+## internal
+nym-task = { path = "../common/task" }
+
+actix-web = "4"
+anyhow = { version = "1.0.66", features = ["backtrace"] }
+array-bytes = "6.0.0"
+async-trait = "0.1.59"
+asynchronous-codec = "0.6.1"
+blake2 = "0.10.6"
+bs58 = "0.4.0"
+bytes = "1.3.0"
+cfg-if = "1.0.0"
+chrono = { version = "0.4.24", default-features = false, features = ["clock"] }
+clap = { version = "4.0.32", features = ["derive"] }
+config = { version = "0.13", default-features = false, features = ["toml"] }
+digest = "0.10.6"
+dirs = "5.0.0"
+futures = "0.3.18"
+futures-util = "0.3.25"
+lazy_static = "1.4.0"
+libp2p = { version = "0.51.3", default-features = false, features = ["dns", "gossipsub", "kad", "macros", "noise", "request-response", "serde", "tcp", "tokio", "yamux"] }
+libp2p-identity = "0.1.0"
+log = "0.4.14"
+lru = "0.10.0"
+nym-config = { path = "../common/config" }
+nym-ephemera-common = { path = "../common/cosmwasm-smart-contracts/ephemera" }
+pretty_env_logger = "0.4"
+refinery = { version = "0.8.7", features = ["rusqlite"], optional = true }
+reqwest = { version = "0.11.6", features = ["json"] }
+rocksdb = { version = "0.20.1", optional = true }
+rusqlite = { version = "0.27.0", features = ["bundled"], optional = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_derive = "1.0.149"
+serde_json = "1.0.91"
+thiserror = "1.0.37"
+tokio = { version = "1", features = ["macros", "net","rt-multi-thread"] }
+tokio-tungstenite = "0.18.0"
+tokio-util = { version = "0.7.4", features = ["full"] }
+toml = "0.7.0"
+unsigned-varint = "0.7.1"
+utoipa = { version = "3.0.1", features = ["actix_extras"] }
+utoipa-swagger-ui = { version = "3.0.2", features = ["actix-web"] }
+uuid = { version = "1.2.2", features = ["v4"] }
+
+# Temporary fix to https://github.com/bluejekyll/trust-dns/issues/1946
+enum-as-inner = "=0.5.1"
+
+[dev-dependencies]
+assert_matches = "1.5.0"
+rand = "0.8.5"
+
+[features]
+default = ["sqlite_storage"]
+rocksdb_storage = ["rocksdb"]
+sqlite_storage = ["rusqlite", "refinery"]

ephemera/README.md

@@ -0,0 +1,142 @@
+# Ephemera - reliable broadcast protocol implementation
+
+Ephemera does reliable broadcast for blocks.
+
+## Short Overview
+
+All Ephemera nodes accept messages submitted by clients. Node then gossips these to other nodes in the cluster. After certain interval,
+a node collects messages and produces a block. Then it does reliable broadcast for the block with other nodes in the cluster.
+
+Ephemera doesn't have the concept of (decentralised) leader at the moment. It's up to an _Application_ to decide which block to use. 
+For example in case of Nym-Api, it is the first block submitted to a "Smart Contract".
+
+At the same time, the purpose of blocks is to reach consensus about which messages are included. It's just that Ephemera doesn't make the final decision,
+instead it leaves that to an _Application_.
+
+## Main concepts
+
+- **Node** - a single instance of Ephemera.
+- **Cluster** - a set of nodes participating in reliable broadcast.
+- **EphemeraMessage** - a message submitted by a client.
+- **Block** - a set of messages collected by a node.
+- **Application(ABCI)** - a trait which Ephemera users implement to accept messages and blocks.
+  - check_tx
+  - check_block
+  - accept_block
+
+## How to run
+
+[README](../scripts/README.md)
+
+## HTTP API
+
+See [Rust](src/api/http/mod.rs)
+
+### Endpoints
+
+**NODE**
+- `/ephemera/node/health`
+- `/ephemera/node/config`
+
+**BLOCKS**
+- `/ephemera/broadcast/block/{hash}`
+- `/ephemera/broadcast/block/height/{height}`
+- `/ephemera/broadcast/blocks/last`
+- `/ephemera/broadcast/block/certificates/{hash}`
+- `/ephemera/broadcast/block/broadcast_info/{hash}`
+
+**GROUP**
+- `/ephemera/broadcast/group/info`
+
+**MESSAGES**
+- `/ephemera/broadcast/submit_message`
+
+**DHT**
+- `/ephemera/dht/query/{key}`
+- `/ephemera/dht/store`
+
+## Rust API
+
+Almost identical to HTTP API.
+
+See [Rust](src/api/mod.rs)
+
+## Application(Ephemera ABCI)
+
+Cosmos style ABCI application hook
+- `check_tx`
+- `check_block`
+- `deliver_block`
+
+See [Rust](src/api/application.rs)
+
+## Examples
+
+### Ephemera HTTP and WS external interfaces example/tests
+
+See [README.md](../examples/http-ws-sync/README.md)
+
+### Nym Api simulation
+
+See [README.md](../examples/nym-api/README.md)
+
+### http API example/tests
+
+See [README.md](../examples/cluster-http-api/README.md)
+
+### Membership over HTTP API example/tests
+
+See [README.md](../examples/members_provider_http/README.md)
+
+## About reliable broadcast and consensus
+
+In blockchain technology blocks have two main purposes:
+1. To maintain chain of blocks, so that the validity of each block can be cryptographically verified by the previous blocks
+2. As a unit of consensus, each block contains a set of transactions/messages/actions that are agreed upon by
+   the network. This set of transactions is chosen from the global set of all possible transactions that are pending.
+   We call the set of transactions in a block consensus because the set of nodes trying to achieve global shared state
+   agreed on this particular set of transactions.
+
+Ephemera is not a blockchain. But it uses blocks to agree on the set of transactions in a block.
+But at the same time it doesn't behave like a blockchain consensus algorithm.
+We may say that it allows each application that uses Ephemera to "propose" something what can be
+afterwards to be used to achieve consensus.
+
+### In Summary
+
+1. Ephemera provides functionality to reach agreement on a single value between a set of nodes.
+2. Ephemera also provides the concept of a block, which application can take advantage of to reach consensus externally.
+
+### Reliable broadcast, consensus and blocks
+
+In distributed systems(including byzantine), we try to solve the problem of reaching to a commons state between a set of nodes.
+
+One way to define this problem is using the following properties:
+1.
+    1) Agreement: All nodes agree on the same value.(TODO clarify)
+    2) Consensus: All nodes agree on the same value.(TODO clarify)
+2. Validity: All nodes agree on a value that is valid.
+3. Termination: All nodes eventually agree on a value.
+
+Reliable broadcast ensures the properties of 1.1 and 1.2. It's left to a particular consensus algorithm to ensure the termination property.
+
+One important feature of consensus in blockchain is that it guarantees total ordering of transactions.
+Reliable broadcast with blocks helps to ensure this total ordering.
+
+### Ephemera specific properties
+
+Because Ephemera doesn't use the idea of leader, we can say that it solves consensus partially.
+It allows each instance to create a block. And then it's up to an application to decide which block to use.
+
+Also, as it doesn't implement a full consensus algorithm, it doesn't ensure the termination.
+There's no algorithm in place what tries to reach a consensus about a single block globally and sequentially
+in time.
+
+When a block contains a single message, then it's semantically equivalent to a reliable broadcast.
+
+But when a block contains multiple messages, then it can be part of a consensus process. Except that in Ephemera each node
+can create a block. To achieve consensus in a more traditional sense, it needs an application help if more strict
+consensus is required.
+
+For example, Nym-Api allows each node to create a block but uses external coordinator(a smart contract)
+to decide which block to use.

ephemera/bin/main.rs

@@ -0,0 +1,12 @@
+use clap::Parser;
+
+use ephemera::cli::Cli;
+use ephemera::logging;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    logging::init();
+
+    Cli::parse().execute().await?;
+    Ok(())
+}

ephemera/migrations/V1__block.sql

@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS blocks (
+    id          INTEGER      NOT NULL PRIMARY KEY AUTOINCREMENT,
+    block_hash  TEXT         NOT NULL UNIQUE,
+    height      TEXT         NOT NULL UNIQUE,
+    block       BLOB         NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS block_certificates (
+    id              INTEGER      NOT NULL PRIMARY KEY AUTOINCREMENT,
+    block_hash      TEXT         NOT NULL UNIQUE,
+    certificates    BLOB         NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS block_broadcast_group (
+    id              INTEGER      NOT NULL PRIMARY KEY AUTOINCREMENT,
+    block_hash      TEXT         NOT NULL UNIQUE,
+    members         BLOB         NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS block_merkle_tree (
+    id              INTEGER      NOT NULL PRIMARY KEY AUTOINCREMENT,
+    block_hash      TEXT         NOT NULL UNIQUE,
+    merkle_tree     BLOB         NOT NULL
+);

ephemera/src/api/application.rs

@@ -0,0 +1,104 @@
+use log::trace;
+use thiserror::Error;
+
+use crate::api::types::{ApiBlock, ApiEphemeraMessage};
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum RemoveMessages {
+    /// Remove all messages from the mempool
+    All,
+    /// Remove only inclued messages from the mempool
+    Selected(Vec<ApiEphemeraMessage>),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum CheckBlockResult {
+    /// Accept the block
+    Accept,
+    /// Reject the block with a reason.
+    Reject,
+    /// Reject the block and remove messages from the mempool
+    RejectAndRemoveMessages(RemoveMessages),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct CheckBlockResponse {
+    pub accept: bool,
+    pub reason: Option<String>,
+}
+
+#[derive(Error, Debug)]
+pub enum Error {
+    //Just a placeholder for now
+    #[error("ApplicationError: {0}")]
+    Application(#[from] anyhow::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Cosmos style ABCI application hook
+///
+/// These functions should be relatively fast, as they are called synchronously by Ephemera main loop.
+pub trait Application {
+    /// It's called when receiving a new message from network before adding it to the mempool.
+    /// It's up to the application to decide whether the message is valid or not.
+    /// Basic check could be for example signature verification.
+    ///
+    /// # Arguments
+    /// * `message` - message to be checked
+    ///
+    /// # Returns
+    /// * `true` - if the message is valid
+    /// * `false` - if the message is invalid
+    ///
+    /// # Errors
+    /// * `Error::General` - if there was an error during validation
+    fn check_tx(&self, message: ApiEphemeraMessage) -> Result<bool>;
+
+    /// Ephemera produces new blocks with configured interval.
+    /// Application can decide whether to accept the block or not.
+    /// For example, if the block doesn't contain any transactions, it can be rejected.
+    ///
+    /// # Arguments
+    /// * `block` - block to be checked
+    ///
+    /// # Returns
+    /// * `CheckBlockResult::Accept` - if the block is valid
+    /// * `CheckBlockResult::Reject` - if the block is invalid
+    /// * `CheckBlockResult::RejectAndRemoveMessages` - if the block is invalid and some messages should be removed from the mempool
+    ///
+    /// # Errors
+    /// * `Error::General` - if there was an error during validation
+    fn check_block(&self, block: &ApiBlock) -> Result<CheckBlockResult>;
+
+    /// Deliver Block is called after block is confirmed by Ephemera and persisted to the storage.
+    ///
+    /// # Arguments
+    /// * `block` - block to be delivered
+    ///
+    /// # Errors
+    /// * `Error::General` - if there was an error during validation
+    fn deliver_block(&self, block: ApiBlock) -> Result<()>;
+}
+
+/// Dummy application which doesn't do any validation.
+/// Might be useful for testing.
+#[derive(Default)]
+pub struct Dummy;
+
+impl Application for Dummy {
+    fn check_tx(&self, tx: ApiEphemeraMessage) -> Result<bool> {
+        trace!("check_tx: {tx:?}");
+        Ok(true)
+    }
+
+    fn check_block(&self, block: &ApiBlock) -> Result<CheckBlockResult> {
+        trace!("accept_block: {block:?}");
+        Ok(CheckBlockResult::Accept)
+    }
+
+    fn deliver_block(&self, block: ApiBlock) -> Result<()> {
+        trace!("deliver_block: {block:?}");
+        Ok(())
+    }
+}

ephemera/src/api/http/client.rs

@@ -0,0 +1,497 @@
+use std::time::Duration;
+
+use thiserror::Error;
+
+use crate::api::types::{ApiBlockBroadcastInfo, ApiBroadcastInfo, ApiHealth};
+use crate::ephemera_api::{
+    ApiBlock, ApiCertificate, ApiDhtQueryRequest, ApiDhtQueryResponse, ApiDhtStoreRequest,
+    ApiEphemeraConfig, ApiEphemeraMessage, ApiVerifyMessageInBlock,
+};
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error(transparent)]
+    Internal(#[from] reqwest::Error),
+    #[error("Unexpected response: {status} {body}")]
+    UnexpectedResponse {
+        status: reqwest::StatusCode,
+        body: String,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Client to interact with the node over HTTP api.
+pub struct Client {
+    pub(crate) client: reqwest::Client,
+    pub(crate) url: String,
+}
+
+impl Client {
+    /// Create a http new client.
+    ///
+    /// # Arguments
+    /// * `url` - The url of the node api endpoint.
+    #[must_use]
+    pub fn new(url: String) -> Self {
+        let client = reqwest::Client::new();
+        Self { client, url }
+    }
+
+    /// Create a new client.
+    ///
+    /// # Arguments
+    /// * `url` - The url of the node api endpoint.
+    /// * `timeout_sec` - Request timeout in seconds.
+    ///
+    /// # Panics
+    /// If the client cannot be created.
+    #[must_use]
+    pub fn new_with_timeout(url: String, timeout_sec: u64) -> Self {
+        let client = reqwest::ClientBuilder::new()
+            .timeout(Duration::from_secs(timeout_sec))
+            .build()
+            .unwrap();
+        Self { client, url }
+    }
+
+    /// Get the health of the node.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{Client, ApiHealth};
+    ///
+    /// #[tokio::main]
+    ///async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///   let client = Client::new("http://localhost:7000".to_string());
+    ///   let health = client.health().await.unwrap();
+    ///    Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Returns
+    /// * [`ApiHealth`] - The health of the node.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn health(&self) -> Result<ApiHealth> {
+        self.query("ephemera/node/health").await
+    }
+
+    /// Get the block by hash.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiBlock, Client};
+    ///
+    /// #[tokio::main]
+    ///async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///     let client = Client::new("http://localhost:7000".to_string());
+    ///     let block = client.get_block_by_hash("9D2LaY17rbnxfgKUbvcsJ5cB2BRHEd8fPJwsBnDHNGBX").await?;
+    ///     Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Returns
+    /// * Option<[`ApiBlock`]> - The block.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_block_by_hash(&self, hash: &str) -> Result<Option<ApiBlock>> {
+        let url = format!("ephemera/broadcast/block/{hash}",);
+        self.query_optional(&url).await
+    }
+
+    /// Get the block certificates by hash.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiCertificate, Client};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    let client = Client::new("http://localhost:7000".to_string());
+    ///    let certificates = client.get_block_certificates("9D2LaY17rbnxfgKUbvcsJ5cB2BRHEd8fPJwsBnDHNGBX").await?;
+    ///    Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `hash` - The hash of the block.
+    ///
+    /// # Returns
+    /// * Option<Vec<[`ApiCertificate`]>> - The block certificates.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_block_certificates(&self, hash: &str) -> Result<Option<Vec<ApiCertificate>>> {
+        let url = format!("ephemera/broadcast/block/certificates/{hash}",);
+        self.query_optional(&url).await
+    }
+
+    /// Get the block by height.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiBlock, Client};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///   let client = Client::new("http://localhost:7000/".to_string());
+    ///   let block = client.get_block_by_height(1).await?;
+    ///   Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `height` - The height of the block.
+    ///
+    /// # Returns
+    /// * Option<[`ApiBlock`]> - The block.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_block_by_height(&self, height: u64) -> Result<Option<ApiBlock>> {
+        let url = format!("ephemera/broadcast/block/height/{height}",);
+        self.query_optional(&url).await
+    }
+
+    /// Get the last block.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiBlock, Client};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    let client = Client::new("http://localhost:7000/".to_string());
+    ///    let block = client.get_last_block().await?;
+    ///    Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Returns
+    /// * [`ApiBlock`] - The last block.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_last_block(&self) -> Result<ApiBlock> {
+        self.query("ephemera/broadcast/blocks/last").await
+    }
+
+    /// Get the node configuration.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiEphemeraConfig, Client};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    let client = Client::new("http://localhost:7000/".to_string());
+    ///    let config = client.get_ephemera_config().await?;
+    ///    Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Returns
+    /// * [`ApiEphemeraConfig`] - The node configuration.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_ephemera_config(&self) -> Result<ApiEphemeraConfig> {
+        self.query("ephemera/node/config").await
+    }
+
+    /// Submit a message to the node.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::{ApiEphemeraMessage, Client};
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///   let client = Client::new("http://localhost:7000/".to_string());
+    ///   let message = unimplemented!("See how to create a ApiEphemeraMessage");
+    ///   client.submit_message(message).await?;
+    ///   Ok(())
+    /// }
+    ///
+    /// ```
+    ///
+    /// # Arguments
+    /// * `message` - The message to submit.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn submit_message(&self, message: ApiEphemeraMessage) -> Result<()> {
+        let url = format!("{}/{}", self.url, "ephemera/broadcast/submit_message");
+        let response = self.client.post(&url).json(&message).send().await?;
+        if response.status().is_success() {
+            Ok(())
+        } else {
+            Err(Error::UnexpectedResponse {
+                status: response.status(),
+                body: response.text().await?,
+            })
+        }
+    }
+
+    ///Store Key Value pair in the DHT.
+    ///
+    /// # Example
+    ///```no_run
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    use ephemera::ephemera_api::Client;
+    ///    let client = Client::new("http://localhost:7000/".to_string());
+    ///    let request = unimplemented!("See how to create a ApiDhtStoreRequest");
+    ///    client.store_in_dht(request).await?;
+    ///  Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `request` - Key Value pair to store.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn store_in_dht(&self, request: ApiDhtStoreRequest) -> Result<()> {
+        let url = format!("{}/{}", self.url, "ephemera/dht/store");
+        let response = self.client.post(&url).json(&request).send().await?;
+        if response.status().is_success() {
+            Ok(())
+        } else {
+            Err(Error::UnexpectedResponse {
+                status: response.status(),
+                body: response.text().await?,
+            })
+        }
+    }
+
+    ///Store Key Value pair in the DHT.
+    ///
+    /// # Example
+    ///```no_run
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    use ephemera::ephemera_api::Client;
+    ///    let client = Client::new("http://localhost:7000/".to_string());
+    ///    let key = &[1, 2, 3];
+    ///    let value = &[4, 5, 6];
+    ///    client.store_in_dht_key_value(key, value).await?;
+    ///    Ok(())
+    /// }
+    /// ```
+    /// # Arguments
+    /// * `key` - Key to use to store the value.
+    /// * `value` - Value to store.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn store_in_dht_key_value(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        let request = ApiDhtStoreRequest::new(key, value);
+        self.store_in_dht(request).await
+    }
+
+    /// Query the DHT for a given key.
+    ///
+    /// # Example
+    ///```no_run
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///    use ephemera::ephemera_api::{ApiDhtQueryRequest, Client};
+    ///    let client = Client::new("http://localhost:7000/".to_string());
+    ///    let request = ApiDhtQueryRequest::new(&[1, 2, 3]);
+    ///    let response = client.query_dht(request).await?;
+    ///    Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `request` - Key to query.
+    ///
+    /// # Returns
+    /// * Option<[`ApiDhtQueryResponse`]> - The value stored in the DHT for the given key.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn query_dht(
+        &self,
+        request: ApiDhtQueryRequest,
+    ) -> Result<Option<ApiDhtQueryResponse>> {
+        let url = format!("ephemera/dht/query/{}", request.key_encoded());
+        self.query_optional(&url).await
+    }
+
+    /// Query the DHT for a given key.
+    ///
+    /// # Example
+    /// ```no_run
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///   use ephemera::ephemera_api::Client;
+    ///   let client = Client::new("http://localhost:7000/".to_string());
+    ///   let key = &[1, 2, 3];
+    ///   let response = client.query_dht_key(key).await?;
+    ///   Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `key` - Key to query.
+    ///
+    /// # Returns
+    /// * Option<[`ApiDhtQueryResponse`]> - The value stored in the DHT for the given key.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn query_dht_key(&self, key: &[u8]) -> Result<Option<ApiDhtQueryResponse>> {
+        let request = ApiDhtQueryRequest::new(key);
+        self.query_dht(request).await
+    }
+
+    /// Get broadcast group info.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::Client;
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///   let client = Client::new("http://localhost:7000/".to_string());
+    ///   let info = client.broadcast_info().await?;
+    ///   Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Returns
+    /// * [`ApiBroadcastInfo`] - The broadcast group info.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn broadcast_info(&self) -> Result<ApiBroadcastInfo> {
+        self.query("ephemera/broadcast/group/info").await
+    }
+
+    /// Get block broadcast info
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::Client;
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    ///  let client = Client::new("http://localhost:7000/".to_string());
+    ///  let info = client.get_block_broadcast_info("hash").await?;
+    ///  Ok(())
+    /// }
+    ///```
+    ///
+    /// # Arguments
+    /// * `hash` - Hash of the block to query.
+    ///
+    /// # Returns
+    /// * Some([`ApiBlockBroadcastInfo`]) - The block broadcast info.
+    /// * None - If the block is not found.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn get_block_broadcast_info(
+        &self,
+        hash: &str,
+    ) -> Result<Option<ApiBlockBroadcastInfo>> {
+        let url = format!("ephemera/broadcast/block/broadcast_info/{hash}",);
+        self.query_optional(&url).await
+    }
+
+    /// Verifies if given message is in block identified by block hash
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::ephemera_api::Client;
+    ///
+    /// #[tokio::main]
+    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    /// let client = Client::new("http://localhost:7000/".to_string());
+    /// let is_in_block = client.verify_message_in_block("block_hash", "message_hash", 0).await?;
+    /// Ok(())
+    /// }
+    /// ```
+    ///
+    /// # Arguments
+    /// * `block_hash` - Hash of the block to query.
+    /// * `message_hash` - Hash of the message to query.
+    /// * `message_index` - Index of the message in the block.
+    ///
+    /// # Returns
+    /// * bool - True if the message is in the block, false otherwise.
+    ///
+    /// # Errors
+    /// If the request fails.
+    pub async fn verify_message_in_block(
+        &self,
+        block_hash: &str,
+        message_hash: &str,
+        message_index: usize,
+    ) -> Result<bool> {
+        let request = ApiVerifyMessageInBlock::new(
+            block_hash.to_string(),
+            message_hash.to_string(),
+            message_index,
+        );
+
+        let url = format!("{}/{}", self.url, "ephemera/messages/verify");
+        let response = self.client.post(&url).json(&request).send().await?;
+        if response.status().is_success() {
+            let body = response.json::<bool>().await?;
+            Ok(body)
+        } else {
+            Err(Error::UnexpectedResponse {
+                status: response.status(),
+                body: response.text().await?,
+            })
+        }
+    }
+
+    async fn query_optional<T: for<'de> serde::Deserialize<'de>>(
+        &self,
+        path: &str,
+    ) -> Result<Option<T>> {
+        let url = format!("{}/{}", self.url, path);
+        match self.client.get(&url).send().await {
+            Ok(response) => {
+                if response.status().is_success() {
+                    let body = response.json::<T>().await?;
+                    Ok(Some(body))
+                } else if response.status() == reqwest::StatusCode::NOT_FOUND {
+                    Ok(None)
+                } else {
+                    return Err(Error::UnexpectedResponse {
+                        status: response.status(),
+                        body: response.text().await?,
+                    });
+                }
+            }
+            Err(err) => Err(err.into()),
+        }
+    }
+
+    async fn query<T: for<'de> serde::Deserialize<'de>>(&self, path: &str) -> Result<T> {
+        let url = format!("{}/{}", self.url, path);
+        match self.client.get(&url).send().await {
+            Ok(response) => {
+                if response.status().is_success() {
+                    let body = response.json::<T>().await?;
+                    Ok(body)
+                } else {
+                    Err(Error::UnexpectedResponse {
+                        status: response.status(),
+                        body: response.text().await?,
+                    })
+                }
+            }
+            Err(err) => Err(err.into()),
+        }
+    }
+}

ephemera/src/api/http/mod.rs

@@ -0,0 +1,88 @@
+use actix_web::{dev::Server, http::KeepAlive, web::Data, App, HttpServer};
+use log::info;
+use utoipa::OpenApi;
+use utoipa_swagger_ui::SwaggerUi;
+
+use crate::api::CommandExecutor;
+use crate::core::builder::NodeInfo;
+
+pub(crate) mod client;
+pub(crate) mod query;
+pub(crate) mod submit;
+
+/// Starts the HTTP server.
+pub(crate) fn init(node_info: &NodeInfo, api: CommandExecutor) -> anyhow::Result<Server> {
+    print_startup_messages(node_info);
+
+    let server = HttpServer::new(move || {
+        App::new()
+            .app_data(Data::new(api.clone()))
+            .service(query::health)
+            .service(query::block_by_hash)
+            .service(query::block_certificates)
+            .service(query::block_by_height)
+            .service(query::block_broadcast_group)
+            .service(query::last_block)
+            .service(query::node_config)
+            .service(query::query_dht)
+            .service(query::broadcast_info)
+            .service(submit::submit_message)
+            .service(submit::store_in_dht)
+            .service(submit::verify_message_in_block)
+            .service(swagger_ui())
+    })
+    .keep_alive(KeepAlive::Os)
+    .bind((node_info.ip.as_str(), node_info.initial_config.http.port))?
+    .run();
+    Ok(server)
+}
+
+/// Builds the Swagger UI.
+///
+/// Note that all routes you want Swagger docs for must be in the `paths` annotation.
+fn swagger_ui() -> SwaggerUi {
+    use crate::api::types;
+    #[derive(OpenApi)]
+    #[openapi(
+        paths(
+            query::health,
+            query::block_by_hash,
+            query::block_certificates,
+            query::block_by_height,
+            query::last_block,
+            query::block_broadcast_group,
+            query::node_config,
+            query::query_dht,
+            query::broadcast_info,
+            submit::submit_message,
+            submit::store_in_dht,
+            submit::verify_message_in_block
+        ),
+        components(schemas(
+            types::ApiBlock,
+            types::ApiEphemeraMessage,
+            types::ApiCertificate,
+            types::ApiSignature,
+            types::ApiPublicKey,
+            types::ApiHealth,
+            types::HealthStatus,
+            types::ApiEphemeraConfig,
+            types::ApiDhtStoreRequest,
+            types::ApiDhtQueryRequest,
+            types::ApiDhtQueryResponse,
+            types::ApiBroadcastInfo,
+            types::ApiVerifyMessageInBlock,
+        ))
+    )]
+    struct ApiDoc;
+    SwaggerUi::new("/swagger-ui/{_:.*}").url("/api-doc/openapi.json", ApiDoc::openapi())
+}
+
+/// Prints messages saying which ports HTTP is running on, and some helpful pointers
+/// `OpenAPI` and `Swagger UI` endpoints.
+fn print_startup_messages(info: &NodeInfo) {
+    let http_root = info.api_address_http();
+    info!("Server running on {}", http_root);
+    info!("Swagger UI: {}/swagger-ui/", http_root);
+    info!("OpenAPI spec is at: {}/api-doc/openapi.json", http_root);
+}

ephemera/src/api/http/query.rs

@@ -0,0 +1,182 @@
+use actix_web::{get, web, HttpResponse, Responder};
+use log::error;
+
+use crate::{
+    api::{types::ApiHealth, types::HealthStatus::Healthy, CommandExecutor},
+    ephemera_api::{ApiDhtQueryRequest, ApiDhtQueryResponse},
+};
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Endpoint to check if the server is running")),
+)]
+#[get("/ephemera/node/health")]
+#[allow(clippy::unused_async)]
+pub(crate) async fn health() -> impl Responder {
+    HttpResponse::Ok().json(ApiHealth { status: Healthy })
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get current broadcast group"),
+(status = 500, description = "Server failed to process request")),
+)]
+#[get("/ephemera/broadcast/group/info")]
+pub(crate) async fn broadcast_info(api: web::Data<CommandExecutor>) -> impl Responder {
+    match api.get_broadcast_info().await {
+        Ok(group) => HttpResponse::Ok().json(group),
+        Err(err) => {
+            error!("Failed to get current broadcast group: {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "GET block by hash"),
+(status = 404, description = "Block not found"),
+(status = 500, description = "Server failed to process request")),
+params(("hash", description = "Block hash")),
+)]
+#[get("/ephemera/broadcast/block/{hash}")]
+pub(crate) async fn block_by_hash(
+    hash: web::Path<String>,
+    api: web::Data<CommandExecutor>,
+) -> impl Responder {
+    match api.get_block_by_id(hash.into_inner()).await {
+        Ok(Some(block)) => HttpResponse::Ok().json(block),
+        Ok(_) => HttpResponse::NotFound().json("Block not found"),
+        Err(err) => {
+            error!("Failed to get block by hash: {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get block signatures"),
+(status = 404, description = "Certificates not found"),
+(status = 500, description = "Server failed to process request")),
+params(("hash", description = "Block hash")),
+)]
+#[get("/ephemera/broadcast/block/certificates/{hash}")]
+pub(crate) async fn block_certificates(
+    hash: web::Path<String>,
+    api: web::Data<CommandExecutor>,
+) -> impl Responder {
+    let id = hash.into_inner();
+    match api.get_block_certificates(id.clone()).await {
+        Ok(Some(signatures)) => HttpResponse::Ok().json(signatures),
+        Ok(_) => HttpResponse::NotFound().json("Certificates not found"),
+        Err(err) => {
+            error!("Failed to get signatures {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get block by height"),
+(status = 404, description = "Block not found"),
+(status = 500, description = "Server failed to process request")),
+params(("height", description = "Block height")),
+)]
+#[get("/ephemera/broadcast/block/height/{height}")]
+pub(crate) async fn block_by_height(
+    height: web::Path<u64>,
+    api: web::Data<CommandExecutor>,
+) -> impl Responder {
+    match api.get_block_by_height(height.into_inner()).await {
+        Ok(Some(block)) => HttpResponse::Ok().json(block),
+        Ok(_) => HttpResponse::NotFound().json("Block not found"),
+        Err(err) => {
+            error!("Failed to get block {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get last block"),
+(status = 500, description = "Server failed to process request")),
+)]
+//Need to use plural(blocks), otherwise overlaps with block_by_id route
+#[get("/ephemera/broadcast/blocks/last")]
+pub(crate) async fn last_block(api: web::Data<CommandExecutor>) -> impl Responder {
+    match api.get_last_block().await {
+        Ok(block) => HttpResponse::Ok().json(block),
+        Err(err) => {
+            error!("Failed to get block {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get block broadcast group"),
+(status = 404, description = "Block not found"),
+(status = 500, description = "Server failed to process request")),
+params(("hash", description = "Block hash")),
+)]
+#[get("/ephemera/broadcast/block/broadcast_info/{hash}")]
+pub(crate) async fn block_broadcast_group(
+    hash: web::Path<String>,
+    api: web::Data<CommandExecutor>,
+) -> impl Responder {
+    let hash = hash.into_inner();
+    match api.get_block_broadcast_info(hash).await {
+        Ok(Some(group)) => HttpResponse::Ok().json(group),
+        Ok(_) => HttpResponse::NotFound().json("Block not found"),
+        Err(err) => {
+            error!("Failed to get block broadcast group {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Get node config"),
+(status = 500, description = "Server failed to process request")),
+)]
+#[get("/ephemera/node/config")]
+pub(crate) async fn node_config(api: web::Data<CommandExecutor>) -> impl Responder {
+    match api.get_node_config().await {
+        Ok(config) => HttpResponse::Ok().json(config),
+        Err(err) => {
+            error!("Failed to get node config {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+responses(
+(status = 200, description = "Query dht"),
+(status = 500, description = "Server failed to process request")),
+params(("query", description = "Dht query")),
+)]
+#[get("/ephemera/dht/query/{key}")]
+pub(crate) async fn query_dht(
+    api: web::Data<CommandExecutor>,
+    key: web::Path<String>,
+) -> impl Responder {
+    let key = ApiDhtQueryRequest::parse_key(key.into_inner().as_str());
+
+    match api.query_dht(key).await {
+        Ok(Some((key, value))) => {
+            let response = ApiDhtQueryResponse::new(key, value);
+            HttpResponse::Ok().json(response)
+        }
+        Ok(_) => HttpResponse::NotFound().json("Not found"),
+        Err(err) => {
+            error!("Failed to query dht {err}",);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}

ephemera/src/api/http/submit.rs

@@ -0,0 +1,88 @@
+use actix_web::{post, web, HttpResponse};
+use log::{debug, error};
+
+use crate::api::types::ApiVerifyMessageInBlock;
+use crate::api::{
+    types::{ApiDhtStoreRequest, ApiEphemeraMessage},
+    ApiError, CommandExecutor,
+};
+
+#[utoipa::path(
+request_body = ApiEphemeraMessage,
+responses(
+(status = 200, description = "Send a message to an Ephemera node which will be broadcast to the network"),
+(status = 500, description = "Server failed to process request")),
+params(("message", description = "Message to send"))
+)]
+#[post("/ephemera/broadcast/submit_message")]
+pub(crate) async fn submit_message(
+    message: web::Json<ApiEphemeraMessage>,
+    api: web::Data<CommandExecutor>,
+) -> HttpResponse {
+    match api.send_ephemera_message(message.into_inner()).await {
+        Ok(_) => HttpResponse::Ok().json("Message submitted"),
+        Err(err) => {
+            if let ApiError::DuplicateMessage = err {
+                debug!("Message already submitted {err:?}");
+                HttpResponse::BadRequest().json("Message already submitted")
+            } else {
+                error!("Error submitting message: {}", err);
+                HttpResponse::InternalServerError().json("Server failed to process request")
+            }
+        }
+    }
+}
+
+#[utoipa::path(
+request_body = ApiDhtStoreRequest,
+responses(
+(status = 200, description = "Request to store a value in the DHT"),
+(status = 500, description = "Server failed to process request")),
+params(
+("request", description = "Dht store request")
+)
+)]
+#[post("/ephemera/dht/store")]
+pub(crate) async fn store_in_dht(
+    request: web::Json<ApiDhtStoreRequest>,
+    api: web::Data<CommandExecutor>,
+) -> HttpResponse {
+    let request = request.into_inner();
+
+    let key = request.key();
+    let value = request.value();
+
+    match api.store_in_dht(key, value).await {
+        Ok(_) => HttpResponse::Ok().json("Store request submitted"),
+        Err(err) => {
+            error!("Error storing in dht: {}", err);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}
+
+#[utoipa::path(
+request_body = ApiVerifyMessageInBlock,
+responses(
+(status = 200, description = "Verifies if given message is in block identified by block hash.\
+Returns true if message is in block, false otherwise. False can also mean that block or message \
+does not exist in that block."),
+(status = 500, description = "Server failed to process request")),
+params(
+("request", description = "Verify message request")
+)
+)]
+#[post("/ephemera/messages/verify")]
+pub(crate) async fn verify_message_in_block(
+    request: web::Json<ApiVerifyMessageInBlock>,
+    api: web::Data<CommandExecutor>,
+) -> HttpResponse {
+    let request = request.into_inner();
+    match api.verify_message_in_block(request).await {
+        Ok(valid) => HttpResponse::Ok().json(valid),
+        Err(err) => {
+            error!("Error verifying message: {}", err);
+            HttpResponse::InternalServerError().json("Server failed to process request")
+        }
+    }
+}

ephemera/src/api/mod.rs

@@ -0,0 +1,309 @@
+//! # Ephemera API
+//!
+//! This module contains all the types and functions available as part of Ephemera public API.
+//!
+//! This API is also available over HTTP.
+
+use std::fmt::Display;
+
+use log::{error, trace};
+use tokio::sync::{
+    mpsc::{channel, Receiver, Sender},
+    oneshot,
+};
+
+use crate::api::types::{
+    ApiBlock, ApiBlockBroadcastInfo, ApiBroadcastInfo, ApiCertificate, ApiEphemeraConfig,
+    ApiEphemeraMessage, ApiError, ApiVerifyMessageInBlock,
+};
+
+pub(crate) mod application;
+pub(crate) mod http;
+pub(crate) mod types;
+
+/// Kademlia DHT key
+pub(crate) type DhtKey = Vec<u8>;
+
+/// Kademlia DHT value
+pub(crate) type DhtValue = Vec<u8>;
+
+/// Kademlia DHT key/value pair
+pub(crate) type DhtKV = (DhtKey, DhtValue);
+
+pub(crate) type Result<T> = std::result::Result<T, ApiError>;
+
+#[derive(Debug)]
+pub(crate) enum ToEphemeraApiCmd {
+    SubmitEphemeraMessage(Box<ApiEphemeraMessage>, oneshot::Sender<Result<()>>),
+    QueryBlockByHeight(u64, oneshot::Sender<Result<Option<ApiBlock>>>),
+    QueryBlockByHash(String, oneshot::Sender<Result<Option<ApiBlock>>>),
+    QueryLastBlock(oneshot::Sender<Result<ApiBlock>>),
+    QueryBlockCertificates(String, oneshot::Sender<Result<Option<Vec<ApiCertificate>>>>),
+    QueryDht(DhtKey, oneshot::Sender<Result<Option<DhtKV>>>),
+    StoreInDht(DhtKey, DhtValue, oneshot::Sender<Result<()>>),
+    QueryEphemeraConfig(oneshot::Sender<Result<ApiEphemeraConfig>>),
+    QueryBroadcastGroup(oneshot::Sender<Result<ApiBroadcastInfo>>),
+    QueryBlockBroadcastInfo(
+        String,
+        oneshot::Sender<Result<Option<ApiBlockBroadcastInfo>>>,
+    ),
+    VerifyMessageInBlock(String, String, usize, oneshot::Sender<Result<bool>>),
+}
+
+impl Display for ToEphemeraApiCmd {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ToEphemeraApiCmd::SubmitEphemeraMessage(message, _) => {
+                write!(f, "SubmitEphemeraMessage({message})",)
+            }
+            ToEphemeraApiCmd::QueryBlockByHeight(height, _) => {
+                write!(f, "QueryBlockByHeight({height})",)
+            }
+            ToEphemeraApiCmd::QueryBlockByHash(hash, _) => write!(f, "QueryBlockByHash({hash})",),
+            ToEphemeraApiCmd::QueryLastBlock(_) => write!(f, "QueryLastBlock"),
+            ToEphemeraApiCmd::QueryBlockCertificates(id, _) => {
+                write!(f, "QueryBlockSignatures{id}")
+            }
+            ToEphemeraApiCmd::QueryDht(_, _) => {
+                write!(f, "QueryDht")
+            }
+            ToEphemeraApiCmd::StoreInDht(_, _, _) => {
+                write!(f, "StoreInDht")
+            }
+            ToEphemeraApiCmd::QueryEphemeraConfig(_) => {
+                write!(f, "EphemeraConfig")
+            }
+            ToEphemeraApiCmd::QueryBroadcastGroup(_) => {
+                write!(f, "BroadcastGroup")
+            }
+            ToEphemeraApiCmd::QueryBlockBroadcastInfo(hash, ..) => {
+                write!(f, "BlockBroadcastInfo({hash})")
+            }
+            ToEphemeraApiCmd::VerifyMessageInBlock(block_id, message_id, height, _) => {
+                write!(
+                    f,
+                    "VerifyMessageInBlock({block_id}, {message_id}, {height})",
+                )
+            }
+        }
+    }
+}
+
+pub(crate) struct ApiListener {
+    pub(crate) messages_rcv: Receiver<ToEphemeraApiCmd>,
+}
+
+impl ApiListener {
+    pub(crate) fn new(messages_rcv: Receiver<ToEphemeraApiCmd>) -> Self {
+        Self { messages_rcv }
+    }
+}
+
+#[derive(Clone)]
+pub struct CommandExecutor {
+    pub(crate) commands_channel: Sender<ToEphemeraApiCmd>,
+}
+
+impl CommandExecutor {
+    pub(crate) fn new() -> (CommandExecutor, ApiListener) {
+        let (commands_channel, signed_messages_rcv) = channel(100);
+        let api_listener = ApiListener::new(signed_messages_rcv);
+        let api = CommandExecutor { commands_channel };
+        (api, api_listener)
+    }
+
+    /// Returns block with given id if it exists
+    ///
+    /// # Arguments
+    /// * `block_id` - Block id
+    ///
+    /// # Returns
+    /// * `ApiBlock` - Block
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_block_by_id(&self, block_id: String) -> Result<Option<ApiBlock>> {
+        trace!("get_block_by_id({:?})", block_id);
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::QueryBlockByHash(block_id, tx))
+            .await
+    }
+
+    /// Returns block with given height if it exists
+    ///
+    /// # Arguments
+    /// * `height` - Block height
+    ///
+    /// # Returns
+    /// * `ApiBlock` - Block
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_block_by_height(&self, height: u64) -> Result<Option<ApiBlock>> {
+        trace!("get_block_by_height({:?})", height);
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::QueryBlockByHeight(height, tx))
+            .await
+    }
+
+    /// Returns last block. Which has maximum height and is stored in database
+    ///
+    /// # Returns
+    /// * `ApiBlock` - Last block
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_last_block(&self) -> Result<ApiBlock> {
+        trace!("get_last_block()");
+        self.send_and_wait_response(ToEphemeraApiCmd::QueryLastBlock)
+            .await
+    }
+
+    /// Returns signatures for given block id
+    ///
+    /// # Arguments
+    /// * `block_hash` - Block id
+    ///
+    /// # Returns
+    /// * `Vec<ApiCertificate>` - Certificates
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_block_certificates(
+        &self,
+        block_hash: String,
+    ) -> Result<Option<Vec<ApiCertificate>>> {
+        trace!("get_block_certificates({block_hash:?})",);
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::QueryBlockCertificates(block_hash, tx))
+            .await
+    }
+
+    /// Queries DHT for given key
+    ///
+    /// # Arguments
+    /// * `key` - DHT key
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    ///
+    /// # Returns
+    /// * `Some((key, value))` - If key is found
+    /// * `None` - If key is not found
+    pub async fn query_dht(&self, key: DhtKey) -> Result<Option<(DhtKey, DhtValue)>> {
+        trace!("get_dht({key:?})");
+        //TODO: this needs timeout(somewhere around dht query functionality)
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::QueryDht(key, tx))
+            .await
+    }
+
+    /// Stores given key-value pair in DHT
+    ///
+    /// # Arguments
+    /// * `key` - DHT key
+    /// * `value` - DHT value
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn store_in_dht(&self, key: DhtKey, value: DhtValue) -> Result<()> {
+        trace!("store_in_dht({key:?}, {value:?})");
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::StoreInDht(key, value, tx))
+            .await
+    }
+
+    /// Returns node configuration
+    ///
+    /// # Returns
+    /// * `ApiEphemeraConfig` - Node configuration
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_node_config(&self) -> Result<ApiEphemeraConfig> {
+        trace!("get_node_config()");
+        self.send_and_wait_response(ToEphemeraApiCmd::QueryEphemeraConfig)
+            .await
+    }
+
+    /// Returns broadcast group
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    ///
+    /// # Return
+    /// * `ApiBroadcastInfo` - Broadcast group
+    pub async fn get_broadcast_info(&self) -> Result<ApiBroadcastInfo> {
+        trace!("get_broadcast_group()");
+        self.send_and_wait_response(ToEphemeraApiCmd::QueryBroadcastGroup)
+            .await
+    }
+
+    /// Returns block broadcast info.
+    ///
+    /// # Arguments
+    /// * `block_hash` - Block hash
+    ///
+    /// # Return
+    /// * `ApiBlockBroadcastInfo` - Block broadcast info
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn get_block_broadcast_info(
+        &self,
+        block_hash: String,
+    ) -> Result<Option<ApiBlockBroadcastInfo>> {
+        trace!("get_broadcast_group()");
+        self.send_and_wait_response(|tx| ToEphemeraApiCmd::QueryBlockBroadcastInfo(block_hash, tx))
+            .await
+    }
+
+    /// Send a message to Ephemera which should then be included in mempool  and broadcast to all peers
+    ///
+    /// # Arguments
+    /// * `message` - Message to be sent
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn send_ephemera_message(&self, message: ApiEphemeraMessage) -> Result<()> {
+        trace!("send_ephemera_message({message})",);
+        self.send_and_wait_response(|tx| {
+            ToEphemeraApiCmd::SubmitEphemeraMessage(message.into(), tx)
+        })
+        .await
+    }
+
+    /// Verifies if given message is in block identified by block hash
+    /// Returns true if message is in block, false otherwise. False can also mean that block or message
+    /// does not exist.
+    ///
+    /// # Arguments
+    /// * `request` - Message and block hash
+    ///
+    /// # Errors
+    /// * `ApiError::InternalError` - If there is an internal error
+    pub async fn verify_message_in_block(&self, request: ApiVerifyMessageInBlock) -> Result<bool> {
+        trace!("verify_message_in_block({request})",);
+        let block_hash = request.block_hash;
+        let message_hash = request.message_hash;
+        let index = request.message_index;
+        self.send_and_wait_response(|tx| {
+            ToEphemeraApiCmd::VerifyMessageInBlock(block_hash, message_hash, index, tx)
+        })
+        .await
+    }
+
+    async fn send_and_wait_response<F, R>(&self, f: F) -> Result<R>
+    where
+        F: FnOnce(oneshot::Sender<Result<R>>) -> ToEphemeraApiCmd,
+        R: Send + 'static,
+    {
+        let (tx, rcv) = oneshot::channel();
+        let cmd = f(tx);
+        if let Err(e) = self.commands_channel.send(cmd).await {
+            error!("Failed to send command to Ephemera: {e:?}",);
+            return Err(ApiError::Internal(
+                "Failed to receive response from Ephemera".to_string(),
+            ));
+        }
+        rcv.await.map_err(|e| {
+            error!("Failed to receive response from Ephemera: {e:?}",);
+            ApiError::Internal("Failed to receive response from Ephemera".to_string())
+        })?
+    }
+}

ephemera/src/api/types.rs

@@ -0,0 +1,659 @@
+//! This module contains all the types that are used in the API.
+//!
+//! - `ApiEphemeraMessage`
+//! - `RawApiEphemeraMessage`
+//! - `ApiBlock`
+//! - `ApiCertificate`
+//! - `Health`
+//! - `ApiError`
+//! - `ApiEphemeraConfig`
+//! - `ApiDhtQueryRequest`
+//! - `ApiDhtQueryResponse`
+//! - `ApiDhtStoreRequest`
+//! - `ApiBroadcastInfo`
+//! - `ApiBlockBroadcastInfo`
+//! - `ApiVerifyMessageInBlock`
+
+use std::collections::HashSet;
+use std::fmt::Display;
+
+use array_bytes::{bytes2hex, hex2bytes};
+use log::error;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use utoipa::ToSchema;
+
+use crate::peer::{PeerId, ToPeerId};
+use crate::utilities::codec::{Codec, DecodingError, EncodingError, EphemeraCodec};
+use crate::{
+    block::types::{block::Block, block::BlockHeader, message::EphemeraMessage},
+    codec::{Decode, Encode},
+    crypto::{Keypair, PublicKey},
+    ephemera_api,
+    utilities::{
+        crypto::{Certificate, Signature},
+        time::EphemeraTime,
+    },
+};
+
+#[derive(Error, Debug)]
+pub enum ApiError {
+    #[error("Application rejected ephemera message")]
+    ApplicationRejectedMessage,
+    #[error("Duplicate message")]
+    DuplicateMessage,
+    #[error("Invalid hash: {0}")]
+    InvalidHash(String),
+    #[error("ApplicationError: {0}")]
+    Application(#[from] ephemera_api::ApplicationError),
+    #[error("Internal error: {0}")]
+    Internal(String),
+}
+
+/// # Ephemera message.
+///
+/// A message submitted to an Ephemera node will be gossiped to other nodes.
+/// And will be eventually included in a Ephemera block.
+///
+/// It needs to signed by the sender. The signature is included in the certificate.
+///
+/// The fields of the message what are signed:
+/// - timestamp
+/// - label
+/// - data
+///
+/// Currently it's up provided [`ephemera_api::application::Application`] to verify the signature.
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiEphemeraMessage {
+    /// The timestamp of the message.
+    pub timestamp: u64,
+    /// The label of the message. It can be used to identify the type of a message for example.
+    pub label: String,
+    /// The data of the message. It is application specific.
+    pub data: Vec<u8>,
+    /// The certificate of the message. All messages are required to be signed.
+    pub certificate: ApiCertificate,
+}
+
+impl ApiEphemeraMessage {
+    #[must_use]
+    pub fn new(raw_message: RawApiEphemeraMessage, certificate: ApiCertificate) -> Self {
+        Self {
+            timestamp: raw_message.timestamp,
+            label: raw_message.label,
+            data: raw_message.data,
+            certificate,
+        }
+    }
+
+    /// Generates the message hash.
+    ///
+    /// # Errors
+    /// - If internal hash function fails.
+    pub fn hash(&self) -> anyhow::Result<String> {
+        let em = EphemeraMessage::from(self.clone());
+        let hash = em.hash_with_default_hasher()?.to_string();
+        Ok(hash)
+    }
+}
+
+/// `RawApiEphemeraMessage` contains the fields of the `ApiEphemeraMessage` that are signed.
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct RawApiEphemeraMessage {
+    /// The timestamp of the message. It's initialized when the message is created.
+    /// It uses UTC time.
+    pub timestamp: u64,
+    /// The label of the message. It can be used to identify the type of a message without decoding full data.
+    pub label: String,
+    /// The data of the message. It is application specific.
+    pub data: Vec<u8>,
+}
+
+impl RawApiEphemeraMessage {
+    #[must_use]
+    pub fn new(label: String, data: Vec<u8>) -> Self {
+        Self {
+            timestamp: EphemeraTime::now(),
+            label,
+            data,
+        }
+    }
+
+    /// Signs the message with the given keypair.
+    ///
+    /// # Signing example
+    ///
+    /// ```
+    /// use ephemera::codec::Encode;
+    /// use ephemera::crypto::{EphemeraKeypair, EphemeraPublicKey, Keypair};
+    /// use ephemera::ephemera_api::{ApiEphemeraMessage, RawApiEphemeraMessage};
+    ///
+    /// let keypair = Keypair::generate(None);
+    /// let raw_message = RawApiEphemeraMessage::new("test".to_string(), vec![]);
+    ///
+    /// let signed_message:ApiEphemeraMessage = raw_message.sign(&keypair).unwrap();
+    ///
+    /// assert_eq!(signed_message.certificate.public_key, keypair.public_key().into());
+    ///
+    /// let bytes = raw_message.encode().unwrap();
+    /// assert!(keypair.public_key().verify(&bytes, &signed_message.certificate.signature.into()));
+    /// ```
+    ///
+    /// # Errors
+    /// - If the message can't be encoded.
+    /// - If the message can't be signed.
+    pub fn sign(&self, keypair: &Keypair) -> anyhow::Result<ApiEphemeraMessage> {
+        let certificate = Certificate::prepare(keypair, &self)?;
+        let message = ApiEphemeraMessage::new(self.clone(), certificate.into());
+        Ok(message)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
+pub struct ApiBlockHeader {
+    /// The timestamp of the block. It's initialized when the block is created.
+    /// It uses UTC time.
+    pub timestamp: u64,
+    /// The PeerId of the block producer instance.
+    pub creator: PeerId,
+    /// The height of the block.
+    pub height: u64,
+    /// The hash of the current block.
+    pub hash: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiBlock {
+    pub header: ApiBlockHeader,
+    pub messages: Vec<ApiEphemeraMessage>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
+pub struct ApiRawBlock {
+    pub(crate) header: ApiBlockHeader,
+    pub(crate) messages: Vec<ApiEphemeraMessage>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiCertificate {
+    pub signature: ApiSignature,
+    pub public_key: ApiPublicKey,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiSignature(pub(crate) Signature);
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiPublicKey(pub(crate) PublicKey);
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiEphemeraConfig {
+    /// The address of the node. It's the address what Ephemera instance uses to communicate with other nodes.
+    pub protocol_address: String,
+    /// The HTTP API address of the node.
+    pub api_address: String,
+    /// The WebSocket address of the node.
+    pub websocket_address: String,
+    /// Node's public key.
+    ///
+    /// # Converting to string and back example
+    /// ```
+    /// use ephemera::crypto::{EphemeraKeypair, Keypair, PublicKey};
+    ///
+    /// let keypair = Keypair::generate(None);
+    /// let public_key = keypair.public_key().to_string();
+    ///
+    /// let from_str = public_key.parse::<PublicKey>().unwrap();
+    ///
+    /// assert_eq!(keypair.public_key(), from_str);
+    /// ```
+    pub public_key: String,
+    /// True if the node is a block producer. It's a configuration option.
+    pub block_producer: bool,
+    /// The interval of block creation in seconds. It's a configuration option.
+    pub block_creation_interval_sec: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiDhtQueryRequest {
+    /// The key to query for in hex format.
+    key: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiDhtQueryResponse {
+    /// The key that was queried for in hex format.
+    key: String,
+    /// The value that was stored under the queried key in hex format.
+    value: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub enum HealthStatus {
+    Healthy,
+    Unhealthy,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiHealth {
+    pub(crate) status: HealthStatus,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiDhtStoreRequest {
+    /// The key to store the value under in hex format.
+    key: String,
+    /// The value to store in hex format.
+    value: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiBroadcastInfo {
+    /// The PeerId of the local node.
+    pub local_peer_id: PeerId,
+    /// The list of the current members of the network.
+    pub current_members: HashSet<PeerId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiBlockBroadcastInfo {
+    pub local_peer_id: PeerId,
+    pub broadcast_group: Vec<PeerId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, ToSchema)]
+pub struct ApiVerifyMessageInBlock {
+    pub block_hash: String,
+    pub message_hash: String,
+    pub message_index: usize,
+}
+
+impl ApiVerifyMessageInBlock {
+    #[must_use]
+    pub fn new(block_hash: String, message_hash: String, message_index: usize) -> Self {
+        Self {
+            block_hash,
+            message_hash,
+            message_index,
+        }
+    }
+}
+
+impl Display for ApiVerifyMessageInBlock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{{ block_hash: {}, message_hash: {} }}",
+            self.block_hash, self.message_hash
+        )
+    }
+}
+
+impl ApiBlockBroadcastInfo {
+    pub(crate) fn new(local_peer_id: PeerId, broadcast_group: Vec<PeerId>) -> Self {
+        Self {
+            local_peer_id,
+            broadcast_group,
+        }
+    }
+}
+
+impl ApiBroadcastInfo {
+    pub(crate) fn new(current_members: HashSet<PeerId>, local_peer_id: PeerId) -> Self {
+        Self {
+            local_peer_id,
+            current_members,
+        }
+    }
+}
+
+impl Display for ApiBroadcastInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let current_members = self.current_members.iter().map(ToString::to_string);
+        write!(
+            f,
+            "{{ local_peer_id: {}, current_members: {current_members:?} }}",
+            self.local_peer_id,
+        )
+    }
+}
+
+impl Display for ApiEphemeraMessage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ApiEphemeraMessage(timestamp: {}, label: {})",
+            self.timestamp, self.label,
+        )
+    }
+}
+
+impl From<ApiEphemeraMessage> for RawApiEphemeraMessage {
+    fn from(message: ApiEphemeraMessage) -> Self {
+        RawApiEphemeraMessage {
+            timestamp: message.timestamp,
+            label: message.label,
+            data: message.data,
+        }
+    }
+}
+
+impl From<ApiEphemeraMessage> for EphemeraMessage {
+    fn from(message: ApiEphemeraMessage) -> Self {
+        Self {
+            timestamp: message.timestamp,
+            label: message.label,
+            data: message.data,
+            certificate: message.certificate.into(),
+        }
+    }
+}
+
+impl Decode for RawApiEphemeraMessage {
+    type Output = Self;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+impl Encode for RawApiEphemeraMessage {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(self)
+    }
+}
+
+impl Encode for &RawApiEphemeraMessage {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(self)
+    }
+}
+
+impl Display for ApiBlockHeader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ApiBlockHeader(timestamp: {}, creator: {}, height: {}, hash: {})",
+            self.timestamp, self.creator, self.height, self.hash,
+        )
+    }
+}
+
+impl ApiBlock {
+    #[must_use]
+    pub fn as_raw_block(&self) -> ApiRawBlock {
+        ApiRawBlock {
+            header: self.header.clone(),
+            messages: self.messages.clone(),
+        }
+    }
+
+    #[must_use]
+    pub fn message_count(&self) -> usize {
+        self.messages.len()
+    }
+
+    #[must_use]
+    pub fn hash(&self) -> String {
+        self.header.hash.clone()
+    }
+
+    /// # Errors
+    /// - If the block is invalid.
+    /// - If the block's certificate is invalid.
+    /// - If the block's certificate is not signed by the block's creator.
+    pub fn verify(&self, certificate: &ApiCertificate) -> Result<bool, ApiError> {
+        let block: Block = self.clone().try_into()?;
+        let valid = block.verify(&(certificate.clone()).into()).map_err(|e| {
+            error!("Failed to verify block: {}", e);
+            ApiError::Internal("Failed to verify block certificate".to_string())
+        })?;
+        Ok(valid)
+    }
+}
+
+impl Display for ApiBlock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "ApiBlock(header: {}, message_count: {})",
+            self.header,
+            self.message_count()
+        )
+    }
+}
+
+impl ApiRawBlock {
+    pub fn new(header: ApiBlockHeader, messages: Vec<ApiEphemeraMessage>) -> Self {
+        Self { header, messages }
+    }
+}
+
+impl ApiCertificate {
+    /// # Errors
+    /// - `EncodingError` if the message cannot be encoded.
+    /// - `KeyPairError` if the message cannot be signed.
+    pub fn prepare<D: Encode>(key_pair: &Keypair, data: &D) -> anyhow::Result<Self> {
+        Certificate::prepare(key_pair, data).map(Into::into)
+    }
+
+    /// # Errors
+    /// -`EncodingError` if the message cannot be encoded.
+    pub fn verify<D: Encode>(&self, data: &D) -> anyhow::Result<bool> {
+        let certificate: Certificate = (self.clone()).into();
+        Certificate::verify(&certificate, data)
+    }
+}
+
+impl From<EphemeraMessage> for ApiEphemeraMessage {
+    fn from(ephemera_message: EphemeraMessage) -> Self {
+        Self {
+            timestamp: ephemera_message.timestamp,
+            label: ephemera_message.label,
+            data: ephemera_message.data,
+            certificate: ApiCertificate {
+                signature: ephemera_message.certificate.signature.into(),
+                public_key: ephemera_message.certificate.public_key.into(),
+            },
+        }
+    }
+}
+
+impl From<Certificate> for ApiCertificate {
+    fn from(signature: Certificate) -> Self {
+        Self {
+            signature: signature.signature.into(),
+            public_key: signature.public_key.into(),
+        }
+    }
+}
+
+impl From<ApiCertificate> for Certificate {
+    fn from(value: ApiCertificate) -> Self {
+        Certificate {
+            signature: value.signature.into(),
+            public_key: value.public_key.into(),
+        }
+    }
+}
+
+impl From<&Block> for &ApiBlock {
+    fn from(block: &Block) -> Self {
+        let api_block: ApiBlock = block.clone().into();
+        Box::leak(Box::new(api_block))
+    }
+}
+
+impl From<Signature> for ApiSignature {
+    fn from(signature: Signature) -> Self {
+        Self(signature)
+    }
+}
+
+impl From<ApiSignature> for Signature {
+    fn from(signature: ApiSignature) -> Self {
+        signature.0
+    }
+}
+
+impl ApiPublicKey {
+    pub fn peer_id(&self) -> String {
+        self.0.peer_id().to_string()
+    }
+}
+
+impl From<PublicKey> for ApiPublicKey {
+    fn from(public_key: PublicKey) -> Self {
+        Self(public_key)
+    }
+}
+
+impl From<ApiPublicKey> for PublicKey {
+    fn from(public_key: ApiPublicKey) -> Self {
+        public_key.0
+    }
+}
+
+impl From<Block> for ApiBlock {
+    fn from(block: Block) -> Self {
+        Self {
+            header: ApiBlockHeader {
+                timestamp: block.header.timestamp,
+                creator: block.header.creator,
+                height: block.header.height,
+                hash: block.header.hash.to_string(),
+            },
+            messages: block.messages.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+impl TryFrom<ApiBlock> for Block {
+    type Error = ApiError;
+
+    fn try_from(api_block: ApiBlock) -> Result<Self, ApiError> {
+        let messages: Vec<EphemeraMessage> = api_block
+            .messages
+            .into_iter()
+            .map(Into::into)
+            .collect::<Vec<EphemeraMessage>>();
+        Ok(Self {
+            header: BlockHeader {
+                timestamp: api_block.header.timestamp,
+                creator: api_block.header.creator,
+                height: api_block.header.height,
+                hash: api_block.header.hash.parse().map_err(|e| {
+                    error!("Failed to parse block hash: {}", e);
+                    ApiError::Internal("Failed to parse block hash".to_string())
+                })?,
+            },
+            messages,
+        })
+    }
+}
+
+impl ApiDhtStoreRequest {
+    #[must_use]
+    pub fn new(key: &[u8], value: &[u8]) -> Self {
+        let key = bytes2hex("0x", key);
+        let value = bytes2hex("0x", value);
+        Self { key, value }
+    }
+
+    #[allow(clippy::missing_panics_doc)]
+    #[must_use]
+    pub fn key(&self) -> Vec<u8> {
+        //We can unwrap here because the key is always valid.
+        hex2bytes(&self.key).unwrap()
+    }
+
+    #[allow(clippy::missing_panics_doc)]
+    #[must_use]
+    pub fn value(&self) -> Vec<u8> {
+        //We can unwrap here because the value is always valid.
+        hex2bytes(&self.value).unwrap()
+    }
+}
+
+impl ApiDhtQueryRequest {
+    #[must_use]
+    pub fn new(key: &[u8]) -> Self {
+        let key = bytes2hex("0x", key);
+        Self { key }
+    }
+
+    #[must_use]
+    pub fn key_encoded(&self) -> String {
+        self.key.clone()
+    }
+
+    #[allow(clippy::missing_panics_doc)]
+    #[must_use]
+    pub fn key(&self) -> Vec<u8> {
+        //We can unwrap here because the value is always valid.
+        hex2bytes(&self.key).unwrap()
+    }
+
+    pub(crate) fn parse_key(key: &str) -> Vec<u8> {
+        hex2bytes(key).unwrap()
+    }
+}
+
+impl ApiDhtQueryResponse {
+    pub(crate) fn new(key: Vec<u8>, value: Vec<u8>) -> Self {
+        let key = bytes2hex("0x", key);
+        let value = bytes2hex("0x", value);
+        Self { key, value }
+    }
+
+    #[allow(clippy::missing_panics_doc)]
+    #[must_use]
+    pub fn key(&self) -> Vec<u8> {
+        //We can unwrap here because the key is always valid.
+        hex2bytes(&self.key).unwrap()
+    }
+
+    #[allow(clippy::missing_panics_doc)]
+    #[must_use]
+    pub fn value(&self) -> Vec<u8> {
+        //We can unwrap here because the value is always valid.
+        hex2bytes(&self.value).unwrap()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::crypto::EphemeraKeypair;
+    use crate::crypto::Keypair;
+
+    use super::*;
+
+    #[test]
+    fn test_message_sign_ok() {
+        let message_signing_keypair = Keypair::generate(None);
+
+        let message = RawApiEphemeraMessage::new("test".to_string(), vec![1, 2, 3]);
+        let signed_message = message
+            .sign(&message_signing_keypair)
+            .expect("Failed to sign message");
+
+        let certificate = signed_message.certificate;
+
+        assert!(certificate.verify(&message).unwrap());
+    }
+
+    #[test]
+    fn test_message_sign_fail() {
+        let message_signing_keypair = Keypair::generate(None);
+
+        let message = RawApiEphemeraMessage::new("test1".to_string(), vec![1, 2, 3]);
+        let signed_message = message
+            .sign(&message_signing_keypair)
+            .expect("Failed to sign message");
+
+        let certificate = signed_message.certificate;
+
+        let modified_message = RawApiEphemeraMessage::new("test2".to_string(), vec![1, 2, 3]);
+        assert!(!certificate.verify(&modified_message).unwrap());
+    }
+}

ephemera/src/block/builder.rs

@@ -0,0 +1,74 @@
+use std::collections::HashSet;
+use std::{sync::Arc, time::Duration};
+
+use log::{debug, info};
+
+use crate::block::manager::State;
+use crate::peer::ToPeerId;
+use crate::{
+    block::{
+        manager::{BlockChainState, BlockManager},
+        message_pool::MessagePool,
+        producer::BlockProducer,
+        types::block::Block,
+    },
+    broadcast::signing::BlockSigner,
+    config::BlockManagerConfiguration,
+    crypto::Keypair,
+    storage::EphemeraDatabase,
+};
+
+pub(crate) struct BlockManagerBuilder {
+    config: BlockManagerConfiguration,
+    block_producer: BlockProducer,
+    keypair: Arc<Keypair>,
+}
+
+impl BlockManagerBuilder {
+    pub(crate) fn new(config: BlockManagerConfiguration, keypair: Arc<Keypair>) -> Self {
+        let block_producer = BlockProducer::new(keypair.peer_id());
+        Self {
+            config,
+            block_producer,
+            keypair,
+        }
+    }
+
+    pub(crate) fn build<D: EphemeraDatabase + ?Sized>(
+        self,
+        storage: &mut D,
+    ) -> anyhow::Result<BlockManager> {
+        let mut most_recent_block = storage.get_last_block()?;
+        if most_recent_block.is_none() {
+            //Although Ephemera is not a blockchain(chain of historically dependent blocks),
+            //it's helpful to have some sort of notion of progress in time. So we use the concept of height.
+            //The genesis block helps to define the start of it.
+
+            info!("No last block found in database. Creating genesis block.");
+
+            let genesis_block = Block::new_genesis_block(self.block_producer.peer_id);
+            storage.store_block(&genesis_block, HashSet::new(), HashSet::new())?;
+            most_recent_block = Some(genesis_block);
+        }
+
+        let last_created_block = most_recent_block.expect("Block should be present");
+        debug!("Most recent block: {:?}", last_created_block);
+
+        let block_signer = BlockSigner::new(self.keypair.clone());
+        let message_pool = MessagePool::new();
+        let block_chain_state = BlockChainState::new(last_created_block);
+        let block_creation_interval =
+            tokio::time::interval(Duration::from_secs(self.config.creation_interval_sec));
+
+        Ok(BlockManager {
+            config: self.config,
+            block_producer: self.block_producer,
+            block_signer,
+            message_pool,
+            block_chain_state,
+            state: State::Paused,
+            backoff: None,
+            block_creation_interval,
+        })
+    }
+}

ephemera/src/block/manager.rs

@@ -0,0 +1,688 @@
+use std::collections::HashSet;
+use std::future::Future;
+use std::task::Poll;
+use std::time::Duration;
+use std::{
+    num::NonZeroUsize,
+    pin::Pin,
+    task,
+    task::Poll::{Pending, Ready},
+};
+
+use anyhow::anyhow;
+use futures::Stream;
+use futures_util::FutureExt;
+use log::{debug, error, info, trace};
+use lru::LruCache;
+use thiserror::Error;
+use tokio::time;
+use tokio::time::{Instant, Interval};
+
+use crate::network::PeerId;
+use crate::peer::ToPeerId;
+use crate::{
+    api::application::RemoveMessages,
+    block::{
+        message_pool::MessagePool,
+        producer::BlockProducer,
+        types::{block::Block, message::EphemeraMessage},
+    },
+    broadcast::signing::BlockSigner,
+    config::BlockManagerConfiguration,
+    utilities::{crypto::Certificate, hash::Hash},
+};
+
+pub(crate) type Result<T> = std::result::Result<T, BlockManagerError>;
+
+#[derive(Error, Debug)]
+pub(crate) enum BlockManagerError {
+    #[error("Message is already in pool: {0}")]
+    DuplicateMessage(String),
+    //Just a placeholder for now
+    #[error("BlockManagerError: {0}")]
+    BlockManager(#[from] anyhow::Error),
+}
+
+/// It helps to use atomic state management for new blocks.
+pub(crate) struct BlockChainState {
+    pub(crate) last_blocks: LruCache<Hash, Block>,
+    /// Last block that we created.
+    /// It's not Option because we always have genesis block
+    last_produced_block: Option<Block>,
+    /// Last block that we accepted
+    /// It's not Option because we always have genesis block
+    last_committed_block: Block,
+}
+
+impl BlockChainState {
+    pub(crate) fn new(last_committed_block: Block) -> Self {
+        Self {
+            //1000 is just a "big enough".
+            last_blocks: LruCache::new(NonZeroUsize::new(1000).unwrap()),
+            last_produced_block: None,
+            last_committed_block,
+        }
+    }
+
+    fn mark_last_produced_block_as_committed(&mut self) {
+        self.last_committed_block = self
+            .last_produced_block
+            .take()
+            .expect("Block should be present");
+    }
+
+    fn is_last_produced_block(&self, hash: Hash) -> bool {
+        match self.last_produced_block.as_ref() {
+            Some(block) => block.get_hash() == hash,
+            None => false,
+        }
+    }
+
+    fn is_last_produced_block_is_pending(&self) -> bool {
+        self.last_produced_block.is_some()
+    }
+
+    fn next_block_height(&self) -> u64 {
+        self.last_committed_block.get_height() + 1
+    }
+
+    fn remove_last_produced_block(&mut self) -> Block {
+        self.last_produced_block
+            .take()
+            .expect("Block should be present")
+    }
+}
+
+pub(crate) enum State {
+    Paused,
+    Running,
+}
+
+#[derive(Debug)]
+pub(crate) struct BackOffInterval {
+    /// Maximum number of attempts before this backoff expires.
+    maximum_times: u32,
+    /// Number of attempts that have been made so far.
+    nr_of_attempts: u32,
+    /// Backoff rate. Previous delay is multiplied by this rate to get next delay.
+    backoff_rate: u32,
+    /// Delay between before next attempt.
+    delay: Interval,
+}
+
+impl BackOffInterval {
+    fn new(maximum_times: u32, backoff_rate: u32, initial_wait: Duration) -> Self {
+        let delay = time::interval_at(Instant::now() + initial_wait, initial_wait);
+        Self {
+            maximum_times,
+            nr_of_attempts: 0,
+            backoff_rate,
+            delay,
+        }
+    }
+
+    fn is_expired(&self) -> bool {
+        self.nr_of_attempts >= self.maximum_times
+    }
+}
+
+impl Future for BackOffInterval {
+    type Output = ();
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
+        if self.nr_of_attempts >= self.maximum_times {
+            debug!("Backoff expired after {} attempts", self.nr_of_attempts);
+            return Pending;
+        }
+
+        match Pin::new(&mut self.delay).poll_tick(cx) {
+            Ready(_) => {
+                self.nr_of_attempts += 1;
+                let next_tick = Instant::now()
+                    + self.delay.period() * self.backoff_rate.pow(self.nr_of_attempts);
+                debug!("Backoff attempt: {}", self.nr_of_attempts);
+                self.delay = time::interval_at(next_tick, self.delay.period());
+                Ready(())
+            }
+            Pending => Pending,
+        }
+    }
+}
+
+pub(crate) struct BlockManager {
+    pub(crate) config: BlockManagerConfiguration,
+    /// Block producer. Simple helper that creates blocks
+    pub(crate) block_producer: BlockProducer,
+    /// Message pool. Contains all messages that we received from the network and not included in any(committed) block yet.
+    pub(crate) message_pool: MessagePool,
+    /// Delay between block creation attempts.
+    pub(crate) block_creation_interval: Interval,
+    /// Backoff between block creation attempts. When `last_produced_block` is not committed during
+    /// certain time window, and normal delay is not passed yet, we use backoff delay to try again.
+    pub(crate) backoff: Option<BackOffInterval>,
+    /// Signs and verifies blocks
+    pub(crate) block_signer: BlockSigner,
+    /// State management for new blocks
+    pub(crate) block_chain_state: BlockChainState,
+    /// Current state of the block manager
+    pub(crate) state: State,
+}
+
+impl BlockManager {
+    pub(crate) fn on_new_message(&mut self, msg: EphemeraMessage) -> Result<()> {
+        trace!("Message received: {:?}", msg);
+
+        let message_hash = msg.hash_with_default_hasher()?;
+        if self.message_pool.contains(&message_hash) {
+            return Err(BlockManagerError::DuplicateMessage(
+                message_hash.to_string(),
+            ));
+        }
+
+        self.message_pool.add_message(msg)?;
+        Ok(())
+    }
+
+    pub(crate) fn on_block(
+        &mut self,
+        sender: &PeerId,
+        block: &Block,
+        certificate: &Certificate,
+    ) -> Result<()> {
+        let hash = block.hash_with_default_hasher()?;
+
+        trace!(
+            "Received block: {:?} from peer {sender:?}",
+            block.get_hash()
+        );
+
+        //Reject blocks with invalid hash
+        if block.header.hash != hash {
+            return Err(anyhow!("Block hash is invalid: {} != {hash}", block.header.hash).into());
+        }
+
+        //Block signer should be also its sender
+        let signer_peer_id = certificate.public_key.peer_id();
+        if *sender != signer_peer_id {
+            return Err(anyhow!(
+                "Block signer is not the block sender: {sender:?} != {signer_peer_id:?}",
+            )
+            .into());
+        }
+
+        //Verify that block signature is valid
+        if self.block_signer.verify_block(block, certificate).is_err() {
+            return Err(anyhow!("Block signature is invalid: {hash}").into());
+        }
+
+        self.block_chain_state.last_blocks.put(hash, block.clone());
+        Ok(())
+    }
+
+    pub(crate) fn sign_block(&mut self, block: &Block) -> Result<Certificate> {
+        let hash = block.hash_with_default_hasher()?;
+
+        trace!("Signing block: {block}");
+
+        let certificate = self.block_signer.sign_block(block, &hash)?;
+
+        trace!("Block certificate: {certificate:?}",);
+
+        Ok(certificate)
+    }
+
+    pub(crate) fn on_application_rejected_block(
+        &mut self,
+        messages_to_remove: RemoveMessages,
+    ) -> Result<()> {
+        debug!("Application rejected last created block");
+
+        let last_produced_block = self.block_chain_state.remove_last_produced_block();
+        match messages_to_remove {
+            RemoveMessages::All => {
+                let messages = last_produced_block
+                    .messages
+                    .into_iter()
+                    .map(Into::into)
+                    .collect::<Vec<_>>();
+
+                debug!("Removing block messages from pool: all: {messages:?}",);
+                self.message_pool.remove_messages(&messages)?;
+            }
+            RemoveMessages::Selected(messages) => {
+                debug!("Removing block messages from pool: selected: {messages:?}",);
+                let messages = messages.into_iter().map(Into::into).collect::<Vec<_>>();
+                self.message_pool.remove_messages(messages.as_slice())?;
+            }
+        };
+        Ok(())
+    }
+
+    /// After a block gets committed, clear up mempool from its messages
+    pub(crate) fn on_block_committed(&mut self, block: &Block) -> Result<()> {
+        info!("Block committed: {}", block);
+
+        let hash = &block.header.hash;
+
+        if !self.block_chain_state.is_last_produced_block(*hash) {
+            let last_produced_block = self
+                .block_chain_state
+                .last_produced_block
+                .as_ref()
+                .expect("Last produced block should be present");
+            log::error!(
+                "Received unexpected committed block: {hash}, was expecting: {}",
+                last_produced_block.get_hash()
+            );
+            panic!("Received committed block which isn't last produced block, this is a bug!");
+        }
+
+        match self.message_pool.remove_messages(&block.messages) {
+            Ok(_) => {
+                self.block_chain_state
+                    .mark_last_produced_block_as_committed();
+            }
+            Err(e) => {
+                return Err(anyhow!("Failed to remove messages from mempool: {}", e).into());
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn get_block_by_hash(&mut self, block_id: &Hash) -> Option<Block> {
+        self.block_chain_state.last_blocks.get(block_id).cloned()
+    }
+
+    pub(crate) fn get_block_certificates(&mut self, hash: &Hash) -> Option<&HashSet<Certificate>> {
+        self.block_signer.get_block_certificates(hash)
+    }
+
+    pub(crate) fn stop(&mut self) {
+        debug!("Stopping block creation");
+        self.state = State::Paused;
+        self.backoff = None;
+    }
+
+    pub(crate) fn start(&mut self) {
+        if !self.config.producer {
+            return;
+        }
+        if let State::Running = self.state {
+            return;
+        }
+        debug!("Starting block creation");
+        self.state = State::Running;
+        self.block_creation_interval =
+            tokio::time::interval(Duration::from_secs(self.config.creation_interval_sec));
+    }
+}
+
+//Produces blocks at a predefined interval.
+//If blocks will be actually broadcast depends on the application.
+impl Stream for BlockManager {
+    type Item = (Block, Certificate);
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut task::Context) -> Poll<Option<Self::Item>> {
+        //Optionally it is possible to turn off block production and let the node behave just as voter.
+        //For example for testing purposes.
+        if !self.config.producer {
+            return Pending;
+        }
+
+        //It is dynamically turned off when node is not part of most recent broadcast group.
+        if let State::Paused = self.state {
+            return Pending;
+        }
+
+        let is_previous_pending = self.block_chain_state.is_last_produced_block_is_pending();
+        if !is_previous_pending {
+            self.backoff = None;
+        }
+
+        if self.block_creation_interval.poll_tick(cx).is_pending() {
+            if let Some(mut backoff) = self.backoff.take() {
+                if backoff.is_expired() {
+                    return Pending;
+                }
+                if backoff.poll_unpin(cx).is_pending() {
+                    self.backoff = Some(backoff);
+                    return Pending;
+                }
+                self.backoff = Some(backoff);
+            } else {
+                return Pending;
+            }
+        } else {
+            self.backoff = None;
+        }
+
+        //If backoff is expired and we still don't have previous block committed
+        let repeat_previous = is_previous_pending && self.config.repeat_last_block_messages;
+
+        let pending_messages = if repeat_previous {
+            let block = self
+                .block_chain_state
+                .last_produced_block
+                .clone()
+                .expect("Block should be present");
+
+            //Use only previous block messages but create new block with new timestamp.
+            debug!("Producing block with previous messages");
+            block.messages
+        } else {
+            debug!("Producing block with new messages");
+            self.message_pool.get_messages()
+        };
+
+        let new_height = self.block_chain_state.next_block_height();
+        let created_block = self
+            .block_producer
+            .create_block(new_height, pending_messages);
+
+        if let Ok(block) = created_block {
+            info!("Created block: {}", block);
+
+            let hash = block.get_hash();
+            self.block_chain_state.last_produced_block = Some(block.clone());
+            self.block_chain_state.last_blocks.put(hash, block.clone());
+
+            let certificate = self
+                .block_signer
+                .sign_block(&block, &hash)
+                .expect("Failed to sign block");
+
+            if self.backoff.is_none() {
+                let backoff = BackOffInterval::new(100, 2, Duration::from_secs(10));
+                self.backoff = Some(backoff);
+            }
+
+            Ready(Some((block, certificate)))
+        } else {
+            error!("Error producing block: {:?}", created_block);
+            Pending
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use assert_matches::assert_matches;
+    use futures_util::StreamExt;
+
+    use crate::crypto::{EphemeraKeypair, Keypair};
+    use crate::ephemera_api::RawApiEphemeraMessage;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_add_message() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+        let hash = signed_message.hash_with_default_hasher().unwrap();
+
+        manager.on_new_message(signed_message).unwrap();
+
+        assert!(manager.message_pool.contains(&hash));
+    }
+
+    #[tokio::test]
+    async fn test_add_duplicate_message() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+
+        manager.on_new_message(signed_message.clone()).unwrap();
+
+        assert_matches!(
+            manager.on_new_message(signed_message),
+            Err(BlockManagerError::DuplicateMessage(_))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_accept_valid_block() {
+        let (mut manager, peer_id) = block_manager_with_defaults();
+
+        let block = block();
+        let certificate = manager.sign_block(&block).unwrap();
+
+        let result = manager.on_block(&peer_id, &block, &certificate);
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_reject_invalid_sender() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let block = block();
+        let certificate = manager.sign_block(&block).unwrap();
+
+        let invalid_peer_id = PeerId::from_public_key(&Keypair::generate(None).public_key());
+        let result = manager.on_block(&invalid_peer_id, &block, &certificate);
+
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_reject_invalid_hash() {
+        let (mut manager, peer_id) = block_manager_with_defaults();
+
+        let mut block = block();
+        let certificate = manager.sign_block(&block).unwrap();
+
+        block.header.hash = Hash::new([0; 32]);
+        let result = manager.on_block(&peer_id, &block, &certificate);
+
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_reject_invalid_signature() {
+        let (mut manager, peer_id) = block_manager_with_defaults();
+
+        let correct_block = block();
+        let fake_block = block();
+
+        let fake_certificate = manager.sign_block(&fake_block).unwrap();
+        let correct_certificate = manager.sign_block(&correct_block).unwrap();
+
+        let result = manager.on_block(&peer_id, &correct_block, &fake_certificate);
+        assert!(result.is_err());
+
+        let result = manager.on_block(&peer_id, &fake_block, &correct_certificate);
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_next_block_empty() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let (block, _) = manager.next().await.unwrap();
+        assert_eq!(block.header.height, 1);
+        assert!(block.messages.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_next_block_with_message() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        match manager.next().await {
+            Some((block, _)) => {
+                assert_eq!(block.header.height, 1);
+                assert_eq!(block.messages.len(), 1);
+            }
+            None => {
+                panic!("No block produced");
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_next_block_previous_not_committed_repeat() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let (block1, _) = manager.next().await.unwrap();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let (block2, _) = manager.next().await.unwrap();
+
+        assert_eq!(block1.messages.len(), block2.messages.len());
+        assert_eq!(block1.header.height, block2.header.height);
+    }
+
+    #[tokio::test]
+    async fn test_next_block_previous_not_committed_repeat_false() {
+        let config = BlockManagerConfiguration::new(true, 0, false);
+        let (mut manager, _) = block_manager_with_config(config);
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let (block1, _) = manager.next().await.unwrap();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let (block2, _) = manager.next().await.unwrap();
+
+        assert_eq!(block1.messages.len(), 1);
+        assert_eq!(block2.messages.len(), 2);
+        //We create new block but don't leave gap
+        assert_eq!(block1.header.height, block2.header.height);
+    }
+
+    #[tokio::test]
+    async fn test_on_committed_with_correct_pending_block() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let (block, _) = manager.next().await.unwrap();
+
+        let result = manager.on_block_committed(&block);
+
+        assert!(result.is_ok());
+        assert!(manager.message_pool.get_messages().is_empty());
+    }
+
+    #[tokio::test]
+    #[should_panic]
+    async fn test_on_committed_with_invalid_pending_block() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        manager.next().await.unwrap();
+
+        //Create invalid block
+        let wrong_block = block();
+
+        //This shouldn't remove messages from the pool
+        manager.on_block_committed(&wrong_block).unwrap();
+    }
+
+    #[tokio::test]
+    async fn application_rejected_messages_all() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        //Add messages to pool
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        let signed_message = message("test");
+        manager.on_new_message(signed_message).unwrap();
+
+        //Produce new block
+        manager.next().await.unwrap();
+
+        //Application Rejects the block with ALL messages
+        manager
+            .on_application_rejected_block(RemoveMessages::All)
+            .unwrap();
+
+        assert!(manager.message_pool.get_messages().is_empty());
+    }
+
+    #[tokio::test]
+    async fn application_rejected_messages_selected() {
+        let (mut manager, _) = block_manager_with_defaults();
+
+        //Add messages to pool
+        let signed_message1 = message("test");
+        manager.on_new_message(signed_message1.clone()).unwrap();
+
+        let signed_message2 = message("test");
+        manager.on_new_message(signed_message2.clone()).unwrap();
+
+        //Produce new block
+        manager.next().await.unwrap();
+
+        //Application Rejects the block with ALL messages
+        manager
+            .on_application_rejected_block(RemoveMessages::Selected(vec![signed_message2.into()]))
+            .unwrap();
+
+        assert_eq!(manager.message_pool.get_messages().len(), 1);
+        let message = manager
+            .message_pool
+            .get_messages()
+            .into_iter()
+            .next()
+            .unwrap();
+        assert_eq!(message, signed_message1);
+    }
+
+    fn block_manager_with_defaults() -> (BlockManager, PeerId) {
+        let config = BlockManagerConfiguration::new(true, 0, true);
+        block_manager_with_config(config)
+    }
+
+    fn block_manager_with_config(config: BlockManagerConfiguration) -> (BlockManager, PeerId) {
+        let keypair: Arc<Keypair> = Keypair::generate(None).into();
+        let peer_id = keypair.public_key().peer_id();
+        let genesis_block = Block::new_genesis_block(peer_id);
+        let block_chain_state = BlockChainState::new(genesis_block);
+        (
+            BlockManager {
+                config,
+                block_producer: BlockProducer::new(peer_id),
+                message_pool: MessagePool::new(),
+                block_creation_interval: tokio::time::interval(Duration::from_millis(1)),
+                backoff: None,
+                block_signer: BlockSigner::new(keypair),
+                block_chain_state,
+                state: State::Running,
+            },
+            peer_id,
+        )
+    }
+
+    fn block() -> Block {
+        let keypair: Arc<Keypair> = Keypair::generate(None).into();
+        let peer_id = keypair.public_key().peer_id();
+        let mut producer = BlockProducer::new(peer_id);
+        producer.create_block(1, vec![]).unwrap()
+    }
+
+    fn message(label: &str) -> EphemeraMessage {
+        let message1 = RawApiEphemeraMessage::new(label.into(), vec![1, 2, 3]);
+        let keypair = Keypair::generate(None);
+        let signed_message1 = message1.sign(&keypair).expect("Failed to sign message");
+        signed_message1.into()
+    }
+}

ephemera/src/block/message_pool.rs

@@ -0,0 +1,87 @@
+//! Message pool for Ephemera messages
+//!
+//! It stores pending Ephemera messages which will be added to a future block.
+//! It doesn't have any other logic than just storing messages.
+//!
+//! It's up to the user provided [`crate::ephemera_api::Application::check_tx`] to decide which messages to include.
+
+use std::collections::HashMap;
+
+use log::{trace, warn};
+
+use crate::block::types::message::EphemeraMessage;
+use crate::utilities::hash::Hash;
+
+pub(crate) struct MessagePool {
+    pending_messages: HashMap<Hash, EphemeraMessage>,
+}
+
+impl MessagePool {
+    pub(super) fn new() -> Self {
+        Self {
+            pending_messages: HashMap::default(),
+        }
+    }
+
+    pub(crate) fn contains(&self, hash: &Hash) -> bool {
+        self.pending_messages.contains_key(hash)
+    }
+
+    pub(super) fn add_message(&mut self, msg: EphemeraMessage) -> anyhow::Result<()> {
+        trace!("Adding message to pool: {:?}", msg);
+
+        let msg_hash = msg.hash_with_default_hasher()?;
+
+        self.pending_messages.insert(msg_hash, msg);
+
+        trace!("Message pool size: {:?}", self.pending_messages.len());
+        Ok(())
+    }
+
+    pub(super) fn remove_messages(&mut self, messages: &[EphemeraMessage]) -> anyhow::Result<()> {
+        trace!(
+            "Mempool size before removing messages {}",
+            self.pending_messages.len()
+        );
+        for msg in messages {
+            let hash = msg.hash_with_default_hasher()?;
+            if self.pending_messages.remove(&hash).is_none() {
+                warn!("Message not found in pool: {:?}", msg);
+            }
+        }
+        trace!(
+            "Mempool size after removing messages {}",
+            self.pending_messages.len()
+        );
+        Ok(())
+    }
+
+    /// Returns a `Vec` of all `EphemeraMessage`s in the message pool.
+    /// The message pool is not cleared.
+    pub(super) fn get_messages(&self) -> Vec<EphemeraMessage> {
+        self.pending_messages.values().cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::block::message_pool::MessagePool;
+    use crate::block::types::message::EphemeraMessage;
+    use crate::crypto::{EphemeraKeypair, Keypair};
+    use crate::ephemera_api::RawApiEphemeraMessage;
+
+    #[test]
+    fn test_add_remove() {
+        let keypair = Keypair::generate(None);
+
+        let message = RawApiEphemeraMessage::new("test".to_string(), vec![1, 2, 3]);
+        let signed_message = message.sign(&keypair).expect("Failed to sign message");
+        let signed_message: EphemeraMessage = signed_message.into();
+
+        let mut pool = MessagePool::new();
+        pool.add_message(signed_message.clone()).unwrap();
+        pool.remove_messages(&[signed_message]).unwrap();
+
+        assert_eq!(pool.get_messages().len(), 0);
+    }
+}

ephemera/src/block/mod.rs

@@ -0,0 +1,26 @@
+//! # Block manager
+//!
+//! Block manager is quite simple. It keeps pending messages in memory and puts all of them into a block
+//! at predefined intervals. That's all it does.
+//!
+//! If the block actually will be broadcast or not is decided by the application. If not, it will produce next block with
+//! the same messages plus the new ones.
+//!
+//! When application shuts down, pending messages are lost.
+//!
+//! When a block gets accepted by reliable broadcast then Block Manager will remove all messages included in the block from the
+//! pending messages queue.
+//!
+//! # Synchronization and duplicate messages in sequence of blocks
+//!
+//! When previous block hasn't been accepted yet, then the next block will contain the same messages as the previous one.
+//! One way to solve this is that an application itself keeps track of duplicate messages and discards them if necessary.
+//!
+//! But it seems a reasonable assumption that in general duplicate messages are unwanted. Therefore, Ephemera solves this
+//! by dropping previous blocks which get Finalised/Committed after a new block has been created.
+
+pub(crate) mod builder;
+pub(crate) mod manager;
+pub(crate) mod message_pool;
+pub(crate) mod producer;
+pub(crate) mod types;

ephemera/src/block/producer.rs

@@ -0,0 +1,92 @@
+use crate::block::{
+    types::block::{Block, RawBlock, RawBlockHeader},
+    types::message::EphemeraMessage,
+};
+use crate::peer::PeerId;
+use log::trace;
+
+pub(crate) struct BlockProducer {
+    pub(crate) peer_id: PeerId,
+}
+
+impl BlockProducer {
+    pub(super) fn new(peer_id: PeerId) -> Self {
+        Self { peer_id }
+    }
+
+    pub(super) fn create_block(
+        &mut self,
+        height: u64,
+        pending_messages: Vec<EphemeraMessage>,
+    ) -> anyhow::Result<Block> {
+        trace!("Pending messages for new block: {:?}", pending_messages);
+        let block = self.new_block(height, pending_messages)?;
+        Ok(block)
+    }
+
+    fn new_block(&self, height: u64, mut messages: Vec<EphemeraMessage>) -> anyhow::Result<Block> {
+        //Ordering is fundamental for block hash. Simple sort is fine for now.
+        messages.sort();
+
+        let raw_header = RawBlockHeader::new(self.peer_id, height);
+        let raw_block = RawBlock::new(raw_header, messages);
+
+        //Better idea is probably combine header hash with Merkle tree root hash
+        let block_hash = raw_block.hash_with_default_hasher()?;
+
+        let block = Block::new(raw_block, block_hash);
+        Ok(block)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::crypto::{EphemeraKeypair, Keypair};
+    use crate::ephemera_api::RawApiEphemeraMessage;
+    use std::cmp::Ordering;
+
+    use super::*;
+
+    #[test]
+    fn test_produce_block() {
+        let peer_id = PeerId::random();
+
+        let mut block_producer = BlockProducer::new(peer_id);
+
+        let message = RawApiEphemeraMessage::new("test".to_string(), vec![1, 2, 3]);
+        let signed_message = message
+            .sign(&Keypair::generate(None))
+            .expect("Failed to sign message");
+        let signed_message1: EphemeraMessage = signed_message.into();
+
+        let message = RawApiEphemeraMessage::new("test".to_string(), vec![1, 2, 3]);
+        let signed_message = message
+            .sign(&Keypair::generate(None))
+            .expect("Failed to sign message");
+        let signed_message2: EphemeraMessage = signed_message.into();
+
+        let messages = vec![signed_message1.clone(), signed_message2.clone()];
+
+        let block = block_producer.create_block(1, messages).unwrap();
+
+        assert_eq!(block.header.height, 1);
+        assert_eq!(block.header.creator, peer_id);
+        assert_eq!(block.messages.len(), 2);
+
+        //Nondeterministic because of timestamp
+        match signed_message1.cmp(&signed_message2) {
+            Ordering::Less => {
+                assert_eq!(block.messages[0], signed_message1);
+                assert_eq!(block.messages[1], signed_message2);
+            }
+            Ordering::Greater => {
+                assert_eq!(block.messages[0], signed_message2);
+                assert_eq!(block.messages[1], signed_message1);
+            }
+
+            Ordering::Equal => {
+                panic!("Messages are equal");
+            }
+        }
+    }
+}

ephemera/src/block/types/block.rs

@@ -0,0 +1,327 @@
+use std::fmt::{Debug, Display};
+
+use serde::{Deserialize, Serialize};
+
+use crate::utilities::merkle::MerkleTree;
+use crate::{
+    block::types::message::EphemeraMessage,
+    codec::{Decode, Encode},
+    crypto::Keypair,
+    peer::PeerId,
+    utilities::{
+        codec::{Codec, DecodingError, EncodingError, EphemeraCodec},
+        crypto::Certificate,
+        hash::{EphemeraHash, EphemeraHasher},
+        hash::{Hash, Hasher},
+        time::EphemeraTime,
+    },
+};
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub(crate) struct BlockHeader {
+    pub(crate) timestamp: u64,
+    pub(crate) creator: PeerId,
+    pub(crate) height: u64,
+    pub(crate) hash: Hash,
+}
+
+impl BlockHeader {
+    pub(crate) fn new(raw_header: &RawBlockHeader, hash: Hash) -> Self {
+        Self {
+            timestamp: raw_header.timestamp,
+            creator: raw_header.creator,
+            height: raw_header.height,
+            hash,
+        }
+    }
+}
+
+impl Display for BlockHeader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let hash = &self.hash;
+        let time = self.timestamp;
+        let creator = &self.creator;
+        let height = self.height;
+        write!(
+            f,
+            "hash: {hash}, timestamp: {time}, creator: {creator}, height: {height}",
+        )
+    }
+}
+
+impl Encode for BlockHeader {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}
+
+impl Decode for BlockHeader {
+    type Output = Self;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+impl EphemeraHash for BlockHeader {
+    fn hash<H: EphemeraHasher>(&self, state: &mut H) -> anyhow::Result<()> {
+        let bytes = Codec::encode(&self)?;
+        state.update(&bytes);
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
+pub(crate) struct RawBlockHeader {
+    pub(crate) timestamp: u64,
+    pub(crate) creator: PeerId,
+    pub(crate) height: u64,
+}
+
+impl RawBlockHeader {
+    pub(crate) fn new(creator: PeerId, height: u64) -> Self {
+        Self {
+            timestamp: EphemeraTime::now(),
+            creator,
+            height,
+        }
+    }
+
+    pub(crate) fn hash_with_default_hasher(&self) -> anyhow::Result<Hash> {
+        let mut hasher = Hasher::default();
+        self.hash(&mut hasher)?;
+        let header_hash = hasher.finish().into();
+        Ok(header_hash)
+    }
+}
+
+impl Display for RawBlockHeader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let creator = &self.creator;
+        let height = self.height;
+        write!(f, "creator: {creator}, height: {height}",)
+    }
+}
+
+impl From<BlockHeader> for RawBlockHeader {
+    fn from(block_header: BlockHeader) -> Self {
+        Self {
+            timestamp: block_header.timestamp,
+            creator: block_header.creator,
+            height: block_header.height,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub(crate) struct Block {
+    pub(crate) header: BlockHeader,
+    pub(crate) messages: Vec<EphemeraMessage>,
+}
+
+impl Block {
+    pub(crate) fn new(raw_block: RawBlock, block_hash: Hash) -> Self {
+        let header = BlockHeader::new(&raw_block.header, block_hash);
+        Self {
+            header,
+            messages: raw_block.messages,
+        }
+    }
+
+    pub(crate) fn get_hash(&self) -> Hash {
+        self.header.hash
+    }
+
+    pub(crate) fn get_height(&self) -> u64 {
+        self.header.height
+    }
+
+    pub(crate) fn new_genesis_block(creator: PeerId) -> Self {
+        let mut block = Self {
+            header: BlockHeader {
+                timestamp: EphemeraTime::now(),
+                creator,
+                height: 0,
+                hash: Hash::new([0; 32]),
+            },
+            messages: Vec::new(),
+        };
+
+        let hash = block
+            .hash_with_default_hasher()
+            .expect("Failed to hash genesis block");
+        block.header.hash = hash;
+        block
+    }
+
+    pub(crate) fn sign(&self, keypair: &Keypair) -> anyhow::Result<Certificate> {
+        let raw_block: RawBlock = self.clone().into();
+        let certificate = Certificate::prepare(keypair, &raw_block)?;
+        Ok(certificate)
+    }
+
+    pub(crate) fn verify(&self, certificate: &Certificate) -> anyhow::Result<bool> {
+        let raw_block: RawBlock = self.clone().into();
+        certificate.verify(&raw_block)
+    }
+
+    pub(crate) fn hash_with_default_hasher(&self) -> anyhow::Result<Hash> {
+        let raw_block: RawBlock = self.clone().into();
+        raw_block.hash_with_default_hasher()
+    }
+
+    pub(crate) fn merkle_tree(&self) -> anyhow::Result<MerkleTree> {
+        merkle_tree(&self.messages)
+    }
+}
+
+impl Display for Block {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let header = &self.header;
+        write!(f, "{header}, nr of messages: {}", self.messages.len())
+    }
+}
+
+impl Encode for Block {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}
+
+impl Decode for Block {
+    type Output = Block;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub(crate) struct RawBlock {
+    pub(crate) header: RawBlockHeader,
+    pub(crate) messages: Vec<EphemeraMessage>,
+}
+
+impl RawBlock {
+    pub(crate) fn new(header: RawBlockHeader, messages: Vec<EphemeraMessage>) -> Self {
+        Self { header, messages }
+    }
+
+    pub(crate) fn hash_with_default_hasher(&self) -> anyhow::Result<Hash> {
+        let header_hash = self.header.hash_with_default_hasher()?;
+        let merkle_root = merkle_tree(&self.messages)?.root_hash();
+        let block_hash = Hasher::digest(&[header_hash.inner(), merkle_root.inner()].concat());
+        Ok(block_hash.into())
+    }
+}
+
+impl From<Block> for RawBlock {
+    fn from(block: Block) -> Self {
+        Self {
+            header: block.header.into(),
+            messages: block.messages,
+        }
+    }
+}
+
+impl Encode for RawBlockHeader {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}
+
+impl Decode for RawBlockHeader {
+    type Output = RawBlockHeader;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+impl Encode for RawBlock {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}
+
+impl Decode for RawBlock {
+    type Output = RawBlock;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+impl EphemeraHash for RawBlockHeader {
+    fn hash<H: EphemeraHasher>(&self, state: &mut H) -> anyhow::Result<()> {
+        state.update(&self.encode()?);
+        Ok(())
+    }
+}
+
+impl EphemeraHash for RawBlock {
+    fn hash<H: EphemeraHasher>(&self, state: &mut H) -> anyhow::Result<()> {
+        self.header.hash(state)?;
+        for message in &self.messages {
+            message.hash(state)?;
+        }
+        Ok(())
+    }
+}
+
+pub(crate) fn merkle_tree(messages: &[EphemeraMessage]) -> anyhow::Result<MerkleTree> {
+    let message_hashes = messages
+        .iter()
+        .map(EphemeraMessage::hash_with_default_hasher)
+        .collect::<anyhow::Result<Vec<Hash>>>()?;
+    let merkle_tree = MerkleTree::build_tree(&message_hashes);
+    Ok(merkle_tree)
+}
+
+#[cfg(test)]
+mod test {
+    use crate::block::types::message::RawEphemeraMessage;
+    use crate::crypto::EphemeraKeypair;
+
+    use super::*;
+
+    #[test]
+    fn test_block_hash_no_messages() {
+        let block = Block::new_genesis_block(PeerId::random());
+        let block_hash = block.hash_with_default_hasher().unwrap();
+        assert_eq!(block_hash, block.get_hash());
+    }
+
+    #[test]
+    fn test_block_hash_with_messages() {
+        let messages = create_ephemera_messages(10);
+        let message_hashes = messages
+            .iter()
+            .map(EphemeraMessage::hash_with_default_hasher)
+            .collect::<anyhow::Result<Vec<Hash>>>()
+            .unwrap();
+
+        let raw_block = RawBlock::new(RawBlockHeader::new(PeerId::random(), 0), messages);
+        let block_hash = raw_block.hash_with_default_hasher().unwrap();
+
+        let header_hash = raw_block.header.hash_with_default_hasher().unwrap();
+        let merkle_root = MerkleTree::build_tree(&message_hashes).root_hash();
+        let expected_block_hash =
+            Hasher::digest(&[header_hash.inner(), merkle_root.inner()].concat());
+
+        assert_eq!(block_hash, expected_block_hash.into());
+    }
+
+    fn create_ephemera_messages(n: usize) -> Vec<EphemeraMessage> {
+        let keypair = Keypair::generate(None);
+        let mut messages = Vec::new();
+        for i in 0..n {
+            let label = format!("test {i}",);
+            let message = RawEphemeraMessage::new(label, vec![0; 32]);
+            let certificate = Certificate::prepare(&keypair, &message).unwrap();
+            let ephemera_message = EphemeraMessage::new(message, certificate);
+            messages.push(ephemera_message);
+        }
+        messages
+    }
+}

ephemera/src/block/types/message.rs

@@ -0,0 +1,99 @@
+use serde::{Deserialize, Serialize};
+
+use crate::utilities::codec::{Codec, DecodingError, EncodingError, EphemeraCodec};
+use crate::{
+    codec::{Decode, Encode},
+    utilities::{
+        crypto::Certificate,
+        hash::{EphemeraHash, EphemeraHasher},
+        hash::{Hash, Hasher},
+    },
+};
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
+pub(crate) struct EphemeraMessage {
+    ///Timestamp of the message
+    pub(crate) timestamp: u64,
+    ///Application specific logical identifier of the message
+    pub(crate) label: String,
+    ///Application specific data
+    pub(crate) data: Vec<u8>,
+    ///Signature of the raw message
+    pub(crate) certificate: Certificate,
+}
+
+impl EphemeraMessage {
+    #[cfg(test)]
+    pub(crate) fn new(raw_message: RawEphemeraMessage, certificate: Certificate) -> Self {
+        Self {
+            timestamp: raw_message.timestamp,
+            label: raw_message.label,
+            data: raw_message.data,
+            certificate,
+        }
+    }
+
+    pub(crate) fn hash_with_default_hasher(&self) -> anyhow::Result<Hash> {
+        let mut hasher = Hasher::default();
+        self.hash(&mut hasher)?;
+        let hash = hasher.finish().into();
+        Ok(hash)
+    }
+}
+
+impl Encode for EphemeraMessage {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}
+
+impl Decode for EphemeraMessage {
+    type Output = Self;
+
+    fn decode(bytes: &[u8]) -> Result<Self::Output, DecodingError> {
+        Codec::decode(bytes)
+    }
+}
+
+impl EphemeraHash for EphemeraMessage {
+    fn hash<H: EphemeraHasher>(&self, state: &mut H) -> anyhow::Result<()> {
+        state.update(&self.encode()?);
+        Ok(())
+    }
+}
+
+/// Raw message represents all the data what will be signed.
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
+pub(crate) struct RawEphemeraMessage {
+    pub(crate) timestamp: u64,
+    pub(crate) label: String,
+    pub(crate) data: Vec<u8>,
+}
+
+impl RawEphemeraMessage {
+    #[cfg(test)]
+    pub(crate) fn new(label: String, data: Vec<u8>) -> Self {
+        use crate::utilities::time::EphemeraTime;
+        Self {
+            timestamp: EphemeraTime::now(),
+            label,
+            data,
+        }
+    }
+}
+
+impl From<EphemeraMessage> for RawEphemeraMessage {
+    fn from(message: EphemeraMessage) -> Self {
+        Self {
+            timestamp: message.timestamp,
+            label: message.label,
+            data: message.data,
+        }
+    }
+}
+
+impl Encode for RawEphemeraMessage {
+    fn encode(&self) -> Result<Vec<u8>, EncodingError> {
+        Codec::encode(&self)
+    }
+}

ephemera/src/block/types/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod block;
+pub(crate) mod message;

ephemera/src/broadcast/bracha/broadcast.rs

@@ -0,0 +1,320 @@
+use std::num::NonZeroUsize;
+
+use log::{debug, trace};
+use lru::LruCache;
+
+use crate::broadcast::bracha::quorum::BrachaMessageType;
+use crate::peer::PeerId;
+use crate::{
+    block::types::block::Block,
+    broadcast::{
+        bracha::quorum::Quorum,
+        MessageType::{Echo, Vote},
+        ProtocolContext, RawRbMsg,
+    },
+    utilities::hash::Hash,
+};
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug)]
+pub(crate) enum BroadcastResponse {
+    Broadcast(RawRbMsg),
+    Deliver(Hash),
+    Drop(Hash),
+}
+
+pub(crate) struct Broadcaster {
+    /// Local peer id
+    local_peer_id: PeerId,
+    /// We keep a context for each block we are processing.
+    contexts: LruCache<Hash, ProtocolContext>,
+    /// Current cluster size
+    cluster_size: usize,
+}
+
+impl Broadcaster {
+    pub fn new(peer_id: PeerId) -> Broadcaster {
+        Broadcaster {
+            //At any given time we are processing in parallel about n messages, where n is the number of peers in the group.
+            //This is just large enough buffer.
+            contexts: LruCache::new(NonZeroUsize::new(1000).unwrap()),
+            cluster_size: 0,
+            local_peer_id: peer_id,
+        }
+    }
+
+    pub(crate) fn new_broadcast(&mut self, block: Block) -> anyhow::Result<BroadcastResponse> {
+        debug!("Starting broadcast for new block {:?}", block.get_hash());
+        self.handle(&RawRbMsg::new(block, self.local_peer_id))
+    }
+
+    pub(crate) fn handle(&mut self, rb_msg: &RawRbMsg) -> anyhow::Result<BroadcastResponse> {
+        trace!("Processing new broadcast message: {:?}", rb_msg);
+
+        let block = rb_msg.block();
+        let hash = block.hash_with_default_hasher()?;
+
+        let ctx = self.contexts.get_or_insert(hash, || {
+            ProtocolContext::new(hash, self.local_peer_id, Quorum::new(self.cluster_size))
+        });
+
+        if ctx.delivered {
+            trace!("Block {hash:?} already delivered");
+            return Ok(BroadcastResponse::Drop(hash));
+        }
+
+        match rb_msg.message_type {
+            Echo(_) => {
+                trace!("Processing ECHO {:?}", rb_msg.id);
+                Ok(self.process_echo(rb_msg, hash))
+            }
+            Vote(_) => {
+                trace!("Processing VOTE {:?}", rb_msg.id);
+                Ok(self.process_vote(rb_msg, hash))
+            }
+        }
+    }
+
+    fn process_echo(&mut self, rb_msg: &RawRbMsg, hash: Hash) -> BroadcastResponse {
+        let ctx = self.contexts.get_mut(&hash).expect("Context not found");
+
+        if self.local_peer_id != rb_msg.original_sender {
+            trace!("Adding echo from {:?}", rb_msg.original_sender);
+            ctx.add_echo(rb_msg.original_sender);
+        }
+
+        if !ctx.echoed() {
+            ctx.add_echo(self.local_peer_id);
+
+            trace!("Sending echo reply for {hash:?}",);
+            return BroadcastResponse::Broadcast(
+                rb_msg.echo_reply(self.local_peer_id, rb_msg.block()),
+            );
+        }
+
+        if !ctx.voted()
+            && ctx
+                .quorum
+                .check_threshold(ctx, BrachaMessageType::Echo)
+                .is_vote()
+        {
+            ctx.add_vote(self.local_peer_id);
+
+            trace!("Sending vote reply for {hash:?}",);
+            return BroadcastResponse::Broadcast(
+                rb_msg.vote_reply(self.local_peer_id, rb_msg.block()),
+            );
+        }
+
+        BroadcastResponse::Drop(hash)
+    }
+
+    fn process_vote(&mut self, rb_msg: &RawRbMsg, hash: Hash) -> BroadcastResponse {
+        let block = rb_msg.block();
+        let ctx = self.contexts.get_mut(&hash).expect("Context not found");
+
+        if self.local_peer_id != rb_msg.original_sender {
+            trace!("Adding vote from {:?}", rb_msg.original_sender);
+            ctx.add_vote(rb_msg.original_sender);
+        }
+
+        if ctx
+            .quorum
+            .check_threshold(ctx, BrachaMessageType::Vote)
+            .is_vote()
+        {
+            ctx.add_vote(self.local_peer_id);
+
+            trace!("Sending vote reply for {hash:?}",);
+            return BroadcastResponse::Broadcast(rb_msg.vote_reply(self.local_peer_id, block));
+        }
+
+        if ctx
+            .quorum
+            .check_threshold(ctx, BrachaMessageType::Vote)
+            .is_deliver()
+        {
+            trace!("Commit complete for {:?}", rb_msg.id);
+
+            ctx.delivered = true;
+
+            return BroadcastResponse::Deliver(hash);
+        }
+
+        BroadcastResponse::Drop(hash)
+    }
+
+    pub(crate) fn group_updated(&mut self, size: usize) {
+        self.cluster_size = size;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    //1.make sure before voting enough echo messages are received
+    //2.make sure before delivering enough vote messages are received
+    //a)Either f + 1
+    //b)Or n - f
+
+    //3.make sure that duplicate messages doesn't have impact
+
+    //4. "Ideally" make sure that when group changes, the ongoing broadcast can deal with it
+
+    use std::iter;
+
+    use assert_matches::assert_matches;
+
+    use crate::broadcast::bracha::broadcast::BroadcastResponse;
+    use crate::peer::PeerId;
+    use crate::utilities::hash::Hash;
+    use crate::{
+        block::types::block::{Block, RawBlock, RawBlockHeader},
+        broadcast::{self, bracha::broadcast::Broadcaster, RawRbMsg},
+    };
+
+    #[test]
+    fn test_state_transitions_from_start_to_end() {
+        let peers: Vec<PeerId> = iter::repeat_with(PeerId::random).take(10).collect();
+        let local_peer_id = peers[0];
+        let block_creator_peer_id = peers[1];
+
+        let mut broadcaster = Broadcaster::new(local_peer_id);
+        broadcaster.group_updated(peers.len());
+
+        let (block_hash, block) = create_block(block_creator_peer_id);
+
+        //After this echo set contains local and block creator(msg sender)
+        receive_echo_first_message(&mut broadcaster, &block, block_creator_peer_id);
+
+        let ctx = broadcaster.contexts.get(&block_hash).unwrap();
+        assert_eq!(ctx.echo.len(), 2);
+        assert!(ctx.echoed());
+        assert!(!ctx.voted());
+
+        receive_nr_of_echo_messages_below_vote_threshold(&mut broadcaster, &block, &peers[2..6]);
+
+        let ctx = broadcaster.contexts.get(&block_hash).unwrap();
+        assert_eq!(ctx.echo.len(), 6);
+        assert!(ctx.echoed());
+        assert!(!ctx.voted());
+
+        receive_echo_threshold_message(&mut broadcaster, &block, *peers.get(7).unwrap());
+
+        let ctx = broadcaster.contexts.get(&block_hash).unwrap();
+        assert_eq!(ctx.echo.len(), 7);
+        assert_eq!(ctx.vote.len(), 1);
+        assert!(ctx.echoed());
+        assert!(ctx.voted());
+
+        receive_nr_of_vote_messages_below_deliver_threshold(&mut broadcaster, &block, &peers[2..7]);
+
+        let ctx = broadcaster.contexts.get(&block_hash).unwrap();
+        assert_eq!(ctx.echo.len(), 7);
+        assert_eq!(ctx.vote.len(), 6);
+        assert!(ctx.echoed());
+        assert!(ctx.voted());
+
+        receive_threshold_vote_message_for_deliver(
+            &mut broadcaster,
+            &block,
+            *peers.get(8).unwrap(),
+        );
+    }
+
+    fn receive_threshold_vote_message_for_deliver(
+        broadcaster: &mut Broadcaster,
+        block: &Block,
+        peer_id: PeerId,
+    ) {
+        let rb_msg = RawRbMsg::new(block.clone(), PeerId::random());
+        let rb_msg = rb_msg.vote_reply(peer_id, block.clone());
+
+        let response = handle_double(broadcaster, &rb_msg);
+
+        assert_matches!(response, BroadcastResponse::Deliver(_));
+    }
+
+    fn receive_nr_of_echo_messages_below_vote_threshold(
+        broadcaster: &mut Broadcaster,
+        block: &Block,
+        peers: &[PeerId],
+    ) {
+        for peer_id in peers {
+            let rb_msg = RawRbMsg::new(block.clone(), *peer_id);
+
+            let response = handle_double(broadcaster, &rb_msg);
+
+            assert_matches!(response, BroadcastResponse::Drop(_));
+        }
+    }
+
+    fn receive_nr_of_vote_messages_below_deliver_threshold(
+        broadcaster: &mut Broadcaster,
+        block: &Block,
+        peers: &[PeerId],
+    ) {
+        for peer_id in peers {
+            let rb_msg = RawRbMsg::new(block.clone(), PeerId::random());
+            let rb_msg = rb_msg.vote_reply(*peer_id, block.clone());
+
+            let response = handle_double(broadcaster, &rb_msg);
+            assert_matches!(response, BroadcastResponse::Drop(_));
+        }
+    }
+
+    fn receive_echo_first_message(
+        broadcaster: &mut Broadcaster,
+        block: &Block,
+        block_creator: PeerId,
+    ) {
+        let rb_msg = RawRbMsg::new(block.clone(), block_creator);
+        let response = handle_double(broadcaster, &rb_msg);
+
+        assert_matches!(
+            response,
+            BroadcastResponse::Broadcast(RawRbMsg {
+                id: _,
+                request_id: _,
+                original_sender: _,
+                timestamp: _,
+                message_type: broadcast::MessageType::Echo(_),
+            })
+        );
+    }
+
+    fn receive_echo_threshold_message(
+        broadcaster: &mut Broadcaster,
+        block: &Block,
+        peer_id: PeerId,
+    ) {
+        let rb_msg = RawRbMsg::new(block.clone(), peer_id);
+
+        let response = handle_double(broadcaster, &rb_msg);
+        assert_matches!(
+            response,
+            BroadcastResponse::Broadcast(RawRbMsg {
+                id: _,
+                request_id: _,
+                original_sender: _,
+                timestamp: _,
+                message_type: broadcast::MessageType::Vote(_),
+            })
+        );
+    }
+
+    fn create_block(block_creator_peer_id: PeerId) -> (Hash, Block) {
+        let header = RawBlockHeader::new(block_creator_peer_id, 0);
+        let raw_block = RawBlock::new(header, vec![]);
+        let block_hash = raw_block.hash_with_default_hasher().unwrap();
+        let block = Block::new(raw_block, block_hash);
+        (block_hash, block)
+    }
+
+    //make sure that duplicate messages doesn't have impact
+    fn handle_double(broadcaster: &mut Broadcaster, rb_msg: &RawRbMsg) -> BroadcastResponse {
+        let response = broadcaster.handle(rb_msg).unwrap();
+        broadcaster.handle(rb_msg).unwrap();
+        response
+    }
+}

ephemera/src/broadcast/bracha/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod broadcast;
+pub(crate) mod quorum;

ephemera/src/broadcast/bracha/quorum.rs

@@ -0,0 +1,254 @@
+use log::trace;
+
+use crate::broadcast::{MessageType, ProtocolContext};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BrachaMessageType {
+    Echo,
+    Vote,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum BrachaAction {
+    Vote,
+    Deliver,
+    Ignore,
+}
+
+impl BrachaAction {
+    pub(crate) fn is_vote(self) -> bool {
+        matches!(self, BrachaAction::Vote)
+    }
+
+    pub(crate) fn is_deliver(self) -> bool {
+        matches!(self, BrachaAction::Deliver)
+    }
+}
+
+impl From<MessageType> for BrachaMessageType {
+    fn from(message_type: MessageType) -> Self {
+        match message_type {
+            MessageType::Echo(_) => BrachaMessageType::Echo,
+            MessageType::Vote(_) => BrachaMessageType::Vote,
+        }
+    }
+}
+
+const MAX_FAULTY_RATIO: f64 = 1.0 / 3.0;
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct Quorum {
+    pub(crate) cluster_size: usize,
+    pub(crate) max_faulty_nodes: usize,
+}
+
+impl Quorum {
+    pub fn new(cluster_size: usize) -> Self {
+        let max_faulty_nodes = Quorum::max_faulty_nodes(cluster_size);
+        Self {
+            cluster_size,
+            max_faulty_nodes,
+        }
+    }
+
+    pub(crate) fn check_threshold(
+        &self,
+        ctx: &ProtocolContext,
+        phase: BrachaMessageType,
+    ) -> BrachaAction {
+        if self.cluster_size == 0 {
+            return BrachaAction::Ignore;
+        }
+
+        match phase {
+            BrachaMessageType::Echo => {
+                if ctx.echo.len() >= self.cluster_size - self.max_faulty_nodes {
+                    trace!(
+                        "Echo threshold reached: Echoed:{} / Threshold:{} for Block:{}",
+                        ctx.echo.len(),
+                        self.cluster_size - self.max_faulty_nodes,
+                        ctx.hash
+                    );
+                    BrachaAction::Vote
+                } else {
+                    trace!(
+                        "Echo threshold not reached: Echoed:{} / Threshold:{} for Block:{}",
+                        ctx.echo.len(),
+                        self.cluster_size - self.max_faulty_nodes,
+                        ctx.hash
+                    );
+                    BrachaAction::Ignore
+                }
+            }
+            BrachaMessageType::Vote => {
+                if !ctx.voted() {
+                    // f + 1 votes are enough to send our vote
+                    if ctx.vote.len() >= self.max_faulty_nodes {
+                        trace!(
+                            "Vote send threshold reached: Voted:{} / Threshold:{} for Block:{}",
+                            ctx.vote.len(),
+                            self.max_faulty_nodes + 1,
+                            ctx.hash
+                        );
+                        return BrachaAction::Vote;
+                    }
+                }
+
+                if ctx.voted() {
+                    // n-f votes are enough to deliver the value
+                    if ctx.vote.len() >= self.cluster_size - self.max_faulty_nodes {
+                        trace!(
+                            "Deliver threshold reached: Voted:{} / Threshold:{} for Block:{}",
+                            ctx.vote.len(),
+                            self.cluster_size - self.max_faulty_nodes,
+                            ctx.hash
+                        );
+                        return BrachaAction::Deliver;
+                    }
+                }
+
+                trace!(
+                    "Vote threshold not reached: Voted:{} / Threshold:{} for Block:{}",
+                    ctx.vote.len(),
+                    self.max_faulty_nodes + 1,
+                    ctx.hash
+                );
+                BrachaAction::Ignore
+            }
+        }
+    }
+
+    pub(crate) fn cluster_size_info(cluster_size: usize) -> String {
+        let max_faulty_nodes = Quorum::max_faulty_nodes(cluster_size);
+        format!("Cluster size: {cluster_size} / Max faulty nodes: {max_faulty_nodes}",)
+    }
+
+    #[allow(
+        clippy::cast_precision_loss,
+        clippy::cast_sign_loss,
+        clippy::cast_possible_truncation
+    )]
+    fn max_faulty_nodes(cluster_size: usize) -> usize {
+        (cluster_size as f64 * MAX_FAULTY_RATIO).floor() as usize
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use crate::broadcast::{
+        bracha::quorum::{BrachaAction, BrachaMessageType, Quorum},
+        ProtocolContext,
+    };
+    use crate::peer::PeerId;
+
+    #[test]
+    fn test_max_faulty_nodes() {
+        let quorum = Quorum::new(10);
+        assert_eq!(quorum.max_faulty_nodes, 3);
+    }
+
+    #[test]
+    fn test_vote_threshold_from_n_minus_f_peers() {
+        let quorum = Quorum::new(10);
+
+        let ctx = ctx_with_nr_echoes(0);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Echo),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_echoes(3);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Echo),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_echoes(8);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Echo),
+            BrachaAction::Vote
+        );
+    }
+
+    #[test]
+    fn test_vote_threshold_from_f_plus_one_peers() {
+        let quorum = Quorum::new(10);
+
+        let ctx = ctx_with_nr_votes(0, None);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_votes(2, None);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_votes(5, None);
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Vote
+        );
+    }
+
+    #[test]
+    fn test_deliver_threshold_from_n_minus_f_peers() {
+        let quorum = Quorum::new(10);
+
+        let local_peer_id = PeerId::random();
+        let ctx = ctx_with_nr_votes(0, local_peer_id.into());
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_votes(3, local_peer_id.into());
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Ignore
+        );
+
+        let ctx = ctx_with_nr_votes(7, local_peer_id.into());
+        assert_eq!(
+            quorum.check_threshold(&ctx, BrachaMessageType::Vote),
+            BrachaAction::Deliver
+        );
+    }
+
+    fn ctx_with_nr_echoes(n: usize) -> ProtocolContext {
+        let mut ctx = ProtocolContext {
+            local_peer_id: PeerId::random(),
+            hash: [0; 32].into(),
+            echo: HashSet::default(),
+            vote: HashSet::default(),
+            quorum: Quorum::new(10),
+            delivered: false,
+        };
+        for _ in 0..n {
+            ctx.echo.insert(PeerId::random());
+        }
+        ctx
+    }
+
+    fn ctx_with_nr_votes(n: usize, local_peer_id: Option<PeerId>) -> ProtocolContext {
+        let mut ctx = ProtocolContext {
+            local_peer_id: local_peer_id.unwrap_or(PeerId::random()),
+            hash: [0; 32].into(),
+            echo: HashSet::default(),
+            vote: HashSet::default(),
+            quorum: Quorum::new(10),
+            delivered: false,
+        };
+        for _ in 0..n {
+            ctx.vote.insert(PeerId::random());
+        }
+        if let Some(id) = local_peer_id {
+            ctx.vote.insert(id);
+        }
+        ctx
+    }
+}

ephemera/src/broadcast/group.rs

@@ -0,0 +1,249 @@
+use std::collections::HashSet;
+use std::num::NonZeroUsize;
+
+use log::warn;
+use lru::LruCache;
+
+use crate::peer::PeerId;
+use crate::utilities::hash::Hash;
+
+pub(crate) struct BroadcastGroup {
+    /// The id of current group. Incremented every time a new snapshot is added.
+    pub(crate) current_id: u64,
+    /// A cache of the group snapshots.
+    pub(crate) snapshots: LruCache<u64, HashSet<PeerId>>,
+    /// A cache of the groups for each block.
+    pub(crate) broadcast_groups: LruCache<Hash, u64>,
+}
+
+impl BroadcastGroup {
+    pub(crate) fn new() -> BroadcastGroup {
+        let mut snapshots = LruCache::new(NonZeroUsize::new(100).unwrap());
+        snapshots.put(0, HashSet::new());
+        BroadcastGroup {
+            current_id: 0,
+            snapshots,
+            broadcast_groups: LruCache::new(NonZeroUsize::new(100).unwrap()),
+        }
+    }
+
+    pub(crate) fn add_snapshot(&mut self, snapshot: HashSet<PeerId>) {
+        self.current_id += 1;
+        self.snapshots.put(self.current_id, snapshot);
+    }
+
+    pub(crate) fn is_member(&mut self, id: u64, peer_id: &PeerId) -> bool {
+        self.snapshots
+            .get(&id)
+            .map_or(false, |s| s.contains(peer_id))
+    }
+
+    pub(crate) fn is_empty(&mut self) -> bool {
+        self.snapshots
+            .get(&self.current_id)
+            .map_or(true, HashSet::is_empty)
+    }
+
+    // Returns empty snapshots(inserted in 'new' fn) if we haven't received any yet.
+    pub(crate) fn current(&mut self) -> &HashSet<PeerId> {
+        self.snapshots
+            .get(&self.current_id)
+            .expect("Current group should always exist")
+    }
+
+    // Checks if creator and sender are part of the expected group.
+    // If we see hash first time, it checks against the current group. And if check passes, it
+    // associates the hash with the current group.
+    pub(crate) fn check_membership(
+        &mut self,
+        hash: Hash,
+        block_creator: &PeerId,
+        message_sender: &PeerId,
+    ) -> bool {
+        //We see this block first time
+        if !self.broadcast_groups.contains(&hash) {
+            //This can happen at startup for example when node is not ready yet(caught up with the network)
+            if self.is_empty() {
+                warn!(
+                    "Received new block {:?} but current group is empty, rejecting the block",
+                    hash
+                );
+                return false;
+            }
+        }
+
+        //Make sure that the sender peer_id and block peer_id are part of the block initial group
+        //1. If the block is new, the group is the current one
+        //2. If the block is old, the group is the one that was used when the block was created
+
+        //It's needed to make sure that
+        //1. The peer is authenticated(part of the network)
+        //2. Block processing is consistent regarding the group across rounds
+
+        let membership_id = *self.broadcast_groups.get(&hash).unwrap_or(&self.current_id);
+
+        //Node is excluded from group for some reason(for example health checks failed)
+        if !self.is_member(membership_id, message_sender) {
+            warn!(
+                "Received new block {} but sender {} is not part of the current group",
+                hash, message_sender
+            );
+            return false;
+        }
+
+        //Node is excluded from group for some reason(for example health checks failed)
+        if !self.is_member(membership_id, block_creator) {
+            warn!(
+                "Received new block {} but sender {} is not part of the current group",
+                hash, message_sender
+            );
+            return false;
+        }
+
+        self.broadcast_groups.put(hash, membership_id);
+
+        true
+    }
+
+    pub(crate) fn get_group_by_block_hash(&mut self, hash: Hash) -> Option<&HashSet<PeerId>> {
+        let membership_id = *self.broadcast_groups.get(&hash)?;
+        self.snapshots.get(&membership_id)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use crate::broadcast::group::BroadcastGroup;
+    use crate::peer::PeerId;
+    use crate::utilities::hash::Hash;
+
+    #[test]
+    fn test_no_snapshot() {
+        let group = BroadcastGroup::new();
+        assert_eq!(group.current_id, 0);
+        //Including initial default snapshot
+        assert_eq!(group.snapshots.len(), 1);
+    }
+
+    #[test]
+    fn test_multiple_snapshots() {
+        let (mut group, snapshots) = group_with_snapshots(10);
+        assert_eq!(group.current_id, 10);
+        //Including initial default snapshot
+        assert_eq!(group.snapshots.len(), 11);
+
+        for (i, sn) in snapshots
+            .iter()
+            .enumerate()
+            .map(|(i, sn)| ((i + 1) as u64, sn))
+        {
+            let gsn = group.snapshots.get(&i).unwrap();
+            assert_eq!(sn, gsn);
+        }
+    }
+
+    #[test]
+    fn check_membership_empty_group() {
+        let mut group = BroadcastGroup::new();
+        let hash = Hash::new([0; 32]);
+        assert!(!group.check_membership(hash, &PeerId::random(), &PeerId::random()));
+        assert!(!group.broadcast_groups.contains(&hash));
+    }
+
+    #[test]
+    fn check_membership_creator_nor_sender_not_member() {
+        let (mut group, _snapshots) = group_with_snapshots(1);
+        assert!(!group.check_membership(Hash::new([0; 32]), &PeerId::random(), &PeerId::random()));
+        assert!(!group.broadcast_groups.contains(&Hash::new([0; 32])));
+    }
+
+    #[test]
+    fn check_membership_creator_not_member() {
+        let (mut group, snapshots) = group_with_snapshots(1);
+        let sender = snapshots[0].clone().into_iter().next().unwrap();
+
+        let hash = Hash::new([0; 32]);
+        assert!(!group.check_membership(hash, &PeerId::random(), &sender));
+        assert!(!group.broadcast_groups.contains(&hash));
+    }
+
+    #[test]
+    fn check_membership_sender_not_member() {
+        let (mut group, snapshots) = group_with_snapshots(1);
+        let creator = snapshots[0].clone().into_iter().next().unwrap();
+        let hash = Hash::new([0; 32]);
+        assert!(!group.check_membership(hash, &creator, &PeerId::random()));
+        assert!(!group.broadcast_groups.contains(&hash));
+    }
+
+    #[test]
+    fn check_snapshot_membership_both_are_members() {
+        let (mut group, snapshots) = group_with_snapshots(1);
+        let creator = snapshots[0].clone().into_iter().next().unwrap();
+        let sender = creator;
+        let hash = Hash::new([0; 32]);
+        assert!(group.check_membership(hash, &creator, &sender));
+        assert!(group.broadcast_groups.contains(&hash));
+    }
+
+    #[test]
+    fn check_snapshot_membership_of_current_snapshot() {
+        let (mut group, snapshots) = group_with_snapshots(2);
+        let current_snapshot = snapshots[1].clone();
+        let creator = current_snapshot.into_iter().next().unwrap();
+        let sender = creator;
+
+        let hash = Hash::new([0; 32]);
+        assert!(group.check_membership(hash, &creator, &sender));
+        assert!(group.broadcast_groups.contains(&hash));
+
+        //Remove the current snapshot
+        group.snapshots.pop(&group.current_id);
+
+        //Membership should fail
+        assert!(!group.check_membership(hash, &creator, &sender));
+    }
+
+    #[test]
+    fn check_snapshot_membership_of_previous_snapshot() {
+        let mut group = BroadcastGroup::new();
+        let first_snapshot = create_snapshot();
+        group.add_snapshot(first_snapshot.clone());
+
+        let creator = first_snapshot.into_iter().next().unwrap();
+        let sender = creator;
+        let hash = Hash::new([0; 32]);
+        assert!(group.check_membership(hash, &creator, &sender));
+        assert!(group.broadcast_groups.contains(&hash));
+
+        //Add second snapshot
+        group.add_snapshot(create_snapshot());
+
+        //Remove the current snapshot
+        group.snapshots.pop(&group.current_id);
+
+        //Membership should still pass
+        assert!(group.broadcast_groups.contains(&hash));
+        assert!(group.check_membership(hash, &creator, &sender));
+    }
+
+    fn group_with_snapshots(count: usize) -> (BroadcastGroup, Vec<HashSet<PeerId>>) {
+        let mut group = BroadcastGroup::new();
+        let mut snapshots = Vec::new();
+        for _ in 0..count {
+            let snapshot = create_snapshot();
+            snapshots.push(snapshot.clone());
+            group.add_snapshot(snapshot);
+        }
+        (group, snapshots)
+    }
+
+    fn create_snapshot() -> HashSet<PeerId> {
+        let mut snapshot = HashSet::new();
+        let peer_id = PeerId::random();
+        snapshot.insert(peer_id);
+        snapshot
+    }
+}

ephemera/src/broadcast/mod.rs

@@ -0,0 +1,191 @@
+//! Simple reliable broadcast protocol(Bracha) implementation
+//!
+use std::collections::HashSet;
+use std::fmt::{Debug, Display};
+
+use serde_derive::{Deserialize, Serialize};
+
+use crate::broadcast::bracha::quorum::Quorum;
+use crate::{
+    block::types::block::Block,
+    peer::PeerId,
+    utilities::{
+        crypto::Certificate,
+        hash::Hash,
+        id::{EphemeraId, EphemeraIdentifier},
+        time::EphemeraTime,
+    },
+};
+
+pub(crate) mod bracha;
+pub(crate) mod group;
+pub(crate) mod signing;
+
+/// Context keeps the broadcast state for a block
+#[derive(Debug, Clone)]
+pub(crate) struct ProtocolContext {
+    pub(crate) local_peer_id: PeerId,
+    /// Block hash
+    pub(crate) hash: Hash,
+    /// Peers that sent prepare message(this peer included)
+    pub(crate) echo: HashSet<PeerId>,
+    /// Peers that sent commit message(this peer included)
+    pub(crate) vote: HashSet<PeerId>,
+    /// Quorum logic for Bracha protocol
+    pub(crate) quorum: Quorum,
+    /// Flag indicating if the message was delivered to the client
+    pub(crate) delivered: bool,
+}
+
+impl ProtocolContext {
+    pub(crate) fn new(hash: Hash, local_peer_id: PeerId, quorum: Quorum) -> ProtocolContext {
+        ProtocolContext {
+            local_peer_id,
+            hash,
+            echo: HashSet::new(),
+            vote: HashSet::new(),
+            quorum,
+            delivered: false,
+        }
+    }
+
+    fn add_echo(&mut self, peer: PeerId) {
+        self.echo.insert(peer);
+    }
+
+    fn add_vote(&mut self, peer: PeerId) {
+        self.vote.insert(peer);
+    }
+
+    fn echoed(&self) -> bool {
+        self.echo.contains(&self.local_peer_id)
+    }
+
+    fn voted(&self) -> bool {
+        self.vote.contains(&self.local_peer_id)
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub(crate) struct RbMsg {
+    ///Unique id of the message which stays the same throughout the protocol
+    pub(crate) id: EphemeraId,
+    ///Distinct id of the message which changes when the message is rebroadcast
+    pub(crate) request_id: EphemeraId,
+    ///Id of the peer that CREATED the message(not necessarily the one that sent it, with gossip it can come through a different peer)
+    pub(crate) original_sender: PeerId,
+    ///When the message was created by the sender.
+    pub(crate) timestamp: u64,
+    ///Current phase of the protocol(Echo, Vote)
+    pub(crate) phase: MessageType,
+    ///Signature of the message
+    pub(crate) certificate: Certificate,
+}
+
+impl RbMsg {
+    pub(crate) fn new(raw: RawRbMsg, signature: Certificate) -> RbMsg {
+        RbMsg {
+            id: raw.id,
+            request_id: raw.request_id,
+            original_sender: raw.original_sender,
+            timestamp: raw.timestamp,
+            phase: raw.message_type,
+            certificate: signature,
+        }
+    }
+
+    pub(crate) fn block(&self) -> &Block {
+        match &self.phase {
+            MessageType::Echo(block) | MessageType::Vote(block) => block,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub(crate) struct RawRbMsg {
+    pub(crate) id: EphemeraId,
+    pub(crate) request_id: EphemeraId,
+    pub(crate) original_sender: PeerId,
+    pub(crate) timestamp: u64,
+    pub(crate) message_type: MessageType,
+}
+
+impl RawRbMsg {
+    pub(crate) fn new(block: Block, original_sender: PeerId) -> RawRbMsg {
+        RawRbMsg {
+            id: EphemeraId::generate(),
+            request_id: EphemeraId::generate(),
+            original_sender,
+            timestamp: EphemeraTime::now(),
+            message_type: MessageType::Echo(block),
+        }
+    }
+
+    pub(crate) fn block(&self) -> Block {
+        match &self.message_type {
+            MessageType::Echo(block) | MessageType::Vote(block) => block.clone(),
+        }
+    }
+
+    pub(crate) fn reply(&self, local_id: PeerId, phase: MessageType) -> Self {
+        RawRbMsg {
+            id: self.id.clone(),
+            request_id: EphemeraId::generate(),
+            original_sender: local_id,
+            timestamp: EphemeraTime::now(),
+            message_type: phase,
+        }
+    }
+
+    pub(crate) fn echo_reply(&self, local_id: PeerId, data: Block) -> Self {
+        self.reply(local_id, MessageType::Echo(data))
+    }
+
+    pub(crate) fn vote_reply(&self, local_id: PeerId, data: Block) -> Self {
+        self.reply(local_id, MessageType::Vote(data))
+    }
+}
+
+impl From<RbMsg> for RawRbMsg {
+    fn from(msg: RbMsg) -> Self {
+        RawRbMsg {
+            id: msg.id,
+            request_id: msg.request_id,
+            original_sender: msg.original_sender,
+            timestamp: msg.timestamp,
+            message_type: msg.phase,
+        }
+    }
+}
+
+impl Display for RbMsg {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "[id: {}, peer: {}, block: {}, phase: {:?}]",
+            self.id,
+            self.original_sender,
+            self.block().get_hash(),
+            self.phase
+        )
+    }
+}
+
+impl Debug for RbMsg {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "[id: {}, peer: {}, block: {}, phase: {:?}]",
+            self.id,
+            self.original_sender,
+            self.block().get_hash(),
+            self.phase
+        )
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub(crate) enum MessageType {
+    Echo(Block),
+    Vote(Block),
+}

ephemera/src/broadcast/signing.rs

@@ -0,0 +1,150 @@
+use std::collections::HashSet;
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+
+use log::trace;
+use lru::LruCache;
+
+use crate::{
+    block::types::block::{Block, RawBlock},
+    crypto::Keypair,
+    utilities::{codec::Encode, crypto::Certificate, crypto::EphemeraPublicKey, hash::Hash},
+};
+
+pub(crate) struct BlockSigner {
+    /// All signatures of the last blocks that we received from the network(+ our own)
+    verified_signatures: LruCache<Hash, HashSet<Certificate>>,
+    /// Our own keypair
+    signing_keypair: Arc<Keypair>,
+}
+
+impl BlockSigner {
+    pub fn new(keypair: Arc<Keypair>) -> Self {
+        Self {
+            verified_signatures: LruCache::new(NonZeroUsize::new(1000).unwrap()),
+            signing_keypair: keypair,
+        }
+    }
+
+    pub(crate) fn get_block_certificates(
+        &mut self,
+        block_id: &Hash,
+    ) -> Option<&HashSet<Certificate>> {
+        self.verified_signatures.get(block_id)
+    }
+
+    pub(crate) fn sign_block(&mut self, block: &Block, hash: &Hash) -> anyhow::Result<Certificate> {
+        trace!("Signing block: {:?}", block);
+
+        let certificate = block.sign(self.signing_keypair.as_ref())?;
+        self.add_certificate(hash, certificate.clone());
+        Ok(certificate)
+    }
+
+    /// This verification is part of reliable broadcast and verifies only the
+    /// signature of the sender.
+    pub(crate) fn verify_block(
+        &mut self,
+        block: &Block,
+        certificate: &Certificate,
+    ) -> anyhow::Result<()> {
+        trace!("Verifying block: {block:?} against certificate {certificate:?}");
+
+        let raw_block: RawBlock = (*block).clone().into();
+        let raw_block = raw_block.encode()?;
+
+        if certificate
+            .public_key
+            .verify(&raw_block, &certificate.signature)
+        {
+            self.add_certificate(&block.header.hash, certificate.clone());
+            Ok(())
+        } else {
+            anyhow::bail!("Invalid block certificate");
+        }
+    }
+
+    fn add_certificate(&mut self, hash: &Hash, certificate: Certificate) {
+        trace!("Adding certificate to block: {}", hash);
+        self.verified_signatures
+            .get_or_insert_mut(*hash, HashSet::new)
+            .insert(certificate);
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::block::types::block::RawBlockHeader;
+    use crate::block::types::message::{EphemeraMessage, RawEphemeraMessage};
+    use crate::crypto::EphemeraKeypair;
+    use crate::peer::ToPeerId;
+
+    use super::*;
+
+    #[test]
+    fn test_sign_verify_block_ok() {
+        let mut signer = BlockSigner::new(Arc::new(Keypair::generate(None)));
+
+        let message_signing_keypair = Keypair::generate(None);
+
+        let block = new_block(&message_signing_keypair, "label1");
+        let hash = block.hash_with_default_hasher().unwrap();
+
+        let certificate = signer.sign_block(&block, &hash).unwrap();
+
+        assert!(signer.verify_block(&block, &certificate).is_ok());
+    }
+
+    #[test]
+    fn test_sign_signatures_cached_correctly() {
+        let mut signer = BlockSigner::new(Arc::new(Keypair::generate(None)));
+
+        let block = new_block(&Keypair::generate(None), "label1");
+        let hash = block.hash_with_default_hasher().unwrap();
+
+        //Signed by node 1
+        let certificate1 = block.sign(&Keypair::generate(None)).unwrap();
+        signer.verify_block(&block, &certificate1).unwrap();
+        //Signed by node 2
+        let certificate2 = block.sign(&Keypair::generate(None)).unwrap();
+        signer.verify_block(&block, &certificate2).unwrap();
+
+        let block_certificates = signer.get_block_certificates(&hash).unwrap();
+        assert_eq!(block_certificates.len(), 2);
+    }
+
+    #[test]
+    fn test_sign_verify_block_fail() {
+        let mut signer = BlockSigner::new(Arc::new(Keypair::generate(None)));
+        let message_signing_keypair = Keypair::generate(None);
+
+        let block = new_block(&message_signing_keypair, "label1");
+        let certificate = block.sign(&message_signing_keypair).unwrap();
+
+        let modified_block = new_block(&message_signing_keypair, "label2");
+
+        assert!(signer.verify_block(&modified_block, &certificate).is_err());
+    }
+
+    fn new_block(keypair: &Keypair, message_label: &str) -> Block {
+        let peer_id = keypair.public_key().peer_id();
+
+        let raw_ephemera_message =
+            RawEphemeraMessage::new(message_label.to_string(), "payload".as_bytes().to_vec());
+
+        let message_certificate = Certificate::prepare(keypair, &raw_ephemera_message).unwrap();
+        let messages = vec![EphemeraMessage::new(
+            raw_ephemera_message,
+            message_certificate,
+        )];
+
+        let raw_block_header = RawBlockHeader::new(peer_id, 0);
+        let raw_block = RawBlock::new(raw_block_header, messages);
+
+        let block_hash = raw_block
+            .hash_with_default_hasher()
+            .expect("Hashing failed");
+
+        Block::new(raw_block, block_hash)
+    }
+}

ephemera/src/cli/config.rs

@@ -0,0 +1,132 @@
+use std::fs;
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use clap::Parser;
+use log::{error, info};
+use toml::{Table, Value};
+
+use crate::config::Configuration;
+
+#[derive(Debug, Clone, Parser)]
+pub struct UpdateConfigCmd {
+    #[clap(long)]
+    pub config_path: String,
+    #[clap(long)]
+    pub property: String,
+    #[clap(long)]
+    pub value: String,
+}
+
+impl UpdateConfigCmd {
+    /// # Panics
+    /// Panics if the config file does not exist.
+    pub fn execute(self) {
+        let path: PathBuf = self.config_path.clone().into();
+        match Configuration::try_load(path.clone()) {
+            Ok(_) => {
+                info!("Updating config: {:?}", self);
+
+                let toml_str = fs::read_to_string(path.clone()).unwrap();
+                let table = toml_str.parse::<Table>().unwrap();
+
+                let keys = self.property.split('.').collect::<Vec<&str>>();
+
+                if !table.contains_key(keys[0]) {
+                    println!("Key '{}' does not exist", keys[0]);
+                    return;
+                }
+
+                let mut visitor = ConfigVisitor::new(keys, self.value);
+                let table = visitor.process(table);
+
+                let toml_str = toml::to_string(&table).unwrap();
+                fs::write(path, toml_str).unwrap();
+            }
+            Err(err) => {
+                error!("Error loading configuration file: {err:?}");
+            }
+        }
+    }
+}
+
+struct ConfigVisitor<'a> {
+    keys: Vec<&'a str>,
+    value: String,
+    in_array: bool,
+    data_in_array: Vec<Value>,
+}
+
+impl<'a> ConfigVisitor<'a> {
+    fn new(keys: Vec<&'a str>, value: String) -> Self {
+        Self {
+            keys,
+            value,
+            in_array: false,
+            data_in_array: vec![],
+        }
+    }
+
+    #[allow(clippy::needless_pass_by_value)]
+    fn process(&mut self, table: Table) -> Table {
+        let key = self.keys[0];
+        let value = table.get(key).unwrap();
+        self.process_value(table.clone(), value.clone(), 0)
+    }
+
+    fn process_value(&mut self, mut table: Table, value: Value, key_index: usize) -> Table {
+        let root_key = self.keys[key_index];
+        match value {
+            Value::String(str) => {
+                println!("Old value: {str}",);
+                let value = String::from(&self.value);
+                println!("New value: {value}",);
+                table.insert(root_key.to_string(), Value::String(value));
+            }
+            Value::Integer(i) => {
+                println!("Old value: {i}",);
+                let value = i64::from_str(&self.value).unwrap();
+                println!("New value: {value}",);
+                table.insert(root_key.to_string(), Value::Integer(value));
+            }
+            Value::Float(f) => {
+                println!("Old value: {f}",);
+                let value = f64::from_str(&self.value).unwrap();
+                println!("New value: {value}",);
+                table.insert(root_key.to_string(), Value::Float(value));
+            }
+            Value::Boolean(b) => {
+                println!("Old value: {b}",);
+                let value = bool::from_str(&self.value).unwrap();
+                println!("New value: {value}",);
+                table.insert(root_key.to_string(), Value::Boolean(value));
+            }
+            Value::Datetime(_) => {
+                println!("Datetime not supported");
+            }
+            Value::Array(ar) => {
+                self.in_array = true;
+                for value in ar {
+                    self.process_value(table.clone(), value, key_index);
+                }
+                table.remove(root_key);
+                table.insert(
+                    root_key.to_string(),
+                    Value::Array(self.data_in_array.clone()),
+                );
+                self.data_in_array.clear();
+                self.in_array = false;
+            }
+            Value::Table(t) => {
+                let value = t.get(self.keys[key_index + 1]).cloned().unwrap();
+                let updated_table = self.process_value(t, value, key_index + 1);
+                if self.in_array {
+                    self.data_in_array.push(updated_table.into());
+                    return table.clone();
+                }
+                table.insert(root_key.to_string(), updated_table.into());
+            }
+        }
+        table
+    }
+}

ephemera/src/cli/crypto.rs

@@ -0,0 +1,15 @@
+use crate::crypto::{EphemeraKeypair, Keypair};
+use clap::Parser;
+
+use crate::utilities::crypto::EphemeraPublicKey;
+
+#[derive(Debug, Clone, Parser)]
+pub struct GenerateKeypairCmd;
+
+impl GenerateKeypairCmd {
+    pub fn execute() {
+        let keypair = Keypair::generate(None);
+        println!("Keypair: {:>5}", keypair.to_base58());
+        println!("Public key: {:>5}", keypair.public_key().to_base58());
+    }
+}

ephemera/src/cli/init.rs

@@ -0,0 +1,165 @@
+use clap::{Args, Parser};
+use serde::{Deserialize, Serialize};
+
+use crate::config::{
+    BlockManagerConfiguration, Configuration, DatabaseConfiguration, HttpConfiguration,
+    Libp2pConfiguration, MembershipKind as ConfigMembershipKind, NodeConfiguration,
+    WebsocketConfiguration,
+};
+use crate::crypto::{EphemeraKeypair, Keypair};
+
+//network settings
+const DEFAULT_LISTEN_ADDRESS: &str = "127.0.0.1";
+const DEFAULT_PROT_LISTEN_PORT: u16 = 3000;
+const DEFAULT_WS_LISTEN_PORT: u16 = 3001;
+const DEFAULT_HTTP_LISTEN_PORT: u16 = 3002;
+
+//libp2p settings
+const DEFAULT_MESSAGES_TOPIC_NAME: &str = "nym-ephemera-proposed";
+const DEFAULT_HEARTBEAT_INTERVAL_SEC: u64 = 1;
+
+#[derive(Args, Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+#[group(required = true, multiple = false)]
+pub struct MembershipKind {
+    /// Requires the threshold of peers returned by membership provider to be online
+    #[clap(long)]
+    threshold: Option<u64>,
+    /// Requires that all peers returned by membership provider peers to be online
+    #[clap(long)]
+    all: bool,
+    /// Membership is just all online peers from the set returned by membership provider
+    #[clap(long)]
+    any: bool,
+}
+
+impl Default for MembershipKind {
+    fn default() -> Self {
+        MembershipKind {
+            threshold: None,
+            all: false,
+            any: true,
+        }
+    }
+}
+
+impl From<MembershipKind> for ConfigMembershipKind {
+    fn from(kind: MembershipKind) -> Self {
+        match (kind.threshold, kind.all, kind.any) {
+            //FIXME: use threshold value
+            (Some(_), false, false) => ConfigMembershipKind::Threshold,
+            (None, true, false) => ConfigMembershipKind::AllOnline,
+            (None, false, true) => ConfigMembershipKind::AnyOnline,
+            _ => panic!("Invalid membership kind"),
+        }
+    }
+}
+
+#[derive(Parser, Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct Cmd {
+    #[clap(long)]
+    /// The IP address to listen on
+    pub ephemera_ip: Option<String>,
+    /// The port which Ephemera uses for peer to peer communication
+    #[clap(long)]
+    pub ephemera_protocol_port: Option<u16>,
+    /// The port which Ephemera listens on for websocket subscriptions
+    #[clap(long)]
+    pub ephemera_websocket_port: Option<u16>,
+    /// The port which Ephemera listens on for http api
+    #[clap(long)]
+    pub ephemera_http_api_port: Option<u16>,
+    /// Either this node produces blocks or not
+    #[clap(skip)]
+    pub block_producer: bool,
+    /// At which interval to produce blocks
+    #[clap(skip)]
+    pub block_creation_interval_sec: u64,
+    /// When next block is created before previous one is finished, should we repeat it with the same messages
+    #[clap(skip)]
+    pub repeat_last_block_messages: bool,
+    /// The interval at which Ephemera requests the list of members
+    #[clap(skip)]
+    pub members_provider_delay_sec: u64,
+    /// A rule how to choose members based on their online status
+    #[clap(skip)]
+    pub membership_kind: MembershipKind,
+}
+
+impl Default for Cmd {
+    fn default() -> Self {
+        Cmd {
+            ephemera_ip: Some(String::from(DEFAULT_LISTEN_ADDRESS)),
+            ephemera_protocol_port: Some(DEFAULT_PROT_LISTEN_PORT),
+            ephemera_websocket_port: Some(DEFAULT_WS_LISTEN_PORT),
+            ephemera_http_api_port: Some(DEFAULT_HTTP_LISTEN_PORT),
+            block_producer: true,
+            block_creation_interval_sec: 30,
+            repeat_last_block_messages: false,
+            members_provider_delay_sec: 60 * 60,
+            membership_kind: MembershipKind::default(),
+        }
+    }
+}
+
+impl Cmd {
+    /// # Panics
+    /// Panics if the config file already exists.
+    pub fn execute(self, id: Option<&str>) {
+        assert!(
+            Configuration::try_load_from_home_dir().is_err(),
+            "Configuration file already exists",
+        );
+
+        let path = Configuration::ephemera_root_dir(id).unwrap();
+        println!("Creating ephemera node configuration in: {path:?}",);
+
+        let db_dir = path.join("db");
+        let rocksdb_path = db_dir.join("rocksdb");
+        let sqlite_path = db_dir.join("ephemera.sqlite");
+        std::fs::create_dir_all(&rocksdb_path).unwrap();
+        std::fs::File::create(&sqlite_path).unwrap();
+
+        let keypair = Keypair::generate(None);
+        let private_key = keypair.to_base58();
+
+        let default_cfg = Self::default();
+
+        let configuration = Configuration {
+            node: NodeConfiguration {
+                ip: self.ephemera_ip.unwrap_or(default_cfg.ephemera_ip.unwrap()),
+                private_key,
+            },
+            libp2p: Libp2pConfiguration {
+                port: self
+                    .ephemera_protocol_port
+                    .unwrap_or(default_cfg.ephemera_protocol_port.unwrap()),
+                ephemera_msg_topic_name: DEFAULT_MESSAGES_TOPIC_NAME.to_string(),
+                heartbeat_interval_sec: DEFAULT_HEARTBEAT_INTERVAL_SEC,
+                members_provider_delay_sec: self.members_provider_delay_sec,
+                membership_kind: self.membership_kind.into(),
+            },
+            storage: DatabaseConfiguration {
+                rocksdb_path: rocksdb_path.as_os_str().to_str().unwrap().to_string(),
+                sqlite_path: sqlite_path.as_os_str().to_str().unwrap().to_string(),
+                create_if_not_exists: true,
+            },
+            websocket: WebsocketConfiguration {
+                port: self
+                    .ephemera_websocket_port
+                    .unwrap_or(default_cfg.ephemera_websocket_port.unwrap()),
+            },
+            http: HttpConfiguration {
+                port: self
+                    .ephemera_http_api_port
+                    .unwrap_or(default_cfg.ephemera_http_api_port.unwrap()),
+            },
+            block_manager: BlockManagerConfiguration {
+                producer: self.block_producer,
+                creation_interval_sec: self.block_creation_interval_sec,
+                repeat_last_block_messages: self.repeat_last_block_messages,
+            },
+        };
+
+        configuration.try_write_home_dir(id).ok();
+    }
+}

ephemera/src/cli/mod.rs

@@ -0,0 +1,49 @@
+use crate::cli::crypto::GenerateKeypairCmd;
+use clap::Parser;
+
+pub mod config;
+mod crypto;
+pub mod init;
+pub mod peers;
+pub mod run_node;
+
+pub const PEERS_CONFIG_FILE: &str = "peers.toml";
+
+#[derive(Parser)]
+#[command()]
+pub struct Cli {
+    #[command(subcommand)]
+    pub subcommand: Subcommand,
+}
+
+#[derive(clap::Subcommand)]
+pub enum Subcommand {
+    InitConfig(init::Cmd),
+    InitLocalPeersConfig(peers::CreateLocalPeersConfiguration),
+    RunNode(run_node::RunExternalNodeCmd),
+    GenerateKeypair(crypto::GenerateKeypairCmd),
+    UpdateConfig(config::UpdateConfigCmd),
+}
+
+impl Cli {
+    /// # Errors
+    /// Returns an error if the subcommand fails.
+    pub async fn execute(self) -> anyhow::Result<()> {
+        match self.subcommand {
+            Subcommand::InitConfig(init) => {
+                init.execute(None);
+            }
+            Subcommand::InitLocalPeersConfig(add_local_peers) => {
+                add_local_peers.execute();
+            }
+            Subcommand::RunNode(run_node) => run_node.execute().await?,
+            Subcommand::GenerateKeypair(_) => {
+                GenerateKeypairCmd::execute();
+            }
+            Subcommand::UpdateConfig(update_config) => {
+                update_config.execute();
+            }
+        }
+        Ok(())
+    }
+}

ephemera/src/cli/peers.rs

@@ -0,0 +1,65 @@
+//! This is used to create local peers configuration file. Useful for local cluster development.
+
+use crate::cli::PEERS_CONFIG_FILE;
+use clap::Parser;
+
+use crate::config::Configuration;
+use crate::crypto::{EphemeraKeypair, EphemeraPublicKey, Keypair};
+use crate::membership::PeerSetting;
+use crate::network::members::ConfigPeers;
+
+#[derive(Debug, Clone, Parser)]
+pub struct CreateLocalPeersConfiguration;
+
+impl CreateLocalPeersConfiguration {
+    /// # Panics
+    /// Panics if the configuration file cannot be written.
+    pub fn execute(self) {
+        let peers = Self::from_ephemera_dev_cluster_conf().unwrap();
+        let config_peers = ConfigPeers::new(peers);
+
+        let peers_conf_path = Configuration::ephemera_root_dir(None)
+            .unwrap()
+            .join(PEERS_CONFIG_FILE);
+
+        config_peers.try_write(peers_conf_path).unwrap();
+    }
+
+    //LOCAL DEV CLUSTER ONLY
+    //Get peers from dev Ephemera cluster config files
+    pub(crate) fn from_ephemera_dev_cluster_conf() -> anyhow::Result<Vec<PeerSetting>> {
+        let ephemera_root_dir = Configuration::ephemera_root_dir(None).unwrap();
+
+        let mut peers = vec![];
+
+        let home_dir = std::fs::read_dir(ephemera_root_dir)?;
+        for entry in home_dir {
+            let path = entry?.path();
+            if path.is_dir() {
+                let cosmos_address = path.file_name().unwrap().to_str().unwrap();
+
+                println!("Reading peer info config from node {cosmos_address}",);
+
+                let conf = Configuration::try_load_from_home_dir().unwrap_or_else(|_| {
+                    panic!("Error loading configuration for node {cosmos_address}")
+                });
+
+                let node_info = conf.node;
+
+                let keypair = bs58::decode(&node_info.private_key).into_vec().unwrap();
+                let keypair = Keypair::from_bytes(&keypair).unwrap();
+
+                let peer = PeerSetting {
+                    cosmos_address: cosmos_address.to_string(),
+                    address: format!("/ip4/{}/tcp/{}", node_info.ip, conf.libp2p.port),
+                    public_key: keypair.public_key().to_base58(),
+                };
+                peers.push(peer);
+
+                println!("Loaded config for node {cosmos_address}",);
+            }
+        }
+
+        Ok(peers)
+    }
+}

ephemera/src/cli/run_node.rs

@@ -0,0 +1,140 @@
+use anyhow::anyhow;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use clap::Parser;
+use log::trace;
+use nym_task::TaskManager;
+use reqwest::Url;
+
+use crate::ephemera_api::ApplicationResult;
+use crate::utilities::codec::{Codec, EphemeraCodec};
+use crate::{
+    api::application::CheckBlockResult,
+    cli::PEERS_CONFIG_FILE,
+    config::Configuration,
+    crypto::EphemeraKeypair,
+    crypto::Keypair,
+    ephemera_api::{ApiBlock, ApiEphemeraMessage, Application, Dummy, RawApiEphemeraMessage},
+    network::members::ConfigMembersProvider,
+    EphemeraStarterInit,
+};
+
+#[derive(Clone, Debug)]
+pub struct HttpMembersProviderArg {
+    pub url: Url,
+}
+
+impl FromStr for HttpMembersProviderArg {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(HttpMembersProviderArg { url: s.parse()? })
+    }
+}
+
+#[derive(Parser)]
+pub struct RunExternalNodeCmd {
+    #[clap(short, long)]
+    pub config_file: String,
+    #[clap(short, long)]
+    pub peers_config: String,
+}
+
+impl RunExternalNodeCmd {
+    /// # Errors
+    /// If the members provider cannot be created.
+    ///
+    /// # Panics
+    /// If the ephemera cannot be created.
+    pub async fn execute(&self) -> anyhow::Result<()> {
+        let ephemera_conf = match Configuration::try_load(self.config_file.clone()) {
+            Ok(conf) => conf,
+            Err(err) => anyhow::bail!("Error loading configuration file: {err:?}"),
+        };
+
+        let members_provider = Self::config_members_provider_with_path(self.peers_config.clone())?;
+        let ephemera = EphemeraStarterInit::new(ephemera_conf.clone())
+            .unwrap()
+            .with_application(Dummy)
+            .with_members_provider(members_provider)?
+            .build();
+
+        let shutdown = TaskManager::new(10);
+
+        let shutdown_listener = shutdown.subscribe();
+        tokio::spawn(ephemera.run(shutdown_listener));
+
+        if let Err(err) = shutdown.catch_interrupt().await {
+            Err(anyhow!("Shutdown error {:?}", err))
+        } else {
+            Ok(())
+        }
+    }
+
+    #[allow(dead_code)]
+    fn config_members_provider() -> anyhow::Result<ConfigMembersProvider> {
+        let peers_conf_path = Configuration::ephemera_root_dir(None)
+            .unwrap()
+            .join(PEERS_CONFIG_FILE);
+
+        let peers_conf = match ConfigMembersProvider::init(peers_conf_path) {
+            Ok(conf) => conf,
+            Err(err) => anyhow::bail!("Error loading peers file: {err:?}"),
+        };
+        Ok(peers_conf)
+    }
+
+    #[allow(dead_code)]
+    fn config_members_provider_with_path(
+        peers_conf_path: String,
+    ) -> anyhow::Result<ConfigMembersProvider> {
+        let peers_conf = match ConfigMembersProvider::init(peers_conf_path) {
+            Ok(conf) => conf,
+            Err(err) => anyhow::bail!("Error loading peers file: {err:?}"),
+        };
+        Ok(peers_conf)
+    }
+}
+
+pub struct SignatureVerificationApplication {
+    keypair: Arc<Keypair>,
+}
+
+impl SignatureVerificationApplication {
+    #[must_use]
+    pub fn new(keypair: Arc<Keypair>) -> Self {
+        Self { keypair }
+    }
+
+    pub(crate) fn verify_message(&self, msg: ApiEphemeraMessage) -> anyhow::Result<()> {
+        let signature = msg.certificate.clone();
+        let raw_message: RawApiEphemeraMessage = msg.into();
+        let encoded_message = Codec::encode(&raw_message)?;
+        if self
+            .keypair
+            .verify(&encoded_message, &signature.signature.into())
+        {
+            Ok(())
+        } else {
+            anyhow::bail!("Invalid signature")
+        }
+    }
+}
+
+impl Application for SignatureVerificationApplication {
+    fn check_tx(&self, tx: ApiEphemeraMessage) -> ApplicationResult<bool> {
+        trace!("SignatureVerificationApplicationHook::check_tx");
+        self.verify_message(tx)?;
+        Ok(true)
+    }
+
+    fn check_block(&self, _block: &ApiBlock) -> ApplicationResult<CheckBlockResult> {
+        Ok(CheckBlockResult::Accept)
+    }
+
+    fn deliver_block(&self, _block: ApiBlock) -> ApplicationResult<()> {
+        trace!("SignatureVerificationApplicationHook::deliver_block");
+        Ok(())
+    }
+}

ephemera/src/config/mod.rs

@@ -0,0 +1,334 @@
+//! Configuration for Ephemeris node. It contains mandatory settings for a node to start.
+//!
+//! Default location for the configuration file is `~/.nym/ephemera/ephemera.toml`.
+//! Or relative to a node specific directory `~/.nym/ephemera/<node_name>/ephemera.toml`.
+
+use std::io::Write;
+use std::path::PathBuf;
+
+use config::ConfigError;
+use log::{error, info};
+use nym_config::{DEFAULT_NYM_APIS_DIR, NYM_DIR};
+use serde_derive::{Deserialize, Serialize};
+use thiserror::Error;
+
+//TODO - validate configuration at load time
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct Configuration {
+    /// Configuration related to node instance identity
+    pub node: NodeConfiguration,
+    /// Configuration for libp2p network
+    pub libp2p: Libp2pConfiguration,
+    /// Configuration for Ephemera embedded database
+    pub storage: DatabaseConfiguration,
+    /// Configuration for websocket
+    pub websocket: WebsocketConfiguration,
+    /// Configuration for embedded http API server
+    pub http: HttpConfiguration,
+    /// Configuration related to block creation
+    pub block_manager: BlockManagerConfiguration,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct NodeConfiguration {
+    /// Node IP shared by all Ephemera services like libp2p, websocket, http.
+    /// If separate IP/DNS is needed for each service, it should be configured outside of Ephemera.
+    pub ip: String,
+    //FIXME: dev only
+    /// If private key is stored in configuration as plain text, read it from here.
+    /// Private key is mandatory for a node to be able to function in the network.
+    /// It is used to signe protocol messages and identify node in the network.
+    pub private_key: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum MembershipKind {
+    /// Mandatory minimum membership size is defined by threshold of all peers returned by membership provider.
+    /// Threshold value is defined the ratio of peers that need to be available.
+    /// For example, if the threshold is 0.5, then at least 50% of the peers need to be available.
+    Threshold,
+    /// Mandatory minimum membership size is all peers who are online.
+    AnyOnline,
+    /// Mandatory minimum membership size is all peers returned by membership provider.
+    AllOnline,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct Libp2pConfiguration {
+    /// Port to listen on for libp2p internal connections
+    pub port: u16,
+    /// Gossipsub topic to gossip Ephemera messages between peers. Ephemera listens messages
+    /// only from this topic. Invalid topic configuration means that Ephemera is not able to
+    /// reach messages from other peers.
+    pub ephemera_msg_topic_name: String,
+    /// Gossipsub interval to check its mesh health.
+    pub heartbeat_interval_sec: u64,
+    /// How often Ephemera checks its membership rendezvous endpoint. It's configurable with second granularity.
+    /// But in general it's up to the user and depends how rendezvous endpoint is configured.
+    ///
+    /// Ephemera uses rendezvous endpoint as authority to tell which nodes are authorized to participate.
+    /// So it should be configured and implemented in a manner that nodes always have the most up to date and
+    /// accurate information.
+    pub members_provider_delay_sec: u64,
+    /// Defines how the actual membership is decided. See `[ephemera:]` for more details.
+    pub membership_kind: MembershipKind,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct DatabaseConfiguration {
+    /// Path to the RocksDb database directory
+    pub rocksdb_path: String,
+    /// Path to the SQLite database file
+    pub sqlite_path: String,
+    /// If to create database if it does not exist
+    pub create_if_not_exists: bool,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct WebsocketConfiguration {
+    /// Port to listen on for WebSocket subscriptions.
+    pub port: u16,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct HttpConfiguration {
+    /// Port to listen on for HTTP API requests
+    pub port: u16,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct BlockManagerConfiguration {
+    /// By default every node is block producer.
+    ///
+    /// But in trusted settings it might be useful configure only one or selected nodes to be block producer.
+    /// The rest of nodes still participate in the message gossiping and reliable broadcast.
+    pub producer: bool,
+    /// Interval in seconds between block creation
+    /// Blocks are "proposed" at this interval.
+    pub creation_interval_sec: u64,
+    /// Ephemera creates blocks at fixed interval and doesn't have any consensus algorithm to make progress
+    /// if the most recent block fails to go through reliable broadcast.
+    /// This flag tells what to do when at next interval previous block is not yet delivered.
+    ///
+    /// If true, Ephemera will repeat messages from the previous block. Otherwise it will take all messages
+    /// from mempool as normally.
+    pub repeat_last_block_messages: bool,
+}
+
+impl BlockManagerConfiguration {
+    pub fn new(producer: bool, creation_interval_sec: u64, repeat_last_block: bool) -> Self {
+        BlockManagerConfiguration {
+            producer,
+            creation_interval_sec,
+            repeat_last_block_messages: repeat_last_block,
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum Error {
+    /// This is returned if configuration file exists and user tries to create new one.
+    #[error("Configuration file exists: '{0}'")]
+    Exists(String),
+    /// This is returned if configuration file does not exist and user tries to load it.
+    #[error("Configuration file does not exists: '{0}'")]
+    NotFound(String),
+    #[error("Configuration file does not exist")]
+    /// This is returned if IoError happens during configuration file read/write.
+    IoError(#[from] std::io::Error),
+    /// This is returned if configuration file is invalid.
+    #[error("Configuration file is invalid: '{0}'")]
+    InvalidFormat(String),
+    /// This is returned if configuration file path is invalid.
+    #[error("Configuration file path is invalid: '{0}'")]
+    InvalidPath(String),
+    /// Technical error happens during parsing.
+    #[error("{}", .0)]
+    Other(String),
+}
+
+impl From<ConfigError> for Error {
+    fn from(err: ConfigError) -> Self {
+        match err {
+            ConfigError::NotFound(err) => Error::NotFound(err),
+            ConfigError::PathParse(err) => {
+                Error::InvalidPath(format!("Invalid path to configuration file: {err:?}",))
+            }
+            ConfigError::FileParse { uri, cause } => {
+                Error::InvalidFormat(format!("Invalid configuration file: {uri:?}: {cause:?}",))
+            }
+            _ => Error::Other(err.to_string()),
+        }
+    }
+}
+
+const EPHEMERA_DIR_NAME: &str = "ephemera";
+const EPHEMERA_CONFIG_FILE: &str = "ephemera.toml";
+
+type Result<T> = std::result::Result<T, Error>;
+
+impl Configuration {
+    /// Tries to read Ephemera node configuration file (`ephemera.toml`) from the given path.
+    ///
+    /// # Arguments
+    /// * `path` - Path to the configuration file.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file does not exist or is invalid.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use ephemera::configuration::Configuration;
+    ///
+    /// let config = Configuration::try_load("/ephemera/ephemera.toml");
+    /// ```
+    pub fn try_load<P: Into<PathBuf>>(path: P) -> Result<Configuration> {
+        let buf = path.into();
+        log::debug!("Loading configuration from {:?}", buf);
+        let config = config::Config::builder()
+            .add_source(config::File::from(buf))
+            .build()
+            .map_err(Error::from)?;
+
+        config.try_deserialize().map_err(Error::from)
+    }
+
+    /// Tries to read Ephemera node configuration from default
+    /// default Ephemera configuration directory(`~.nym/ephemera`). Full path resolves
+    /// as `~/.nym/ephemera/<node_name>/<file>`.
+    ///
+    /// # Arguments
+    /// * `node_name` - Name of the node.
+    /// * `file` - Name of the configuration file.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file does not exist or is invalid.
+    pub fn try_load_node_from_home_dir(file: &str) -> Result<Configuration> {
+        let file_path = Self::ephemera_node_dir(None)?.join(file);
+        Configuration::try_load(file_path)
+    }
+
+    /// Tries to read Ephemera node configuration from default Ephemera configuration directory(`~.nym/ephemera`).
+    /// Full path resolves as `~/.nym/ephemera/<node_name>/ephemera.toml`.
+    ///
+    /// # Arguments
+    /// * `node_name` - Name of the node.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file does not exist or is invalid.
+    pub fn try_load_from_home_dir() -> Result<Configuration> {
+        let file_path = Configuration::ephemera_config_file_home(None)?;
+        let config = config::Config::builder()
+            .add_source(config::File::from(file_path))
+            .build()
+            .map_err(Error::from)?;
+
+        config.try_deserialize().map_err(Error::from)
+    }
+
+    /// Tries to write(create) Ephemera node configuration file (`ephemera.toml`) relative to default
+    /// Ephemera configuration directory(`~.nym/ephemera`). Full path resolves as `~/.nym/ephemera/<node_name>/ephemera.toml`.
+    ///
+    /// # Arguments
+    /// * `id` - Id of the node.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file already exists.
+    ///
+    /// # Panics
+    /// Panics if the configuration file cannot be written.
+    pub fn try_write_home_dir(&self, id: Option<&str>) -> Result<()> {
+        let conf_path = Configuration::ephemera_node_dir(id)?;
+        if !conf_path.exists() {
+            std::fs::create_dir_all(conf_path)?;
+        }
+
+        let file_path = Configuration::ephemera_config_file_home(id)?;
+        if file_path.exists() {
+            return Err(Error::Exists(file_path.to_str().unwrap().to_string()));
+        }
+
+        self.write(&file_path)?;
+        Ok(())
+    }
+
+    /// Tries to write(update) Ephemera node configuration file (`ephemera.toml`) relative to default
+    /// Ephemera configuration directory(`~.nym/ephemera`). Full path resolves as `~/.nym/ephemera/<node_name>/ephemera.toml`.
+    /// If the file does not exist, update will be refused.
+    ///
+    /// # Arguments
+    /// * `node_name` - Name of the node.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file does not exist.
+    ///
+    /// # Panics
+    /// Panics if the configuration file cannot be written.
+    pub fn try_update_home_dir(&self) -> Result<()> {
+        let file_path = Configuration::ephemera_config_file_home(None)?;
+        if !file_path.exists() {
+            error!(
+                "Configuration file does not exist {}",
+                file_path.to_str().unwrap()
+            );
+            return Err(Error::NotFound(file_path.to_str().unwrap().to_string()));
+        }
+        self.write(&file_path)?;
+        Ok(())
+    }
+
+    /// Returns node configuration file path relative to default Ephemera configuration directory(`~.nym/ephemera`).
+    /// Full path resolves as `~/.nym/ephemera/<id>/ephemera.toml`.
+    ///
+    /// # Arguments
+    /// * `id` - Id of the node.
+    ///
+    /// # Errors
+    /// Returns an error if the configuration file path cannot be resolved.
+    pub fn ephemera_config_file_home(id: Option<&str>) -> Result<PathBuf> {
+        Ok(Self::ephemera_node_dir(id)?.join(EPHEMERA_CONFIG_FILE))
+    }
+
+    /// Returns default Ephemera configuration directory(`~.nym/ephemera`).
+    ///
+    /// # Errors
+    /// Returns an error if the configuration directory cannot be resolved.
+    pub fn ephemera_root_dir(id: Option<&str>) -> Result<PathBuf> {
+        let id = id.unwrap_or_default();
+        dirs::home_dir()
+            .map(|home| {
+                home.join(NYM_DIR)
+                    .join(DEFAULT_NYM_APIS_DIR)
+                    .join(id)
+                    .join(EPHEMERA_DIR_NAME)
+            })
+            .ok_or(Error::Other("Could not find home directory".to_string()))
+    }
+
+    pub(crate) fn ephemera_node_dir(id: Option<&str>) -> Result<PathBuf> {
+        Self::ephemera_root_dir(id)
+    }
+
+    fn write(&self, file_path: &PathBuf) -> Result<()> {
+        //TODO: use toml or config crate, not both
+        let config = toml::to_string(&self).map_err(|e| {
+            Error::InvalidFormat(format!("Failed to serialize configuration: {e}",))
+        })?;
+
+        let config = format!(
+            "#This file is generated by cli and automatically overwritten every time when cli is run\n{config}",
+        );
+
+        if file_path.exists() {
+            info!("Updating configuration file: '{}'", file_path.display());
+        } else {
+            info!("Writing configuration to file: '{}'", file_path.display());
+        }
+
+        let mut file = std::fs::File::create(file_path)?;
+        file.write_all(config.as_bytes())?;
+
+        Ok(())
+    }
+}

ephemera/src/core/api_cmd.rs

@@ -0,0 +1,390 @@
+use std::num::NonZeroUsize;
+
+use log::{debug, error, trace};
+use lru::LruCache;
+use tokio::sync::oneshot::Sender;
+
+use crate::api::types::{ApiBlockBroadcastInfo, ApiBroadcastInfo};
+use crate::api::{DhtKV, DhtKey, DhtValue};
+use crate::ephemera_api::ApiEphemeraMessage;
+use crate::peer::ToPeerId;
+use crate::{
+    api::{
+        self,
+        application::Application,
+        types::{ApiBlock, ApiCertificate, ApiError},
+        ToEphemeraApiCmd,
+    },
+    block::{manager::BlockManagerError, types::message},
+    crypto::EphemeraKeypair,
+    ephemera_api::ApiEphemeraConfig,
+    network::libp2p::ephemera_sender::EphemeraEvent,
+    Ephemera,
+};
+
+type DhtPendingQueryReply = Sender<Result<Option<(Vec<u8>, Vec<u8>)>, ApiError>>;
+
+pub(crate) struct ApiCmdProcessor {
+    pub(crate) dht_query_cache: LruCache<Vec<u8>, Vec<DhtPendingQueryReply>>,
+}
+
+impl ApiCmdProcessor {
+    pub(crate) fn new() -> Self {
+        Self {
+            dht_query_cache: LruCache::new(NonZeroUsize::new(1000).unwrap()),
+        }
+    }
+
+    pub(crate) async fn process_api_requests<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        cmd: ToEphemeraApiCmd,
+    ) -> api::Result<()> {
+        trace!("Processing API request: {:?}", cmd);
+        match cmd {
+            ToEphemeraApiCmd::SubmitEphemeraMessage(api_msg, reply) => {
+                // Ask application to decide if we should accept this message.
+                Self::submit_message(ephemera, api_msg, reply).await?;
+            }
+
+            ToEphemeraApiCmd::QueryBlockByHash(block_hash, reply) => {
+                Self::query_block_by_hash(ephemera, &block_hash, reply).await;
+            }
+
+            ToEphemeraApiCmd::QueryBlockByHeight(height, reply) => {
+                Self::query_block_by_height(ephemera, height, reply).await;
+            }
+
+            ToEphemeraApiCmd::QueryLastBlock(reply) => {
+                Self::query_last_block(ephemera, reply).await;
+            }
+
+            ToEphemeraApiCmd::QueryBlockCertificates(block_id, reply) => {
+                Self::query_block_certificates(ephemera, &block_id, reply).await;
+            }
+
+            ToEphemeraApiCmd::QueryDht(key, reply) => {
+                Self::query_dht(ephemera, key, reply).await;
+            }
+
+            ToEphemeraApiCmd::StoreInDht(key, value, reply) => {
+                Self::store_in_dht(ephemera, key, value, reply).await;
+            }
+
+            ToEphemeraApiCmd::QueryEphemeraConfig(reply) => {
+                Self::ephemera_config(ephemera, reply);
+            }
+
+            ToEphemeraApiCmd::QueryBroadcastGroup(reply) => {
+                Self::broadcast_group(ephemera, reply);
+            }
+            ToEphemeraApiCmd::QueryBlockBroadcastInfo(hash, reply) => {
+                Self::query_block_broadcast_info(ephemera, &hash, reply).await;
+            }
+            ToEphemeraApiCmd::VerifyMessageInBlock(block_hash, message_hash, index, reply) => {
+                Self::verify_message_in_block(ephemera, block_hash, message_hash, index, reply)
+                    .await;
+            }
+        }
+        Ok(())
+    }
+
+    fn broadcast_group<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        reply: Sender<api::Result<ApiBroadcastInfo>>,
+    ) {
+        let group_peers = ephemera.broadcast_group.current();
+
+        let bc = ApiBroadcastInfo::new(group_peers.clone(), ephemera.node_info.peer_id);
+        reply
+            .send(Ok(bc))
+            .expect("Error sending BroadcastGroup response to api");
+    }
+
+    fn ephemera_config<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        reply: Sender<api::Result<ApiEphemeraConfig>>,
+    ) {
+        let node_info = ephemera.node_info.clone();
+        let api_config = ApiEphemeraConfig {
+            protocol_address: node_info.protocol_address(),
+            api_address: node_info.api_address_http(),
+            websocket_address: node_info.ws_address_ws(),
+            public_key: node_info.keypair.public_key().to_string(),
+            block_producer: node_info.initial_config.block_manager.producer,
+            block_creation_interval_sec: node_info
+                .initial_config
+                .block_manager
+                .creation_interval_sec,
+        };
+        reply
+            .send(Ok(api_config))
+            .expect("Error sending EphemeraConfig response to api");
+    }
+
+    async fn store_in_dht<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        key: DhtKey,
+        value: DhtValue,
+        reply: Sender<api::Result<()>>,
+    ) {
+        let response = match ephemera
+            .to_network
+            .send_ephemera_event(EphemeraEvent::StoreInDht { key, value })
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(err) => {
+                error!("Error sending StoreInDht to network: {:?}", err);
+                Err(ApiError::Internal("Failed to store in DHT".to_string()))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending StoreInDht response to api");
+    }
+
+    async fn query_dht<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        key: DhtKey,
+        reply: Sender<api::Result<Option<DhtKV>>>,
+    ) {
+        match ephemera
+            .to_network
+            .send_ephemera_event(EphemeraEvent::QueryDht { key: key.clone() })
+            .await
+        {
+            Ok(_) => {
+                //Save the reply channel in a map and send the reply when we get the response from the network
+                ephemera
+                    .api_cmd_processor
+                    .dht_query_cache
+                    .get_or_insert_mut(key, Vec::new)
+                    .push(reply);
+            }
+            Err(err) => {
+                error!("Error sending QueryDht to network: {:?}", err);
+                reply
+                    .send(Err(ApiError::Internal("Failed to query DHT".to_string())))
+                    .expect("Error sending QueryDht response to api");
+            }
+        };
+    }
+
+    async fn query_block_certificates<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        block_id: &str,
+        reply: Sender<api::Result<Option<Vec<ApiCertificate>>>>,
+    ) {
+        let response = match ephemera
+            .storage
+            .lock()
+            .await
+            .get_block_certificates(block_id)
+        {
+            Ok(signatures) => {
+                let certificates = signatures.map(|s| {
+                    s.into_iter()
+                        .map(Into::into)
+                        .collect::<Vec<ApiCertificate>>()
+                });
+                Ok(certificates)
+            }
+            Err(err) => {
+                error!("Error querying block certificates: {:?}", err);
+                Err(ApiError::Internal(
+                    "Failed to query block certificates".to_string(),
+                ))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending QueryBlockSignatures response to api");
+    }
+
+    async fn query_last_block<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        reply: Sender<api::Result<ApiBlock>>,
+    ) {
+        let response = match ephemera.storage.lock().await.get_last_block() {
+            Ok(Some(block)) => Ok(block.into()),
+            Ok(None) => Err(ApiError::Internal(
+                "No blocks found, this is a bug!".to_string(),
+            )),
+            Err(err) => {
+                error!("Error querying last block: {:?}", err);
+                Err(ApiError::Internal("Failed to query last block".to_string()))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending QueryLastBlock response to api");
+    }
+
+    async fn query_block_by_height<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        height: u64,
+        reply: Sender<api::Result<Option<ApiBlock>>>,
+    ) {
+        let response = match ephemera.storage.lock().await.get_block_by_height(height) {
+            Ok(Some(block)) => {
+                let api_block: ApiBlock = block.into();
+                Ok(api_block.into())
+            }
+            Ok(None) => Ok(None),
+            Err(err) => {
+                error!("Error querying block by height: {:?}", err);
+                Err(ApiError::Internal(
+                    "Failed to query block by height".to_string(),
+                ))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending QueryBlockByHeight response to api");
+    }
+
+    async fn query_block_by_hash<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        block_hash: &str,
+        reply: Sender<api::Result<Option<ApiBlock>>>,
+    ) {
+        let response = match ephemera.storage.lock().await.get_block_by_hash(block_hash) {
+            Ok(Some(block)) => {
+                let api_block: ApiBlock = block.into();
+                Ok(api_block.into())
+            }
+            Ok(None) => Ok(None),
+            Err(err) => {
+                error!("Error querying block by id: {:?}", err);
+                Err(ApiError::Internal(
+                    "Failed to query block by id".to_string(),
+                ))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending QueryBlockByHash response to api");
+    }
+
+    async fn submit_message<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        api_msg: Box<ApiEphemeraMessage>,
+        reply: Sender<api::Result<()>>,
+    ) -> api::Result<()> {
+        let response = match ephemera.application.check_tx(*api_msg.clone()) {
+            Ok(true) => {
+                trace!("Application accepted ephemera message: {:?}", api_msg);
+
+                // Send to BlockManager to verify it and put into memory pool
+                let ephemera_msg: message::EphemeraMessage = (*api_msg).into();
+                match ephemera.block_manager.on_new_message(ephemera_msg.clone()) {
+                    Ok(_) => {
+                        //Gossip to network for other nodes to receive
+                        match ephemera
+                            .to_network
+                            .send_ephemera_event(EphemeraEvent::EphemeraMessage(
+                                ephemera_msg.into(),
+                            ))
+                            .await
+                        {
+                            Ok(_) => Ok(()),
+                            Err(err) => {
+                                error!("Error sending EphemeraMessage to network: {:?}", err);
+                                Err(ApiError::Internal("Failed to submit message".to_string()))
+                            }
+                        }
+                    }
+                    Err(err) => match err {
+                        BlockManagerError::DuplicateMessage(_) => Err(ApiError::DuplicateMessage),
+                        BlockManagerError::BlockManager(err) => {
+                            error!("Error submitting message to block manager: {:?}", err);
+                            Err(ApiError::Internal("Failed to submit message".to_string()))
+                        }
+                    },
+                }
+            }
+            Ok(false) => {
+                debug!("Application rejected ephemera message: {:?}", api_msg);
+                Err(ApiError::ApplicationRejectedMessage)
+            }
+            Err(err) => {
+                error!("Application rejected ephemera message: {:?}", err);
+                Err(ApiError::Application(err))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending SubmitEphemeraMessage response to api");
+        Ok(())
+    }
+
+    async fn query_block_broadcast_info<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        block_id: &str,
+        reply: Sender<api::Result<Option<ApiBlockBroadcastInfo>>>,
+    ) {
+        let response = match ephemera
+            .storage
+            .lock()
+            .await
+            .get_block_broadcast_group(block_id)
+        {
+            Ok(Some(peers)) => {
+                let local_peer = ephemera.node_info.keypair.peer_id();
+                Ok(Some(ApiBlockBroadcastInfo::new(local_peer, peers)))
+            }
+            Ok(None) => Ok(None),
+            Err(err) => {
+                error!("Error querying block broadcast info: {:?}", err);
+                Err(ApiError::Internal(
+                    "Failed to query block broadcast info".to_string(),
+                ))
+            }
+        };
+        reply
+            .send(response)
+            .expect("Error sending QueryBlockBroadcastGroup response to api");
+    }
+    async fn verify_message_in_block<A: Application>(
+        ephemera: &mut Ephemera<A>,
+        block_hash: String,
+        message_hash: String,
+        index: usize,
+        reply: Sender<api::Result<bool>>,
+    ) {
+        let message_hash_hash = message_hash.parse();
+        if message_hash_hash.is_err() {
+            reply
+                .send(Err(ApiError::InvalidHash(
+                    "Failed to parse message hash".to_string(),
+                )))
+                .expect("Error sending VerifyMessageInBlock response to api");
+            return;
+        }
+
+        let message_hash_hash = message_hash_hash.unwrap();
+
+        let storage = ephemera.storage.lock().await;
+        match storage.get_block_merkle_tree(&block_hash) {
+            Ok(Some(tree)) => {
+                let result = tree.verify_leaf_at_index(message_hash_hash, index);
+                reply
+                    .send(Ok(result))
+                    .expect("Error sending VerifyMessageInBlock response to api");
+            }
+            Ok(None) => {
+                reply
+                    .send(Ok(false))
+                    .expect("Error sending VerifyMessageInBlock response to api");
+            }
+            Err(err) => {
+                error!("Error querying block merkle tree: {:?}", err);
+                reply
+                    .send(Err(ApiError::Internal(
+                        "Failed to verify message".to_string(),
+                    )))
+                    .expect("Error sending VerifyMessageInBlock response to api");
+            }
+        }
+    }
+}

ephemera/src/core/builder.rs

@@ -0,0 +1,413 @@
+use std::fmt::Display;
+use std::future::Future;
+use std::sync::Arc;
+
+use futures_util::future::BoxFuture;
+use futures_util::FutureExt;
+use log::{debug, error, info};
+use tokio::sync::Mutex;
+
+use crate::core::shutdown::Shutdown;
+#[cfg(feature = "rocksdb_storage")]
+use crate::storage::rocksdb::RocksDbStorage;
+#[cfg(feature = "sqlite_storage")]
+use crate::storage::sqlite::SqliteStorage;
+use crate::{
+    api::{application::Application, http, ApiListener, CommandExecutor},
+    block::{builder::BlockManagerBuilder, manager::BlockManager},
+    broadcast::bracha::broadcast::Broadcaster,
+    broadcast::group::BroadcastGroup,
+    config::Configuration,
+    core::{
+        api_cmd::ApiCmdProcessor,
+        shutdown::{Handle, ShutdownManager},
+    },
+    crypto::Keypair,
+    membership,
+    membership::PeerInfo,
+    network::libp2p::{
+        ephemera_sender::EphemeraToNetworkSender, network_sender::NetCommunicationReceiver,
+        swarm_network::SwarmNetwork,
+    },
+    peer::{PeerId, ToPeerId},
+    storage::EphemeraDatabase,
+    utilities::crypto::key_manager::KeyManager,
+    websocket::ws_manager::{WsManager, WsMessageBroadcaster},
+    Ephemera,
+};
+
+#[derive(Clone)]
+pub(crate) struct NodeInfo {
+    pub(crate) ip: String,
+    pub(crate) protocol_port: u16,
+    pub(crate) http_port: u16,
+    pub(crate) ws_port: u16,
+    pub(crate) peer_id: PeerId,
+    pub(crate) keypair: Arc<Keypair>,
+    pub(crate) initial_config: Configuration,
+}
+
+impl NodeInfo {
+    pub(crate) fn new(config: Configuration) -> anyhow::Result<Self> {
+        let keypair = KeyManager::read_keypair_from_str(&config.node.private_key)?;
+        let info = Self {
+            ip: config.node.ip.clone(),
+            protocol_port: config.libp2p.port,
+            http_port: config.http.port,
+            ws_port: config.websocket.port,
+            peer_id: keypair.peer_id(),
+            keypair,
+            initial_config: config,
+        };
+        Ok(info)
+    }
+
+    pub(crate) fn protocol_address(&self) -> String {
+        format!("/ip4/{}/tcp/{}", self.ip, self.protocol_port)
+    }
+
+    pub(crate) fn api_address_http(&self) -> String {
+        format!("http://{}:{}", self.ip, self.http_port)
+    }
+
+    pub(crate) fn ws_address_ws(&self) -> String {
+        format!("ws://{}:{}", self.ip, self.ws_port)
+    }
+
+    pub(crate) fn ws_address_ip_port(&self) -> String {
+        format!("{}:{}", self.ip, self.ws_port)
+    }
+}
+
+impl Display for NodeInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "NodeInfo {{ ip: {}, protocol_port: {}, http_port: {}, ws_port: {}, peer_id: {}, keypair: {} }}",
+            self.ip,
+            self.protocol_port,
+            self.http_port,
+            self.ws_port,
+            self.peer_id,
+            self.keypair
+        )
+    }
+}
+
+#[derive(Clone)]
+pub struct EphemeraHandle {
+    /// Ephemera API
+    pub api: CommandExecutor,
+    /// Allows to send shutdown signal to the node
+    pub shutdown: Handle,
+}
+
+pub struct EphemeraStarterInit {
+    config: Configuration,
+    node_info: NodeInfo,
+    broadcaster: Broadcaster,
+    api_listener: ApiListener,
+    api: CommandExecutor,
+}
+
+impl EphemeraStarterInit {
+    /// Initialize Ephemera builder
+    ///
+    /// # Arguments
+    /// * `config` - [Configuration]
+    ///
+    /// # Returns
+    /// [`EphemeraStarterInit`]
+    ///
+    /// # Errors
+    /// * If the node configuration is invalid
+    pub fn new(config: Configuration) -> anyhow::Result<Self> {
+        let instance_info = NodeInfo::new(config.clone())?;
+        let broadcaster = Broadcaster::new(instance_info.peer_id);
+        let (api, api_listener) = CommandExecutor::new();
+
+        let builder = EphemeraStarterInit {
+            config,
+            node_info: instance_info,
+            broadcaster,
+            api_listener,
+            api,
+        };
+        Ok(builder)
+    }
+
+    pub fn with_application<A: Application>(
+        self,
+        application: A,
+    ) -> EphemeraStarterWithApplication<A> {
+        EphemeraStarterWithApplication {
+            init: self,
+            application,
+        }
+    }
+}
+
+#[derive(Default)]
+struct ServiceInfo {
+    ws_message_broadcast: Option<WsMessageBroadcaster>,
+    from_network: Option<NetCommunicationReceiver>,
+    to_network: Option<EphemeraToNetworkSender>,
+}
+
+pub struct EphemeraStarterWithApplication<A: Application> {
+    init: EphemeraStarterInit,
+    application: A,
+}
+
+impl<A: Application> EphemeraStarterWithApplication<A> {
+    /// Initialize Ephemera with the given application.
+    /// It also tries to open the database connection.
+    ///
+    /// # Arguments
+    /// * `application` - [Application] to be used
+    ///
+    /// # Returns
+    /// [`EphemeraStarterWithApplication`]
+    ///
+    /// # Errors
+    /// * If the node configuration is invalid or the database connection cannot be opened
+    pub fn with_members_provider<
+        P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+    >(
+        mut self,
+        provider: P,
+    ) -> anyhow::Result<EphemeraStarterWithProvider<A>> {
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "sqlite_storage")] {
+                let mut storage = self.connect_sqlite()?;
+                debug!("Connected to sqlite database")
+            } else if #[cfg(feature = "rocksdb_storage")] {
+                let mut storage = self.connect_rocksdb()?;
+                debug!("Connected to rocksdb database")
+            } else {
+                compile_error!("Must enable either sqlite or rocksdb feature");
+            }
+        }
+
+        let block_manager = self.init_block_manager(&mut storage)?;
+
+        let (mut shutdown_manager, shutdown_handle) = ShutdownManager::init();
+
+        let mut service_data = ServiceInfo::default();
+        let services = self.init_services(&mut service_data, &mut shutdown_manager, provider)?;
+
+        Ok(EphemeraStarterWithProvider {
+            with_application: self,
+            block_manager: Some(block_manager),
+            service_data,
+            services,
+            storage: Some(Box::new(storage)),
+            shutdown_manager: Some(shutdown_manager),
+            shutdown_handle: Some(shutdown_handle),
+        })
+    }
+
+    //allocate database connection
+    #[cfg(feature = "rocksdb_storage")]
+    fn connect_rocksdb(&self) -> anyhow::Result<RocksDbStorage> {
+        info!("Opening database...");
+        RocksDbStorage::open(self.init.config.storage.clone())
+            .map_err(|e| anyhow::anyhow!("Failed to open database: {}", e))
+    }
+
+    #[cfg(feature = "sqlite_storage")]
+    fn connect_sqlite(&mut self) -> anyhow::Result<SqliteStorage> {
+        info!("Opening database...");
+        SqliteStorage::open(self.init.config.storage.clone())
+            .map_err(|e| anyhow::anyhow!("Failed to open database: {}", e))
+    }
+
+    fn init_block_manager<D: EphemeraDatabase + ?Sized>(
+        &mut self,
+        db: &mut D,
+    ) -> anyhow::Result<BlockManager> {
+        let block_manager_configuration = self.init.config.block_manager.clone();
+        let keypair = self.init.node_info.keypair.clone();
+        let builder = BlockManagerBuilder::new(block_manager_configuration, keypair);
+        builder.build(db)
+    }
+
+    fn init_services<
+        P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+    >(
+        &mut self,
+        service_data: &mut ServiceInfo,
+        shutdown_manager: &mut ShutdownManager,
+        provider: P,
+    ) -> anyhow::Result<Vec<BoxFuture<'static, anyhow::Result<()>>>> {
+        let services = vec![
+            self.init_libp2p(service_data, shutdown_manager.subscribe(), provider)?,
+            self.init_http(shutdown_manager.subscribe())?,
+            self.init_websocket(service_data, shutdown_manager.subscribe()),
+        ];
+        Ok(services)
+    }
+
+    fn init_websocket(
+        &mut self,
+        service_data: &mut ServiceInfo,
+        mut shutdown: Shutdown,
+    ) -> BoxFuture<'static, anyhow::Result<()>> {
+        let (mut websocket, ws_message_broadcast) =
+            WsManager::new(self.init.node_info.ws_address_ip_port());
+
+        service_data.ws_message_broadcast = Some(ws_message_broadcast);
+
+        async move {
+            websocket.listen().await?;
+
+            tokio::select! {
+                _ = shutdown.shutdown_signal_rcv.recv() => {
+                    info!("Shutting down websocket manager");
+                }
+                ws_stopped = websocket.run() => {
+                    match ws_stopped {
+                        Ok(_) => info!("Websocket stopped unexpectedly"),
+                        Err(e) => error!("Websocket stopped with error: {}", e),
+                    }
+                }
+            }
+            info!("Websocket task finished");
+            Ok(())
+        }
+        .boxed()
+    }
+
+    fn init_http(
+        &mut self,
+        mut shutdown: Shutdown,
+    ) -> anyhow::Result<BoxFuture<'static, anyhow::Result<()>>> {
+        let http = http::init(&self.init.node_info, self.init.api.clone())?;
+
+        let fut = async move {
+            let server_handle = http.handle();
+            tokio::select! {
+                _ = shutdown.shutdown_signal_rcv.recv() => {
+                    info!("Shutting down http server");
+                    server_handle.stop(true).await;
+                }
+                http_stopped = http => {
+                    match http_stopped {
+                        Ok(_) => info!("Http server stopped unexpectedly"),
+                        Err(e) => error!("Http server stopped with error: {}", e),
+                    }
+                }
+            }
+            info!("Http task finished");
+            Ok(())
+        }
+        .boxed();
+        Ok(fut)
+    }
+
+    fn init_libp2p<
+        P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+    >(
+        &mut self,
+        service_data: &mut ServiceInfo,
+        mut shutdown: Shutdown,
+        provider: P,
+    ) -> anyhow::Result<BoxFuture<'static, anyhow::Result<()>>> {
+        info!("Starting network...",);
+
+        let (mut network, from_network, to_network) =
+            SwarmNetwork::new(self.init.node_info.clone(), provider)?;
+
+        service_data.from_network = Some(from_network);
+        service_data.to_network = Some(to_network);
+
+        let libp2p = async move {
+            network.listen()?;
+
+            tokio::select! {
+                _ = shutdown.shutdown_signal_rcv.recv() => {
+                    info!("Shutting down network");
+                }
+                nw_stopped = network.start() => {
+                    match nw_stopped {
+                        Ok(_) => info!("Network stopped unexpectedly"),
+                        Err(e) => error!("Network stopped with error: {e}",),
+                    }
+                }
+            }
+            info!("Network task finished");
+            Ok(())
+        }
+        .boxed();
+        Ok(libp2p)
+    }
+}
+
+pub struct EphemeraStarterWithProvider<A>
+where
+    A: Application + 'static,
+{
+    with_application: EphemeraStarterWithApplication<A>,
+    block_manager: Option<BlockManager>,
+    service_data: ServiceInfo,
+    storage: Option<Box<dyn EphemeraDatabase>>,
+    services: Vec<BoxFuture<'static, anyhow::Result<()>>>,
+    shutdown_manager: Option<ShutdownManager>,
+    shutdown_handle: Option<Handle>,
+}
+
+impl<A> EphemeraStarterWithProvider<A>
+where
+    A: Application + 'static,
+{
+    pub fn build(self) -> Ephemera<A> {
+        self.ephemera()
+    }
+
+    fn ephemera(mut self) -> Ephemera<A> {
+        let ephemera_handle = EphemeraHandle {
+            api: self.with_application.init.api,
+            shutdown: self.shutdown_handle.take().unwrap(),
+        };
+
+        let node_info = self.with_application.init.node_info;
+        let application = self.with_application.application;
+        let block_manager = self.block_manager.expect("Block manager not initialized");
+        let broadcaster = self.with_application.init.broadcaster;
+        let from_network = self
+            .service_data
+            .from_network
+            .expect("From network not initialized");
+        let to_network = self
+            .service_data
+            .to_network
+            .expect("To network not initialized");
+        let storage = self.storage.expect("Storage not initialized");
+        let ws_message_broadcast = self
+            .service_data
+            .ws_message_broadcast
+            .expect("WS message broadcast not initialized");
+        let api_listener = self.with_application.init.api_listener;
+        let shutdown_manager = self
+            .shutdown_manager
+            .expect("Shutdown manager not initialized");
+        let services = self.services;
+
+        Ephemera {
+            node_info,
+            block_manager,
+            broadcaster,
+            from_network,
+            to_network,
+            broadcast_group: BroadcastGroup::new(),
+            storage: Arc::new(Mutex::new(storage)),
+            ws_message_broadcast,
+            api_listener,
+            api_cmd_processor: ApiCmdProcessor::new(),
+            application: application.into(),
+            ephemera_handle,
+            shutdown_manager,
+            services,
+        }
+    }
+}

ephemera/src/core/ephemera.rs

@@ -0,0 +1,421 @@
+use std::sync::Arc;
+
+use anyhow::anyhow;
+use futures_util::future::BoxFuture;
+use futures_util::StreamExt;
+use log::{debug, error, info, trace};
+use nym_task::TaskClient;
+use thiserror::Error;
+use tokio::sync::Mutex;
+
+use crate::broadcast::bracha::quorum::Quorum;
+use crate::storage::DatabaseError;
+use crate::{
+    api::{application::Application, application::CheckBlockResult, ApiListener},
+    block::{manager::BlockManager, types::block::Block},
+    broadcast::{
+        bracha::broadcast::BroadcastResponse, bracha::broadcast::Broadcaster,
+        group::BroadcastGroup, RbMsg,
+    },
+    core::{
+        api_cmd::ApiCmdProcessor,
+        builder::{EphemeraHandle, NodeInfo},
+        shutdown::ShutdownManager,
+    },
+    network::{
+        libp2p::network_sender::GroupChangeEvent,
+        libp2p::{
+            ephemera_sender::{EphemeraEvent, EphemeraToNetworkSender},
+            network_sender::{NetCommunicationReceiver, NetworkEvent},
+        },
+    },
+    storage::EphemeraDatabase,
+    utilities::crypto::Certificate,
+    websocket::ws_manager::WsMessageBroadcaster,
+};
+
+#[derive(Error, Debug)]
+enum EphemeraCoreError {
+    #[error("DatabaseFailure: {0}")]
+    DatabaseFailure(DatabaseError),
+    //Just a placeholder now
+    #[error("EphemeraCore: {0}")]
+    EphemeraCore(#[from] anyhow::Error),
+}
+
+type Result<T> = std::result::Result<T, EphemeraCoreError>;
+
+pub struct Ephemera<A: Application> {
+    /// Node info
+    pub(crate) node_info: NodeInfo,
+
+    /// Block manager responsibility includes:
+    /// - Block production and signing
+    /// - Block verification for externally received blocks
+    /// - Message verification sent by clients and gossiped other nodes
+    pub(crate) block_manager: BlockManager,
+
+    /// Broadcaster is making sure that blocks are deterministically agreed by all nodes.
+    pub(crate) broadcaster: Broadcaster,
+
+    /// A component which receives messages from network.
+    pub(crate) from_network: NetCommunicationReceiver,
+
+    /// A component which sends messages to network.
+    pub(crate) to_network: EphemeraToNetworkSender,
+
+    /// A component which keeps track of broadcast group over time.
+    pub(crate) broadcast_group: BroadcastGroup,
+
+    /// A component which has mutable access to database.
+    pub(crate) storage: Arc<Mutex<Box<dyn EphemeraDatabase>>>,
+
+    /// A component which broadcasts messages to websocket clients.
+    pub(crate) ws_message_broadcast: WsMessageBroadcaster,
+
+    /// A component which listens API requests.
+    pub(crate) api_listener: ApiListener,
+
+    /// A component which processes API requests.
+    pub(crate) api_cmd_processor: ApiCmdProcessor,
+
+    /// An implementation of Application trait. Provides callbacks to broadcast.
+    pub(crate) application: Arc<A>,
+
+    ///Interface to external Rust code
+    pub(crate) ephemera_handle: EphemeraHandle,
+
+    /// A component which handles shutdown.
+    pub(crate) shutdown_manager: ShutdownManager,
+
+    /// A list of services which are running in background.
+    pub(crate) services: Vec<BoxFuture<'static, anyhow::Result<()>>>,
+}
+
+impl<A: Application> Ephemera<A> {
+    ///Provides external api for Rust code to interact with ephemera node.
+    #[must_use]
+    pub fn handle(&self) -> EphemeraHandle {
+        self.ephemera_handle.clone()
+    }
+
+    /// Main loop of ephemera node.
+    /// 1. Block manager generates new blocks or receives blocks from network.
+    /// 2. Run reliable broadcast.
+    /// 3. Process reliable broadcast result.
+    /// 4. Process http api request
+    /// 5. Process rust api request
+    /// 6. Publish(gossip) messages to network
+    /// 7. Publish blocks to network
+    /// 8. Broadcast messages to websocket clients
+    pub async fn run(mut self, mut shutdown: TaskClient) {
+        info!("Starting ephemera services");
+        for service in self.services.drain(..) {
+            let handle = tokio::spawn(service);
+            self.shutdown_manager.add_handle(handle);
+        }
+
+        info!("Starting ephemera main loop");
+
+        while !shutdown.is_shutdown() {
+            tokio::select! {
+                biased;
+                _ = shutdown.recv() => {
+                    trace!("UpdateHandler: Received shutdown");
+                    self.shutdown_manager.stop().await;
+                    break;
+                }
+
+                // GENERATING NEW BLOCKS
+                Some((new_block, certificate)) = self.block_manager.next() => {
+                    if let Err(err) = self.process_new_local_block(new_block, certificate).await{
+                        error!("Error processing new block: {:?}", err);
+                    }
+                }
+
+                // PROCESSING NETWORK EVENTS
+                Some(net_event) = self.from_network.net_event_rcv.recv() => {
+                    if let Err(err) = self.process_network_event(net_event).await{
+                        error!("Error processing network event: {:?}", err);
+                        if let EphemeraCoreError::DatabaseFailure(_) = err {
+                            info!("Database failure. Shutting down ephemera");
+                            self.shutdown_manager.stop().await;
+                            break;
+                        }
+                    }
+                }
+
+                //PROCESSING EXTERNAL API REQUESTS
+                api = self.api_listener.messages_rcv.recv() => {
+                    match api {
+                        Some(api_msg) => {
+                            if let Err(err) = ApiCmdProcessor::process_api_requests(&mut self, api_msg).await{
+                                error!("Error processing api request: {:?}", err);
+                            }
+                        }
+                        None => {
+                            error!("Error: Api listener channel closed");
+                            //TODO: handle shutdown
+                        }
+                    }
+                }
+            }
+        }
+        info!("Ephemera main loop finished");
+    }
+
+    async fn process_network_event(&mut self, net_event: NetworkEvent) -> Result<()> {
+        trace!("New network event: {:?}", net_event);
+
+        match net_event {
+            NetworkEvent::EphemeraMessage(em) => {
+                let api_msg = (*em.clone()).into();
+                trace!("New ephemera message from network: {:?}", api_msg);
+
+                //Only Application checks if messages are valid(possibly message origin).
+                //For messages we don't check if sender belongs to group.
+
+                // Ask application to decide if we should accept this message.
+                match self.application.check_tx(api_msg) {
+                    Ok(true) => {
+                        trace!("Application accepted message: {:?}", em);
+
+                        // Send to BlockManager to store in mempool.
+                        if let Err(err) = self.block_manager.on_new_message(*em) {
+                            error!("Error sending signed message to block manager: {:?}", err);
+                        }
+                    }
+                    Ok(false) => {
+                        trace!("Application rejected message: {:?}", em);
+                    }
+                    Err(err) => {
+                        error!("Application check_tx failed: {:?}", err);
+                    }
+                }
+            }
+            NetworkEvent::BroadcastMessage(rb_msg) => {
+                self.process_block_from_network(*rb_msg).await?;
+            }
+            NetworkEvent::GroupUpdate(event) => {
+                self.process_group_update(event);
+            }
+            NetworkEvent::QueryDhtResponse { key, value } => {
+                match self.api_cmd_processor.dht_query_cache.pop(&key) {
+                    Some(replies) => {
+                        for reply in replies {
+                            let response = Ok(Some((key.clone(), value.clone())));
+                            if let Err(err) = reply.send(response) {
+                                error!("Error sending dht query response: {:?}", err);
+                            }
+                        }
+                    }
+                    None => {
+                        trace!(
+                            "No dht query cache found for key: {:?}",
+                            String::from_utf8(key)
+                        );
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn process_group_update(&mut self, event: GroupChangeEvent) {
+        match event {
+            GroupChangeEvent::PeersUpdated(peers) => {
+                info!("New group: {:?}", peers);
+                info!("{}", Quorum::cluster_size_info(peers.len()));
+                self.broadcaster.group_updated(peers.len());
+                self.broadcast_group.add_snapshot(peers);
+                self.block_manager.start();
+            }
+            GroupChangeEvent::LocalPeerRemoved(peers) | GroupChangeEvent::NotEnoughPeers(peers) => {
+                info!("New group: {:?}", peers);
+                info!("Group update: Local peer removed or not enough peers");
+                self.broadcaster.group_updated(0);
+                self.broadcast_group.add_snapshot(peers);
+                self.block_manager.stop();
+            }
+        }
+    }
+
+    async fn process_new_local_block(
+        &mut self,
+        new_block: Block,
+        certificate: Certificate,
+    ) -> Result<()> {
+        debug!("New block from block manager: {:?}", new_block.get_hash());
+
+        let hash = new_block.header.hash;
+        let block_creator = &self.node_info.peer_id;
+        let sender = &self.node_info.peer_id;
+
+        // Check if block matches group membership.
+        if !self
+            .broadcast_group
+            .check_membership(hash, block_creator, sender)
+        {
+            debug!("Membership check rejected block: {:?}", new_block);
+            return Ok(());
+        }
+
+        //Ephemera ABCI
+        match self.application.check_block(&new_block.clone().into()) {
+            Ok(response) => match response {
+                CheckBlockResult::Accept => {
+                    debug!("Application accepted new block: {hash:?}",);
+                }
+                CheckBlockResult::Reject => {
+                    debug!("Application rejected block: {hash:?}",);
+                    return Ok(());
+                }
+                CheckBlockResult::RejectAndRemoveMessages(messages_to_remove) => {
+                    debug!("Application rejected block: {:?}", messages_to_remove);
+                    self.block_manager
+                        .on_application_rejected_block(messages_to_remove)
+                        .map_err(|err| {
+                            anyhow!("Error rejecting block from block manager: {:?}", err)
+                        })?;
+                }
+            },
+            Err(err) => {
+                return Err(anyhow!("Application check_block failed: {:?}", err).into());
+            }
+        }
+
+        //Block manager generated new block that nobody hasn't seen yet.
+        //We start reliable broadcaster protocol to broadcaster it to other nodes.
+        match self.broadcaster.new_broadcast(new_block) {
+            Ok(resp) => {
+                if let BroadcastResponse::Broadcast(msg) = resp {
+                    trace!("Broadcasting new block: {:?}", msg);
+
+                    let rb_msg = RbMsg::new(msg, certificate);
+                    self.to_network
+                        .send_ephemera_event(EphemeraEvent::ProtocolMessage(rb_msg.into()))
+                        .await?;
+                }
+            }
+            Err(err) => {
+                error!("Error starting new broadcast: {:?}", err);
+            }
+        }
+        Ok(())
+    }
+
+    //TODO: should we accept more blocks(certificates) from peers after its committed?
+    async fn process_block_from_network(&mut self, msg: RbMsg) -> Result<()> {
+        let msg_id = msg.id.clone();
+        let block = msg.block();
+        let block_creator = &block.header.creator;
+        let sender = &msg.original_sender;
+        let hash = block.header.hash;
+        let certificate = msg.certificate.clone();
+
+        trace!("New broadcast message from network: {:?}", msg);
+
+        if !self
+            .broadcast_group
+            .check_membership(hash, block_creator, sender)
+        {
+            return Err(anyhow!("Block doesn't match broacast group").into());
+        }
+
+        if let Err(err) = self.block_manager.on_block(sender, block, &certificate) {
+            return Err(anyhow!("Error sending block to block manager: {:?}", err).into());
+        }
+        let raw_mgs = msg.into();
+        match self.broadcaster.handle(&raw_mgs) {
+            Ok(resp) => {
+                match resp {
+                    BroadcastResponse::Broadcast(msg) => {
+                        trace!("Broadcasting block to network: {:?}", msg);
+
+                        match self.block_manager.sign_block(&msg.block()) {
+                            Ok(certificate) => {
+                                let rb_msg = RbMsg::new(msg, certificate);
+                                self.to_network
+                                    .send_ephemera_event(EphemeraEvent::ProtocolMessage(
+                                        rb_msg.into(),
+                                    ))
+                                    .await?;
+                            }
+                            Err(err) => {
+                                return Err(anyhow!("Error signing block: {:?}", err).into());
+                            }
+                        }
+                    }
+                    BroadcastResponse::Deliver(hash) => {
+                        trace!("Block broadcast complete: {hash:?}",);
+                        let block = self.block_manager.get_block_by_hash(&hash);
+                        match block {
+                            Some(block) => {
+                                if block.header.creator == self.node_info.peer_id {
+                                    info!("Block committed, ready to deliver...: {hash:?}",);
+
+                                    //BlockManager
+                                    self.block_manager.on_block_committed(&block).map_err(|e| {
+                                        anyhow!(
+                                            "Error: BlockManager failed to process block: {e:?}",
+                                        )
+                                    })?;
+
+                                    //Save to database
+                                    let certificates = self
+                                        .block_manager
+                                        .get_block_certificates(&block.header.hash)
+                                        .ok_or(anyhow!(
+                                            "Error: Block certificates not found for block: {hash:?}"
+                                        ))?;
+                                    let members = self
+                                        .broadcast_group
+                                        .get_group_by_block_hash(block.get_hash())
+                                        .ok_or(anyhow!(
+                                            "Error: Group not found for block: {hash:?}"
+                                        ))?;
+
+                                    if let Err(e) = self.storage.lock().await.store_block(
+                                        &block,
+                                        certificates.clone(),
+                                        members.clone(),
+                                    ) {
+                                        return Err(EphemeraCoreError::DatabaseFailure(e));
+                                    }
+
+                                    // It is open question how much Application `deliver_block` failure should affect
+                                    // continuing with next block.
+                                    //Application(ABCI)
+                                    self.application
+                                        .deliver_block(Into::into(block.clone()))
+                                        .map_err(|e| {
+                                            anyhow!(
+                                                "Error: Deliver block to Application failed: {e:?}",
+                                            )
+                                        })?;
+
+                                    //WS
+                                    self.ws_message_broadcast.send_block(&block)?;
+                                    info!("Block broadcast complete: {hash:?}",);
+                                }
+                            }
+                            None => {
+                                return Err(
+                                    anyhow!("Error: Block not found in block manager").into()
+                                );
+                            }
+                        }
+                    }
+                    BroadcastResponse::Drop(hash) => {
+                        trace!("Ignoring broadcast message {:?}[block {:?}]", msg_id, hash);
+                        return Ok(());
+                    }
+                }
+            }
+            Err(err) => {
+                error!("Error handling broadcast message: {:?}", err);
+            }
+        }
+        Ok(())
+    }
+}

ephemera/src/core/mod.rs

@@ -0,0 +1,4 @@
+pub(crate) mod api_cmd;
+pub(crate) mod builder;
+pub(crate) mod ephemera;
+pub(crate) mod shutdown;

ephemera/src/core/shutdown.rs

@@ -0,0 +1,77 @@
+use log::info;
+
+use tokio::sync::broadcast;
+use tokio::task::JoinHandle;
+
+pub(crate) struct ShutdownManager {
+    pub(crate) shutdown_tx: broadcast::Sender<()>,
+    pub(crate) _shutdown_rcv: broadcast::Receiver<()>,
+    handles: Vec<JoinHandle<anyhow::Result<()>>>,
+}
+
+pub(crate) struct Shutdown {
+    pub(crate) shutdown_signal_rcv: broadcast::Receiver<()>,
+}
+
+#[derive(Clone)]
+pub struct Handle {
+    pub(crate) shutdown_started: bool,
+}
+
+impl Handle {
+    /// Shutdown the node.
+    /// This will send a shutdown signal to all tasks and wait for them to finish.
+    ///
+    /// # Errors
+    /// This will return an error if shutdown signal can't be sent.
+    ///
+    /// # Panics
+    /// This will panic if shutdown signal can't be sent.
+    pub fn shutdown(&mut self) {
+        self.shutdown_started = true;
+    }
+}
+
+impl ShutdownManager {
+    pub(crate) fn init() -> (ShutdownManager, Handle) {
+        let (shutdown_tx, shutdown_rcv) = broadcast::channel(1);
+        let shutdown_handle = Handle {
+            shutdown_started: false,
+        };
+        let manager = Self {
+            shutdown_tx,
+            _shutdown_rcv: shutdown_rcv,
+            handles: vec![],
+        };
+        (manager, shutdown_handle)
+    }
+
+    pub async fn stop(self) {
+        info!("Starting Ephemera shutdown");
+        self.shutdown_tx.send(()).unwrap();
+        info!("Waiting for tasks to finish");
+        for (i, handle) in self
+            .handles
+            .into_iter()
+            .enumerate()
+            .map(|(i, h)| (i + 1, h))
+        {
+            match handle.await.unwrap() {
+                Ok(_) => info!("Task {i} finished successfully"),
+                Err(e) => info!("Task {i} finished with error: {e}",),
+            }
+        }
+        info!("All tasks finished");
+    }
+
+    pub(crate) fn subscribe(&self) -> Shutdown {
+        let shutdown = self.shutdown_tx.subscribe();
+        Shutdown {
+            shutdown_signal_rcv: shutdown,
+        }
+    }
+
+    pub(crate) fn add_handle(&mut self, handle: JoinHandle<anyhow::Result<()>>) {
+        self.handles.push(handle);
+    }
+}

ephemera/src/lib.rs

@@ -0,0 +1,118 @@
+//! # Ephemera Node
+
+//! An Ephemera node does reliable broadcast of inbound messages to all other Ephemera nodes in the cluster.
+//!
+//! Each node has a unique ID, and each message is signed by the node that first received it.
+//! Messages are then re-broadcast and re-signed by all other nodes in the cluster.
+//!
+//! At the end of the process, each message is signed by every node in the cluster, and each node has also
+//! signed all messages that were broadcast by other nodes. This means that nodes are unable to repudiate messages
+//! once they are seen and signed, so there is a strong guarantee of message integrity within the cluster.
+//!
+//! # Why would I want this?
+//!
+//! Let's say you have blockchain system that needs to ship large amounts of information around, but the information
+//! is relatively short-lived. You could use a blockchain to store the information, but that would be expensive,
+//! slow, and wasteful. Instead, you could use Ephemera to broadcast the information to all nodes in the cluster,
+//! and then store only a cryptographic commitment in the blockchain's data store.
+//!
+//! Ephemera nodes then keep messages around for inspection in a data availability layer (accessible over HTTP)
+//! so that interested parties can verify correctness. Ephemeral information can then be automatically discarded
+//! once it's no longer useful.
+//!
+//! This gives very similar guarantees to a blockchain, but without incurring the permanent storage costs.
+//!
+//! Note that it *requires* a blockchain to be present.
+
+//'Denying' everything and allowing exceptions seems better than other way around.
+#![deny(clippy::pedantic)]
+
+// PUBLIC MODULES
+
+pub use crate::core::builder::{
+    EphemeraStarterInit, EphemeraStarterWithApplication, EphemeraStarterWithProvider,
+};
+pub use crate::core::ephemera::Ephemera;
+pub use crate::core::shutdown::Handle as ShutdownHandle;
+
+/// Ephemera API. Public interface and types.
+pub mod ephemera_api {
+    pub use crate::api::{
+        application::{
+            Application, CheckBlockResult, Dummy, Error as ApplicationError, RemoveMessages,
+            Result as ApplicationResult,
+        },
+        http::client::{Client, Error as HttpClientError, Result as HttpClientResult},
+        types::{
+            ApiBlock, ApiBlockBroadcastInfo, ApiBroadcastInfo, ApiCertificate, ApiDhtQueryRequest,
+            ApiDhtQueryResponse, ApiDhtStoreRequest, ApiEphemeraConfig, ApiEphemeraMessage,
+            ApiError, ApiHealth, ApiVerifyMessageInBlock, RawApiEphemeraMessage,
+        },
+        CommandExecutor,
+    };
+}
+
+/// Peer identification
+#[allow(clippy::module_name_repetitions)]
+pub mod peer {
+    pub use super::network::{PeerId, PeerIdError, ToPeerId};
+}
+
+/// Ephemera membership. How to find other nodes in the cluster.
+pub mod membership {
+    pub use super::network::members::{
+        ConfigMembersProvider, DummyMembersProvider, PeerInfo, PeerSetting, ProviderError, Result,
+    };
+}
+
+/// Ephemera keypair and public key
+pub mod crypto {
+    pub use super::utilities::crypto::{
+        EphemeraKeypair, EphemeraPublicKey, KeyPairError, Keypair, PublicKey,
+    };
+}
+
+/// Ephemera codec to encode and decode messages
+pub mod codec {
+    pub use super::utilities::codec::{Decode, Encode};
+}
+
+/// Ephemera node configuration
+pub mod configuration {
+    pub use super::config::Configuration;
+}
+
+/// Ephemera CLI. Helpers for creating configuration, running node, etc.
+pub mod cli;
+
+/// Utilities to set up logging.
+pub mod logging;
+
+// PRIVATE MODULES
+
+/// External interface for Ephemera
+mod api;
+
+/// Block creation code
+mod block;
+
+/// Ephemera reliable broadcast
+mod broadcast;
+
+/// Ephemera configuration
+mod config;
+
+/// Ephemera core. Ephemera builder and instance.
+mod core;
+
+/// Ephemera networking with peers
+mod network;
+
+/// Ephemera storage. Block storage and certificate storage.
+mod storage;
+
+/// Ephemera utilities. Crypto, codec, etc.
+mod utilities;
+
+/// Ephemera websocket. Websocket server where external clients can subscribe.
+mod websocket;

ephemera/src/logging/mod.rs

@@ -0,0 +1,19 @@
+pub fn init() {
+    if let Ok(directives) = ::std::env::var("RUST_LOG") {
+        println!("Logging enabled with directives: {directives}",);
+        pretty_env_logger::formatted_timed_builder()
+            .parse_filters(&directives)
+            .format_timestamp_millis()
+            .init();
+    } else {
+        println!("Logging disabled");
+    }
+}
+
+pub fn init_with_directives(directives: &str) {
+    println!("Logging enabled with directives: {directives}",);
+    pretty_env_logger::formatted_timed_builder()
+        .parse_filters(directives)
+        .format_timestamp_millis()
+        .init();
+}

ephemera/src/network/libp2p/behaviours/membership/behaviour.rs

@@ -0,0 +1,556 @@
+//! # Membership behaviour.
+//!
+//! Ephemera `reliable broadcast` needs to know the list of peers who participate in the protocol.
+//! Also it's not enough to have just the list but to make sure that they are actually online.
+//! When the list of available peers changes, `reliable broadcast` needs to be notified so that it can adjust accordingly.
+//!
+//! This behaviour is responsible for keeping membership up to date.
+//!
+//! `MembersProviderFuture`: `Future<Output = membership::Result<Vec<PeerInfo>>> + Send + 'static`.
+//!
+//! User provides a [`MembersProviderFuture`] implementation to the [Behaviour] which is responsible for fetching the list of peers.
+//!
+//! [Behaviour] accepts only peers that are actually online.
+//!
+//! When peers become available or unavailable, [Behaviour] adjusts the list of connected peers accordingly and notifies `reliable broadcast`
+//! about the membership change.
+//!
+//! It is configurable what `threshold` of peers(from the total list provided by [`MembersProviderFuture`]) should be available at any given time.
+//! Or if just to use all peers who are online. See [`MembershipKind`] for more details.
+//!
+//! Ideally [`MembersProviderFuture`] can depend on a resource that gives reliable results. Some kind of registry which itself keeps track of actually online nodes.
+//! As Ephemera uses only peers provided by [`MembersProviderFuture`], it depends on its accuracy.
+//! At the same time it tries to be flexible and robust to handle less reliable [`MembersProviderFuture`] implementations.
+
+// When peer gets disconnected, we try to dial it and if that fails, we update group.
+// (it may connect us meanwhile).
+// Although we can retry to connect to disconnected peers, it's simpler if we just assume that when
+// they come online again, they will connect us.
+
+//So the main goal is to remove offline peers from the group so that rb can make progress.
+
+//a)when peer disconnects, we try to dial it and if that fails, we update group.
+//b)when peer connects, we will update the group.
+
+use std::future::Future;
+use std::time::Duration;
+use std::{
+    collections::HashMap,
+    collections::HashSet,
+    fmt::Debug,
+    task::{Context, Poll},
+};
+
+use futures_util::FutureExt;
+use libp2p::core::Endpoint;
+use libp2p::swarm::{ConnectionDenied, NotifyHandler, THandler};
+use libp2p::{
+    swarm::ToSwarm,
+    swarm::{
+        behaviour::ConnectionEstablished,
+        dial_opts::{DialOpts, PeerCondition},
+        ConnectionClosed, ConnectionId, DialFailure, FromSwarm, NetworkBehaviour, PollParameters,
+        THandlerInEvent, THandlerOutEvent,
+    },
+    Multiaddr,
+};
+use libp2p_identity::PeerId;
+use log::{debug, error, trace, warn};
+use tokio::time;
+use tokio::time::{Instant, Interval};
+
+use crate::network::libp2p::behaviours::membership::handler::ToHandler;
+use crate::network::libp2p::behaviours::membership::{Membership, MEMBERSHIP_SYNC_INTERVAL_SEC};
+use crate::network::Peer;
+use crate::{
+    membership,
+    network::{
+        libp2p::behaviours::{
+            membership::connections::ConnectedPeers,
+            membership::protocol::ProtocolMessage,
+            membership::{handler::Handler, MAX_DIAL_ATTEMPT_ROUNDS},
+            membership::{MembershipKind, Memberships},
+        },
+        members::PeerInfo,
+    },
+};
+
+/// [`MembersProviderFuture`] state when we are trying to connect to new peers.
+///
+/// We try to connect few times before giving up. Generally speaking an another peer is either online or offline
+/// at any given time. But it has been helpful for testing when whole cluster comes up around the same time.
+#[derive(Debug, Default)]
+struct PendingPeersUpdate {
+    /// Peers that we are haven't tried to connect to yet.
+    waiting_to_dial: HashSet<PeerId>,
+    /// Number of dial attempts per round.
+    dial_attempts: usize,
+    /// How long we wait between dial attempts.
+    interval_between_dial_attempts: Option<Interval>,
+}
+
+#[derive(Debug, Default)]
+struct SyncPeers {
+    pending_peers: Vec<PeerId>,
+}
+
+impl SyncPeers {
+    fn new(pending_peers: Vec<PeerId>) -> Self {
+        Self { pending_peers }
+    }
+}
+
+/// Behaviour states.
+enum State {
+    /// Waiting for new peers from the members provider trait.
+    WaitingPeers,
+    /// Trying to connect to new peers.
+    WaitingDial(PendingPeersUpdate),
+    /// We have finished trying to connect to new peers and going to report it.
+    NotifyPeersUpdated,
+    /// Notify other members that we have updated members.
+    SyncPeers(SyncPeers),
+}
+
+/// Events that can be emitted by the `Behaviour`.
+pub(crate) enum Event {
+    /// We have received new peers from the members provider trait.
+    /// We are going to try to connect to them.
+    PeerUpdatePending,
+    /// We have finished trying to connect to new peers and going to report it.
+    PeersUpdated(HashSet<PeerId>),
+    /// MembersProviderFuture reported us new peers and this set doesn't contain our local peer.
+    LocalRemoved(HashSet<PeerId>),
+    /// MembersProviderFuture reported us new peers and we failed to connect to enough of them.
+    NotEnoughPeers(HashSet<PeerId>),
+}
+
+pub(crate) struct Behaviour<P>
+where
+    P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + 'static,
+{
+    /// All peers that are part of the current group.
+    memberships: Memberships,
+    /// Local peer id.
+    local_peer_id: PeerId,
+    /// Future that provides new peers.
+    members_provider: P,
+    /// Interval between requesting new peers from the members provider.
+    members_provider_interval: Option<Interval>,
+    /// Delay between dial attempts.
+    members_provider_delay: Duration,
+    /// Current behaviour state.
+    state: State,
+    /// Current state of all incoming and outgoing connections.
+    all_connections: ConnectedPeers,
+    /// Membership kind.
+    membership_kind: MembershipKind,
+    /// Last time we broadcast SYNC
+    last_sync_time: Instant,
+    /// Minimum time between members provider updates.
+    minimum_time_between_sync: Duration,
+}
+
+impl<P> Behaviour<P>
+where
+    P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    pub(crate) fn new(
+        members_provider: P,
+        members_provider_delay: Duration,
+        local_peer_id: PeerId,
+        membership_kind: MembershipKind,
+    ) -> Self {
+        let initial_delay = Instant::now() + Duration::from_secs(5);
+        let delay = tokio::time::interval_at(initial_delay, members_provider_delay);
+        Behaviour {
+            memberships: Memberships::new(),
+            local_peer_id,
+            members_provider,
+            members_provider_interval: Some(delay),
+            members_provider_delay,
+            state: State::WaitingPeers,
+            all_connections: ConnectedPeers::default(),
+            membership_kind,
+            last_sync_time: Instant::now(),
+            minimum_time_between_sync: Duration::from_secs(MEMBERSHIP_SYNC_INTERVAL_SEC),
+        }
+    }
+
+    /// Returns the list of peers that are part of current group.
+    pub(crate) fn active_peer_ids(&mut self) -> &HashSet<PeerId> {
+        self.memberships.current().connected_peers()
+    }
+
+    pub(crate) fn active_peer_ids_with_local(&mut self) -> HashSet<PeerId> {
+        self.memberships.current().connected_peer_ids_with_local()
+    }
+
+    fn waiting_peers(&mut self, cx: &mut Context) -> Poll<ToSwarm<Event, ToHandler>> {
+        if let Some(mut tick) = self.members_provider_interval.take() {
+            if !tick.poll_tick(cx).is_ready() {
+                self.members_provider_interval = Some(tick);
+                return Poll::Pending;
+            }
+        }
+        let peers = match self.members_provider.poll_unpin(cx) {
+            Poll::Ready(peers) => {
+                let wait_time = Instant::now() + self.members_provider_delay;
+                self.members_provider_interval =
+                    time::interval_at(wait_time, self.members_provider_delay).into();
+                self.last_sync_time = Instant::now();
+                peers
+            }
+            Poll::Pending => {
+                return Poll::Pending;
+            }
+        };
+
+        match peers {
+            Ok(peers) => {
+                if peers.is_empty() {
+                    //Not sure what to do here. Tempted to think that if this happens
+                    //we should ignore it and assume that this is probably a bug in the membership service.
+
+                    warn!("Received empty peers from provider. To try again before preconfigured interval, please restart the node.");
+                    return Poll::Ready(ToSwarm::GenerateEvent(Event::NotEnoughPeers(
+                        HashSet::default(),
+                    )));
+                }
+
+                let mut new_peers = HashMap::new();
+
+                for peer_info in peers {
+                    match <PeerInfo as TryInto<Peer>>::try_into(peer_info) {
+                        Ok(peer) => {
+                            debug!(
+                                "Received peer: {:?}, {:?}",
+                                peer.peer_id.inner(),
+                                peer.cosmos_address
+                            );
+                            new_peers.insert(*peer.peer_id.inner(), peer);
+                        }
+                        Err(err) => {
+                            error!("Error while converting peer info to peer: {}", err);
+                        }
+                    }
+                }
+
+                //If we are not part of the new membership, notify immediately
+                if !new_peers.contains_key(&self.local_peer_id) {
+                    debug!(
+                        "Local peer {:?} is not part of the new membership. Notifying immediately.",
+                        self.local_peer_id
+                    );
+                    let pending_membership = Membership::new(new_peers.clone());
+                    self.memberships.set_pending(pending_membership);
+                    self.state = State::NotifyPeersUpdated;
+                    return Poll::Pending;
+                }
+
+                let mut pending_membership =
+                    Membership::new_with_local(new_peers.clone(), self.local_peer_id);
+                let mut pending_update = PendingPeersUpdate::default();
+
+                for peer_id in new_peers.keys() {
+                    if self.all_connections.is_peer_connected(peer_id) {
+                        pending_membership.peer_connected(*peer_id);
+                    } else {
+                        pending_update.waiting_to_dial.insert(*peer_id);
+                    }
+                }
+
+                self.memberships.set_pending(pending_membership);
+
+                //It seems that all peers from updated membership set are already connected
+                if pending_update.waiting_to_dial.is_empty() {
+                    self.state = State::NotifyPeersUpdated;
+                    Poll::Pending
+                } else {
+                    self.state = State::WaitingDial(pending_update);
+
+                    //Just let the rest of the system to know that we are in the middle of updating membership
+                    Poll::Ready(ToSwarm::GenerateEvent(Event::PeerUpdatePending))
+                }
+            }
+            Err(err) => {
+                error!("Error while getting peers from provider: {:?}", err);
+                Poll::Ready(ToSwarm::GenerateEvent(Event::NotEnoughPeers(
+                    HashSet::default(),
+                )))
+            }
+        }
+    }
+
+    fn waiting_dial(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<ToSwarm<Event, THandlerInEvent<Self>>> {
+        if let State::WaitingDial(PendingPeersUpdate {
+            waiting_to_dial,
+            dial_attempts,
+            interval_between_dial_attempts,
+        }) = &mut self.state
+        {
+            //Refresh the list of connected peers
+            for peer_id in self.all_connections.all_connected_peers_ref() {
+                waiting_to_dial.remove(peer_id);
+            }
+
+            //FIXME
+            waiting_to_dial.remove(&self.local_peer_id);
+
+            let pending_membership = self
+                .memberships
+                .pending()
+                .expect("Pending membership should be set");
+
+            //With each 'poll' we can tell Swarm to dial one peer
+            let next_waiting = waiting_to_dial.iter().next().copied();
+
+            if let Some(peer_id) = next_waiting {
+                waiting_to_dial.remove(&peer_id);
+
+                let address = pending_membership
+                    .peer_address(&peer_id)
+                    .expect("Peer should exist");
+
+                trace!("Dialing peer: {:?} {:?}", peer_id, address);
+
+                let opts = DialOpts::peer_id(peer_id)
+                    .condition(PeerCondition::NotDialing)
+                    .addresses(vec![address.clone()])
+                    .build();
+
+                Poll::Ready(ToSwarm::Dial { opts })
+            } else {
+                let all_peers = pending_membership.all_peer_ids();
+                let connected_peers = pending_membership.connected_peers();
+
+                //Exclude local peer
+                let all_connected = connected_peers.len() == all_peers.len() - 1;
+
+                if all_connected || *dial_attempts >= MAX_DIAL_ATTEMPT_ROUNDS {
+                    interval_between_dial_attempts.take();
+                    self.state = State::NotifyPeersUpdated;
+                    return Poll::Pending;
+                }
+                //Try again few times before notifying the rest of the system about membership update.
+                //Dialing attempt
+                if let Some(interval) = interval_between_dial_attempts {
+                    if interval.poll_tick(cx) == Poll::Pending {
+                        return Poll::Pending;
+                    }
+                    *dial_attempts += 1;
+                    trace!("Next attempt({dial_attempts:?}) to dial failed peers");
+                } else {
+                    let start_at = Instant::now() + Duration::from_secs(5);
+                    *interval_between_dial_attempts =
+                        Some(time::interval_at(start_at, Duration::from_secs(10)));
+                }
+                if *dial_attempts > 0 {
+                    waiting_to_dial.extend(all_peers.difference(connected_peers).copied());
+                }
+
+                Poll::Pending
+            }
+        } else {
+            unreachable!()
+        }
+    }
+
+    fn notify_peers_updated(&mut self) -> Poll<ToSwarm<Event, ToHandler>> {
+        if let Some(membership) = self.memberships.remove_pending() {
+            self.memberships.update(membership);
+        }
+
+        let membership = self.memberships.current();
+        let membership_connected_peers = membership.connected_peer_ids();
+
+        let event = if membership.includes_local() {
+            if self.membership_kind.accept(membership) {
+                debug!("Membership accepted by kind: {:?}", self.membership_kind);
+                Event::PeersUpdated(membership_connected_peers)
+            } else {
+                debug!("Membership rejected by kind: {:?}", self.membership_kind);
+                Event::NotEnoughPeers(membership_connected_peers)
+            }
+        } else {
+            debug!("Membership does not include local peer");
+            Event::LocalRemoved(membership_connected_peers)
+        };
+
+        //TODO: this list should also include "old" peers(peers who aren't part of new membership).
+        let connected_peers = membership
+            .connected_peer_ids()
+            .into_iter()
+            .collect::<Vec<_>>();
+        self.state = State::SyncPeers(SyncPeers::new(connected_peers));
+        Poll::Ready(ToSwarm::GenerateEvent(event))
+    }
+
+    fn sync_peers(&mut self) -> Poll<ToSwarm<Event, ToHandler>> {
+        if let State::SyncPeers(SyncPeers { pending_peers }) = &mut self.state {
+            match pending_peers.pop() {
+                None => {
+                    self.state = State::WaitingPeers;
+                    Poll::Pending
+                }
+                Some(peer_id) => {
+                    debug!("Notifying {peer_id:?} about membership update",);
+                    Poll::Ready(ToSwarm::NotifyHandler {
+                        peer_id,
+                        handler: NotifyHandler::Any,
+                        event: ToHandler::Message(ProtocolMessage::Sync),
+                    })
+                }
+            }
+        } else {
+            unreachable!("State should be SyncPeers")
+        }
+    }
+}
+
+impl<P> NetworkBehaviour for Behaviour<P>
+where
+    P: Future<Output = membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    type ConnectionHandler = Handler;
+    type OutEvent = Event;
+
+    fn handle_pending_inbound_connection(
+        &mut self,
+        _connection_id: ConnectionId,
+        _local_addr: &Multiaddr,
+        _remote_addr: &Multiaddr,
+    ) -> Result<(), ConnectionDenied> {
+        //TODO: we can refuse connections from peers that are not part of the current membership.
+        Ok(())
+    }
+
+    fn handle_established_inbound_connection(
+        &mut self,
+        _connection_id: ConnectionId,
+        peer: PeerId,
+        _local_addr: &Multiaddr,
+        _remote_addr: &Multiaddr,
+    ) -> Result<THandler<Self>, ConnectionDenied> {
+        trace!("Established inbound connection with peer: {:?}", peer);
+        Ok(Handler::new())
+    }
+
+    fn handle_pending_outbound_connection(
+        &mut self,
+        _connection_id: ConnectionId,
+        maybe_peer: Option<PeerId>,
+        _addresses: &[Multiaddr],
+        _effective_role: Endpoint,
+    ) -> Result<Vec<Multiaddr>, ConnectionDenied> {
+        //FIXME: deprecated
+        #[allow(deprecated)]
+        match maybe_peer {
+            Some(peer_id) => Ok(self.addresses_of_peer(&peer_id)),
+            None => Ok(vec![]),
+        }
+    }
+
+    fn handle_established_outbound_connection(
+        &mut self,
+        _connection_id: ConnectionId,
+        peer: PeerId,
+        addr: &Multiaddr,
+        _role_override: Endpoint,
+    ) -> Result<THandler<Self>, ConnectionDenied> {
+        trace!(
+            "Established outbound connection with peer: {:?} {:?}",
+            peer,
+            addr
+        );
+        Ok(Handler::new())
+    }
+
+    ///Membership behaviour is responsible for providing addresses to another Swarm behaviours.
+    fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
+        self.memberships
+            .current()
+            .peer_address(peer_id)
+            .cloned()
+            .map_or(vec![], |addr| vec![addr])
+    }
+
+    fn on_swarm_event(&mut self, event: FromSwarm<Self::ConnectionHandler>) {
+        match event {
+            FromSwarm::ConnectionEstablished(ConnectionEstablished {
+                peer_id,
+                connection_id: _,
+                endpoint,
+                failed_addresses: _,
+                other_established: _,
+            }) => {
+                self.all_connections
+                    .insert(peer_id, endpoint.clone().into());
+                if let Some(pending) = self.memberships.pending_mut() {
+                    pending.peer_connected(peer_id);
+                }
+                trace!("{:?}", self.all_connections);
+            }
+
+            FromSwarm::ConnectionClosed(ConnectionClosed {
+                peer_id,
+                connection_id: _,
+                endpoint,
+                handler: _h,
+                remaining_established: _,
+            }) => {
+                self.all_connections
+                    .remove(&peer_id, &endpoint.clone().into());
+                if let Some(pending) = self.memberships.pending_mut() {
+                    pending.peer_disconnected(&peer_id);
+                }
+                debug!("{:?}", self.all_connections);
+            }
+            FromSwarm::DialFailure(DialFailure {
+                peer_id: Some(peer_id),
+                error,
+                connection_id: _,
+            }) => {
+                trace!("Dial failure: {:?} {:?}", peer_id, error);
+            }
+            _ => {}
+        }
+    }
+
+    fn on_connection_handler_event(
+        &mut self,
+        peer_id: PeerId,
+        _connection_id: ConnectionId,
+        event: THandlerOutEvent<Self>,
+    ) {
+        trace!(
+            "Received event from connection handler: {:?} from peer: {:?}",
+            event,
+            peer_id
+        );
+
+        //TODO: we may need to check who sent the update: probably we should accept only updates from members who we already know
+        if let State::WaitingPeers = self.state {
+            if self.last_sync_time + self.minimum_time_between_sync < Instant::now() {
+                self.members_provider_interval = None;
+                debug!("Received sync notification from peer {peer_id:?}, requesting membership update");
+            }
+        }
+    }
+
+    fn poll(
+        &mut self,
+        cx: &mut Context<'_>,
+        _params: &mut impl PollParameters,
+    ) -> Poll<ToSwarm<Self::OutEvent, THandlerInEvent<Self>>> {
+        match &mut self.state {
+            State::WaitingPeers => self.waiting_peers(cx),
+            State::WaitingDial(_) => self.waiting_dial(cx),
+            State::NotifyPeersUpdated => self.notify_peers_updated(),
+            State::SyncPeers(_) => self.sync_peers(),
+        }
+    }
+}

ephemera/src/network/libp2p/behaviours/membership/connections.rs

@@ -0,0 +1,88 @@
+use std::collections::{HashMap, HashSet};
+use std::fmt::Debug;
+
+use libp2p::core::ConnectedPoint;
+use libp2p::Multiaddr;
+use serde::Serialize;
+
+#[derive(PartialEq, Eq, Debug, Clone, Hash, Serialize)]
+pub(crate) enum Endpoint {
+    Dialer {
+        address: Multiaddr,
+    },
+    Listener {
+        local_address: Multiaddr,
+        remote_address: Multiaddr,
+    },
+}
+
+impl From<ConnectedPoint> for Endpoint {
+    fn from(connected_point: ConnectedPoint) -> Self {
+        match connected_point {
+            ConnectedPoint::Dialer { address, .. } => Endpoint::Dialer { address },
+            ConnectedPoint::Listener {
+                local_addr,
+                send_back_addr,
+            } => Endpoint::Listener {
+                local_address: local_addr,
+                remote_address: send_back_addr,
+            },
+        }
+    }
+}
+
+#[derive(Debug, Default, Serialize)]
+pub(crate) struct Connections {
+    dialer: HashSet<Endpoint>,
+    listener: HashSet<Endpoint>,
+}
+
+impl Connections {
+    fn insert(&mut self, connected_point: Endpoint) {
+        match connected_point {
+            Endpoint::Dialer { .. } => {
+                self.dialer.insert(connected_point);
+            }
+            Endpoint::Listener { .. } => {
+                self.listener.insert(connected_point);
+            }
+        }
+    }
+
+    fn remove(&mut self, connected_point: &Endpoint) {
+        match connected_point {
+            Endpoint::Dialer { .. } => {
+                self.dialer.remove(connected_point);
+            }
+            Endpoint::Listener { .. } => {
+                self.listener.remove(connected_point);
+            }
+        }
+    }
+}
+
+#[derive(Debug, Default, Serialize)]
+pub(crate) struct ConnectedPeers {
+    connections: HashMap<libp2p_identity::PeerId, Connections>,
+}
+
+impl ConnectedPeers {
+    pub(crate) fn is_peer_connected(&self, peer_id: &libp2p_identity::PeerId) -> bool {
+        self.connections.contains_key(peer_id)
+    }
+
+    pub(crate) fn all_connected_peers_ref(&self) -> Vec<&libp2p_identity::PeerId> {
+        self.connections.keys().collect()
+    }
+
+    pub(crate) fn insert(&mut self, peer_id: libp2p_identity::PeerId, connected_point: Endpoint) {
+        let connections = self.connections.entry(peer_id).or_default();
+        connections.insert(connected_point);
+    }
+
+    pub(crate) fn remove(&mut self, peer_id: &libp2p_identity::PeerId, connected_point: &Endpoint) {
+        if let Some(connections) = self.connections.get_mut(peer_id) {
+            connections.remove(connected_point);
+        }
+    }
+}

ephemera/src/network/libp2p/behaviours/membership/handler.rs

@@ -0,0 +1,340 @@
+//! Because each Ephemera instance requests peers at arbitrary time, a node needs to notify other
+//! peers when it has just requested an update. That helps to keep the whole cluster in sync and avoid
+//! nodes' membership diverging.
+//!
+//! Overall this synchronizes all nodes' view of the membership.
+//!
+//! Current approach is a bit 'burst'. It makes all nodes to request membership info at the same time.
+//!
+//! TODO
+//! Because we actually can verify peers' membership, it would be possible that one peer(or subset of peers) requests the
+//! peers from a rendezvous point and then sends the list to the other peers. Or possibly only the difference.
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use asynchronous_codec::Framed;
+use futures::Sink;
+use futures_util::StreamExt;
+use libp2p::{
+    swarm::handler::{
+        DialUpgradeError, FullyNegotiatedInbound, FullyNegotiatedOutbound, ListenUpgradeError,
+    },
+    swarm::NegotiatedSubstream,
+    swarm::{
+        handler::ConnectionEvent, ConnectionHandler, ConnectionHandlerEvent, KeepAlive,
+        SubstreamProtocol,
+    },
+};
+use log::{debug, error};
+use thiserror::Error;
+
+use crate::network::libp2p::behaviours::membership::protocol::{
+    MembershipCodec, Protocol, ProtocolMessage,
+};
+
+#[derive(Error, Debug)]
+pub(crate) enum Error {
+    #[error("HandlerError: {0}")]
+    Handler(#[from] anyhow::Error),
+}
+
+//Because we keep long lived connections, we need to restrict number of substream attempts.
+//Here we don't need more than 1 because it's a 'quiet' protocol.
+const MAX_SUBSTREAM_ATTEMPTS: usize = 1;
+
+enum InboundSubstreamState {
+    WaitingInput(Framed<NegotiatedSubstream, MembershipCodec>),
+    Closing(Framed<NegotiatedSubstream, MembershipCodec>),
+}
+
+enum OutboundSubstreamState {
+    WaitingOutput(Framed<NegotiatedSubstream, MembershipCodec>),
+    PendingSend(
+        Framed<NegotiatedSubstream, MembershipCodec>,
+        ProtocolMessage,
+    ),
+    PendingFlush(Framed<NegotiatedSubstream, MembershipCodec>),
+}
+
+pub(crate) struct Handler {
+    outbound_substream: Option<OutboundSubstreamState>,
+    inbound_substream: Option<InboundSubstreamState>,
+    send_queue: Vec<ProtocolMessage>,
+    outbound_substream_establishing: bool,
+    outbound_substream_attempts: usize,
+    inbound_substream_attempts: usize,
+}
+
+impl Handler {
+    pub(crate) fn new() -> Self {
+        Self {
+            outbound_substream: None,
+            inbound_substream: None,
+            send_queue: vec![],
+            outbound_substream_establishing: false,
+            outbound_substream_attempts: 0,
+            inbound_substream_attempts: 0,
+        }
+    }
+
+    //Process inbound stream messages
+    //WAITING_INPUT
+    //  - if message received, send it to behaviour
+    //  - if receive error or None, close substream
+    //
+    //CLOSING
+    //  - Wait buffer to be flushed
+    //  - Close substream
+    fn process_inbound_stream(
+        &mut self,
+        cx: &mut Context,
+    ) -> Option<Poll<ConnectionHandlerEvent<Protocol, (), FromHandler, Error>>> {
+        loop {
+            match std::mem::take(&mut self.inbound_substream) {
+                // inbound idle state
+                Some(InboundSubstreamState::WaitingInput(mut substream)) => {
+                    match substream.poll_next_unpin(cx) {
+                        Poll::Ready(Some(Ok(message))) => {
+                            self.inbound_substream =
+                                Some(InboundSubstreamState::WaitingInput(substream));
+
+                            let from_handler = FromHandler::Message(message);
+
+                            return Poll::Ready(ConnectionHandlerEvent::Custom(from_handler))
+                                .into();
+                        }
+                        Poll::Ready(Some(Err(err))) => {
+                            error!("Failed to read from substream: {err}",);
+                            self.inbound_substream =
+                                Some(InboundSubstreamState::Closing(substream));
+                        }
+                        Poll::Ready(None) => {
+                            debug!("Inbound stream closed by remote");
+                            self.inbound_substream =
+                                Some(InboundSubstreamState::Closing(substream));
+                        }
+                        Poll::Pending => {
+                            self.inbound_substream =
+                                Some(InboundSubstreamState::WaitingInput(substream));
+                            break;
+                        }
+                    }
+                }
+                Some(InboundSubstreamState::Closing(mut substream)) => {
+                    match Sink::poll_close(Pin::new(&mut substream), cx) {
+                        Poll::Ready(res) => {
+                            if let Err(e) = res {
+                                error!("Inbound substream error while closing: {e}");
+                            }
+                            self.inbound_substream = None;
+                            break;
+                        }
+                        Poll::Pending => {
+                            self.inbound_substream =
+                                Some(InboundSubstreamState::Closing(substream));
+                            break;
+                        }
+                    }
+                }
+                None => {
+                    self.inbound_substream = None;
+                    break;
+                }
+            }
+        }
+        None
+    }
+
+    //Process outbound stream messages
+    //WAITING_OUTPUT
+    //  - if send queue is not empty, go to PENDING_SEND
+    //
+    //PENDING_SEND
+    //  - send message to substream
+    //  - if send error, mark substream as Closing
+    //
+    //PENDING_FLUSH
+    //  - flush substream
+    //  - if flush error, mark substream as Closing
+    fn process_outbound_stream(&mut self, cx: &mut Context) {
+        loop {
+            match std::mem::take(&mut self.outbound_substream) {
+                // outbound idle state
+                Some(OutboundSubstreamState::WaitingOutput(substream)) => {
+                    if let Some(message) = self.send_queue.pop() {
+                        self.outbound_substream =
+                            Some(OutboundSubstreamState::PendingSend(substream, message));
+                        continue;
+                    }
+
+                    self.outbound_substream =
+                        Some(OutboundSubstreamState::WaitingOutput(substream));
+                    break;
+                }
+                Some(OutboundSubstreamState::PendingSend(mut substream, message)) => {
+                    match Sink::poll_ready(Pin::new(&mut substream), cx) {
+                        Poll::Ready(Ok(())) => {
+                            match Sink::start_send(Pin::new(&mut substream), message) {
+                                Ok(()) => {
+                                    self.outbound_substream =
+                                        Some(OutboundSubstreamState::PendingFlush(substream));
+                                }
+                                Err(e) => {
+                                    debug!("Failed to send message on outbound stream: {e}");
+                                    self.outbound_substream = None;
+                                    break;
+                                }
+                            }
+                        }
+                        Poll::Ready(Err(e)) => {
+                            debug!("Failed to send message on outbound stream: {e}");
+                            self.outbound_substream = None;
+                            break;
+                        }
+                        Poll::Pending => {
+                            self.outbound_substream =
+                                Some(OutboundSubstreamState::PendingSend(substream, message));
+                            break;
+                        }
+                    }
+                }
+                Some(OutboundSubstreamState::PendingFlush(mut substream)) => {
+                    match Sink::poll_flush(Pin::new(&mut substream), cx) {
+                        Poll::Ready(Ok(())) => {
+                            self.outbound_substream =
+                                Some(OutboundSubstreamState::WaitingOutput(substream));
+                        }
+                        Poll::Ready(Err(e)) => {
+                            debug!("Failed to flush outbound stream: {e}");
+                            self.outbound_substream = None;
+                            break;
+                        }
+                        Poll::Pending => {
+                            self.outbound_substream =
+                                Some(OutboundSubstreamState::PendingFlush(substream));
+                            break;
+                        }
+                    }
+                }
+                None => {
+                    self.outbound_substream = None;
+                    break;
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum FromHandler {
+    Message(ProtocolMessage),
+}
+
+#[derive(Debug)]
+pub(crate) enum ToHandler {
+    Message(ProtocolMessage),
+}
+
+impl ConnectionHandler for Handler {
+    type InEvent = ToHandler;
+    type OutEvent = FromHandler;
+    type Error = Error;
+    type InboundProtocol = Protocol;
+    type OutboundProtocol = Protocol;
+    type InboundOpenInfo = ();
+    type OutboundOpenInfo = ();
+
+    fn listen_protocol(&self) -> SubstreamProtocol<Protocol, ()> {
+        SubstreamProtocol::new(Protocol, ())
+    }
+
+    fn connection_keep_alive(&self) -> KeepAlive {
+        //we could add idle timeout here
+        KeepAlive::Yes
+    }
+
+    fn poll(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<
+        ConnectionHandlerEvent<
+            Self::OutboundProtocol,
+            Self::OutboundOpenInfo,
+            Self::OutEvent,
+            Self::Error,
+        >,
+    > {
+        //  poll STATE_MACHINE
+        //- Request outbound substream if neccessary
+        //- poll inbound substream
+        //- poll outbound substream
+
+        //Establish new connection when behaviour wants to send a message and we don't have an outbound substream yet
+        if !self.send_queue.is_empty()
+            && self.outbound_substream.is_none()
+            && !self.outbound_substream_establishing
+        {
+            self.outbound_substream_establishing = true;
+            return Poll::Ready(ConnectionHandlerEvent::OutboundSubstreamRequest {
+                protocol: SubstreamProtocol::new(Protocol, ()),
+            });
+        }
+
+        if let Some(res) = self.process_inbound_stream(cx) {
+            return res;
+        }
+
+        self.process_outbound_stream(cx);
+
+        Poll::Pending
+    }
+
+    fn on_behaviour_event(&mut self, event: Self::InEvent) {
+        match event {
+            ToHandler::Message(message) => {
+                self.send_queue.push(message);
+            }
+        }
+    }
+
+    fn on_connection_event(
+        &mut self,
+        event: ConnectionEvent<
+            Self::InboundProtocol,
+            Self::OutboundProtocol,
+            Self::InboundOpenInfo,
+            Self::OutboundOpenInfo,
+        >,
+    ) {
+        match event {
+            ConnectionEvent::FullyNegotiatedInbound(FullyNegotiatedInbound {
+                protocol: stream,
+                info: _,
+            }) => {
+                if self.inbound_substream_attempts > MAX_SUBSTREAM_ATTEMPTS {
+                    log::warn!("Too many inbound substream attempts, refusing stream");
+                    return;
+                }
+                self.inbound_substream_attempts += 1;
+                self.inbound_substream = Some(InboundSubstreamState::WaitingInput(stream));
+            }
+            ConnectionEvent::FullyNegotiatedOutbound(FullyNegotiatedOutbound {
+                protocol,
+                info: _,
+            }) => {
+                if self.outbound_substream_attempts > MAX_SUBSTREAM_ATTEMPTS {
+                    log::warn!("Too many outbound substream attempts, refusing stream");
+                    return;
+                }
+                self.outbound_substream = Some(OutboundSubstreamState::WaitingOutput(protocol));
+            }
+            ConnectionEvent::DialUpgradeError(DialUpgradeError { info, error }) => {
+                error!("DialUpgradeError: info: {:?}, error: {:?}", info, error);
+            }
+            ConnectionEvent::ListenUpgradeError(ListenUpgradeError { info, error }) => {
+                error!("ListenUpgradeError: info: {:?}, error: {:?}", info, error);
+            }
+            ConnectionEvent::AddressChange(_) => {}
+        }
+    }
+}

ephemera/src/network/libp2p/behaviours/membership/mod.rs

@@ -0,0 +1,190 @@
+//! In Ephemera, membership of reliable broadcast protocol is decided by membership provider.
+//! Only peers who are returned by [`crate::membership::MembersProviderFut`] are allowed to participate.
+
+use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
+
+use libp2p_identity::PeerId;
+use lru::LruCache;
+
+use crate::network::Peer;
+
+pub(crate) mod behaviour;
+mod connections;
+mod handler;
+mod protocol;
+
+const MAX_DIAL_ATTEMPT_ROUNDS: usize = 6;
+
+/// Minimum percentage of available nodes to consider the network healthy.
+//TODO: make this configurable
+const MEMBERSHIP_MINIMUM_AVAILABLE_NODES_RATIO: f64 = 0.8;
+
+/// Minimum time between syncs of membership.
+const MEMBERSHIP_SYNC_INTERVAL_SEC: u64 = 60;
+
+/// Maximum percentage of nodes that can change in a single membership update.
+/// In general it should be considered a security risk if it has changed too much.
+/// //TODO: make this configurable
+const _MEMBERSHIP_MAXIMUM_ALLOWED_CHANGE_RATIO: f64 = 0.2;
+
+/// Membership provider returns list of peers. But it is up to the Ephemera user to decide
+/// how reliable the list is. For example, it can contain peers who are offline.
+
+/// This enum defines how the actual membership is decided.
+#[derive(Debug)]
+pub(crate) enum MembershipKind {
+    /// Mandatory minimum membership size is defined by threshold of all peers returned by membership provider.
+    Threshold(f64),
+    /// Mandatory minimum membership size is all peers who are online.
+    AnyOnline,
+    /// Mandatory minimum membership size is all peers returned by membership provider.
+    AllOnline,
+}
+
+impl MembershipKind {
+    #[allow(
+        clippy::cast_precision_loss,
+        clippy::cast_sign_loss,
+        clippy::cast_possible_truncation
+    )]
+    pub(crate) fn accept(&self, membership: &Membership) -> bool {
+        let total_number_of_peers = membership.all_members.len();
+        let connected_peers = membership.connected_peers_ids.len();
+        match self {
+            MembershipKind::Threshold(threshold) => {
+                let minimum_available_nodes = (total_number_of_peers as f64 * threshold) as usize;
+                connected_peers >= minimum_available_nodes
+            }
+            MembershipKind::AnyOnline => connected_peers > 0,
+            MembershipKind::AllOnline => connected_peers == total_number_of_peers,
+        }
+    }
+}
+
+pub(crate) struct Memberships {
+    snapshots: LruCache<u64, Membership>,
+    current: u64,
+    /// This is set when we get new peers set from [crate::membership::MembersProviderFut]
+    /// but haven't yet activated it.
+    pending_membership: Option<Membership>,
+}
+
+impl Memberships {
+    pub(crate) fn new() -> Self {
+        let mut snapshots = LruCache::new(NonZeroUsize::new(1000).unwrap());
+        snapshots.put(0, Membership::new(HashMap::default()));
+        Self {
+            snapshots,
+            current: 0,
+            pending_membership: None,
+        }
+    }
+
+    pub(crate) fn current(&mut self) -> &Membership {
+        //Unwrap is safe because we always have current membership
+        self.snapshots.get(&self.current).unwrap()
+    }
+
+    pub(crate) fn update(&mut self, membership: Membership) {
+        self.current += 1;
+        self.snapshots.put(self.current, membership);
+    }
+
+    pub(crate) fn set_pending(&mut self, membership: Membership) {
+        self.pending_membership = Some(membership);
+    }
+
+    pub(crate) fn remove_pending(&mut self) -> Option<Membership> {
+        self.pending_membership.take()
+    }
+
+    pub(crate) fn pending(&self) -> Option<&Membership> {
+        self.pending_membership.as_ref()
+    }
+
+    pub(crate) fn pending_mut(&mut self) -> Option<&mut Membership> {
+        self.pending_membership.as_mut()
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Membership {
+    local_peer_id: PeerId,
+    all_members: HashMap<PeerId, Peer>,
+    all_peers_ids: HashSet<PeerId>,
+    connected_peers_ids: HashSet<PeerId>,
+}
+
+impl Membership {
+    pub(crate) fn new_with_local(
+        all_members: HashMap<PeerId, Peer>,
+        local_peer_id: PeerId,
+    ) -> Self {
+        let all_peers_ids = all_members.keys().copied().collect();
+        Self {
+            local_peer_id,
+            all_members,
+            all_peers_ids,
+            connected_peers_ids: HashSet::new(),
+        }
+    }
+
+    pub(crate) fn new(all_members: HashMap<PeerId, Peer>) -> Self {
+        let all_peers_ids = all_members.keys().copied().collect();
+        Self {
+            local_peer_id: PeerId::random(),
+            all_members,
+            all_peers_ids,
+            connected_peers_ids: HashSet::new(),
+        }
+    }
+
+    pub(crate) fn includes_local(&self) -> bool {
+        self.all_members.contains_key(&self.local_peer_id)
+    }
+
+    pub(crate) fn peer_connected(&mut self, peer_id: PeerId) {
+        self.connected_peers_ids.insert(peer_id);
+    }
+
+    pub(crate) fn peer_disconnected(&mut self, peer_id: &PeerId) {
+        self.connected_peers_ids.remove(peer_id);
+    }
+
+    pub(crate) fn all_peer_ids(&self) -> &HashSet<PeerId> {
+        &self.all_peers_ids
+    }
+
+    pub(crate) fn connected_peer_ids(&self) -> HashSet<PeerId> {
+        self.connected_peers_ids.clone()
+    }
+
+    pub(crate) fn connected_peer_ids_with_local(&self) -> HashSet<PeerId> {
+        let mut active_peers = self.connected_peers_ids.clone();
+        active_peers.insert(self.local_peer_id);
+        active_peers
+    }
+
+    pub(crate) fn connected_peers(&self) -> &HashSet<PeerId> {
+        &self.connected_peers_ids
+    }
+
+    pub(crate) fn peer_address(&self, peer_id: &PeerId) -> Option<&libp2p::Multiaddr> {
+        self.all_members
+            .get(peer_id)
+            .map(|peer| peer.address.inner())
+    }
+}
+
+impl From<crate::config::MembershipKind> for MembershipKind {
+    fn from(kind: crate::config::MembershipKind) -> Self {
+        match kind {
+            crate::config::MembershipKind::Threshold => {
+                MembershipKind::Threshold(MEMBERSHIP_MINIMUM_AVAILABLE_NODES_RATIO)
+            }
+            crate::config::MembershipKind::AnyOnline => MembershipKind::AnyOnline,
+            crate::config::MembershipKind::AllOnline => MembershipKind::AllOnline,
+        }
+    }
+}

ephemera/src/network/libp2p/behaviours/membership/protocol.rs

@@ -0,0 +1,100 @@
+use std::future::Future;
+use std::iter;
+use std::pin::Pin;
+
+use asynchronous_codec::{Decoder, Encoder, Framed};
+use bytes::BytesMut;
+use futures::{AsyncRead, AsyncWrite};
+use futures_util::future;
+use libp2p::core::UpgradeInfo;
+use libp2p::{InboundUpgrade, OutboundUpgrade};
+use log::trace;
+use serde::{Deserialize, Serialize};
+
+use crate::utilities::codec::varint_bytes::{read_length_prefixed, write_length_prefixed};
+
+pub const PROTOCOL_NAME: &[u8] = b"/ephemera/membership/1.0.0";
+
+pub(crate) struct Protocol;
+
+impl UpgradeInfo for Protocol {
+    type Info = &'static [u8];
+    type InfoIter = iter::Once<Self::Info>;
+
+    fn protocol_info(&self) -> Self::InfoIter {
+        iter::once(PROTOCOL_NAME)
+    }
+}
+
+impl<C> InboundUpgrade<C> for Protocol
+where
+    C: AsyncRead + AsyncWrite + Unpin + Send + 'static,
+{
+    type Output = Framed<C, MembershipCodec>;
+    type Error = anyhow::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Output, Self::Error>> + Send>>;
+
+    fn upgrade_inbound(self, socket: C, _: Self::Info) -> Self::Future {
+        trace!(
+            "Inbound upgrade for protocol: {}",
+            String::from_utf8_lossy(PROTOCOL_NAME)
+        );
+        Box::pin(future::ok(Framed::new(socket, MembershipCodec {})))
+    }
+}
+
+impl<C> OutboundUpgrade<C> for Protocol
+where
+    C: AsyncRead + AsyncWrite + Unpin + Send + 'static,
+{
+    type Output = Framed<C, MembershipCodec>;
+    type Error = anyhow::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Output, Self::Error>> + Send>>;
+
+    fn upgrade_outbound(self, socket: C, _: Self::Info) -> Self::Future {
+        trace!(
+            "Outbound upgrade for protocol: {}",
+            String::from_utf8_lossy(PROTOCOL_NAME)
+        );
+        Box::pin(future::ok(Framed::new(socket, MembershipCodec {})))
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) enum ProtocolMessage {
+    Sync,
+}
+
+pub(crate) struct MembershipCodec {}
+
+impl Encoder for MembershipCodec {
+    type Item = ProtocolMessage;
+    type Error = anyhow::Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        //FIXME: switch to binary
+        let data = serde_json::to_vec(&item).unwrap();
+        write_length_prefixed(dst, data);
+        Ok(())
+    }
+}
+
+impl Decoder for MembershipCodec {
+    type Item = ProtocolMessage;
+    type Error = anyhow::Error;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        if src.is_empty() {
+            return Ok(None);
+        }
+        let data = read_length_prefixed(src, 1024 * 1024)?;
+        match data {
+            None => Ok(None),
+            Some(data) => {
+                //FIXME: switch to binary
+                let msg = serde_json::from_slice(&data)?;
+                Ok(msg)
+            }
+        }
+    }
+}

ephemera/src/network/libp2p/behaviours/mod.rs

@@ -0,0 +1,183 @@
+use std::future::Future;
+use std::{iter, sync::Arc, time::Duration};
+
+use libp2p::{
+    core::{muxing::StreamMuxerBox, transport::Boxed},
+    dns, gossipsub,
+    gossipsub::{IdentTopic as Topic, MessageAuthenticity, ValidationMode},
+    kad, noise, request_response as libp2p_request_response,
+    swarm::NetworkBehaviour,
+    tcp::{tokio::Transport as TokioTransport, Config as TokioConfig},
+    yamux, PeerId as Libp2pPeerId, Transport,
+};
+use log::info;
+
+use crate::membership::PeerInfo;
+use crate::network::libp2p::behaviours::membership::MembershipKind;
+use crate::{
+    broadcast::RbMsg,
+    crypto::Keypair,
+    network::libp2p::behaviours::request_response::{
+        RbMsgMessagesCodec, RbMsgProtocol, RbMsgResponse,
+    },
+    peer::{PeerId, ToPeerId},
+    utilities::hash::{EphemeraHasher, Hasher},
+};
+
+pub(crate) mod membership;
+pub(crate) mod request_response;
+
+#[derive(NetworkBehaviour)]
+#[behaviour(out_event = "GroupBehaviourEvent")]
+pub(crate) struct GroupNetworkBehaviour<P>
+where
+    P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + 'static,
+{
+    pub(crate) members_provider: membership::behaviour::Behaviour<P>,
+    pub(crate) gossipsub: gossipsub::Behaviour,
+    pub(crate) request_response: libp2p_request_response::Behaviour<RbMsgMessagesCodec>,
+    pub(crate) kademlia: kad::Kademlia<kad::store::MemoryStore>,
+}
+
+#[allow(clippy::large_enum_variant)]
+pub(crate) enum GroupBehaviourEvent {
+    Gossipsub(gossipsub::Event),
+    RequestResponse(libp2p_request_response::Event<RbMsg, RbMsgResponse>),
+    Membership(membership::behaviour::Event),
+    Kademlia(kad::KademliaEvent),
+}
+
+impl From<gossipsub::Event> for GroupBehaviourEvent {
+    fn from(event: gossipsub::Event) -> Self {
+        GroupBehaviourEvent::Gossipsub(event)
+    }
+}
+
+impl From<libp2p_request_response::Event<RbMsg, RbMsgResponse>> for GroupBehaviourEvent {
+    fn from(event: libp2p_request_response::Event<RbMsg, RbMsgResponse>) -> Self {
+        GroupBehaviourEvent::RequestResponse(event)
+    }
+}
+
+impl From<membership::behaviour::Event> for GroupBehaviourEvent {
+    fn from(event: membership::behaviour::Event) -> Self {
+        GroupBehaviourEvent::Membership(event)
+    }
+}
+
+impl From<kad::KademliaEvent> for GroupBehaviourEvent {
+    fn from(event: kad::KademliaEvent) -> Self {
+        GroupBehaviourEvent::Kademlia(event)
+    }
+}
+
+//Create combined behaviour.
+//Gossipsub takes care of message delivery semantics
+//Membership takes care of providing peers who are part of the reliable broadcast group
+//Kademlia takes provides closest neighbours and general DHT functionality
+pub(crate) fn create_behaviour<P>(
+    keypair: &Arc<Keypair>,
+    ephemera_msg_topic: &Topic,
+    members_provider: P,
+    members_provider_delay: Duration,
+    membership_kind: MembershipKind,
+) -> GroupNetworkBehaviour<P>
+where
+    P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    //TODO: review behaviours config(eg. gossipsub minimum peers, kademlia ttl, request-response timeouts etc.)
+    let local_peer_id = keypair.peer_id();
+    let gossipsub = create_gossipsub(keypair, ephemera_msg_topic);
+    let request_response = create_request_response();
+    let rendezvous_behaviour = create_membership(
+        members_provider,
+        members_provider_delay,
+        membership_kind,
+        local_peer_id,
+    );
+    let kademlia = create_kademlia(keypair);
+
+    GroupNetworkBehaviour {
+        members_provider: rendezvous_behaviour,
+        gossipsub,
+        request_response,
+        kademlia,
+    }
+}
+
+// Configure networking messaging stack(Gossipsub)
+pub(crate) fn create_gossipsub(local_key: &Arc<Keypair>, topic: &Topic) -> gossipsub::Behaviour {
+    let gossipsub_config = gossipsub::ConfigBuilder::default()
+        //TODO: settings from config
+        .heartbeat_interval(Duration::from_secs(5))
+        .message_id_fn(|msg: &gossipsub::Message| Hasher::digest(&msg.data).into())
+        .validation_mode(ValidationMode::Strict)
+        .build()
+        .expect("Valid config");
+
+    let mut behaviour = gossipsub::Behaviour::new(
+        MessageAuthenticity::Signed(local_key.inner().clone()),
+        gossipsub_config,
+    )
+    .expect("Correct configuration");
+
+    info!("Subscribing to topic: {}", topic);
+    behaviour.subscribe(topic).expect("Valid topic");
+    behaviour
+}
+
+pub(crate) fn create_request_response() -> libp2p_request_response::Behaviour<RbMsgMessagesCodec> {
+    let config = libp2p_request_response::Config::default();
+    libp2p_request_response::Behaviour::new(
+        RbMsgMessagesCodec,
+        iter::once((
+            RbMsgProtocol,
+            libp2p_request_response::ProtocolSupport::Full,
+        )),
+        config,
+    )
+}
+
+pub(crate) fn create_membership<P>(
+    members_provider: P,
+    members_provider_delay: Duration,
+    membership_kind: MembershipKind,
+    local_peer_id: PeerId,
+) -> membership::behaviour::Behaviour<P>
+where
+    P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    membership::behaviour::Behaviour::new(
+        members_provider,
+        members_provider_delay,
+        local_peer_id.into(),
+        membership_kind,
+    )
+}
+
+pub(super) fn create_kademlia(local_key: &Arc<Keypair>) -> kad::Kademlia<kad::store::MemoryStore> {
+    let peer_id = local_key.peer_id();
+    let mut cfg = kad::KademliaConfig::default();
+    cfg.set_query_timeout(Duration::from_secs(5 * 60));
+    let store = kad::store::MemoryStore::new(peer_id.0);
+    kad::Kademlia::with_config(*peer_id.inner(), store, cfg)
+}
+
+//Configure networking connection stack(Tcp, Noise, Yamux)
+//Tcp protocol for networking
+//Noise protocol for encryption
+//Yamux protocol for multiplexing
+pub(crate) fn create_transport(
+    local_key: &Arc<Keypair>,
+) -> anyhow::Result<Boxed<(Libp2pPeerId, StreamMuxerBox)>> {
+    let transport = TokioTransport::new(TokioConfig::default().nodelay(true));
+    let transport = dns::TokioDnsConfig::system(transport)?;
+
+    let noise_config = noise::Config::new(local_key.inner())?;
+    Ok(transport
+        .upgrade(libp2p::core::upgrade::Version::V1)
+        .authenticate(noise_config)
+        .multiplex(yamux::Config::default())
+        .timeout(Duration::from_secs(20))
+        .boxed())
+}

ephemera/src/network/libp2p/behaviours/request_response/mod.rs

@@ -0,0 +1,101 @@
+use async_trait::async_trait;
+use futures::{AsyncRead, AsyncWrite};
+use libp2p::request_response;
+use log::trace;
+use serde::{Deserialize, Serialize};
+
+use crate::broadcast::RbMsg;
+use crate::utilities::codec::varint_async::{read_length_prefixed, write_length_prefixed};
+use crate::utilities::id::EphemeraId;
+
+#[derive(Clone)]
+pub(crate) struct RbMsgMessagesCodec;
+
+impl RbMsgMessagesCodec {}
+
+#[derive(Clone)]
+pub(crate) struct RbMsgProtocol;
+
+impl request_response::ProtocolName for RbMsgProtocol {
+    fn protocol_name(&self) -> &[u8] {
+        "/ephemera/reliable_broadcast/1.0.0".as_bytes()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct RbMsgResponse {
+    pub(crate) id: EphemeraId,
+}
+
+impl RbMsgResponse {
+    pub(crate) fn new(id: EphemeraId) -> Self {
+        Self { id }
+    }
+}
+
+#[async_trait]
+impl request_response::Codec for RbMsgMessagesCodec {
+    type Protocol = RbMsgProtocol;
+    type Request = RbMsg;
+    type Response = RbMsgResponse;
+
+    async fn read_request<T>(
+        &mut self,
+        _: &Self::Protocol,
+        io: &mut T,
+    ) -> Result<Self::Request, std::io::Error>
+    where
+        T: AsyncRead + Unpin + Send,
+    {
+        //FIXME: max size
+        let data = read_length_prefixed(io, 1024 * 1024).await?;
+        //FIXME: switch to binary
+        let msg = serde_json::from_slice(&data)?;
+        trace!("Received request {:?}", msg);
+        Ok(msg)
+    }
+
+    async fn read_response<T>(
+        &mut self,
+        _: &Self::Protocol,
+        io: &mut T,
+    ) -> std::io::Result<Self::Response>
+    where
+        T: AsyncRead + Unpin + Send,
+    {
+        //FIXME: max size
+        let response = read_length_prefixed(io, 1024 * 1024).await?;
+        //FIXME: switch to binary
+        let response = serde_json::from_slice(&response)?;
+        trace!("Received response {:?}", response);
+        Ok(response)
+    }
+
+    async fn write_request<T>(
+        &mut self,
+        _: &Self::Protocol,
+        io: &mut T,
+        req: Self::Request,
+    ) -> Result<(), std::io::Error>
+    where
+        T: AsyncWrite + Unpin + Send,
+    {
+        let data = serde_json::to_vec(&req).unwrap();
+        write_length_prefixed(io, data).await?;
+        Ok(())
+    }
+
+    async fn write_response<T>(
+        &mut self,
+        _: &Self::Protocol,
+        io: &mut T,
+        response: Self::Response,
+    ) -> std::io::Result<()>
+    where
+        T: AsyncWrite + Unpin + Send,
+    {
+        let response = serde_json::to_vec(&response).unwrap();
+        write_length_prefixed(io, response).await?;
+        Ok(())
+    }
+}

ephemera/src/network/libp2p/ephemera_sender.rs

@@ -0,0 +1,58 @@
+use log::trace;
+use tokio::sync::mpsc;
+
+use crate::block::types::message::EphemeraMessage;
+use crate::broadcast::RbMsg;
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum EphemeraEvent {
+    EphemeraMessage(Box<EphemeraMessage>),
+    ProtocolMessage(Box<RbMsg>),
+    StoreInDht { key: Vec<u8>, value: Vec<u8> },
+    QueryDht { key: Vec<u8> },
+}
+
+pub(crate) struct EphemeraToNetwork;
+
+impl EphemeraToNetwork {
+    pub(crate) fn init() -> (EphemeraToNetworkSender, EphemeraToNetworkReceiver) {
+        let (net_event_tx, net_event_rcv) = mpsc::channel(1000);
+
+        let receiver = EphemeraToNetworkReceiver::new(net_event_rcv);
+        let sender = EphemeraToNetworkSender::new(net_event_tx);
+
+        (sender, receiver)
+    }
+}
+
+//Receives messages from the network
+pub(crate) struct EphemeraToNetworkReceiver {
+    pub(crate) net_event_rcv: mpsc::Receiver<EphemeraEvent>,
+}
+
+impl EphemeraToNetworkReceiver {
+    pub(crate) fn new(net_event_rcv: mpsc::Receiver<EphemeraEvent>) -> Self {
+        Self { net_event_rcv }
+    }
+}
+
+//Sends messages to the network
+pub(crate) struct EphemeraToNetworkSender {
+    pub(crate) network_event_sender_tx: mpsc::Sender<EphemeraEvent>,
+}
+
+impl EphemeraToNetworkSender {
+    pub(crate) fn new(network_event_sender_tx: mpsc::Sender<EphemeraEvent>) -> Self {
+        Self {
+            network_event_sender_tx,
+        }
+    }
+
+    pub(crate) async fn send_ephemera_event(&mut self, event: EphemeraEvent) -> anyhow::Result<()> {
+        trace!("Network event: {:?}", event);
+        self.network_event_sender_tx
+            .send(event)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))
+    }
+}

ephemera/src/network/libp2p/mod.rs

@@ -0,0 +1,4 @@
+mod behaviours;
+pub(crate) mod ephemera_sender;
+pub(crate) mod network_sender;
+pub(crate) mod swarm_network;

ephemera/src/network/libp2p/network_sender.rs

@@ -0,0 +1,67 @@
+use log::trace;
+use std::collections::HashSet;
+use tokio::sync::mpsc;
+
+use crate::block::types::message::EphemeraMessage;
+use crate::broadcast::RbMsg;
+use crate::peer::PeerId;
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum GroupChangeEvent {
+    PeersUpdated(HashSet<PeerId>),
+    LocalPeerRemoved(HashSet<PeerId>),
+    NotEnoughPeers(HashSet<PeerId>),
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum NetworkEvent {
+    EphemeraMessage(Box<EphemeraMessage>),
+    BroadcastMessage(Box<RbMsg>),
+    GroupUpdate(GroupChangeEvent),
+    QueryDhtResponse { key: Vec<u8>, value: Vec<u8> },
+}
+
+pub(crate) struct EphemeraNetworkCommunication;
+
+impl EphemeraNetworkCommunication {
+    pub(crate) fn init() -> (NetCommunicationSender, NetCommunicationReceiver) {
+        let (net_event_tx, net_event_rcv) = mpsc::channel(1000);
+
+        let receiver = NetCommunicationReceiver::new(net_event_rcv);
+        let sender = NetCommunicationSender::new(net_event_tx);
+
+        (sender, receiver)
+    }
+}
+
+//Receives messages from the network
+pub(crate) struct NetCommunicationReceiver {
+    pub(crate) net_event_rcv: mpsc::Receiver<NetworkEvent>,
+}
+
+impl NetCommunicationReceiver {
+    pub(crate) fn new(net_event_rcv: mpsc::Receiver<NetworkEvent>) -> Self {
+        Self { net_event_rcv }
+    }
+}
+
+//Sends messages to the network
+pub(crate) struct NetCommunicationSender {
+    pub(crate) network_event_sender_tx: mpsc::Sender<NetworkEvent>,
+}
+
+impl NetCommunicationSender {
+    pub(crate) fn new(network_event_sender_tx: mpsc::Sender<NetworkEvent>) -> Self {
+        Self {
+            network_event_sender_tx,
+        }
+    }
+
+    pub(crate) async fn send_network_event(&mut self, event: NetworkEvent) -> anyhow::Result<()> {
+        trace!("Network event: {:?}", event);
+        self.network_event_sender_tx
+            .send(event)
+            .await
+            .map_err(|e| anyhow::anyhow!(e))
+    }
+}

ephemera/src/network/libp2p/swarm_network.rs

@@ -0,0 +1,604 @@
+use std::collections::HashSet;
+use std::future::Future;
+use std::str::FromStr;
+
+use futures::StreamExt;
+use libp2p::kad::{GetClosestPeersResult, GetRecordResult};
+use libp2p::swarm::{NetworkBehaviour, SwarmBuilder};
+use libp2p::{
+    gossipsub, gossipsub::IdentTopic as Topic, kad, request_response, swarm::SwarmEvent, Multiaddr,
+    Swarm,
+};
+use log::{debug, error, info, trace};
+
+use crate::membership::PeerInfo;
+use crate::{
+    block::types::message::EphemeraMessage,
+    broadcast::RbMsg,
+    codec::Encode,
+    core::builder::NodeInfo,
+    network::libp2p::behaviours,
+    network::libp2p::{
+        behaviours::{
+            create_behaviour, create_transport, request_response::RbMsgResponse,
+            GroupBehaviourEvent, GroupNetworkBehaviour,
+        },
+        ephemera_sender::{
+            EphemeraEvent, EphemeraToNetwork, EphemeraToNetworkReceiver, EphemeraToNetworkSender,
+        },
+        network_sender::{
+            EphemeraNetworkCommunication, GroupChangeEvent,
+            GroupChangeEvent::{LocalPeerRemoved, NotEnoughPeers},
+            NetCommunicationReceiver, NetCommunicationSender, NetworkEvent,
+        },
+    },
+};
+
+pub(crate) type InitSwarm<P> = (
+    SwarmNetwork<P>,
+    NetCommunicationReceiver,
+    EphemeraToNetworkSender,
+);
+
+pub struct SwarmNetwork<P>
+where
+    P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    node_info: NodeInfo,
+    swarm: Swarm<GroupNetworkBehaviour<P>>,
+    from_ephemera_rcv: EphemeraToNetworkReceiver,
+    to_ephemera_tx: NetCommunicationSender,
+    ephemera_msg_topic: Topic,
+}
+
+impl<P> SwarmNetwork<P>
+where
+    P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + Unpin + 'static,
+{
+    pub(crate) fn new(node_info: NodeInfo, members_provider: P) -> anyhow::Result<InitSwarm<P>>
+    where
+        P: Future<Output = crate::membership::Result<Vec<PeerInfo>>> + Send + 'static,
+    {
+        let (from_ephemera_tx, from_ephemera_rcv) = EphemeraToNetwork::init();
+        let (to_ephemera_tx, to_ephemera_rcv) = EphemeraNetworkCommunication::init();
+
+        let libp2p_configuration = node_info.initial_config.libp2p.clone();
+
+        let local_key = node_info.keypair.clone();
+        let peer_id = node_info.peer_id;
+        let ephemera_msg_topic = Topic::new(&libp2p_configuration.ephemera_msg_topic_name);
+
+        let transport = create_transport(&local_key)?;
+
+        let members_provider_delay =
+            std::time::Duration::from_secs(libp2p_configuration.members_provider_delay_sec);
+        let behaviour = create_behaviour(
+            &local_key,
+            &ephemera_msg_topic,
+            members_provider,
+            members_provider_delay,
+            libp2p_configuration.membership_kind.into(),
+        );
+
+        let swarm = SwarmBuilder::with_tokio_executor(transport, behaviour, peer_id.into()).build();
+
+        let network = SwarmNetwork {
+            node_info,
+            swarm,
+            from_ephemera_rcv,
+            to_ephemera_tx,
+            ephemera_msg_topic,
+        };
+
+        Ok((network, to_ephemera_rcv, from_ephemera_tx))
+    }
+
+    pub(crate) fn listen(&mut self) -> anyhow::Result<()> {
+        let address =
+            Multiaddr::from_str(&self.node_info.protocol_address()).expect("Invalid multi-address");
+        self.swarm.listen_on(address.clone())?;
+
+        info!("Listening on {address:?}");
+        Ok(())
+    }
+
+    pub(crate) async fn start(mut self) -> anyhow::Result<()> {
+        loop {
+            tokio::select! {
+                swarm_event = self.swarm.next() => {
+                    match swarm_event{
+                        Some(event) => {
+                            if let Err(err) = self.handle_incoming_messages(event).await{
+                                error!("Error handling swarm event: {:?}", err);
+                            }
+                        }
+                        None => {
+                            anyhow::bail!("Swarm event channel closed");
+                        }
+                    }
+                },
+                Some(event) = self.from_ephemera_rcv.net_event_rcv.recv() => {
+                    self.process_ephemera_events(event);
+                }
+            }
+        }
+    }
+
+    fn process_ephemera_events(&mut self, event: EphemeraEvent) {
+        match event {
+            EphemeraEvent::EphemeraMessage(em) => {
+                self.send_ephemera_message(em.as_ref());
+            }
+            EphemeraEvent::ProtocolMessage(pm) => {
+                self.send_broadcast_message(pm.as_ref());
+            }
+            EphemeraEvent::StoreInDht { key, value } => {
+                let record = kad::Record::new(key, value);
+                let quorum = kad::Quorum::One;
+                match self
+                    .swarm
+                    .behaviour_mut()
+                    .kademlia
+                    .put_record(record, quorum)
+                {
+                    Ok(ok) => {
+                        trace!("StoreDht: {:?}", ok);
+                    }
+                    Err(err) => {
+                        error!("StoreDht: {:?}", err);
+                    }
+                }
+            }
+            EphemeraEvent::QueryDht { key } => {
+                let kad_key = kad::record::Key::new::<Vec<u8>>(key.as_ref());
+                let query_id = self.swarm.behaviour_mut().kademlia.get_record(kad_key);
+                trace!("QueryDht: {:?}", query_id);
+            }
+        }
+    }
+
+    async fn handle_incoming_messages<E>(
+        &mut self,
+        swarm_event: SwarmEvent<GroupBehaviourEvent, E>,
+    ) -> anyhow::Result<()> {
+        if let SwarmEvent::Behaviour(b) = swarm_event {
+            if let Err(err) = self.process_group_behaviour_event(b).await {
+                error!("Error handling behaviour event: {:?}", err);
+            }
+        } else {
+            Self::process_other_swarm_events(swarm_event);
+        }
+        Ok(())
+    }
+
+    async fn process_group_behaviour_event(
+        &mut self,
+        event: GroupBehaviourEvent,
+    ) -> anyhow::Result<()> {
+        match event {
+            GroupBehaviourEvent::Gossipsub(gs) => {
+                if let Err(err) = self.process_gossipsub_event(gs).await {
+                    error!("Error processing gossipsub event: {:?}", err);
+                }
+            }
+            GroupBehaviourEvent::RequestResponse(request_response) => {
+                if let Err(err) = self.process_request_response(request_response).await {
+                    error!("Error processing request response: {:?}", err);
+                }
+            }
+
+            GroupBehaviourEvent::Membership(event) => {
+                if let Err(err) = self.process_members_provider_event(event).await {
+                    error!("Error processing rendezvous event: {:?}", err);
+                }
+            }
+            GroupBehaviourEvent::Kademlia(ev) => {
+                if let Err(err) = self.process_kad_event(ev).await {
+                    error!("Error processing kademlia event: {:?}", err);
+                }
+            } // GroupBehaviourEvent::Ping(_) => {}
+        }
+        Ok(())
+    }
+
+    async fn process_gossipsub_event(&mut self, event: gossipsub::Event) -> anyhow::Result<()> {
+        match event {
+            gossipsub::Event::Message {
+                propagation_source: _,
+                message_id: _,
+                message,
+            } => {
+                let msg: EphemeraMessage = serde_json::from_slice(&message.data[..])?;
+                self.to_ephemera_tx
+                    .send_network_event(NetworkEvent::EphemeraMessage(msg.into()))
+                    .await?;
+            }
+
+            gossipsub::Event::Subscribed { peer_id, topic } => {
+                trace!("Peer {peer_id:?} subscribed to topic {topic:?}");
+            }
+            gossipsub::Event::Unsubscribed { peer_id, topic } => {
+                trace!("Peer {peer_id:?} unsubscribed from topic {topic:?}");
+            }
+            gossipsub::Event::GossipsubNotSupported { peer_id } => {
+                trace!("Peer {peer_id:?} does not support gossipsub");
+            }
+        }
+        Ok(())
+    }
+
+    async fn process_request_response(
+        &mut self,
+        event: request_response::Event<RbMsg, RbMsgResponse>,
+    ) -> anyhow::Result<()> {
+        match event {
+            request_response::Event::Message { peer, message } => match message {
+                request_response::Message::Request {
+                    request_id: _,
+                    request,
+                    channel,
+                } => {
+                    let rb_id = request.id.clone();
+                    trace!("Received request {:?}", request);
+                    self.to_ephemera_tx
+                        .send_network_event(NetworkEvent::BroadcastMessage(request.into()))
+                        .await?;
+                    if let Err(err) = self
+                        .swarm
+                        .behaviour_mut()
+                        .request_response
+                        .send_response(channel, RbMsgResponse::new(rb_id))
+                    {
+                        error!("Error sending response: {:?}", err);
+                    }
+                }
+                request_response::Message::Response {
+                    request_id,
+                    response,
+                } => {
+                    trace!("Received response {response:?} from peer: {peer:?}, request_id: {request_id:?}",);
+                }
+            },
+            request_response::Event::OutboundFailure {
+                peer,
+                request_id,
+                error,
+            } => {
+                error!("Outbound failure: {error:?}, peer:{peer:?}, request_id:{request_id:?}",);
+            }
+            request_response::Event::InboundFailure {
+                peer,
+                request_id,
+                error,
+            } => {
+                error!("Inbound failure: {error:?}, peer:{peer:?}, request_id:{request_id:?}",);
+            }
+            request_response::Event::ResponseSent { peer, request_id } => {
+                trace!("Response sent to peer: {peer:?}, {request_id:?}",);
+            }
+        }
+        Ok(())
+    }
+
+    async fn process_members_provider_event(
+        &mut self,
+        event: behaviours::membership::behaviour::Event,
+    ) -> anyhow::Result<()> {
+        match event {
+            behaviours::membership::behaviour::Event::PeersUpdated(peers_ids) => {
+                info!("Peers updated: {:?}", peers_ids);
+
+                let local_peer_id = *self.swarm.local_peer_id();
+                for peer_id in peers_ids {
+                    if peer_id == local_peer_id {
+                        continue;
+                    }
+                    //FIXME: deprecated
+                    #[allow(deprecated)]
+                    let address = self
+                        .swarm
+                        .behaviour_mut()
+                        .members_provider
+                        .addresses_of_peer(&peer_id);
+                    if let Some(address) = address.first() {
+                        self.swarm
+                            .behaviour_mut()
+                            .kademlia
+                            .add_address(&peer_id, address.clone());
+                    }
+                }
+
+                let query_id = self
+                    .swarm
+                    .behaviour_mut()
+                    .kademlia
+                    .get_closest_peers(libp2p::PeerId::random());
+                debug!("Neighbours: {:?}", query_id);
+            }
+            behaviours::membership::behaviour::Event::PeerUpdatePending => {
+                info!("Peer update pending");
+            }
+            behaviours::membership::behaviour::Event::LocalRemoved(peers_ids) => {
+                //TODO: should pause all network block and message activities...?
+                let peers_ids = peers_ids.into_iter().map(Into::into).collect();
+                let update = NetworkEvent::GroupUpdate(LocalPeerRemoved(peers_ids));
+                self.to_ephemera_tx.send_network_event(update).await?;
+            }
+            behaviours::membership::behaviour::Event::NotEnoughPeers(peers_ids) => {
+                //TODO: should pause all network block and message activities...?
+                let peers_ids = peers_ids.into_iter().map(Into::into).collect();
+                let update = NetworkEvent::GroupUpdate(NotEnoughPeers(peers_ids));
+                self.to_ephemera_tx.send_network_event(update).await?;
+            }
+        }
+        Ok(())
+    }
+
+    async fn process_kad_event(&mut self, event: kad::KademliaEvent) -> anyhow::Result<()> {
+        match event {
+            kad::KademliaEvent::OutboundQueryProgressed {
+                id,
+                result,
+                stats,
+                step,
+            } => {
+                trace!(
+                    "Outbound query progressed: id:{:?}, result:{:?}, stats:{:?}, step:{:?}",
+                    id,
+                    result,
+                    stats,
+                    step
+                );
+                match result {
+                    kad::QueryResult::GetClosestPeers(gcp) => {
+                        self.process_closest_peers(gcp).await?;
+                    }
+                    kad::QueryResult::GetRecord(get_res) => {
+                        self.process_get_record(get_res).await?;
+                    }
+                    kad::QueryResult::Bootstrap(bt) => {
+                        trace!("Bootstrap: {:?}", bt);
+                    }
+                    kad::QueryResult::GetProviders(gp) => {
+                        trace!("GetProviders: {:?}", gp);
+                    }
+                    kad::QueryResult::StartProviding(sp) => {
+                        trace!("StartProviding: {:?}", sp);
+                    }
+                    kad::QueryResult::RepublishProvider(rp) => {
+                        trace!("RepublishProvider: {:?}", rp);
+                    }
+                    kad::QueryResult::PutRecord(pr) => {
+                        trace!("PutRecord: {:?}", pr);
+                    }
+                    kad::QueryResult::RepublishRecord(rr) => {
+                        trace!("RepublishRecord: {:?}", rr);
+                    }
+                }
+            }
+
+            kad::KademliaEvent::InboundRequest { request } => {
+                trace!("Inbound request: {:?}", request);
+            }
+
+            kad::KademliaEvent::RoutingUpdated {
+                peer: peer_id,
+                is_new_peer: _,
+                addresses,
+                bucket_range: _,
+                old_peer: _,
+            } => {
+                trace!("Routing updated: peer:{peer_id}, addresses:{addresses:?}",);
+            }
+            kad::KademliaEvent::UnroutablePeer { peer } => {
+                trace!("Unroutable peer: {:?}", peer);
+            }
+            kad::KademliaEvent::RoutablePeer { peer, address } => {
+                trace!("Routable peer: {:?}, address: {:?}", peer, address);
+            }
+            kad::KademliaEvent::PendingRoutablePeer { peer, address } => {
+                trace!("Pending routable peer: {:?}, address: {:?}", peer, address);
+            }
+        }
+        Ok(())
+    }
+
+    async fn process_get_record(&mut self, get_res: GetRecordResult) -> anyhow::Result<()> {
+        trace!("GetRecord: {:?}", get_res);
+        match get_res {
+            Ok(ok) => match ok {
+                kad::GetRecordOk::FoundRecord(fr) => {
+                    let record = fr.record;
+                    let event = NetworkEvent::QueryDhtResponse {
+                        key: record.key.to_vec(),
+                        value: record.value,
+                    };
+                    self.to_ephemera_tx.send_network_event(event).await?;
+                }
+                kad::GetRecordOk::FinishedWithNoAdditionalRecord { .. } => {
+                    trace!("FinishedWithNoAdditionalRecord");
+                }
+            },
+            Err(err) => {
+                trace!("Not getting record: {:?}", err);
+            }
+        }
+        Ok(())
+    }
+
+    async fn process_closest_peers(&mut self, gcp: GetClosestPeersResult) -> anyhow::Result<()> {
+        trace!("GetClosestPeers: {:?}", gcp);
+        //TODO: we need also to make sure that we have enough peers
+        // (Repeat if not enough, may need to wait network to stabilize)
+        match gcp {
+            Ok(cp) => {
+                if cp.peers.is_empty() {
+                    log::warn!("No peers found");
+                    return Ok(());
+                }
+
+                let gossipsub = &mut self.swarm.behaviour_mut().gossipsub;
+                for peer_id in cp.peers {
+                    gossipsub.add_explicit_peer(&peer_id);
+                }
+
+                let active_peers = self
+                    .swarm
+                    .behaviour_mut()
+                    .members_provider
+                    .active_peer_ids_with_local();
+                let active_peers = active_peers
+                    .into_iter()
+                    .map(Into::into)
+                    .collect::<HashSet<_>>();
+                let group_update =
+                    NetworkEvent::GroupUpdate(GroupChangeEvent::PeersUpdated(active_peers));
+                self.to_ephemera_tx.send_network_event(group_update).await?;
+            }
+            Err(err) => {
+                error!("Error getting closest peers: {:?}", err);
+            }
+        }
+        Ok(())
+    }
+
+    fn send_broadcast_message(&mut self, msg: &RbMsg) {
+        trace!("Sending broadcast message: {:?}", msg);
+        let local_peer_id = *self.swarm.local_peer_id();
+        let behaviours = self.swarm.behaviour_mut();
+        for peer in behaviours.members_provider.active_peer_ids() {
+            trace!("Sending broadcast message: {:?} to peer: {peer:?}", msg.id,);
+            if *peer == local_peer_id {
+                continue;
+            }
+            behaviours.request_response.send_request(peer, msg.clone());
+        }
+    }
+
+    fn send_ephemera_message(&mut self, msg: &EphemeraMessage) {
+        trace!("Sending Ephemera message: {:?}", msg);
+        match msg.encode() {
+            Ok(vec) => {
+                let topic = self.ephemera_msg_topic.clone();
+                if let Err(err) = self.swarm.behaviour_mut().gossipsub.publish(topic, vec) {
+                    error!("Error publishing message: {}", err);
+                }
+            }
+            Err(err) => {
+                error!("Error serializing message: {}", err);
+            }
+        }
+    }
+
+    //Just logging
+    #[allow(clippy::too_many_lines)]
+    fn process_other_swarm_events<E>(swarm_event: SwarmEvent<GroupBehaviourEvent, E>) {
+        match swarm_event {
+            SwarmEvent::ConnectionEstablished {
+                peer_id,
+                endpoint,
+                num_established,
+                concurrent_dial_errors,
+                established_in,
+            } => {
+                trace!("Connection established: peer_id:{:?}, endpoint:{:?}, num_established:{:?}, concurrent_dial_errors:{:?}, established_in:{:?}", peer_id, endpoint, num_established, concurrent_dial_errors, established_in);
+            }
+            SwarmEvent::ConnectionClosed {
+                peer_id,
+                endpoint,
+                num_established,
+                cause: _,
+            } => {
+                trace!(
+                    "Connection closed: peer_id:{:?}, endpoint:{:?}, num_established:{:?}",
+                    peer_id,
+                    endpoint,
+                    num_established
+                );
+            }
+            SwarmEvent::IncomingConnection {
+                local_addr,
+                send_back_addr,
+            } => {
+                trace!(
+                    "Incoming connection: local_addr:{:?}, send_back_addr:{:?}",
+                    local_addr,
+                    send_back_addr
+                );
+            }
+            SwarmEvent::IncomingConnectionError {
+                local_addr,
+                send_back_addr,
+                error,
+            } => {
+                trace!(
+                    "Incoming connection error: local_addr:{:?}, send_back_addr:{:?}, error:{:?}",
+                    local_addr,
+                    send_back_addr,
+                    error
+                );
+            }
+            SwarmEvent::OutgoingConnectionError { peer_id, error } => {
+                trace!(
+                    "Outgoing connection error: peer_id:{:?}, error:{:?}",
+                    peer_id,
+                    error
+                );
+            }
+            #[allow(deprecated)]
+            SwarmEvent::BannedPeer { peer_id, endpoint } => {
+                trace!(
+                    "Banned peer: peer_id:{:?}, endpoint:{:?}",
+                    peer_id,
+                    endpoint
+                );
+            }
+            SwarmEvent::NewListenAddr {
+                listener_id,
+                address,
+            } => {
+                trace!(
+                    "New listen address: listener_id:{:?}, address:{:?}",
+                    listener_id,
+                    address
+                );
+            }
+            SwarmEvent::ExpiredListenAddr {
+                listener_id,
+                address,
+            } => {
+                trace!(
+                    "Expired listen address: listener_id:{:?}, address:{:?}",
+                    listener_id,
+                    address
+                );
+            }
+            SwarmEvent::ListenerClosed {
+                listener_id,
+                addresses,
+                reason,
+            } => {
+                trace!(
+                    "Listener closed: listener_id:{:?}, addresses:{:?}, reason:{:?}",
+                    listener_id,
+                    addresses,
+                    reason
+                );
+            }
+            SwarmEvent::ListenerError { listener_id, error } => {
+                trace!(
+                    "Listener error: listener_id:{:?}, error:{:?}",
+                    listener_id,
+                    error
+                );
+            }
+            SwarmEvent::Dialing(peer_id) => {
+                trace!("Dialing: {peer_id:?}",);
+            }
+
+            SwarmEvent::Behaviour(_) => {
+                trace!("Unexpected behaviour event");
+            }
+        }
+    }
+}

ephemera/src/network/members/mod.rs

@@ -0,0 +1,252 @@
+use std::fmt::Display;
+use std::future::Future;
+use std::io::Write;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use futures_util::{future, FutureExt};
+use log::error;
+use nym_ephemera_common::types::JsonPeerInfo;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::crypto::PublicKey;
+use crate::network::{Address, Peer};
+use crate::peer::PeerId;
+
+/// Information about an Ephemera peer.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct PeerInfo {
+    /// The cosmos address of the peer, used in interacting with the chain.
+    pub cosmos_address: String,
+    /// The address of the peer.
+    /// Expected formats:
+    /// 1. `<IP>:<PORT>`
+    /// 2. `/ip4/<IP>/tcp/<PORT>` - this is the format used by libp2p multiaddr
+    pub address: String,
+    /// The public key of the peer. It uniquely identifies the peer.
+    /// Public key is used to derive the peer id.
+    pub pub_key: PublicKey,
+}
+
+impl Display for PeerInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "cosmos address {}, address {}, public key {}",
+            self.cosmos_address, self.address, self.pub_key
+        )
+    }
+}
+
+impl TryFrom<PeerInfo> for Peer {
+    type Error = anyhow::Error;
+
+    fn try_from(value: PeerInfo) -> std::result::Result<Self, Self::Error> {
+        let address: Address = value.address.parse()?;
+        let public_key = value.pub_key;
+        Ok(Self {
+            cosmos_address: value.cosmos_address,
+            address,
+            public_key: public_key.clone(),
+            peer_id: PeerId::from_public_key(&public_key),
+        })
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum ProviderError {
+    #[error("ResourceUnavailable: {0}")]
+    ResourceUnavailable(String),
+    #[error("MembersProvider: {0}")]
+    MembersProvider(#[from] anyhow::Error),
+    #[error("Could not get peers - {0}")]
+    GetPeers(String),
+}
+
+pub type Result<T> = std::result::Result<T, ProviderError>;
+
+/// A membership provider that does nothing.
+/// Might be useful for testing.
+pub struct DummyMembersProvider;
+
+#[allow(clippy::missing_errors_doc, clippy::unused_async)]
+impl DummyMembersProvider {
+    pub async fn empty_peers_list() -> Result<Vec<PeerInfo>> {
+        Ok(vec![])
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum ConfigMembersProviderError {
+    #[error("ConfigDoesNotExist: '{0}'")]
+    NotExist(String),
+    #[error("ParsingFailed: {0}")]
+    ParsingFailed(#[from] config::ConfigError),
+    #[error("TomlError: {0}")]
+    TomlError(#[from] toml::ser::Error),
+    #[error("IoError: {0}")]
+    IoError(#[from] std::io::Error),
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct PeerSetting {
+    /// The cosmos address of the peer, used in interacting with the chain.
+    pub cosmos_address: String,
+    /// The address of the peer.
+    /// Expected formats:
+    /// 1. `<IP>:<PORT>`
+    /// 2. `/ip4/<IP>/tcp/<PORT>` - this is the format used by libp2p multiaddr
+    pub address: String,
+    ///Serialized public key.
+    ///
+    /// # Converting to string and back example
+    ///```
+    /// use ephemera::crypto::{EphemeraKeypair, EphemeraPublicKey, Keypair, PublicKey};
+    ///
+    /// let public_key = Keypair::generate(None).public_key();
+    ///
+    /// let public_key_str = public_key.to_string();
+    ///
+    /// let public_key_parsed = public_key_str.parse::<PublicKey>().unwrap();
+    ///
+    /// assert_eq!(public_key, public_key_parsed);
+    /// ```
+    pub public_key: String,
+}
+
+impl TryFrom<PeerSetting> for PeerInfo {
+    type Error = anyhow::Error;
+
+    fn try_from(setting: PeerSetting) -> std::result::Result<Self, Self::Error> {
+        let pub_key = setting.public_key.parse::<PublicKey>()?;
+        Ok(PeerInfo {
+            cosmos_address: setting.cosmos_address,
+            address: setting.address,
+            pub_key,
+        })
+    }
+}
+
+///[`ProviderFut`] that reads the peers from a toml config file.
+///
+/// # Configuration example
+/// ```toml
+/// [[peers]]
+/// name = "node1"
+/// address = "/ip4/127.0.0.1/tcp/3000"
+/// pub_key = "4XTTMEghav9LZThm6opUaHrdGEEYUkrfkakVg4VAetetBZDWJ"
+///
+/// [[peers]]
+/// name = "node2"
+/// address = "/ip4/127.0.0.1/tcp/3001"
+/// pub_key = "4XTTMFQt2tgNRmwRgEAaGQe2NXygsK6Vr3pkuBfYezhDfoVty"
+/// ```
+pub struct ConfigMembersProvider {
+    config_location: PathBuf,
+}
+
+impl ConfigMembersProvider {
+    /// Creates a new [`ConfigMembersProvider`] instance.
+    ///
+    /// # Arguments
+    /// * `path` - Path to the peers toml config file.
+    ///
+    /// # Errors
+    /// Returns [`ConfigMembersProviderError::NotExist`] if the file does not exist.
+    /// Returns [`ConfigMembersProviderError::ParsingFailed`] if the file is not a valid members file.
+    pub fn init<I: Into<PathBuf>>(
+        path: I,
+    ) -> std::result::Result<Self, ConfigMembersProviderError> {
+        let path_buf = path.into();
+        if !path_buf.exists() {
+            return Err(ConfigMembersProviderError::NotExist(
+                path_buf.to_string_lossy().to_string(),
+            ));
+        }
+
+        let provider = Self {
+            config_location: path_buf,
+        };
+
+        if provider.read_config().is_err() {
+            return Err(ConfigMembersProviderError::ParsingFailed(
+                config::ConfigError::Message("Failed to parse config".to_string()),
+            ));
+        }
+
+        Ok(provider)
+    }
+
+    pub(crate) fn read_config(&self) -> Result<Vec<PeerInfo>> {
+        let config_peers = ConfigPeers::try_load(self.config_location.clone())
+            .map_err(|err| anyhow::anyhow!(err))?;
+
+        let peers = config_peers
+            .peers
+            .iter()
+            .map(|peer| PeerInfo::try_from(peer.clone()))
+            .collect::<anyhow::Result<Vec<PeerInfo>>>()?;
+        Ok(peers)
+    }
+}
+
+impl Future for ConfigMembersProvider {
+    type Output = Result<Vec<PeerInfo>>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        future::ready(self.read_config()).poll_unpin(cx)
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct ConfigPeers {
+    peers: Vec<PeerSetting>,
+}
+
+impl ConfigPeers {
+    pub(crate) fn new(peers: Vec<PeerSetting>) -> Self {
+        Self { peers }
+    }
+
+    pub(crate) fn try_load<I: Into<PathBuf>>(
+        path: I,
+    ) -> std::result::Result<ConfigPeers, ConfigMembersProviderError> {
+        let path = path.into();
+        let config = config::Config::builder()
+            .add_source(config::File::from(path))
+            .build()?;
+
+        config.try_deserialize().map_err(Into::into)
+    }
+
+    pub(crate) fn try_write<I: Into<PathBuf>>(
+        &self,
+        path: I,
+    ) -> std::result::Result<(), ConfigMembersProviderError> {
+        let config = toml::to_string(&self)?;
+
+        let config = format!(
+            "#This file is generated by cli and automatically overwritten every time when cli is §\n{config}",
+        );
+
+        let mut file = std::fs::File::create(path.into())?;
+        file.write_all(config.as_bytes())?;
+
+        Ok(())
+    }
+}
+
+impl TryFrom<JsonPeerInfo> for PeerInfo {
+    type Error = anyhow::Error;
+
+    fn try_from(json_peer_info: JsonPeerInfo) -> std::result::Result<Self, Self::Error> {
+        let pub_key = json_peer_info.public_key.parse::<PublicKey>()?;
+        Ok(PeerInfo {
+            cosmos_address: json_peer_info.cosmos_address.to_string(),
+            address: json_peer_info.ip_address,
+            pub_key,
+        })
+    }
+}

ephemera/src/network/mod.rs

@@ -0,0 +1,237 @@
+use std::fmt::Display;
+use std::net::IpAddr;
+use std::str::FromStr;
+
+use ::libp2p::{multiaddr::Protocol, Multiaddr};
+use libp2p_identity::PeerId as Libp2pPeerId;
+use log::info;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::crypto::PublicKey;
+
+pub(crate) mod libp2p;
+pub(crate) mod members;
+
+pub(crate) type PeerIdType = Libp2pPeerId;
+
+#[derive(Debug, Error)]
+pub enum PeerIdError {
+    #[error("Invalid peer ID: {0}")]
+    InvalidPeerId(String),
+}
+
+/// Unique identifier of a peer.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct PeerId(pub(crate) PeerIdType);
+
+impl PeerId {
+    #[must_use]
+    pub fn random() -> Self {
+        Self(PeerIdType::random())
+    }
+
+    /// Returns the internal representation of the peer ID.
+    pub(crate) fn inner(&self) -> &PeerIdType {
+        &self.0
+    }
+
+    /// Returns a raw representation of the peer ID.
+    #[must_use]
+    pub fn to_bytes(&self) -> Vec<u8> {
+        self.0.to_bytes()
+    }
+
+    /// Returns a peer ID from a raw representation.
+    ///
+    /// # Returns
+    /// A `PeerId` if the bytes are valid.
+    ///
+    /// # Errors
+    /// An error if input has wrong format. This function is reverse to [`PeerId::to_bytes`].
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, PeerIdError> {
+        Ok(Self(PeerIdType::from_bytes(bytes).map_err(|e| {
+            PeerIdError::InvalidPeerId(format!("Invalid peer ID: {e}"))
+        })?))
+    }
+
+    /// Builds a `PeerId` from a public key.
+    #[must_use]
+    pub fn from_public_key(public_key: &PublicKey) -> Self {
+        Self(PeerIdType::from_public_key(public_key.inner()))
+    }
+}
+
+impl From<PeerId> for libp2p_identity::PeerId {
+    fn from(peer_id: PeerId) -> Self {
+        peer_id.0
+    }
+}
+
+impl From<libp2p_identity::PeerId> for PeerId {
+    fn from(peer_id: libp2p_identity::PeerId) -> Self {
+        Self(peer_id)
+    }
+}
+
+impl Display for PeerId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+pub trait ToPeerId {
+    fn peer_id(&self) -> PeerId;
+}
+
+/// A peer of the network.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(crate) struct Peer {
+    /// The peer's ID. It identifies the peer uniquely and is derived from its public key.
+    ///
+    /// # Deriving PeerId from PublicKey example
+    ///
+    /// ```
+    /// use ephemera::crypto::{EphemeraKeypair, Keypair, PublicKey};
+    /// use ephemera::peer::{PeerId, ToPeerId};
+    ///
+    /// let public_key = Keypair::generate(None).public_key();
+    ///
+    /// let peer_id = PeerId::from_public_key(&public_key);
+    ///
+    /// assert_eq!(peer_id, public_key.peer_id());
+    ///
+    /// ```
+    pub peer_id: PeerId,
+    /// The peer's public key. It matches PeerId.
+    pub public_key: PublicKey,
+    /// The peer's address.
+    pub address: Address,
+    /// The cosmos address of the peer, used in interacting with the chain.
+    pub cosmos_address: String,
+}
+
+#[derive(Error, Debug)]
+pub enum AddressError {
+    #[error("Failed to parse address: {0}")]
+    ParsingError(String),
+}
+
+/// Ephemera node address.
+///
+/// Supported formats:
+/// 1. `<IP>:<PORT>`
+/// 2. `/ip4/<IP>/tcp/<PORT>` - this is format used by libp2p multiaddr.
+/// 3. `/dns4/<NAME>/tcp/<PORT>` - this is format used by libp2p multiaddr.
+/// See [libp2p/multiaddress](https://github.com/libp2p/specs/blob/master/addressing/README.md) for more details.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Address(pub Multiaddr);
+
+impl Address {
+    pub fn inner(&self) -> &Multiaddr {
+        &self.0
+    }
+}
+
+impl From<Multiaddr> for Address {
+    fn from(multiaddr: Multiaddr) -> Self {
+        Self(multiaddr)
+    }
+}
+
+impl FromStr for Address {
+    type Err = AddressError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let address: Option<Multiaddr> = match Multiaddr::from_str(s) {
+            Ok(multiaddr) => Some(multiaddr),
+            Err(err) => {
+                info!("Failed to parse multiaddr: {}", err);
+                None
+            }
+        };
+
+        let multi_address = address.or_else(|| match std::net::SocketAddr::from_str(s) {
+            Ok(sa) => {
+                let mut multiaddr = Multiaddr::empty();
+                match sa {
+                    std::net::SocketAddr::V4(v4) => {
+                        multiaddr.push(Protocol::Ip4(*v4.ip()));
+                        multiaddr.push(Protocol::Tcp(v4.port()));
+                    }
+                    std::net::SocketAddr::V6(v6) => {
+                        multiaddr.push(Protocol::Ip6(*v6.ip()));
+                        multiaddr.push(Protocol::Tcp(v6.port()));
+                    }
+                }
+
+                Some(multiaddr)
+            }
+            Err(err) => {
+                info!("Failed to parse socket addr: {err}");
+                None
+            }
+        });
+
+        match multi_address {
+            Some(multi_address) => Ok(Self(multi_address)),
+            None => Err(AddressError::ParsingError(s.to_string())),
+        }
+    }
+}
+
+impl TryFrom<Address> for (IpAddr, u16) {
+    type Error = std::io::Error;
+
+    fn try_from(addr: Address) -> Result<Self, Self::Error> {
+        let mut multiaddr = addr.0;
+        if let Some(Protocol::Tcp(port)) = multiaddr.pop() {
+            if let Some(Protocol::Ip4(ip)) = multiaddr.pop() {
+                return Ok((IpAddr::V4(ip), port));
+            }
+        }
+        Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "invalid address",
+        ))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_parse_multiaddr() {
+        "/ip4/127.0.0.1/tcp/1234".parse::<Address>().unwrap();
+    }
+
+    #[test]
+    fn test_parse_ip_port() {
+        "127.0.0.1:1234".parse::<Address>().unwrap();
+    }
+
+    #[test]
+    fn test_fail_parse_multiaddr_without_port() {
+        let result = "/ip4/127.0.0.1/tcp/".parse::<Address>();
+        assert!(matches!(result, Err(AddressError::ParsingError(_))));
+    }
+
+    #[test]
+    fn test_fail_parse_multiaddr_without_ip() {
+        let result = "/ip4//tcp/1234".parse::<Address>();
+        assert!(matches!(result, Err(AddressError::ParsingError(_))));
+    }
+
+    #[test]
+    fn test_fail_parse_ip_port_without_port() {
+        let result = "127.0.0.1".parse::<Address>();
+        assert!(matches!(result, Err(AddressError::ParsingError(_))));
+    }
+
+    #[test]
+    fn test_fail_parse_ip_port_without_ip() {
+        let result = "1234".parse::<Address>();
+        assert!(matches!(result, Err(AddressError::ParsingError(_))));
+    }
+}

ephemera/src/storage/mod.rs

@@ -0,0 +1,65 @@
+//! # Database
+//!
+//! It supports `SqlLite` and `RocksDB`.
+//!
+//! ## `RocksDB`
+//!
+//! `SqlLite` is used by default.
+//!
+//! ## `SqlLite`
+//!
+//! To use `SqlLite`, you need to compile with the `sqlite_storage` feature and with `--no-default-features` flag.
+
+use std::collections::HashSet;
+
+use thiserror::Error;
+
+use crate::block::types::block::Block;
+use crate::peer::PeerId;
+use crate::utilities::crypto::Certificate;
+use crate::utilities::merkle::MerkleTree;
+
+#[cfg(feature = "rocksdb_storage")]
+pub(crate) mod rocksdb;
+
+#[cfg(feature = "sqlite_storage")]
+pub(crate) mod sqlite;
+
+pub(crate) type Result<T> = std::result::Result<T, DatabaseError>;
+
+#[derive(Error, Debug)]
+pub(crate) enum DatabaseError {
+    //Pessimistically assume that most database errors are fatal and unrecoverable.
+    #[error("Database failure: {0}")]
+    DatabaseFailure(#[from] anyhow::Error),
+}
+
+pub(crate) trait EphemeraDatabase: Send {
+    /// Returns block by its id. Block ids are generated by Ephemera
+    fn get_block_by_hash(&self, block_hash: &str) -> Result<Option<Block>>;
+
+    /// Returns last committed/finalised block.
+    fn get_last_block(&self) -> Result<Option<Block>>;
+
+    /// Returns block by its height
+    fn get_block_by_height(&self, height: u64) -> Result<Option<Block>>;
+
+    /// Returns block certificates.
+    ///
+    /// Certificates were created as part of broadcast protocol and signed by peers who participated.
+    fn get_block_certificates(&self, block_hash: &str) -> Result<Option<Vec<Certificate>>>;
+
+    /// Returns peers who participated in block broadcast.
+    fn get_block_broadcast_group(&self, block_hash: &str) -> Result<Option<Vec<PeerId>>>;
+
+    /// Stores block and its signatures
+    fn store_block(
+        &mut self,
+        block: &Block,
+        certificates: HashSet<Certificate>,
+        members: HashSet<PeerId>,
+    ) -> Result<()>;
+
+    /// Returns block merkle tree
+    fn get_block_merkle_tree(&self, block_hash: &str) -> Result<Option<MerkleTree>>;
+}

ephemera/src/storage/rocksdb/mod.rs

@@ -0,0 +1,125 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use log::info;
+use rocksdb::{TransactionDB, TransactionDBOptions};
+
+use crate::block::types::block::Block;
+use crate::config::DatabaseConfiguration;
+use crate::peer::PeerId;
+use crate::storage::rocksdb::query::Database;
+use crate::storage::rocksdb::store::DbStore;
+use crate::storage::EphemeraDatabase;
+use crate::storage::Result;
+use crate::utilities::crypto::Certificate;
+use crate::utilities::merkle::MerkleTree;
+
+pub(crate) mod query;
+pub(crate) mod store;
+
+pub(crate) struct RocksDbStorage {
+    pub(crate) db_store: DbStore,
+    pub(crate) db_query: Database,
+}
+
+const PREFIX_LAST_BLOCK_KEY: &str = "last_block";
+const PREFIX_BLOCK_HASH: &str = "block_hash";
+const PREFIX_BLOCK_HEIGHT: &str = "block_height";
+const PREFIX_CERTIFICATES: &str = "block_certificates";
+const PREFIX_MEMBERS: &str = "block_members";
+const MERKLE_TREE: &str = "merkle_tree";
+
+impl RocksDbStorage {
+    pub fn open(db_conf: &DatabaseConfiguration) -> Result<Self> {
+        info!("Opening RocksDB database at {}", db_conf.rocksdb_path);
+
+        let mut options = rocksdb::Options::default();
+        options.create_if_missing(db_conf.create_if_not_exists);
+
+        let db = TransactionDB::open(
+            &options,
+            &TransactionDBOptions::default(),
+            db_conf.rocksdb_path.clone(),
+        )
+        .map_err(|err| anyhow::anyhow!(err))?;
+
+        let db = Arc::new(db);
+        let db_store = DbStore::new(db.clone());
+        let db_query = Database::new(db);
+        let storage = Self { db_store, db_query };
+
+        info!("Opened RocksDB database at {}", db_conf.rocksdb_path);
+        Ok(storage)
+    }
+}
+
+impl EphemeraDatabase for RocksDbStorage {
+    fn get_block_by_hash(&self, block_id: &str) -> Result<Option<Block>> {
+        self.db_query
+            .get_block_by_hash(block_id)
+            .map_err(Into::into)
+    }
+
+    fn get_last_block(&self) -> Result<Option<Block>> {
+        self.db_query.get_last_block().map_err(Into::into)
+    }
+
+    fn get_block_by_height(&self, height: u64) -> Result<Option<Block>> {
+        self.db_query
+            .get_block_by_height(height)
+            .map_err(Into::into)
+    }
+
+    fn get_block_certificates(&self, block_id: &str) -> Result<Option<Vec<Certificate>>> {
+        self.db_query
+            .get_block_certificates(block_id)
+            .map_err(Into::into)
+    }
+
+    fn get_block_broadcast_group(&self, block_id: &str) -> Result<Option<Vec<PeerId>>> {
+        self.db_query
+            .get_block_broadcast_group(block_id)
+            .map_err(Into::into)
+    }
+
+    fn store_block(
+        &mut self,
+        block: &Block,
+        certificates: HashSet<Certificate>,
+        members: HashSet<PeerId>,
+    ) -> Result<()> {
+        self.db_store
+            .store_block(block, certificates, members)
+            .map_err(Into::into)
+    }
+
+    fn get_block_merkle_tree(&self, block_hash: &str) -> Result<Option<MerkleTree>> {
+        self.db_query
+            .get_block_merkle_tree(block_hash)
+            .map_err(Into::into)
+    }
+}
+
+fn block_hash_key(block_hash: &str) -> String {
+    format!("{PREFIX_BLOCK_HASH}:{block_hash}")
+}
+
+fn block_height_key(height: &u64) -> String {
+    format!("{PREFIX_BLOCK_HEIGHT}:{height}")
+}
+
+fn last_block_key() -> String {
+    PREFIX_LAST_BLOCK_KEY.to_string()
+}
+
+fn certificates_key(block_hash: &str) -> String {
+    format!("{PREFIX_CERTIFICATES}:{block_hash}",)
+}
+
+fn members_key(block_hash: &str) -> String {
+    format!("{PREFIX_MEMBERS}:{block_hash}",)
+}
+
+fn merkle_tree_key(block_hash: &str) -> String {
+    format!("{MERKLE_TREE}:{block_hash}",)
+}

ephemera/src/storage/rocksdb/query.rs

@@ -0,0 +1,118 @@
+use std::sync::Arc;
+
+use log::trace;
+use rocksdb::TransactionDB;
+
+use crate::block::types::block::Block;
+use crate::network::PeerId;
+use crate::storage::rocksdb::{
+    block_hash_key, block_height_key, certificates_key, last_block_key, members_key,
+    merkle_tree_key,
+};
+use crate::utilities::crypto::Certificate;
+use crate::utilities::merkle::MerkleTree;
+
+pub struct Database {
+    database: Arc<TransactionDB>,
+}
+
+impl Database {
+    #[allow(dead_code)]
+    pub fn new(db: Arc<TransactionDB>) -> Database {
+        Database { database: db }
+    }
+
+    pub(crate) fn get_block_by_hash(&self, block_hash: &str) -> anyhow::Result<Option<Block>> {
+        trace!("Getting block by id: {:?}", block_hash);
+
+        let block_hash_key = block_hash_key(block_hash);
+
+        let block = if let Some(block) = self.database.get(block_hash_key)? {
+            let block = serde_json::from_slice::<Block>(&block)?;
+            trace!("Found block: {}", block.header);
+            Some(block)
+        } else {
+            trace!("Didn't find block");
+            None
+        };
+        Ok(block)
+    }
+
+    pub(crate) fn get_last_block(&self) -> anyhow::Result<Option<Block>> {
+        trace!("Getting last block");
+
+        if let Some(block_hash) = self.database.get(last_block_key())? {
+            let block_hash = String::from_utf8(block_hash)?;
+            self.get_block_by_hash(&block_hash)
+        } else {
+            trace!("Unable to get last block");
+            Ok(None)
+        }
+    }
+
+    pub(crate) fn get_block_by_height(&self, height: u64) -> anyhow::Result<Option<Block>> {
+        trace!("Getting block by height: {}", height);
+
+        if let Some(block_hash) = self.database.get(block_height_key(&height))? {
+            let block_hash = String::from_utf8(block_hash)?;
+            self.get_block_by_hash(&block_hash)
+        } else {
+            trace!("Didn't find block");
+            Ok(None)
+        }
+    }
+
+    pub(crate) fn get_block_certificates(
+        &self,
+        block_hash: &str,
+    ) -> anyhow::Result<Option<Vec<Certificate>>> {
+        trace!("Getting block signatures: {}", block_hash);
+
+        let certificates_key = certificates_key(block_hash);
+
+        if let Some(certificates) = self.database.get(certificates_key)? {
+            let certificates: Vec<Certificate> = serde_json::from_slice(&certificates)?;
+            trace!("Found certificates: {:?}", certificates);
+            Ok(Some(certificates))
+        } else {
+            trace!("Didn't find signatures");
+            Ok(None)
+        }
+    }
+
+    pub(crate) fn get_block_broadcast_group(
+        &self,
+        block_hash: &str,
+    ) -> anyhow::Result<Option<Vec<PeerId>>> {
+        trace!("Getting block broadcast group: {}", block_hash);
+
+        let members_key = members_key(block_hash);
+
+        if let Some(members) = self.database.get(members_key)? {
+            let members: Vec<PeerId> = serde_json::from_slice(&members)?;
+            trace!("Found members: {:?}", members);
+            Ok(Some(members))
+        } else {
+            trace!("Didn't find members");
+            Ok(None)
+        }
+    }
+
+    pub(crate) fn get_block_merkle_tree(
+        &self,
+        block_hash: &str,
+    ) -> anyhow::Result<Option<MerkleTree>> {
+        trace!("Getting block merkle tree: {}", block_hash);
+
+        let merkle_tree_key = merkle_tree_key(block_hash);
+
+        if let Some(merkle_tree) = self.database.get(merkle_tree_key)? {
+            let merkle_tree: MerkleTree = serde_json::from_slice(&merkle_tree)?;
+            trace!("Found merkle tree: {:?}", merkle_tree);
+            Ok(Some(merkle_tree))
+        } else {
+            trace!("Didn't find merkle tree");
+            Ok(None)
+        }
+    }
+}