rust-sdk: handle surb storage (#2870)

* clients: make surb storage more flexible

- in the rust-sdk we make the surb storage generic and pluggable, with
  the fs_backend the default.
- make it possible to disable fs_backend at runtime

* Add comment

* changelog: add note

* client-core: tidy up some minor things
This commit is contained in:
Jon Häggblad
2023-01-19 13:30:43 +01:00
committed by GitHub
parent 886e2ed5e7
commit c6a5e08188
17 changed files with 262 additions and 64 deletions
+3
View File
@@ -7,8 +7,11 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
### Added
- dkg rerun from scratch and dkg-specific epochs ([#2839])
- nym-sdk: add support for surb storage ([#2870])
[#2839]: https://github.com/nymtech/nym/pull/2839
[#2870]: https://github.com/nymtech/nym/pull/2870
## [v1.1.6] (2023-01-17)
@@ -386,22 +386,32 @@ where
where
<B as ReplyStorageBackend>::StorageError: Sync + Send,
{
let persistent_storage = PersistentReplyStorage::new(backend);
let mem_store = persistent_storage
.load_state_from_backend()
.await
.map_err(|err| ClientCoreError::SurbStorageError {
source: Box::new(err),
})?;
let store_clone = mem_store.clone();
spawn_future(async move {
persistent_storage
.flush_on_shutdown(store_clone, shutdown)
if backend.is_active() {
log::trace!("Setup persistent reply storage");
let persistent_storage = PersistentReplyStorage::new(backend);
let mem_store = persistent_storage
.load_state_from_backend()
.await
});
.map_err(|err| ClientCoreError::SurbStorageError {
source: Box::new(err),
})?;
Ok(mem_store)
let store_clone = mem_store.clone();
spawn_future(async move {
persistent_storage
.flush_on_shutdown(store_clone, shutdown)
.await
});
Ok(mem_store)
} else {
log::trace!("Setup inactive reply storage");
Ok(backend
.get_inactive_storage()
.map_err(|err| ClientCoreError::SurbStorageError {
source: Box::new(err),
})?)
}
}
pub async fn start_base(mut self) -> Result<BaseClient, ClientCoreError>
@@ -43,6 +43,14 @@ async fn setup_fresh_backend<P: AsRef<Path>>(
Ok(storage_backend)
}
fn setup_inactive_backend(debug_config: &DebugConfig) -> fs_backend::Backend {
info!("creating inactive surb database");
fs_backend::Backend::new_inactive(
debug_config.minimum_reply_surb_storage_threshold,
debug_config.maximum_reply_surb_storage_threshold,
)
}
fn archive_corrupted_database<P: AsRef<Path>>(db_path: P) -> io::Result<()> {
let db_path = db_path.as_ref();
debug_assert!(db_path.exists());
@@ -65,24 +73,29 @@ fn archive_corrupted_database<P: AsRef<Path>>(db_path: P) -> io::Result<()> {
}
pub async fn setup_fs_reply_surb_backend<P: AsRef<Path>>(
db_path: P,
db_path: Option<P>,
debug_config: &DebugConfig,
) -> Result<fs_backend::Backend, ClientCoreError> {
// if the database file doesnt exist, initialise fresh storage, otherwise attempt to load the existing one
let db_path = db_path.as_ref();
if db_path.exists() {
info!("loading existing surb database");
match fs_backend::Backend::try_load(db_path).await {
Ok(backend) => Ok(backend),
Err(err) => {
error!("failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future");
if let Some(db_path) = db_path {
// if the database file doesnt exist, initialise fresh storage, otherwise attempt to load
// the existing one
let db_path = db_path.as_ref();
if db_path.exists() {
info!("loading existing surb database");
match fs_backend::Backend::try_load(db_path).await {
Ok(backend) => Ok(backend),
Err(err) => {
error!("failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future");
archive_corrupted_database(db_path)?;
setup_fresh_backend(db_path, debug_config).await
archive_corrupted_database(db_path)?;
setup_fresh_backend(db_path, debug_config).await
}
}
} else {
setup_fresh_backend(db_path, debug_config).await
}
} else {
setup_fresh_backend(db_path, debug_config).await
Ok(setup_inactive_backend(debug_config))
}
}
@@ -41,4 +41,8 @@ impl ReplyStorageBackend for Backend {
async fn load_surb_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError> {
self.empty.load_surb_storage().await
}
fn get_inactive_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError> {
self.empty.get_inactive_storage()
}
}
@@ -10,6 +10,12 @@ pub enum StorageError {
#[error("the provided database path doesn't have a filename defined")]
DatabasePathWithoutFilename { provided_path: PathBuf },
#[error("unable to create the directory for the database")]
DatabasePathUnableToCreateParentDirectory {
provided_path: PathBuf,
source: io::Error,
},
#[error("failed to rename our databse file - {source}")]
DatabaseRenameError {
#[source]
@@ -20,6 +20,16 @@ impl StorageManager {
database_path: P,
fresh: bool,
) -> Result<Self, StorageError> {
// ensure the whole directory structure exists
if let Some(parent_dir) = database_path.as_ref().parent() {
std::fs::create_dir_all(parent_dir).map_err(|source| {
StorageError::DatabasePathUnableToCreateParentDirectory {
provided_path: database_path.as_ref().to_path_buf(),
source,
}
})?;
}
let mut opts = sqlx::sqlite::SqliteConnectOptions::new()
.filename(database_path)
.create_if_missing(fresh);
@@ -22,11 +22,49 @@ mod error;
mod manager;
mod models;
#[derive(Debug)]
enum StorageManagerState {
Storage(StorageManager),
Inactive(InactiveMetadata),
}
// When the storage backaed is initialized as inactive, it will still contain metadata parameters
// that will be needed when the in-mem storage is fetched for use.
#[derive(Debug)]
struct InactiveMetadata {
pub minimum_reply_surb_storage_threshold: usize,
pub maximum_reply_surb_storage_threshold: usize,
}
impl StorageManagerState {
fn get(&self) -> &StorageManager {
match self {
StorageManagerState::Storage(manager) => manager,
StorageManagerState::Inactive(_) => {
panic!("tried to get storage of an inactive backend")
}
}
}
fn get_mut(&mut self) -> &mut StorageManager {
match self {
StorageManagerState::Storage(manager) => manager,
StorageManagerState::Inactive(_) => {
panic!("tried to get storage of an inactive backend")
}
}
}
fn is_active(&self) -> bool {
matches!(self, StorageManagerState::Storage(_))
}
}
#[derive(Debug)]
pub struct Backend {
temporary_old_path: Option<PathBuf>,
database_path: PathBuf,
manager: StorageManager,
manager: StorageManagerState,
}
impl Backend {
@@ -40,17 +78,32 @@ impl Backend {
});
}
let manager = StorageManager::init(database_path, true).await?;
manager.create_status_table().await?;
let backend = Backend {
temporary_old_path: None,
database_path: owned_path,
manager: StorageManager::init(database_path, true).await?,
manager: StorageManagerState::Storage(manager),
};
backend.manager.create_status_table().await?;
Ok(backend)
}
pub fn new_inactive(
minimum_reply_surb_storage_threshold: usize,
maximum_reply_surb_storage_threshold: usize,
) -> Self {
Backend {
temporary_old_path: None,
database_path: PathBuf::new(),
manager: StorageManagerState::Inactive(InactiveMetadata {
minimum_reply_surb_storage_threshold,
maximum_reply_surb_storage_threshold,
}),
}
}
pub async fn try_load<P: AsRef<Path>>(database_path: P) -> Result<Self, StorageError> {
let owned_path: PathBuf = database_path.as_ref().into();
if owned_path.file_name().is_none() {
@@ -119,12 +172,12 @@ impl Backend {
Ok(Backend {
temporary_old_path: None,
database_path: owned_path,
manager,
manager: StorageManagerState::Storage(manager),
})
}
async fn close_pool(&mut self) {
self.manager.connection_pool.close().await;
self.manager.get_mut().connection_pool.close().await;
}
async fn rotate(&mut self) -> Result<(), StorageError> {
@@ -143,8 +196,9 @@ impl Backend {
fs::rename(&self.database_path, &temp_old)
.map_err(|err| StorageError::DatabaseRenameError { source: err })?;
self.manager = StorageManager::init(&self.database_path, true).await?;
self.manager.create_status_table().await?;
self.manager =
StorageManagerState::Storage(StorageManager::init(&self.database_path, true).await?);
self.manager.get_mut().create_status_table().await?;
self.temporary_old_path = Some(temp_old);
Ok(())
@@ -161,26 +215,27 @@ impl Backend {
}
async fn start_storage_flush(&self) -> Result<(), StorageError> {
Ok(self.manager.set_flush_status(true).await?)
Ok(self.manager.get().set_flush_status(true).await?)
}
async fn end_storage_flush(&self) -> Result<(), StorageError> {
self.manager
.get()
.set_previous_flush_timestamp(OffsetDateTime::now_utc().unix_timestamp())
.await?;
Ok(self.manager.set_flush_status(false).await?)
Ok(self.manager.get().set_flush_status(false).await?)
}
async fn start_client_use(&self) -> Result<(), StorageError> {
Ok(self.manager.set_client_in_use_status(true).await?)
Ok(self.manager.get().set_client_in_use_status(true).await?)
}
async fn stop_client_use(&self) -> Result<(), StorageError> {
Ok(self.manager.set_client_in_use_status(false).await?)
Ok(self.manager.get().set_client_in_use_status(false).await?)
}
async fn get_stored_tags(&self) -> Result<UsedSenderTags, StorageError> {
let stored = self.manager.get_tags().await?;
let stored = self.manager.get().get_tags().await?;
// stop at the first instance of corruption. if even a single entry is malformed,
// something weird has happened and we can't trust the rest of the data
@@ -196,6 +251,7 @@ impl Backend {
for map_ref in tags.as_raw_iter() {
let (recipient, tag) = map_ref.pair();
self.manager
.get()
.insert_tag(StoredSenderTag::new(*recipient, *tag))
.await?;
}
@@ -203,7 +259,7 @@ impl Backend {
}
async fn get_stored_reply_keys(&self) -> Result<SentReplyKeys, StorageError> {
let stored = self.manager.get_reply_keys().await?;
let stored = self.manager.get().get_reply_keys().await?;
// stop at the first instance of corruption. if even a single entry is malformed,
// something weird has happened and we can't trust the rest of the data
@@ -219,6 +275,7 @@ impl Backend {
for map_ref in reply_keys.as_raw_iter() {
let (digest, key) = map_ref.pair();
self.manager
.get()
.insert_reply_key(StoredReplyKey::new(*digest, *key))
.await?;
}
@@ -226,7 +283,7 @@ impl Backend {
}
async fn get_stored_reply_surbs(&self) -> Result<ReceivedReplySurbsMap, StorageError> {
let surb_senders = self.manager.get_surb_senders().await?;
let surb_senders = self.manager.get().get_surb_senders().await?;
let metadata = self.get_reply_surb_storage_metadata().await?;
let mut received_surbs = Vec::with_capacity(surb_senders.len());
@@ -236,6 +293,7 @@ impl Backend {
sender.try_into()?;
let stored_surbs = self
.manager
.get()
.get_reply_surbs(sender_id)
.await?
.into_iter()
@@ -263,6 +321,7 @@ impl Backend {
let (tag, received_surbs) = map_ref.pair();
let sender_id = self
.manager
.get()
.insert_surb_sender(StoredSurbSender::new(
*tag,
received_surbs.surbs_last_received_at(),
@@ -271,6 +330,7 @@ impl Backend {
for reply_surb in received_surbs.surbs_ref() {
self.manager
.get()
.insert_reply_surb(StoredReplySurb::new(sender_id, reply_surb))
.await?
}
@@ -282,6 +342,7 @@ impl Backend {
&self,
) -> Result<ReplySurbStorageMetadata, StorageError> {
self.manager
.get()
.get_reply_surb_storage_metadata()
.await
.map_err(Into::into)
@@ -292,6 +353,7 @@ impl Backend {
reply_surbs: &ReceivedReplySurbsMap,
) -> Result<(), StorageError> {
self.manager
.get()
.insert_reply_surb_storage_metadata(ReplySurbStorageMetadata::new(
reply_surbs.min_surb_threshold(),
reply_surbs.max_surb_threshold(),
@@ -305,6 +367,10 @@ impl Backend {
impl ReplyStorageBackend for Backend {
type StorageError = error::StorageError;
fn is_active(&self) -> bool {
self.manager.is_active()
}
async fn start_storage_session(&self) -> Result<(), Self::StorageError> {
self.start_client_use().await
}
@@ -342,6 +408,18 @@ impl ReplyStorageBackend for Backend {
Ok(CombinedReplyStorage::load(reply_keys, reply_surbs, tags))
}
fn get_inactive_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError> {
match self.manager {
StorageManagerState::Storage(_) => {
panic!("tried to get inactive storage from an active storage backend")
}
StorageManagerState::Inactive(ref state) => Ok(CombinedReplyStorage::new(
state.minimum_reply_surb_storage_threshold,
state.maximum_reply_surb_storage_threshold,
)),
}
}
async fn stop_storage_session(self) -> Result<(), Self::StorageError> {
self.stop_client_use().await
}
@@ -50,12 +50,23 @@ impl ReplyStorageBackend for Empty {
self.max_surb_threshold,
))
}
fn get_inactive_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError> {
Ok(CombinedReplyStorage::new(
self.min_surb_threshold,
self.max_surb_threshold,
))
}
}
#[async_trait]
pub trait ReplyStorageBackend: Sized {
type StorageError: Error + 'static;
fn is_active(&self) -> bool {
true
}
async fn start_storage_session(&self) -> Result<(), Self::StorageError> {
Ok(())
}
@@ -73,6 +84,11 @@ pub trait ReplyStorageBackend: Sized {
async fn load_surb_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError>;
/// In the case the storage backend is initialized in an inactive state (persisting data is
/// disabled), we might still need to fetch the (in-mem) storage and the parameters it was
/// created with.
fn get_inactive_storage(&self) -> Result<CombinedReplyStorage, Self::StorageError>;
async fn stop_storage_session(self) -> Result<(), Self::StorageError> {
Ok(())
}
+2 -2
View File
@@ -124,7 +124,7 @@ impl SocketClient {
self.key_manager,
Some(Self::create_bandwidth_controller(&self.config).await),
non_wasm_helpers::setup_fs_reply_surb_backend(
self.config.get_base().get_reply_surb_database_path(),
Some(self.config.get_base().get_reply_surb_database_path()),
self.config.get_debug_settings(),
)
.await?,
@@ -161,7 +161,7 @@ impl SocketClient {
self.key_manager,
Some(Self::create_bandwidth_controller(&self.config).await),
non_wasm_helpers::setup_fs_reply_surb_backend(
self.config.get_base().get_reply_surb_database_path(),
Some(self.config.get_base().get_reply_surb_database_path()),
self.config.get_debug_settings(),
)
.await?,
+1 -1
View File
@@ -193,7 +193,7 @@ impl NymClient {
self.key_manager,
Some(Self::create_bandwidth_controller(&self.config).await),
non_wasm_helpers::setup_fs_reply_surb_backend(
self.config.get_base().get_reply_surb_database_path(),
Some(self.config.get_base().get_reply_surb_database_path()),
self.config.get_debug_settings(),
)
.await?,
@@ -10,7 +10,9 @@ async fn main() {
let config = mixnet::Config::new(user_chosen_gateway_id, nym_api_endpoints);
let mut client = mixnet::ClientBuilder::new(Some(config), None).unwrap();
let mut client = mixnet::MixnetClientBuilder::new(Some(config), None)
.await
.unwrap();
// Just some plain data to pretend we have some external storage that the application
// implementer is using.
+3 -1
View File
@@ -14,7 +14,9 @@ async fn main() {
let keys = mixnet::StoragePaths::new_from_dir(mixnet::KeyMode::Keep, &config_dir).unwrap();
// Provide key paths for the client to read/write keys to.
let client = mixnet::ClientBuilder::new(None, Some(keys)).unwrap();
let client = mixnet::MixnetClientBuilder::new(None, Some(keys))
.await
.unwrap();
// Connect to the mixnet, now we're listening for incoming
let mut client = client.connect_to_mixnet().await.unwrap();
+1 -1
View File
@@ -5,7 +5,7 @@ async fn main() {
logging::setup_logging();
// Passing no config makes the client fire up an ephemeral session and figure shit out on its own
let mut client = mixnet::Client::connect().await.unwrap();
let mut client = mixnet::MixnetClient::connect().await.unwrap();
// Be able to get our client address
let our_address = client.nym_address();
@@ -5,8 +5,9 @@ async fn main() {
logging::setup_logging();
// Create client builder, including ephemeral keys. The builder can be usable in the context
// where you don't want to connect just yet
let client = mixnet::ClientBuilder::new(None, None).unwrap();
// where you don't want to connect just yet.
// Since not storage paths are given, the surb storage will be inactive.
let client = mixnet::MixnetClientBuilder::new(None, None).await.unwrap();
// Now we connect to the mixnet, using ephemeral keys already created
let mut client = client.connect_to_mixnet().await.unwrap();
+1 -1
View File
@@ -13,5 +13,5 @@ pub use nymsphinx::{
pub use keys::{Keys, KeysArc};
pub use paths::{GatewayKeyMode, KeyMode, StoragePaths};
pub use client::{Client, ClientBuilder};
pub use client::{MixnetClient, MixnetClientBuilder};
pub use config::Config;
+66 -13
View File
@@ -10,6 +10,7 @@ use client_core::{
inbound_messages::InputMessage,
key_manager::KeyManager,
received_buffer::ReconstructedMessagesReceiver,
replies::reply_storage::{self, ReplyStorageBackend},
},
config::{persistence::key_pathfinder::ClientKeyPathfinder, GatewayEndpointConfig},
};
@@ -26,7 +27,7 @@ use validator_client::nyxd::SigningNyxdClient;
use super::{connection_state::BuilderState, Config, GatewayKeyMode, Keys, KeysArc, StoragePaths};
use crate::error::{Error, Result};
pub struct ClientBuilder {
pub struct MixnetClientBuilder<B: ReplyStorageBackend> {
/// Keys handled by the client
key_manager: KeyManager,
@@ -39,15 +40,67 @@ pub struct ClientBuilder {
/// The client can be in one of multiple states, depending on how it is created and if it's
/// connected to the mixnet.
state: BuilderState,
/// The storage backend for reply-SURBs
reply_storage_backend: B,
}
impl ClientBuilder {
impl MixnetClientBuilder<reply_storage::fs_backend::Backend> {
/// Create a new mixnet client. If no config options are supplied, creates a new client with
/// ephemeral keys stored in RAM, which will be discarded at application close.
///
/// Callers have the option of supplying futher parameters to store persistent identities at a
/// location on-disk, if desired.
pub fn new(config_option: Option<Config>, paths: Option<StoragePaths>) -> Result<Self> {
pub async fn new(config: Option<Config>, paths: Option<StoragePaths>) -> Result<Self> {
let config = config.unwrap_or_default();
let reply_surb_database_path = paths.as_ref().map(|p| p.reply_surb_database_path.clone());
let reply_storage_backend = non_wasm_helpers::setup_fs_reply_surb_backend(
reply_surb_database_path,
&config.debug_config,
)
.await?;
MixnetClientBuilder::new_with_custom_storage(Some(config), paths, reply_storage_backend)
}
}
impl MixnetClientBuilder<reply_storage::Empty> {
/// Create a new mixnet client. If no config options are supplied, creates a new client with
/// ephemeral keys stored in RAM, which will be discarded at application close.
///
/// Callers have the option of supplying futher parameters to store persistent identities at a
/// location on-disk, if desired.
pub fn new_without_storage(
config: Option<Config>,
paths: Option<StoragePaths>,
) -> Result<Self> {
let config = config.unwrap_or_default();
let reply_storage_backend =
non_wasm_helpers::setup_empty_reply_surb_backend(&config.debug_config);
MixnetClientBuilder::new_with_custom_storage(Some(config), paths, reply_storage_backend)
}
}
impl<B> MixnetClientBuilder<B>
where
B: ReplyStorageBackend + Sync + Send + 'static,
{
/// Create a new mixnet client. If no config options are supplied, creates a new client with
/// ephemeral keys stored in RAM, which will be discarded at application close.
///
/// Callers have the option of supplying futher parameters to store persistent identities at a
/// location on-disk, if desired.
///
/// A custom storage backend can be passed in.
pub fn new_with_custom_storage(
config_option: Option<Config>,
paths: Option<StoragePaths>,
reply_storage_backend: B,
) -> Result<Self> {
let config = config_option.unwrap_or_default();
// If we are provided paths to keys, use them if they are available. And if they are
@@ -87,6 +140,7 @@ impl ClientBuilder {
config,
storage_paths: paths,
state: BuilderState::New,
reply_storage_backend,
})
}
@@ -184,7 +238,10 @@ impl ClientBuilder {
}
/// Connects to the mixnet via the gateway in the client config
pub async fn connect_to_mixnet(mut self) -> Result<Client> {
pub async fn connect_to_mixnet(mut self) -> Result<MixnetClient>
where
<B as ReplyStorageBackend>::StorageError: Sync + Send,
{
// For some simple cases we can figure how to setup gateway without it having to have been
// called in advance.
if matches!(self.state, BuilderState::New) {
@@ -221,16 +278,12 @@ impl ClientBuilder {
// TODO: we currently don't support having a bandwidth controller
let bandwidth_controller = None;
// TODO: currently we only support in-memory reply surb storage.
let reply_storage_backend =
non_wasm_helpers::setup_empty_reply_surb_backend(&self.config.debug_config);
let base_builder: BaseClientBuilder<_, SigningNyxdClient> = BaseClientBuilder::new(
&gateway_endpoint_config,
&self.config.debug_config,
self.key_manager.clone(),
bandwidth_controller,
reply_storage_backend,
self.reply_storage_backend,
CredentialsToggle::Disabled,
self.config.nym_api_endpoints.clone(),
);
@@ -243,7 +296,7 @@ impl ClientBuilder {
// Register our receiver
let reconstructed_receiver = client_output.register_receiver().unwrap();
Ok(Client {
Ok(MixnetClient {
nym_address,
key_manager: self.key_manager,
client_input,
@@ -255,7 +308,7 @@ impl ClientBuilder {
}
}
pub struct Client {
pub struct MixnetClient {
nym_address: Recipient,
/// Keys handled by the client
@@ -274,9 +327,9 @@ pub struct Client {
task_manager: TaskManager,
}
impl Client {
impl MixnetClient {
pub async fn connect() -> Result<Self> {
let client = ClientBuilder::new(None, None)?;
let client = MixnetClientBuilder::new_without_storage(None, None)?;
client.connect_to_mixnet().await
}
+1 -1
View File
@@ -61,7 +61,7 @@ pub struct StoragePaths {
impl StoragePaths {
pub fn new_from_dir(operating_mode: KeyMode, dir: &Path) -> Result<Self> {
if !dir.is_file() {
if dir.is_file() {
return Err(Error::ExpectedDirectory(dir.to_owned()));
}