fix: ignore corrupted surb storage and instead create fresh one (#2711)

* checking for correct surb metadata on db load * archiving corrupted database on load and attempting to start fresh session instead * checking for data corruption by looking at flush timestamp * Moving public import to separate section
2022-12-15 15:45:54 +00:00
parent c0ffbb0cb2
commit 4d7e21e0f2
2 changed files with 70 additions and 26 deletions
@@ -8,6 +8,57 @@ use crate::config::DebugConfig;
 use crate::error::ClientCoreError;
 use log::{error, info};
 use std::path::Path;
+use std::{fs, io};
+use time::OffsetDateTime;
+
+async fn setup_fresh_backend<P: AsRef<Path>>(
+    db_path: P,
+    debug_config: &DebugConfig,
+) -> Result<fs_backend::Backend, ClientCoreError<fs_backend::Backend>> {
+    info!("creating fresh surb database");
+    let mut storage_backend = match fs_backend::Backend::init(db_path).await {
+        Ok(backend) => backend,
+        Err(err) => {
+            error!("failed to setup persistent storage backend for our reply needs: {err}");
+            return Err(ClientCoreError::SurbStorageError { source: err });
+        }
+    };
+
+    // while I kinda hate that we're going to be creating `CombinedReplyStorage` twice,
+    // it will only be happening on the very first run and in practice won't incur huge
+    // costs since the storage is going to be empty
+    let mem_store = CombinedReplyStorage::new(
+        debug_config.minimum_reply_surb_storage_threshold,
+        debug_config.maximum_reply_surb_storage_threshold,
+    );
+    storage_backend
+        .init_fresh(&mem_store)
+        .await
+        .map_err(|err| ClientCoreError::SurbStorageError { source: err })?;
+
+    Ok(storage_backend)
+}
+
+fn archive_corrupted_database<P: AsRef<Path>>(db_path: P) -> io::Result<()> {
+    let db_path = db_path.as_ref();
+    debug_assert!(db_path.exists());
+
+    let now = OffsetDateTime::now_utc().unix_timestamp();
+
+    let suffix = format!("_{now}.corrupted");
+
+    let new_extension =
+        if let Some(existing_extension) = db_path.extension().and_then(|ext| ext.to_str()) {
+            format!("{existing_extension}.{}", suffix)
+        } else {
+            suffix
+        };
+
+    let mut renamed = db_path.to_owned();
+    renamed.set_extension(new_extension);
+
+    fs::rename(db_path, renamed)
+}

 pub async fn setup_fs_reply_surb_backend<P: AsRef<Path>>(
    db_path: P,
@@ -20,32 +71,13 @@ pub async fn setup_fs_reply_surb_backend<P: AsRef<Path>>(
        match fs_backend::Backend::try_load(db_path).await {
            Ok(backend) => Ok(backend),
            Err(err) => {
-                error!("failed to setup persistent storage backend for our reply needs: {err}");
-                Err(ClientCoreError::SurbStorageError { source: err })
+                error!("failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future");
+
+                archive_corrupted_database(db_path)?;
+                setup_fresh_backend(db_path, debug_config).await
            }
        }
    } else {
-        info!("creating fresh surb database");
-        let mut storage_backend = match fs_backend::Backend::init(db_path).await {
-            Ok(backend) => backend,
-            Err(err) => {
-                error!("failed to setup persistent storage backend for our reply needs: {err}");
-                return Err(ClientCoreError::SurbStorageError { source: err });
-            }
-        };
-
-        // while I kinda hate that we're going to be creating `CombinedReplyStorage` twice,
-        // it will only be happening on the very first run and in practice won't incur huge
-        // costs since the storage is going to be empty
-        let mem_store = CombinedReplyStorage::new(
-            debug_config.minimum_reply_surb_storage_threshold,
-            debug_config.maximum_reply_surb_storage_threshold,
-        );
-        storage_backend
-            .init_fresh(&mem_store)
-            .await
-            .map_err(|err| ClientCoreError::SurbStorageError { source: err })?;
-
-        Ok(storage_backend)
+        setup_fresh_backend(db_path, debug_config).await
    }
 }
@@ -1,7 +1,6 @@
 // Copyright 2022 - Nym Technologies SA <contact@nymtech.net>
 // SPDX-License-Identifier: Apache-2.0

-pub use self::error::StorageError;
 use crate::client::replies::reply_storage::backend::fs_backend::manager::StorageManager;
 use crate::client::replies::reply_storage::backend::fs_backend::models::{
    ReplySurbStorageMetadata, StoredReplyKey, StoredReplySurb, StoredSenderTag, StoredSurbSender,
@@ -17,6 +16,8 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use time::OffsetDateTime;

+pub use self::error::StorageError;
+
 mod error;
 mod manager;
 mod models;
@@ -66,6 +67,12 @@ impl Backend {
            return Err(StorageError::IncompleteDataFlush);
        }

+        let last_flush_timestamp = manager.get_previous_flush_timestamp().await?;
+        if last_flush_timestamp == 0 {
+            // either this client has been running since 1970 or the flush failed
+            return Err(StorageError::IncompleteDataFlush);
+        }
+
        // the process has gone down without full graceful shutdown,
        // meaning the database doesn't contain valid data anymore
        // so we have to purge it
@@ -75,7 +82,12 @@ impl Backend {
            manager.delete_all_reply_keys().await?;
        }

-        let last_flush_timestamp = manager.get_previous_flush_timestamp().await?;
+        if let Err(err) = manager.get_reply_surb_storage_metadata().await {
+            // we can't recover here, we HAVE TO initialise fresh (because we don't know correct starting metadata)
+            error!("it seems the client has been shutdown gracefully - we're missing valid surb data dump. the existing database cannot be used");
+            return Err(err.into());
+        }
+
        let last_flush = match OffsetDateTime::from_unix_timestamp(last_flush_timestamp) {
            Ok(last_flush) => last_flush,
            Err(err) => {