110e0e143f
* db: migrate from lmdb-zero to heed * fix: check resizing operation and wait to avoid crash with multiple batches, fix exists check at batch * build: fix missing deps at Cargo.lock * lmdb: single environment, migrate existing databases with provided non-default environment name * fix: revert chunk size to 128mb * lmdb: ability to use multiple shared environments * build: remove unused dependency * fix: resize to have correct multiplier of the system page size * lmdb: speed up prefix iter by storing keys * lmdb: default env name * lmdb: wait db resize before read, reduce timeout before resizing * lmdb: use static reader for iterator, count existing batches for stable resize * fix: check batches count on resize waiting * lmdb: use separate databases instead of prefixes, use default db for values without prefixes, migrate old environment * fix: pop pos key * lmdb: count all open transactions to finish before resizing * lmdb: immediate resize if there are no open transactions * lmdb: remove env state when there are no more stores * lmdb: use atomic for resize and resize checking flags * lmdb: sleep 10ms when waiting all opened txs to be closed * lmdb: use atomic open txs and stores count * lmdb: use index to detect separator, ignore unknown db key to not have a panic * lmdb: store max 10k keys in the iterator * lmdb: check iter result on getting total * lmdb: handle errors at iterator * lmdb: handle an error when db with provided key not found * lmdb: fix iterate over 10k keys * lmdb: document migration resize safety * lmdb: fix iter test * lmdb: clear new db after unsuccessful migration, handle read error on migration to interrupt process * store: bring back old key methods to reproduce data migration * lmdb: return an error on unsuccessful migration * lmdb: migration test, clean data after allocate test * lmdb: info migration log * fix: move iterator before handling an error to allow skip bad value * lmdb: return an error if removal of old DB file failed after migration * lmdb: lifetime for iterator, use write transaction at batch iterator * lmdb: migration progress * fix: tests * lmdb: immediately set resizing flag, ignore resizing flag while there are more than 0 opened txs to avoid stuck, optimize tx counter for some operations * lmdb: key for successful migration * lmdb: fix put database creation at separate block to avoid lifetime issues when returning an error on migration
118 lines
3.6 KiB
Rust
118 lines
3.6 KiB
Rust
// Copyright 2021 The Grin Developers
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use grin_chain as chain;
|
|
use grin_core as core;
|
|
|
|
use grin_util as util;
|
|
|
|
use std::fs::{self, File};
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
|
|
use crate::chain::store::ChainStore;
|
|
use crate::chain::txhashset;
|
|
use crate::core::core::hash::Hashed;
|
|
use crate::core::core::BlockHeader;
|
|
use crate::core::global;
|
|
use crate::util::file;
|
|
|
|
fn clean_output_dir(dir_name: &str) {
|
|
let _ = fs::remove_dir_all(dir_name);
|
|
}
|
|
|
|
#[test]
|
|
fn test_unexpected_zip() {
|
|
global::set_local_chain_type(global::ChainTypes::AutomatedTesting);
|
|
let db_root = format!(".grin_txhashset_zip");
|
|
clean_output_dir(&db_root);
|
|
{
|
|
let chain_store = ChainStore::new(&db_root, None).unwrap();
|
|
let store = Arc::new(chain_store);
|
|
txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
|
|
let head = BlockHeader::default();
|
|
// First check if everything works out of the box
|
|
assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
|
|
let zip_path = Path::new(&db_root).join(format!(
|
|
"txhashset_snapshot_{}.zip",
|
|
head.hash().to_string()
|
|
));
|
|
let zip_file = File::open(&zip_path).unwrap();
|
|
assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok());
|
|
// Remove temp txhashset dir
|
|
let _ = fs::remove_dir_all(
|
|
Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())),
|
|
);
|
|
// Then add strange files in the original txhashset folder
|
|
File::create(&Path::new(&db_root).join("txhashset").join("badfile"))
|
|
.expect("problem creating a file");
|
|
File::create(
|
|
&Path::new(&db_root)
|
|
.join("txhashset")
|
|
.join("output")
|
|
.join("badfile"),
|
|
)
|
|
.expect("problem creating a file");
|
|
|
|
let files = file::list_files(&Path::new(&db_root).join("txhashset"));
|
|
let expected_files: Vec<_> = vec![
|
|
"badfile",
|
|
"kernel/pmmr_data.bin",
|
|
"kernel/pmmr_hash.bin",
|
|
"kernel/pmmr_size.bin",
|
|
"output/badfile",
|
|
"output/pmmr_data.bin",
|
|
"output/pmmr_hash.bin",
|
|
"rangeproof/pmmr_data.bin",
|
|
"rangeproof/pmmr_hash.bin",
|
|
];
|
|
assert_eq!(
|
|
files,
|
|
expected_files
|
|
.iter()
|
|
.map(|x| PathBuf::from(x))
|
|
.collect::<Vec<_>>()
|
|
);
|
|
|
|
assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
|
|
let _ = fs::remove_dir_all(
|
|
Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())),
|
|
);
|
|
let zip_file = File::open(zip_path).unwrap();
|
|
let _ = fs::remove_dir_all(Path::new(&db_root).join("txhashset"));
|
|
assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok());
|
|
|
|
// Check that the new txhashset dir contains *only* the expected files
|
|
// No "badfiles" and no "size" file.
|
|
let files = file::list_files(&Path::new(&db_root).join("txhashset"));
|
|
let expected_files: Vec<_> = vec![
|
|
"kernel/pmmr_data.bin",
|
|
"kernel/pmmr_hash.bin",
|
|
"output/pmmr_data.bin",
|
|
"output/pmmr_hash.bin",
|
|
"rangeproof/pmmr_data.bin",
|
|
"rangeproof/pmmr_hash.bin",
|
|
];
|
|
assert_eq!(
|
|
files,
|
|
expected_files
|
|
.iter()
|
|
.map(|x| PathBuf::from(x))
|
|
.collect::<Vec<_>>()
|
|
);
|
|
}
|
|
// Cleanup chain directory
|
|
clean_output_dir(&db_root);
|
|
}
|