Feature/slate serialization (#2534)

* - Add backwards compatability - Add hex serialization * rustfmt * rustfmt * Windows Compatibility Fixes #1 (#2535) * initial changes for windows build and unit/integration tests * rustfmt * wallet+store tests * rustfmt * fix linux daemonize * better encapsulate file rename * rustfmt * remove daemonize commands * rustfmt * remove server start/stop commands * add ability to drop pmmr backend files explicitly for txhashset unzip * rustfmt * fix pmmr tests * rustfmt * Windows TUI Fix (#2555) * switch pancurses backend to win32 * revert changes to restore test * compatibility fix + debug messages * rustfmt * Add content disposition for OK responses (#2545) * Testing http send and fixing accordingly * add repost method into wallet owner api (#2553) * add repost method into wallet owner api * rustfmt * Add ability to compare selection strategies (#2516) Before tx creation user can estimate fee and locked amount with different selection strategies by providing `-e` flag for `wallet send` command.
2019-02-13 15:29:44 +02:00
parent 0d36acf01b
commit ee4eed71ea
39 changed files with 2408 additions and 1221 deletions
@@ -73,11 +73,19 @@ pub fn new_named_env(path: String, name: String, max_readers: Option<u32>) -> lm
 	env_builder.set_maxdbs(8).unwrap();
 	// half a TB should give us plenty room, will be an issue on 32 bits
 	// (which we don't support anyway)
-	env_builder
-		.set_mapsize(549_755_813_888)
-		.unwrap_or_else(|e| {
-			panic!("Unable to allocate LMDB space: {:?}", e);
-		});
+
+	#[cfg(not(target_os = "windows"))]
+	env_builder.set_mapsize(5_368_709_120).unwrap_or_else(|e| {
+		panic!("Unable to allocate LMDB space: {:?}", e);
+	});
+	//TODO: This is temporary to support (beta) windows support
+	//Windows allocates the entire file at once, so this needs to
+	//be changed to allocate as little as possible and increase as needed
+	#[cfg(target_os = "windows")]
+	env_builder.set_mapsize(524_288_000).unwrap_or_else(|e| {
+		panic!("Unable to allocate LMDB space: {:?}", e);
+	});
+
 	if let Some(max_readers) = max_readers {
 		env_builder
 			.set_maxreaders(max_readers)
@@ -152,6 +152,12 @@ impl<T: PMMRable> Backend<T> for PMMRBackend<T> {
 		self.data_file.path()
 	}

+	/// Release underlying data files
+	fn release_files(&mut self) {
+		self.data_file.release();
+		self.hash_file.release();
+	}
+
 	fn snapshot(&self, header: &BlockHeader) -> Result<(), String> {
 		self.leaf_set
 			.snapshot(header)
@@ -208,8 +214,8 @@ impl<T: PMMRable> PMMRBackend<T> {
 		Ok(PMMRBackend {
 			data_dir: data_dir.to_path_buf(),
 			prunable,
-			hash_file,
-			data_file,
+			hash_file: hash_file,
+			data_file: data_file,
 			leaf_set,
 			prune_list,
 		})
@@ -334,20 +340,11 @@ impl<T: PMMRable> PMMRBackend<T> {
 			}
 			self.prune_list.flush()?;
 		}
-
 		// 4. Rename the compact copy of hash file and reopen it.
-		fs::rename(
-			tmp_prune_file_hash.clone(),
-			self.data_dir.join(PMMR_HASH_FILE),
-		)?;
-		self.hash_file = DataFile::open(self.data_dir.join(PMMR_HASH_FILE))?;
+		self.hash_file.replace(Path::new(&tmp_prune_file_hash))?;

 		// 5. Rename the compact copy of the data file and reopen it.
-		fs::rename(
-			tmp_prune_file_data.clone(),
-			self.data_dir.join(PMMR_DATA_FILE),
-		)?;
-		self.data_file = DataFile::open(self.data_dir.join(PMMR_DATA_FILE))?;
+		self.data_file.replace(Path::new(&tmp_prune_file_data))?;

 		// 6. Write the leaf_set to disk.
 		// Optimize the bitmap storage in the process.
@@ -101,6 +101,17 @@ where
 		self.file.path()
 	}

+	/// Replace underlying file with another, deleting original
+	pub fn replace(&mut self, with: &Path) -> io::Result<()> {
+		self.file.replace(with)?;
+		Ok(())
+	}
+
+	/// Drop underlying file handles
+	pub fn release(&mut self) {
+		self.file.release();
+	}
+
 	/// Write the file out to disk, pruning removed elements.
 	pub fn save_prune<F>(&self, target: &str, prune_offs: &[u64], prune_cb: F) -> io::Result<()>
 	where
@@ -125,7 +136,7 @@ where
 /// latter by truncating the underlying file and re-creating the mmap.
 pub struct AppendOnlyFile {
 	path: PathBuf,
-	file: File,
+	file: Option<File>,
 	mmap: Option<memmap::Mmap>,
 	buffer_start: usize,
 	buffer: Vec<u8>,
@@ -135,28 +146,36 @@ pub struct AppendOnlyFile {
 impl AppendOnlyFile {
 	/// Open a file (existing or not) as append-only, backed by a mmap.
 	pub fn open<P: AsRef<Path>>(path: P) -> io::Result<AppendOnlyFile> {
-		let file = OpenOptions::new()
-			.read(true)
-			.append(true)
-			.create(true)
-			.open(&path)?;
 		let mut aof = AppendOnlyFile {
-			file,
+			file: None,
 			path: path.as_ref().to_path_buf(),
 			mmap: None,
 			buffer_start: 0,
 			buffer: vec![],
 			buffer_start_bak: 0,
 		};
-		// If we have a non-empty file then mmap it.
-		let sz = aof.size();
-		if sz > 0 {
-			aof.buffer_start = sz as usize;
-			aof.mmap = Some(unsafe { memmap::Mmap::map(&aof.file)? });
-		}
+		aof.init()?;
 		Ok(aof)
 	}

+	/// (Re)init an underlying file and its associated memmap
+	pub fn init(&mut self) -> io::Result<()> {
+		self.file = Some(
+			OpenOptions::new()
+				.read(true)
+				.append(true)
+				.create(true)
+				.open(self.path.clone())?,
+		);
+		// If we have a non-empty file then mmap it.
+		let sz = self.size();
+		if sz > 0 {
+			self.buffer_start = sz as usize;
+			self.mmap = Some(unsafe { memmap::Mmap::map(&self.file.as_ref().unwrap())? });
+		}
+		Ok(())
+	}
+
 	/// Append data to the file. Until the append-only file is synced, data is
 	/// only written to memory.
 	pub fn append(&mut self, bytes: &mut [u8]) {
@@ -193,21 +212,37 @@ impl AppendOnlyFile {
 	pub fn flush(&mut self) -> io::Result<()> {
 		if self.buffer_start_bak > 0 {
 			// Flushing a rewound state, we need to truncate via set_len() before applying.
-			self.file.set_len(self.buffer_start as u64)?;
+			// Drop and recreate, or windows throws an access error
+			self.mmap = None;
+			self.file = None;
+			{
+				let file = OpenOptions::new()
+					.read(true)
+					.create(true)
+					.write(true)
+					.open(&self.path)?;
+				file.set_len(self.buffer_start as u64)?;
+			}
+			let file = OpenOptions::new()
+				.read(true)
+				.create(true)
+				.append(true)
+				.open(&self.path)?;
+			self.file = Some(file);
 			self.buffer_start_bak = 0;
 		}

 		self.buffer_start += self.buffer.len();
-		self.file.write_all(&self.buffer[..])?;
-		self.file.sync_all()?;
+		self.file.as_mut().unwrap().write_all(&self.buffer[..])?;
+		self.file.as_mut().unwrap().sync_all()?;

 		self.buffer = vec![];

 		// Note: file must be non-empty to memory map it
-		if self.file.metadata()?.len() == 0 {
+		if self.file.as_ref().unwrap().metadata()?.len() == 0 {
 			self.mmap = None;
 		} else {
-			self.mmap = Some(unsafe { memmap::Mmap::map(&self.file)? });
+			self.mmap = Some(unsafe { memmap::Mmap::map(&self.file.as_ref().unwrap())? });
 		}

 		Ok(())
@@ -313,6 +348,23 @@ impl AppendOnlyFile {
 		}
 	}

+	/// Replace the underlying file with another file
+	/// deleting the original
+	pub fn replace(&mut self, with: &Path) -> io::Result<()> {
+		self.mmap = None;
+		self.file = None;
+		fs::remove_file(&self.path)?;
+		fs::rename(with, &self.path)?;
+		self.init()?;
+		Ok(())
+	}
+
+	/// Release underlying file handles
+	pub fn release(&mut self) {
+		self.mmap = None;
+		self.file = None;
+	}
+
 	/// Current size of the file in bytes.
 	pub fn size(&self) -> u64 {
 		fs::metadata(&self.path).map(|md| md.len()).unwrap_or(0)