From 1be60922c28f3e5e2c9e3e2df6a736704a33bc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Wed, 8 Nov 2023 17:16:56 +0000 Subject: [PATCH] binary upgrade logic --- tools/nymvisor/Cargo.toml | 5 +- tools/nymvisor/src/cli/run.rs | 3 + tools/nymvisor/src/config/mod.rs | 30 +++++- tools/nymvisor/src/config/template.rs | 4 + tools/nymvisor/src/daemon/mod.rs | 28 +++++- tools/nymvisor/src/error.rs | 74 ++++++++++++++- tools/nymvisor/src/helpers.rs | 1 + tools/nymvisor/src/tasks/launcher/mod.rs | 13 ++- tools/nymvisor/src/upgrades/download.rs | 111 +++++++++++++++++++++++ tools/nymvisor/src/upgrades/mod.rs | 100 ++++++++++++++++++-- tools/nymvisor/src/upgrades/types.rs | 88 +++++++++++++++++- 11 files changed, 438 insertions(+), 19 deletions(-) create mode 100644 tools/nymvisor/src/upgrades/download.rs diff --git a/tools/nymvisor/Cargo.toml b/tools/nymvisor/Cargo.toml index 9639f2f2c8..1b858ee724 100644 --- a/tools/nymvisor/Cargo.toml +++ b/tools/nymvisor/Cargo.toml @@ -13,14 +13,15 @@ license.workspace = true [dependencies] anyhow = { workspace = true } base64 = "0.21.5" +bytes = { version = "1.5.0", features = ["std"]} clap = { workspace = true, features = ["derive"] } dotenvy = { workspace = true } futures = { workspace = true } humantime = "2.1.0" humantime-serde = "1.1.1" lazy_static = { workspace = true } -nix = { version = "0.27.1", features = ["signal"] } -reqwest = { workspace = true, features = ["json"] } +nix = { version = "0.27.1", features = ["signal", "fs"] } +reqwest = { workspace = true, features = ["json", "stream"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } time = { workspace = true, features = [ "serde-human-readable" ] } diff --git a/tools/nymvisor/src/cli/run.rs b/tools/nymvisor/src/cli/run.rs index cb23e2c2de..fb9343b338 100644 --- a/tools/nymvisor/src/cli/run.rs +++ b/tools/nymvisor/src/cli/run.rs @@ -5,6 +5,7 @@ use crate::cli::try_load_current_config; use crate::daemon::Daemon; use crate::env::Env; use crate::error::NymvisorError; +use crate::upgrades::upgrade_binary; use async_file_watcher::AsyncFileWatcher; use futures::channel::mpsc; use futures::future::{AbortHandle, Abortable}; @@ -32,6 +33,8 @@ pub(crate) fn execute(args: Args) -> Result<(), NymvisorError> { info!("starting nymvisor for {}", config.daemon.name); + // upgrade_binary(&config)?; + // TODO: experiment with the minimal runtime // look at futures::executor::LocalPool // well, if the creation of the runtime failed, there isn't much we could do diff --git a/tools/nymvisor/src/config/mod.rs b/tools/nymvisor/src/config/mod.rs index 36bdd54272..135d4f09d1 100644 --- a/tools/nymvisor/src/config/mod.rs +++ b/tools/nymvisor/src/config/mod.rs @@ -27,6 +27,8 @@ pub(crate) const DEFAULT_BASE_UPSTREAM_UPGRADE_INFO_SOURCE: &str = "https://nymtech.net/.wellknown/"; pub(crate) const UPGRADE_PLAN_FILENAME: &str = "upgrade-plan.json"; +pub(crate) const UPGRADE_HISTORY_FILENAME: &str = "upgrade-history.json"; +pub(crate) const UPGRADE_LOCK_FILENAME: &str = "upgrade.lock"; pub(crate) const UPGRADE_INFO_FILENAME: &str = "upgrade-info.json"; pub(crate) const NYMVISOR_DIR: &str = "nymvisor"; pub(crate) const BACKUP_DIR: &str = "backups"; @@ -287,10 +289,26 @@ impl Config { if name == GENESIS_DIR { self.genesis_daemon_dir().join(UPGRADE_INFO_FILENAME) } else { - self.upgrades_dir().join(name).join(UPGRADE_INFO_FILENAME) + self.upgrade_dir(name).join(UPGRADE_INFO_FILENAME) } } + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrades/ + pub fn upgrade_dir>(&self, upgrade_name: P) -> PathBuf { + self.upgrades_dir().join(upgrade_name) + } + + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrades//bin + pub fn upgrade_binary_dir>(&self, upgrade_name: P) -> PathBuf { + self.upgrade_dir(upgrade_name).join(BIN_DIR) + } + + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrades//bin/nym-api + pub fn upgrade_binary>(&self, upgrade_name: P) -> PathBuf { + self.upgrade_binary_dir(upgrade_name) + .join(&self.daemon.name) + } + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrades/ pub fn upgrades_dir(&self) -> PathBuf { self.upgrade_data_dir().join(UPGRADES_DIR) @@ -301,6 +319,16 @@ impl Config { self.upgrade_data_dir().join(UPGRADE_PLAN_FILENAME) } + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrade-history.json + pub fn upgrade_history_filepath(&self) -> PathBuf { + self.upgrade_data_dir().join(UPGRADE_HISTORY_FILENAME) + } + + // e.g. $HOME/.nym/nymvisors/data/nym-api/upgrade.lock + pub fn upgrade_lock_filepath(&self) -> PathBuf { + self.upgrade_data_dir().join(UPGRADE_LOCK_FILENAME) + } + pub fn upstream_upgrade_url(&self) -> Url { if let Some(absolute_url) = &self.daemon.debug.absolute_upstream_upgrade_url { absolute_url.clone() diff --git a/tools/nymvisor/src/config/template.rs b/tools/nymvisor/src/config/template.rs index bbc9c946b3..fcfd66a8e2 100644 --- a/tools/nymvisor/src/config/template.rs +++ b/tools/nymvisor/src/config/template.rs @@ -20,6 +20,10 @@ id = '{{ nymvisor.id }}' # Can be overridden with $NYMVISOR_UPSTREAM_BASE_UPGRADE_URL environmental variable. upstream_base_upgrade_url = '{{ nymvisor.upstream_base_upgrade_url }}' +# Specifies the rate of polling the upstream url for upgrade information. +# Can be overridden with $NYMVISOR_UPSTREAM_POLLING_RATE +upstream_polling_rate = '{{ nymvisor.upstream_polling_rate }}' + # If set to true, this will disable `nymvisor` logs (but not the underlying process) # default: false # Can be overridden with $NYMVISOR_DISABLE_LOGS environmental variable. diff --git a/tools/nymvisor/src/daemon/mod.rs b/tools/nymvisor/src/daemon/mod.rs index 8dff09d0a1..a4c613345a 100644 --- a/tools/nymvisor/src/daemon/mod.rs +++ b/tools/nymvisor/src/daemon/mod.rs @@ -8,7 +8,9 @@ use nix::unistd::Pid; use nym_bin_common::build_information::BinaryBuildInformationOwned; use std::ffi::OsStr; use std::fmt::Debug; +use std::fs; use std::future::Future; +use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; use std::pin::Pin; use std::process::{ExitStatus, Stdio}; @@ -60,7 +62,7 @@ impl Daemon { self } - #[instrument] + #[instrument(skip(self), fields(self.executable_path = ?self.executable_path))] pub(crate) fn get_build_information( &self, ) -> Result { @@ -86,8 +88,28 @@ impl Daemon { .map_err(|source| NymvisorError::DaemonBuildInformationParseFailure { source }) } - pub(crate) fn verify_binary(&self) { - todo!() + #[instrument(skip(self), fields(self.executable_path = ?self.executable_path))] + pub(crate) fn verify_binary(&self) -> Result<(), NymvisorError> { + let metadata = fs::metadata(&self.executable_path).expect("error handling"); + + if !metadata.is_file() { + todo!("error not a file") + } + + let mut permissions = metadata.permissions(); + let mode = permissions.mode(); + let is_executable = mode & 0o111 != 0; + if !is_executable { + warn!( + "the binary does not seem to have executable bits sets. attempting to fix that..." + ); + let new_mode = mode | 0o111; // Set the three execute bits to on (a+x). + permissions.set_mode(new_mode); + + fs::set_permissions(&self.executable_path, permissions).expect("error handling"); + } + + Ok(()) } pub(crate) fn execute_async(&self, args: I) -> Result diff --git a/tools/nymvisor/src/error.rs b/tools/nymvisor/src/error.rs index 45d6559920..10d912e896 100644 --- a/tools/nymvisor/src/error.rs +++ b/tools/nymvisor/src/error.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use async_file_watcher::NotifyError; +use nix::errno::Errno; use nix::sys::signal::Signal; use nym_bin_common::build_information::BinaryBuildInformationOwned; use std::ffi::OsString; @@ -81,8 +82,43 @@ pub(crate) enum NymvisorError { source: io::Error, }, - #[error("could not acquire the lock at {} to update the upgrade-plan.info file. It is either held by another process or this nymvisor has experienced a critical failure during previous upgrade attempt", lock_path.display())] - UnableToAcquireUpgradePlanLock { lock_path: PathBuf }, + #[error( + "failed to load upgrade history using path '{}'. detailed message: {source}", path.display() + )] + UpgradeHistoryLoadFailure { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error( + "failed to save upgrade history using path '{}'. detailed message: {source}", path.display() + )] + UpgradeHistorySaveFailure { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("could not acquire the lock at {} to perform binary upgrade with error code {libc_code}. It is either held by another process or this nymvisor has experienced a critical failure during previous upgrade attempt", lock_path.display())] + UnableToAcquireUpgradePlanLock { + lock_path: PathBuf, + libc_code: Errno, + }, + + #[error("could not create the lock file at {} to perform binary upgrade: {source}", path.display())] + LockFileCreationFailure { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("could not remove the lock file at {} after performing binary upgrade: {source}", path.display())] + LockFileRemovalFailure { + path: PathBuf, + #[source] + source: io::Error, + }, #[error("failed to initialise the path '{}': {source}", path.display())] PathInitFailure { @@ -143,6 +179,13 @@ pub(crate) enum NymvisorError { source: io::Error, }, + #[error("failed to remove symlink at '{}': {source}", path.display())] + SymlinkRemovalFailure { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("the value of daemon home has to be provided by either `--daemon-home` flag or `$DAEMON_HOME` environmental variable")] DaemonHomeUnavailable, @@ -205,12 +248,37 @@ pub(crate) enum NymvisorError { source: NotifyError, }, - #[error("failed to query the upstream url ('{url}') - source")] + #[error("failed to query the upstream url ('{url}'): {source}")] UpstreamQueryFailure { url: Url, #[source] source: reqwest::Error, }, + + #[error( + "attempted to perform binary upgrade with no upgrades queued up in the upgrade plan file" + )] + NoQueuedUpgrades, + + #[error("could not find the upgrade binary at {} while the binary download is disabled", path.display())] + NoUpgradeBinaryWithDisabledDownload { path: PathBuf }, + + #[error("upgrade '{upgrade_name}' does not have any valid download URLs for the current arch '{arch}'")] + NoDownloadUrls { upgrade_name: String, arch: String }, + + #[error("failed to download the upgrade binary from '{url}': {source}")] + UpgradeDownloadFailure { + url: Url, + #[source] + source: reqwest::Error, + }, + + #[error("failed to create daemon binary at {}: {source}", path.display())] + DaemonBinaryCreationFailure { + path: PathBuf, + #[source] + source: io::Error, + }, } impl From for NymvisorError { diff --git a/tools/nymvisor/src/helpers.rs b/tools/nymvisor/src/helpers.rs index 38dc36ebc3..67bf4e9a2d 100644 --- a/tools/nymvisor/src/helpers.rs +++ b/tools/nymvisor/src/helpers.rs @@ -1,2 +1,3 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 + diff --git a/tools/nymvisor/src/tasks/launcher/mod.rs b/tools/nymvisor/src/tasks/launcher/mod.rs index 844a6f5079..ebabdf5d77 100644 --- a/tools/nymvisor/src/tasks/launcher/mod.rs +++ b/tools/nymvisor/src/tasks/launcher/mod.rs @@ -39,7 +39,7 @@ impl DaemonLauncher { } self.perform_backup()?; - upgrade_binary()?; + upgrade_binary(&self.config).await?; todo!() } @@ -149,3 +149,14 @@ impl DaemonLauncher { todo!() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + #[test] + fn foo() { + println!("{}", env::consts::OS); // Prints the current OS. + } +} diff --git a/tools/nymvisor/src/upgrades/download.rs b/tools/nymvisor/src/upgrades/download.rs new file mode 100644 index 0000000000..4d13944af8 --- /dev/null +++ b/tools/nymvisor/src/upgrades/download.rs @@ -0,0 +1,111 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::config::Config; +use crate::error::NymvisorError; +use crate::upgrades::types::UpgradeInfo; +use bytes::Buf; +use futures::stream::StreamExt; +use std::io::BufWriter; +use std::time::Duration; +use std::{env, fs, io}; +use tracing::info; + +const LOGGING_RATE: Duration = Duration::from_millis(25); + +fn log_progress_bar(downloaded: u64, length: u64) { + let percentage = downloaded as f32 * 100. / length as f32; + + let width = 40; + let filled = (percentage * width as f32 / 100.) as usize; + let empty = width - filled; + + let filled = format!("{:#^width$}", "", width = filled); + let empty = format!("{: ^width$}", "", width = empty); + + let mb_downloaded = downloaded as f64 / (1024. * 1024.); + let mb_total = length as f64 / (1024. * 1024.); + + info!("[{filled}{empty}] {mb_downloaded:.2}MB/{mb_total:.2}MB ({percentage:.2}%)"); +} + +pub(super) async fn download_upgrade_binary( + config: &Config, + info: &UpgradeInfo, +) -> Result<(), NymvisorError> { + info!("attempting to download the upgrade binary"); + let Some(download_url) = info.platforms.get(&os_arch()) else { + return Err(NymvisorError::NoDownloadUrls { + upgrade_name: info.name.clone(), + arch: os_arch(), + }); + }; + + fs::create_dir_all(config.upgrade_binary_dir(&info.name)).map_err(|source| { + NymvisorError::PathInitFailure { + path: config.upgrade_binary_dir(&info.name), + source, + } + })?; + + let target = config.upgrade_binary(&info.name); + let response = reqwest::get(download_url.url.clone()) + .await + .map_err(|source| NymvisorError::UpgradeDownloadFailure { + url: download_url.url.clone(), + source, + })?; + + let maybe_length = response.content_length(); + let mut source = response.bytes_stream(); + + let output_binary = + fs::File::create(&target).map_err(|source| NymvisorError::DaemonBinaryCreationFailure { + path: target.clone(), + source, + })?; + let mut out = BufWriter::new(output_binary); + + info!("beginning the download"); + let mut downloaded = 0; + let mut last_logged = tokio::time::Instant::now(); + while let Some(chunk) = source.next().await { + let mut bytes = chunk + .map_err(|err_source| NymvisorError::UpgradeDownloadFailure { + url: download_url.url.clone(), + source: err_source, + })? + .reader(); + + downloaded += io::copy(&mut bytes, &mut out).map_err(|err_source| { + NymvisorError::DaemonBinaryCreationFailure { + path: target.clone(), + source: err_source, + } + })?; + + if let Some(length) = maybe_length { + if last_logged.elapsed() > LOGGING_RATE { + log_progress_bar(downloaded, length); + last_logged = tokio::time::Instant::now(); + } + } + } + if let Some(length) = maybe_length { + log_progress_bar(length, length) + } + info!("finished the download"); + + Ok(()) +} + +fn os_arch() -> String { + let os = env::consts::OS; + let arch = env::consts::ARCH; + // a special case for macos because of course it's its own special snowflake + if os == "macos" { + format!("darwin-{arch}") + } else { + format!("{os}-{arch}") + } +} diff --git a/tools/nymvisor/src/upgrades/mod.rs b/tools/nymvisor/src/upgrades/mod.rs index 8e21ac2e14..ecdcf8fd5c 100644 --- a/tools/nymvisor/src/upgrades/mod.rs +++ b/tools/nymvisor/src/upgrades/mod.rs @@ -1,20 +1,106 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::config::Config; +use crate::daemon::Daemon; use crate::error::NymvisorError; +use crate::upgrades::download::download_upgrade_binary; +use crate::upgrades::types::{UpgradeHistory, UpgradePlan}; +use nix::fcntl::{flock, FlockArg}; +use std::fs; +use std::fs::File; +use std::os::fd::AsRawFd; +use std::path::PathBuf; +use tracing::{debug, info}; +pub(crate) mod download; mod serde_helpers; pub(crate) mod types; -pub(crate) fn upgrade_binary() -> Result<(), NymvisorError> { - // lock +pub(crate) async fn upgrade_binary(config: &Config) -> Result<(), NymvisorError> { + info!("attempting to perform binary upgrade"); - /* - if binary already exist => swap symlink, write history and we're done - otherwise deal with download + let mut plan = UpgradePlan::try_load(config.upgrade_plan_filepath())?; + let Some(next) = plan.pop_next_upgrade() else { + return Err(NymvisorError::NoQueuedUpgrades); + }; + let upgrade_name = next.name.clone(); - */ + let history_path = config.upgrade_history_filepath(); + let mut upgrade_history = if history_path.exists() { + UpgradeHistory::try_load(history_path)? + } else { + UpgradeHistory::new() + }; - todo!() + debug!("creating the lock file"); + let lock_path = config.upgrade_lock_filepath(); + let lock_file = + File::create(&lock_path).map_err(|source| NymvisorError::LockFileCreationFailure { + path: lock_path.clone(), + source, + })?; + let lock_fd = lock_file.as_raw_fd(); + + debug!("attempting to acquire the lock"); + if let Err(err) = flock(lock_fd, FlockArg::LockExclusiveNonblock) { + return Err(NymvisorError::UnableToAcquireUpgradePlanLock { + lock_path, + libc_code: err, + }); + } + + let upgrade_binary_path = config.upgrade_binary(&upgrade_name); + + if !upgrade_binary_path.exists() { + if !config.daemon.debug.allow_binaries_download { + return Err(NymvisorError::NoUpgradeBinaryWithDisabledDownload { + path: upgrade_binary_path, + }); + } + info!( + "upgrade binary not found at {}. attempting to to download it", + upgrade_binary_path.display() + ); + download_upgrade_binary(config, &next).await?; + } + + let tmp_daemon = Daemon::new(upgrade_binary_path); + tmp_daemon.verify_binary()?; + + let new_bin_info = tmp_daemon.get_build_information()?; + if new_bin_info.build_version != next.version { + // TODO: if it was downloaded, maybe we should download to some .tmp file first? + todo!() + } + + plan.update_on_disk()?; + upgrade_history.insert_new_upgrade(next)?; + set_upgrade_link(config, config.upgrade_binary_dir(&upgrade_name))?; + + // finally remove the lock file + fs::remove_file(&lock_path).map_err(|source| NymvisorError::LockFileRemovalFailure { + path: lock_path.clone(), + source, + }) +} + +fn set_upgrade_link(config: &Config, upgrade_path: PathBuf) -> Result<(), NymvisorError> { + // remove the existing symlink if it exists + let link = config.current_daemon_dir(); + if fs::read_link(&link).is_ok() { + fs::remove_file(&link).map_err(|source| NymvisorError::SymlinkRemovalFailure { + path: link.clone(), + source, + })?; + } + + std::os::unix::fs::symlink(&upgrade_path, &link).map_err(|source| { + NymvisorError::SymlinkCreationFailure { + source_path: upgrade_path, + target_path: link, + source, + } + }) } diff --git a/tools/nymvisor/src/upgrades/types.rs b/tools/nymvisor/src/upgrades/types.rs index 4fa5b97ce5..6dc7dc0722 100644 --- a/tools/nymvisor/src/upgrades/types.rs +++ b/tools/nymvisor/src/upgrades/types.rs @@ -40,7 +40,7 @@ impl UpgradePlan { self.queued_up.push_back(upgrade); } - fn update_on_disk(&self) -> Result<(), NymvisorError> { + pub(crate) fn update_on_disk(&self) -> Result<(), NymvisorError> { // it should be impossible to update an existing upgrade plan that wasn't loaded from disk assert!(self._save_path.is_some()); @@ -77,10 +77,18 @@ impl UpgradePlan { &self.current } + pub(crate) fn set_current(&mut self, new_current: UpgradeInfo) { + self.current = new_current + } + pub(crate) fn next_upgrade(&self) -> Option<&UpgradeInfo> { self.queued_up.front() } + pub(crate) fn pop_next_upgrade(&mut self) -> Option { + self.queued_up.pop_front() + } + pub(crate) fn has_planned(&self, upgrade: &UpgradeInfo) -> bool { for planned in &self.queued_up { if planned.version == upgrade.version { @@ -238,12 +246,88 @@ impl UpgradeInfo { #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "lowercase")] -pub struct UpgradeHistory(Vec); +pub struct UpgradeHistory { + // metadata indicating save location of the underlying file + #[serde(skip)] + _save_path: Option, + + history: Vec, +} #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "lowercase")] pub struct UpgradeHistoryEntry { #[serde(with = "time::serde::rfc3339")] performed_at: OffsetDateTime, + info: UpgradeInfo, } + +impl UpgradeHistoryEntry { + fn new(info: UpgradeInfo) -> Self { + UpgradeHistoryEntry { + performed_at: OffsetDateTime::now_utc(), + info, + } + } +} + +impl UpgradeHistory { + pub(crate) fn new() -> Self { + UpgradeHistory { + _save_path: None, + history: vec![], + } + } + + pub(crate) fn update_on_disk(&self) -> Result<(), NymvisorError> { + // it should be impossible to update an existing upgrade history that wasn't loaded from disk + assert!(self._save_path.is_some()); + + // safety: the except here is fine as this failure implies failure in the underlying logic of the code + // as opposed to user error + #[allow(clippy::expect_used)] + let save_path = self + ._save_path + .as_ref() + .expect("loaded upgrade history does not have an associate save path!"); + + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(save_path) + .map_err(|source| NymvisorError::UpgradeHistorySaveFailure { + path: save_path.to_path_buf(), + source, + })?; + + // we're not using any non-standard serializer and thus the serialization should not ever fail + #[allow(clippy::expect_used)] + serde_json::to_writer_pretty(file, self) + .expect("unexpected UpgradeHistory serialization failure"); + Ok(()) + } + + fn push_upgrade(&mut self, upgrade: UpgradeInfo) { + self.history.push(UpgradeHistoryEntry::new(upgrade)); + } + + pub(crate) fn insert_new_upgrade(&mut self, upgrade: UpgradeInfo) -> Result<(), NymvisorError> { + self.push_upgrade(upgrade); + self.update_on_disk() + } + + pub(crate) fn try_load>(path: P) -> Result { + let path = path.as_ref(); + std::fs::File::open(path) + .and_then(|file| { + serde_json::from_reader(file) + .map_err(|serde_json_err| io::Error::new(io::ErrorKind::Other, serde_json_err)) + }) + .map_err(|source| NymvisorError::UpgradeHistoryLoadFailure { + path: path.to_path_buf(), + source, + }) + } +}