From 2aa5e25532d6e75dfc0de6e5282c84d23f77571e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Wed, 13 Mar 2024 10:18:03 +0000 Subject: [PATCH] removed the usage of allow.list --- common/types/src/gateway.rs | 9 +- gateway/src/commands/helpers.rs | 16 - gateway/src/commands/init.rs | 8 - gateway/src/commands/run.rs | 7 - .../src/commands/setup_network_requester.rs | 7 - gateway/src/http/mod.rs | 15 +- gateway/src/node/helpers.rs | 3 - .../network-requester/src/cli/init.rs | 7 - .../network-requester/src/cli/mod.rs | 5 - .../network-requester/src/cli/run.rs | 7 - .../network-requester/src/config/mod.rs | 7 +- .../src/config/persistence.rs | 2 + .../network-requester/src/core.rs | 25 +- .../request_filter/allowed_hosts/filter.rs | 559 ------------------ .../src/request_filter/allowed_hosts/group.rs | 61 -- .../src/request_filter/allowed_hosts/host.rs | 35 -- .../src/request_filter/allowed_hosts/hosts.rs | 203 ------- .../src/request_filter/allowed_hosts/mod.rs | 13 - .../allowed_hosts/standard_list.rs | 113 ---- .../allowed_hosts/stored_allowed_hosts.rs | 127 ---- .../src/request_filter/exit_policy/mod.rs | 25 +- .../src/request_filter/mod.rs | 133 +---- 22 files changed, 53 insertions(+), 1334 deletions(-) delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/filter.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/group.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/host.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/hosts.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/mod.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/standard_list.rs delete mode 100644 service-providers/network-requester/src/request_filter/allowed_hosts/stored_allowed_hosts.rs diff --git a/common/types/src/gateway.rs b/common/types/src/gateway.rs index 90d012a5ff..834775c492 100644 --- a/common/types/src/gateway.rs +++ b/common/types/src/gateway.rs @@ -124,15 +124,12 @@ pub struct GatewayNetworkRequesterDetails { pub encryption_key: String, pub open_proxy: bool, - pub exit_policy: bool, pub enabled_statistics: bool, // just a convenience wrapper around all the keys pub address: String, pub config_path: String, - pub allow_list_path: String, - pub unknown_list_path: String, } impl fmt::Display for GatewayNetworkRequesterDetails { @@ -146,11 +143,7 @@ impl fmt::Display for GatewayNetworkRequesterDetails { writeln!(f, "\taddress: {}", self.address)?; writeln!(f, "\tuses open proxy: {}", self.open_proxy)?; - writeln!(f, "\tuses exit policy: {}", self.exit_policy)?; - writeln!(f, "\tsends statistics: {}", self.enabled_statistics)?; - - writeln!(f, "\tallow list path: {}", self.allow_list_path)?; - writeln!(f, "\tunknown list path: {}", self.unknown_list_path) + writeln!(f, "\tsends statistics: {}", self.enabled_statistics) } } diff --git a/gateway/src/commands/helpers.rs b/gateway/src/commands/helpers.rs index 968b9b14c6..2bd3f3834c 100644 --- a/gateway/src/commands/helpers.rs +++ b/gateway/src/commands/helpers.rs @@ -109,7 +109,6 @@ pub(crate) struct OverrideNetworkRequesterConfig { pub(crate) medium_toggle: bool, pub(crate) open_proxy: Option, - pub(crate) enable_exit_policy: Option, pub(crate) enable_statistics: Option, pub(crate) statistics_recipient: Option, @@ -229,10 +228,6 @@ pub(crate) fn override_network_requester_config( nym_network_requester::Config::with_open_proxy, opts.open_proxy, ) - .with_optional( - nym_network_requester::Config::with_old_allow_list, - opts.enable_exit_policy.map(|e| !e), - ) .with_optional( nym_network_requester::Config::with_enabled_statistics, opts.enable_statistics, @@ -314,21 +309,10 @@ pub(crate) async fn initialise_local_network_requester( enabled: gateway_config.network_requester.enabled, identity_key: address.identity().to_string(), encryption_key: address.encryption_key().to_string(), - exit_policy: !nr_cfg.network_requester.use_deprecated_allow_list, open_proxy: nr_cfg.network_requester.open_proxy, enabled_statistics: nr_cfg.network_requester.enabled_statistics, address: address.to_string(), config_path: nr_cfg_path.display().to_string(), - allow_list_path: nr_cfg - .storage_paths - .allowed_list_location - .display() - .to_string(), - unknown_list_path: nr_cfg - .storage_paths - .unknown_list_location - .display() - .to_string(), }) } diff --git a/gateway/src/commands/init.rs b/gateway/src/commands/init.rs index 010d270222..b2be9129d4 100644 --- a/gateway/src/commands/init.rs +++ b/gateway/src/commands/init.rs @@ -133,12 +133,6 @@ pub struct Init { )] medium_toggle: bool, - /// Specifies whether this network requester will run using the default ExitPolicy - /// as opposed to the allow list. - /// Note: this setting will become the default in the future releases. - #[clap(long)] - with_exit_policy: Option, - #[clap(short, long, default_value_t = OutputFormat::default())] output: OutputFormat, } @@ -173,7 +167,6 @@ impl<'a> From<&'a Init> for OverrideNetworkRequesterConfig { no_cover: value.no_cover, medium_toggle: value.medium_toggle, open_proxy: value.open_proxy, - enable_exit_policy: value.with_exit_policy, enable_statistics: value.enable_statistics, statistics_recipient: value.statistics_recipient.clone(), } @@ -303,7 +296,6 @@ mod tests { fastmode: false, no_cover: false, medium_toggle: false, - with_exit_policy: None, }; std::env::set_var(BECH32_PREFIX, "n"); diff --git a/gateway/src/commands/run.rs b/gateway/src/commands/run.rs index 34c1b6be8d..bc52ff056e 100644 --- a/gateway/src/commands/run.rs +++ b/gateway/src/commands/run.rs @@ -132,12 +132,6 @@ pub struct Run { #[arg(long, group = "network", hide = true)] custom_mixnet: Option, - /// Specifies whether this network requester will run using the default ExitPolicy - /// as opposed to the allow list. - /// Note: this setting will become the default in the future releases. - #[arg(long)] - with_exit_policy: Option, - #[arg(short, long, default_value_t = OutputFormat::default())] output: OutputFormat, @@ -175,7 +169,6 @@ impl<'a> From<&'a Run> for OverrideNetworkRequesterConfig { no_cover: value.no_cover, medium_toggle: value.medium_toggle, open_proxy: value.open_proxy, - enable_exit_policy: value.with_exit_policy, enable_statistics: value.enable_statistics, statistics_recipient: value.statistics_recipient.clone(), } diff --git a/gateway/src/commands/setup_network_requester.rs b/gateway/src/commands/setup_network_requester.rs index c9de302200..b9368ce2e0 100644 --- a/gateway/src/commands/setup_network_requester.rs +++ b/gateway/src/commands/setup_network_requester.rs @@ -59,12 +59,6 @@ pub struct CmdArgs { )] medium_toggle: bool, - /// Specifies whether this network requester will run using the default ExitPolicy - /// as opposed to the allow list. - /// Note: this setting will become the default in the future releases. - #[clap(long)] - with_exit_policy: Option, - #[clap(short, long, default_value_t = OutputFormat::default())] output: OutputFormat, } @@ -76,7 +70,6 @@ impl<'a> From<&'a CmdArgs> for OverrideNetworkRequesterConfig { no_cover: value.no_cover, medium_toggle: value.medium_toggle, open_proxy: value.open_proxy, - enable_exit_policy: value.with_exit_policy, enable_statistics: value.enable_statistics, statistics_recipient: value.statistics_recipient.clone(), } diff --git a/gateway/src/http/mod.rs b/gateway/src/http/mod.rs index 7b63a522fb..9fcaa10209 100644 --- a/gateway/src/http/mod.rs +++ b/gateway/src/http/mod.rs @@ -199,13 +199,11 @@ impl<'a> HttpApiBuilder<'a> { // once we start refreshing it, we'll have to change it, but at that point // the allow list will be probably be completely removed and thus the pointer management // will be much easier - let Some(exit_policy) = request_filter.current_exit_policy_filter() else { - warn!("this node does not use an exit policy. no changes will be made"); - return self; - }; + + let upstream = request_filter.current_exit_policy_filter().upstream(); // if there's no upstream (i.e. open proxy), we couldn't have possibly updated it : ) - let last_updated = if exit_policy.upstream().is_some() { + let last_updated = if upstream.is_some() { #[allow(clippy::expect_used)] std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -217,12 +215,9 @@ impl<'a> HttpApiBuilder<'a> { self.exit_policy = Some(UsedExitPolicy { enabled: true, - upstream_source: exit_policy - .upstream() - .map(|u| u.to_string()) - .unwrap_or_default(), + upstream_source: upstream.map(|u| u.to_string()).unwrap_or_default(), last_updated, - policy: Some(exit_policy.policy().clone()), + policy: Some(request_filter.current_exit_policy_filter().policy().clone()), }); self diff --git a/gateway/src/node/helpers.rs b/gateway/src/node/helpers.rs index 3e2d54417a..58a11e3f6c 100644 --- a/gateway/src/node/helpers.rs +++ b/gateway/src/node/helpers.rs @@ -64,12 +64,9 @@ pub(crate) async fn node_details( identity_key: nr_identity_public_key.to_base58_string(), encryption_key: nr_encryption_key.to_base58_string(), open_proxy: cfg.network_requester.open_proxy, - exit_policy: !cfg.network_requester.use_deprecated_allow_list, enabled_statistics: cfg.network_requester.enabled_statistics, address: address.to_string(), config_path: display_path(nr_cfg_path), - allow_list_path: display_path(&cfg.storage_paths.allowed_list_location), - unknown_list_path: display_path(&cfg.storage_paths.unknown_list_location), }) } else { None diff --git a/service-providers/network-requester/src/cli/init.rs b/service-providers/network-requester/src/cli/init.rs index b9cef1f16a..5f155bf144 100644 --- a/service-providers/network-requester/src/cli/init.rs +++ b/service-providers/network-requester/src/cli/init.rs @@ -57,12 +57,6 @@ pub(crate) struct Init { #[clap(long)] statistics_recipient: Option, - /// Specifies whether this network requester will run using the default ExitPolicy - /// as opposed to the allow list. - /// Note: this setting will become the default in the future releases. - #[clap(long)] - with_exit_policy: Option, - #[clap(short, long, default_value_t = OutputFormat::default())] output: OutputFormat, } @@ -76,7 +70,6 @@ impl From for OverrideConfig { medium_toggle: false, nyxd_urls: init_config.common_args.nyxd_urls, enabled_credentials_mode: init_config.common_args.enabled_credentials_mode, - enable_exit_policy: init_config.with_exit_policy, open_proxy: init_config.open_proxy, enable_statistics: init_config.enable_statistics, statistics_recipient: init_config.statistics_recipient, diff --git a/service-providers/network-requester/src/cli/mod.rs b/service-providers/network-requester/src/cli/mod.rs index 53f01c5c8a..5cfa1f046a 100644 --- a/service-providers/network-requester/src/cli/mod.rs +++ b/service-providers/network-requester/src/cli/mod.rs @@ -104,7 +104,6 @@ pub(crate) struct OverrideConfig { medium_toggle: bool, nyxd_urls: Option>, enabled_credentials_mode: Option, - enable_exit_policy: Option, open_proxy: Option, enable_statistics: Option, @@ -165,10 +164,6 @@ pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Confi args.enabled_credentials_mode.map(|b| !b), ) .with_optional(Config::with_open_proxy, args.open_proxy) - .with_optional( - Config::with_old_allow_list, - args.enable_exit_policy.map(|e| !e), - ) .with_optional(Config::with_enabled_statistics, args.enable_statistics) .with_optional(Config::with_statistics_recipient, args.statistics_recipient) } diff --git a/service-providers/network-requester/src/cli/run.rs b/service-providers/network-requester/src/cli/run.rs index cd3a5ad653..07d9612898 100644 --- a/service-providers/network-requester/src/cli/run.rs +++ b/service-providers/network-requester/src/cli/run.rs @@ -40,12 +40,6 @@ pub(crate) struct Run { conflicts_with = "fastmode" )] medium_toggle: bool, - - /// Specifies whether this network requester will run using the default ExitPolicy - /// as opposed to the allow list. - /// Note: this setting will become the default in the future releases. - #[clap(long)] - with_exit_policy: Option, } impl From for OverrideConfig { @@ -57,7 +51,6 @@ impl From for OverrideConfig { medium_toggle: run_config.medium_toggle, nyxd_urls: run_config.common_args.nyxd_urls, enabled_credentials_mode: run_config.common_args.enabled_credentials_mode, - enable_exit_policy: run_config.with_exit_policy, open_proxy: run_config.open_proxy, enable_statistics: run_config.enable_statistics, statistics_recipient: run_config.statistics_recipient, diff --git a/service-providers/network-requester/src/config/mod.rs b/service-providers/network-requester/src/config/mod.rs index 53cdc143f4..a1bc434d92 100644 --- a/service-providers/network-requester/src/config/mod.rs +++ b/service-providers/network-requester/src/config/mod.rs @@ -166,12 +166,6 @@ impl Config { self } - #[must_use] - pub fn with_old_allow_list(mut self, use_old_allow_list: bool) -> Self { - self.network_requester.use_deprecated_allow_list = use_old_allow_list; - self - } - #[must_use] pub fn with_enabled_statistics(mut self, enabled_statistics: bool) -> Self { self.network_requester.enabled_statistics = enabled_statistics; @@ -251,6 +245,7 @@ pub struct NetworkRequester { /// Specifies whether this network requester should be using the deprecated allow-list, /// as opposed to the new ExitPolicy. /// Note: this field will be removed in a near future. + #[deprecated] pub use_deprecated_allow_list: bool, /// Specifies the url for an upstream source of the exit policy used by this node. diff --git a/service-providers/network-requester/src/config/persistence.rs b/service-providers/network-requester/src/config/persistence.rs index cc3d4378ba..6d81c3c77e 100644 --- a/service-providers/network-requester/src/config/persistence.rs +++ b/service-providers/network-requester/src/config/persistence.rs @@ -16,10 +16,12 @@ pub struct NetworkRequesterPaths { /// Deprecated /// Location of the file containing our allow.list + #[deprecated] pub allowed_list_location: PathBuf, /// Deprecated /// Location of the file containing our unknown.list + #[deprecated] pub unknown_list_location: PathBuf, /// Location of the file containing our description diff --git a/service-providers/network-requester/src/core.rs b/service-providers/network-requester/src/core.rs index 7affa6c849..b84124aff0 100644 --- a/service-providers/network-requester/src/core.rs +++ b/service-providers/network-requester/src/core.rs @@ -327,9 +327,6 @@ impl NRServiceProviderBuilder { }); let request_filter = RequestFilter::new(&self.config).await?; - request_filter - .start_update_tasks(&self.config.network_requester_debug, &shutdown) - .await; let mut service_provider = NRServiceProvider { config: self.config, @@ -605,20 +602,14 @@ impl NRServiceProvider { QueryResponse::Description("Description (placeholder)".to_string()), ), QueryRequest::ExitPolicy => { - let response = match self.request_filter.current_exit_policy_filter() { - Some(exit_policy_filter) => QueryResponse::ExitPolicy { - enabled: true, - upstream: exit_policy_filter - .upstream() - .map(|u| u.to_string()) - .unwrap_or_default(), - policy: Some(exit_policy_filter.policy().clone()), - }, - None => QueryResponse::ExitPolicy { - enabled: false, - upstream: "".to_string(), - policy: None, - }, + let exit_policy_filter = self.request_filter.current_exit_policy_filter(); + let response = QueryResponse::ExitPolicy { + enabled: true, + upstream: exit_policy_filter + .upstream() + .map(|u| u.to_string()) + .unwrap_or_default(), + policy: Some(exit_policy_filter.policy().clone()), }; Socks5Response::new_query(protocol_version, response) diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/filter.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/filter.rs deleted file mode 100644 index a721c91089..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/filter.rs +++ /dev/null @@ -1,559 +0,0 @@ -// Copyright 2020-2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use super::HostsStore; -use crate::request_filter::allowed_hosts::group::HostsGroup; -use crate::request_filter::allowed_hosts::standard_list::StandardList; -use crate::request_filter::allowed_hosts::stored_allowed_hosts::StoredAllowedHosts; -use publicsuffix::Psl; -use std::{ - net::{IpAddr, SocketAddr}, - str::FromStr, -}; -use tokio::sync::Mutex; -use url::Url; - -#[derive(Debug)] -enum RequestHost { - IpAddr(IpAddr), - SocketAddr(SocketAddr), - RootDomain(String), -} - -/// Filters outbound requests based on what's in an `allowed_hosts` list. -/// -/// Requests to unknown hosts are automatically written to an `unknown_hosts` -/// list so that they can be copy/pasted into the `allowed_hosts` list if desired. -/// This may be handy for service provider node operators who want to be able to look in the -/// `unknown_hosts` file and allow new hosts (e.g. if a wallet has added a new outbound request -/// which needs to be allowed). -/// -/// We rely on the list of domains at https://publicsuffix.org/ to figure out what the root -/// domain is for a given request. This allows us to distinguish all the rules for e.g. -/// .com, .co.uk, .co.jp, uk.com, etc, so that we can distinguish correct root-ish -/// domains as allowed. That list is loaded once at startup from Mozilla's canonical -/// publicsuffix list. -pub(crate) struct OutboundRequestFilter { - pub(super) allowed_hosts: StoredAllowedHosts, - pub(super) standard_list: StandardList, - root_domain_list: publicsuffix::List, - unknown_hosts: Mutex, -} - -/// The official URL of the list -pub const LIST_URL: &str = "https://publicsuffix.org/list/public_suffix_list.dat"; - -async fn request(u: Url) -> String { - reqwest::get(u) - .await - .expect("failed to get LIST_URL") - .text() - .await - .expect("failed to get text") -} - -async fn fetch_list() -> publicsuffix::List { - let str = request(Url::parse(LIST_URL).unwrap()).await; - publicsuffix::List::from_str(&str).unwrap() -} - -impl OutboundRequestFilter { - /// Create a new `OutboundRequestFilter` with the given `allowed_hosts` and `unknown_hosts` lists. - /// - /// Automatically fetches the latest https://publicsuffix.org/ domain list from the internet so that - /// the requester can properly parse all of the world's top-level domains in an up-to-date fashion. - /// - /// Automatcially fetches the latest standard allowed list from the Nym website, so that all - /// requesters are able to support the same minimal functionality out of the box. - pub(crate) async fn new( - allowed_hosts: StoredAllowedHosts, - standard_list: StandardList, - unknown_hosts: HostsStore, - ) -> OutboundRequestFilter { - let domain_list = fetch_list().await; - - OutboundRequestFilter { - allowed_hosts, - standard_list, - root_domain_list: domain_list, - unknown_hosts: Mutex::new(unknown_hosts), - } - } - - pub(crate) fn standard_list(&self) -> StandardList { - self.standard_list.clone() - } - - pub(crate) fn allowed_hosts(&self) -> StoredAllowedHosts { - self.allowed_hosts.clone() - } - - async fn check_allowed_hosts(&self, host: &RequestHost) -> bool { - let guard = self.allowed_hosts.get().await; - self.check_group(&guard.data, host) - } - - async fn check_standard_list(&self, host: &RequestHost) -> bool { - let guard = self.standard_list.get().await; - self.check_group(&guard, host) - } - - fn check_group(&self, group: &HostsGroup, host: &RequestHost) -> bool { - match host { - RequestHost::IpAddr(ip_addr) => group.contains_ip_address(*ip_addr), - RequestHost::SocketAddr(socket_addr) => group.contains_ip_address(socket_addr.ip()), - RequestHost::RootDomain(domain) => group.contains_domain(domain), - } - } - - async fn add_to_unknown_hosts(&self, host: RequestHost) { - let mut guard = self.unknown_hosts.lock().await; - match host { - RequestHost::IpAddr(ip_addr) => guard.add_ip(ip_addr), - RequestHost::SocketAddr(socket_addr) => guard.add_ip(socket_addr.ip()), - RequestHost::RootDomain(domain) => guard.add_domain(&domain), - } - } - - fn parse_request_host(&self, host: &str) -> Option { - // first check if it's a socket address (ip:port) - // (this check is performed to not incorrectly strip what we think might be a port - // from ipv6 address, as for example ::1 contains colons but has no port - if let Ok(socketaddr) = host.parse::() { - Some(RequestHost::SocketAddr(socketaddr)) - // then check if it was an ip address - } else if let Ok(ipaddr) = host.parse::() { - Some(RequestHost::IpAddr(ipaddr)) - // finally, then assume it might be a domain - } else { - // check root - let trimmed = Self::trim_port(host); - // if this failed, it was probably some nonsense - self.get_domain_root(&trimmed).map(RequestHost::RootDomain) - } - } - - async fn check_request_host(&self, request_host: &RequestHost) -> bool { - // first check our own allow list - let local_allowed = self.check_allowed_hosts(request_host).await; - - // if it's locally allowed, no point in checking the standard list - if local_allowed { - return true; - } - - // if that failed, check the standard list - self.check_standard_list(request_host).await - } - - /// Returns `true` if a host's root domain is in the `allowed_hosts` list. - /// - /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. - pub(crate) async fn check(&self, host: &str) -> bool { - let allowed = match self.parse_request_host(host) { - Some(request_host) => { - let res = self.check_request_host(&request_host).await; - if !res { - self.add_to_unknown_hosts(request_host).await - } - res - } - None => false, - }; - - if !allowed { - log::warn!("Blocked outbound connection to {host}, add it to allowed.list if needed",); - } - - allowed - } - - fn trim_port(host: &str) -> String { - let mut tmp: Vec<_> = host.split(':').collect(); - if tmp.len() > 1 { - tmp.pop(); // get rid of last element (port) - tmp.join(":") //rejoin - } else { - host.to_string() - } - } - - /// Attempts to get the root domain, shorn of subdomains, using publicsuffix. - /// If the domain is itself registered in publicsuffix (e.g. s3.amazonaws.com), - /// then just use the full address as root. - fn get_domain_root(&self, host: &str) -> Option { - if let Some(domain) = self.root_domain_list.domain(host.as_bytes()) { - return String::from_utf8(domain.as_bytes().to_vec()).ok(); - } else if let Some(suffix) = self.root_domain_list.suffix(host.as_bytes()) { - if suffix.is_known() { - return String::from_utf8(suffix.as_bytes().to_vec()).ok(); - } else { - return None; - } - } - log::warn!("Error parsing domain: {:?}", host); - None // domain couldn't be parsed - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::ops::{Deref, DerefMut}; - - struct HostsStoreFixture { - inner: HostsStore, - // take ownership of temp file that will get cleaned up on drop (i.e. when test finishes) - _tmp_file: tempfile::NamedTempFile, - } - - impl Deref for HostsStoreFixture { - type Target = HostsStore; - - fn deref(&self) -> &Self::Target { - &self.inner - } - } - - impl DerefMut for HostsStoreFixture { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } - } - - impl HostsStoreFixture { - fn new() -> HostsStoreFixture { - let tmp_file = tempfile::NamedTempFile::new().unwrap(); - let inner = HostsStore::new(&tmp_file); - HostsStoreFixture { - inner, - _tmp_file: tmp_file, - } - } - } - - struct OutboundRequestFilterFixture { - inner: OutboundRequestFilter, - // take ownership of temp files that will get cleaned up on drop (i.e. when test finishes) - _allow_tmp_file: tempfile::NamedTempFile, - _unknown_tmp_file: tempfile::NamedTempFile, - } - - impl Deref for OutboundRequestFilterFixture { - type Target = OutboundRequestFilter; - - fn deref(&self) -> &Self::Target { - &self.inner - } - } - - impl DerefMut for OutboundRequestFilterFixture { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } - } - - impl OutboundRequestFilterFixture { - async fn new() -> OutboundRequestFilterFixture { - let allow_tmp_file = tempfile::NamedTempFile::new().unwrap(); - let unknown_tmp_file = tempfile::NamedTempFile::new().unwrap(); - let allowed = HostsStore::new(&allow_tmp_file); - let unknown = HostsStore::new(&unknown_tmp_file); - let standard = StandardList::new(); - - let inner = OutboundRequestFilter::new(allowed.into(), standard, unknown).await; - OutboundRequestFilterFixture { - inner, - _allow_tmp_file: allow_tmp_file, - _unknown_tmp_file: unknown_tmp_file, - } - } - } - - async fn setup_empty() -> OutboundRequestFilterFixture { - OutboundRequestFilterFixture::new().await - } - - async fn setup_with_allowed(allowed: &[&str]) -> OutboundRequestFilterFixture { - let allow_tmp_file = tempfile::NamedTempFile::new().unwrap(); - let unknown_tmp_file = tempfile::NamedTempFile::new().unwrap(); - let mut allowed_store = HostsStore::new(&allow_tmp_file); - let unknown = HostsStore::new(&unknown_tmp_file); - let standard = StandardList::new(); - - for allow in allowed { - allowed_store.add_host(allow) - } - - let inner = OutboundRequestFilter::new(allowed_store.into(), standard, unknown).await; - OutboundRequestFilterFixture { - inner, - _allow_tmp_file: allow_tmp_file, - _unknown_tmp_file: unknown_tmp_file, - } - } - - #[cfg(test)] - mod trimming_port_information { - use super::*; - - #[test] - fn happens_when_port_exists() { - let host = "nymtech.net:9999"; - assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); - } - - #[test] - fn doesnt_happen_when_no_port_exists() { - let host = "nymtech.net"; - assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); - } - } - - #[cfg(test)] - mod getting_the_domain_root { - use super::*; - - #[tokio::test] - async fn leaves_a_com_alone() { - let filter = setup_empty().await; - assert_eq!( - Some("domain.com".to_string()), - filter.get_domain_root("domain.com") - ) - } - - #[tokio::test] - async fn trims_subdomains_from_com() { - let filter = setup_empty().await; - assert_eq!( - Some("domain.com".to_string()), - filter.get_domain_root("foomp.domain.com") - ) - } - - #[tokio::test] - async fn works_for_non_com_roots() { - let filter = setup_empty().await; - assert_eq!( - Some("domain.co.uk".to_string()), - filter.get_domain_root("domain.co.uk") - ) - } - - #[tokio::test] - async fn works_for_non_com_roots_with_subdomains() { - let filter = setup_empty().await; - assert_eq!( - Some("domain.co.uk".to_string()), - filter.get_domain_root("foomp.domain.co.uk") - ) - } - - #[tokio::test] - async fn returns_none_on_garbage() { - let filter = setup_empty().await; - assert_eq!(None, filter.get_domain_root("::/&&%@")); - } - - #[tokio::test] - async fn returns_full_on_suffix_domains() { - let filter = setup_empty().await; - dbg!(filter.get_domain_root("s3.amazonaws.com")); - assert_eq!( - Some("s3.amazonaws.com".to_string()), - filter.get_domain_root("s3.amazonaws.com") - ); - } - } - - #[cfg(test)] - mod requests_to_unknown_hosts { - use super::*; - - #[tokio::test] - async fn are_not_allowed() { - let host = "unknown.com"; - let filter = setup_empty().await; - assert!(!filter.check(host).await); - } - - #[tokio::test] - async fn get_appended_once_to_the_unknown_hosts_list() { - let host = "unknown.com"; - let filter = setup_empty().await; - filter.check(host).await; - assert_eq!(1, filter.unknown_hosts.lock().await.data.domains.len()); - assert!(filter - .unknown_hosts - .lock() - .await - .data - .domains - .contains("unknown.com")); - filter.check(host).await; - assert_eq!(1, filter.unknown_hosts.lock().await.data.domains.len()); - assert!(filter - .unknown_hosts - .lock() - .await - .data - .domains - .contains("unknown.com")); - } - } - - #[cfg(test)] - mod requests_to_allowed_hosts { - use super::*; - - #[tokio::test] - async fn are_allowed() { - let host = "nymtech.net"; - - let filter = setup_with_allowed(&["nymtech.net"]).await; - assert!(filter.check(host).await); - } - - #[tokio::test] - async fn are_allowed_for_subdomains() { - let host = "foomp.nymtech.net"; - - let filter = setup_with_allowed(&["nymtech.net"]).await; - assert!(filter.check(host).await); - } - - #[tokio::test] - async fn are_not_appended_to_file() { - let filter = setup_with_allowed(&["nymtech.net"]).await; - - // test initial state - let lines = - HostsStore::load_from_storefile(&filter.allowed_hosts.get().await.storefile) - .unwrap(); - assert_eq!(1, lines.len()); - - assert!(filter.check("nymtech.net").await); - - // test state after we've checked to make sure no unexpected changes - let lines = - HostsStore::load_from_storefile(&filter.allowed_hosts.get().await.storefile) - .unwrap(); - assert_eq!(1, lines.len()); - } - - #[tokio::test] - async fn are_allowed_for_ipv4_addresses() { - let address_good = "1.1.1.1"; - let address_good_port = "1.1.1.1:1234"; - let address_bad = "1.1.1.2"; - - let filter = setup_with_allowed(&["1.1.1.1"]).await; - assert!(filter.check(address_good).await); - assert!(filter.check(address_good_port).await); - assert!(!filter.check(address_bad).await); - } - - #[tokio::test] - async fn are_allowed_for_ipv6_addresses() { - let ip_v6_full = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"; - let ip_v6_full_rendered = "2001:0db8:85a3::8a2e:0370:7334"; - let ip_v6_full_port = "[2001:0db8:85a3::8a2e:0370:7334]:1234"; - - let ip_v6_semi = "2001:0db8::0001:0000"; - let ip_v6_semi_rendered = "2001:db8::1:0"; - - let ip_v6_loopback_port = "[::1]:1234"; - - let filter1 = setup_with_allowed(&[ip_v6_full, ip_v6_semi, "::1"]).await; - let filter2 = - setup_with_allowed(&[ip_v6_full_rendered, ip_v6_semi_rendered, "::1"]).await; - - assert!(filter1.check(ip_v6_full).await); - assert!(filter1.check(ip_v6_full_rendered).await); - assert!(filter1.check(ip_v6_full_port).await); - assert!(filter1.check(ip_v6_semi).await); - assert!(filter1.check(ip_v6_semi_rendered).await); - assert!(filter1.check(ip_v6_loopback_port).await); - - assert!(filter2.check(ip_v6_full).await); - assert!(filter2.check(ip_v6_full_rendered).await); - assert!(filter2.check(ip_v6_full_port).await); - assert!(filter2.check(ip_v6_semi).await); - assert!(filter2.check(ip_v6_semi_rendered).await); - assert!(filter2.check(ip_v6_loopback_port).await); - } - - #[tokio::test] - async fn are_allowed_for_ipv4_address_ranges() { - let range1 = "127.0.0.1/32"; - let range2 = "1.2.3.4/24"; - - let bottom_range2 = "1.2.3.0"; - let top_range2 = "1.2.3.255"; - - let outside_range2 = "1.2.2.4"; - - let filter = setup_with_allowed(&[range1, range2]).await; - assert!(filter.check("127.0.0.1").await); - assert!(filter.check("127.0.0.1:1234").await); - assert!(filter.check(bottom_range2).await); - assert!(filter.check(top_range2).await); - assert!(!filter.check(outside_range2).await); - } - - #[tokio::test] - async fn are_allowed_for_ipv6_address_ranges() { - let range = "2620:0:2d0:200::7/32"; - - let bottom1 = "2620:0:0:0:0:0:0:0"; - let bottom2 = "2620::"; - - let top = "2620:0:ffff:ffff:ffff:ffff:ffff:ffff"; - let mid = "2620:0:42::42"; - - let filter = setup_with_allowed(&[range]).await; - assert!(filter.check(bottom1).await); - assert!(filter.check(bottom2).await); - assert!(filter.check(top).await); - assert!(filter.check(mid).await); - } - } - - #[cfg(test)] - mod creating_a_new_host_store { - use super::*; - - #[test] - fn loads_its_host_list_from_storefile() { - let mut host_store = HostsStoreFixture::new(); - - host_store.add_host("nymtech.net"); - host_store.add_host("edwardsnowden.com"); - host_store.add_host("1.2.3.4"); - host_store.add_host("5.6.7.8/16"); - host_store.add_host("1:2:3::"); - host_store.add_host("5:6:7::/48"); - - assert!(host_store.data.domains.contains("nymtech.net")); - assert!(host_store.data.domains.contains("edwardsnowden.com")); - - assert!(host_store - .data - .ip_nets - .contains(&"1.2.3.4".parse().unwrap())); - assert!(host_store - .data - .ip_nets - .contains(&"5.6.7.8/16".parse().unwrap())); - assert!(host_store - .data - .ip_nets - .contains(&"1:2:3::".parse().unwrap())); - assert!(host_store - .data - .ip_nets - .contains(&"5:6:7::/48".parse().unwrap())); - } - } -} diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/group.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/group.rs deleted file mode 100644 index 2c2da2732a..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/group.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use crate::request_filter::allowed_hosts::host::Host; -use ipnetwork::IpNetwork; -use std::collections::HashSet; -use std::net::IpAddr; - -/// A simpled grouped set of hosts. -/// It ignores any port information. -#[derive(Debug)] -pub(crate) struct HostsGroup { - pub(super) domains: HashSet, - pub(super) ip_nets: HashSet, -} - -impl HostsGroup { - pub(crate) fn new(raw_hosts: Vec) -> HostsGroup { - let mut domains = HashSet::new(); - let mut ip_nets = HashSet::new(); - - for host in raw_hosts { - match host { - Host::Domain(domain) => { - domains.insert(domain); - } - Host::IpNetwork(ipnet) => { - ip_nets.insert(ipnet); - } - } - } - - HostsGroup { domains, ip_nets } - } - - pub(crate) fn contains_domain(&self, host: &str) -> bool { - self.domains.contains(host) - } - - pub(super) fn contains_ip_address(&self, address: IpAddr) -> bool { - for ip_net in &self.ip_nets { - if ip_net.contains(address) { - return true; - } - } - - false - } - - pub(super) fn contains_ip_network(&self, network: IpNetwork) -> bool { - self.ip_nets.contains(&network) - } - - pub(super) fn add_ipnet>(&mut self, network: N) { - self.ip_nets.insert(network.into()); - } - - pub(super) fn add_domain(&mut self, domain: &str) { - self.domains.insert(domain.to_string()); - } -} diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/host.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/host.rs deleted file mode 100644 index 7f86e16aa8..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/host.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use ipnetwork::IpNetwork; -use std::str::FromStr; - -// used for parsing file content -#[derive(Debug)] -pub(crate) enum Host { - Domain(String), - IpNetwork(IpNetwork), -} - -impl> From for Host { - fn from(raw: S) -> Self { - // SAFETY: unwrap here is fine as `FromStr` implementation returns `Infallible` error. - raw.as_ref().parse().unwrap() - } -} - -impl FromStr for Host { - type Err = core::convert::Infallible; - - fn from_str(s: &str) -> Result { - if let Ok(ipnet) = s.parse() { - Ok(Host::IpNetwork(ipnet)) - } else { - // TODO: perhaps in the future it should do some domain validation? - // - // So for example if somebody put some nonsense in the whitelist file like "foomp", - // it would get rejected? - Ok(Host::Domain(s.to_string())) - } - } -} diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/hosts.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/hosts.rs deleted file mode 100644 index b3dc697464..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/hosts.rs +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use super::host::Host; -use crate::request_filter::allowed_hosts::group::HostsGroup; -use ipnetwork::IpNetwork; -use std::{ - fs::{self, File, OpenOptions}, - io::{self, BufRead, BufReader}, - net::IpAddr, - path::{Path, PathBuf}, -}; - -/// A simple file-backed store for information about allowed / unknown hosts. -/// It ignores any port information. -#[derive(Debug)] -pub(crate) struct HostsStore { - pub(super) storefile: PathBuf, - pub(super) data: HostsGroup, -} - -impl HostsStore { - /// Constructs a new HostsStore. If the storefile does not exist, it will be created. - /// - /// You can inject a list of standard hosts that you want to support, in addition to the ones - /// in the user-defined storefile. - pub(crate) fn new>(storefile: P) -> HostsStore { - let storefile = storefile.as_ref().to_path_buf(); - if storefile.exists() && !storefile.is_file() { - // there's no error handling in here and I'm not going to be changing it right now. - panic!( - "the provided storefile {:?} is not a valid file!", - storefile - ) - } - - Self::setup_storefile(&storefile); - let hosts = Self::load_from_storefile(&storefile) - .unwrap_or_else(|_| panic!("Could not load hosts from storefile at {storefile:?}")); - - HostsStore { - storefile, - data: HostsGroup::new(hosts), - } - } - - pub(crate) fn try_reload(&mut self) -> io::Result<()> { - let hosts = Self::load_from_storefile(&self.storefile)?; - self.data = HostsGroup::new(hosts); - Ok(()) - } - - pub(crate) fn contains_domain(&self, host: &str) -> bool { - self.data.contains_domain(host) - } - - pub(super) fn contains_ip_address(&self, address: IpAddr) -> bool { - self.data.contains_ip_address(address) - } - - #[allow(unused)] - pub(super) fn contains_ipnetwork(&self, network: IpNetwork) -> bool { - self.data.contains_ip_network(network) - } - - #[allow(unused)] - pub(super) fn add_host>(&mut self, host: H) { - match host.into() { - Host::Domain(domain) => self.add_domain(&domain), - Host::IpNetwork(ipnet) => self.add_ipnet(ipnet), - } - } - - #[allow(unused)] - pub(super) fn add_ipnet(&mut self, network: IpNetwork) { - if !self.contains_ipnetwork(network) { - self.data.add_ipnet(network); - self.append_to_file(&network.to_string()); - } - } - - pub(super) fn add_ip(&mut self, ip: IpAddr) { - if !self.contains_ip_address(ip) { - self.data.add_ipnet(ip); - self.append_to_file(&ip.to_string()); - } - } - - pub(super) fn add_domain(&mut self, domain: &str) { - if !self.contains_domain(domain) { - self.data.add_domain(domain); - self.append_to_file(domain); - } - } - - /// Appends a line of `text` to the storefile at `path` - pub(super) fn append(path: &Path, text: &str) { - use std::io::Write; - let mut file = OpenOptions::new().append(true).open(path).unwrap(); - - if let Err(e) = writeln!(file, "{text}") { - log::error!("Couldn't write to file: {e}"); - } - } - - fn append_to_file(&self, host: &str) { - HostsStore::append(&self.storefile, host); - } - - fn setup_storefile(file: &PathBuf) { - if !file.exists() { - let parent_dir = file - .parent() - .expect("parent dir does not exist for {file:?}"); - fs::create_dir_all(parent_dir) - .unwrap_or_else(|_| panic!("could not create storage directory at {parent_dir:?}")); - log::trace!("Creating: {file:?}"); - File::create(file).unwrap(); - } - } - - /// Loads the storefile contents into memory. - pub(super) fn load_from_storefile

(filename: P) -> io::Result> - where - P: AsRef, - { - log::trace!("Loading from storefile: {}", filename.as_ref().display()); - let file = File::open(filename)?; - let reader = BufReader::new(&file); - let hosts = reader - .lines() - .filter_map(|line| { - let line = line.expect("failed to read input file line!"); - trim_comment(&line) - }) - .map(Host::from) - .collect(); - Ok(hosts) - } -} - -fn trim_comment(line: &str) -> Option { - if let Some(content) = line.split('#').next() { - let trim_content = content.trim().to_string(); - if trim_content.is_empty() { - None - } else { - Some(trim_content) - } - } else { - None - } -} - -#[cfg(test)] -mod constructor_tests { - use super::*; - - #[test] - fn works_with_no_standard_hosts() { - let temp_file = tempfile::NamedTempFile::new().unwrap(); - let store = HostsStore::new(temp_file); - assert_eq!(store.data.domains.len(), 0); - } - - #[test] - fn trim_comments() { - let entries = vec![ - "# keybase", - "keybaseapi.com", - "", - "gist.githubusercontent.com", - " ", - "# healthcheck # foo", - "nymtech.net", - "# blockstream green wallet", - "blockstream.info", - "greenaddress.it", - "91.108.56.0/22", - "2001:b28:f23d::/48", - "2001:67c:4e8::/48", - "# nym matrix server", - "# monero desktop - mainnet", - ]; - let filtered_entries: Vec<_> = entries - .iter() - .filter_map(|line| trim_comment(line)) - .collect(); - assert_eq!( - filtered_entries, - vec![ - "keybaseapi.com", - "gist.githubusercontent.com", - "nymtech.net", - "blockstream.info", - "greenaddress.it", - "91.108.56.0/22", - "2001:b28:f23d::/48", - "2001:67c:4e8::/48", - ] - ); - } -} diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/mod.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/mod.rs deleted file mode 100644 index 225f6418b5..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -mod filter; -mod group; -mod host; -mod hosts; -pub(crate) mod standard_list; -pub(crate) mod stored_allowed_hosts; - -pub(crate) use filter::OutboundRequestFilter; -pub(crate) use hosts::HostsStore; -pub(crate) use standard_list::StandardList; diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/standard_list.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/standard_list.rs deleted file mode 100644 index 79b7c75901..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/standard_list.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use crate::request_filter::allowed_hosts::group::HostsGroup; -use crate::request_filter::allowed_hosts::host::Host; -use nym_task::TaskClient; -use regex::Regex; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{RwLock, RwLockReadGuard}; - -const STANDARD_LIST_URL: &str = - "https://nymtech.net/.wellknown/network-requester/standard-allowed-list.txt"; - -fn remove_comments(text: String) -> String { - if let Ok(regex) = Regex::new(r"#.*\n") { - regex.replace_all(&text, "").into_owned() - } else { - log::warn!("Failed to strip comments from standard allowed list"); - text - } -} - -/// Fetch the standard allowed list from nymtech.net -pub(crate) async fn fetch() -> Vec { - log::info!("Refreshing standard allowed hosts"); - let text = get_standard_allowed_list().await; - remove_comments(text) - .split_whitespace() - .map(Into::into) - .collect() -} - -async fn get_standard_allowed_list() -> String { - reqwest::get(STANDARD_LIST_URL) - .await - .expect("failed to get allowed hosts") - .text() - .await - .expect("failed to get allowed hosts text") -} - -#[derive(Clone, Debug)] -pub(crate) struct StandardList { - inner: Arc>, -} - -impl StandardList { - // note: standard list will be fetched immediately when `StandardListUpdater::run` is called - // (because first `tick()` of tokio interval fires up immediately) - pub(crate) fn new() -> Self { - StandardList { - inner: Arc::new(RwLock::new(HostsGroup::new(Vec::new()))), - } - } - - pub(crate) async fn update(&self) { - let raw_standard_list = fetch().await; - log::debug!("fetched allowed hosts: {:?}", raw_standard_list); - - let new_data = HostsGroup::new(raw_standard_list); - *self.inner.write().await = new_data - } - - pub(crate) async fn get(&self) -> RwLockReadGuard<'_, HostsGroup> { - self.inner.read().await - } -} - -pub(crate) struct StandardListUpdater { - update_interval: Duration, - standard_list: StandardList, - - // Listens to shutdown commands from higher up - shutdown_listener: TaskClient, -} - -impl StandardListUpdater { - pub(crate) fn new( - update_interval: Duration, - standard_list: StandardList, - shutdown_listener: TaskClient, - ) -> Self { - Self { - update_interval, - standard_list, - shutdown_listener, - } - } - - pub(crate) async fn run(&mut self) { - let mut update_interval = tokio::time::interval(self.update_interval); - - while !self.shutdown_listener.is_shutdown() { - tokio::select! { - biased; - _ = self.shutdown_listener.recv() => { - log::trace!("StandardListUpdater: Received shutdown"); - } - _ = update_interval.tick() => { - log::debug!("updating standard list"); - self.standard_list.update().await - } - } - } - - log::debug!("StandardListUpdater: Exiting"); - } - - pub(crate) fn start(mut self) { - tokio::spawn(async move { self.run().await }); - } -} diff --git a/service-providers/network-requester/src/request_filter/allowed_hosts/stored_allowed_hosts.rs b/service-providers/network-requester/src/request_filter/allowed_hosts/stored_allowed_hosts.rs deleted file mode 100644 index a2e2537dba..0000000000 --- a/service-providers/network-requester/src/request_filter/allowed_hosts/stored_allowed_hosts.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: GPL-3.0-only - -use crate::request_filter::allowed_hosts::HostsStore; -use futures::channel::mpsc; -use futures::StreamExt; -use nym_async_file_watcher::{AsyncFileWatcher, FileWatcherEventReceiver}; -use nym_task::TaskClient; -use std::io; -use std::path::Path; -use std::sync::Arc; -use tokio::sync::{RwLock, RwLockReadGuard}; - -#[derive(Debug, Clone)] -pub(crate) struct StoredAllowedHosts { - inner: Arc>, -} - -impl StoredAllowedHosts { - pub(crate) fn new>(path: P) -> Self { - let allowed_hosts = HostsStore::new(path); - - StoredAllowedHosts { - inner: Arc::new(RwLock::new(allowed_hosts)), - } - } - - pub(crate) async fn reload(&self) -> io::Result<()> { - log::debug!("reloading stored allowed hosts"); - self.inner.write().await.try_reload() - } - - pub(crate) async fn get(&self) -> RwLockReadGuard<'_, HostsStore> { - self.inner.read().await - } -} - -impl From for StoredAllowedHosts { - fn from(value: HostsStore) -> Self { - StoredAllowedHosts { - inner: Arc::new(RwLock::new(value)), - } - } -} - -pub(crate) struct StoredAllowedHostsReloader { - stored_hosts: StoredAllowedHosts, - events_receiver: FileWatcherEventReceiver, - - // Listens to shutdown commands from higher up - shutdown_listener: TaskClient, -} - -impl StoredAllowedHostsReloader { - pub(crate) fn new( - stored_hosts: StoredAllowedHosts, - events_receiver: FileWatcherEventReceiver, - shutdown_listener: TaskClient, - ) -> Self { - StoredAllowedHostsReloader { - events_receiver, - stored_hosts, - shutdown_listener, - } - } - - pub(crate) async fn run(&mut self) { - while !self.shutdown_listener.is_shutdown() { - tokio::select! { - biased; - _ = self.shutdown_listener.recv() => { - log::trace!("StoredAllowedHostsReloader: Received shutdown"); - } - event = self.events_receiver.next() => { - let Some(event) = event else { - log::trace!("StoredAllowedHostsReloader: sender channel has terminated"); - break - }; - log::debug!("the file has changed - {event:?}"); - log::debug!("reloading stored hosts"); - if let Err(err) = self.stored_hosts.reload().await { - log::error!("failed to reload stored hosts: {err}") - } - } - } - } - - log::debug!("StoredAllowedHostsReloader: Exiting"); - } - - pub(crate) fn start(mut self) { - tokio::spawn(async move { self.run().await }); - } -} - -async fn run_watcher(mut watcher: AsyncFileWatcher, mut shutdown: TaskClient) { - tokio::select! { - biased; - _ = shutdown.recv() => { - log::trace!("AsyncFileWatcher: Received shutdown"); - } - res = watcher.watch() => { - log::trace!("AsyncFileWatcher: finished with {res:?}"); - } - } - log::debug!("AsyncFileWatcher: Exiting"); -} - -fn start_watcher(watcher: AsyncFileWatcher, shutdown: TaskClient) { - tokio::spawn(async move { run_watcher(watcher, shutdown).await }); -} - -pub(crate) async fn start_allowed_list_reloader( - stored_list: StoredAllowedHosts, - shutdown_listener: TaskClient, -) { - let (events_sender, events_receiver) = mpsc::unbounded(); - let file = stored_list.get().await.storefile.clone(); - - let watcher = AsyncFileWatcher::new_file_changes_watcher(file, events_sender) - .expect("failed to create file watcher"); - let reloader = - StoredAllowedHostsReloader::new(stored_list, events_receiver, shutdown_listener.clone()); - - start_watcher(watcher, shutdown_listener); - reloader.start() -} diff --git a/service-providers/network-requester/src/request_filter/exit_policy/mod.rs b/service-providers/network-requester/src/request_filter/exit_policy/mod.rs index 5fdcc28b57..d3c3a99681 100644 --- a/service-providers/network-requester/src/request_filter/exit_policy/mod.rs +++ b/service-providers/network-requester/src/request_filter/exit_policy/mod.rs @@ -1,6 +1,7 @@ -// Copyright 2023 - Nym Technologies SA +// Copyright 2023-2024 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only +use crate::config::Config; use crate::error::NetworkRequesterError; use log::trace; use nym_exit_policy::client::get_exit_policy; @@ -15,6 +16,12 @@ pub struct ExitPolicyRequestFilter { policy: ExitPolicy, } +impl From for ExitPolicyRequestFilter { + fn from(value: ExitPolicy) -> Self { + ExitPolicyRequestFilter::new_from_policy(value) + } +} + impl ExitPolicyRequestFilter { pub(crate) async fn new_upstream(url: impl IntoUrl) -> Result { let url = url @@ -27,7 +34,21 @@ impl ExitPolicyRequestFilter { }) } - pub(crate) fn new(policy: ExitPolicy) -> Self { + pub(crate) async fn new(config: &Config) -> Result { + let policy_filter = if config.network_requester.open_proxy { + ExitPolicyRequestFilter::new_from_policy(ExitPolicy::new_open()) + } else { + let upstream_url = config + .network_requester + .upstream_exit_policy_url + .as_ref() + .ok_or(NetworkRequesterError::NoUpstreamExitPolicy)?; + ExitPolicyRequestFilter::new_upstream(upstream_url.clone()).await? + }; + Ok(policy_filter) + } + + pub fn new_from_policy(policy: ExitPolicy) -> Self { ExitPolicyRequestFilter { upstream: None, policy, diff --git a/service-providers/network-requester/src/request_filter/mod.rs b/service-providers/network-requester/src/request_filter/mod.rs index 25f34ae591..b1534ce7b8 100644 --- a/service-providers/network-requester/src/request_filter/mod.rs +++ b/service-providers/network-requester/src/request_filter/mod.rs @@ -1,143 +1,36 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use crate::config::{self, Config}; +use crate::config::Config; use crate::error::NetworkRequesterError; -use crate::request_filter::allowed_hosts::standard_list::StandardListUpdater; -use crate::request_filter::allowed_hosts::stored_allowed_hosts::{ - start_allowed_list_reloader, StoredAllowedHosts, -}; -use crate::request_filter::allowed_hosts::{OutboundRequestFilter, StandardList}; -use crate::request_filter::exit_policy::ExitPolicyRequestFilter; -use log::{info, warn}; -use nym_exit_policy::ExitPolicy; +use log::warn; use nym_socks5_requests::RemoteAddress; -use nym_task::TaskHandle; use std::sync::Arc; -/// Old request filtering based on the allowed.list files. -pub mod allowed_hosts; pub mod exit_policy; -enum RequestFilterInner { - AllowList { - open_proxy: bool, - filter: OutboundRequestFilter, - }, - ExitPolicy { - policy_filter: ExitPolicyRequestFilter, - }, -} +pub use exit_policy::ExitPolicyRequestFilter; #[derive(Clone)] pub struct RequestFilter { - inner: Arc, + inner: Arc, } impl RequestFilter { pub(crate) async fn new(config: &Config) -> Result { - if config.network_requester.use_deprecated_allow_list { - info!("setting up allow-list based 'OutboundRequestFilter'..."); - Ok(Self::new_allow_list_request_filter(config).await) - } else { - info!("setting up ExitPolicy based request filter..."); - Self::new_exit_policy_filter(config).await - } - } - - pub fn current_exit_policy_filter(&self) -> Option<&ExitPolicyRequestFilter> { - match &*self.inner { - RequestFilterInner::AllowList { .. } => None, - RequestFilterInner::ExitPolicy { policy_filter } => Some(policy_filter), - } - } - - pub(crate) async fn start_update_tasks( - &self, - config: &config::Debug, - task_handle: &TaskHandle, - ) { - match &*self.inner { - RequestFilterInner::AllowList { open_proxy, filter } => { - // if we're running in open proxy, we don't have to spawn any refreshers, - // after all, we're going to be accepting all requests regardless - // of the local allow list or the standard list - if *open_proxy { - return; - } - - // start the standard list updater - StandardListUpdater::new( - config.standard_list_update_interval, - filter.standard_list(), - task_handle.get_handle().named("StandardListUpdater"), - ) - .start(); - - // start the allowed.list watcher and updater - start_allowed_list_reloader( - filter.allowed_hosts(), - task_handle - .get_handle() - .named("stored_allowed_hosts_reloader"), - ) - .await; - } - RequestFilterInner::ExitPolicy { .. } => { - // nothing to do for the exit policy (yet; we might add a refresher at some point) - } - } - } - - async fn new_allow_list_request_filter(config: &Config) -> Self { - let standard_list = StandardList::new(); - let allowed_hosts = StoredAllowedHosts::new(&config.storage_paths.allowed_list_location); - let unknown_hosts = - allowed_hosts::HostsStore::new(&config.storage_paths.unknown_list_location); - - // TODO: technically if we're running open proxy, we don't have to be loading anything here - RequestFilter { - inner: Arc::new(RequestFilterInner::AllowList { - open_proxy: config.network_requester.open_proxy, - filter: OutboundRequestFilter::new(allowed_hosts, standard_list, unknown_hosts) - .await, - }), - } - } - - async fn new_exit_policy_filter(config: &Config) -> Result { - let policy_filter = if config.network_requester.open_proxy { - ExitPolicyRequestFilter::new(ExitPolicy::new_open()) - } else { - let upstream_url = config - .network_requester - .upstream_exit_policy_url - .as_ref() - .ok_or(NetworkRequesterError::NoUpstreamExitPolicy)?; - ExitPolicyRequestFilter::new_upstream(upstream_url.clone()).await? - }; Ok(RequestFilter { - inner: Arc::new(RequestFilterInner::ExitPolicy { policy_filter }), + inner: Arc::new(ExitPolicyRequestFilter::new(config).await?), }) } + pub fn current_exit_policy_filter(&self) -> &ExitPolicyRequestFilter { + &*self.inner + } + pub(crate) async fn check_address(&self, address: &RemoteAddress) -> bool { - match &*self.inner { - RequestFilterInner::AllowList { open_proxy, filter } => { - if *open_proxy { - return true; - } - filter.check(address).await - } - RequestFilterInner::ExitPolicy { policy_filter } => { - match policy_filter.check(address).await { - Err(err) => { - warn!("failed to validate '{address}' against the exit policy: {err}"); - false - } - Ok(res) => res, - } - } - } + self.inner.check(address).await.unwrap_or_else(|err| { + warn!("failed to validate '{address}' against the exit policy: {err}"); + false + }) } }