GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed the usage of allow.list

Browse Source
This commit is contained in:
Jędrzej Stuczyński
2024-03-13 10:18:03 +00:00
parent 588ed7dead
commit 2aa5e25532
22 changed files with 53 additions and 1334 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/types/src/gateway.rs
+1 -8
View File
@@ -124,15 +124,12 @@ pub struct GatewayNetworkRequesterDetails {
pub encryption_key: String,
pub open_proxy: bool,
pub exit_policy: bool,
pub enabled_statistics: bool,
// just a convenience wrapper around all the keys
pub address: String,
pub config_path: String,
pub allow_list_path: String,
pub unknown_list_path: String,
}
impl fmt::Display for GatewayNetworkRequesterDetails {
@@ -146,11 +143,7 @@ impl fmt::Display for GatewayNetworkRequesterDetails {
writeln!(f, "\taddress: {}", self.address)?;
writeln!(f, "\tuses open proxy: {}", self.open_proxy)?;
writeln!(f, "\tuses exit policy: {}", self.exit_policy)?;
writeln!(f, "\tsends statistics: {}", self.enabled_statistics)?;
writeln!(f, "\tallow list path: {}", self.allow_list_path)?;
writeln!(f, "\tunknown list path: {}", self.unknown_list_path)
writeln!(f, "\tsends statistics: {}", self.enabled_statistics)
}
}
gateway/src/commands/helpers.rs
-16
View File
@@ -109,7 +109,6 @@ pub(crate) struct OverrideNetworkRequesterConfig {
pub(crate) medium_toggle: bool,
pub(crate) open_proxy: Option<bool>,
pub(crate) enable_exit_policy: Option<bool>,
pub(crate) enable_statistics: Option<bool>,
pub(crate) statistics_recipient: Option<String>,
@@ -229,10 +228,6 @@ pub(crate) fn override_network_requester_config(
nym_network_requester::Config::with_open_proxy,
opts.open_proxy,
)
.with_optional(
nym_network_requester::Config::with_old_allow_list,
opts.enable_exit_policy.map(|e| !e),
)
.with_optional(
nym_network_requester::Config::with_enabled_statistics,
opts.enable_statistics,
@@ -314,21 +309,10 @@ pub(crate) async fn initialise_local_network_requester(
enabled: gateway_config.network_requester.enabled,
identity_key: address.identity().to_string(),
encryption_key: address.encryption_key().to_string(),
exit_policy: !nr_cfg.network_requester.use_deprecated_allow_list,
open_proxy: nr_cfg.network_requester.open_proxy,
enabled_statistics: nr_cfg.network_requester.enabled_statistics,
address: address.to_string(),
config_path: nr_cfg_path.display().to_string(),
allow_list_path: nr_cfg
.storage_paths
.allowed_list_location
.display()
.to_string(),
unknown_list_path: nr_cfg
.storage_paths
.unknown_list_location
.display()
.to_string(),
})
}
gateway/src/commands/init.rs
-8
View File
@@ -133,12 +133,6 @@ pub struct Init {
)]
medium_toggle: bool,
/// Specifies whether this network requester will run using the default ExitPolicy
/// as opposed to the allow list.
/// Note: this setting will become the default in the future releases.
#[clap(long)]
with_exit_policy: Option<bool>,
#[clap(short, long, default_value_t = OutputFormat::default())]
output: OutputFormat,
}
@@ -173,7 +167,6 @@ impl<'a> From<&'a Init> for OverrideNetworkRequesterConfig {
no_cover: value.no_cover,
medium_toggle: value.medium_toggle,
open_proxy: value.open_proxy,
enable_exit_policy: value.with_exit_policy,
enable_statistics: value.enable_statistics,
statistics_recipient: value.statistics_recipient.clone(),
}
@@ -303,7 +296,6 @@ mod tests {
fastmode: false,
no_cover: false,
medium_toggle: false,
with_exit_policy: None,
};
std::env::set_var(BECH32_PREFIX, "n");
gateway/src/commands/run.rs
-7
View File
@@ -132,12 +132,6 @@ pub struct Run {
#[arg(long, group = "network", hide = true)]
custom_mixnet: Option<PathBuf>,
/// Specifies whether this network requester will run using the default ExitPolicy
/// as opposed to the allow list.
/// Note: this setting will become the default in the future releases.
#[arg(long)]
with_exit_policy: Option<bool>,
#[arg(short, long, default_value_t = OutputFormat::default())]
output: OutputFormat,
@@ -175,7 +169,6 @@ impl<'a> From<&'a Run> for OverrideNetworkRequesterConfig {
no_cover: value.no_cover,
medium_toggle: value.medium_toggle,
open_proxy: value.open_proxy,
enable_exit_policy: value.with_exit_policy,
enable_statistics: value.enable_statistics,
statistics_recipient: value.statistics_recipient.clone(),
}
gateway/src/commands/setup_network_requester.rs
-7
View File
@@ -59,12 +59,6 @@ pub struct CmdArgs {
)]
medium_toggle: bool,
/// Specifies whether this network requester will run using the default ExitPolicy
/// as opposed to the allow list.
/// Note: this setting will become the default in the future releases.
#[clap(long)]
with_exit_policy: Option<bool>,
#[clap(short, long, default_value_t = OutputFormat::default())]
output: OutputFormat,
}
@@ -76,7 +70,6 @@ impl<'a> From<&'a CmdArgs> for OverrideNetworkRequesterConfig {
no_cover: value.no_cover,
medium_toggle: value.medium_toggle,
open_proxy: value.open_proxy,
enable_exit_policy: value.with_exit_policy,
enable_statistics: value.enable_statistics,
statistics_recipient: value.statistics_recipient.clone(),
}
gateway/src/http/mod.rs
+5 -10
View File
@@ -199,13 +199,11 @@ impl<'a> HttpApiBuilder<'a> {
// once we start refreshing it, we'll have to change it, but at that point
// the allow list will be probably be completely removed and thus the pointer management
// will be much easier
let Some(exit_policy) = request_filter.current_exit_policy_filter() else {
warn!("this node does not use an exit policy. no changes will be made");
return self;
};
let upstream = request_filter.current_exit_policy_filter().upstream();
// if there's no upstream (i.e. open proxy), we couldn't have possibly updated it : )
let last_updated = if exit_policy.upstream().is_some() {
let last_updated = if upstream.is_some() {
#[allow(clippy::expect_used)]
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -217,12 +215,9 @@ impl<'a> HttpApiBuilder<'a> {
self.exit_policy = Some(UsedExitPolicy {
enabled: true,
upstream_source: exit_policy
.upstream()
.map(|u| u.to_string())
.unwrap_or_default(),
upstream_source: upstream.map(|u| u.to_string()).unwrap_or_default(),
last_updated,
policy: Some(exit_policy.policy().clone()),
policy: Some(request_filter.current_exit_policy_filter().policy().clone()),
});
self
gateway/src/node/helpers.rs
-3
View File
@@ -64,12 +64,9 @@ pub(crate) async fn node_details(
identity_key: nr_identity_public_key.to_base58_string(),
encryption_key: nr_encryption_key.to_base58_string(),
open_proxy: cfg.network_requester.open_proxy,
exit_policy: !cfg.network_requester.use_deprecated_allow_list,
enabled_statistics: cfg.network_requester.enabled_statistics,
address: address.to_string(),
config_path: display_path(nr_cfg_path),
allow_list_path: display_path(&cfg.storage_paths.allowed_list_location),
unknown_list_path: display_path(&cfg.storage_paths.unknown_list_location),
})
} else {
None
service-providers/network-requester/src/cli/init.rs
-7
View File
@@ -57,12 +57,6 @@ pub(crate) struct Init {
#[clap(long)]
statistics_recipient: Option<String>,
/// Specifies whether this network requester will run using the default ExitPolicy
/// as opposed to the allow list.
/// Note: this setting will become the default in the future releases.
#[clap(long)]
with_exit_policy: Option<bool>,
#[clap(short, long, default_value_t = OutputFormat::default())]
output: OutputFormat,
}
@@ -76,7 +70,6 @@ impl From<Init> for OverrideConfig {
medium_toggle: false,
nyxd_urls: init_config.common_args.nyxd_urls,
enabled_credentials_mode: init_config.common_args.enabled_credentials_mode,
enable_exit_policy: init_config.with_exit_policy,
open_proxy: init_config.open_proxy,
enable_statistics: init_config.enable_statistics,
statistics_recipient: init_config.statistics_recipient,
service-providers/network-requester/src/cli/mod.rs
-5
View File
@@ -104,7 +104,6 @@ pub(crate) struct OverrideConfig {
medium_toggle: bool,
nyxd_urls: Option<Vec<url::Url>>,
enabled_credentials_mode: Option<bool>,
enable_exit_policy: Option<bool>,
open_proxy: Option<bool>,
enable_statistics: Option<bool>,
@@ -165,10 +164,6 @@ pub(crate) fn override_config(mut config: Config, args: OverrideConfig) -> Confi
args.enabled_credentials_mode.map(|b| !b),
)
.with_optional(Config::with_open_proxy, args.open_proxy)
.with_optional(
Config::with_old_allow_list,
args.enable_exit_policy.map(|e| !e),
)
.with_optional(Config::with_enabled_statistics, args.enable_statistics)
.with_optional(Config::with_statistics_recipient, args.statistics_recipient)
}
service-providers/network-requester/src/cli/run.rs
-7
View File
@@ -40,12 +40,6 @@ pub(crate) struct Run {
conflicts_with = "fastmode"
)]
medium_toggle: bool,
/// Specifies whether this network requester will run using the default ExitPolicy
/// as opposed to the allow list.
/// Note: this setting will become the default in the future releases.
#[clap(long)]
with_exit_policy: Option<bool>,
}
impl From<Run> for OverrideConfig {
@@ -57,7 +51,6 @@ impl From<Run> for OverrideConfig {
medium_toggle: run_config.medium_toggle,
nyxd_urls: run_config.common_args.nyxd_urls,
enabled_credentials_mode: run_config.common_args.enabled_credentials_mode,
enable_exit_policy: run_config.with_exit_policy,
open_proxy: run_config.open_proxy,
enable_statistics: run_config.enable_statistics,
statistics_recipient: run_config.statistics_recipient,
service-providers/network-requester/src/config/mod.rs
+1 -6
View File
@@ -166,12 +166,6 @@ impl Config {
self
}
#[must_use]
pub fn with_old_allow_list(mut self, use_old_allow_list: bool) -> Self {
self.network_requester.use_deprecated_allow_list = use_old_allow_list;
self
}
#[must_use]
pub fn with_enabled_statistics(mut self, enabled_statistics: bool) -> Self {
self.network_requester.enabled_statistics = enabled_statistics;
@@ -251,6 +245,7 @@ pub struct NetworkRequester {
/// Specifies whether this network requester should be using the deprecated allow-list,
/// as opposed to the new ExitPolicy.
/// Note: this field will be removed in a near future.
#[deprecated]
pub use_deprecated_allow_list: bool,
/// Specifies the url for an upstream source of the exit policy used by this node.
service-providers/network-requester/src/config/persistence.rs
+2
View File
@@ -16,10 +16,12 @@ pub struct NetworkRequesterPaths {
/// Deprecated
/// Location of the file containing our allow.list
#[deprecated]
pub allowed_list_location: PathBuf,
/// Deprecated
/// Location of the file containing our unknown.list
#[deprecated]
pub unknown_list_location: PathBuf,
/// Location of the file containing our description
service-providers/network-requester/src/core.rs
+8 -17
View File
@@ -327,9 +327,6 @@ impl NRServiceProviderBuilder {
});
let request_filter = RequestFilter::new(&self.config).await?;
request_filter
.start_update_tasks(&self.config.network_requester_debug, &shutdown)
.await;
let mut service_provider = NRServiceProvider {
config: self.config,
@@ -605,20 +602,14 @@ impl NRServiceProvider {
QueryResponse::Description("Description (placeholder)".to_string()),
),
QueryRequest::ExitPolicy => {
let response = match self.request_filter.current_exit_policy_filter() {
Some(exit_policy_filter) => QueryResponse::ExitPolicy {
enabled: true,
upstream: exit_policy_filter
.upstream()
.map(|u| u.to_string())
.unwrap_or_default(),
policy: Some(exit_policy_filter.policy().clone()),
},
None => QueryResponse::ExitPolicy {
enabled: false,
upstream: "".to_string(),
policy: None,
},
let exit_policy_filter = self.request_filter.current_exit_policy_filter();
let response = QueryResponse::ExitPolicy {
enabled: true,
upstream: exit_policy_filter
.upstream()
.map(|u| u.to_string())
.unwrap_or_default(),
policy: Some(exit_policy_filter.policy().clone()),
};
Socks5Response::new_query(protocol_version, response)
service-providers/network-requester/src/request_filter/allowed_hosts/filter.rs
-559
View File
@@ -1,559 +0,0 @@
// Copyright 2020-2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use super::HostsStore;
use crate::request_filter::allowed_hosts::group::HostsGroup;
use crate::request_filter::allowed_hosts::standard_list::StandardList;
use crate::request_filter::allowed_hosts::stored_allowed_hosts::StoredAllowedHosts;
use publicsuffix::Psl;
use std::{
net::{IpAddr, SocketAddr},
str::FromStr,
};
use tokio::sync::Mutex;
use url::Url;
#[derive(Debug)]
enum RequestHost {
IpAddr(IpAddr),
SocketAddr(SocketAddr),
RootDomain(String),
}
/// Filters outbound requests based on what's in an `allowed_hosts` list.
///
/// Requests to unknown hosts are automatically written to an `unknown_hosts`
/// list so that they can be copy/pasted into the `allowed_hosts` list if desired.
/// This may be handy for service provider node operators who want to be able to look in the
/// `unknown_hosts` file and allow new hosts (e.g. if a wallet has added a new outbound request
/// which needs to be allowed).
///
/// We rely on the list of domains at https://publicsuffix.org/ to figure out what the root
/// domain is for a given request. This allows us to distinguish all the rules for e.g.
/// .com, .co.uk, .co.jp, uk.com, etc, so that we can distinguish correct root-ish
/// domains as allowed. That list is loaded once at startup from Mozilla's canonical
/// publicsuffix list.
pub(crate) struct OutboundRequestFilter {
pub(super) allowed_hosts: StoredAllowedHosts,
pub(super) standard_list: StandardList,
root_domain_list: publicsuffix::List,
unknown_hosts: Mutex<HostsStore>,
}
/// The official URL of the list
pub const LIST_URL: &str = "https://publicsuffix.org/list/public_suffix_list.dat";
async fn request(u: Url) -> String {
reqwest::get(u)
.await
.expect("failed to get LIST_URL")
.text()
.await
.expect("failed to get text")
}
async fn fetch_list() -> publicsuffix::List {
let str = request(Url::parse(LIST_URL).unwrap()).await;
publicsuffix::List::from_str(&str).unwrap()
}
impl OutboundRequestFilter {
/// Create a new `OutboundRequestFilter` with the given `allowed_hosts` and `unknown_hosts` lists.
///
/// Automatically fetches the latest https://publicsuffix.org/ domain list from the internet so that
/// the requester can properly parse all of the world's top-level domains in an up-to-date fashion.
///
/// Automatcially fetches the latest standard allowed list from the Nym website, so that all
/// requesters are able to support the same minimal functionality out of the box.
pub(crate) async fn new(
allowed_hosts: StoredAllowedHosts,
standard_list: StandardList,
unknown_hosts: HostsStore,
) -> OutboundRequestFilter {
let domain_list = fetch_list().await;
OutboundRequestFilter {
allowed_hosts,
standard_list,
root_domain_list: domain_list,
unknown_hosts: Mutex::new(unknown_hosts),
}
}
pub(crate) fn standard_list(&self) -> StandardList {
self.standard_list.clone()
}
pub(crate) fn allowed_hosts(&self) -> StoredAllowedHosts {
self.allowed_hosts.clone()
}
async fn check_allowed_hosts(&self, host: &RequestHost) -> bool {
let guard = self.allowed_hosts.get().await;
self.check_group(&guard.data, host)
}
async fn check_standard_list(&self, host: &RequestHost) -> bool {
let guard = self.standard_list.get().await;
self.check_group(&guard, host)
}
fn check_group(&self, group: &HostsGroup, host: &RequestHost) -> bool {
match host {
RequestHost::IpAddr(ip_addr) => group.contains_ip_address(*ip_addr),
RequestHost::SocketAddr(socket_addr) => group.contains_ip_address(socket_addr.ip()),
RequestHost::RootDomain(domain) => group.contains_domain(domain),
}
}
async fn add_to_unknown_hosts(&self, host: RequestHost) {
let mut guard = self.unknown_hosts.lock().await;
match host {
RequestHost::IpAddr(ip_addr) => guard.add_ip(ip_addr),
RequestHost::SocketAddr(socket_addr) => guard.add_ip(socket_addr.ip()),
RequestHost::RootDomain(domain) => guard.add_domain(&domain),
}
}
fn parse_request_host(&self, host: &str) -> Option<RequestHost> {
// first check if it's a socket address (ip:port)
// (this check is performed to not incorrectly strip what we think might be a port
// from ipv6 address, as for example ::1 contains colons but has no port
if let Ok(socketaddr) = host.parse::<SocketAddr>() {
Some(RequestHost::SocketAddr(socketaddr))
// then check if it was an ip address
} else if let Ok(ipaddr) = host.parse::<IpAddr>() {
Some(RequestHost::IpAddr(ipaddr))
// finally, then assume it might be a domain
} else {
// check root
let trimmed = Self::trim_port(host);
// if this failed, it was probably some nonsense
self.get_domain_root(&trimmed).map(RequestHost::RootDomain)
}
}
async fn check_request_host(&self, request_host: &RequestHost) -> bool {
// first check our own allow list
let local_allowed = self.check_allowed_hosts(request_host).await;
// if it's locally allowed, no point in checking the standard list
if local_allowed {
return true;
}
// if that failed, check the standard list
self.check_standard_list(request_host).await
}
/// Returns `true` if a host's root domain is in the `allowed_hosts` list.
///
/// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile.
pub(crate) async fn check(&self, host: &str) -> bool {
let allowed = match self.parse_request_host(host) {
Some(request_host) => {
let res = self.check_request_host(&request_host).await;
if !res {
self.add_to_unknown_hosts(request_host).await
}
res
}
None => false,
};
if !allowed {
log::warn!("Blocked outbound connection to {host}, add it to allowed.list if needed",);
}
allowed
}
fn trim_port(host: &str) -> String {
let mut tmp: Vec<_> = host.split(':').collect();
if tmp.len() > 1 {
tmp.pop(); // get rid of last element (port)
tmp.join(":") //rejoin
} else {
host.to_string()
}
}
/// Attempts to get the root domain, shorn of subdomains, using publicsuffix.
/// If the domain is itself registered in publicsuffix (e.g. s3.amazonaws.com),
/// then just use the full address as root.
fn get_domain_root(&self, host: &str) -> Option<String> {
if let Some(domain) = self.root_domain_list.domain(host.as_bytes()) {
return String::from_utf8(domain.as_bytes().to_vec()).ok();
} else if let Some(suffix) = self.root_domain_list.suffix(host.as_bytes()) {
if suffix.is_known() {
return String::from_utf8(suffix.as_bytes().to_vec()).ok();
} else {
return None;
}
}
log::warn!("Error parsing domain: {:?}", host);
None // domain couldn't be parsed
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::ops::{Deref, DerefMut};
struct HostsStoreFixture {
inner: HostsStore,
// take ownership of temp file that will get cleaned up on drop (i.e. when test finishes)
_tmp_file: tempfile::NamedTempFile,
}
impl Deref for HostsStoreFixture {
type Target = HostsStore;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl DerefMut for HostsStoreFixture {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
impl HostsStoreFixture {
fn new() -> HostsStoreFixture {
let tmp_file = tempfile::NamedTempFile::new().unwrap();
let inner = HostsStore::new(&tmp_file);
HostsStoreFixture {
inner,
_tmp_file: tmp_file,
}
}
}
struct OutboundRequestFilterFixture {
inner: OutboundRequestFilter,
// take ownership of temp files that will get cleaned up on drop (i.e. when test finishes)
_allow_tmp_file: tempfile::NamedTempFile,
_unknown_tmp_file: tempfile::NamedTempFile,
}
impl Deref for OutboundRequestFilterFixture {
type Target = OutboundRequestFilter;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl DerefMut for OutboundRequestFilterFixture {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
impl OutboundRequestFilterFixture {
async fn new() -> OutboundRequestFilterFixture {
let allow_tmp_file = tempfile::NamedTempFile::new().unwrap();
let unknown_tmp_file = tempfile::NamedTempFile::new().unwrap();
let allowed = HostsStore::new(&allow_tmp_file);
let unknown = HostsStore::new(&unknown_tmp_file);
let standard = StandardList::new();
let inner = OutboundRequestFilter::new(allowed.into(), standard, unknown).await;
OutboundRequestFilterFixture {
inner,
_allow_tmp_file: allow_tmp_file,
_unknown_tmp_file: unknown_tmp_file,
}
}
}
async fn setup_empty() -> OutboundRequestFilterFixture {
OutboundRequestFilterFixture::new().await
}
async fn setup_with_allowed(allowed: &[&str]) -> OutboundRequestFilterFixture {
let allow_tmp_file = tempfile::NamedTempFile::new().unwrap();
let unknown_tmp_file = tempfile::NamedTempFile::new().unwrap();
let mut allowed_store = HostsStore::new(&allow_tmp_file);
let unknown = HostsStore::new(&unknown_tmp_file);
let standard = StandardList::new();
for allow in allowed {
allowed_store.add_host(allow)
}
let inner = OutboundRequestFilter::new(allowed_store.into(), standard, unknown).await;
OutboundRequestFilterFixture {
inner,
_allow_tmp_file: allow_tmp_file,
_unknown_tmp_file: unknown_tmp_file,
}
}
#[cfg(test)]
mod trimming_port_information {
use super::*;
#[test]
fn happens_when_port_exists() {
let host = "nymtech.net:9999";
assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host));
}
#[test]
fn doesnt_happen_when_no_port_exists() {
let host = "nymtech.net";
assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host));
}
}
#[cfg(test)]
mod getting_the_domain_root {
use super::*;
#[tokio::test]
async fn leaves_a_com_alone() {
let filter = setup_empty().await;
assert_eq!(
Some("domain.com".to_string()),
filter.get_domain_root("domain.com")
)
}
#[tokio::test]
async fn trims_subdomains_from_com() {
let filter = setup_empty().await;
assert_eq!(
Some("domain.com".to_string()),
filter.get_domain_root("foomp.domain.com")
)
}
#[tokio::test]
async fn works_for_non_com_roots() {
let filter = setup_empty().await;
assert_eq!(
Some("domain.co.uk".to_string()),
filter.get_domain_root("domain.co.uk")
)
}
#[tokio::test]
async fn works_for_non_com_roots_with_subdomains() {
let filter = setup_empty().await;
assert_eq!(
Some("domain.co.uk".to_string()),
filter.get_domain_root("foomp.domain.co.uk")
)
}
#[tokio::test]
async fn returns_none_on_garbage() {
let filter = setup_empty().await;
assert_eq!(None, filter.get_domain_root("::/&&%@"));
}
#[tokio::test]
async fn returns_full_on_suffix_domains() {
let filter = setup_empty().await;
dbg!(filter.get_domain_root("s3.amazonaws.com"));
assert_eq!(
Some("s3.amazonaws.com".to_string()),
filter.get_domain_root("s3.amazonaws.com")
);
}
}
#[cfg(test)]
mod requests_to_unknown_hosts {
use super::*;
#[tokio::test]
async fn are_not_allowed() {
let host = "unknown.com";
let filter = setup_empty().await;
assert!(!filter.check(host).await);
}
#[tokio::test]
async fn get_appended_once_to_the_unknown_hosts_list() {
let host = "unknown.com";
let filter = setup_empty().await;
filter.check(host).await;
assert_eq!(1, filter.unknown_hosts.lock().await.data.domains.len());
assert!(filter
.unknown_hosts
.lock()
.await
.data
.domains
.contains("unknown.com"));
filter.check(host).await;
assert_eq!(1, filter.unknown_hosts.lock().await.data.domains.len());
assert!(filter
.unknown_hosts
.lock()
.await
.data
.domains
.contains("unknown.com"));
}
}
#[cfg(test)]
mod requests_to_allowed_hosts {
use super::*;
#[tokio::test]
async fn are_allowed() {
let host = "nymtech.net";
let filter = setup_with_allowed(&["nymtech.net"]).await;
assert!(filter.check(host).await);
}
#[tokio::test]
async fn are_allowed_for_subdomains() {
let host = "foomp.nymtech.net";
let filter = setup_with_allowed(&["nymtech.net"]).await;
assert!(filter.check(host).await);
}
#[tokio::test]
async fn are_not_appended_to_file() {
let filter = setup_with_allowed(&["nymtech.net"]).await;
// test initial state
let lines =
HostsStore::load_from_storefile(&filter.allowed_hosts.get().await.storefile)
.unwrap();
assert_eq!(1, lines.len());
assert!(filter.check("nymtech.net").await);
// test state after we've checked to make sure no unexpected changes
let lines =
HostsStore::load_from_storefile(&filter.allowed_hosts.get().await.storefile)
.unwrap();
assert_eq!(1, lines.len());
}
#[tokio::test]
async fn are_allowed_for_ipv4_addresses() {
let address_good = "1.1.1.1";
let address_good_port = "1.1.1.1:1234";
let address_bad = "1.1.1.2";
let filter = setup_with_allowed(&["1.1.1.1"]).await;
assert!(filter.check(address_good).await);
assert!(filter.check(address_good_port).await);
assert!(!filter.check(address_bad).await);
}
#[tokio::test]
async fn are_allowed_for_ipv6_addresses() {
let ip_v6_full = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
let ip_v6_full_rendered = "2001:0db8:85a3::8a2e:0370:7334";
let ip_v6_full_port = "[2001:0db8:85a3::8a2e:0370:7334]:1234";
let ip_v6_semi = "2001:0db8::0001:0000";
let ip_v6_semi_rendered = "2001:db8::1:0";
let ip_v6_loopback_port = "[::1]:1234";
let filter1 = setup_with_allowed(&[ip_v6_full, ip_v6_semi, "::1"]).await;
let filter2 =
setup_with_allowed(&[ip_v6_full_rendered, ip_v6_semi_rendered, "::1"]).await;
assert!(filter1.check(ip_v6_full).await);
assert!(filter1.check(ip_v6_full_rendered).await);
assert!(filter1.check(ip_v6_full_port).await);
assert!(filter1.check(ip_v6_semi).await);
assert!(filter1.check(ip_v6_semi_rendered).await);
assert!(filter1.check(ip_v6_loopback_port).await);
assert!(filter2.check(ip_v6_full).await);
assert!(filter2.check(ip_v6_full_rendered).await);
assert!(filter2.check(ip_v6_full_port).await);
assert!(filter2.check(ip_v6_semi).await);
assert!(filter2.check(ip_v6_semi_rendered).await);
assert!(filter2.check(ip_v6_loopback_port).await);
}
#[tokio::test]
async fn are_allowed_for_ipv4_address_ranges() {
let range1 = "127.0.0.1/32";
let range2 = "1.2.3.4/24";
let bottom_range2 = "1.2.3.0";
let top_range2 = "1.2.3.255";
let outside_range2 = "1.2.2.4";
let filter = setup_with_allowed(&[range1, range2]).await;
assert!(filter.check("127.0.0.1").await);
assert!(filter.check("127.0.0.1:1234").await);
assert!(filter.check(bottom_range2).await);
assert!(filter.check(top_range2).await);
assert!(!filter.check(outside_range2).await);
}
#[tokio::test]
async fn are_allowed_for_ipv6_address_ranges() {
let range = "2620:0:2d0:200::7/32";
let bottom1 = "2620:0:0:0:0:0:0:0";
let bottom2 = "2620::";
let top = "2620:0:ffff:ffff:ffff:ffff:ffff:ffff";
let mid = "2620:0:42::42";
let filter = setup_with_allowed(&[range]).await;
assert!(filter.check(bottom1).await);
assert!(filter.check(bottom2).await);
assert!(filter.check(top).await);
assert!(filter.check(mid).await);
}
}
#[cfg(test)]
mod creating_a_new_host_store {
use super::*;
#[test]
fn loads_its_host_list_from_storefile() {
let mut host_store = HostsStoreFixture::new();
host_store.add_host("nymtech.net");
host_store.add_host("edwardsnowden.com");
host_store.add_host("1.2.3.4");
host_store.add_host("5.6.7.8/16");
host_store.add_host("1:2:3::");
host_store.add_host("5:6:7::/48");
assert!(host_store.data.domains.contains("nymtech.net"));
assert!(host_store.data.domains.contains("edwardsnowden.com"));
assert!(host_store
.data
.ip_nets
.contains(&"1.2.3.4".parse().unwrap()));
assert!(host_store
.data
.ip_nets
.contains(&"5.6.7.8/16".parse().unwrap()));
assert!(host_store
.data
.ip_nets
.contains(&"1:2:3::".parse().unwrap()));
assert!(host_store
.data
.ip_nets
.contains(&"5:6:7::/48".parse().unwrap()));
}
}
}
service-providers/network-requester/src/request_filter/allowed_hosts/group.rs
-61
View File
@@ -1,61 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::request_filter::allowed_hosts::host::Host;
use ipnetwork::IpNetwork;
use std::collections::HashSet;
use std::net::IpAddr;
/// A simpled grouped set of hosts.
/// It ignores any port information.
#[derive(Debug)]
pub(crate) struct HostsGroup {
pub(super) domains: HashSet<String>,
pub(super) ip_nets: HashSet<IpNetwork>,
}
impl HostsGroup {
pub(crate) fn new(raw_hosts: Vec<Host>) -> HostsGroup {
let mut domains = HashSet::new();
let mut ip_nets = HashSet::new();
for host in raw_hosts {
match host {
Host::Domain(domain) => {
domains.insert(domain);
}
Host::IpNetwork(ipnet) => {
ip_nets.insert(ipnet);
}
}
}
HostsGroup { domains, ip_nets }
}
pub(crate) fn contains_domain(&self, host: &str) -> bool {
self.domains.contains(host)
}
pub(super) fn contains_ip_address(&self, address: IpAddr) -> bool {
for ip_net in &self.ip_nets {
if ip_net.contains(address) {
return true;
}
}
false
}
pub(super) fn contains_ip_network(&self, network: IpNetwork) -> bool {
self.ip_nets.contains(&network)
}
pub(super) fn add_ipnet<N: Into<IpNetwork>>(&mut self, network: N) {
self.ip_nets.insert(network.into());
}
pub(super) fn add_domain(&mut self, domain: &str) {
self.domains.insert(domain.to_string());
}
}
service-providers/network-requester/src/request_filter/allowed_hosts/host.rs
-35
View File
@@ -1,35 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use ipnetwork::IpNetwork;
use std::str::FromStr;
// used for parsing file content
#[derive(Debug)]
pub(crate) enum Host {
Domain(String),
IpNetwork(IpNetwork),
}
impl<S: AsRef<str>> From<S> for Host {
fn from(raw: S) -> Self {
// SAFETY: unwrap here is fine as `FromStr` implementation returns `Infallible` error.
raw.as_ref().parse().unwrap()
}
}
impl FromStr for Host {
type Err = core::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
if let Ok(ipnet) = s.parse() {
Ok(Host::IpNetwork(ipnet))
} else {
// TODO: perhaps in the future it should do some domain validation?
//
// So for example if somebody put some nonsense in the whitelist file like "foomp",
// it would get rejected?
Ok(Host::Domain(s.to_string()))
}
}
}
service-providers/network-requester/src/request_filter/allowed_hosts/hosts.rs
-203
View File
@@ -1,203 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use super::host::Host;
use crate::request_filter::allowed_hosts::group::HostsGroup;
use ipnetwork::IpNetwork;
use std::{
fs::{self, File, OpenOptions},
io::{self, BufRead, BufReader},
net::IpAddr,
path::{Path, PathBuf},
};
/// A simple file-backed store for information about allowed / unknown hosts.
/// It ignores any port information.
#[derive(Debug)]
pub(crate) struct HostsStore {
pub(super) storefile: PathBuf,
pub(super) data: HostsGroup,
}
impl HostsStore {
/// Constructs a new HostsStore. If the storefile does not exist, it will be created.
///
/// You can inject a list of standard hosts that you want to support, in addition to the ones
/// in the user-defined storefile.
pub(crate) fn new<P: AsRef<Path>>(storefile: P) -> HostsStore {
let storefile = storefile.as_ref().to_path_buf();
if storefile.exists() && !storefile.is_file() {
// there's no error handling in here and I'm not going to be changing it right now.
panic!(
"the provided storefile {:?} is not a valid file!",
storefile
)
}
Self::setup_storefile(&storefile);
let hosts = Self::load_from_storefile(&storefile)
.unwrap_or_else(|_| panic!("Could not load hosts from storefile at {storefile:?}"));
HostsStore {
storefile,
data: HostsGroup::new(hosts),
}
}
pub(crate) fn try_reload(&mut self) -> io::Result<()> {
let hosts = Self::load_from_storefile(&self.storefile)?;
self.data = HostsGroup::new(hosts);
Ok(())
}
pub(crate) fn contains_domain(&self, host: &str) -> bool {
self.data.contains_domain(host)
}
pub(super) fn contains_ip_address(&self, address: IpAddr) -> bool {
self.data.contains_ip_address(address)
}
#[allow(unused)]
pub(super) fn contains_ipnetwork(&self, network: IpNetwork) -> bool {
self.data.contains_ip_network(network)
}
#[allow(unused)]
pub(super) fn add_host<H: Into<Host>>(&mut self, host: H) {
match host.into() {
Host::Domain(domain) => self.add_domain(&domain),
Host::IpNetwork(ipnet) => self.add_ipnet(ipnet),
}
}
#[allow(unused)]
pub(super) fn add_ipnet(&mut self, network: IpNetwork) {
if !self.contains_ipnetwork(network) {
self.data.add_ipnet(network);
self.append_to_file(&network.to_string());
}
}
pub(super) fn add_ip(&mut self, ip: IpAddr) {
if !self.contains_ip_address(ip) {
self.data.add_ipnet(ip);
self.append_to_file(&ip.to_string());
}
}
pub(super) fn add_domain(&mut self, domain: &str) {
if !self.contains_domain(domain) {
self.data.add_domain(domain);
self.append_to_file(domain);
}
}
/// Appends a line of `text` to the storefile at `path`
pub(super) fn append(path: &Path, text: &str) {
use std::io::Write;
let mut file = OpenOptions::new().append(true).open(path).unwrap();
if let Err(e) = writeln!(file, "{text}") {
log::error!("Couldn't write to file: {e}");
}
}
fn append_to_file(&self, host: &str) {
HostsStore::append(&self.storefile, host);
}
fn setup_storefile(file: &PathBuf) {
if !file.exists() {
let parent_dir = file
.parent()
.expect("parent dir does not exist for {file:?}");
fs::create_dir_all(parent_dir)
.unwrap_or_else(|_| panic!("could not create storage directory at {parent_dir:?}"));
log::trace!("Creating: {file:?}");
File::create(file).unwrap();
}
}
/// Loads the storefile contents into memory.
pub(super) fn load_from_storefile<P>(filename: P) -> io::Result<Vec<Host>>
where
P: AsRef<Path>,
{
log::trace!("Loading from storefile: {}", filename.as_ref().display());
let file = File::open(filename)?;
let reader = BufReader::new(&file);
let hosts = reader
.lines()
.filter_map(|line| {
let line = line.expect("failed to read input file line!");
trim_comment(&line)
})
.map(Host::from)
.collect();
Ok(hosts)
}
}
fn trim_comment(line: &str) -> Option<String> {
if let Some(content) = line.split('#').next() {
let trim_content = content.trim().to_string();
if trim_content.is_empty() {
None
} else {
Some(trim_content)
}
} else {
None
}
}
#[cfg(test)]
mod constructor_tests {
use super::*;
#[test]
fn works_with_no_standard_hosts() {
let temp_file = tempfile::NamedTempFile::new().unwrap();
let store = HostsStore::new(temp_file);
assert_eq!(store.data.domains.len(), 0);
}
#[test]
fn trim_comments() {
let entries = vec![
"# keybase",
"keybaseapi.com",
"",
"gist.githubusercontent.com",
" ",
"# healthcheck # foo",
"nymtech.net",
"# blockstream green wallet",
"blockstream.info",
"greenaddress.it",
"91.108.56.0/22",
"2001:b28:f23d::/48",
"2001:67c:4e8::/48",
"# nym matrix server",
"# monero desktop - mainnet",
];
let filtered_entries: Vec<_> = entries
.iter()
.filter_map(|line| trim_comment(line))
.collect();
assert_eq!(
filtered_entries,
vec![
"keybaseapi.com",
"gist.githubusercontent.com",
"nymtech.net",
"blockstream.info",
"greenaddress.it",
"91.108.56.0/22",
"2001:b28:f23d::/48",
"2001:67c:4e8::/48",
]
);
}
}
service-providers/network-requester/src/request_filter/allowed_hosts/mod.rs
-13
View File
@@ -1,13 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
mod filter;
mod group;
mod host;
mod hosts;
pub(crate) mod standard_list;
pub(crate) mod stored_allowed_hosts;
pub(crate) use filter::OutboundRequestFilter;
pub(crate) use hosts::HostsStore;
pub(crate) use standard_list::StandardList;
service-providers/network-requester/src/request_filter/allowed_hosts/standard_list.rs
-113
View File
@@ -1,113 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::request_filter::allowed_hosts::group::HostsGroup;
use crate::request_filter::allowed_hosts::host::Host;
use nym_task::TaskClient;
use regex::Regex;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{RwLock, RwLockReadGuard};
const STANDARD_LIST_URL: &str =
"https://nymtech.net/.wellknown/network-requester/standard-allowed-list.txt";
fn remove_comments(text: String) -> String {
if let Ok(regex) = Regex::new(r"#.*\n") {
regex.replace_all(&text, "").into_owned()
} else {
log::warn!("Failed to strip comments from standard allowed list");
text
}
}
/// Fetch the standard allowed list from nymtech.net
pub(crate) async fn fetch() -> Vec<Host> {
log::info!("Refreshing standard allowed hosts");
let text = get_standard_allowed_list().await;
remove_comments(text)
.split_whitespace()
.map(Into::into)
.collect()
}
async fn get_standard_allowed_list() -> String {
reqwest::get(STANDARD_LIST_URL)
.await
.expect("failed to get allowed hosts")
.text()
.await
.expect("failed to get allowed hosts text")
}
#[derive(Clone, Debug)]
pub(crate) struct StandardList {
inner: Arc<RwLock<HostsGroup>>,
}
impl StandardList {
// note: standard list will be fetched immediately when `StandardListUpdater::run` is called
// (because first `tick()` of tokio interval fires up immediately)
pub(crate) fn new() -> Self {
StandardList {
inner: Arc::new(RwLock::new(HostsGroup::new(Vec::new()))),
}
}
pub(crate) async fn update(&self) {
let raw_standard_list = fetch().await;
log::debug!("fetched allowed hosts: {:?}", raw_standard_list);
let new_data = HostsGroup::new(raw_standard_list);
*self.inner.write().await = new_data
}
pub(crate) async fn get(&self) -> RwLockReadGuard<'_, HostsGroup> {
self.inner.read().await
}
}
pub(crate) struct StandardListUpdater {
update_interval: Duration,
standard_list: StandardList,
// Listens to shutdown commands from higher up
shutdown_listener: TaskClient,
}
impl StandardListUpdater {
pub(crate) fn new(
update_interval: Duration,
standard_list: StandardList,
shutdown_listener: TaskClient,
) -> Self {
Self {
update_interval,
standard_list,
shutdown_listener,
}
}
pub(crate) async fn run(&mut self) {
let mut update_interval = tokio::time::interval(self.update_interval);
while !self.shutdown_listener.is_shutdown() {
tokio::select! {
biased;
_ = self.shutdown_listener.recv() => {
log::trace!("StandardListUpdater: Received shutdown");
}
_ = update_interval.tick() => {
log::debug!("updating standard list");
self.standard_list.update().await
}
}
}
log::debug!("StandardListUpdater: Exiting");
}
pub(crate) fn start(mut self) {
tokio::spawn(async move { self.run().await });
}
}
service-providers/network-requester/src/request_filter/allowed_hosts/stored_allowed_hosts.rs
-127
View File
@@ -1,127 +0,0 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::request_filter::allowed_hosts::HostsStore;
use futures::channel::mpsc;
use futures::StreamExt;
use nym_async_file_watcher::{AsyncFileWatcher, FileWatcherEventReceiver};
use nym_task::TaskClient;
use std::io;
use std::path::Path;
use std::sync::Arc;
use tokio::sync::{RwLock, RwLockReadGuard};
#[derive(Debug, Clone)]
pub(crate) struct StoredAllowedHosts {
inner: Arc<RwLock<HostsStore>>,
}
impl StoredAllowedHosts {
pub(crate) fn new<P: AsRef<Path>>(path: P) -> Self {
let allowed_hosts = HostsStore::new(path);
StoredAllowedHosts {
inner: Arc::new(RwLock::new(allowed_hosts)),
}
}
pub(crate) async fn reload(&self) -> io::Result<()> {
log::debug!("reloading stored allowed hosts");
self.inner.write().await.try_reload()
}
pub(crate) async fn get(&self) -> RwLockReadGuard<'_, HostsStore> {
self.inner.read().await
}
}
impl From<HostsStore> for StoredAllowedHosts {
fn from(value: HostsStore) -> Self {
StoredAllowedHosts {
inner: Arc::new(RwLock::new(value)),
}
}
}
pub(crate) struct StoredAllowedHostsReloader {
stored_hosts: StoredAllowedHosts,
events_receiver: FileWatcherEventReceiver,
// Listens to shutdown commands from higher up
shutdown_listener: TaskClient,
}
impl StoredAllowedHostsReloader {
pub(crate) fn new(
stored_hosts: StoredAllowedHosts,
events_receiver: FileWatcherEventReceiver,
shutdown_listener: TaskClient,
) -> Self {
StoredAllowedHostsReloader {
events_receiver,
stored_hosts,
shutdown_listener,
}
}
pub(crate) async fn run(&mut self) {
while !self.shutdown_listener.is_shutdown() {
tokio::select! {
biased;
_ = self.shutdown_listener.recv() => {
log::trace!("StoredAllowedHostsReloader: Received shutdown");
}
event = self.events_receiver.next() => {
let Some(event) = event else {
log::trace!("StoredAllowedHostsReloader: sender channel has terminated");
break
};
log::debug!("the file has changed - {event:?}");
log::debug!("reloading stored hosts");
if let Err(err) = self.stored_hosts.reload().await {
log::error!("failed to reload stored hosts: {err}")
}
}
}
}
log::debug!("StoredAllowedHostsReloader: Exiting");
}
pub(crate) fn start(mut self) {
tokio::spawn(async move { self.run().await });
}
}
async fn run_watcher(mut watcher: AsyncFileWatcher, mut shutdown: TaskClient) {
tokio::select! {
biased;
_ = shutdown.recv() => {
log::trace!("AsyncFileWatcher: Received shutdown");
}
res = watcher.watch() => {
log::trace!("AsyncFileWatcher: finished with {res:?}");
}
}
log::debug!("AsyncFileWatcher: Exiting");
}
fn start_watcher(watcher: AsyncFileWatcher, shutdown: TaskClient) {
tokio::spawn(async move { run_watcher(watcher, shutdown).await });
}
pub(crate) async fn start_allowed_list_reloader(
stored_list: StoredAllowedHosts,
shutdown_listener: TaskClient,
) {
let (events_sender, events_receiver) = mpsc::unbounded();
let file = stored_list.get().await.storefile.clone();
let watcher = AsyncFileWatcher::new_file_changes_watcher(file, events_sender)
.expect("failed to create file watcher");
let reloader =
StoredAllowedHostsReloader::new(stored_list, events_receiver, shutdown_listener.clone());
start_watcher(watcher, shutdown_listener);
reloader.start()
}
service-providers/network-requester/src/request_filter/exit_policy/mod.rs
+23 -2
View File
@@ -1,6 +1,7 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// Copyright 2023-2024 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::config::Config;
use crate::error::NetworkRequesterError;
use log::trace;
use nym_exit_policy::client::get_exit_policy;
@@ -15,6 +16,12 @@ pub struct ExitPolicyRequestFilter {
policy: ExitPolicy,
}
impl From<ExitPolicy> for ExitPolicyRequestFilter {
fn from(value: ExitPolicy) -> Self {
ExitPolicyRequestFilter::new_from_policy(value)
}
}
impl ExitPolicyRequestFilter {
pub(crate) async fn new_upstream(url: impl IntoUrl) -> Result<Self, NetworkRequesterError> {
let url = url
@@ -27,7 +34,21 @@ impl ExitPolicyRequestFilter {
})
}
pub(crate) fn new(policy: ExitPolicy) -> Self {
pub(crate) async fn new(config: &Config) -> Result<Self, NetworkRequesterError> {
let policy_filter = if config.network_requester.open_proxy {
ExitPolicyRequestFilter::new_from_policy(ExitPolicy::new_open())
} else {
let upstream_url = config
.network_requester
.upstream_exit_policy_url
.as_ref()
.ok_or(NetworkRequesterError::NoUpstreamExitPolicy)?;
ExitPolicyRequestFilter::new_upstream(upstream_url.clone()).await?
};
Ok(policy_filter)
}
pub fn new_from_policy(policy: ExitPolicy) -> Self {
ExitPolicyRequestFilter {
upstream: None,
policy,
service-providers/network-requester/src/request_filter/mod.rs
+13 -120
View File
@@ -1,143 +1,36 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::config::{self, Config};
use crate::config::Config;
use crate::error::NetworkRequesterError;
use crate::request_filter::allowed_hosts::standard_list::StandardListUpdater;
use crate::request_filter::allowed_hosts::stored_allowed_hosts::{
start_allowed_list_reloader, StoredAllowedHosts,
};
use crate::request_filter::allowed_hosts::{OutboundRequestFilter, StandardList};
use crate::request_filter::exit_policy::ExitPolicyRequestFilter;
use log::{info, warn};
use nym_exit_policy::ExitPolicy;
use log::warn;
use nym_socks5_requests::RemoteAddress;
use nym_task::TaskHandle;
use std::sync::Arc;
/// Old request filtering based on the allowed.list files.
pub mod allowed_hosts;
pub mod exit_policy;
enum RequestFilterInner {
AllowList {
open_proxy: bool,
filter: OutboundRequestFilter,
},
ExitPolicy {
policy_filter: ExitPolicyRequestFilter,
},
}
pub use exit_policy::ExitPolicyRequestFilter;
#[derive(Clone)]
pub struct RequestFilter {
inner: Arc<RequestFilterInner>,
inner: Arc<ExitPolicyRequestFilter>,
}
impl RequestFilter {
pub(crate) async fn new(config: &Config) -> Result<Self, NetworkRequesterError> {
if config.network_requester.use_deprecated_allow_list {
info!("setting up allow-list based 'OutboundRequestFilter'...");
Ok(Self::new_allow_list_request_filter(config).await)
} else {
info!("setting up ExitPolicy based request filter...");
Self::new_exit_policy_filter(config).await
}
}
pub fn current_exit_policy_filter(&self) -> Option<&ExitPolicyRequestFilter> {
match &*self.inner {
RequestFilterInner::AllowList { .. } => None,
RequestFilterInner::ExitPolicy { policy_filter } => Some(policy_filter),
}
}
pub(crate) async fn start_update_tasks(
&self,
config: &config::Debug,
task_handle: &TaskHandle,
) {
match &*self.inner {
RequestFilterInner::AllowList { open_proxy, filter } => {
// if we're running in open proxy, we don't have to spawn any refreshers,
// after all, we're going to be accepting all requests regardless
// of the local allow list or the standard list
if *open_proxy {
return;
}
// start the standard list updater
StandardListUpdater::new(
config.standard_list_update_interval,
filter.standard_list(),
task_handle.get_handle().named("StandardListUpdater"),
)
.start();
// start the allowed.list watcher and updater
start_allowed_list_reloader(
filter.allowed_hosts(),
task_handle
.get_handle()
.named("stored_allowed_hosts_reloader"),
)
.await;
}
RequestFilterInner::ExitPolicy { .. } => {
// nothing to do for the exit policy (yet; we might add a refresher at some point)
}
}
}
async fn new_allow_list_request_filter(config: &Config) -> Self {
let standard_list = StandardList::new();
let allowed_hosts = StoredAllowedHosts::new(&config.storage_paths.allowed_list_location);
let unknown_hosts =
allowed_hosts::HostsStore::new(&config.storage_paths.unknown_list_location);
// TODO: technically if we're running open proxy, we don't have to be loading anything here
RequestFilter {
inner: Arc::new(RequestFilterInner::AllowList {
open_proxy: config.network_requester.open_proxy,
filter: OutboundRequestFilter::new(allowed_hosts, standard_list, unknown_hosts)
.await,
}),
}
}
async fn new_exit_policy_filter(config: &Config) -> Result<Self, NetworkRequesterError> {
let policy_filter = if config.network_requester.open_proxy {
ExitPolicyRequestFilter::new(ExitPolicy::new_open())
} else {
let upstream_url = config
.network_requester
.upstream_exit_policy_url
.as_ref()
.ok_or(NetworkRequesterError::NoUpstreamExitPolicy)?;
ExitPolicyRequestFilter::new_upstream(upstream_url.clone()).await?
};
Ok(RequestFilter {
inner: Arc::new(RequestFilterInner::ExitPolicy { policy_filter }),
inner: Arc::new(ExitPolicyRequestFilter::new(config).await?),
})
}
pub fn current_exit_policy_filter(&self) -> &ExitPolicyRequestFilter {
&*self.inner
}
pub(crate) async fn check_address(&self, address: &RemoteAddress) -> bool {
match &*self.inner {
RequestFilterInner::AllowList { open_proxy, filter } => {
if *open_proxy {
return true;
}
filter.check(address).await
}
RequestFilterInner::ExitPolicy { policy_filter } => {
match policy_filter.check(address).await {
Err(err) => {
warn!("failed to validate '{address}' against the exit policy: {err}");
false
}
Ok(res) => res,
}
}
}
self.inner.check(address).await.unwrap_or_else(|err| {
warn!("failed to validate '{address}' against the exit policy: {err}");
false
})
}
}