From d1b7ed8603be0bf90ba05c8b73690fec6037c8ee Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 2 Sep 2020 15:25:00 +0100 Subject: [PATCH 01/23] Moving main upwards --- service-providers/simple-socks5/src/main.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/service-providers/simple-socks5/src/main.rs b/service-providers/simple-socks5/src/main.rs index 0532cb725a..9095c91dcf 100644 --- a/service-providers/simple-socks5/src/main.rs +++ b/service-providers/simple-socks5/src/main.rs @@ -2,6 +2,15 @@ mod connection; mod core; mod websocket; +#[tokio::main] +async fn main() { + setup_logging(); + let uri = "ws://localhost:1977"; + println!("Starting socks5 service provider:"); + let mut server = core::ServiceProvider::new(uri.into()); + server.run().await; +} + fn setup_logging() { let mut log_builder = pretty_env_logger::formatted_timed_builder(); if let Ok(s) = ::std::env::var("RUST_LOG") { @@ -19,12 +28,3 @@ fn setup_logging() { .filter_module("want", log::LevelFilter::Warn) .init(); } - -#[tokio::main] -async fn main() { - setup_logging(); - let uri = "ws://localhost:1977"; - println!("Starting socks5 service provider:"); - let mut server = core::ServiceProvider::new(uri.into()); - server.run().await; -} From 822f6350cba00523a27650aac7bcc89d50df8a71 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 3 Sep 2020 11:47:28 +0100 Subject: [PATCH 02/23] WIP commit --- .gitignore | 1 + .../simple-socks5/src/allowed_hosts.rs | 113 ++++++++++++++++++ service-providers/simple-socks5/src/main.rs | 1 + 3 files changed, 115 insertions(+) create mode 100644 service-providers/simple-socks5/src/allowed_hosts.rs diff --git a/.gitignore b/.gitignore index 4675987263..6b5b8997ef 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ scripts/run_mix.sh scripts/start_local_tmux_network.sh /.floo /.flooignore +/service-providers/simple-socks5/allowed.list diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs new file mode 100644 index 0000000000..d8cd8262dc --- /dev/null +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -0,0 +1,113 @@ +struct OutboundRequestFilter { + allowed_hosts: Persistence, + unknown_hosts: Persistence, +} + +impl OutboundRequestFilter { + pub(crate) fn new( + allowed_hosts: Persistence, + unknown_hosts: Persistence, + ) -> OutboundRequestFilter { + OutboundRequestFilter { + allowed_hosts, + unknown_hosts, + } + } + + pub(crate) fn check(&self, host: &str) -> bool { + if self.allowed_hosts.contains(host.to_string()) { + true + } else { + self.unknown_hosts.maybe_add(host.to_string()); + false + } + } + + pub(crate) fn is_unknown(&self, host: &str) -> bool { + true + } +} + +#[derive(Debug)] +struct Persistence { + hosts: Vec, +} + +impl Persistence { + fn new(hosts: Vec) -> Persistence { + Persistence { hosts } + } + + fn contains(&self, host: String) -> bool { + self.hosts.contains(&host) + } + + fn maybe_add(&mut self, host: String) { + if !self.contains(host) { + self.hosts.push(host); + } + } + + /// Reloads the allowed.list and unknown.list files into memory. Used primarily for testing. + fn reload_from_disk(&self) {} + + fn append_to_file(&self, host: String) {} +} +// Appender + +#[cfg(test)] +mod requests_to_unknown_hosts { + use super::*; + + fn setup() -> OutboundRequestFilter { + let allowed = Persistence::new(vec![]); + let unknown = Persistence::new(vec![]); + OutboundRequestFilter::new(allowed, unknown) + } + + #[test] + fn are_not_allowed() { + let host = "unknown.com"; + let filter = setup(); + assert_eq!(false, filter.check(&host)); + } + + #[test] + fn get_saved_to_file() { + let host = "unknown.com"; + let filter = setup(); + filter.check(host); + assert!(true, filter.is_unknown(host)); + } + + #[test] + fn get_appended_once_to_the_unknown_hosts_list() { + let host = "unknown.com"; + let filter = setup(); + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + } +} + +#[cfg(test)] +mod requests_to_allowed_hosts { + use super::*; + + fn setup() -> OutboundRequestFilter { + let allowed = Persistence::new(vec!["nymtech.net".to_string()]); + let unknown = Persistence::new(vec![]); + OutboundRequestFilter::new(allowed, unknown) + } + + #[test] + fn are_allowed() { + let host = "nymtech.net"; + let filter = setup(); + assert_eq!(true, filter.check(host)); + } + + // #[test] + // fn are_not_appended_to_file() { + // todo!() + // } +} diff --git a/service-providers/simple-socks5/src/main.rs b/service-providers/simple-socks5/src/main.rs index 9095c91dcf..3082577fcb 100644 --- a/service-providers/simple-socks5/src/main.rs +++ b/service-providers/simple-socks5/src/main.rs @@ -1,3 +1,4 @@ +mod allowed_hosts; mod connection; mod core; mod websocket; From dc5576e1d7e1bdf98bd78088790e8505e1de7253 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 3 Sep 2020 15:02:17 +0100 Subject: [PATCH 03/23] Checking outbound requests starting to happen --- .../simple-socks5/src/allowed_hosts.rs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index d8cd8262dc..7e4b758dab 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -14,16 +14,17 @@ impl OutboundRequestFilter { } } - pub(crate) fn check(&self, host: &str) -> bool { - if self.allowed_hosts.contains(host.to_string()) { + pub(crate) fn check(&mut self, host: &str) -> bool { + let hostname = host.to_string().clone(); + if self.allowed_hosts.contains(hostname.clone()) { true } else { - self.unknown_hosts.maybe_add(host.to_string()); + self.unknown_hosts.maybe_add(hostname); false } } - pub(crate) fn is_unknown(&self, host: &str) -> bool { + pub(crate) fn is_unknown(&self, _host: &str) -> bool { true } } @@ -43,7 +44,7 @@ impl Persistence { } fn maybe_add(&mut self, host: String) { - if !self.contains(host) { + if !self.contains(host.clone()) { self.hosts.push(host); } } @@ -51,7 +52,7 @@ impl Persistence { /// Reloads the allowed.list and unknown.list files into memory. Used primarily for testing. fn reload_from_disk(&self) {} - fn append_to_file(&self, host: String) {} + fn append_to_file(&self, _host: String) {} } // Appender @@ -68,14 +69,14 @@ mod requests_to_unknown_hosts { #[test] fn are_not_allowed() { let host = "unknown.com"; - let filter = setup(); + let mut filter = setup(); assert_eq!(false, filter.check(&host)); } #[test] fn get_saved_to_file() { let host = "unknown.com"; - let filter = setup(); + let mut filter = setup(); filter.check(host); assert!(true, filter.is_unknown(host)); } @@ -83,7 +84,7 @@ mod requests_to_unknown_hosts { #[test] fn get_appended_once_to_the_unknown_hosts_list() { let host = "unknown.com"; - let filter = setup(); + let mut filter = setup(); filter.check(host); assert_eq!(1, filter.unknown_hosts.hosts.len()); } @@ -102,7 +103,7 @@ mod requests_to_allowed_hosts { #[test] fn are_allowed() { let host = "nymtech.net"; - let filter = setup(); + let mut filter = setup(); assert_eq!(true, filter.check(host)); } From 918e134c37d16ab71758a46d950a7755fc6aabd3 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 3 Sep 2020 15:04:06 +0100 Subject: [PATCH 04/23] Renamed Persistence to HostsStore --- .../simple-socks5/src/allowed_hosts.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 7e4b758dab..3be1d75be4 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -1,12 +1,12 @@ struct OutboundRequestFilter { - allowed_hosts: Persistence, - unknown_hosts: Persistence, + allowed_hosts: HostsStore, + unknown_hosts: HostsStore, } impl OutboundRequestFilter { pub(crate) fn new( - allowed_hosts: Persistence, - unknown_hosts: Persistence, + allowed_hosts: HostsStore, + unknown_hosts: HostsStore, ) -> OutboundRequestFilter { OutboundRequestFilter { allowed_hosts, @@ -30,13 +30,13 @@ impl OutboundRequestFilter { } #[derive(Debug)] -struct Persistence { +struct HostsStore { hosts: Vec, } -impl Persistence { - fn new(hosts: Vec) -> Persistence { - Persistence { hosts } +impl HostsStore { + fn new(hosts: Vec) -> HostsStore { + HostsStore { hosts } } fn contains(&self, host: String) -> bool { @@ -61,8 +61,8 @@ mod requests_to_unknown_hosts { use super::*; fn setup() -> OutboundRequestFilter { - let allowed = Persistence::new(vec![]); - let unknown = Persistence::new(vec![]); + let allowed = HostsStore::new(vec![]); + let unknown = HostsStore::new(vec![]); OutboundRequestFilter::new(allowed, unknown) } @@ -95,8 +95,8 @@ mod requests_to_allowed_hosts { use super::*; fn setup() -> OutboundRequestFilter { - let allowed = Persistence::new(vec!["nymtech.net".to_string()]); - let unknown = Persistence::new(vec![]); + let allowed = HostsStore::new(vec!["nymtech.net".to_string()]); + let unknown = HostsStore::new(vec![]); OutboundRequestFilter::new(allowed, unknown) } From 6d1a499cbeaa0888ed938cf2f95c4d1863c27fd2 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 3 Sep 2020 15:10:21 +0100 Subject: [PATCH 05/23] Ensured that unknowns are only added once --- service-providers/simple-socks5/src/allowed_hosts.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 3be1d75be4..8a406f8236 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -87,6 +87,11 @@ mod requests_to_unknown_hosts { let mut filter = setup(); filter.check(host); assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); } } From 4cb9720758c320d3addee13ee873c15f9ab6da0f Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 3 Sep 2020 15:10:31 +0100 Subject: [PATCH 06/23] Ownership improvements --- .../simple-socks5/src/allowed_hosts.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 8a406f8236..21cfd0f6b5 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -15,11 +15,10 @@ impl OutboundRequestFilter { } pub(crate) fn check(&mut self, host: &str) -> bool { - let hostname = host.to_string().clone(); - if self.allowed_hosts.contains(hostname.clone()) { + if self.allowed_hosts.contains(host) { true } else { - self.unknown_hosts.maybe_add(hostname); + self.unknown_hosts.maybe_add(host); false } } @@ -39,13 +38,13 @@ impl HostsStore { HostsStore { hosts } } - fn contains(&self, host: String) -> bool { - self.hosts.contains(&host) + fn contains(&self, host: &str) -> bool { + self.hosts.contains(&host.to_string()) } - fn maybe_add(&mut self, host: String) { - if !self.contains(host.clone()) { - self.hosts.push(host); + fn maybe_add(&mut self, host: &str) { + if !self.contains(&host) { + self.hosts.push(host.to_string()); } } From 80b16069e8450bcc6c614dda7f58ce8d1909113c Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 10:47:43 +0100 Subject: [PATCH 07/23] Adding rand crate so we can generate test output files --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index bbccf14293..10774192ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2768,6 +2768,7 @@ dependencies = [ "ordered-buffer", "pretty_env_logger", "proxy-helpers", + "rand 0.7.3", "socks5-requests", "tokio 0.2.22", "tokio-tungstenite", From c4e9577f78f61a0c2046768ee1b27eb2a77a58a1 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 10:47:57 +0100 Subject: [PATCH 08/23] Adding dirs crate so we can save to disk --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 10774192ec..b6c445dc93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2762,6 +2762,7 @@ name = "simple-socks5" version = "0.1.0" dependencies = [ "clap", + "dirs 2.0.2", "futures 0.3.4", "log 0.4.8", "nymsphinx", From 244a8bf95723c766d3864f1845f87cd6e86742eb Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 10:48:11 +0100 Subject: [PATCH 09/23] ibid --- service-providers/simple-socks5/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service-providers/simple-socks5/Cargo.toml b/service-providers/simple-socks5/Cargo.toml index 2ccc4775f2..ef4c17b9c4 100644 --- a/service-providers/simple-socks5/Cargo.toml +++ b/service-providers/simple-socks5/Cargo.toml @@ -19,3 +19,6 @@ ordered-buffer = {path = "../../common/socks5/ordered-buffer"} socks5-requests = { path = "../../common/socks5/requests" } proxy-helpers = { path = "../../common/socks5/proxy-helpers" } websocket-requests = { path = "../../clients/native/websocket-requests" } + +[dev-dependencies] +rand = "0.7" \ No newline at end of file From 334e8f20965a21facf0939307c24f0bfeeaa4e26 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 10:48:17 +0100 Subject: [PATCH 10/23] ibid --- service-providers/simple-socks5/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/service-providers/simple-socks5/Cargo.toml b/service-providers/simple-socks5/Cargo.toml index ef4c17b9c4..823bf8942c 100644 --- a/service-providers/simple-socks5/Cargo.toml +++ b/service-providers/simple-socks5/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] clap = "2.33.0" +dirs = "2.0.2" futures = "0.3" log = "0.4" pretty_env_logger = "0.3" From 5340716cce4064ae77d66bf77e74924d729a717f Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 10:53:02 +0100 Subject: [PATCH 11/23] Unknown hosts are written to file --- .../simple-socks5/src/allowed_hosts.rs | 184 ++++++++++++------ 1 file changed, 127 insertions(+), 57 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 21cfd0f6b5..a82567afe6 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -1,3 +1,6 @@ +use std::fs::{self}; +use std::path::PathBuf; + struct OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, @@ -23,96 +26,163 @@ impl OutboundRequestFilter { } } - pub(crate) fn is_unknown(&self, _host: &str) -> bool { - true + pub(crate) fn is_unknown(&self, host: &str) -> bool { + !self.allowed_hosts.contains(host) } } #[derive(Debug)] struct HostsStore { + storefile: String, hosts: Vec, } impl HostsStore { - fn new(hosts: Vec) -> HostsStore { - HostsStore { hosts } + fn new(filename: &str, hosts: Vec) -> HostsStore { + let dirpath = HostsStore::setup_storage_path(); + let filepath = dirpath.join(filename); + HostsStore { + storefile: filepath.to_str().unwrap().to_string(), + hosts, + } } fn contains(&self, host: &str) -> bool { self.hosts.contains(&host.to_string()) } + fn ensure_directory_exists(dirpath: &PathBuf) { + fs::create_dir_all(dirpath); + } + fn maybe_add(&mut self, host: &str) { if !self.contains(&host) { self.hosts.push(host.to_string()); + self.append_to_file(host); } } + fn append_to_file(&self, host: &str) -> std::io::Result<()> { + fs::write(&self.storefile, host)?; + Ok(()) + } + fn setup_storage_path() -> PathBuf { + let os_home_dir = dirs::home_dir().expect("no home directory known for this OS"); // grabs the OS default home dir + let dirpath = os_home_dir + .join(".nym") + .join("service-providers") + .join("socks5"); + HostsStore::ensure_directory_exists(&dirpath); + dirpath + } + /// Reloads the allowed.list and unknown.list files into memory. Used primarily for testing. fn reload_from_disk(&self) {} - - fn append_to_file(&self, _host: String) {} } // Appender #[cfg(test)] -mod requests_to_unknown_hosts { +mod tests { use super::*; - fn setup() -> OutboundRequestFilter { - let allowed = HostsStore::new(vec![]); - let unknown = HostsStore::new(vec![]); - OutboundRequestFilter::new(allowed, unknown) + fn random_string() -> String { + format!("{:?}", rand::random::()) } - #[test] - fn are_not_allowed() { - let host = "unknown.com"; - let mut filter = setup(); - assert_eq!(false, filter.check(&host)); + #[cfg(test)] + mod requests_to_unknown_hosts { + use super::*; + + fn setup() -> OutboundRequestFilter { + let allowed = HostsStore::new(&format!("allowed-{}.list", random_string()), vec![]); + let unknown = HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); + OutboundRequestFilter::new(allowed, unknown) + } + + #[test] + fn are_not_allowed() { + let host = "unknown.com"; + let mut filter = setup(); + assert_eq!(false, filter.check(&host)); + } + + #[test] + fn get_saved_to_file() { + let host = "unknown.com"; + let mut filter = setup(); + filter.check(host); + assert!(true, filter.is_unknown(host)); + } + + #[test] + fn get_appended_once_to_the_unknown_hosts_list() { + let host = "unknown.com"; + let mut filter = setup(); + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + } + + #[test] + fn are_written_once_to_file() { + let host = "unknown.com"; + let mut filter = setup(); + filter.check(host); + let lines = lines_from_file(&filter.unknown_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); + + filter.check(host); + let lines = lines_from_file(&filter.unknown_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); + } + + use io::BufReader; + use std::fs::File; + use std::io; + use std::io::BufRead; + use std::path::Path; + + fn lines_from_file

(filename: P) -> io::Result> + where + P: AsRef, + { + let file = File::open(filename)?; + let reader = BufReader::new(&file); + let lines: Vec = reader.lines().collect::>().unwrap(); + Ok(lines) + } } + #[cfg(test)] + mod requests_to_allowed_hosts { + use super::*; + fn setup() -> OutboundRequestFilter { + let allowed = HostsStore::new( + &format!("allowed-{}.list", random_string()), + vec!["nymtech.net".to_string()], + ); + let unknown = HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); - #[test] - fn get_saved_to_file() { - let host = "unknown.com"; - let mut filter = setup(); - filter.check(host); - assert!(true, filter.is_unknown(host)); - } + OutboundRequestFilter::new(allowed, unknown) + } + #[test] + fn are_allowed() { + let host = "nymtech.net"; + let mut filter = setup(); + assert_eq!(true, filter.check(host)); + } - #[test] - fn get_appended_once_to_the_unknown_hosts_list() { - let host = "unknown.com"; - let mut filter = setup(); - filter.check(host); - assert_eq!(1, filter.unknown_hosts.hosts.len()); - assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); - - filter.check(host); - assert_eq!(1, filter.unknown_hosts.hosts.len()); - assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + #[test] + fn are_not_unknown() { + let host = "nymtech.net"; + let mut filter = setup(); + assert_eq!(false, filter.is_unknown(host)); + } + // #[test] + // fn are_not_appended_to_file() { + // todo!() + // } } } - -#[cfg(test)] -mod requests_to_allowed_hosts { - use super::*; - - fn setup() -> OutboundRequestFilter { - let allowed = HostsStore::new(vec!["nymtech.net".to_string()]); - let unknown = HostsStore::new(vec![]); - OutboundRequestFilter::new(allowed, unknown) - } - - #[test] - fn are_allowed() { - let host = "nymtech.net"; - let mut filter = setup(); - assert_eq!(true, filter.check(host)); - } - - // #[test] - // fn are_not_appended_to_file() { - // todo!() - // } -} From 11000a99c1fd71a591966948623f82c0ef8c8275 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 13:04:20 +0100 Subject: [PATCH 12/23] Tests ensuring that unknown requests are written only once --- .../simple-socks5/src/allowed_hosts.rs | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index a82567afe6..85d08df721 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -85,10 +85,6 @@ impl HostsStore { mod tests { use super::*; - fn random_string() -> String { - format!("{:?}", rand::random::()) - } - #[cfg(test)] mod requests_to_unknown_hosts { use super::*; @@ -138,34 +134,19 @@ mod tests { let lines = lines_from_file(&filter.unknown_hosts.storefile).unwrap(); assert_eq!(1, lines.len()); } - - use io::BufReader; - use std::fs::File; - use std::io; - use std::io::BufRead; - use std::path::Path; - - fn lines_from_file

(filename: P) -> io::Result> - where - P: AsRef, - { - let file = File::open(filename)?; - let reader = BufReader::new(&file); - let lines: Vec = reader.lines().collect::>().unwrap(); - Ok(lines) - } } #[cfg(test)] mod requests_to_allowed_hosts { use super::*; fn setup() -> OutboundRequestFilter { - let allowed = HostsStore::new( + let allowed_hosts = HostsStore::new( &format!("allowed-{}.list", random_string()), vec!["nymtech.net".to_string()], ); - let unknown = HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); + let unknown_hosts = + HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); - OutboundRequestFilter::new(allowed, unknown) + OutboundRequestFilter::new(allowed_hosts, unknown_hosts) } #[test] fn are_allowed() { @@ -180,9 +161,41 @@ mod tests { let mut filter = setup(); assert_eq!(false, filter.is_unknown(host)); } - // #[test] - // fn are_not_appended_to_file() { - // todo!() - // } + + #[test] + fn are_not_appended_to_file() { + let host = "nymtech.net"; + let mut filter = setup(); + + // test initial state + let lines = lines_from_file(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(0, lines.len()); + + filter.check(host); + + // test state after we've checked to make sure no unexpected changes + let lines = lines_from_file(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(0, lines.len()); + } + } + + fn random_string() -> String { + format!("{:?}", rand::random::()) + } + + use io::BufReader; + use std::fs::File; + use std::io; + use std::io::BufRead; + use std::path::Path; + + fn lines_from_file

(filename: P) -> io::Result> + where + P: AsRef, + { + let file = File::open(filename)?; + let reader = BufReader::new(&file); + let lines: Vec = reader.lines().collect::>().unwrap(); + Ok(lines) } } From ac3829c65f939d47d1bd1da0830c86706c4f030b Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 14:02:46 +0100 Subject: [PATCH 13/23] Fixed up paths for easier testing --- .../simple-socks5/src/allowed_hosts.rs | 72 ++++++++++++------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 85d08df721..f72c6ad1e1 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -1,6 +1,11 @@ +use fs::File; use std::fs::{self}; use std::path::PathBuf; +/// Filters outbound requests based on what's in an `allowed_hosts` list. +/// +/// Requests to unknown hosts are automatically written to an `unknown_hosts` +/// list so that they can be copy/pasted into the `allowed_hosts` list if desired. struct OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, @@ -31,29 +36,34 @@ impl OutboundRequestFilter { } } +/// A simple file-based store for information about allowed / unknown hosts. #[derive(Debug)] struct HostsStore { - storefile: String, + storefile: PathBuf, hosts: Vec, } impl HostsStore { - fn new(filename: &str, hosts: Vec) -> HostsStore { - let dirpath = HostsStore::setup_storage_path(); - let filepath = dirpath.join(filename); - HostsStore { - storefile: filepath.to_str().unwrap().to_string(), - hosts, - } + /// Constructs a new HostsStore + fn new(base_dir: PathBuf, filename: PathBuf, hosts: Vec) -> HostsStore { + let storefile = HostsStore::setup_storefile(base_dir, filename); + HostsStore { storefile, hosts } + } + + /// Returns the default base directory for the storefile. + /// + /// This is split out so we can easily inject our own base_dir for unit tests. + pub fn default_base_dir() -> PathBuf { + dirs::home_dir() + .expect("no home directory known for this OS") + .join(".nym") } fn contains(&self, host: &str) -> bool { self.hosts.contains(&host.to_string()) } - fn ensure_directory_exists(dirpath: &PathBuf) { - fs::create_dir_all(dirpath); - } + fn ensure_storefile_exists(dirpath: &PathBuf) {} fn maybe_add(&mut self, host: &str) { if !self.contains(&host) { @@ -66,14 +76,18 @@ impl HostsStore { fs::write(&self.storefile, host)?; Ok(()) } - fn setup_storage_path() -> PathBuf { - let os_home_dir = dirs::home_dir().expect("no home directory known for this OS"); // grabs the OS default home dir - let dirpath = os_home_dir - .join(".nym") - .join("service-providers") - .join("socks5"); - HostsStore::ensure_directory_exists(&dirpath); - dirpath + fn setup_storefile(base_dir: PathBuf, filename: PathBuf) -> PathBuf { + let dirpath = base_dir.join("service-providers").join("socks5"); + fs::create_dir_all(&dirpath).expect(&format!( + "could not create storage directory at {:?}", + dirpath + )); + let storefile = dirpath.join(filename); + let exists = std::path::Path::new(&storefile).exists(); + if !exists { + File::create(&storefile).unwrap(); + } + storefile } /// Reloads the allowed.list and unknown.list files into memory. Used primarily for testing. @@ -90,8 +104,11 @@ mod tests { use super::*; fn setup() -> OutboundRequestFilter { - let allowed = HostsStore::new(&format!("allowed-{}.list", random_string()), vec![]); - let unknown = HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); + let base_dir: PathBuf = ["/tmp/nym-tests"].iter().collect(); + let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); + let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); + let allowed = HostsStore::new(base_dir.clone(), allowed_filename, vec![]); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename, vec![]); OutboundRequestFilter::new(allowed, unknown) } @@ -139,14 +156,17 @@ mod tests { mod requests_to_allowed_hosts { use super::*; fn setup() -> OutboundRequestFilter { - let allowed_hosts = HostsStore::new( - &format!("allowed-{}.list", random_string()), + let base_dir: PathBuf = ["/tmp/nym"].iter().collect(); + let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); + let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); + let allowed = HostsStore::new( + base_dir.clone(), + allowed_filename, vec!["nymtech.net".to_string()], ); - let unknown_hosts = - HostsStore::new(&format!("unknown-{}.list", random_string()), vec![]); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename, vec![]); - OutboundRequestFilter::new(allowed_hosts, unknown_hosts) + OutboundRequestFilter::new(allowed, unknown) } #[test] fn are_allowed() { From b7a8aefbde82c6c975a8ba57be0584ce0e4158ca Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 15:08:10 +0100 Subject: [PATCH 14/23] Loading host lists from disk --- .../simple-socks5/src/allowed_hosts.rs | 170 ++++++++++-------- 1 file changed, 91 insertions(+), 79 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index f72c6ad1e1..b414f0e1af 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -1,5 +1,10 @@ -use fs::File; -use std::fs::{self}; +use fs::OpenOptions; +use io::BufReader; +use std::fs; +use std::fs::File; +use std::io; +use std::io::BufRead; +use std::path::Path; use std::path::PathBuf; /// Filters outbound requests based on what's in an `allowed_hosts` list. @@ -22,6 +27,9 @@ impl OutboundRequestFilter { } } + /// Returns `true` if a host is in the `allowed_hosts` list. + /// + /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. pub(crate) fn check(&mut self, host: &str) -> bool { if self.allowed_hosts.contains(host) { true @@ -30,10 +38,6 @@ impl OutboundRequestFilter { false } } - - pub(crate) fn is_unknown(&self, host: &str) -> bool { - !self.allowed_hosts.contains(host) - } } /// A simple file-based store for information about allowed / unknown hosts. @@ -45,11 +49,39 @@ struct HostsStore { impl HostsStore { /// Constructs a new HostsStore - fn new(base_dir: PathBuf, filename: PathBuf, hosts: Vec) -> HostsStore { + fn new(base_dir: PathBuf, filename: PathBuf) -> HostsStore { let storefile = HostsStore::setup_storefile(base_dir, filename); + let hosts = HostsStore::load_from_storefile(&storefile).expect(&format!( + "Could not load hosts from storefile at {:?}", + storefile + )); HostsStore { storefile, hosts } } + fn append(path: &PathBuf, text: &str) { + use std::io::Write; + let mut file = OpenOptions::new() + .write(true) + .append(true) + .open(path) + .unwrap(); + + if let Err(e) = writeln!(file, "{}", text) { + eprintln!("Couldn't write to file: {}", e); + } + } + + fn append_to_file(&self, host: &str) { + fs::write(&self.storefile, host).expect(&format!( + "Could not write to storage file at {:?}", + self.storefile + )); + } + + fn contains(&self, host: &str) -> bool { + self.hosts.contains(&host.to_string()) + } + /// Returns the default base directory for the storefile. /// /// This is split out so we can easily inject our own base_dir for unit tests. @@ -59,12 +91,6 @@ impl HostsStore { .join(".nym") } - fn contains(&self, host: &str) -> bool { - self.hosts.contains(&host.to_string()) - } - - fn ensure_storefile_exists(dirpath: &PathBuf) {} - fn maybe_add(&mut self, host: &str) { if !self.contains(&host) { self.hosts.push(host.to_string()); @@ -72,10 +98,6 @@ impl HostsStore { } } - fn append_to_file(&self, host: &str) -> std::io::Result<()> { - fs::write(&self.storefile, host)?; - Ok(()) - } fn setup_storefile(base_dir: PathBuf, filename: PathBuf) -> PathBuf { let dirpath = base_dir.join("service-providers").join("socks5"); fs::create_dir_all(&dirpath).expect(&format!( @@ -90,10 +112,17 @@ impl HostsStore { storefile } - /// Reloads the allowed.list and unknown.list files into memory. Used primarily for testing. - fn reload_from_disk(&self) {} + /// Loads the storefile contents into memory. + fn load_from_storefile

(filename: P) -> io::Result> + where + P: AsRef, + { + let file = File::open(filename)?; + let reader = BufReader::new(&file); + let lines: Vec = reader.lines().collect::>().unwrap(); + Ok(lines) + } } -// Appender #[cfg(test)] mod tests { @@ -104,11 +133,11 @@ mod tests { use super::*; fn setup() -> OutboundRequestFilter { - let base_dir: PathBuf = ["/tmp/nym-tests"].iter().collect(); + let base_dir = test_base_dir(); let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); - let allowed = HostsStore::new(base_dir.clone(), allowed_filename, vec![]); - let unknown = HostsStore::new(base_dir.clone(), unknown_filename, vec![]); + let allowed = HostsStore::new(base_dir.clone(), allowed_filename); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename); OutboundRequestFilter::new(allowed, unknown) } @@ -119,14 +148,6 @@ mod tests { assert_eq!(false, filter.check(&host)); } - #[test] - fn get_saved_to_file() { - let host = "unknown.com"; - let mut filter = setup(); - filter.check(host); - assert!(true, filter.is_unknown(host)); - } - #[test] fn get_appended_once_to_the_unknown_hosts_list() { let host = "unknown.com"; @@ -138,64 +159,40 @@ mod tests { assert_eq!(1, filter.unknown_hosts.hosts.len()); assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); } - - #[test] - fn are_written_once_to_file() { - let host = "unknown.com"; - let mut filter = setup(); - filter.check(host); - let lines = lines_from_file(&filter.unknown_hosts.storefile).unwrap(); - assert_eq!(1, lines.len()); - - filter.check(host); - let lines = lines_from_file(&filter.unknown_hosts.storefile).unwrap(); - assert_eq!(1, lines.len()); - } } #[cfg(test)] mod requests_to_allowed_hosts { use super::*; fn setup() -> OutboundRequestFilter { - let base_dir: PathBuf = ["/tmp/nym"].iter().collect(); - let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); - let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); - let allowed = HostsStore::new( - base_dir.clone(), - allowed_filename, - vec!["nymtech.net".to_string()], - ); - let unknown = HostsStore::new(base_dir.clone(), unknown_filename, vec![]); + let (allowed_storefile, base_dir1, allowed_filename) = create_test_storefile(); + let (_, base_dir2, unknown_filename) = create_test_storefile(); + HostsStore::append(&allowed_storefile, "nymtech.net"); + let allowed = HostsStore::new(base_dir1, allowed_filename.to_path_buf()); + let unknown = HostsStore::new(base_dir2, unknown_filename.to_path_buf()); OutboundRequestFilter::new(allowed, unknown) } #[test] fn are_allowed() { let host = "nymtech.net"; + let mut filter = setup(); assert_eq!(true, filter.check(host)); } - #[test] - fn are_not_unknown() { - let host = "nymtech.net"; - let mut filter = setup(); - assert_eq!(false, filter.is_unknown(host)); - } - #[test] fn are_not_appended_to_file() { - let host = "nymtech.net"; let mut filter = setup(); // test initial state - let lines = lines_from_file(&filter.allowed_hosts.storefile).unwrap(); - assert_eq!(0, lines.len()); + let lines = HostsStore::load_from_storefile(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); - filter.check(host); + filter.check("nymtech.net"); // test state after we've checked to make sure no unexpected changes - let lines = lines_from_file(&filter.allowed_hosts.storefile).unwrap(); - assert_eq!(0, lines.len()); + let lines = HostsStore::load_from_storefile(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); } } @@ -203,19 +200,34 @@ mod tests { format!("{:?}", rand::random::()) } - use io::BufReader; - use std::fs::File; - use std::io; - use std::io::BufRead; - use std::path::Path; + fn test_base_dir() -> PathBuf { + ["/tmp/nym-tests"].iter().collect() + } - fn lines_from_file

(filename: P) -> io::Result> - where - P: AsRef, - { - let file = File::open(filename)?; - let reader = BufReader::new(&file); - let lines: Vec = reader.lines().collect::>().unwrap(); - Ok(lines) + fn create_test_storefile() -> (PathBuf, PathBuf, PathBuf) { + let base_dir = test_base_dir(); + let filename = PathBuf::from(format!("hosts-store-{}.list", random_string())); + let dirpath = base_dir.join("service-providers").join("socks5"); + fs::create_dir_all(&dirpath).expect(&format!( + "could not create storage directory at {:?}", + dirpath + )); + let storefile = dirpath.join(&filename); + File::create(&storefile).unwrap(); + (storefile, base_dir, filename) + } + + #[cfg(test)] + mod creating_a_new_host_store { + use super::*; + #[test] + fn loads_its_host_list_from_storefile() { + let (storefile, base_dir, filename) = create_test_storefile(); + HostsStore::append(&storefile, "nymtech.net"); + HostsStore::append(&storefile, "edwardsnowden.com"); + + let host_store = HostsStore::new(base_dir, filename); + assert_eq!(vec!["nymtech.net", "edwardsnowden.com"], host_store.hosts); + } } } From 4d4ee26a9b7f053ac8902fda90efaa8684a14468 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 4 Sep 2020 20:27:19 +0100 Subject: [PATCH 15/23] Saving domain roots in unknown file --- Cargo.lock | 24 ++++ service-providers/simple-socks5/Cargo.toml | 2 + .../simple-socks5/src/allowed_hosts.rs | 110 ++++++++++++++++-- service-providers/simple-socks5/src/core.rs | 22 +++- 4 files changed, 146 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6c445dc93..5829bc10a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,6 +805,15 @@ dependencies = [ "termcolor", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check 0.9.1", +] + [[package]] name = "extend" version = "0.1.1" @@ -2281,6 +2290,20 @@ dependencies = [ "tokio-test", ] +[[package]] +name = "publicsuffix" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" +dependencies = [ + "error-chain", + "idna 0.2.0", + "lazy_static", + "native-tls", + "regex", + "url 2.1.1", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2769,6 +2792,7 @@ dependencies = [ "ordered-buffer", "pretty_env_logger", "proxy-helpers", + "publicsuffix", "rand 0.7.3", "socks5-requests", "tokio 0.2.22", diff --git a/service-providers/simple-socks5/Cargo.toml b/service-providers/simple-socks5/Cargo.toml index 823bf8942c..5a6e5a73b3 100644 --- a/service-providers/simple-socks5/Cargo.toml +++ b/service-providers/simple-socks5/Cargo.toml @@ -14,7 +14,9 @@ log = "0.4" pretty_env_logger = "0.3" tokio = { version = "0.2", features = ["stream", "tcp", "rt-threaded", "macros"] } tokio-tungstenite = "0.11.0" +publicsuffix = "1.3.1" +# internal nymsphinx = { path = "../../common/nymsphinx" } ordered-buffer = {path = "../../common/socks5/ordered-buffer"} socks5-requests = { path = "../../common/socks5/requests" } diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index b414f0e1af..1ddc267391 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -1,5 +1,6 @@ use fs::OpenOptions; use io::BufReader; +use publicsuffix::{errors, List}; use std::fs; use std::fs::File; use std::io; @@ -11,7 +12,10 @@ use std::path::PathBuf; /// /// Requests to unknown hosts are automatically written to an `unknown_hosts` /// list so that they can be copy/pasted into the `allowed_hosts` list if desired. -struct OutboundRequestFilter { +/// This may be handy for service provider node operators who want to be able to look in the +/// `unknown_hosts` file and allow new hosts (e.g. if a wallet has added a new outbound request +/// which needs to be allowed). +pub(crate) struct OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, } @@ -27,29 +31,50 @@ impl OutboundRequestFilter { } } - /// Returns `true` if a host is in the `allowed_hosts` list. + /// Returns `true` if a host's domain is in the `allowed_hosts` list. /// /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. pub(crate) fn check(&mut self, host: &str) -> bool { - if self.allowed_hosts.contains(host) { + let trimmed = Self::trim_port(host); + let domain_root = Self::get_domain_root(&trimmed).unwrap(); + if self.allowed_hosts.contains(&domain_root) { true } else { - self.unknown_hosts.maybe_add(host); + self.unknown_hosts.maybe_add(&domain_root); false } } + + fn trim_port(host: &str) -> String { + let mut tmp: Vec<&str> = host.split(":").collect(); + if tmp.len() > 1 { + tmp.pop(); // get rid of last element (port) + let out = tmp.join(":"); + out + } else { + host.to_string() + } + } + + /// Attempts to get the root domain, shorn of port, subdomains, etc. + fn get_domain_root(host: &str) -> Result { + let list = List::fetch()?; + let domain = list.parse_domain(host)?; + let root = domain.root().unwrap(); + Ok(root.to_string()) + } } /// A simple file-based store for information about allowed / unknown hosts. #[derive(Debug)] -struct HostsStore { +pub(crate) struct HostsStore { storefile: PathBuf, hosts: Vec, } impl HostsStore { /// Constructs a new HostsStore - fn new(base_dir: PathBuf, filename: PathBuf) -> HostsStore { + pub(crate) fn new(base_dir: PathBuf, filename: PathBuf) -> HostsStore { let storefile = HostsStore::setup_storefile(base_dir, filename); let hosts = HostsStore::load_from_storefile(&storefile).expect(&format!( "Could not load hosts from storefile at {:?}", @@ -72,10 +97,7 @@ impl HostsStore { } fn append_to_file(&self, host: &str) { - fs::write(&self.storefile, host).expect(&format!( - "Could not write to storage file at {:?}", - self.storefile - )); + HostsStore::append(&self.storefile, host); } fn contains(&self, host: &str) -> bool { @@ -92,7 +114,7 @@ impl HostsStore { } fn maybe_add(&mut self, host: &str) { - if !self.contains(&host) { + if !self.contains(host) { self.hosts.push(host.to_string()); self.append_to_file(host); } @@ -128,6 +150,64 @@ impl HostsStore { mod tests { use super::*; + #[cfg(test)] + mod trimming_port_information { + use super::*; + + #[test] + fn happens_when_port_exists() { + let host = "nymtech.net:9999"; + assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); + } + + #[test] + fn doesnt_happen_when_no_port_exists() { + let host = "nymtech.net"; + assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); + } + } + + #[cfg(test)] + mod getting_the_root_domain { + use super::*; + + #[test] + fn gets_a_com_tld_ok() { + let host = "domain.com"; + assert_eq!( + "domain.com", + OutboundRequestFilter::get_domain_root(host).unwrap() + ) + } + + #[test] + fn trims_subdomains() { + let host = "foomp.domain.com"; + assert_eq!( + "domain.com", + OutboundRequestFilter::get_domain_root(host).unwrap() + ) + } + + #[test] + fn works_for_non_com_roots() { + let host = "domain.co.uk"; + assert_eq!( + "domain.co.uk", + OutboundRequestFilter::get_domain_root(host).unwrap() + ) + } + + #[test] + fn works_for_non_com_roots_with_subdomains() { + let host = "foomp.domain.co.uk"; + assert_eq!( + "domain.co.uk", + OutboundRequestFilter::get_domain_root(host).unwrap() + ) + } + } + #[cfg(test)] mod requests_to_unknown_hosts { use super::*; @@ -180,6 +260,14 @@ mod tests { assert_eq!(true, filter.check(host)); } + #[test] + fn are_allowed_for_subdomains() { + let host = "foomp.nymtech.net"; + + let mut filter = setup(); + assert_eq!(true, filter.check(host)); + } + #[test] fn are_not_appended_to_file() { let mut filter = setup(); diff --git a/service-providers/simple-socks5/src/core.rs b/service-providers/simple-socks5/src/core.rs index b0e37cd737..d061d83de2 100644 --- a/service-providers/simple-socks5/src/core.rs +++ b/service-providers/simple-socks5/src/core.rs @@ -1,3 +1,4 @@ +use crate::allowed_hosts::{HostsStore, OutboundRequestFilter}; use crate::connection::Connection; use crate::websocket; use futures::channel::mpsc; @@ -8,6 +9,7 @@ use nymsphinx::addressing::clients::Recipient; use ordered_buffer::OrderedMessageBuffer; use proxy_helpers::connection_controller::{Controller, ControllerCommand}; use socks5_requests::{Request, Response}; +use std::path::PathBuf; use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::protocol::Message; use tokio_tungstenite::WebSocketStream; @@ -16,11 +18,25 @@ use websocket_requests::{requests::ClientRequest, responses::ServerResponse}; pub struct ServiceProvider { listening_address: String, + outbound_request_filter: OutboundRequestFilter, } impl ServiceProvider { pub fn new(listening_address: String) -> ServiceProvider { - ServiceProvider { listening_address } + let allowed_hosts = HostsStore::new( + HostsStore::default_base_dir(), + PathBuf::from("allowed.list"), + ); + + let unknown_hosts = HostsStore::new( + HostsStore::default_base_dir(), + PathBuf::from("unknown.list"), + ); + let outbound_request_filter = OutboundRequestFilter::new(allowed_hosts, unknown_hosts); + ServiceProvider { + listening_address, + outbound_request_filter, + } } /// Listens for any messages from `mix_reader` that should be written back to the mix network @@ -112,6 +128,10 @@ impl ServiceProvider { message, return_address, } => { + if !self.outbound_request_filter.check(&remote_addr) { + continue; + } + let controller_sender_clone = controller_sender.clone(); let mut ordered_buffer = OrderedMessageBuffer::new(); ordered_buffer.write(message); From d3cb0d25c81152b049a943b3a615b0b2a5435ad9 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 11:59:58 +0100 Subject: [PATCH 16/23] Only fetching the publicsuffix domains list once --- .../simple-socks5/src/allowed_hosts.rs | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 1ddc267391..a4c65b7b4e 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -18,6 +18,7 @@ use std::path::PathBuf; pub(crate) struct OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, + domain_list: publicsuffix::List, } impl OutboundRequestFilter { @@ -25,18 +26,27 @@ impl OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, ) -> OutboundRequestFilter { + let domain_list = match Self::fetch_domain_list() { + Ok(list) => list, + Err(e) => panic!("Couldn't fetch domain list for request filtering, do you have an internet connection?: {:?}", e), + }; OutboundRequestFilter { allowed_hosts, + domain_list, unknown_hosts, } } + fn fetch_domain_list() -> Result { + Ok(publicsuffix::List::fetch()?) + } + /// Returns `true` if a host's domain is in the `allowed_hosts` list. /// /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. pub(crate) fn check(&mut self, host: &str) -> bool { let trimmed = Self::trim_port(host); - let domain_root = Self::get_domain_root(&trimmed).unwrap(); + let domain_root = self.get_domain_root(&trimmed).unwrap(); if self.allowed_hosts.contains(&domain_root) { true } else { @@ -49,7 +59,7 @@ impl OutboundRequestFilter { let mut tmp: Vec<&str> = host.split(":").collect(); if tmp.len() > 1 { tmp.pop(); // get rid of last element (port) - let out = tmp.join(":"); + let out = tmp.join(":"); //rejoin out } else { host.to_string() @@ -57,9 +67,8 @@ impl OutboundRequestFilter { } /// Attempts to get the root domain, shorn of port, subdomains, etc. - fn get_domain_root(host: &str) -> Result { - let list = List::fetch()?; - let domain = list.parse_domain(host)?; + fn get_domain_root(&self, host: &str) -> Result { + let domain = self.domain_list.parse_domain(host)?; let root = domain.root().unwrap(); Ok(root.to_string()) } @@ -171,39 +180,45 @@ mod tests { mod getting_the_root_domain { use super::*; - #[test] - fn gets_a_com_tld_ok() { - let host = "domain.com"; - assert_eq!( - "domain.com", - OutboundRequestFilter::get_domain_root(host).unwrap() - ) + fn setup() -> OutboundRequestFilter { + let base_dir = test_base_dir(); + let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); + let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); + let allowed = HostsStore::new(base_dir.clone(), allowed_filename); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename); + OutboundRequestFilter::new(allowed, unknown) } #[test] - fn trims_subdomains() { - let host = "foomp.domain.com"; + fn leaves_a_com_alone() { + let filter = setup(); + assert_eq!("domain.com", filter.get_domain_root("domain.com").unwrap()) + } + + #[test] + fn trims_subdomains_from_com() { + let filter = setup(); assert_eq!( "domain.com", - OutboundRequestFilter::get_domain_root(host).unwrap() + filter.get_domain_root("foomp.domain.com").unwrap() ) } #[test] fn works_for_non_com_roots() { - let host = "domain.co.uk"; + let filter = setup(); assert_eq!( "domain.co.uk", - OutboundRequestFilter::get_domain_root(host).unwrap() + filter.get_domain_root("domain.co.uk").unwrap() ) } #[test] fn works_for_non_com_roots_with_subdomains() { - let host = "foomp.domain.co.uk"; + let filter = setup(); assert_eq!( "domain.co.uk", - OutboundRequestFilter::get_domain_root(host).unwrap() + filter.get_domain_root("foomp.domain.co.uk").unwrap() ) } } From 9ce8c424554dd6b7d34263ff2e5139c769be1d2b Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 12:35:34 +0100 Subject: [PATCH 17/23] Documenting use of publicsuffix list --- service-providers/simple-socks5/src/allowed_hosts.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index a4c65b7b4e..b9ae58fa75 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -15,6 +15,11 @@ use std::path::PathBuf; /// This may be handy for service provider node operators who want to be able to look in the /// `unknown_hosts` file and allow new hosts (e.g. if a wallet has added a new outbound request /// which needs to be allowed). +/// +/// We rely on the list of domains at https://publicsuffix.org/ to figure out what the root +/// domain is for a given request. This allows us to distinguish all the rules for e.g. +/// .com, .co.uk, .co.jp, uk.com, etc, so that we can distinguish correct root-ish +/// domains as allowed. That list is loaded once at startup from the network. pub(crate) struct OutboundRequestFilter { allowed_hosts: HostsStore, unknown_hosts: HostsStore, From 4c055a8e358efe0d67c4172ae55b7cf241fc94f3 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 13:00:18 +0100 Subject: [PATCH 18/23] Beefed up error handling --- .../simple-socks5/src/allowed_hosts.rs | 64 +++++++++++++------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index b9ae58fa75..340b33a8eb 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -51,13 +51,18 @@ impl OutboundRequestFilter { /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. pub(crate) fn check(&mut self, host: &str) -> bool { let trimmed = Self::trim_port(host); - let domain_root = self.get_domain_root(&trimmed).unwrap(); - if self.allowed_hosts.contains(&domain_root) { - true - } else { - self.unknown_hosts.maybe_add(&domain_root); - false - } + match self.get_domain_root(&trimmed) { + Some(domain_root) => { + if self.allowed_hosts.contains(&domain_root) { + return true; + } else { + // not in allowed list but it's a domain + self.unknown_hosts.maybe_add(&domain_root); + return false; + } + } + None => return false, // the domain was probably nonsense. I've chosen not to log it unknown_hosts for now. + }; } fn trim_port(host: &str) -> String { @@ -71,11 +76,15 @@ impl OutboundRequestFilter { } } - /// Attempts to get the root domain, shorn of port, subdomains, etc. - fn get_domain_root(&self, host: &str) -> Result { - let domain = self.domain_list.parse_domain(host)?; - let root = domain.root().unwrap(); - Ok(root.to_string()) + /// Attempts to get the root domain, shorn of subdomains, using publicsuffix. + fn get_domain_root(&self, host: &str) -> Option { + match self.domain_list.parse_domain(host) { + Ok(d) => match d.root() { + Some(root) => return Some(root.to_string()), + None => return None, // no domain root matches + }, + Err(_) => return None, // domain couldn't be parsed + } } } @@ -182,7 +191,7 @@ mod tests { } #[cfg(test)] - mod getting_the_root_domain { + mod getting_the_domain_root { use super::*; fn setup() -> OutboundRequestFilter { @@ -197,15 +206,18 @@ mod tests { #[test] fn leaves_a_com_alone() { let filter = setup(); - assert_eq!("domain.com", filter.get_domain_root("domain.com").unwrap()) + assert_eq!( + Some("domain.com".to_string()), + filter.get_domain_root("domain.com") + ) } #[test] fn trims_subdomains_from_com() { let filter = setup(); assert_eq!( - "domain.com", - filter.get_domain_root("foomp.domain.com").unwrap() + Some("domain.com".to_string()), + filter.get_domain_root("foomp.domain.com") ) } @@ -213,8 +225,8 @@ mod tests { fn works_for_non_com_roots() { let filter = setup(); assert_eq!( - "domain.co.uk", - filter.get_domain_root("domain.co.uk").unwrap() + Some("domain.co.uk".to_string()), + filter.get_domain_root("domain.co.uk") ) } @@ -222,10 +234,22 @@ mod tests { fn works_for_non_com_roots_with_subdomains() { let filter = setup(); assert_eq!( - "domain.co.uk", - filter.get_domain_root("foomp.domain.co.uk").unwrap() + Some("domain.co.uk".to_string()), + filter.get_domain_root("foomp.domain.co.uk") ) } + + #[test] + fn returns_none_on_garbage() { + let filter = setup(); + assert_eq!(None, filter.get_domain_root("::/&&%@")); + } + + #[test] + fn returns_none_on_nonsense_domains() { + let filter = setup(); + assert_eq!(None, filter.get_domain_root("flappappa")); + } } #[cfg(test)] From 4b16d14121d5cf9da5ac721771e848cc05a6ca4d Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 14:13:35 +0100 Subject: [PATCH 19/23] Knocking down log level on high water mark --- common/socks5/ordered-buffer/src/buffer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/socks5/ordered-buffer/src/buffer.rs b/common/socks5/ordered-buffer/src/buffer.rs index 1424ab715e..4684672de8 100644 --- a/common/socks5/ordered-buffer/src/buffer.rs +++ b/common/socks5/ordered-buffer/src/buffer.rs @@ -61,7 +61,7 @@ impl OrderedMessageBuffer { let high_water = index; self.next_index = high_water; - warn!("Next high water mark is: {}", high_water); + info!("Next high water mark is: {}", high_water); // dig out the bytes from inside the struct let data: Vec = contiguous_messages From eb1737264e9fb71661f573a633a967364d8aeaf9 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 14:14:03 +0100 Subject: [PATCH 20/23] Minor cleanup on domain parsing --- .../simple-socks5/src/allowed_hosts.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs index 340b33a8eb..44fddbba5f 100644 --- a/service-providers/simple-socks5/src/allowed_hosts.rs +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -22,8 +22,8 @@ use std::path::PathBuf; /// domains as allowed. That list is loaded once at startup from the network. pub(crate) struct OutboundRequestFilter { allowed_hosts: HostsStore, - unknown_hosts: HostsStore, domain_list: publicsuffix::List, + unknown_hosts: HostsStore, } impl OutboundRequestFilter { @@ -46,7 +46,7 @@ impl OutboundRequestFilter { Ok(publicsuffix::List::fetch()?) } - /// Returns `true` if a host's domain is in the `allowed_hosts` list. + /// Returns `true` if a host's root domain is in the `allowed_hosts` list. /// /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. pub(crate) fn check(&mut self, host: &str) -> bool { @@ -57,11 +57,17 @@ impl OutboundRequestFilter { return true; } else { // not in allowed list but it's a domain + log::warn!( + "Blocked outbound connection to {:?}, add it to allowed.list if needed", + &domain_root + ); self.unknown_hosts.maybe_add(&domain_root); - return false; + return false; // domain is unknown } } - None => return false, // the domain was probably nonsense. I've chosen not to log it unknown_hosts for now. + None => { + return false; // the host was either an IP or nonsense. For this release, we'll ignore it. + } }; } @@ -83,7 +89,10 @@ impl OutboundRequestFilter { Some(root) => return Some(root.to_string()), None => return None, // no domain root matches }, - Err(_) => return None, // domain couldn't be parsed + Err(_) => { + log::warn!("Error parsing domain: {:?}", host); + return None; // domain couldn't be parsed + } } } } From 80a86e3bec94f92b6711b68be2faaeb2fc790de1 Mon Sep 17 00:00:00 2001 From: Dave Date: Sat, 5 Sep 2020 14:14:29 +0100 Subject: [PATCH 21/23] Noted failing filter check --- service-providers/simple-socks5/src/core.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/service-providers/simple-socks5/src/core.rs b/service-providers/simple-socks5/src/core.rs index d061d83de2..ec9fb69be8 100644 --- a/service-providers/simple-socks5/src/core.rs +++ b/service-providers/simple-socks5/src/core.rs @@ -129,6 +129,7 @@ impl ServiceProvider { return_address, } => { if !self.outbound_request_filter.check(&remote_addr) { + log::info!("Domain {:?} failed filter check", remote_addr); continue; } From 406086aec796d7a23774f4e75b857bdb5dd22f7a Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 8 Sep 2020 12:01:43 +0100 Subject: [PATCH 22/23] Removing log message --- common/socks5/ordered-buffer/src/buffer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/common/socks5/ordered-buffer/src/buffer.rs b/common/socks5/ordered-buffer/src/buffer.rs index 4684672de8..e8c9d35083 100644 --- a/common/socks5/ordered-buffer/src/buffer.rs +++ b/common/socks5/ordered-buffer/src/buffer.rs @@ -61,7 +61,6 @@ impl OrderedMessageBuffer { let high_water = index; self.next_index = high_water; - info!("Next high water mark is: {}", high_water); // dig out the bytes from inside the struct let data: Vec = contiguous_messages From 83e78e7df13075726b1952546bfb4a205097e037 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 8 Sep 2020 12:02:04 +0100 Subject: [PATCH 23/23] Setting log message to trace --- common/socks5/ordered-buffer/src/buffer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/common/socks5/ordered-buffer/src/buffer.rs b/common/socks5/ordered-buffer/src/buffer.rs index e8c9d35083..ca26e35ccf 100644 --- a/common/socks5/ordered-buffer/src/buffer.rs +++ b/common/socks5/ordered-buffer/src/buffer.rs @@ -61,6 +61,7 @@ impl OrderedMessageBuffer { let high_water = index; self.next_index = high_water; + trace!("Next high water mark is: {}", high_water); // dig out the bytes from inside the struct let data: Vec = contiguous_messages