diff --git a/.gitignore b/.gitignore index 4675987263..6b5b8997ef 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ scripts/run_mix.sh scripts/start_local_tmux_network.sh /.floo /.flooignore +/service-providers/simple-socks5/allowed.list diff --git a/Cargo.lock b/Cargo.lock index bbccf14293..5829bc10a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,6 +805,15 @@ dependencies = [ "termcolor", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check 0.9.1", +] + [[package]] name = "extend" version = "0.1.1" @@ -2281,6 +2290,20 @@ dependencies = [ "tokio-test", ] +[[package]] +name = "publicsuffix" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" +dependencies = [ + "error-chain", + "idna 0.2.0", + "lazy_static", + "native-tls", + "regex", + "url 2.1.1", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2762,12 +2785,15 @@ name = "simple-socks5" version = "0.1.0" dependencies = [ "clap", + "dirs 2.0.2", "futures 0.3.4", "log 0.4.8", "nymsphinx", "ordered-buffer", "pretty_env_logger", "proxy-helpers", + "publicsuffix", + "rand 0.7.3", "socks5-requests", "tokio 0.2.22", "tokio-tungstenite", diff --git a/common/socks5/ordered-buffer/src/buffer.rs b/common/socks5/ordered-buffer/src/buffer.rs index 1424ab715e..ca26e35ccf 100644 --- a/common/socks5/ordered-buffer/src/buffer.rs +++ b/common/socks5/ordered-buffer/src/buffer.rs @@ -61,7 +61,7 @@ impl OrderedMessageBuffer { let high_water = index; self.next_index = high_water; - warn!("Next high water mark is: {}", high_water); + trace!("Next high water mark is: {}", high_water); // dig out the bytes from inside the struct let data: Vec = contiguous_messages diff --git a/service-providers/simple-socks5/Cargo.toml b/service-providers/simple-socks5/Cargo.toml index 2ccc4775f2..5a6e5a73b3 100644 --- a/service-providers/simple-socks5/Cargo.toml +++ b/service-providers/simple-socks5/Cargo.toml @@ -8,14 +8,20 @@ edition = "2018" [dependencies] clap = "2.33.0" +dirs = "2.0.2" futures = "0.3" log = "0.4" pretty_env_logger = "0.3" tokio = { version = "0.2", features = ["stream", "tcp", "rt-threaded", "macros"] } tokio-tungstenite = "0.11.0" +publicsuffix = "1.3.1" +# internal nymsphinx = { path = "../../common/nymsphinx" } ordered-buffer = {path = "../../common/socks5/ordered-buffer"} socks5-requests = { path = "../../common/socks5/requests" } proxy-helpers = { path = "../../common/socks5/proxy-helpers" } websocket-requests = { path = "../../clients/native/websocket-requests" } + +[dev-dependencies] +rand = "0.7" \ No newline at end of file diff --git a/service-providers/simple-socks5/src/allowed_hosts.rs b/service-providers/simple-socks5/src/allowed_hosts.rs new file mode 100644 index 0000000000..44fddbba5f --- /dev/null +++ b/service-providers/simple-socks5/src/allowed_hosts.rs @@ -0,0 +1,374 @@ +use fs::OpenOptions; +use io::BufReader; +use publicsuffix::{errors, List}; +use std::fs; +use std::fs::File; +use std::io; +use std::io::BufRead; +use std::path::Path; +use std::path::PathBuf; + +/// Filters outbound requests based on what's in an `allowed_hosts` list. +/// +/// Requests to unknown hosts are automatically written to an `unknown_hosts` +/// list so that they can be copy/pasted into the `allowed_hosts` list if desired. +/// This may be handy for service provider node operators who want to be able to look in the +/// `unknown_hosts` file and allow new hosts (e.g. if a wallet has added a new outbound request +/// which needs to be allowed). +/// +/// We rely on the list of domains at https://publicsuffix.org/ to figure out what the root +/// domain is for a given request. This allows us to distinguish all the rules for e.g. +/// .com, .co.uk, .co.jp, uk.com, etc, so that we can distinguish correct root-ish +/// domains as allowed. That list is loaded once at startup from the network. +pub(crate) struct OutboundRequestFilter { + allowed_hosts: HostsStore, + domain_list: publicsuffix::List, + unknown_hosts: HostsStore, +} + +impl OutboundRequestFilter { + pub(crate) fn new( + allowed_hosts: HostsStore, + unknown_hosts: HostsStore, + ) -> OutboundRequestFilter { + let domain_list = match Self::fetch_domain_list() { + Ok(list) => list, + Err(e) => panic!("Couldn't fetch domain list for request filtering, do you have an internet connection?: {:?}", e), + }; + OutboundRequestFilter { + allowed_hosts, + domain_list, + unknown_hosts, + } + } + + fn fetch_domain_list() -> Result { + Ok(publicsuffix::List::fetch()?) + } + + /// Returns `true` if a host's root domain is in the `allowed_hosts` list. + /// + /// If it's not in the list, return `false` and write it to the `unknown_hosts` storefile. + pub(crate) fn check(&mut self, host: &str) -> bool { + let trimmed = Self::trim_port(host); + match self.get_domain_root(&trimmed) { + Some(domain_root) => { + if self.allowed_hosts.contains(&domain_root) { + return true; + } else { + // not in allowed list but it's a domain + log::warn!( + "Blocked outbound connection to {:?}, add it to allowed.list if needed", + &domain_root + ); + self.unknown_hosts.maybe_add(&domain_root); + return false; // domain is unknown + } + } + None => { + return false; // the host was either an IP or nonsense. For this release, we'll ignore it. + } + }; + } + + fn trim_port(host: &str) -> String { + let mut tmp: Vec<&str> = host.split(":").collect(); + if tmp.len() > 1 { + tmp.pop(); // get rid of last element (port) + let out = tmp.join(":"); //rejoin + out + } else { + host.to_string() + } + } + + /// Attempts to get the root domain, shorn of subdomains, using publicsuffix. + fn get_domain_root(&self, host: &str) -> Option { + match self.domain_list.parse_domain(host) { + Ok(d) => match d.root() { + Some(root) => return Some(root.to_string()), + None => return None, // no domain root matches + }, + Err(_) => { + log::warn!("Error parsing domain: {:?}", host); + return None; // domain couldn't be parsed + } + } + } +} + +/// A simple file-based store for information about allowed / unknown hosts. +#[derive(Debug)] +pub(crate) struct HostsStore { + storefile: PathBuf, + hosts: Vec, +} + +impl HostsStore { + /// Constructs a new HostsStore + pub(crate) fn new(base_dir: PathBuf, filename: PathBuf) -> HostsStore { + let storefile = HostsStore::setup_storefile(base_dir, filename); + let hosts = HostsStore::load_from_storefile(&storefile).expect(&format!( + "Could not load hosts from storefile at {:?}", + storefile + )); + HostsStore { storefile, hosts } + } + + fn append(path: &PathBuf, text: &str) { + use std::io::Write; + let mut file = OpenOptions::new() + .write(true) + .append(true) + .open(path) + .unwrap(); + + if let Err(e) = writeln!(file, "{}", text) { + eprintln!("Couldn't write to file: {}", e); + } + } + + fn append_to_file(&self, host: &str) { + HostsStore::append(&self.storefile, host); + } + + fn contains(&self, host: &str) -> bool { + self.hosts.contains(&host.to_string()) + } + + /// Returns the default base directory for the storefile. + /// + /// This is split out so we can easily inject our own base_dir for unit tests. + pub fn default_base_dir() -> PathBuf { + dirs::home_dir() + .expect("no home directory known for this OS") + .join(".nym") + } + + fn maybe_add(&mut self, host: &str) { + if !self.contains(host) { + self.hosts.push(host.to_string()); + self.append_to_file(host); + } + } + + fn setup_storefile(base_dir: PathBuf, filename: PathBuf) -> PathBuf { + let dirpath = base_dir.join("service-providers").join("socks5"); + fs::create_dir_all(&dirpath).expect(&format!( + "could not create storage directory at {:?}", + dirpath + )); + let storefile = dirpath.join(filename); + let exists = std::path::Path::new(&storefile).exists(); + if !exists { + File::create(&storefile).unwrap(); + } + storefile + } + + /// Loads the storefile contents into memory. + fn load_from_storefile

(filename: P) -> io::Result> + where + P: AsRef, + { + let file = File::open(filename)?; + let reader = BufReader::new(&file); + let lines: Vec = reader.lines().collect::>().unwrap(); + Ok(lines) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(test)] + mod trimming_port_information { + use super::*; + + #[test] + fn happens_when_port_exists() { + let host = "nymtech.net:9999"; + assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); + } + + #[test] + fn doesnt_happen_when_no_port_exists() { + let host = "nymtech.net"; + assert_eq!("nymtech.net", OutboundRequestFilter::trim_port(host)); + } + } + + #[cfg(test)] + mod getting_the_domain_root { + use super::*; + + fn setup() -> OutboundRequestFilter { + let base_dir = test_base_dir(); + let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); + let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); + let allowed = HostsStore::new(base_dir.clone(), allowed_filename); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename); + OutboundRequestFilter::new(allowed, unknown) + } + + #[test] + fn leaves_a_com_alone() { + let filter = setup(); + assert_eq!( + Some("domain.com".to_string()), + filter.get_domain_root("domain.com") + ) + } + + #[test] + fn trims_subdomains_from_com() { + let filter = setup(); + assert_eq!( + Some("domain.com".to_string()), + filter.get_domain_root("foomp.domain.com") + ) + } + + #[test] + fn works_for_non_com_roots() { + let filter = setup(); + assert_eq!( + Some("domain.co.uk".to_string()), + filter.get_domain_root("domain.co.uk") + ) + } + + #[test] + fn works_for_non_com_roots_with_subdomains() { + let filter = setup(); + assert_eq!( + Some("domain.co.uk".to_string()), + filter.get_domain_root("foomp.domain.co.uk") + ) + } + + #[test] + fn returns_none_on_garbage() { + let filter = setup(); + assert_eq!(None, filter.get_domain_root("::/&&%@")); + } + + #[test] + fn returns_none_on_nonsense_domains() { + let filter = setup(); + assert_eq!(None, filter.get_domain_root("flappappa")); + } + } + + #[cfg(test)] + mod requests_to_unknown_hosts { + use super::*; + + fn setup() -> OutboundRequestFilter { + let base_dir = test_base_dir(); + let allowed_filename = PathBuf::from(format!("allowed-{}.list", random_string())); + let unknown_filename = PathBuf::from(&format!("unknown-{}.list", random_string())); + let allowed = HostsStore::new(base_dir.clone(), allowed_filename); + let unknown = HostsStore::new(base_dir.clone(), unknown_filename); + OutboundRequestFilter::new(allowed, unknown) + } + + #[test] + fn are_not_allowed() { + let host = "unknown.com"; + let mut filter = setup(); + assert_eq!(false, filter.check(&host)); + } + + #[test] + fn get_appended_once_to_the_unknown_hosts_list() { + let host = "unknown.com"; + let mut filter = setup(); + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + filter.check(host); + assert_eq!(1, filter.unknown_hosts.hosts.len()); + assert_eq!("unknown.com", filter.unknown_hosts.hosts.first().unwrap()); + } + } + #[cfg(test)] + mod requests_to_allowed_hosts { + use super::*; + fn setup() -> OutboundRequestFilter { + let (allowed_storefile, base_dir1, allowed_filename) = create_test_storefile(); + let (_, base_dir2, unknown_filename) = create_test_storefile(); + HostsStore::append(&allowed_storefile, "nymtech.net"); + + let allowed = HostsStore::new(base_dir1, allowed_filename.to_path_buf()); + let unknown = HostsStore::new(base_dir2, unknown_filename.to_path_buf()); + OutboundRequestFilter::new(allowed, unknown) + } + #[test] + fn are_allowed() { + let host = "nymtech.net"; + + let mut filter = setup(); + assert_eq!(true, filter.check(host)); + } + + #[test] + fn are_allowed_for_subdomains() { + let host = "foomp.nymtech.net"; + + let mut filter = setup(); + assert_eq!(true, filter.check(host)); + } + + #[test] + fn are_not_appended_to_file() { + let mut filter = setup(); + + // test initial state + let lines = HostsStore::load_from_storefile(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); + + filter.check("nymtech.net"); + + // test state after we've checked to make sure no unexpected changes + let lines = HostsStore::load_from_storefile(&filter.allowed_hosts.storefile).unwrap(); + assert_eq!(1, lines.len()); + } + } + + fn random_string() -> String { + format!("{:?}", rand::random::()) + } + + fn test_base_dir() -> PathBuf { + ["/tmp/nym-tests"].iter().collect() + } + + fn create_test_storefile() -> (PathBuf, PathBuf, PathBuf) { + let base_dir = test_base_dir(); + let filename = PathBuf::from(format!("hosts-store-{}.list", random_string())); + let dirpath = base_dir.join("service-providers").join("socks5"); + fs::create_dir_all(&dirpath).expect(&format!( + "could not create storage directory at {:?}", + dirpath + )); + let storefile = dirpath.join(&filename); + File::create(&storefile).unwrap(); + (storefile, base_dir, filename) + } + + #[cfg(test)] + mod creating_a_new_host_store { + use super::*; + #[test] + fn loads_its_host_list_from_storefile() { + let (storefile, base_dir, filename) = create_test_storefile(); + HostsStore::append(&storefile, "nymtech.net"); + HostsStore::append(&storefile, "edwardsnowden.com"); + + let host_store = HostsStore::new(base_dir, filename); + assert_eq!(vec!["nymtech.net", "edwardsnowden.com"], host_store.hosts); + } + } +} diff --git a/service-providers/simple-socks5/src/core.rs b/service-providers/simple-socks5/src/core.rs index b0e37cd737..ec9fb69be8 100644 --- a/service-providers/simple-socks5/src/core.rs +++ b/service-providers/simple-socks5/src/core.rs @@ -1,3 +1,4 @@ +use crate::allowed_hosts::{HostsStore, OutboundRequestFilter}; use crate::connection::Connection; use crate::websocket; use futures::channel::mpsc; @@ -8,6 +9,7 @@ use nymsphinx::addressing::clients::Recipient; use ordered_buffer::OrderedMessageBuffer; use proxy_helpers::connection_controller::{Controller, ControllerCommand}; use socks5_requests::{Request, Response}; +use std::path::PathBuf; use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::protocol::Message; use tokio_tungstenite::WebSocketStream; @@ -16,11 +18,25 @@ use websocket_requests::{requests::ClientRequest, responses::ServerResponse}; pub struct ServiceProvider { listening_address: String, + outbound_request_filter: OutboundRequestFilter, } impl ServiceProvider { pub fn new(listening_address: String) -> ServiceProvider { - ServiceProvider { listening_address } + let allowed_hosts = HostsStore::new( + HostsStore::default_base_dir(), + PathBuf::from("allowed.list"), + ); + + let unknown_hosts = HostsStore::new( + HostsStore::default_base_dir(), + PathBuf::from("unknown.list"), + ); + let outbound_request_filter = OutboundRequestFilter::new(allowed_hosts, unknown_hosts); + ServiceProvider { + listening_address, + outbound_request_filter, + } } /// Listens for any messages from `mix_reader` that should be written back to the mix network @@ -112,6 +128,11 @@ impl ServiceProvider { message, return_address, } => { + if !self.outbound_request_filter.check(&remote_addr) { + log::info!("Domain {:?} failed filter check", remote_addr); + continue; + } + let controller_sender_clone = controller_sender.clone(); let mut ordered_buffer = OrderedMessageBuffer::new(); ordered_buffer.write(message); diff --git a/service-providers/simple-socks5/src/main.rs b/service-providers/simple-socks5/src/main.rs index 0532cb725a..3082577fcb 100644 --- a/service-providers/simple-socks5/src/main.rs +++ b/service-providers/simple-socks5/src/main.rs @@ -1,7 +1,17 @@ +mod allowed_hosts; mod connection; mod core; mod websocket; +#[tokio::main] +async fn main() { + setup_logging(); + let uri = "ws://localhost:1977"; + println!("Starting socks5 service provider:"); + let mut server = core::ServiceProvider::new(uri.into()); + server.run().await; +} + fn setup_logging() { let mut log_builder = pretty_env_logger::formatted_timed_builder(); if let Ok(s) = ::std::env::var("RUST_LOG") { @@ -19,12 +29,3 @@ fn setup_logging() { .filter_module("want", log::LevelFilter::Warn) .init(); } - -#[tokio::main] -async fn main() { - setup_logging(); - let uri = "ws://localhost:1977"; - println!("Starting socks5 service provider:"); - let mut server = core::ServiceProvider::new(uri.into()); - server.run().await; -}