diff --git a/Cargo.lock b/Cargo.lock index b3cced4dbb..69b2e902cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1669,6 +1669,7 @@ dependencies = [ "isocountry", "itertools", "log", + "maxminddb", "mixnet-contract-common", "network-defaults", "okapi", @@ -2642,6 +2643,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +[[package]] +name = "ipnetwork" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355" +dependencies = [ + "serde", +] + [[package]] name = "ipnetwork" version = "0.20.0" @@ -2915,6 +2925,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "maxminddb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2ba61113f9f7a9f0e87c519682d39c43a6f3f79c2cc42c3ba3dda83b1fa334" +dependencies = [ + "ipnetwork 0.18.0", + "log", + "memchr", + "serde", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -3371,7 +3393,7 @@ dependencies = [ "completions", "dirs", "futures", - "ipnetwork", + "ipnetwork 0.20.0", "log", "network-defaults", "nymsphinx", diff --git a/explorer-api/Cargo.toml b/explorer-api/Cargo.toml index 2e4f4fbef4..f743b34611 100644 --- a/explorer-api/Cargo.toml +++ b/explorer-api/Cargo.toml @@ -23,6 +23,7 @@ serde = "1.0.126" serde_json = "1.0.66" thiserror = "1.0.29" tokio = {version = "1.19.1", features = ["full"] } +maxminddb = "0.23.0" mixnet-contract-common = { path = "../common/cosmwasm-smart-contracts/mixnet-contract" } network-defaults = { path = "../common/network-defaults" } diff --git a/explorer-api/README.md b/explorer-api/README.md index 0555275f14..e5d78f590f 100644 --- a/explorer-api/README.md +++ b/explorer-api/README.md @@ -11,10 +11,16 @@ Features: ## Running -Supply the environment variable `GEO_IP_SERVICE_API_KEY` with a key from https://app.ipbase.com/. +Supply the environment variable `GEOIP_DATABASE_PATH` with a path +that points to a GeoIP2 database file in binary format. Run as a service and reverse proxy with `nginx` to add `https` with Lets Encrypt. +Setup nginx to inject the request IP to the header `X-Real-IP`. + +Use https://github.com/maxmind/geoipupdate to automatically +provide and update the GeoIP2 database file. + # TODO / Known Issues ## TODO diff --git a/explorer-api/src/buy_terms/http.rs b/explorer-api/src/buy_terms/http.rs new file mode 100644 index 0000000000..5594a1a3be --- /dev/null +++ b/explorer-api/src/buy_terms/http.rs @@ -0,0 +1,26 @@ +// Copyright 2022 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::geo_ip::location::Location; +use crate::state::ExplorerApiStateContext; +use rocket::response::status; +use rocket::serde::json::Json; +use rocket::{Route, State}; +use rocket_okapi::okapi::openapi3::OpenApi; +use rocket_okapi::settings::OpenApiSettings; + +pub fn nym_terms_make_default_routes(settings: &OpenApiSettings) -> (Vec, OpenApi) { + openapi_get_routes_spec![settings: terms] +} + +#[openapi(tag = "terms")] +#[get("/")] +pub(crate) async fn terms( + _state: &State, + location: Location, +) -> Result, status::Forbidden> { + if location.iso_alpha2 == "US" { + return Err(status::Forbidden(Some("US government sucks".to_string()))); + } + Ok(Json("Nym Terms & Conditions: Welcome".to_string())) +} diff --git a/explorer-api/src/buy_terms/mod.rs b/explorer-api/src/buy_terms/mod.rs new file mode 100644 index 0000000000..d064c8bd6f --- /dev/null +++ b/explorer-api/src/buy_terms/mod.rs @@ -0,0 +1 @@ +pub(crate) mod http; diff --git a/explorer-api/src/country_statistics/geolocate.rs b/explorer-api/src/country_statistics/geolocate.rs index 10329c212a..60603a2aaf 100644 --- a/explorer-api/src/country_statistics/geolocate.rs +++ b/explorer-api/src/country_statistics/geolocate.rs @@ -1,12 +1,10 @@ // Copyright 2022 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::mix_nodes::location::{GeoLocation, Location}; +use crate::mix_nodes::location::Location; use crate::state::ExplorerApiStateContext; use log::{info, warn}; -use reqwest::Error as ReqwestError; use task::ShutdownListener; -use thiserror::Error; pub(crate) struct GeoLocateTask { state: ExplorerApiStateContext, @@ -19,13 +17,6 @@ impl GeoLocateTask { } pub(crate) fn start(mut self) { - if ::std::env::var("GEO_IP_SERVICE_API_KEY").is_err() { - error!( - "Env var GEO_IP_SERVICE_API_KEY is not set. Geolocation tasks will not be started." - ); - return; - } - info!("Spawning mix node locator task runner..."); tokio::spawn(async move { let mut interval_timer = tokio::time::interval(std::time::Duration::from_millis(50)); @@ -53,6 +44,8 @@ impl GeoLocateTask { .await .unwrap_or_default(); + let geo_ip = self.state.inner.geo_ip.0.clone(); + for (i, cache_item) in mixnode_bonds.values().enumerate() { if self .state @@ -65,99 +58,50 @@ impl GeoLocateTask { continue; } - // the mix node has not been located or is the cache time has expired - match locate(&cache_item.mix_node().host).await { - Ok(geo_location) => { - let location = Location::new(geo_location); + match geo_ip.query(&cache_item.mix_node().host) { + Ok(opt) => match opt { + Some(location) => { + let location = Location::new(location); - trace!( - "{} mix nodes already located. Ip {} is located in {:#?}", - i, - cache_item.mix_node().host, - location.three_letter_iso_country_code, - ); - - if i > 0 && (i % 100) == 0 { - info!( - "Located {} mixnodes...", - i + 1, + trace!( + "{} mix nodes already located. Ip {} is located in {:#?}", + i, + cache_item.mix_node().host, + location.three_letter_iso_country_code, ); + + if i > 0 && (i % 100) == 0 { + info!("Located {} mixnodes...", i + 1,); + } + + self.state + .inner + .mixnodes + .set_location(cache_item.mix_id(), Some(location)) + .await; + + // one node has been located, so return out of the loop + return; } - - self.state - .inner - .mixnodes - .set_location(cache_item.mix_id(), Some(location)) - .await; - - // one node has been located, so return out of the loop - return; - } - Err(e) => match e { - LocateError::ReqwestError(e) => warn!( - "❌ Oh no! Location for {} failed {}", - cache_item.mix_node().host, e - ), - LocateError::NotFound(e) => { - warn!( - "❌ Location for {} not found. Response body: {}", - cache_item.mix_node().host, e - ); + None => { + warn!("❌ Location for {} not found.", cache_item.mix_node().host); self.state .inner .mixnodes .set_location(cache_item.mix_id(), None) .await; - }, - LocateError::RateLimited(e) => warn!( - "❌ Oh no, we've been rate limited! Location for {} failed. Response body: {}", - cache_item.mix_node().host, e - ), + } }, - } + Err(e) => { + warn!( + "❌ Oh no! Location for {} failed. Error: {:#?}", + cache_item.mix_node().host, + e + ); + } + }; } trace!("All mix nodes located"); } } - -#[derive(Debug, Error)] -enum LocateError { - #[error("Oops, we have made too many requests and are being rate limited. Request body: {0}")] - RateLimited(String), - - #[error("Geolocation not found. Request body: {0}")] - NotFound(String), - - #[error(transparent)] - ReqwestError(#[from] ReqwestError), -} - -async fn locate(ip: &str) -> Result { - let api_key = ::std::env::var("GEO_IP_SERVICE_API_KEY") - .expect("Env var GEO_IP_SERVICE_API_KEY is not set"); - let uri = format!("{}/?apikey={}&ip={}", crate::GEO_IP_SERVICE, api_key, ip); - match reqwest::get(uri.clone()).await { - Ok(response) => { - if response.status() == 429 { - return Err(LocateError::RateLimited( - response - .text() - .await - .unwrap_or_else(|_| "(the response body is empty)".to_string()), - )); - } - if response.status() == 404 { - return Err(LocateError::NotFound( - response - .text() - .await - .unwrap_or_else(|_| "(the response body is empty)".to_string()), - )); - } - let location = response.json::().await?; - Ok(location) - } - Err(e) => Err(LocateError::ReqwestError(e)), - } -} diff --git a/explorer-api/src/geo_ip/GeoLite2-Country.mmdb b/explorer-api/src/geo_ip/GeoLite2-Country.mmdb new file mode 100644 index 0000000000..ef8c639a46 Binary files /dev/null and b/explorer-api/src/geo_ip/GeoLite2-Country.mmdb differ diff --git a/explorer-api/src/geo_ip/location.rs b/explorer-api/src/geo_ip/location.rs new file mode 100644 index 0000000000..82da674638 --- /dev/null +++ b/explorer-api/src/geo_ip/location.rs @@ -0,0 +1,112 @@ +// Copyright 2022 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use isocountry::CountryCode; +use log::warn; +use maxminddb::{geoip2::Country, MaxMindDBError, Reader}; +use std::{net::IpAddr, str::FromStr, sync::Arc}; + +const DEFAULT_DATABASE_PATH: &str = "./src/geo_ip/GeoLite2-Country.mmdb"; + +#[derive(Debug)] +pub enum GeoIpError { + NoValidIP, + InternalError, +} + +// The current State implementation does not allow to fail on state +// creation, ie. returning Result<>. To avoid to use unwrap family, +// as a workaround, wrap the state inside an Option<> +// If Reader::open_readfile fails for some reason db will will be set to None +// and an error will be logged. +pub(crate) struct GeoIp { + pub(crate) db: Option>>, +} + +#[derive(Clone)] +pub(crate) struct ThreadsafeGeoIp(pub Arc); + +pub(crate) struct Location { + /// two-letter country code (ISO 3166-1 alpha-2) + pub(crate) iso_alpha2: String, + /// three-letter country code (ISO 3166-1 alpha-3) + pub(crate) iso_alpha3: String, + /// English country short name (ISO 3166-1) + pub(crate) name: String, +} + +impl GeoIp { + pub fn new() -> Self { + let db_path = std::env::var("GEOIP_DATABASE_PATH").unwrap_or_else(|e| { + warn!( + "Env variable GEOIP_DATABASE_PATH is not set: {} - Fallback to {}", + e, DEFAULT_DATABASE_PATH + ); + DEFAULT_DATABASE_PATH.to_string() + }); + let reader = Reader::open_readfile(&db_path) + .map_err(|e| { + error!("Fail to open GeoLite2 database file {}: {}", db_path, e); + }) + .ok(); + GeoIp { db: reader } + } + + pub fn query(&self, address: &str) -> Result, GeoIpError> { + let ip: IpAddr = FromStr::from_str(address).map_err(|e| { + error!("Fail to create IpAddr from {}: {}", &address, e); + GeoIpError::NoValidIP + })?; + let result = self + .db + .as_ref() + .ok_or_else(|| { + error!("No registered GeoIP database"); + GeoIpError::InternalError + })? + .lookup::(ip); + match &result { + Ok(v) => Ok(Some( + Location::try_from(v).map_err(|_| GeoIpError::InternalError)?, + )), + Err(e) => match e { + MaxMindDBError::AddressNotFoundError(_) => Ok(None), + _ => Err(GeoIpError::InternalError), + }, + } + } +} + +impl<'a> TryFrom<&Country<'a>> for Location { + type Error = String; + + fn try_from(country: &Country) -> Result { + let data = country.country.as_ref().ok_or_else(|| { + warn!("No Country data found"); + "No Country data found" + })?; + let iso_alpha2 = String::from(data.iso_code.ok_or_else(|| { + warn!("No iso alpha-2 code found in Country data {:#?}", data); + "No iso alpha-2 code found in Country data" + })?); + let iso_codes = CountryCode::for_alpha2(&iso_alpha2).map_err(|e| { + let message = format!( + "Fail to get iso codes from iso alpha-2 country code {}: {}", + &iso_alpha2, e + ); + warn!("{}", &message); + message + })?; + Ok(Location { + iso_alpha2, + iso_alpha3: String::from(iso_codes.alpha3()), + name: String::from(iso_codes.name()), + }) + } +} + +impl ThreadsafeGeoIp { + pub fn new() -> Self { + ThreadsafeGeoIp(Arc::new(GeoIp::new())) + } +} diff --git a/explorer-api/src/geo_ip/mod.rs b/explorer-api/src/geo_ip/mod.rs new file mode 100644 index 0000000000..d137da2c97 --- /dev/null +++ b/explorer-api/src/geo_ip/mod.rs @@ -0,0 +1 @@ +pub(crate) mod location; diff --git a/explorer-api/src/guards/location.rs b/explorer-api/src/guards/location.rs new file mode 100644 index 0000000000..dd1bfe1ff8 --- /dev/null +++ b/explorer-api/src/guards/location.rs @@ -0,0 +1,76 @@ +// Copyright 2022 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::geo_ip::location::{GeoIpError, Location}; +use crate::state::ExplorerApiStateContext; +use rocket::http::Status; +use rocket::request::FromRequest; +use rocket::request::Outcome; +use rocket::Request; +use rocket_okapi::gen::OpenApiGenerator; +use rocket_okapi::request::{OpenApiFromRequest, RequestHeaderInput}; + +const IP_HEADER: &str = "X-Real-IP"; + +#[derive(Debug)] +pub enum LocationError { + NoIP, + LocationNotFound, + InternalError, +} + +fn find_location(request: &Request<'_>) -> Result { + let ip = request + .headers() + .get_one(IP_HEADER) + .map(|f| f.to_string()) + .ok_or_else(|| { + error!("Header not found, {}", IP_HEADER); + (Status::Forbidden, LocationError::NoIP) + })?; + + let geo_ip = &request + .rocket() + .state::() + .ok_or((Status::InternalServerError, LocationError::InternalError))? // should never fail + .inner + .geo_ip; + + let location = geo_ip + .0 + .clone() + .query(&ip) + .map_err(|e| match e { + GeoIpError::NoValidIP => (Status::Forbidden, LocationError::NoIP), + GeoIpError::InternalError => { + (Status::InternalServerError, LocationError::InternalError) + } + })? + .ok_or_else(|| { + warn!("Fail to find a matching location for {}", ip); + (Status::Forbidden, LocationError::LocationNotFound) + })?; + Ok(location) +} + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for Location { + type Error = LocationError; + + async fn from_request(request: &'r Request<'_>) -> Outcome { + match find_location(request) { + Ok(loc) => Outcome::Success(loc), + Err(e) => Outcome::Failure(e), + } + } +} + +impl<'a> OpenApiFromRequest<'a> for Location { + fn from_request_input( + _gen: &mut OpenApiGenerator, + _name: String, + _required: bool, + ) -> rocket_okapi::Result { + Ok(RequestHeaderInput::None) + } +} diff --git a/explorer-api/src/guards/mod.rs b/explorer-api/src/guards/mod.rs new file mode 100644 index 0000000000..d137da2c97 --- /dev/null +++ b/explorer-api/src/guards/mod.rs @@ -0,0 +1 @@ +pub(crate) mod location; diff --git a/explorer-api/src/http/mod.rs b/explorer-api/src/http/mod.rs index 31f638ee5f..f5559124a5 100644 --- a/explorer-api/src/http/mod.rs +++ b/explorer-api/src/http/mod.rs @@ -5,6 +5,7 @@ use rocket::{Build, Request, Rocket}; use rocket_cors::{AllowedHeaders, AllowedOrigins}; use rocket_okapi::swagger_ui::make_swagger_ui; +use crate::buy_terms::http::nym_terms_make_default_routes; use crate::country_statistics::http::country_statistics_make_default_routes; use crate::gateways::http::gateways_make_default_routes; use crate::http::swagger::get_docs; @@ -56,6 +57,7 @@ fn configure_rocket(state: ExplorerApiStateContext) -> Rocket { "/overview" => overview_make_default_routes(&openapi_settings), "/ping" => ping_make_default_routes(&openapi_settings), "/validators" => validators_make_default_routes(&openapi_settings), + "/terms" => nym_terms_make_default_routes(&openapi_settings), }; building_rocket diff --git a/explorer-api/src/main.rs b/explorer-api/src/main.rs index 4e144b56a9..cdd0c53608 100644 --- a/explorer-api/src/main.rs +++ b/explorer-api/src/main.rs @@ -8,11 +8,14 @@ use log::info; use network_defaults::setup_env; use task::ShutdownNotifier; +mod buy_terms; pub(crate) mod cache; mod client; pub(crate) mod commands; mod country_statistics; mod gateways; +mod geo_ip; +mod guards; mod helpers; mod http; mod mix_node; @@ -23,7 +26,6 @@ mod state; mod tasks; mod validators; -const GEO_IP_SERVICE: &str = "https://api.ipbase.com/json"; const COUNTRY_DATA_REFRESH_INTERVAL: u64 = 60 * 15; // every 15 minutes #[tokio::main] diff --git a/explorer-api/src/mix_nodes/location.rs b/explorer-api/src/mix_nodes/location.rs index 5591a2b6a7..9e92f8a831 100644 --- a/explorer-api/src/mix_nodes/location.rs +++ b/explorer-api/src/mix_nodes/location.rs @@ -1,7 +1,7 @@ // Copyright 2021 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::mix_nodes::utils::map_2_letter_to_3_letter_country_code; +use crate::geo_ip::location; use mixnet_contract_common::NodeId; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -36,19 +36,14 @@ pub(crate) struct Location { pub(crate) two_letter_iso_country_code: String, pub(crate) three_letter_iso_country_code: String, pub(crate) country_name: String, - pub(crate) lat: f32, - pub(crate) lng: f32, } impl Location { - pub(crate) fn new(geo_location: GeoLocation) -> Self { - let three_letter_iso_country_code = map_2_letter_to_3_letter_country_code(&geo_location); + pub(crate) fn new(location: location::Location) -> Self { Location { - country_name: geo_location.country_name, - two_letter_iso_country_code: geo_location.country_code, - three_letter_iso_country_code, - lat: geo_location.latitude, - lng: geo_location.longitude, + country_name: location.name, + two_letter_iso_country_code: location.iso_alpha2, + three_letter_iso_country_code: location.iso_alpha3, } } } diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs index e2aced241a..82c2aedd1f 100644 --- a/explorer-api/src/mix_nodes/mod.rs +++ b/explorer-api/src/mix_nodes/mod.rs @@ -6,7 +6,6 @@ use std::time::Duration; pub(crate) mod http; pub(crate) mod location; pub(crate) mod models; -pub(crate) mod utils; pub(crate) const CACHE_REFRESH_RATE: Duration = Duration::from_secs(30); pub(crate) const CACHE_ENTRY_TTL: Duration = Duration::from_secs(60); diff --git a/explorer-api/src/state.rs b/explorer-api/src/state.rs index e8f89dbf3f..995de56b0c 100644 --- a/explorer-api/src/state.rs +++ b/explorer-api/src/state.rs @@ -7,6 +7,7 @@ use mixnet_contract_common::{IdentityKeyRef, NodeId}; use serde::{Deserialize, Serialize}; use crate::client::ThreadsafeValidatorClient; +use crate::geo_ip::location::ThreadsafeGeoIp; use validator_client::models::MixNodeBondAnnotated; use crate::country_statistics::country_nodes_distribution::{ @@ -30,6 +31,7 @@ pub struct ExplorerApiState { pub(crate) mixnodes: ThreadsafeMixNodesCache, pub(crate) ping: ThreadsafePingCache, pub(crate) validators: ThreadsafeValidatorCache, + pub(crate) geo_ip: ThreadsafeGeoIp, // TODO: discuss with @MS whether this is an appropriate spot for it pub(crate) validator_client: ThreadsafeValidatorClient, @@ -87,6 +89,7 @@ impl ExplorerApiStateContext { ping: ThreadsafePingCache::new(), validators: ThreadsafeValidatorCache::new(), validator_client: ThreadsafeValidatorClient::new(), + geo_ip: ThreadsafeGeoIp::new(), } } else { warn!( @@ -102,6 +105,7 @@ impl ExplorerApiStateContext { ping: ThreadsafePingCache::new(), validators: ThreadsafeValidatorCache::new(), validator_client: ThreadsafeValidatorClient::new(), + geo_ip: ThreadsafeGeoIp::new(), } } }