From 7e581183f465e1845bb802aba58542a28e703324 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 5 Oct 2021 17:41:38 +0100 Subject: [PATCH 01/13] Network Explorer API: add env var GEO_IP_SERVICE_API_KEY to provide an API key for https://freegeoip.app/ --- explorer-api/README.md | 6 +++++- explorer-api/src/country_statistics/mod.rs | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/explorer-api/README.md b/explorer-api/README.md index 49532965ca..d05c19064b 100644 --- a/explorer-api/README.md +++ b/explorer-api/README.md @@ -14,4 +14,8 @@ TODO: * grab mixnode description info via reqwest and serve it (avoid mixed-content errors) * serve it all over http * dependency injection -* tests \ No newline at end of file +* tests + +## Running + +- Supply the environment variable `GEO_IP_SERVICE_API_KEY` with a key from https://freegeoip.app/ \ No newline at end of file diff --git a/explorer-api/src/country_statistics/mod.rs b/explorer-api/src/country_statistics/mod.rs index 6f588e4eef..681cc7cc32 100644 --- a/explorer-api/src/country_statistics/mod.rs +++ b/explorer-api/src/country_statistics/mod.rs @@ -90,7 +90,15 @@ impl CountryStatistics { } async fn locate(ip: &str) -> Result { - let response = reqwest::get(format!("{}{}", crate::GEO_IP_SERVICE, ip)).await?; + let api_key = ::std::env::var("GEO_IP_SERVICE_API_KEY") + .expect("Env var GEO_IP_SERVICE_API_KEY is not set"); + let response = reqwest::get(format!( + "{}{}?apikey={}", + crate::GEO_IP_SERVICE, + ip, + api_key + )) + .await?; let location = response.json::().await?; Ok(location) } From 81d49dc4b9a01745313d007d533404d978c0022d Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:41:10 +0100 Subject: [PATCH 02/13] Add thiserror crate --- Cargo.lock | 1 + explorer-api/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 32b5f4af88..22217f3cb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1599,6 +1599,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "thiserror", "tokio", "validator-client", ] diff --git a/explorer-api/Cargo.toml b/explorer-api/Cargo.toml index cf97221de4..f9c043bae4 100644 --- a/explorer-api/Cargo.toml +++ b/explorer-api/Cargo.toml @@ -20,6 +20,7 @@ okapi = { version = "0.6.0-alpha-1", features = ["derive_json_schema"] } rocket_okapi = "0.7.0-alpha-1" log = "0.4.0" pretty_env_logger = "0.4.0" +thiserror = "1.0.29" mixnet-contract = { path = "../common/mixnet-contract" } network-defaults = { path = "../common/network-defaults" } From f1a628dee4c0cd2beba2ba75deadb4be85048588 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:41:47 +0100 Subject: [PATCH 03/13] Change freegeoip API hostname --- explorer-api/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer-api/src/main.rs b/explorer-api/src/main.rs index 9d2c586ad3..6cb74e203f 100644 --- a/explorer-api/src/main.rs +++ b/explorer-api/src/main.rs @@ -12,7 +12,7 @@ mod mix_nodes; mod ping; mod state; -const GEO_IP_SERVICE: &str = "https://freegeoip.app/json/"; +const GEO_IP_SERVICE: &str = "https://api.freegeoip.app/json"; #[tokio::main] async fn main() { From e0e1ca992f15221fe17a7b21fb2f463a844f2568 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:44:15 +0100 Subject: [PATCH 04/13] Move locations to their own cache --- explorer-api/src/mix_node/http.rs | 22 +---------- explorer-api/src/mix_nodes/mod.rs | 61 ++++++++++++++++++------------- explorer-api/src/state.rs | 7 +--- 3 files changed, 38 insertions(+), 52 deletions(-) diff --git a/explorer-api/src/mix_node/http.rs b/explorer-api/src/mix_node/http.rs index eeb737b16c..d9cb546c3e 100644 --- a/explorer-api/src/mix_node/http.rs +++ b/explorer-api/src/mix_node/http.rs @@ -34,27 +34,7 @@ pub(crate) struct PrettyMixNodeBondWithLocation { pub(crate) async fn list( state: &State, ) -> Json> { - Json( - state - .inner - .mix_nodes - .get() - .await - .value - .values() - .map(|i| { - let mix_node = i.bond.clone(); - PrettyMixNodeBondWithLocation { - location: i.location.clone(), - bond_amount: mix_node.bond_amount, - total_delegation: mix_node.total_delegation, - owner: mix_node.owner, - layer: mix_node.layer, - mix_node: mix_node.mix_node, - } - }) - .collect::>(), - ) + Json(state.inner.mix_nodes.get_mixnodes_with_location().await) } #[openapi(tag = "mix_node")] diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs index f04a2dbce0..7952c7b0b0 100644 --- a/explorer-api/src/mix_nodes/mod.rs +++ b/explorer-api/src/mix_nodes/mod.rs @@ -7,6 +7,7 @@ use std::time::{Duration, SystemTime}; use rocket::tokio::sync::RwLock; use serde::{Deserialize, Serialize}; +use crate::mix_node::http::PrettyMixNodeBondWithLocation; use crate::mix_nodes::utils::map_2_letter_to_3_letter_country_code; use mixnet_contract::{Delegation, MixNodeBond, RawDelegationData, UnpackedDelegation}; use network_defaults::{ @@ -53,10 +54,6 @@ impl Location { } } -#[derive(Clone, Debug)] -pub(crate) struct MixNodeBondWithLocation { - pub(crate) location: Option, - pub(crate) bond: MixNodeBond, } #[derive(Clone, Debug)] @@ -92,23 +89,28 @@ impl ThreadsafeMixNodesResult { } } + pub(crate) async fn is_location_valid(&self, identity_key: &str) -> bool { + self.inner + .read() + .await + .location_cache + .get(identity_key) + .map(|cache_item| cache_item.valid_until > SystemTime::now()) + .unwrap_or(false) + } + pub(crate) async fn get_location_cache(&self) -> LocationCache { self.inner.read().await.location_cache.clone() } - pub(crate) async fn set_location(&self, identity_key: &str, location: Location) { + pub(crate) async fn set_location(&self, identity_key: &str, location: Option) { let mut guard = self.inner.write().await; // cache the location for this mix node so that it can be used when the mix node list is refreshed - guard - .location_cache - .insert(identity_key.to_string(), location.clone()); - - // add the location to the mix node - guard - .value - .entry(identity_key.to_string()) - .and_modify(|item| item.location = Some(location)); + guard.location_cache.insert( + identity_key.to_string(), + LocationCacheItem::new_from_location(location), + ); } pub(crate) async fn get(&self) -> MixNodesResult { @@ -124,25 +126,34 @@ impl ThreadsafeMixNodesResult { self.inner.read().await.clone() } - pub(crate) async fn refresh_and_get(&self) -> MixNodesResult { - self.refresh().await; - self.inner.read().await.clone() + pub(crate) async fn get_mixnodes_with_location(&self) -> Vec { + let guard = self.inner.read().await; + guard + .value + .values() + .map(|bond| { + let location = guard.location_cache.get(&bond.mix_node.identity_key); + let copy = bond.clone(); + PrettyMixNodeBondWithLocation { + location: location.and_then(|l| l.location.clone()), + bond_amount: copy.bond_amount, + total_delegation: copy.total_delegation, + owner: copy.owner, + layer: copy.layer, + mix_node: copy.mix_node, + } + }) + .collect() } - async fn refresh(&self) { + pub(crate) async fn refresh(&self) { // get mixnodes and cache the new value let value = retrieve_mixnodes().await; let location_cache = self.inner.read().await.location_cache.clone(); *self.inner.write().await = MixNodesResult { value: value .into_iter() - .map(|bond| { - let location = location_cache.get(&bond.mix_node.identity_key).cloned(); // add the location, if we've located this mix node before - ( - bond.mix_node.identity_key.to_string(), - MixNodeBondWithLocation { location, bond }, - ) - }) + .map(|bond| (bond.mix_node.identity_key.to_string(), bond)) .collect(), valid_until: SystemTime::now() + Duration::from_secs(60 * 10), // valid for 10 minutes location_cache, diff --git a/explorer-api/src/state.rs b/explorer-api/src/state.rs index 7f7862eb68..2be0a42f5d 100644 --- a/explorer-api/src/state.rs +++ b/explorer-api/src/state.rs @@ -27,12 +27,7 @@ pub struct ExplorerApiState { impl ExplorerApiState { pub(crate) async fn get_mix_node(&self, pubkey: &str) -> Option { - self.mix_nodes - .get() - .await - .value - .get(pubkey) - .map(|cache_item| cache_item.bond.clone()) + self.mix_nodes.get().await.value.get(pubkey).cloned() } } From 7eacb6b57efaa3195595199c7a51115d00f69a88 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:45:39 +0100 Subject: [PATCH 05/13] Add TTL to location cache --- explorer-api/src/mix_nodes/mod.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs index 7952c7b0b0..e4aede3fd9 100644 --- a/explorer-api/src/mix_nodes/mod.rs +++ b/explorer-api/src/mix_nodes/mod.rs @@ -15,7 +15,7 @@ use network_defaults::{ }; use validator_client::nymd::QueryNymdClient; -pub(crate) type LocationCache = HashMap; +pub(crate) type LocationCache = HashMap; #[derive(Debug, Deserialize)] pub(crate) struct GeoLocation { @@ -32,6 +32,12 @@ pub(crate) struct GeoLocation { pub(crate) metro_code: u32, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct LocationCacheItem { + pub(crate) location: Option, + pub(crate) valid_until: SystemTime, +} + #[derive(Clone, Debug, JsonSchema, Serialize, Deserialize)] pub(crate) struct Location { pub(crate) two_letter_iso_country_code: String, @@ -54,12 +60,19 @@ impl Location { } } +impl LocationCacheItem { + pub(crate) fn new_from_location(location: Option) -> Self { + LocationCacheItem { + location, + valid_until: SystemTime::now() + Duration::from_secs(60 * 60 * 24), // valid for 1 day + } + } } #[derive(Clone, Debug)] pub(crate) struct MixNodesResult { pub(crate) valid_until: SystemTime, - pub(crate) value: HashMap, + pub(crate) value: HashMap, location_cache: LocationCache, } From 72e64cfab7e82b4e736ca5788d3168b8a5c8c1f1 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:46:50 +0100 Subject: [PATCH 06/13] Split the mixnode country distribution calculation into its own task and get data from the location cache --- .../src/country_statistics/distribution.rs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 explorer-api/src/country_statistics/distribution.rs diff --git a/explorer-api/src/country_statistics/distribution.rs b/explorer-api/src/country_statistics/distribution.rs new file mode 100644 index 0000000000..3a9d338f2e --- /dev/null +++ b/explorer-api/src/country_statistics/distribution.rs @@ -0,0 +1,57 @@ +use log::info; + +use crate::country_statistics::country_nodes_distribution::CountryNodesDistribution; +use crate::mix_nodes::Location; +use crate::state::ExplorerApiStateContext; + +pub(crate) struct CountryStatisticsDistributionTask { + state: ExplorerApiStateContext, +} + +impl CountryStatisticsDistributionTask { + pub(crate) fn new(state: ExplorerApiStateContext) -> Self { + CountryStatisticsDistributionTask { state } + } + + pub(crate) fn start(mut self) { + info!("Spawning mix node country distribution task runner..."); + tokio::spawn(async move { + let mut interval_timer = tokio::time::interval(std::time::Duration::from_secs(60 * 15)); // every 15 mins + loop { + // wait for the next interval tick + interval_timer.tick().await; + self.calculate_nodes_per_country().await; + } + }); + } + + /// Retrieves the current list of mixnodes from the validators and calculates how many nodes are in each country + async fn calculate_nodes_per_country(&mut self) { + let cache = self.state.inner.mix_nodes.get_location_cache().await; + + let locations: Vec<&Location> = cache.values().flat_map(|i| i.location.as_ref()).collect(); + + let mut distribution = CountryNodesDistribution::new(); + + info!("Calculating country distribution from located mixnodes..."); + + for location in locations { + *(distribution.entry(location.three_letter_iso_country_code.clone())).or_insert(0) += 1; + } + + // replace the shared distribution to be the new distribution + self.state + .inner + .country_node_distribution + .set_all(distribution) + .await; + + info!( + "Mixnode country distribution done: {:?}", + self.state.inner.country_node_distribution.get_all().await + ); + + // keep state on disk, so that when this process dies it can start up again and users get some data + self.state.write_to_file().await; + } +} From b2ec19ece477c7a6eac8e59e6f21b83edb34e86b Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:47:25 +0100 Subject: [PATCH 07/13] Split geolcation into its own task the picks off mixnodes that don't have locations or ones where the cache TTL has expired --- .../src/country_statistics/geolocate.rs | 139 ++++++++++++++++++ explorer-api/src/country_statistics/mod.rs | 104 +------------ 2 files changed, 141 insertions(+), 102 deletions(-) create mode 100644 explorer-api/src/country_statistics/geolocate.rs diff --git a/explorer-api/src/country_statistics/geolocate.rs b/explorer-api/src/country_statistics/geolocate.rs new file mode 100644 index 0000000000..8e23a1efa0 --- /dev/null +++ b/explorer-api/src/country_statistics/geolocate.rs @@ -0,0 +1,139 @@ +use log::{info, warn}; +use reqwest::Error as ReqwestError; +use thiserror::Error; + +use crate::mix_nodes::{GeoLocation, Location}; +use crate::state::ExplorerApiStateContext; + +pub(crate) struct GeoLocateTask { + state: ExplorerApiStateContext, +} + +impl GeoLocateTask { + pub(crate) fn new(state: ExplorerApiStateContext) -> Self { + GeoLocateTask { state } + } + + pub(crate) fn start(mut self) { + info!("Spawning mix node locator task runner..."); + tokio::spawn(async move { + let mut interval_timer = tokio::time::interval(std::time::Duration::from_millis(50)); + loop { + // wait for the next interval tick + interval_timer.tick().await; + self.locate_mix_nodes().await; + } + }); + } + + async fn locate_mix_nodes(&mut self) { + let mixnode_bonds = self.state.inner.mix_nodes.get().await.value; + + for (i, cache_item) in mixnode_bonds.values().enumerate() { + if self + .state + .inner + .mix_nodes + .is_location_valid(&cache_item.mix_node.identity_key) + .await + { + // when the cached location is valid, don't locate and continue to next mix node + continue; + } + + // the mix node has not been located or is the cache time has expired + match locate(&cache_item.mix_node.host).await { + Ok(geo_location) => { + let location = Location::new(geo_location); + + trace!( + "{} mix nodes already located. Ip {} is located in {:#?}", + i, + cache_item.mix_node.host, + location.three_letter_iso_country_code, + ); + + if i > 0 && (i % 100) == 0 { + info!( + "Located {} mixnodes...", + i + 1, + ); + } + + self.state + .inner + .mix_nodes + .set_location(&cache_item.mix_node.identity_key, Some(location)) + .await; + + // one node has been located, so break out of the loop + return; + } + Err(e) => match e { + LocateError::ReqwestError(e) => warn!( + "❌ Oh no! Location for {} failed {}", + cache_item.mix_node.host, e + ), + LocateError::NotFound(e) => { + error!( + "❌ Location for {} not found. Response body: {}", + cache_item.mix_node.host, e + ); + self.state + .inner + .mix_nodes + .set_location(&cache_item.mix_node.identity_key, None) + .await; + }, + LocateError::RateLimited(e) => error!( + "❌ Oh no, we've been rate limited! Location for {} failed. Response body: {}", + cache_item.mix_node.host, e + ), + }, + } + } + + trace!("All mix nodes located"); + } +} + +#[derive(Debug, Error)] +enum LocateError { + #[error("Oops, we have made too many requests and are being rate limited. Request body: {0}")] + RateLimited(String), + + #[error("Geolocation not found. Request body: {0}")] + NotFound(String), + + #[error(transparent)] + ReqwestError(#[from] ReqwestError), +} + +async fn locate(ip: &str) -> Result { + let api_key = ::std::env::var("GEO_IP_SERVICE_API_KEY") + .expect("Env var GEO_IP_SERVICE_API_KEY is not set"); + let uri = format!("{}/{}?apikey={}", crate::GEO_IP_SERVICE, ip, api_key); + match reqwest::get(uri.clone()).await { + Ok(response) => { + if response.status() == 429 { + return Err(LocateError::RateLimited( + response + .text() + .await + .unwrap_or_else(|_| "(the response body is empty)".to_string()), + )); + } + if response.status() == 404 { + return Err(LocateError::NotFound( + response + .text() + .await + .unwrap_or_else(|_| "(the response body is empty)".to_string()), + )); + } + let location = response.json::().await?; + Ok(location) + } + Err(e) => Err(LocateError::ReqwestError(e)), + } +} diff --git a/explorer-api/src/country_statistics/mod.rs b/explorer-api/src/country_statistics/mod.rs index 681cc7cc32..43cbaadeba 100644 --- a/explorer-api/src/country_statistics/mod.rs +++ b/explorer-api/src/country_statistics/mod.rs @@ -1,104 +1,4 @@ -use log::{info, trace, warn}; -use reqwest::Error as ReqwestError; - -use crate::country_statistics::country_nodes_distribution::CountryNodesDistribution; -use crate::mix_nodes::{GeoLocation, Location}; -use crate::state::ExplorerApiStateContext; - pub mod country_nodes_distribution; +pub mod distribution; +pub mod geolocate; pub mod http; - -pub(crate) struct CountryStatistics { - state: ExplorerApiStateContext, -} - -impl CountryStatistics { - pub(crate) fn new(state: ExplorerApiStateContext) -> Self { - CountryStatistics { state } - } - - pub(crate) fn start(mut self) { - info!("Spawning task runner..."); - tokio::spawn(async move { - let mut interval_timer = tokio::time::interval(std::time::Duration::from_secs(60 * 60)); - loop { - // wait for the next interval tick - interval_timer.tick().await; - - info!("Running task..."); - self.calculate_nodes_per_country().await; - info!("Done"); - } - }); - } - - /// Retrieves the current list of mixnodes from the validators and calculates how many nodes are in each country - async fn calculate_nodes_per_country(&mut self) { - // force the mixnode cache to invalidate - let mixnode_bonds = self.state.inner.mix_nodes.refresh_and_get().await.value; - - let mut distribution = CountryNodesDistribution::new(); - - info!("Locating mixnodes..."); - for (i, cache_item) in mixnode_bonds.values().enumerate() { - match locate(&cache_item.bond.mix_node.host).await { - Ok(geo_location) => { - let location = Location::new(geo_location); - - *(distribution.entry(location.three_letter_iso_country_code.to_string())) - .or_insert(0) += 1; - - trace!( - "Ip {} is located in {:#?}", - cache_item.bond.mix_node.host, - location.three_letter_iso_country_code, - ); - - self.state - .inner - .mix_nodes - .set_location(&cache_item.bond.mix_node.identity_key, location) - .await; - - if (i % 100) == 0 { - info!( - "Located {} mixnodes in {} countries", - i + 1, - distribution.len() - ); - } - } - Err(e) => warn!("❌ Oh no! Location failed {}", e), - } - } - - // replace the shared distribution to be the new distribution - self.state - .inner - .country_node_distribution - .set_all(distribution) - .await; - - info!( - "Locating mixnodes done: {:?}", - self.state.inner.country_node_distribution.get_all().await - ); - - // keep state on disk, so that when this process dies it can start up again and users get some data - self.state.write_to_file().await; - } -} - -async fn locate(ip: &str) -> Result { - let api_key = ::std::env::var("GEO_IP_SERVICE_API_KEY") - .expect("Env var GEO_IP_SERVICE_API_KEY is not set"); - let response = reqwest::get(format!( - "{}{}?apikey={}", - crate::GEO_IP_SERVICE, - ip, - api_key - )) - .await?; - let location = response.json::().await?; - Ok(location) -} From a5091cd124c465d4f61727a032dfba95a52d156f Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:47:57 +0100 Subject: [PATCH 08/13] Split refreshing of mixnode cache into a separate task --- explorer-api/src/mix_nodes/mod.rs | 1 + explorer-api/src/mix_nodes/tasks.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 explorer-api/src/mix_nodes/tasks.rs diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs index e4aede3fd9..4fdc1f0ff7 100644 --- a/explorer-api/src/mix_nodes/mod.rs +++ b/explorer-api/src/mix_nodes/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod tasks; mod utils; use std::collections::HashMap; diff --git a/explorer-api/src/mix_nodes/tasks.rs b/explorer-api/src/mix_nodes/tasks.rs new file mode 100644 index 0000000000..24cff84d8a --- /dev/null +++ b/explorer-api/src/mix_nodes/tasks.rs @@ -0,0 +1,26 @@ +use crate::state::ExplorerApiStateContext; + +pub(crate) struct MixNodesTasks { + state: ExplorerApiStateContext, +} + +impl MixNodesTasks { + pub(crate) fn new(state: ExplorerApiStateContext) -> Self { + MixNodesTasks { state } + } + + pub(crate) fn start(self) { + info!("Spawning mix nodes task runner..."); + tokio::spawn(async move { + let mut interval_timer = tokio::time::interval(std::time::Duration::from_secs(60 * 60)); // every hour + loop { + // wait for the next interval tick + interval_timer.tick().await; + + info!("Updating mix node cache..."); + self.state.inner.mix_nodes.refresh().await; + info!("Done"); + } + }); + } +} From 2fda22e16826050073505901a8d2b5b261b19945 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 6 Oct 2021 20:48:09 +0100 Subject: [PATCH 09/13] Run new tasks at startup --- explorer-api/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/explorer-api/src/main.rs b/explorer-api/src/main.rs index 6cb74e203f..b47e2e38c3 100644 --- a/explorer-api/src/main.rs +++ b/explorer-api/src/main.rs @@ -36,7 +36,12 @@ impl ExplorerApi { info!("Explorer API starting up..."); // spawn concurrent tasks - country_statistics::CountryStatistics::new(self.state.clone()).start(); + mix_nodes::tasks::MixNodesTasks::new(self.state.clone()).start(); + country_statistics::distribution::CountryStatisticsDistributionTask::new( + self.state.clone(), + ) + .start(); + country_statistics::geolocate::GeoLocateTask::new(self.state.clone()).start(); http::start(self.state.clone()); // wait for user to press ctrl+C From d9b08344767c1da6194ea26c3317fa4f621a1f38 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Thu, 7 Oct 2021 15:53:47 +0100 Subject: [PATCH 10/13] Review feedback --- .../src/country_statistics/distribution.rs | 14 ++++++++++---- explorer-api/src/country_statistics/geolocate.rs | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/explorer-api/src/country_statistics/distribution.rs b/explorer-api/src/country_statistics/distribution.rs index 3a9d338f2e..d7e432c245 100644 --- a/explorer-api/src/country_statistics/distribution.rs +++ b/explorer-api/src/country_statistics/distribution.rs @@ -29,14 +29,20 @@ impl CountryStatisticsDistributionTask { async fn calculate_nodes_per_country(&mut self) { let cache = self.state.inner.mix_nodes.get_location_cache().await; - let locations: Vec<&Location> = cache.values().flat_map(|i| i.location.as_ref()).collect(); + let three_letter_iso_country_codes: Vec = cache + .values() + .flat_map(|i| { + i.location + .as_ref() + .map(|j| j.three_letter_iso_country_code.clone()) + }) + .collect(); let mut distribution = CountryNodesDistribution::new(); info!("Calculating country distribution from located mixnodes..."); - - for location in locations { - *(distribution.entry(location.three_letter_iso_country_code.clone())).or_insert(0) += 1; + for three_letter_iso_country_code in three_letter_iso_country_codes { + *(distribution.entry(three_letter_iso_country_code)).or_insert(0) += 1; } // replace the shared distribution to be the new distribution diff --git a/explorer-api/src/country_statistics/geolocate.rs b/explorer-api/src/country_statistics/geolocate.rs index 8e23a1efa0..f03fd8d365 100644 --- a/explorer-api/src/country_statistics/geolocate.rs +++ b/explorer-api/src/country_statistics/geolocate.rs @@ -66,7 +66,7 @@ impl GeoLocateTask { .set_location(&cache_item.mix_node.identity_key, Some(location)) .await; - // one node has been located, so break out of the loop + // one node has been located, so return out of the loop return; } Err(e) => match e { @@ -75,7 +75,7 @@ impl GeoLocateTask { cache_item.mix_node.host, e ), LocateError::NotFound(e) => { - error!( + warn!( "❌ Location for {} not found. Response body: {}", cache_item.mix_node.host, e ); @@ -85,7 +85,7 @@ impl GeoLocateTask { .set_location(&cache_item.mix_node.identity_key, None) .await; }, - LocateError::RateLimited(e) => error!( + LocateError::RateLimited(e) => warn!( "❌ Oh no, we've been rate limited! Location for {} failed. Response body: {}", cache_item.mix_node.host, e ), From e88063150082ef47a83daa008de5e7d852861b96 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 8 Oct 2021 13:08:29 +0100 Subject: [PATCH 11/13] Remove unused imports --- explorer-api/src/country_statistics/distribution.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer-api/src/country_statistics/distribution.rs b/explorer-api/src/country_statistics/distribution.rs index d7e432c245..841e8eac2c 100644 --- a/explorer-api/src/country_statistics/distribution.rs +++ b/explorer-api/src/country_statistics/distribution.rs @@ -1,7 +1,7 @@ use log::info; use crate::country_statistics::country_nodes_distribution::CountryNodesDistribution; -use crate::mix_nodes::Location; + use crate::state::ExplorerApiStateContext; pub(crate) struct CountryStatisticsDistributionTask { From bc923be8620df89208a315c1174da5b29cbc0cce Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 12 Oct 2021 12:44:45 +0100 Subject: [PATCH 12/13] Update README --- explorer-api/README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/explorer-api/README.md b/explorer-api/README.md index d05c19064b..a60b941f7d 100644 --- a/explorer-api/README.md +++ b/explorer-api/README.md @@ -1,21 +1,24 @@ Network Explorer API ==================== -An API that can: +An API that provides data for the [Network Explorer](../explorer). -* calculate how many nodes are in which country, by checking the IPs of all nodes against an external service -* serve "hello world" via HTTP - - -TODO: - -* record the number of mixnodes on a given date and write to a file for later retrieval -* store the nodes per country state in a variable -* grab mixnode description info via reqwest and serve it (avoid mixed-content errors) -* serve it all over http -* dependency injection -* tests +Features: + - geolocates mixnodes using https://freegeoip.app/ + - calculates how many nodes are in each country + - proxies mixnode API requests to add HTTPS + ## Running -- Supply the environment variable `GEO_IP_SERVICE_API_KEY` with a key from https://freegeoip.app/ \ No newline at end of file +Supply the environment variable `GEO_IP_SERVICE_API_KEY` with a key from https://freegeoip.app/. + +Run as a service and reverse proxy with `nginx` to add `https` with Lets Encrypt. + +# TODO / Known Issues + +## TODO + +* record the number of mixnodes on a given date and write to a file for later retrieval +* dependency injection +* tests From 592e52b3339cd51bf80998e3b28d3ba3bef6054c Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 12 Oct 2021 17:28:32 +0100 Subject: [PATCH 13/13] Hush nightly clippy, hush --- explorer-api/src/mix_nodes/mod.rs | 1 + explorer-api/src/state.rs | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs index 4fdc1f0ff7..8aacdfd76f 100644 --- a/explorer-api/src/mix_nodes/mod.rs +++ b/explorer-api/src/mix_nodes/mod.rs @@ -18,6 +18,7 @@ use validator_client::nymd::QueryNymdClient; pub(crate) type LocationCache = HashMap; +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub(crate) struct GeoLocation { pub(crate) ip: String, diff --git a/explorer-api/src/state.rs b/explorer-api/src/state.rs index 2be0a42f5d..3202abc9ea 100644 --- a/explorer-api/src/state.rs +++ b/explorer-api/src/state.rs @@ -41,14 +41,12 @@ pub struct ExplorerApiStateOnDisk { #[derive(Clone)] pub(crate) struct ExplorerApiStateContext { pub(crate) inner: ExplorerApiState, - state_file: String, } impl ExplorerApiStateContext { pub(crate) fn new() -> Self { ExplorerApiStateContext { inner: ExplorerApiStateContext::read_from_file(), - state_file: std::env::var("API_STATE_FILE").unwrap_or_else(|_| STATE_FILE.to_string()), } }