diff --git a/.env.sample-dev b/.env.sample-dev index 2d24384081..d77245ab12 100644 --- a/.env.sample-dev +++ b/.env.sample-dev @@ -3,3 +3,21 @@ RUST_LOG=info RUST_BACKTRACE=1 + +######################################### +# geoipupdate (needed for explorer-api) # +######################################### +# MaxMind account ID (change it to a valid account ID) +GEOIPUPDATE_ACCOUNT_ID=xxx +# MaxMind license key (change it to a valid license key) +GEOIPUPDATE_LICENSE_KEY=xxx +# List of space-separated database edition IDs. Edition IDs may +# consist of letters, digits, and dashes. For example, GeoIP2-City +# would download the GeoIP2 City database (GeoIP2-City). +GEOIPUPDATE_EDITION_IDS=GeoLite2-Country +# The number of hours between geoipupdate runs. If this is not set +# or is set to 0, geoipupdate will run once and exit. +GEOIPUPDATE_FREQUENCY=72 +# The path to the directory where geoipupdate will download the +# database. +GEOIP_DB_DIRECTORY=./explorer-api/geo_ip diff --git a/Cargo.lock b/Cargo.lock index 1847572677..627cb1f2b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1583,6 +1583,7 @@ version = "1.0.1" dependencies = [ "chrono", "clap 3.2.8", + "dotenv", "humantime-serde", "isocountry", "itertools", diff --git a/docker-compose.yml b/docker-compose.yml index b7ce864070..6c756e2ebd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: "3.7" x-network: &NETWORK BECH32_PREFIX: nymt @@ -23,7 +23,7 @@ services: networks: localnet: ipv4_address: 172.168.10.2 - command: [ "genesis" ] + command: ["genesis"] secondary_validator: build: context: docker/validator @@ -41,7 +41,7 @@ services: ipv4_address: 172.168.10.3 depends_on: - "genesis_validator" - command: [ "secondary" ] + command: ["secondary"] # mixnet_contract: # build: docker/mixnet_contract # image: contract:latest @@ -74,34 +74,49 @@ services: - "genesis_validator" - "secondary_validator" -# mongo: -# image: mongo:latest -# command: -# - --storageEngine=wiredTiger -# volumes: -# - mongo_data:/data/db -# block_explorer: -# build: -# context: https://github.com/forbole/big-dipper.git#v0.41.x-7 -# image: block_explorer:v0.41.x-7 -# ports: -# - "3080:3000" -# depends_on: -# - "mongo" -# environment: -# ROOT_URL: ${APP_ROOT_URL:-http://localhost} -# MONGO_URL: mongodb://mongo:27017/meteor -# PORT: 3000 -# METEOR_SETTINGS: ${METEOR_SETTINGS} -# explorer: -# build: -# context: docker/explorer -# image: explorer:latest -# ports: -# - "3040:3000" -# depends_on: -# - "genesis_validator" -# - "block_explorer" + # mongo: + # image: mongo:latest + # command: + # - --storageEngine=wiredTiger + # volumes: + # - mongo_data:/data/db + # block_explorer: + # build: + # context: https://github.com/forbole/big-dipper.git#v0.41.x-7 + # image: block_explorer:v0.41.x-7 + # ports: + # - "3080:3000" + # depends_on: + # - "mongo" + # environment: + # ROOT_URL: ${APP_ROOT_URL:-http://localhost} + # MONGO_URL: mongodb://mongo:27017/meteor + # PORT: 3000 + # METEOR_SETTINGS: ${METEOR_SETTINGS} + # explorer: + # build: + # context: docker/explorer + # image: explorer:latest + # ports: + # - "3040:3000" + # depends_on: + # - "genesis_validator" + # - "block_explorer" + + # service to update geoip binary database, for explorer-api + geoipupdate: + container_name: geoipupdate + image: maxmindinc/geoipupdate + restart: unless-stopped + environment: + GEOIPUPDATE_ACCOUNT_ID: ${GEOIPUPDATE_ACCOUNT_ID} + GEOIPUPDATE_LICENSE_KEY: ${GEOIPUPDATE_LICENSE_KEY} + GEOIPUPDATE_EDITION_IDS: ${GEOIPUPDATE_EDITION_IDS} + GEOIPUPDATE_FREQUENCY: ${GEOIPUPDATE_FREQUENCY} + networks: + - geoipupdate + volumes: + - ${GEOIP_DB_DIRECTORY}:/usr/share/GeoIP volumes: genesis_volume: @@ -111,11 +126,11 @@ volumes: # contract_volume: # mongo_data: - networks: + geoipupdate: localnet: driver: bridge ipam: driver: default config: - - subnet: 172.168.10.0/25 \ No newline at end of file + - subnet: 172.168.10.0/25 diff --git a/explorer-api/.env.sample b/explorer-api/.env.sample new file mode 100644 index 0000000000..1efd2de597 --- /dev/null +++ b/explorer-api/.env.sample @@ -0,0 +1,2 @@ +# The path to the geoip database file +GEOIP_DB_PATH=./geo_ip/GeoLite2-Country.mmdb diff --git a/explorer-api/.gitignore b/explorer-api/.gitignore index 8731db8ba6..66ab76436b 100644 --- a/explorer-api/.gitignore +++ b/explorer-api/.gitignore @@ -1,2 +1,3 @@ target explorer-api-state.json +/geo_ip diff --git a/explorer-api/Cargo.toml b/explorer-api/Cargo.toml index 50be3cc51a..f9edf01b44 100644 --- a/explorer-api/Cargo.toml +++ b/explorer-api/Cargo.toml @@ -24,6 +24,7 @@ serde_json = "1.0.66" thiserror = "1.0.29" tokio = {version = "1.21.2", features = ["full"] } maxminddb = "0.23.0" +dotenv = "0.15.0" mixnet-contract-common = { path = "../common/cosmwasm-smart-contracts/mixnet-contract" } network-defaults = { path = "../common/network-defaults" } diff --git a/explorer-api/README.md b/explorer-api/README.md index e5d78f590f..d4e193dbb0 100644 --- a/explorer-api/README.md +++ b/explorer-api/README.md @@ -1,30 +1,58 @@ -Network Explorer API -==================== +# Network Explorer API An API that provides data for the [Network Explorer](../explorer). Features: - - geolocates mixnodes using https://app.ipbase.com/ - - calculates how many nodes are in each country - - proxies mixnode API requests to add HTTPS - +- geolocates mixnodes using https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +- calculates how many nodes are in each country +- proxies mixnode API requests to add HTTPS + +## GeoIP db install/update + +First we need to install the geoip database. + +We use https://github.com/maxmind/geoipupdate to automatically +download and update GeoLite2 binary database. For convenience we +run it as a docker container. + +At the root of the repo, inside the `docker-compose.yml`, there +is a docker service `geoipupdate` for it. + +Supposed you provided an `.env` file with **all the environment +variables** listed in `.env.sample-dev` (found at the root), +simply run the service through docker: + +```shell +docker compose up -d geoipupdate +``` + +Running this command will automatically install (and update) the +db file inside the directory path provided by `GEOIP_DB_DIRECTORY` +env variable. + ## Running -Supply the environment variable `GEOIP_DATABASE_PATH` with a path -that points to a GeoIP2 database file in binary format. +When starting the explorer-api, supply the environment variable +`GEOIP_DB_PATH`, pointing to the GeoLite2 binary database file. +It should be previously installed thanks to `geoipupdate` service. + +For example: + +```shell +GEOIP_DB_PATH=./geo_ip/GeoLite2-Country.mmdb cargo run +``` + +Note: explorer-api binary reads the provided `.env` file. Run as a service and reverse proxy with `nginx` to add `https` with Lets Encrypt. Setup nginx to inject the request IP to the header `X-Real-IP`. -Use https://github.com/maxmind/geoipupdate to automatically -provide and update the GeoIP2 database file. - # TODO / Known Issues ## TODO -* record the number of mixnodes on a given date and write to a file for later retrieval -* dependency injection -* tests +- record the number of mixnodes on a given date and write to a file for later retrieval +- dependency injection +- tests diff --git a/explorer-api/src/geo_ip/GeoLite2-Country.mmdb b/explorer-api/src/geo_ip/GeoLite2-Country.mmdb deleted file mode 100644 index ef8c639a46..0000000000 Binary files a/explorer-api/src/geo_ip/GeoLite2-Country.mmdb and /dev/null differ diff --git a/explorer-api/src/geo_ip/location.rs b/explorer-api/src/geo_ip/location.rs index 82da674638..643de3e76d 100644 --- a/explorer-api/src/geo_ip/location.rs +++ b/explorer-api/src/geo_ip/location.rs @@ -6,7 +6,7 @@ use log::warn; use maxminddb::{geoip2::Country, MaxMindDBError, Reader}; use std::{net::IpAddr, str::FromStr, sync::Arc}; -const DEFAULT_DATABASE_PATH: &str = "./src/geo_ip/GeoLite2-Country.mmdb"; +const DEFAULT_DATABASE_PATH: &str = "./geo_ip/GeoLite2-Country.mmdb"; #[derive(Debug)] pub enum GeoIpError { @@ -17,7 +17,7 @@ pub enum GeoIpError { // The current State implementation does not allow to fail on state // creation, ie. returning Result<>. To avoid to use unwrap family, // as a workaround, wrap the state inside an Option<> -// If Reader::open_readfile fails for some reason db will will be set to None +// If Reader::open_readfile fails for some reason db will be set to None // and an error will be logged. pub(crate) struct GeoIp { pub(crate) db: Option>>, @@ -37,9 +37,9 @@ pub(crate) struct Location { impl GeoIp { pub fn new() -> Self { - let db_path = std::env::var("GEOIP_DATABASE_PATH").unwrap_or_else(|e| { + let db_path = std::env::var("GEOIP_DB_PATH").unwrap_or_else(|e| { warn!( - "Env variable GEOIP_DATABASE_PATH is not set: {} - Fallback to {}", + "Env variable GEOIP_DB_PATH is not set: {} - Fallback to {}", e, DEFAULT_DATABASE_PATH ); DEFAULT_DATABASE_PATH.to_string() diff --git a/explorer-api/src/main.rs b/explorer-api/src/main.rs index cdd0c53608..d08059442e 100644 --- a/explorer-api/src/main.rs +++ b/explorer-api/src/main.rs @@ -4,6 +4,7 @@ extern crate rocket; extern crate rocket_okapi; use clap::Parser; +use dotenv::dotenv; use log::info; use network_defaults::setup_env; use task::ShutdownNotifier; @@ -30,6 +31,7 @@ const COUNTRY_DATA_REFRESH_INTERVAL: u64 = 60 * 15; // every 15 minutes #[tokio::main] async fn main() { + dotenv().ok(); setup_logging(); let args = commands::Cli::parse(); setup_env(args.config_env_file);