GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(explorer-api): auto update geoip database (#1684)

Browse Source 
* feat(explorer-api): auto update geoip database

* feat(explorer-api): read dotenv file

* fix(explorer-api): gitignore

* feat: move geoipupdate service to root compose file
This commit is contained in:
Pierre Dommerc
2022-10-24 18:17:50 +02:00
committed by GitHub
parent f1624e658e
commit 3c55b28e69
10 changed files with 119 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.sample-dev
+18
View File
@@ -3,3 +3,21 @@
RUST_LOG=info
RUST_BACKTRACE=1
#########################################
# geoipupdate (needed for explorer-api) #
#########################################
# MaxMind account ID (change it to a valid account ID)
GEOIPUPDATE_ACCOUNT_ID=xxx
# MaxMind license key (change it to a valid license key)
GEOIPUPDATE_LICENSE_KEY=xxx
# List of space-separated database edition IDs. Edition IDs may
# consist of letters, digits, and dashes. For example, GeoIP2-City
# would download the GeoIP2 City database (GeoIP2-City).
GEOIPUPDATE_EDITION_IDS=GeoLite2-Country
# The number of hours between geoipupdate runs. If this is not set
# or is set to 0, geoipupdate will run once and exit.
GEOIPUPDATE_FREQUENCY=72
# The path to the directory where geoipupdate will download the
# database.
GEOIP_DB_DIRECTORY=./explorer-api/geo_ip
Cargo.lock
Generated
+1
View File
@@ -1583,6 +1583,7 @@ version = "1.0.1"
dependencies = [
"chrono",
"clap 3.2.8",
"dotenv",
"humantime-serde",
"isocountry",
"itertools",
docker-compose.yml
+48 -33
View File
@@ -1,4 +1,4 @@
version: '3.7'
version: "3.7"
x-network: &NETWORK
BECH32_PREFIX: nymt
@@ -23,7 +23,7 @@ services:
networks:
localnet:
ipv4_address: 172.168.10.2
command: [ "genesis" ]
command: ["genesis"]
secondary_validator:
build:
context: docker/validator
@@ -41,7 +41,7 @@ services:
ipv4_address: 172.168.10.3
depends_on:
- "genesis_validator"
command: [ "secondary" ]
command: ["secondary"]
# mixnet_contract:
# build: docker/mixnet_contract
# image: contract:latest
@@ -74,34 +74,49 @@ services:
- "genesis_validator"
- "secondary_validator"
# mongo:
# image: mongo:latest
# command:
# - --storageEngine=wiredTiger
# volumes:
# - mongo_data:/data/db
# block_explorer:
# build:
# context: https://github.com/forbole/big-dipper.git#v0.41.x-7
# image: block_explorer:v0.41.x-7
# ports:
# - "3080:3000"
# depends_on:
# - "mongo"
# environment:
# ROOT_URL: ${APP_ROOT_URL:-http://localhost}
# MONGO_URL: mongodb://mongo:27017/meteor
# PORT: 3000
# METEOR_SETTINGS: ${METEOR_SETTINGS}
# explorer:
# build:
# context: docker/explorer
# image: explorer:latest
# ports:
# - "3040:3000"
# depends_on:
# - "genesis_validator"
# - "block_explorer"
# mongo:
# image: mongo:latest
# command:
# - --storageEngine=wiredTiger
# volumes:
# - mongo_data:/data/db
# block_explorer:
# build:
# context: https://github.com/forbole/big-dipper.git#v0.41.x-7
# image: block_explorer:v0.41.x-7
# ports:
# - "3080:3000"
# depends_on:
# - "mongo"
# environment:
# ROOT_URL: ${APP_ROOT_URL:-http://localhost}
# MONGO_URL: mongodb://mongo:27017/meteor
# PORT: 3000
# METEOR_SETTINGS: ${METEOR_SETTINGS}
# explorer:
# build:
# context: docker/explorer
# image: explorer:latest
# ports:
# - "3040:3000"
# depends_on:
# - "genesis_validator"
# - "block_explorer"
# service to update geoip binary database, for explorer-api
geoipupdate:
container_name: geoipupdate
image: maxmindinc/geoipupdate
restart: unless-stopped
environment:
GEOIPUPDATE_ACCOUNT_ID: ${GEOIPUPDATE_ACCOUNT_ID}
GEOIPUPDATE_LICENSE_KEY: ${GEOIPUPDATE_LICENSE_KEY}
GEOIPUPDATE_EDITION_IDS: ${GEOIPUPDATE_EDITION_IDS}
GEOIPUPDATE_FREQUENCY: ${GEOIPUPDATE_FREQUENCY}
networks:
- geoipupdate
volumes:
- ${GEOIP_DB_DIRECTORY}:/usr/share/GeoIP
volumes:
genesis_volume:
@@ -111,11 +126,11 @@ volumes:
# contract_volume:
# mongo_data:
networks:
geoipupdate:
localnet:
driver: bridge
ipam:
driver: default
config:
- subnet: 172.168.10.0/25
- subnet: 172.168.10.0/25
explorer-api/.env.sample
+2
View File
@@ -0,0 +1,2 @@
# The path to the geoip database file
GEOIP_DB_PATH=./geo_ip/GeoLite2-Country.mmdb
explorer-api/.gitignore
+1
View File
@@ -1,2 +1,3 @@
target
explorer-api-state.json
/geo_ip
explorer-api/Cargo.toml
+1
View File
@@ -24,6 +24,7 @@ serde_json = "1.0.66"
thiserror = "1.0.29"
tokio = {version = "1.21.2", features = ["full"] }
maxminddb = "0.23.0"
dotenv = "0.15.0"
mixnet-contract-common = { path = "../common/cosmwasm-smart-contracts/mixnet-contract" }
network-defaults = { path = "../common/network-defaults" }
explorer-api/README.md
+42 -14
View File
@@ -1,30 +1,58 @@
Network Explorer API
====================
# Network Explorer API
An API that provides data for the [Network Explorer](../explorer).
Features:
- geolocates mixnodes using https://app.ipbase.com/
- calculates how many nodes are in each country
- proxies mixnode API requests to add HTTPS
- geolocates mixnodes using https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
- calculates how many nodes are in each country
- proxies mixnode API requests to add HTTPS
## GeoIP db install/update
First we need to install the geoip database.
We use https://github.com/maxmind/geoipupdate to automatically
download and update GeoLite2 binary database. For convenience we
run it as a docker container.
At the root of the repo, inside the `docker-compose.yml`, there
is a docker service `geoipupdate` for it.
Supposed you provided an `.env` file with **all the environment
variables** listed in `.env.sample-dev` (found at the root),
simply run the service through docker:
```shell
docker compose up -d geoipupdate
```
Running this command will automatically install (and update) the
db file inside the directory path provided by `GEOIP_DB_DIRECTORY`
env variable.
## Running
Supply the environment variable `GEOIP_DATABASE_PATH` with a path
that points to a GeoIP2 database file in binary format.
When starting the explorer-api, supply the environment variable
`GEOIP_DB_PATH`, pointing to the GeoLite2 binary database file.
It should be previously installed thanks to `geoipupdate` service.
For example:
```shell
GEOIP_DB_PATH=./geo_ip/GeoLite2-Country.mmdb cargo run
```
Note: explorer-api binary reads the provided `.env` file.
Run as a service and reverse proxy with `nginx` to add `https` with Lets Encrypt.
Setup nginx to inject the request IP to the header `X-Real-IP`.
Use https://github.com/maxmind/geoipupdate to automatically
provide and update the GeoIP2 database file.
# TODO / Known Issues
## TODO
* record the number of mixnodes on a given date and write to a file for later retrieval
* dependency injection
* tests
- record the number of mixnodes on a given date and write to a file for later retrieval
- dependency injection
- tests
explorer-api/src/geo_ip/GeoLite2-Country.mmdb
BIN
View File
Binary file not shown.
explorer-api/src/geo_ip/location.rs
+4 -4
View File
@@ -6,7 +6,7 @@ use log::warn;
use maxminddb::{geoip2::Country, MaxMindDBError, Reader};
use std::{net::IpAddr, str::FromStr, sync::Arc};
const DEFAULT_DATABASE_PATH: &str = "./src/geo_ip/GeoLite2-Country.mmdb";
const DEFAULT_DATABASE_PATH: &str = "./geo_ip/GeoLite2-Country.mmdb";
#[derive(Debug)]
pub enum GeoIpError {
@@ -17,7 +17,7 @@ pub enum GeoIpError {
// The current State implementation does not allow to fail on state
// creation, ie. returning Result<>. To avoid to use unwrap family,
// as a workaround, wrap the state inside an Option<>
// If Reader::open_readfile fails for some reason db will will be set to None
// If Reader::open_readfile fails for some reason db will be set to None
// and an error will be logged.
pub(crate) struct GeoIp {
pub(crate) db: Option<Reader<Vec<u8>>>,
@@ -37,9 +37,9 @@ pub(crate) struct Location {
impl GeoIp {
pub fn new() -> Self {
let db_path = std::env::var("GEOIP_DATABASE_PATH").unwrap_or_else(|e| {
let db_path = std::env::var("GEOIP_DB_PATH").unwrap_or_else(|e| {
warn!(
"Env variable GEOIP_DATABASE_PATH is not set: {} - Fallback to {}",
"Env variable GEOIP_DB_PATH is not set: {} - Fallback to {}",
e, DEFAULT_DATABASE_PATH
);
DEFAULT_DATABASE_PATH.to_string()
explorer-api/src/main.rs
+2
View File
@@ -4,6 +4,7 @@ extern crate rocket;
extern crate rocket_okapi;
use clap::Parser;
use dotenv::dotenv;
use log::info;
use network_defaults::setup_env;
use task::ShutdownNotifier;
@@ -30,6 +31,7 @@ const COUNTRY_DATA_REFRESH_INTERVAL: u64 = 60 * 15; // every 15 minutes
#[tokio::main]
async fn main() {
dotenv().ok();
setup_logging();
let args = commands::Cli::parse();
setup_env(args.config_env_file);