Merge branch 'release/v0.5.0-rc.1'

.gitignore

@@ -6,3 +6,4 @@
 target
 .env
 /.vscode/settings.json
+sample-configs/validator-config.toml

.travis.yml

@@ -11,5 +11,5 @@ before_script:
   - rustup component add rustfmt
 script:
   - cargo build
-  - cargo test
+  - cargo test -- --test-threads=1
   - cargo fmt -- --check

CHANGELOG.md

@@ -1,5 +1,51 @@
 # Changelog
 
+## [Unreleased](https://github.com/nymtech/nym/tree/HEAD)
+
+[Full Changelog](https://github.com/nymtech/nym/compare/v0.4.1...HEAD)
+
+**Closed issues:**
+
+- COMPILE: Could not compile project using Cargo [\#118](https://github.com/nymtech/nym/issues/118)
+- Wherever unbounded mpsc channel is used, prefer unbounded\_send\(\) over send\(\).await [\#90](https://github.com/nymtech/nym/issues/90)
+- Add a `Send` method in nym-client [\#81](https://github.com/nymtech/nym/issues/81)
+- Start on Tendermint integration [\#79](https://github.com/nymtech/nym/issues/79)
+- Ditch DummyKeyPair [\#75](https://github.com/nymtech/nym/issues/75)
+- Replace args with proper config files [\#69](https://github.com/nymtech/nym/issues/69)
+- Fix incorrectly used Arcs [\#47](https://github.com/nymtech/nym/issues/47)
+- nym-mixnode mandatory host option [\#26](https://github.com/nymtech/nym/issues/26)
+- Create config struct for mixnode \(possibly also for client\) [\#21](https://github.com/nymtech/nym/issues/21)
+- Reuse TCP socket connection between client and mixnodes [\#20](https://github.com/nymtech/nym/issues/20)
+- Once implementation is available, wherever appropriate, replace `futures::lock::Mutex` with `futures::lock::RwLock` [\#9](https://github.com/nymtech/nym/issues/9)
+- Check if RwLock on MixProcessingData is still needed [\#8](https://github.com/nymtech/nym/issues/8)
+- Reuse TCP socket connection between mixnodes and providers [\#3](https://github.com/nymtech/nym/issues/3)
+- Persistent socket connection with other mixes [\#2](https://github.com/nymtech/nym/issues/2)
+
+**Merged pull requests:**
+
+- Feature/client refactoring [\#128](https://github.com/nymtech/nym/pull/128) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/provider refactoring [\#125](https://github.com/nymtech/nym/pull/125) ([jstuczyn](https://github.com/jstuczyn))
+- all: fixing mis-spelling [\#123](https://github.com/nymtech/nym/pull/123) ([futurechimp](https://github.com/futurechimp))
+- Feature/further clippy fixes [\#121](https://github.com/nymtech/nym/pull/121) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/tokio tungstenite dependency fix [\#120](https://github.com/nymtech/nym/pull/120) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/config files cleanup [\#119](https://github.com/nymtech/nym/pull/119) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/config files [\#117](https://github.com/nymtech/nym/pull/117) ([jstuczyn](https://github.com/jstuczyn))
+- Feature/un genericize keys [\#111](https://github.com/nymtech/nym/pull/111) ([futurechimp](https://github.com/futurechimp))
+- Feature/abci [\#110](https://github.com/nymtech/nym/pull/110) ([futurechimp](https://github.com/futurechimp))
+- Simplified the use of generics on identity keypair by using output types [\#109](https://github.com/nymtech/nym/pull/109) ([jstuczyn](https://github.com/jstuczyn))
+
+## [v0.4.1](https://github.com/nymtech/nym/tree/v0.4.1) (2020-01-29)
+
+[Full Changelog](https://github.com/nymtech/nym/compare/v0.4.0...v0.4.1)
+
+**Closed issues:**
+
+- Change healthcheck to run on provided topology rather than pull one itself [\#95](https://github.com/nymtech/nym/issues/95)
+
+**Merged pull requests:**
+
+- Bugfix/healthcheck on provided topology [\#108](https://github.com/nymtech/nym/pull/108) ([jstuczyn](https://github.com/jstuczyn))
+
 ## [v0.4.0](https://github.com/nymtech/nym/tree/v0.4.0) (2020-01-28)
 
 [Full Changelog](https://github.com/nymtech/nym/compare/v0.4.0-rc.2...v0.4.0)

Cargo.toml

@@ -6,9 +6,11 @@ panic = "abort"
 members = [
     "common/clients/directory-client",
     "common/clients/mix-client",
+    "common/clients/multi-tcp-client",
     "common/clients/provider-client",
     "common/clients/validator-client",
     "common/addressing",
+    "common/config",
     "common/crypto",
     "common/healthcheck",
     "common/pemstore",

common/clients/directory-client/Cargo.toml

@@ -16,4 +16,4 @@ serde = { version = "1.0.104", features = ["derive"] }
 topology = {path = "../../topology"}
 
 [dev-dependencies]
-mockito = "0.22.0"
+mockito = "0.23.0"

common/clients/directory-client/src/lib.rs

@@ -54,7 +54,7 @@ impl DirectoryClient for Client {
         let presence_mix_nodes_post: PresenceMixNodesPost =
             PresenceMixNodesPost::new(config.base_url.clone());
         let presence_providers_post: PresenceProvidersPost =
-            PresenceProvidersPost::new(config.base_url.clone());
+            PresenceProvidersPost::new(config.base_url);
         Client {
             health_check,
             metrics_mixes,

common/clients/directory-client/src/presence/mod.rs

@@ -29,11 +29,10 @@ impl NymTopology for Topology {
         };
         let directory = Client::new(directory_config);
 
-        let topology = directory
+        directory
             .presence_topology
             .get()
-            .expect("Failed to retrieve network topology.");
-        topology
+            .expect("Failed to retrieve network topology.")
     }
 
     fn new_from_nodes(

common/clients/mix-client/Cargo.toml

@@ -18,6 +18,6 @@ addressing = {path = "../../addressing"}
 topology = {path = "../../topology"}
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 # sphinx = { path = "../../../../sphinx"}
 

common/clients/mix-client/src/lib.rs

@@ -9,6 +9,7 @@ pub mod poisson;
 pub struct MixClient {}
 
 impl MixClient {
+    #[allow(clippy::new_without_default)]
     pub fn new() -> MixClient {
         MixClient {}
     }

common/clients/mix-client/src/packet.rs

@@ -3,10 +3,10 @@ use addressing::AddressTypeError;
 use sphinx::route::{Destination, DestinationAddressBytes, SURBIdentifier};
 use sphinx::SphinxPacket;
 use std::net::SocketAddr;
+use std::time;
 use topology::{NymTopology, NymTopologyError};
 
 pub const LOOP_COVER_MESSAGE_PAYLOAD: &[u8] = b"The cake is a lie!";
-pub const LOOP_COVER_MESSAGE_AVERAGE_DELAY: f64 = 2.0;
 
 #[derive(Debug)]
 pub enum SphinxPacketEncapsulationError {
@@ -39,29 +39,49 @@ impl From<AddressTypeError> for SphinxPacketEncapsulationError {
     }
 }
 
+#[deprecated(note = "please use loop_cover_message_route instead")]
 pub fn loop_cover_message<T: NymTopology>(
     our_address: DestinationAddressBytes,
     surb_id: SURBIdentifier,
     topology: &T,
+    average_delay: time::Duration,
 ) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> {
     let destination = Destination::new(our_address, surb_id);
 
+    #[allow(deprecated)]
     encapsulate_message(
         destination,
         LOOP_COVER_MESSAGE_PAYLOAD.to_vec(),
         topology,
-        LOOP_COVER_MESSAGE_AVERAGE_DELAY,
+        average_delay,
     )
 }
 
+pub fn loop_cover_message_route(
+    our_address: DestinationAddressBytes,
+    surb_id: SURBIdentifier,
+    route: Vec<sphinx::route::Node>,
+    average_delay: time::Duration,
+) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> {
+    let destination = Destination::new(our_address, surb_id);
+
+    encapsulate_message_route(
+        destination,
+        LOOP_COVER_MESSAGE_PAYLOAD.to_vec(),
+        route,
+        average_delay,
+    )
+}
+
+#[deprecated(note = "please use encapsulate_message_route instead")]
 pub fn encapsulate_message<T: NymTopology>(
     recipient: Destination,
     message: Vec<u8>,
     topology: &T,
-    average_delay: f64,
+    average_delay: time::Duration,
 ) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> {
     let mut providers = topology.providers();
-    if providers.len() == 0 {
+    if providers.is_empty() {
         return Err(SphinxPacketEncapsulationError::NoValidProvidersError);
     }
     // unwrap is fine here as we asserted there is at least single provider
@@ -69,7 +89,7 @@ pub fn encapsulate_message<T: NymTopology>(
 
     let route = topology.route_to(provider)?;
 
-    let delays = sphinx::header::delays::generate(route.len(), average_delay);
+    let delays = sphinx::header::delays::generate_from_average_duration(route.len(), average_delay);
 
     // build the packet
     let packet = sphinx::SphinxPacket::new(message, &route[..], &recipient, &delays)?;
@@ -80,3 +100,20 @@ pub fn encapsulate_message<T: NymTopology>(
 
     Ok((first_node_address, packet))
 }
+
+pub fn encapsulate_message_route(
+    recipient: Destination,
+    message: Vec<u8>,
+    route: Vec<sphinx::route::Node>,
+    average_delay: time::Duration,
+) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> {
+    let delays = sphinx::header::delays::generate_from_average_duration(route.len(), average_delay);
+
+    // build the packet
+    let packet = sphinx::SphinxPacket::new(message, &route[..], &recipient, &delays)?;
+
+    let first_node_address =
+        addressing::socket_address_from_encoded_bytes(route.first().unwrap().address.to_bytes())?;
+
+    Ok((first_node_address, packet))
+}

common/clients/mix-client/src/poisson.rs

@@ -1,9 +1,10 @@
 use rand_distr::{Distribution, Exp};
+use std::time;
 
-pub fn sample(average_delay: f64) -> f64 {
+pub fn sample(average_duration: time::Duration) -> time::Duration {
     // this is our internal code used by our traffic streams
     // the error is only thrown if average delay is less than 0, which will never happen
     // so call to unwrap is perfectly safe here
-    let exp = Exp::new(1.0 / average_delay).unwrap();
-    exp.sample(&mut rand::thread_rng())
+    let exp = Exp::new(1.0 / average_duration.as_nanos() as f64).unwrap();
+    time::Duration::from_nanos(exp.sample(&mut rand::thread_rng()).round() as u64)
 }

common/clients/multi-tcp-client/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "multi-tcp-client"
+version = "0.1.0"
+authors = ["Jedrzej Stuczynski <andrew@nymtech.net>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+futures = "0.3"
+log = "0.4.8"
+tokio = { version = "0.2", features = ["full"] }

common/clients/multi-tcp-client/src/connection_manager/mod.rs

@@ -0,0 +1,94 @@
+use crate::connection_manager::reconnector::ConnectionReconnector;
+use crate::connection_manager::writer::ConnectionWriter;
+use futures::task::Poll;
+use futures::AsyncWriteExt;
+use log::*;
+use std::io;
+use std::net::SocketAddr;
+use std::time::Duration;
+
+mod reconnector;
+mod writer;
+
+enum ConnectionState<'a> {
+    Writing(ConnectionWriter),
+    Reconnecting(ConnectionReconnector<'a>),
+}
+
+pub(crate) struct ConnectionManager<'a> {
+    address: SocketAddr,
+
+    maximum_reconnection_backoff: Duration,
+    reconnection_backoff: Duration,
+
+    state: ConnectionState<'a>,
+}
+
+impl<'a> ConnectionManager<'a> {
+    pub(crate) async fn new(
+        address: SocketAddr,
+        reconnection_backoff: Duration,
+        maximum_reconnection_backoff: Duration,
+    ) -> ConnectionManager<'a> {
+        // based on initial connection we will either have a writer or a reconnector
+        let state = match tokio::net::TcpStream::connect(address).await {
+            Ok(conn) => ConnectionState::Writing(ConnectionWriter::new(conn)),
+            Err(e) => {
+                warn!(
+                    "failed to establish initial connection to {} ({}). Going into reconnection mode",
+                    address, e
+                );
+                ConnectionState::Reconnecting(ConnectionReconnector::new(
+                    address,
+                    reconnection_backoff,
+                    maximum_reconnection_backoff,
+                ))
+            }
+        };
+
+        ConnectionManager {
+            address,
+            maximum_reconnection_backoff,
+            reconnection_backoff,
+            state,
+        }
+    }
+
+    pub(crate) async fn send(&mut self, msg: &[u8]) -> io::Result<()> {
+        if let ConnectionState::Reconnecting(conn_reconnector) = &mut self.state {
+            // do a single poll rather than await for future to completely resolve
+            let new_connection = match futures::poll!(conn_reconnector) {
+                Poll::Pending => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::BrokenPipe,
+                        "connection is broken - reconnection is in progress",
+                    ))
+                }
+                Poll::Ready(conn) => conn,
+            };
+
+            debug!("Managed to reconnect to {}!", self.address);
+            self.state = ConnectionState::Writing(ConnectionWriter::new(new_connection));
+        }
+
+        // we must be in writing state if we are here, either by being here from beginning or just
+        // transitioning from reconnecting
+        if let ConnectionState::Writing(conn_writer) = &mut self.state {
+            return match conn_writer.write_all(msg).await {
+                // if we failed to write to connection we should reconnect
+                // TODO: is this true? can we fail to write to a connection while it still remains open and valid?
+                Ok(_) => Ok(()),
+                Err(e) => {
+                    trace!("Creating connection reconnector!");
+                    self.state = ConnectionState::Reconnecting(ConnectionReconnector::new(
+                        self.address,
+                        self.reconnection_backoff,
+                        self.maximum_reconnection_backoff,
+                    ));
+                    Err(e)
+                }
+            };
+        };
+        unreachable!()
+    }
+}

common/clients/multi-tcp-client/src/connection_manager/reconnector.rs

@@ -0,0 +1,78 @@
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use log::*;
+use std::future::Future;
+use std::io;
+use std::net::SocketAddr;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+pub(crate) struct ConnectionReconnector<'a> {
+    address: SocketAddr,
+    connection: BoxFuture<'a, io::Result<tokio::net::TcpStream>>,
+
+    current_retry_attempt: u32,
+
+    current_backoff_delay: tokio::time::Delay,
+    maximum_reconnection_backoff: Duration,
+
+    initial_reconnection_backoff: Duration,
+}
+
+impl<'a> ConnectionReconnector<'a> {
+    pub(crate) fn new(
+        address: SocketAddr,
+        initial_reconnection_backoff: Duration,
+        maximum_reconnection_backoff: Duration,
+    ) -> ConnectionReconnector<'a> {
+        ConnectionReconnector {
+            address,
+            connection: tokio::net::TcpStream::connect(address).boxed(),
+            current_backoff_delay: tokio::time::delay_for(Duration::new(0, 0)), // if we can re-establish connection on first try without any backoff that's perfect
+            current_retry_attempt: 0,
+            maximum_reconnection_backoff,
+            initial_reconnection_backoff,
+        }
+    }
+}
+
+impl<'a> Future for ConnectionReconnector<'a> {
+    type Output = tokio::net::TcpStream;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        // see if we are still in exponential backoff
+        if Pin::new(&mut self.current_backoff_delay)
+            .poll(cx)
+            .is_pending()
+        {
+            return Poll::Pending;
+        };
+
+        // see if we managed to resolve the connection yet
+        match Pin::new(&mut self.connection).poll(cx) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(Err(e)) => {
+                warn!(
+                    "we failed to re-establish connection to {} - {:?} (attempt {})",
+                    self.address, e, self.current_retry_attempt
+                );
+                self.current_retry_attempt += 1;
+
+                // we failed to re-establish connection - continue exponential backoff
+                let next_delay = std::cmp::min(
+                    self.maximum_reconnection_backoff,
+                    2_u32.pow(self.current_retry_attempt) * self.initial_reconnection_backoff,
+                );
+
+                self.current_backoff_delay
+                    .reset(tokio::time::Instant::now() + next_delay);
+
+                self.connection = tokio::net::TcpStream::connect(self.address).boxed();
+
+                Poll::Pending
+            }
+            Poll::Ready(Ok(conn)) => Poll::Ready(conn),
+        }
+    }
+}

common/clients/multi-tcp-client/src/connection_manager/writer.rs

@@ -0,0 +1,46 @@
+use futures::task::{Context, Poll};
+use futures::AsyncWrite;
+use std::io;
+use std::pin::Pin;
+use tokio::prelude::*;
+
+pub(crate) struct ConnectionWriter {
+    connection: tokio::net::TcpStream,
+}
+
+impl ConnectionWriter {
+    pub(crate) fn new(connection: tokio::net::TcpStream) -> Self {
+        ConnectionWriter { connection }
+    }
+}
+
+impl AsyncWrite for ConnectionWriter {
+    fn poll_write(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        use tokio::io::AsyncWrite;
+
+        let mut read_buf = [0; 1];
+        match Pin::new(&mut self.connection).poll_read(cx, &mut read_buf) {
+            // at least try the obvious check for if connection is definitely down
+            // TODO: can we do anything else?
+            Poll::Ready(Ok(n)) if n == 0 => Poll::Ready(Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                "trying to write to closed connection",
+            ))),
+            _ => Pin::new(&mut self.connection).poll_write(cx, buf),
+        }
+    }
+
+    fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        use tokio::io::AsyncWrite;
+        Pin::new(&mut self.connection).poll_flush(cx)
+    }
+
+    fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        use tokio::io::AsyncWrite;
+        Pin::new(&mut self.connection).poll_shutdown(cx)
+    }
+}

common/clients/multi-tcp-client/src/lib.rs

@@ -0,0 +1,226 @@
+use crate::connection_manager::ConnectionManager;
+use log::*;
+use std::collections::HashMap;
+use std::io;
+use std::net::SocketAddr;
+use std::time::Duration;
+
+mod connection_manager;
+
+pub struct Config {
+    initial_endpoints: Vec<SocketAddr>,
+    initial_reconnection_backoff: Duration,
+    maximum_reconnection_backoff: Duration,
+}
+
+impl Config {
+    pub fn new(
+        initial_endpoints: Vec<SocketAddr>,
+        initial_reconnection_backoff: Duration,
+        maximum_reconnection_backoff: Duration,
+    ) -> Self {
+        Config {
+            initial_endpoints,
+            initial_reconnection_backoff,
+            maximum_reconnection_backoff,
+        }
+    }
+}
+
+pub struct Client<'a> {
+    connections_managers: HashMap<SocketAddr, ConnectionManager<'a>>,
+    maximum_reconnection_backoff: Duration,
+    initial_reconnection_backoff: Duration,
+}
+
+impl<'a> Client<'a> {
+    pub async fn new(config: Config) -> Client<'a> {
+        let mut connections_managers = HashMap::new();
+        for initial_endpoint in config.initial_endpoints {
+            connections_managers.insert(
+                initial_endpoint,
+                ConnectionManager::new(
+                    initial_endpoint,
+                    config.initial_reconnection_backoff,
+                    config.maximum_reconnection_backoff,
+                )
+                .await,
+            );
+        }
+
+        Client {
+            connections_managers,
+            initial_reconnection_backoff: config.maximum_reconnection_backoff,
+            maximum_reconnection_backoff: config.initial_reconnection_backoff,
+        }
+    }
+
+    pub async fn send(&mut self, address: SocketAddr, message: &[u8]) -> io::Result<()> {
+        if !self.connections_managers.contains_key(&address) {
+            info!(
+                "There is no existing connection to {:?} - it will be established now",
+                address
+            );
+
+            // TODO: now we're blocking to establish TCP connection this need to be changed
+            // so that other connections could progress
+            let new_manager = ConnectionManager::new(
+                address,
+                self.initial_reconnection_backoff,
+                self.maximum_reconnection_backoff,
+            )
+            .await;
+
+            self.connections_managers.insert(address, new_manager);
+        }
+
+        // to optimize later by using channels and separate tokio tasks for each connection handler
+        // because right now say we want to write to addresses A and B -
+        // We have to wait until we're done dealing with A before we can do anything with B
+        self.connections_managers
+            .get_mut(&address)
+            .unwrap()
+            .send(&message)
+            .await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::str;
+    use std::time;
+    use tokio::prelude::*;
+
+    const SERVER_MSG_LEN: usize = 16;
+    const CLOSE_MESSAGE: [u8; SERVER_MSG_LEN] = [0; SERVER_MSG_LEN];
+
+    struct DummyServer {
+        received_buf: Vec<Vec<u8>>,
+        listener: tokio::net::TcpListener,
+    }
+
+    impl DummyServer {
+        async fn new(address: SocketAddr) -> Self {
+            DummyServer {
+                received_buf: Vec::new(),
+                listener: tokio::net::TcpListener::bind(address).await.unwrap(),
+            }
+        }
+
+        fn get_received(&self) -> Vec<Vec<u8>> {
+            self.received_buf.clone()
+        }
+
+        // this is only used in tests so slightly higher logging levels are fine
+        async fn listen_until(mut self, close_message: &[u8]) -> Self {
+            let (mut socket, _) = self.listener.accept().await.unwrap();
+            loop {
+                let mut buf = [0u8; SERVER_MSG_LEN];
+                match socket.read(&mut buf).await {
+                    Ok(n) if n == 0 => {
+                        info!("Remote connection closed");
+                        return self;
+                    }
+                    Ok(n) => {
+                        info!("received ({}) - {:?}", n, str::from_utf8(buf[..n].as_ref()));
+
+                        if buf[..n].as_ref() == close_message {
+                            info!("closing...");
+                            socket.shutdown(std::net::Shutdown::Both).unwrap();
+                            return self;
+                        } else {
+                            self.received_buf.push(buf[..n].to_vec());
+                        }
+                    }
+                    Err(e) => {
+                        panic!("failed to read from socket; err = {:?}", e);
+                    }
+                };
+            }
+        }
+    }
+
+    #[test]
+    fn client_reconnects_to_server_after_it_went_down() {
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let addr = "127.0.0.1:6000".parse().unwrap();
+        let reconnection_backoff = Duration::from_secs(1);
+        let client_config =
+            Config::new(vec![addr], reconnection_backoff, 10 * reconnection_backoff);
+
+        let messages_to_send = vec![[1u8; SERVER_MSG_LEN], [2; SERVER_MSG_LEN]];
+
+        let dummy_server = rt.block_on(DummyServer::new(addr));
+        let finished_dummy_server_future = rt.spawn(dummy_server.listen_until(&CLOSE_MESSAGE));
+
+        let mut c = rt.block_on(Client::new(client_config));
+
+        for msg in &messages_to_send {
+            rt.block_on(c.send(addr, msg)).unwrap();
+        }
+
+        // kill server
+        rt.block_on(c.send(addr, &CLOSE_MESSAGE)).unwrap();
+        let received_messages = rt
+            .block_on(finished_dummy_server_future)
+            .unwrap()
+            .get_received();
+
+        assert_eq!(received_messages, messages_to_send);
+
+        // try to send - go into reconnection
+        let post_kill_message = [3u8; SERVER_MSG_LEN];
+
+        // we are trying to send to killed server
+        assert!(rt.block_on(c.send(addr, &post_kill_message)).is_err());
+
+        let new_dummy_server = rt.block_on(DummyServer::new(addr));
+        let new_server_future = rt.spawn(new_dummy_server.listen_until(&CLOSE_MESSAGE));
+
+        // keep sending after we leave reconnection backoff and reconnect
+        loop {
+            if rt.block_on(c.send(addr, &post_kill_message)).is_ok() {
+                break;
+            }
+            rt.block_on(
+                async move { tokio::time::delay_for(time::Duration::from_millis(50)).await },
+            );
+        }
+
+        // kill the server to ensure it actually got the message
+        rt.block_on(c.send(addr, &CLOSE_MESSAGE)).unwrap();
+        let new_received_messages = rt.block_on(new_server_future).unwrap().get_received();
+        assert_eq!(post_kill_message.to_vec(), new_received_messages[0]);
+    }
+
+    #[test]
+    fn server_receives_all_sent_messages_when_up() {
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let addr = "127.0.0.1:6001".parse().unwrap();
+        let reconnection_backoff = Duration::from_secs(2);
+        let client_config =
+            Config::new(vec![addr], reconnection_backoff, 10 * reconnection_backoff);
+
+        let messages_to_send = vec![[1u8; SERVER_MSG_LEN], [2; SERVER_MSG_LEN]];
+
+        let dummy_server = rt.block_on(DummyServer::new(addr));
+        let finished_dummy_server_future = rt.spawn(dummy_server.listen_until(&CLOSE_MESSAGE));
+
+        let mut c = rt.block_on(Client::new(client_config));
+
+        for msg in &messages_to_send {
+            rt.block_on(c.send(addr, msg)).unwrap();
+        }
+
+        rt.block_on(c.send(addr, &CLOSE_MESSAGE)).unwrap();
+
+        // the server future should have already been resolved
+        let received_messages = rt
+            .block_on(finished_dummy_server_future)
+            .unwrap()
+            .get_received();
+
+        assert_eq!(received_messages, messages_to_send);
+    }
+}

common/clients/provider-client/Cargo.toml

@@ -16,4 +16,4 @@ tokio = { version = "0.2", features = ["full"] }
 sfw-provider-requests = { path = "../../../sfw-provider/sfw-provider-requests" }
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }

common/clients/provider-client/src/lib.rs

@@ -82,14 +82,14 @@ impl ProviderClient {
     }
 
     pub async fn retrieve_messages(&self) -> Result<Vec<Vec<u8>>, ProviderClientError> {
-        let auth_token = match self.auth_token {
-            Some(token) => token,
+        let auth_token = match self.auth_token.as_ref() {
+            Some(token) => token.clone(),
             None => {
                 return Err(ProviderClientError::EmptyAuthTokenError);
             }
         };
 
-        let pull_request = PullRequest::new(self.our_address, auth_token);
+        let pull_request = PullRequest::new(self.our_address.clone(), auth_token);
         let bytes = pull_request.to_bytes();
 
         let response = self.send_request(bytes).await?;
@@ -103,7 +103,7 @@ impl ProviderClient {
             return Err(ProviderClientError::ClientAlreadyRegisteredError);
         }
 
-        let register_request = RegisterRequest::new(self.our_address);
+        let register_request = RegisterRequest::new(self.our_address.clone());
         let bytes = register_request.to_bytes();
 
         let response = self.send_request(bytes).await?;

common/config/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "config"
+version = "0.1.0"
+authors = ["Jedrzej Stuczynski <andrew@nymtech.net>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+handlebars = "3.0.1"
+serde = { version = "1.0.104", features = ["derive"] }
+toml = "0.5.6"

common/config/src/lib.rs

@@ -0,0 +1,64 @@
+use handlebars::Handlebars;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use std::path::PathBuf;
+use std::{fs, io};
+
+pub trait NymConfig: Default + Serialize + DeserializeOwned {
+    fn template() -> &'static str;
+
+    fn config_file_name() -> String;
+
+    fn default_root_directory() -> PathBuf;
+
+    // default, most probable, implementations; can be easily overridden where required
+    fn default_config_directory(id: Option<&str>) -> PathBuf {
+        Self::default_root_directory()
+            .join(id.unwrap_or(""))
+            .join("config")
+    }
+
+    fn default_data_directory(id: Option<&str>) -> PathBuf {
+        Self::default_root_directory()
+            .join(id.unwrap_or(""))
+            .join("data")
+    }
+
+    fn root_directory(&self) -> PathBuf;
+    fn config_directory(&self) -> PathBuf;
+    fn data_directory(&self) -> PathBuf;
+
+    fn save_to_file(&self, custom_location: Option<PathBuf>) -> io::Result<()> {
+        let reg = Handlebars::new();
+        // it's whoever is implementing the trait responsibility to make sure you can execute your own template on your data
+        let templated_config = reg.render_template(Self::template(), self).unwrap();
+
+        // make sure the whole directory structure actually exists
+        match custom_location.clone() {
+            Some(loc) => {
+                if let Some(parent_dir) = loc.parent() {
+                    fs::create_dir_all(parent_dir)
+                } else {
+                    Ok(())
+                }
+            }
+            None => fs::create_dir_all(self.config_directory()),
+        }?;
+
+        fs::write(
+            custom_location
+                .unwrap_or_else(|| self.config_directory().join(Self::config_file_name())),
+            templated_config,
+        )
+    }
+
+    fn load_from_file(custom_location: Option<PathBuf>, id: Option<&str>) -> io::Result<Self> {
+        let config_contents = fs::read_to_string(
+            custom_location
+                .unwrap_or_else(|| Self::default_config_directory(id).join("config.toml")),
+        )?;
+
+        toml::from_str(&config_contents)
+            .map_err(|toml_err| io::Error::new(io::ErrorKind::Other, toml_err))
+    }
+}

common/crypto/Cargo.toml

@@ -15,4 +15,4 @@ rand = "0.7.2"
 rand_os = "0.1"
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }

common/crypto/src/encryption/mod.rs

@@ -1,39 +1,138 @@
-use crate::PemStorable;
+use crate::{PemStorableKey, PemStorableKeyPair};
+use curve25519_dalek::montgomery::MontgomeryPoint;
+use curve25519_dalek::scalar::Scalar;
 
-pub mod x25519;
+// TODO: ensure this is a proper name for this considering we are not implementing entire DH here
 
-pub trait MixnetEncryptionKeyPair<Priv, Pub>
-where
-    Priv: MixnetEncryptionPrivateKey,
-    Pub: MixnetEncryptionPublicKey,
-{
-    fn new() -> Self;
-    fn private_key(&self) -> &Priv;
-    fn public_key(&self) -> &Pub;
-    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self;
+const CURVE_GENERATOR: MontgomeryPoint = curve25519_dalek::constants::X25519_BASEPOINT;
 
-    // TODO: encryption related methods
+pub struct KeyPair {
+    pub(crate) private_key: PrivateKey,
+    pub(crate) public_key: PublicKey,
 }
 
-pub trait MixnetEncryptionPublicKey:
-    Sized + PemStorable + for<'a> From<&'a <Self as MixnetEncryptionPublicKey>::PrivateKeyMaterial>
-{
-    // we need to couple public and private keys together
-    type PrivateKeyMaterial: MixnetEncryptionPrivateKey<PublicKeyMaterial = Self>;
+impl KeyPair {
+    pub fn new() -> Self {
+        let mut rng = rand_os::OsRng::new().unwrap();
+        let private_key_value = Scalar::random(&mut rng);
+        let public_key_value = CURVE_GENERATOR * private_key_value;
 
-    fn to_bytes(&self) -> Vec<u8>;
-    fn from_bytes(b: &[u8]) -> Self;
-}
-
-pub trait MixnetEncryptionPrivateKey: Sized + PemStorable {
-    // we need to couple public and private keys together
-    type PublicKeyMaterial: MixnetEncryptionPublicKey<PrivateKeyMaterial = Self>;
-
-    /// Returns the associated public key
-    fn public_key(&self) -> Self::PublicKeyMaterial {
-        self.into()
+        KeyPair {
+            private_key: PrivateKey(private_key_value),
+            public_key: PublicKey(public_key_value),
+        }
     }
 
-    fn to_bytes(&self) -> Vec<u8>;
-    fn from_bytes(b: &[u8]) -> Self;
+    pub fn private_key(&self) -> &PrivateKey {
+        &self.private_key
+    }
+
+    pub fn public_key(&self) -> &PublicKey {
+        &self.public_key
+    }
+
+    pub fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
+        KeyPair {
+            private_key: PrivateKey::from_bytes(priv_bytes),
+            public_key: PublicKey::from_bytes(pub_bytes),
+        }
+    }
+}
+
+impl Default for KeyPair {
+    fn default() -> Self {
+        KeyPair::new()
+    }
+}
+
+impl PemStorableKeyPair for KeyPair {
+    type PrivatePemKey = PrivateKey;
+    type PublicPemKey = PublicKey;
+
+    fn private_key(&self) -> &Self::PrivatePemKey {
+        self.private_key()
+    }
+
+    fn public_key(&self) -> &Self::PublicPemKey {
+        self.public_key()
+    }
+
+    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
+        Self::from_bytes(priv_bytes, pub_bytes)
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct PrivateKey(pub Scalar);
+
+impl PrivateKey {
+    pub fn to_bytes(&self) -> Vec<u8> {
+        self.0.to_bytes().to_vec()
+    }
+
+    pub fn from_bytes(b: &[u8]) -> Self {
+        let mut bytes = [0; 32];
+        bytes.copy_from_slice(&b[..]);
+        // due to trait restriction we have no choice but to panic if this fails
+        let key = Scalar::from_canonical_bytes(bytes).unwrap();
+        Self(key)
+    }
+
+    pub fn inner(&self) -> Scalar {
+        self.0
+    }
+}
+
+impl PemStorableKey for PrivateKey {
+    fn pem_type(&self) -> String {
+        String::from("X25519 PRIVATE KEY")
+    }
+
+    fn to_bytes(&self) -> Vec<u8> {
+        self.to_bytes()
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct PublicKey(pub MontgomeryPoint);
+
+impl<'a> From<&'a PrivateKey> for PublicKey {
+    fn from(pk: &'a PrivateKey) -> Self {
+        PublicKey(CURVE_GENERATOR * pk.0)
+    }
+}
+
+impl PublicKey {
+    pub fn to_bytes(&self) -> Vec<u8> {
+        self.0.to_bytes().to_vec()
+    }
+
+    pub fn from_bytes(b: &[u8]) -> Self {
+        let mut bytes = [0; 32];
+        bytes.copy_from_slice(&b[..]);
+        let key = MontgomeryPoint(bytes);
+        Self(key)
+    }
+
+    pub fn inner(&self) -> MontgomeryPoint {
+        self.0
+    }
+
+    pub fn to_base58_string(&self) -> String {
+        bs58::encode(&self.to_bytes()).into_string()
+    }
+
+    pub fn from_base58_string(val: String) -> Self {
+        Self::from_bytes(&bs58::decode(&val).into_vec().unwrap())
+    }
+}
+
+impl PemStorableKey for PublicKey {
+    fn pem_type(&self) -> String {
+        String::from("X25519 PUBLIC KEY")
+    }
+
+    fn to_bytes(&self) -> Vec<u8> {
+        self.to_bytes()
+    }
 }

common/crypto/src/encryption/x25519.rs

@@ -1,99 +0,0 @@
-use crate::encryption::{
-    MixnetEncryptionKeyPair, MixnetEncryptionPrivateKey, MixnetEncryptionPublicKey,
-};
-use crate::PemStorable;
-use curve25519_dalek::montgomery::MontgomeryPoint;
-use curve25519_dalek::scalar::Scalar;
-
-// TODO: ensure this is a proper name for this considering we are not implementing entire DH here
-
-const CURVE_GENERATOR: MontgomeryPoint = curve25519_dalek::constants::X25519_BASEPOINT;
-
-pub struct KeyPair {
-    pub(crate) private_key: PrivateKey,
-    pub(crate) public_key: PublicKey,
-}
-
-impl MixnetEncryptionKeyPair<PrivateKey, PublicKey> for KeyPair {
-    fn new() -> Self {
-        let mut rng = rand_os::OsRng::new().unwrap();
-        let private_key_value = Scalar::random(&mut rng);
-        let public_key_value = CURVE_GENERATOR * private_key_value;
-
-        KeyPair {
-            private_key: PrivateKey(private_key_value),
-            public_key: PublicKey(public_key_value),
-        }
-    }
-
-    fn private_key(&self) -> &PrivateKey {
-        &self.private_key
-    }
-
-    fn public_key(&self) -> &PublicKey {
-        &self.public_key
-    }
-
-    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
-        KeyPair {
-            private_key: PrivateKey::from_bytes(priv_bytes),
-            public_key: PublicKey::from_bytes(pub_bytes),
-        }
-    }
-}
-
-// COPY IS DERIVED ONLY TEMPORARILY UNTIL https://github.com/nymtech/nym/issues/47 is fixed
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
-pub struct PrivateKey(pub Scalar);
-
-impl MixnetEncryptionPrivateKey for PrivateKey {
-    type PublicKeyMaterial = PublicKey;
-
-    fn to_bytes(&self) -> Vec<u8> {
-        self.0.to_bytes().to_vec()
-    }
-
-    fn from_bytes(b: &[u8]) -> Self {
-        let mut bytes = [0; 32];
-        bytes.copy_from_slice(&b[..]);
-        // due to trait restriction we have no choice but to panic if this fails
-        let key = Scalar::from_canonical_bytes(bytes).unwrap();
-        Self(key)
-    }
-}
-
-impl PemStorable for PrivateKey {
-    fn pem_type(&self) -> String {
-        String::from("X25519 PRIVATE KEY")
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct PublicKey(pub MontgomeryPoint);
-
-impl<'a> From<&'a PrivateKey> for PublicKey {
-    fn from(pk: &'a PrivateKey) -> Self {
-        PublicKey(CURVE_GENERATOR * pk.0)
-    }
-}
-
-impl MixnetEncryptionPublicKey for PublicKey {
-    type PrivateKeyMaterial = PrivateKey;
-
-    fn to_bytes(&self) -> Vec<u8> {
-        self.0.to_bytes().to_vec()
-    }
-
-    fn from_bytes(b: &[u8]) -> Self {
-        let mut bytes = [0; 32];
-        bytes.copy_from_slice(&b[..]);
-        let key = MontgomeryPoint(bytes);
-        Self(key)
-    }
-}
-
-impl PemStorable for PublicKey {
-    fn pem_type(&self) -> String {
-        String::from("X25519 PUBLIC KEY")
-    }
-}

common/crypto/src/identity/mod.rs

@@ -1,162 +1,138 @@
-use crate::encryption::{
-    MixnetEncryptionKeyPair, MixnetEncryptionPrivateKey, MixnetEncryptionPublicKey,
-};
-use crate::{encryption, PemStorable};
+use crate::{encryption, PemStorableKey, PemStorableKeyPair};
 use bs58;
 use curve25519_dalek::scalar::Scalar;
 use sphinx::route::DestinationAddressBytes;
 
-pub trait MixnetIdentityKeyPair<Priv, Pub>: Clone
-where
-    Priv: MixnetIdentityPrivateKey,
-    Pub: MixnetIdentityPublicKey,
-{
-    fn new() -> Self;
-    fn private_key(&self) -> &Priv;
-    fn public_key(&self) -> &Pub;
-    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self;
-
-    // TODO: signing related methods
-}
-
-pub trait MixnetIdentityPublicKey:
-    Sized
-    + PemStorable
-    + Clone
-    + for<'a> From<&'a <Self as MixnetIdentityPublicKey>::PrivateKeyMaterial>
-{
-    // we need to couple public and private keys together
-    type PrivateKeyMaterial: MixnetIdentityPrivateKey<PublicKeyMaterial = Self>;
-
-    fn derive_address(&self) -> DestinationAddressBytes;
-    fn to_bytes(&self) -> Vec<u8>;
-    fn from_bytes(b: &[u8]) -> Self;
-}
-
-pub trait MixnetIdentityPrivateKey: Sized + PemStorable + Clone {
-    // we need to couple public and private keys together
-    type PublicKeyMaterial: MixnetIdentityPublicKey<PrivateKeyMaterial = Self>;
-
-    /// Returns the associated public key
-    fn public_key(&self) -> Self::PublicKeyMaterial {
-        self.into()
-    }
-
-    fn to_bytes(&self) -> Vec<u8>;
-    fn from_bytes(b: &[u8]) -> Self;
-}
-
-// same for validator
-
 // for time being define a dummy identity using x25519 encryption keys (as we've done so far)
 // and replace it with proper keys, like ed25519 later on
 #[derive(Clone)]
-pub struct DummyMixIdentityKeyPair {
-    pub private_key: DummyMixIdentityPrivateKey,
-    pub public_key: DummyMixIdentityPublicKey,
+pub struct MixIdentityKeyPair {
+    pub private_key: MixIdentityPrivateKey,
+    pub public_key: MixIdentityPublicKey,
 }
 
-impl MixnetIdentityKeyPair<DummyMixIdentityPrivateKey, DummyMixIdentityPublicKey>
-    for DummyMixIdentityKeyPair
-{
-    fn new() -> Self {
-        let keypair = encryption::x25519::KeyPair::new();
-        DummyMixIdentityKeyPair {
-            private_key: DummyMixIdentityPrivateKey(keypair.private_key),
-            public_key: DummyMixIdentityPublicKey(keypair.public_key),
+impl MixIdentityKeyPair {
+    pub fn new() -> Self {
+        let keypair = encryption::KeyPair::new();
+        MixIdentityKeyPair {
+            private_key: MixIdentityPrivateKey(keypair.private_key),
+            public_key: MixIdentityPublicKey(keypair.public_key),
         }
     }
 
-    fn private_key(&self) -> &DummyMixIdentityPrivateKey {
+    pub fn private_key(&self) -> &MixIdentityPrivateKey {
         &self.private_key
     }
 
-    fn public_key(&self) -> &DummyMixIdentityPublicKey {
+    pub fn public_key(&self) -> &MixIdentityPublicKey {
         &self.public_key
     }
 
-    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
-        DummyMixIdentityKeyPair {
-            private_key: DummyMixIdentityPrivateKey::from_bytes(priv_bytes),
-            public_key: DummyMixIdentityPublicKey::from_bytes(pub_bytes),
+    pub fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
+        MixIdentityKeyPair {
+            private_key: MixIdentityPrivateKey::from_bytes(priv_bytes),
+            public_key: MixIdentityPublicKey::from_bytes(pub_bytes),
         }
     }
 }
 
+impl Default for MixIdentityKeyPair {
+    fn default() -> Self {
+        MixIdentityKeyPair::new()
+    }
+}
+
+impl PemStorableKeyPair for MixIdentityKeyPair {
+    type PrivatePemKey = MixIdentityPrivateKey;
+    type PublicPemKey = MixIdentityPublicKey;
+
+    fn private_key(&self) -> &Self::PrivatePemKey {
+        self.private_key()
+    }
+
+    fn public_key(&self) -> &Self::PublicPemKey {
+        self.public_key()
+    }
+
+    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self {
+        Self::from_bytes(priv_bytes, pub_bytes)
+    }
+}
+
 #[derive(Debug, Clone, Eq, PartialEq)]
-pub struct DummyMixIdentityPublicKey(encryption::x25519::PublicKey);
+pub struct MixIdentityPublicKey(encryption::PublicKey);
 
-impl MixnetIdentityPublicKey for DummyMixIdentityPublicKey {
-    type PrivateKeyMaterial = DummyMixIdentityPrivateKey;
-
-    fn derive_address(&self) -> DestinationAddressBytes {
+impl MixIdentityPublicKey {
+    pub fn derive_address(&self) -> DestinationAddressBytes {
         let mut temporary_address = [0u8; 32];
         let public_key_bytes = self.to_bytes();
         temporary_address.copy_from_slice(&public_key_bytes[..]);
 
-        temporary_address
+        DestinationAddressBytes::from_bytes(temporary_address)
     }
 
-    fn to_bytes(&self) -> Vec<u8> {
+    pub fn to_bytes(&self) -> Vec<u8> {
         self.0.to_bytes()
     }
 
-    fn from_bytes(b: &[u8]) -> Self {
-        Self(encryption::x25519::PublicKey::from_bytes(b))
+    pub fn from_bytes(b: &[u8]) -> Self {
+        Self(encryption::PublicKey::from_bytes(b))
     }
-}
 
-impl PemStorable for DummyMixIdentityPublicKey {
-    fn pem_type(&self) -> String {
-        format!("DUMMY KEY BASED ON {}", self.0.pem_type())
-    }
-}
-
-impl DummyMixIdentityPublicKey {
     pub fn to_base58_string(&self) -> String {
         bs58::encode(&self.to_bytes()).into_string()
     }
 
-    #[allow(dead_code)]
-    fn from_base58_string(val: String) -> Self {
+    pub fn from_base58_string(val: String) -> Self {
         Self::from_bytes(&bs58::decode(&val).into_vec().unwrap())
     }
 }
 
-// COPY IS DERIVED ONLY TEMPORARILY UNTIL https://github.com/nymtech/nym/issues/47 is fixed
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
-pub struct DummyMixIdentityPrivateKey(pub encryption::x25519::PrivateKey);
+impl PemStorableKey for MixIdentityPublicKey {
+    fn pem_type(&self) -> String {
+        format!("DUMMY KEY BASED ON {}", self.0.pem_type())
+    }
 
-impl<'a> From<&'a DummyMixIdentityPrivateKey> for DummyMixIdentityPublicKey {
-    fn from(pk: &'a DummyMixIdentityPrivateKey) -> Self {
-        let private_ref = &pk.0;
-        let public: encryption::x25519::PublicKey = private_ref.into();
-        DummyMixIdentityPublicKey(public)
+    fn to_bytes(&self) -> Vec<u8> {
+        self.to_bytes()
     }
 }
 
-impl MixnetIdentityPrivateKey for DummyMixIdentityPrivateKey {
-    type PublicKeyMaterial = DummyMixIdentityPublicKey;
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct MixIdentityPrivateKey(pub encryption::PrivateKey);
 
-    fn to_bytes(&self) -> Vec<u8> {
+impl<'a> From<&'a MixIdentityPrivateKey> for MixIdentityPublicKey {
+    fn from(pk: &'a MixIdentityPrivateKey) -> Self {
+        let private_ref = &pk.0;
+        let public: encryption::PublicKey = private_ref.into();
+        MixIdentityPublicKey(public)
+    }
+}
+
+impl MixIdentityPrivateKey {
+    pub fn to_bytes(&self) -> Vec<u8> {
         self.0.to_bytes()
     }
 
-    fn from_bytes(b: &[u8]) -> Self {
-        Self(encryption::x25519::PrivateKey::from_bytes(b))
+    pub fn from_bytes(b: &[u8]) -> Self {
+        Self(encryption::PrivateKey::from_bytes(b))
     }
 }
 
 // TODO: this will be implemented differently by using the proper trait
-impl DummyMixIdentityPrivateKey {
-    pub fn as_scalar(self) -> Scalar {
-        let encryption_key = self.0;
+impl MixIdentityPrivateKey {
+    pub fn as_scalar(&self) -> Scalar {
+        let encryption_key = &self.0;
         encryption_key.0
     }
 }
 
-impl PemStorable for DummyMixIdentityPrivateKey {
+impl PemStorableKey for MixIdentityPrivateKey {
     fn pem_type(&self) -> String {
         format!("DUMMY KEY BASED ON {}", self.0.pem_type())
     }
+
+    fn to_bytes(&self) -> Vec<u8> {
+        self.to_bytes()
+    }
 }

common/crypto/src/lib.rs

@@ -1,9 +1,22 @@
 pub mod encryption;
 pub mod identity;
 
-// TODO: this trait will need to be moved elsewhere, probably to some 'persistence' crate
-// but since it will need to be used by all identities, it's not really appropriate if it lived in nym-client
+// TODO: ideally those trait should be moved to 'pemstore' crate, however, that would cause
+// circular dependency. The best solution would be to remove dependency on 'crypto' from
+// pemstore by using either dynamic dispatch or generics - perhaps this should be done
+// at some point during one of refactors.
 
-pub trait PemStorable {
+pub trait PemStorableKey {
     fn pem_type(&self) -> String;
+    fn to_bytes(&self) -> Vec<u8>;
+}
+
+pub trait PemStorableKeyPair {
+    type PrivatePemKey: PemStorableKey;
+    type PublicPemKey: PemStorableKey;
+
+    fn private_key(&self) -> &Self::PrivatePemKey;
+    fn public_key(&self) -> &Self::PublicPemKey;
+
+    fn from_bytes(priv_bytes: &[u8], pub_bytes: &[u8]) -> Self;
 }

common/healthcheck/Cargo.toml

@@ -27,7 +27,7 @@ sfw-provider-requests = { path = "../../sfw-provider/sfw-provider-requests" }
 topology = {path = "../topology" }
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 # sphinx = { path = "../../../sphinx"}
 
 [dev-dependencies]

common/healthcheck/src/lib.rs

@@ -1,10 +1,7 @@
 use crate::result::HealthCheckResult;
-use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey};
-use directory_client::requests::presence_topology_get::PresenceTopologyGetRequester;
-use directory_client::DirectoryClient;
-use log::{debug, error, info, trace};
+use crypto::identity::MixIdentityKeyPair;
+use log::trace;
 use std::fmt::{Error, Formatter};
-use std::marker::PhantomData;
 use std::time::Duration;
 use topology::{NymTopology, NymTopologyError};
 
@@ -36,38 +33,22 @@ impl From<topology::NymTopologyError> for HealthCheckerError {
     }
 }
 
-pub struct HealthChecker<IDPair, Priv, Pub>
-where
-    IDPair: MixnetIdentityKeyPair<Priv, Pub>,
-    Priv: MixnetIdentityPrivateKey,
-    Pub: MixnetIdentityPublicKey,
-{
+pub struct HealthChecker {
     num_test_packets: usize,
     resolution_timeout: Duration,
-    identity_keypair: IDPair,
-
-    _phantom_private: PhantomData<Priv>,
-    _phantom_public: PhantomData<Pub>,
+    identity_keypair: MixIdentityKeyPair,
 }
 
-impl<IDPair, Priv, Pub> HealthChecker<IDPair, Priv, Pub>
-where
-    IDPair: crypto::identity::MixnetIdentityKeyPair<Priv, Pub>,
-    Priv: crypto::identity::MixnetIdentityPrivateKey,
-    Pub: crypto::identity::MixnetIdentityPublicKey,
-{
+impl HealthChecker {
     pub fn new(
-        resolution_timeout_f64: f64,
+        resolution_timeout: Duration,
         num_test_packets: usize,
-        identity_keypair: IDPair,
+        identity_keypair: MixIdentityKeyPair,
     ) -> Self {
         HealthChecker {
-            resolution_timeout: Duration::from_secs_f64(resolution_timeout_f64),
+            resolution_timeout,
             num_test_packets,
             identity_keypair,
-
-            _phantom_private: PhantomData,
-            _phantom_public: PhantomData,
         }
     }
 

common/healthcheck/src/path_check.rs

@@ -1,4 +1,4 @@
-use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey};
+use crypto::identity::MixIdentityKeyPair;
 use itertools::Itertools;
 use log::{debug, error, info, trace, warn};
 use mix_client::MixClient;
@@ -27,22 +27,18 @@ pub(crate) struct PathChecker {
 }
 
 impl PathChecker {
-    pub(crate) async fn new<IDPair, Priv, Pub>(
+    pub(crate) async fn new(
         providers: Vec<provider::Node>,
-        identity_keys: &IDPair,
+        identity_keys: &MixIdentityKeyPair,
         check_id: [u8; 16],
-    ) -> Self
-    where
-        IDPair: MixnetIdentityKeyPair<Priv, Pub>,
-        Priv: MixnetIdentityPrivateKey,
-        Pub: MixnetIdentityPublicKey,
-    {
+    ) -> Self {
         let mut provider_clients = HashMap::new();
 
         let address = identity_keys.public_key().derive_address();
 
         for provider in providers {
-            let mut provider_client = ProviderClient::new(provider.client_listener, address, None);
+            let mut provider_client =
+                ProviderClient::new(provider.client_listener, address.clone(), None);
             let insertion_result = match provider_client.register().await {
                 Ok(token) => {
                     debug!("registered at provider {}", provider.pub_key);
@@ -78,7 +74,7 @@ impl PathChecker {
 
     // iteration is used to distinguish packets sent through the same path (as the healthcheck
     // may try to send say 10 packets through given path)
-    fn unique_path_key(path: &Vec<SphinxNode>, check_id: [u8; 16], iteration: u8) -> Vec<u8> {
+    fn unique_path_key(path: &[SphinxNode], check_id: [u8; 16], iteration: u8) -> Vec<u8> {
         check_id
             .iter()
             .cloned()
@@ -178,8 +174,8 @@ impl PathChecker {
         self.update_path_statuses(provider_messages);
     }
 
-    pub(crate) async fn send_test_packet(&mut self, path: &Vec<SphinxNode>, iteration: u8) {
-        if path.len() == 0 {
+    pub(crate) async fn send_test_packet(&mut self, path: &[SphinxNode], iteration: u8) {
+        if path.is_empty() {
             warn!("trying to send test packet through an empty path!");
             return;
         }
@@ -226,7 +222,7 @@ impl PathChecker {
         let first_node_client = self
             .layer_one_clients
             .entry(first_node_key)
-            .or_insert(Some(mix_client::MixClient::new()));
+            .or_insert_with(|| Some(mix_client::MixClient::new()));
 
         if first_node_client.is_none() {
             debug!("we can ignore this path as layer one mix is inaccessible");
@@ -243,7 +239,7 @@ impl PathChecker {
         // we already checked for 'None' case
         let first_node_client = first_node_client.as_ref().unwrap();
 
-        let delays: Vec<_> = path.iter().map(|_| Delay::new(0)).collect();
+        let delays: Vec<_> = path.iter().map(|_| Delay::new_from_nanos(0)).collect();
 
         // all of the data used to create the packet was created by us
         let packet = sphinx::SphinxPacket::new(
@@ -257,7 +253,7 @@ impl PathChecker {
         debug!("sending test packet to {}", first_node_address);
         match first_node_client.send(packet, first_node_address).await {
             Err(err) => {
-                info!("failed to send packet to {} - {}", first_node_address, err);
+                debug!("failed to send packet to {} - {}", first_node_address, err);
                 if self
                     .paths_status
                     .insert(path_identifier, PathStatus::Unhealthy)

common/healthcheck/src/result.rs

@@ -1,7 +1,7 @@
 use crate::path_check::{PathChecker, PathStatus};
 use crate::score::NodeScore;
-use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey};
-use log::{debug, error, info, warn};
+use crypto::identity::MixIdentityKeyPair;
+use log::{debug, error, warn};
 use rand_os::rand_core::RngCore;
 use sphinx::route::NodeAddressBytes;
 use std::collections::HashMap;
@@ -16,7 +16,7 @@ impl std::fmt::Display for HealthCheckResult {
     fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
         write!(f, "NETWORK HEALTH\n==============\n")?;
         for score in self.0.iter() {
-            write!(f, "{}\n", score)?
+            writeln!(f, "{}", score)?
         }
         Ok(())
     }
@@ -34,12 +34,8 @@ impl HealthCheckResult {
 
         let health = mixes
             .into_iter()
-            .map(|node| NodeScore::from_mixnode(node))
-            .chain(
-                providers
-                    .into_iter()
-                    .map(|node| NodeScore::from_provider(node)),
-            )
+            .map(NodeScore::from_mixnode)
+            .chain(providers.into_iter().map(NodeScore::from_provider))
             .collect();
 
         HealthCheckResult(health)
@@ -102,18 +98,12 @@ impl HealthCheckResult {
         id
     }
 
-    pub async fn calculate<T, IDPair, Priv, Pub>(
+    pub async fn calculate<T: NymTopology>(
         topology: &T,
         iterations: usize,
         resolution_timeout: Duration,
-        identity_keys: &IDPair,
-    ) -> Self
-    where
-        T: NymTopology,
-        IDPair: MixnetIdentityKeyPair<Priv, Pub>,
-        Priv: MixnetIdentityPrivateKey,
-        Pub: MixnetIdentityPublicKey,
-    {
+        identity_keys: &MixIdentityKeyPair,
+    ) -> Self {
         // currently healthchecker supports only up to 255 iterations - if we somehow
         // find we need more, it's relatively easy change
         assert!(iterations <= 255);
@@ -150,7 +140,7 @@ impl HealthCheckResult {
             }
         }
 
-        info!(
+        debug!(
             "waiting {:?} for pending requests to resolve",
             resolution_timeout
         );

common/healthcheck/src/score.rs

@@ -37,6 +37,7 @@ pub(crate) struct NodeScore {
 
 impl Ord for NodeScore {
     // order by: version, layer, sent, received, pubkey; ignore addresses
+    #[allow(clippy::comparison_chain)]
     fn cmp(&self, other: &Self) -> Ordering {
         if self.typ > other.typ {
             return Ordering::Greater;
@@ -110,7 +111,7 @@ impl NodeScore {
             pub_key: NodeAddressBytes::from_base58_string(node.pub_key),
             addresses: vec![node.mixnet_listener, node.client_listener],
             version: node.version,
-            layer: format!("provider"),
+            layer: "provider".to_string(),
             packets_sent: 0,
             packets_received: 0,
         }

common/pemstore/src/pemstore.rs

@@ -1,40 +1,35 @@
 use crate::pathfinder::PathFinder;
+use crypto::identity::MixIdentityKeyPair;
+use crypto::PemStorableKey;
+use crypto::{encryption, PemStorableKeyPair};
+use log::info;
 use pem::{encode, parse, Pem};
 use std::fs::File;
 use std::io;
 use std::io::prelude::*;
 use std::path::PathBuf;
 
-#[allow(dead_code)]
-pub fn read_mix_encryption_keypair_from_disk(_id: String) -> crypto::encryption::x25519::KeyPair {
-    unimplemented!()
-}
-
 pub struct PemStore {
+    #[allow(dead_code)]
     config_dir: PathBuf,
-    private_mix_key: PathBuf,
-    public_mix_key: PathBuf,
+    private_mix_key_file: PathBuf,
+    public_mix_key_file: PathBuf,
 }
 
 impl PemStore {
     pub fn new<P: PathFinder>(pathfinder: P) -> PemStore {
         PemStore {
             config_dir: pathfinder.config_dir(),
-            private_mix_key: pathfinder.private_identity_key(),
-            public_mix_key: pathfinder.public_identity_key(),
+            private_mix_key_file: pathfinder.private_identity_key(),
+            public_mix_key_file: pathfinder.public_identity_key(),
         }
     }
 
-    pub fn read_identity<IDPair, Priv, Pub>(&self) -> io::Result<IDPair>
-    where
-        IDPair: crypto::identity::MixnetIdentityKeyPair<Priv, Pub>,
-        Priv: crypto::identity::MixnetIdentityPrivateKey,
-        Pub: crypto::identity::MixnetIdentityPublicKey,
-    {
-        let private_pem = self.read_pem_file(self.private_mix_key.clone())?;
-        let public_pem = self.read_pem_file(self.public_mix_key.clone())?;
+    pub fn read_keypair<T: PemStorableKeyPair>(&self) -> io::Result<T> {
+        let private_pem = self.read_pem_file(self.private_mix_key_file.clone())?;
+        let public_pem = self.read_pem_file(self.public_mix_key_file.clone())?;
 
-        let key_pair = IDPair::from_bytes(&private_pem.contents, &public_pem.contents);
+        let key_pair = T::from_bytes(&private_pem.contents, &public_pem.contents);
 
         if key_pair.private_key().pem_type() != private_pem.tag {
             return Err(io::Error::new(
@@ -53,39 +48,62 @@ impl PemStore {
         Ok(key_pair)
     }
 
+    pub fn read_encryption(&self) -> io::Result<encryption::KeyPair> {
+        self.read_keypair()
+    }
+
+    pub fn read_identity(&self) -> io::Result<MixIdentityKeyPair> {
+        self.read_keypair()
+    }
+
     fn read_pem_file(&self, filepath: PathBuf) -> io::Result<Pem> {
         let mut pem_bytes = File::open(filepath)?;
         let mut buf = Vec::new();
         pem_bytes.read_to_end(&mut buf)?;
         parse(&buf).map_err(|e| io::Error::new(io::ErrorKind::Other, e))
     }
-    // This should be refactored and made more generic for when we have other kinds of
-    // KeyPairs that we want to persist (e.g. validator keypairs, or keys for
-    // signing vs encryption). However, for the moment, it does the job.
-    pub fn write_identity<IDPair, Priv, Pub>(&self, key_pair: IDPair) -> io::Result<()>
-    where
-        IDPair: crypto::identity::MixnetIdentityKeyPair<Priv, Pub>,
-        Priv: crypto::identity::MixnetIdentityPrivateKey,
-        Pub: crypto::identity::MixnetIdentityPublicKey,
-    {
-        std::fs::create_dir_all(self.config_dir.clone())?;
 
+    fn write_keypair(&self, key_pair: impl PemStorableKeyPair) -> io::Result<()> {
         let private_key = key_pair.private_key();
         let public_key = key_pair.public_key();
+
         self.write_pem_file(
-            self.private_mix_key.clone(),
+            self.private_mix_key_file.clone(),
             private_key.to_bytes(),
             private_key.pem_type(),
         )?;
+        info!(
+            "Written private key to {:?}",
+            self.private_mix_key_file.clone()
+        );
         self.write_pem_file(
-            self.public_mix_key.clone(),
+            self.public_mix_key_file.clone(),
             public_key.to_bytes(),
             public_key.pem_type(),
         )?;
+        info!(
+            "Written public key to {:?}",
+            self.public_mix_key_file.clone()
+        );
         Ok(())
     }
 
+    // This should be refactored and made more generic for when we have other kinds of
+    // KeyPairs that we want to persist (e.g. validator keypairs, or keys for
+    // signing vs encryption). However, for the moment, it does the job.
+    pub fn write_identity(&self, key_pair: MixIdentityKeyPair) -> io::Result<()> {
+        self.write_keypair(key_pair)
+    }
+
+    pub fn write_encryption_keys(&self, key_pair: encryption::KeyPair) -> io::Result<()> {
+        self.write_keypair(key_pair)
+    }
+
     fn write_pem_file(&self, filepath: PathBuf, data: Vec<u8>, tag: String) -> io::Result<()> {
+        // ensure the whole directory structure exists
+        if let Some(parent_dir) = filepath.parent() {
+            std::fs::create_dir_all(parent_dir)?;
+        }
         let pem = Pem {
             tag,
             contents: data,

common/topology/Cargo.toml

@@ -19,5 +19,5 @@ addressing = {path = "../addressing"}
 version-checker = {path = "../version-checker" }
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 # sphinx = { path = "../../../sphinx"}

common/topology/src/lib.rs

@@ -10,7 +10,9 @@ mod filter;
 pub mod mix;
 pub mod provider;
 
-pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync {
+// TODO: Figure out why 'Clone' was required to have 'TopologyAccessor<T>' working
+// even though it only contains an Arc
+pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync + Clone {
     fn new(directory_server: String) -> Self;
     fn new_from_nodes(
         mix_nodes: Vec<mix::Node>,
@@ -30,7 +32,7 @@ pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync {
             }
             highest_layer = max(highest_layer, mix.layer);
 
-            let layer_nodes = layered_topology.entry(mix.layer).or_insert(Vec::new());
+            let layer_nodes = layered_topology.entry(mix.layer).or_insert_with(Vec::new);
             layer_nodes.push(mix);
         }
 
@@ -40,12 +42,12 @@ pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync {
             if !layered_topology.contains_key(&layer) {
                 missing_layers.push(layer);
             }
-            if layered_topology[&layer].len() == 0 {
+            if layered_topology[&layer].is_empty() {
                 missing_layers.push(layer);
             }
         }
 
-        if missing_layers.len() > 0 {
+        if !missing_layers.is_empty() {
             return Err(NymTopologyError::MissingLayerError(missing_layers));
         }
 
@@ -112,10 +114,7 @@ pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync {
     }
 
     fn can_construct_path_through(&self) -> bool {
-        match self.make_layered_topology() {
-            Ok(_) => true,
-            Err(_) => false,
-        }
+        self.make_layered_topology().is_ok()
     }
 }
 

mixnode/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 build = "build.rs"
 name = "nym-mixnode"
-version = "0.4.1"
+version = "0.5.0-rc.1"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 edition = "2018"
 
@@ -11,18 +11,31 @@ edition = "2018"
 bs58 = "0.3.0"
 clap = "2.33.0"
 curve25519-dalek = "1.2.3"
+dirs = "2.0.2"
 dotenv = "0.15.0"
 futures = "0.3.1"
 log = "0.4"
 pretty_env_logger = "0.3"
+serde = { version = "1.0.104", features = ["derive"] }
 tokio = { version = "0.2", features = ["full"] }
 
 ## internal
 addressing = {path = "../common/addressing" }
+config = {path = "../common/config"}
+crypto = {path = "../common/crypto"}
 directory-client = { path = "../common/clients/directory-client" }
+multi-tcp-client = { path = "../common/clients/multi-tcp-client" }
+pemstore = {path = "../common/pemstore"}
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 
 [build-dependencies]
 built = "0.3.2"
+
+[dev-dependencies]
+tempfile = "3.1.0"
+
+[features]
+qa = []
+local = []

mixnode/src/built_info.rs

@@ -0,0 +1,2 @@
+// The file has been placed there by the build script.
+include!(concat!(env!("OUT_DIR"), "/built.rs"));

mixnode/src/commands/init.rs

@@ -0,0 +1,82 @@
+use crate::commands::override_config;
+use crate::config::persistence::pathfinder::MixNodePathfinder;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+use crypto::encryption;
+use pemstore::pemstore::PemStore;
+
+pub fn command_args<'a, 'b>() -> clap::App<'a, 'b> {
+    App::new("init")
+        .about("Initialise the mixnode")
+        .arg(
+            Arg::with_name("id")
+                .long("id")
+                .help("Id of the nym-mixnode we want to create config for.")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("layer")
+                .long("layer")
+                .help("The mixnet layer of this particular node")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("host")
+                .long("host")
+                .help("The custom host on which the mixnode will be running")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("port")
+                .long("port")
+                .help("The port on which the mixnode will be listening")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("announce-host")
+                .long("announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("announce-port")
+                .long("announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("directory")
+                .long("directory")
+                .help("Address of the directory server the node is sending presence and metrics to")
+                .takes_value(true),
+        )
+}
+
+pub fn execute(matches: &ArgMatches) {
+    let id = matches.value_of("id").unwrap();
+    println!("Initialising mixnode {}...", id);
+
+    let layer = matches.value_of("layer").unwrap().parse().unwrap();
+    let mut config = crate::config::Config::new(id, layer);
+
+    config = override_config(config, matches);
+
+    let sphinx_keys = encryption::KeyPair::new();
+    let pathfinder = MixNodePathfinder::new_from_config(&config);
+    let pem_store = PemStore::new(pathfinder);
+    pem_store
+        .write_encryption_keys(sphinx_keys)
+        .expect("Failed to save sphinx keys");
+    println!("Saved mixnet sphinx keypair");
+
+    let config_save_location = config.get_config_file_save_location();
+    config
+        .save_to_file(None)
+        .expect("Failed to save the config file");
+    println!("Saved configuration file to {:?}", config_save_location);
+
+    println!("Mixnode configuration completed.\n\n\n")
+}

mixnode/src/commands/mod.rs

@@ -0,0 +1,40 @@
+use crate::config::Config;
+use clap::ArgMatches;
+
+pub mod init;
+pub mod run;
+
+pub(crate) fn override_config(mut config: Config, matches: &ArgMatches) -> Config {
+    if let Some(host) = matches.value_of("host") {
+        config = config.with_listening_host(host);
+    }
+
+    if let Some(port) = matches.value_of("port").map(|port| port.parse::<u16>()) {
+        if let Err(err) = port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_listening_port(port.unwrap());
+    }
+
+    if let Some(directory) = matches.value_of("directory") {
+        config = config.with_custom_directory(directory);
+    }
+
+    if let Some(announce_host) = matches.value_of("announce-host") {
+        config = config.with_announce_host(announce_host);
+    }
+
+    if let Some(announce_port) = matches
+        .value_of("announce-port")
+        .map(|port| port.parse::<u16>())
+    {
+        if let Err(err) = announce_port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_announce_port(announce_port.unwrap());
+    }
+
+    config
+}

mixnode/src/commands/run.rs

@@ -0,0 +1,112 @@
+use crate::commands::override_config;
+use crate::config::Config;
+use crate::node::MixNode;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+
+pub fn command_args<'a, 'b>() -> App<'a, 'b> {
+    App::new("run")
+        .about("Starts the mixnode")
+        .arg(
+            Arg::with_name("id")
+                .long("id")
+                .help("Id of the nym-mixnode we want to run")
+                .takes_value(true)
+                .required(true),
+        )
+        // the rest of arguments are optional, they are used to override settings in config file
+        .arg(
+            Arg::with_name("config")
+                .long("config")
+                .help("Custom path to the nym-mixnode configuration file")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("layer")
+                .long("layer")
+                .help("The mixnet layer of this particular node")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("host")
+                .long("host")
+                .help("The custom host on which the mixnode will be running")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("port")
+                .long("port")
+                .help("The port on which the mixnode will be listening")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("announce-host")
+                .long("announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("announce-port")
+                .long("announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("directory")
+                .long("directory")
+                .help("Address of the directory server the node is sending presence and metrics to")
+                .takes_value(true),
+        )
+}
+
+fn show_binding_warning(address: String) {
+    println!("\n##### WARNING #####");
+    println!(
+        "\nYou are trying to bind to {} - you might not be accessible to other nodes\n\
+         You can ignore this warning if you're running setup on a local network \n\
+         or have set a custom 'announce-host'",
+        address
+    );
+    println!("\n##### WARNING #####\n");
+}
+
+fn special_addresses() -> Vec<&'static str> {
+    vec!["localhost", "127.0.0.1", "0.0.0.0", "::1", "[::1]"]
+}
+
+pub fn execute(matches: &ArgMatches) {
+    let id = matches.value_of("id").unwrap();
+
+    println!("Starting mixnode {}...", id);
+
+    let mut config =
+        Config::load_from_file(matches.value_of("config").map(|path| path.into()), Some(id))
+            .expect("Failed to load config file");
+
+    config = override_config(config, matches);
+
+    let listening_ip_string = config.get_listening_address().ip().to_string();
+    if special_addresses().contains(&listening_ip_string.as_ref()) {
+        show_binding_warning(listening_ip_string);
+    }
+
+    println!(
+        "Directory server [presence]: {}",
+        config.get_presence_directory_server()
+    );
+    println!(
+        "Directory server [metrics]: {}",
+        config.get_metrics_directory_server()
+    );
+
+    println!(
+        "Listening for incoming packets on {}",
+        config.get_listening_address()
+    );
+    println!(
+        "Announcing the following socket address: {}",
+        config.get_announce_address()
+    );
+
+    MixNode::new(config).run();
+}

mixnode/src/config/mod.rs

@@ -0,0 +1,362 @@
+use crate::config::template::config_template;
+use config::NymConfig;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::net::{IpAddr, SocketAddr};
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::time;
+
+pub mod persistence;
+mod template;
+
+// 'MIXNODE'
+const DEFAULT_LISTENING_PORT: u16 = 1789;
+
+// 'DEBUG'
+// where applicable, the below are defined in milliseconds
+const DEFAULT_PRESENCE_SENDING_DELAY: u64 = 3000;
+const DEFAULT_METRICS_SENDING_DELAY: u64 = 3000;
+const DEFAULT_PACKET_FORWARDING_INITIAL_BACKOFF: u64 = 10_000; // 10s
+const DEFAULT_PACKET_FORWARDING_MAXIMUM_BACKOFF: u64 = 300_000; // 5min
+
+#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Config {
+    mixnode: MixNode,
+
+    #[serde(default)]
+    logging: Logging,
+    #[serde(default)]
+    debug: Debug,
+}
+
+impl NymConfig for Config {
+    fn template() -> &'static str {
+        config_template()
+    }
+
+    fn config_file_name() -> String {
+        "config.toml".to_string()
+    }
+
+    fn default_root_directory() -> PathBuf {
+        dirs::home_dir()
+            .expect("Failed to evaluate $HOME value")
+            .join(".nym")
+            .join("mixnodes")
+    }
+
+    fn root_directory(&self) -> PathBuf {
+        self.mixnode.nym_root_directory.clone()
+    }
+
+    fn config_directory(&self) -> PathBuf {
+        self.mixnode
+            .nym_root_directory
+            .join(&self.mixnode.id)
+            .join("config")
+    }
+
+    fn data_directory(&self) -> PathBuf {
+        self.mixnode
+            .nym_root_directory
+            .join(&self.mixnode.id)
+            .join("data")
+    }
+}
+
+impl Config {
+    pub fn new<S: Into<String>>(id: S, layer: u64) -> Self {
+        Config::default().with_id(id).with_layer(layer)
+    }
+
+    // builder methods
+    pub fn with_id<S: Into<String>>(mut self, id: S) -> Self {
+        let id = id.into();
+        if self.mixnode.private_sphinx_key_file.as_os_str().is_empty() {
+            self.mixnode.private_sphinx_key_file =
+                self::MixNode::default_private_sphinx_key_file(&id);
+        }
+        if self.mixnode.public_sphinx_key_file.as_os_str().is_empty() {
+            self.mixnode.public_sphinx_key_file =
+                self::MixNode::default_public_sphinx_key_file(&id);
+        }
+        self.mixnode.id = id;
+        self
+    }
+
+    pub fn with_layer(mut self, layer: u64) -> Self {
+        self.mixnode.layer = layer;
+        self
+    }
+
+    // if you want to use distinct servers for metrics and presence
+    // you need to do so in the config.toml file.
+    pub fn with_custom_directory<S: Into<String>>(mut self, directory_server: S) -> Self {
+        let directory_server_string = directory_server.into();
+        self.debug.presence_directory_server = directory_server_string.clone();
+        self.debug.metrics_directory_server = directory_server_string;
+        self
+    }
+
+    pub fn with_listening_host<S: Into<String>>(mut self, host: S) -> Self {
+        // see if the provided `host` is just an ip address or ip:port
+        let host = host.into();
+
+        // is it ip:port?
+        match SocketAddr::from_str(host.as_ref()) {
+            Ok(socket_addr) => {
+                self.mixnode.listening_address = socket_addr;
+                self
+            }
+            // try just for ip
+            Err(_) => match IpAddr::from_str(host.as_ref()) {
+                Ok(ip_addr) => {
+                    self.mixnode.listening_address.set_ip(ip_addr);
+                    self
+                }
+                Err(_) => {
+                    error!(
+                        "failed to make any changes to config - invalid host {}",
+                        host
+                    );
+                    self
+                }
+            },
+        }
+    }
+
+    pub fn with_listening_port(mut self, port: u16) -> Self {
+        self.mixnode.listening_address.set_port(port);
+        self
+    }
+
+    pub fn with_announce_host<S: Into<String>>(mut self, host: S) -> Self {
+        // this is slightly more complicated as we store announce information as String,
+        // since it might not necessarily be a valid SocketAddr (say `nymtech.net:8080` is a valid
+        // announce address, yet invalid SocketAddr`
+
+        // first lets see if we received host:port or just host part of an address
+        let host = host.into();
+        let split_host: Vec<_> = host.split(':').collect();
+        match split_host.len() {
+            1 => {
+                // we provided only 'host' part so we are going to reuse existing port
+                self.mixnode.announce_address =
+                    format!("{}:{}", host, self.mixnode.listening_address.port());
+                self
+            }
+            2 => {
+                // we provided 'host:port' so just put the whole thing there
+                self.mixnode.announce_address = host;
+                self
+            }
+            _ => {
+                // we provided something completely invalid, so don't try to parse it
+                error!(
+                    "failed to make any changes to config - invalid announce host {}",
+                    host
+                );
+                self
+            }
+        }
+    }
+
+    pub fn with_announce_port(mut self, port: u16) -> Self {
+        let current_host: Vec<_> = self.mixnode.announce_address.split(':').collect();
+        debug_assert_eq!(current_host.len(), 2);
+        self.mixnode.announce_address = format!("{}:{}", current_host[0], port);
+        self
+    }
+
+    // getters
+    pub fn get_config_file_save_location(&self) -> PathBuf {
+        self.config_directory().join(Self::config_file_name())
+    }
+
+    pub fn get_private_sphinx_key_file(&self) -> PathBuf {
+        self.mixnode.private_sphinx_key_file.clone()
+    }
+
+    pub fn get_public_sphinx_key_file(&self) -> PathBuf {
+        self.mixnode.public_sphinx_key_file.clone()
+    }
+
+    pub fn get_presence_directory_server(&self) -> String {
+        self.debug.presence_directory_server.clone()
+    }
+
+    pub fn get_presence_sending_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.presence_sending_delay)
+    }
+
+    pub fn get_metrics_directory_server(&self) -> String {
+        self.debug.metrics_directory_server.clone()
+    }
+
+    pub fn get_metrics_sending_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.metrics_sending_delay)
+    }
+
+    pub fn get_layer(&self) -> u64 {
+        self.mixnode.layer
+    }
+
+    pub fn get_listening_address(&self) -> SocketAddr {
+        self.mixnode.listening_address
+    }
+
+    pub fn get_announce_address(&self) -> String {
+        self.mixnode.announce_address.clone()
+    }
+
+    pub fn get_packet_forwarding_initial_backoff(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.packet_forwarding_initial_backoff)
+    }
+
+    pub fn get_packet_forwarding_maximum_backoff(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.packet_forwarding_maximum_backoff)
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct MixNode {
+    /// ID specifies the human readable ID of this particular mixnode.
+    id: String,
+
+    /// Layer of this particular mixnode determining its position in the network.
+    layer: u64,
+
+    /// Socket address to which this mixnode will bind to and will be listening for packets.
+    listening_address: SocketAddr,
+
+    /// Optional address announced to the directory server for the clients to connect to.
+    /// It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+    /// later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+    /// Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+    /// are valid announce addresses, while the later will default to whatever port is used for
+    /// `listening_address`.
+    announce_address: String,
+
+    /// Path to file containing private sphinx key.
+    private_sphinx_key_file: PathBuf,
+
+    /// Path to file containing public sphinx key.
+    public_sphinx_key_file: PathBuf,
+
+    /// nym_home_directory specifies absolute path to the home nym MixNodes directory.
+    /// It is expected to use default value and hence .toml file should not redefine this field.
+    nym_root_directory: PathBuf,
+}
+
+impl MixNode {
+    fn default_private_sphinx_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("private_sphinx.pem")
+    }
+
+    fn default_public_sphinx_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("public_sphinx.pem")
+    }
+}
+
+impl Default for MixNode {
+    fn default() -> Self {
+        MixNode {
+            id: "".to_string(),
+            layer: 0,
+            listening_address: format!("0.0.0.0:{}", DEFAULT_LISTENING_PORT)
+                .parse()
+                .unwrap(),
+            announce_address: format!("127.0.0.1:{}", DEFAULT_LISTENING_PORT),
+            private_sphinx_key_file: Default::default(),
+            public_sphinx_key_file: Default::default(),
+            nym_root_directory: Config::default_root_directory(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Logging {}
+
+impl Default for Logging {
+    fn default() -> Self {
+        Logging {}
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Debug {
+    // The idea of additional 'directory servers' is to let mixes report their presence
+    // and metrics to separate places
+    /// Directory server to which the server will be reporting their presence data.
+    presence_directory_server: String,
+
+    /// Delay between each subsequent presence data being sent.
+    /// The provided value is interpreted as milliseconds.
+    presence_sending_delay: u64,
+
+    /// Directory server to which the server will be reporting their metrics data.
+    metrics_directory_server: String,
+
+    /// Delay between each subsequent metrics data being sent.
+    /// The provided value is interpreted as milliseconds.
+    metrics_sending_delay: u64,
+
+    /// Initial value of an exponential backoff to reconnect to dropped TCP connection when
+    /// forwarding sphinx packets.
+    /// The provided value is interpreted as milliseconds.
+    packet_forwarding_initial_backoff: u64,
+
+    /// Maximum value of an exponential backoff to reconnect to dropped TCP connection when
+    /// forwarding sphinx packets.
+    /// The provided value is interpreted as milliseconds.
+    packet_forwarding_maximum_backoff: u64,
+}
+
+impl Debug {
+    fn default_directory_server() -> String {
+        #[cfg(feature = "qa")]
+        return "https://qa-directory.nymtech.net".to_string();
+        #[cfg(feature = "local")]
+        return "http://localhost:8080".to_string();
+
+        "https://directory.nymtech.net".to_string()
+    }
+}
+
+impl Default for Debug {
+    fn default() -> Self {
+        Debug {
+            presence_directory_server: Self::default_directory_server(),
+            presence_sending_delay: DEFAULT_PRESENCE_SENDING_DELAY,
+            metrics_directory_server: Self::default_directory_server(),
+            metrics_sending_delay: DEFAULT_METRICS_SENDING_DELAY,
+            packet_forwarding_initial_backoff: DEFAULT_PACKET_FORWARDING_INITIAL_BACKOFF,
+            packet_forwarding_maximum_backoff: DEFAULT_PACKET_FORWARDING_MAXIMUM_BACKOFF,
+        }
+    }
+}
+
+#[cfg(test)]
+mod mixnode_config {
+    use super::*;
+
+    #[test]
+    fn after_saving_default_config_the_loaded_one_is_identical() {
+        // need to figure out how to do something similar but without touching the disk
+        // or the file system at all...
+        let temp_location = tempfile::tempdir().unwrap().path().join("config.toml");
+        let default_config = Config::default().with_id("foomp".to_string());
+        default_config
+            .save_to_file(Some(temp_location.clone()))
+            .unwrap();
+
+        let loaded_config = Config::load_from_file(Some(temp_location), None).unwrap();
+
+        assert_eq!(default_config, loaded_config);
+    }
+}

mixnode/src/config/persistence/pathfinder.rs

@@ -0,0 +1,44 @@
+use crate::config::Config;
+use pemstore::pathfinder::PathFinder;
+use std::path::PathBuf;
+
+#[derive(Debug)]
+pub struct MixNodePathfinder {
+    pub config_dir: PathBuf,
+    pub private_sphinx_key: PathBuf,
+    pub public_sphinx_key: PathBuf,
+}
+
+impl MixNodePathfinder {
+    pub fn new_from_config(config: &Config) -> Self {
+        MixNodePathfinder {
+            config_dir: config.get_config_file_save_location(),
+            private_sphinx_key: config.get_private_sphinx_key_file(),
+            public_sphinx_key: config.get_public_sphinx_key_file(),
+        }
+    }
+}
+
+impl PathFinder for MixNodePathfinder {
+    fn config_dir(&self) -> PathBuf {
+        self.config_dir.clone()
+    }
+
+    fn private_identity_key(&self) -> PathBuf {
+        // TEMPORARILY USE SAME KEYS AS ENCRYPTION
+        self.private_sphinx_key.clone()
+    }
+
+    fn public_identity_key(&self) -> PathBuf {
+        // TEMPORARILY USE SAME KEYS AS ENCRYPTION
+        self.public_sphinx_key.clone()
+    }
+
+    fn private_encryption_key(&self) -> Option<PathBuf> {
+        Some(self.private_sphinx_key.clone())
+    }
+
+    fn public_encryption_key(&self) -> Option<PathBuf> {
+        Some(self.public_sphinx_key.clone())
+    }
+}

mixnode/src/config/template.rs

@@ -0,0 +1,81 @@
+pub(crate) fn config_template() -> &'static str {
+    // While using normal toml marshalling would have been way simpler with less overhead,
+    // I think it's useful to have comments attached to the saved config file to explain behaviour of
+    // particular fields.
+    // Note: any changes to the template must be reflected in the appropriate structs in mod.rs.
+    r#"
+# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+##### main base mixnode config options #####
+
+[mixnode]
+# Human readable ID of this particular mixnode.
+id = "{{ mixnode.id }}"
+
+# Layer of this particular mixnode determining its position in the network.
+layer = {{ mixnode.layer }}
+
+# Socket address to which this mixnode will bind to and will be listening for packets.
+listening_address = "{{ mixnode.listening_address }}"
+
+# Path to file containing private identity key.
+private_sphinx_key_file = "{{ mixnode.private_sphinx_key_file }}"
+
+# Path to file containing public sphinx key.
+public_sphinx_key_file = "{{ mixnode.public_sphinx_key_file }}"
+
+##### additional mixnode config options #####
+
+# Optional address announced to the directory server for the clients to connect to.
+# It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+# later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+# Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+# are valid announce addresses, while the later will default to whatever port is used for
+# `listening_address`.
+announce_address = "{{ mixnode.announce_address }}"
+
+##### advanced configuration options #####
+
+# Absolute path to the home Nym Clients directory.
+nym_root_directory = "{{ mixnode.nym_root_directory }}"
+
+
+##### logging configuration options #####
+
+[logging]
+
+# TODO
+
+
+##### debug configuration options #####
+# The following options should not be modified unless you know EXACTLY what you are doing
+# as if set incorrectly, they may impact your anonymity.
+
+[debug]
+
+# Directory server to which the server will be reporting their presence data.
+presence_directory_server = "{{ debug.presence_directory_server}}"
+
+# Delay between each subsequent presence data being sent.
+# The provided value is interpreted as milliseconds.
+presence_sending_delay = {{ debug.presence_sending_delay }}
+
+# Directory server to which the server will be reporting their metrics data.
+metrics_directory_server = "{{ debug.metrics_directory_server }}"
+
+# Delay between each subsequent metrics data being sent.
+# The provided value is interpreted as milliseconds.
+metrics_sending_delay = {{ debug.metrics_sending_delay }}
+
+# Initial value of an exponential backoff to reconnect to dropped TCP connection when
+# forwarding sphinx packets.
+# The provided value is interpreted as milliseconds.
+packet_forwarding_initial_backoff = {{ debug.packet_forwarding_initial_backoff }}
+
+# Maximum value of an exponential backoff to reconnect to dropped TCP connection when
+# forwarding sphinx packets.
+# The provided value is interpreted as milliseconds.
+packet_forwarding_maximum_backoff = {{ debug.packet_forwarding_maximum_backoff }}    
+"#
+}

mixnode/src/main.rs

@@ -1,82 +1,37 @@
-use clap::{App, Arg, ArgMatches, SubCommand};
-use log::*;
-use std::process;
+use clap::{App, ArgMatches};
 
-mod mix_peer;
+pub mod built_info;
+mod commands;
+mod config;
 mod node;
 
 fn main() {
     dotenv::dotenv().ok();
     pretty_env_logger::init();
 
+    println!("{}", banner());
+
     let arg_matches = App::new("Nym Mixnode")
         .version(built_info::PKG_VERSION)
         .author("Nymtech")
         .about("Implementation of the Loopix-based Mixnode")
-        .subcommand(
-            SubCommand::with_name("run")
-                .about("Starts the mixnode")
-                .arg(
-                    Arg::with_name("host")
-                        .long("host")
-                        .help("The custom host on which the mixnode will be running")
-                        .takes_value(true)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("port")
-                        .long("port")
-                        .help("The port on which the mixnode will be listening")
-                        .takes_value(true),
-                )
-                .arg(
-                    Arg::with_name("layer")
-                        .long("layer")
-                        .help("The mixnet layer of this particular node")
-                        .takes_value(true)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("announce_host")
-                        .long("announce-host")
-                        .help("The host that will be reported to the directory server")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("announce_port")
-                        .long("announce-port")
-                        .help("The port that will be reported to the directory server")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("directory")
-                        .long("directory")
-                        .help("Address of the directory server the node is sending presence and metrics to")
-                        .takes_value(true),
-                ),
-        )
+        .subcommand(commands::init::command_args())
+        .subcommand(commands::run::command_args())
         .get_matches();
 
-    if let Err(e) = execute(arg_matches) {
-        error!("{}", e);
-        process::exit(1);
-    }
+    execute(arg_matches);
 }
 
-pub mod built_info {
-    // The file has been placed there by the build script.
-    include!(concat!(env!("OUT_DIR"), "/built.rs"));
-}
-
-fn execute(matches: ArgMatches) -> Result<(), String> {
+fn execute(matches: ArgMatches) {
     match matches.subcommand() {
-        ("run", Some(m)) => Ok(node::runner::start(m)),
-        _ => Err(usage()),
+        ("init", Some(m)) => commands::init::execute(m),
+        ("run", Some(m)) => commands::run::execute(m),
+        _ => println!("{}", usage()),
     }
 }
 
-fn usage() -> String {
-    banner() + "usage: --help to see available options.\n\n"
+fn usage() -> &'static str {
+    "usage: --help to see available options.\n\n"
 }
 
 fn banner() -> String {

mixnode/src/mix_peer.rs

@@ -1,46 +0,0 @@
-use addressing;
-use addressing::AddressTypeError;
-use sphinx::route::NodeAddressBytes;
-use std::error::Error;
-use std::net::SocketAddr;
-use tokio::prelude::*;
-
-#[derive(Debug)]
-pub struct MixPeer {
-    connection: SocketAddr,
-}
-
-#[derive(Debug)]
-pub enum MixPeerError {
-    InvalidAddressError,
-}
-
-impl From<addressing::AddressTypeError> for MixPeerError {
-    fn from(_: AddressTypeError) -> Self {
-        use MixPeerError::*;
-
-        InvalidAddressError
-    }
-}
-
-impl MixPeer {
-    // note that very soon `next_hop_address` will be changed to `next_hop_metadata`
-    pub fn new(next_hop_address: NodeAddressBytes) -> Result<MixPeer, MixPeerError> {
-        let next_hop_socket_address =
-            addressing::socket_address_from_encoded_bytes(next_hop_address.to_bytes())?;
-        Ok(MixPeer {
-            connection: next_hop_socket_address,
-        })
-    }
-
-    pub async fn send(&self, bytes: Vec<u8>) -> Result<(), Box<dyn Error>> {
-        let next_hop_address = self.connection.clone();
-        let mut stream = tokio::net::TcpStream::connect(next_hop_address).await?;
-        stream.write_all(&bytes).await?;
-        Ok(())
-    }
-
-    pub fn to_string(&self) -> String {
-        self.connection.to_string()
-    }
-}

mixnode/src/node/listener.rs

@@ -0,0 +1,101 @@
+use crate::node::packet_processing::{MixProcessingResult, PacketProcessor};
+use futures::channel::mpsc;
+use log::*;
+use std::io;
+use std::net::SocketAddr;
+use tokio::prelude::*;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+async fn process_received_packet(
+    packet_data: [u8; sphinx::PACKET_SIZE],
+    packet_processor: PacketProcessor,
+    forwarding_channel: mpsc::UnboundedSender<(SocketAddr, Vec<u8>)>,
+) {
+    // all processing incl. delay was done, the only thing left is to forward it
+    match packet_processor.process_sphinx_packet(packet_data).await {
+        Err(e) => debug!("We failed to process received sphinx packet - {:?}", e),
+        Ok(res) => match res {
+            MixProcessingResult::ForwardHop(hop_address, hop_data) => {
+                // send our data to tcp client for forwarding. If forwarding fails, then it fails,
+                // it's not like we can do anything about it
+                //
+                // in unbounded_send() failed it means that the receiver channel was disconnected
+                // and hence something weird must have happened without a way of recovering
+                forwarding_channel
+                    .unbounded_send((hop_address, hop_data))
+                    .unwrap();
+                packet_processor.report_sent(hop_address);
+            }
+            MixProcessingResult::LoopMessage => {
+                warn!("Somehow processed a loop cover message that we haven't implemented yet!")
+            }
+        },
+    }
+}
+
+async fn process_socket_connection(
+    mut socket: tokio::net::TcpStream,
+    packet_processor: PacketProcessor,
+    forwarding_channel: mpsc::UnboundedSender<(SocketAddr, Vec<u8>)>,
+) {
+    let mut buf = [0u8; sphinx::PACKET_SIZE];
+    loop {
+        match socket.read(&mut buf).await {
+            // socket closed
+            Ok(n) if n == 0 => {
+                trace!("Remote connection closed.");
+                return;
+            }
+            Ok(n) => {
+                if n != sphinx::PACKET_SIZE {
+                    warn!("read data of different length than expected sphinx packet size - {} (expected {})", n, sphinx::PACKET_SIZE);
+                    continue;
+                }
+
+                // we must be able to handle multiple packets from same connection independently
+                tokio::spawn(process_received_packet(
+                    buf,
+                    // note: processing_data is relatively cheap (and safe) to clone -
+                    // it contains arc to private key and metrics reporter (which is just
+                    // a single mpsc unbounded sender)
+                    packet_processor.clone(),
+                    forwarding_channel.clone(),
+                ))
+            }
+            Err(e) => {
+                warn!(
+                    "failed to read from socket. Closing the connection; err = {:?}",
+                    e
+                );
+                return;
+            }
+        };
+    }
+}
+
+pub(crate) fn run_socket_listener(
+    handle: &Handle,
+    addr: SocketAddr,
+    packet_processor: PacketProcessor,
+    forwarding_channel: mpsc::UnboundedSender<(SocketAddr, Vec<u8>)>,
+) -> JoinHandle<io::Result<()>> {
+    let handle_clone = handle.clone();
+    handle.spawn(async move {
+        let mut listener = tokio::net::TcpListener::bind(addr).await?;
+        loop {
+            let (socket, _) = listener.accept().await?;
+
+            let thread_packet_processor = packet_processor.clone();
+            let forwarding_channel_clone = forwarding_channel.clone();
+            handle_clone.spawn(async move {
+                process_socket_connection(
+                    socket,
+                    thread_packet_processor,
+                    forwarding_channel_clone,
+                )
+                .await;
+            });
+        }
+    })
+}

mixnode/src/node/metrics.rs

@@ -8,60 +8,49 @@ use log::{debug, error};
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
 
-const METRICS_INTERVAL: u64 = 3;
+pub(crate) enum MetricEvent {
+    Sent(String),
+    Received,
+}
 
-#[derive(Debug)]
-pub struct MetricsReporter {
+#[derive(Debug, Clone)]
+// Note: you should NEVER create more than a single instance of this using 'new()'.
+// You should always use .clone() to create additional instances
+struct MixMetrics {
+    inner: Arc<Mutex<MixMetricsInner>>,
+}
+
+struct MixMetricsInner {
     received: u64,
     sent: HashMap<String, u64>,
 }
 
-impl MetricsReporter {
+impl MixMetrics {
     pub(crate) fn new() -> Self {
-        MetricsReporter {
-            received: 0,
-            sent: HashMap::new(),
+        MixMetrics {
+            inner: Arc::new(Mutex::new(MixMetricsInner {
+                received: 0,
+                sent: HashMap::new(),
+            })),
         }
     }
 
-    pub(crate) fn add_arc_mutex(self) -> Arc<Mutex<Self>> {
-        Arc::new(Mutex::new(self))
-    }
-
-    async fn increment_received_metrics(metrics: Arc<Mutex<MetricsReporter>>) {
-        let mut unlocked = metrics.lock().await;
+    async fn increment_received_metrics(&mut self) {
+        let mut unlocked = self.inner.lock().await;
         unlocked.received += 1;
     }
 
-    pub(crate) async fn run_received_metrics_control(
-        metrics: Arc<Mutex<MetricsReporter>>,
-        mut rx: mpsc::Receiver<()>,
-    ) {
-        while let Some(_) = rx.next().await {
-            MetricsReporter::increment_received_metrics(metrics.clone()).await;
-        }
-    }
-
-    async fn increment_sent_metrics(metrics: Arc<Mutex<MetricsReporter>>, sent_to: String) {
-        let mut unlocked = metrics.lock().await;
-        let receiver_count = unlocked.sent.entry(sent_to).or_insert(0);
+    async fn increment_sent_metrics(&mut self, destination: String) {
+        let mut unlocked = self.inner.lock().await;
+        let receiver_count = unlocked.sent.entry(destination).or_insert(0);
         *receiver_count += 1;
     }
 
-    pub(crate) async fn run_sent_metrics_control(
-        metrics: Arc<Mutex<MetricsReporter>>,
-        mut rx: mpsc::Receiver<String>,
-    ) {
-        while let Some(sent_metric) = rx.next().await {
-            MetricsReporter::increment_sent_metrics(metrics.clone(), sent_metric).await;
-        }
-    }
-
-    async fn acquire_and_reset_metrics(
-        metrics: Arc<Mutex<MetricsReporter>>,
-    ) -> (u64, HashMap<String, u64>) {
-        let mut unlocked = metrics.lock().await;
+    async fn acquire_and_reset_metrics(&mut self) -> (u64, HashMap<String, u64>) {
+        let mut unlocked = self.inner.lock().await;
         let received = unlocked.received;
 
         let sent = std::mem::replace(&mut unlocked.sent, HashMap::new());
@@ -69,27 +58,138 @@ impl MetricsReporter {
 
         (received, sent)
     }
+}
 
-    pub(crate) async fn run_metrics_sender(
-        metrics: Arc<Mutex<MetricsReporter>>,
-        cfg: directory_client::Config,
-        pub_key_str: String,
-    ) {
-        let delay_duration = Duration::from_secs(METRICS_INTERVAL);
-        let directory_client = directory_client::Client::new(cfg);
-        loop {
-            tokio::time::delay_for(delay_duration).await;
-            let (received, sent) =
-                MetricsReporter::acquire_and_reset_metrics(metrics.clone()).await;
+struct MetricsReceiver {
+    metrics: MixMetrics,
+    metrics_rx: mpsc::UnboundedReceiver<MetricEvent>,
+}
 
-            match directory_client.metrics_post.post(&MixMetric {
-                pub_key: pub_key_str.clone(),
-                received,
-                sent,
-            }) {
-                Err(err) => error!("failed to send metrics - {:?}", err),
-                Ok(_) => debug!("sent metrics information"),
-            }
+impl MetricsReceiver {
+    fn new(metrics: MixMetrics, metrics_rx: mpsc::UnboundedReceiver<MetricEvent>) -> Self {
+        MetricsReceiver {
+            metrics,
+            metrics_rx,
         }
     }
+
+    fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            while let Some(metrics_data) = self.metrics_rx.next().await {
+                match metrics_data {
+                    MetricEvent::Received => self.metrics.increment_received_metrics().await,
+                    MetricEvent::Sent(destination) => {
+                        self.metrics.increment_sent_metrics(destination).await
+                    }
+                }
+            }
+        })
+    }
+}
+
+struct MetricsSender {
+    metrics: MixMetrics,
+    directory_client: directory_client::Client,
+    pub_key_str: String,
+    sending_delay: Duration,
+}
+
+impl MetricsSender {
+    fn new(
+        metrics: MixMetrics,
+        directory_server: String,
+        pub_key_str: String,
+        sending_delay: Duration,
+    ) -> Self {
+        MetricsSender {
+            metrics,
+            directory_client: directory_client::Client::new(directory_client::Config::new(
+                directory_server,
+            )),
+            pub_key_str,
+            sending_delay,
+        }
+    }
+
+    fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            loop {
+                tokio::time::delay_for(self.sending_delay).await;
+                let (received, sent) = self.metrics.acquire_and_reset_metrics().await;
+
+                match self.directory_client.metrics_post.post(&MixMetric {
+                    pub_key: self.pub_key_str.clone(),
+                    received,
+                    sent,
+                }) {
+                    Err(err) => error!("failed to send metrics - {:?}", err),
+                    Ok(_) => debug!("sent metrics information"),
+                }
+            }
+        })
+    }
+}
+
+#[derive(Clone)]
+pub struct MetricsReporter {
+    metrics_tx: mpsc::UnboundedSender<MetricEvent>,
+}
+
+impl MetricsReporter {
+    pub(crate) fn new(metrics_tx: mpsc::UnboundedSender<MetricEvent>) -> Self {
+        MetricsReporter { metrics_tx }
+    }
+
+    pub(crate) fn report_sent(&self, destination: String) {
+        // in unbounded_send() failed it means that the receiver channel was disconnected
+        // and hence something weird must have happened without a way of recovering
+        self.metrics_tx
+            .unbounded_send(MetricEvent::Sent(destination))
+            .unwrap()
+    }
+
+    pub(crate) fn report_received(&self) {
+        // in unbounded_send() failed it means that the receiver channel was disconnected
+        // and hence something weird must have happened without a way of recovering
+        self.metrics_tx
+            .unbounded_send(MetricEvent::Received)
+            .unwrap()
+    }
+}
+
+// basically an easy single entry point to start all metrics related tasks
+pub struct MetricsController {
+    receiver: MetricsReceiver,
+    reporter: MetricsReporter,
+    sender: MetricsSender,
+}
+
+impl MetricsController {
+    pub(crate) fn new(
+        directory_server: String,
+        pub_key_str: String,
+        sending_delay: Duration,
+    ) -> Self {
+        let (metrics_tx, metrics_rx) = mpsc::unbounded();
+        let shared_metrics = MixMetrics::new();
+
+        MetricsController {
+            sender: MetricsSender::new(
+                shared_metrics.clone(),
+                directory_server,
+                pub_key_str,
+                sending_delay,
+            ),
+            receiver: MetricsReceiver::new(shared_metrics, metrics_rx),
+            reporter: MetricsReporter::new(metrics_tx),
+        }
+    }
+
+    // reporter is how node is going to be accessing the metrics data
+    pub(crate) fn start(self, handle: &Handle) -> MetricsReporter {
+        // TODO: should we do anything with JoinHandle(s) returned by start methods?
+        self.receiver.start(handle);
+        self.sender.start(handle);
+        self.reporter
+    }
 }

mixnode/src/node/mod.rs

@@ -1,292 +1,118 @@
-use crate::mix_peer::MixPeer;
-use crate::node;
-use crate::node::metrics::MetricsReporter;
-use curve25519_dalek::montgomery::MontgomeryPoint;
-use curve25519_dalek::scalar::Scalar;
+use crate::config::persistence::pathfinder::MixNodePathfinder;
+use crate::config::Config;
+use crate::node::packet_processing::PacketProcessor;
+use crypto::encryption;
 use futures::channel::mpsc;
-use futures::lock::Mutex;
-use futures::SinkExt;
 use log::*;
-use sphinx::header::delays::Delay as SphinxDelay;
-use sphinx::{ProcessedPacket, SphinxPacket};
+use pemstore::pemstore::PemStore;
 use std::net::SocketAddr;
-use std::sync::Arc;
-use std::time::Duration;
-use tokio::prelude::*;
 use tokio::runtime::Runtime;
 
+mod listener;
 mod metrics;
+mod packet_forwarding;
+pub(crate) mod packet_processing;
 mod presence;
-pub mod runner;
-
-pub struct Config {
-    announce_address: String,
-    directory_server: String,
-    layer: usize,
-    public_key: MontgomeryPoint,
-    secret_key: Scalar,
-    socket_address: SocketAddr,
-}
-
-impl Config {
-    pub fn public_key_string(&self) -> String {
-        let key_bytes = self.public_key.to_bytes().to_vec();
-        bs58::encode(&key_bytes).into_string()
-    }
-}
-
-#[derive(Debug)]
-pub enum MixProcessingError {
-    SphinxRecoveryError,
-    ReceivedFinalHopError,
-    SphinxProcessingError,
-    InvalidHopAddress,
-}
-
-impl From<sphinx::ProcessingError> for MixProcessingError {
-    // for time being just have a single error instance for all possible results of sphinx::ProcessingError
-    fn from(_: sphinx::ProcessingError) -> Self {
-        use MixProcessingError::*;
-
-        SphinxRecoveryError
-    }
-}
-
-struct ForwardingData {
-    packet: SphinxPacket,
-    delay: SphinxDelay,
-    recipient: MixPeer,
-    sent_metrics_tx: mpsc::Sender<String>,
-}
-
-// TODO: this will need to be changed if MixPeer will live longer than our Forwarding Data
-impl ForwardingData {
-    fn new(
-        packet: SphinxPacket,
-        delay: SphinxDelay,
-        recipient: MixPeer,
-        sent_metrics_tx: mpsc::Sender<String>,
-    ) -> Self {
-        ForwardingData {
-            packet,
-            delay,
-            recipient,
-            sent_metrics_tx,
-        }
-    }
-}
-
-// ProcessingData defines all data required to correctly unwrap sphinx packets
-struct ProcessingData {
-    secret_key: Scalar,
-    received_metrics_tx: mpsc::Sender<()>,
-    sent_metrics_tx: mpsc::Sender<String>,
-}
-
-impl ProcessingData {
-    fn new(
-        secret_key: Scalar,
-        received_metrics_tx: mpsc::Sender<()>,
-        sent_metrics_tx: mpsc::Sender<String>,
-    ) -> Self {
-        ProcessingData {
-            secret_key,
-            received_metrics_tx,
-            sent_metrics_tx,
-        }
-    }
-
-    fn add_arc_mutex(self) -> Arc<Mutex<Self>> {
-        Arc::new(Mutex::new(self))
-    }
-}
-
-struct PacketProcessor;
-
-impl PacketProcessor {
-    pub async fn process_sphinx_data_packet(
-        packet_data: &[u8],
-        processing_data: Arc<Mutex<ProcessingData>>,
-    ) -> Result<ForwardingData, MixProcessingError> {
-        // we received something resembling a sphinx packet, report it!
-        let processing_data = processing_data.lock().await;
-        let mut received_metrics_tx = processing_data.received_metrics_tx.clone();
-
-        // if unwrap failed it means our metrics reporter died, so we should exit application and
-        // force restart
-        if received_metrics_tx.send(()).await.is_err() {
-            error!("failed to send metrics data to the controller - the underlying thread probably died!");
-            std::process::exit(1);
-        }
-
-        let packet = SphinxPacket::from_bytes(packet_data.to_vec())?;
-        let (next_packet, next_hop_address, delay) =
-            match packet.process(processing_data.secret_key) {
-                Ok(ProcessedPacket::ProcessedPacketForwardHop(packet, address, delay)) => {
-                    (packet, address, delay)
-                }
-                Ok(_) => return Err(MixProcessingError::ReceivedFinalHopError),
-                Err(e) => {
-                    warn!("Failed to unwrap Sphinx packet: {:?}", e);
-                    return Err(MixProcessingError::SphinxProcessingError);
-                }
-            };
-
-        let next_mix = match MixPeer::new(next_hop_address) {
-            Ok(next_mix) => next_mix,
-            Err(_) => return Err(MixProcessingError::InvalidHopAddress),
-        };
-
-        let fwd_data = ForwardingData::new(
-            next_packet,
-            delay,
-            next_mix,
-            processing_data.sent_metrics_tx.clone(),
-        );
-        Ok(fwd_data)
-    }
-
-    async fn wait_and_forward(mut forwarding_data: ForwardingData) {
-        let delay_duration = Duration::from_nanos(forwarding_data.delay.get_value());
-        tokio::time::delay_for(delay_duration).await;
-
-        if forwarding_data
-            .sent_metrics_tx
-            .send(forwarding_data.recipient.to_string())
-            .await
-            .is_err()
-        {
-            error!("failed to send metrics data to the controller - the underlying thread probably died!");
-            std::process::exit(1);
-        }
-
-        trace!("RECIPIENT: {:?}", forwarding_data.recipient);
-        match forwarding_data
-            .recipient
-            .send(forwarding_data.packet.to_bytes())
-            .await
-        {
-            Ok(()) => (),
-            Err(e) => {
-                warn!(
-                    "failed to write bytes to next mix peer. err = {:?}",
-                    e.to_string()
-                );
-            }
-        }
-    }
-}
 
 // the MixNode will live for whole duration of this program
 pub struct MixNode {
-    directory_server: String,
-    network_address: SocketAddr,
-    public_key: MontgomeryPoint,
-    secret_key: Scalar,
-    // TODO: use it later to enforce forward travel
-    //    layer: usize,
+    runtime: Runtime,
+    config: Config,
+    sphinx_keypair: encryption::KeyPair,
 }
 
 impl MixNode {
-    pub fn new(config: &Config) -> Self {
+    fn load_sphinx_keys(config_file: &Config) -> encryption::KeyPair {
+        let sphinx_keypair = PemStore::new(MixNodePathfinder::new_from_config(&config_file))
+            .read_encryption()
+            .expect("Failed to read stored sphinx key files");
+        println!(
+            "Public encryption key: {}\nFor time being, it is identical to identity keys",
+            sphinx_keypair.public_key().to_base58_string()
+        );
+        sphinx_keypair
+    }
+
+    pub fn new(config: Config) -> Self {
+        let sphinx_keypair = Self::load_sphinx_keys(&config);
+
         MixNode {
-            directory_server: config.directory_server.clone(),
-            network_address: config.socket_address,
-            secret_key: config.secret_key,
-            public_key: config.public_key,
-            //            layer: config.layer,
+            runtime: Runtime::new().unwrap(),
+            config,
+            sphinx_keypair,
         }
     }
 
-    async fn process_socket_connection(
-        mut socket: tokio::net::TcpStream,
-        processing_data: Arc<Mutex<ProcessingData>>,
+    fn start_presence_notifier(&self) {
+        info!("Starting presence notifier...");
+        let notifier_config = presence::NotifierConfig::new(
+            self.config.get_presence_directory_server(),
+            self.config.get_announce_address(),
+            self.sphinx_keypair.public_key().to_base58_string(),
+            self.config.get_layer(),
+            self.config.get_presence_sending_delay(),
+        );
+        presence::Notifier::new(notifier_config).start(self.runtime.handle());
+    }
+
+    fn start_metrics_reporter(&self) -> metrics::MetricsReporter {
+        info!("Starting metrics reporter...");
+        metrics::MetricsController::new(
+            self.config.get_metrics_directory_server(),
+            self.sphinx_keypair.public_key().to_base58_string(),
+            self.config.get_metrics_sending_delay(),
+        )
+        .start(self.runtime.handle())
+    }
+
+    fn start_socket_listener(
+        &self,
+        metrics_reporter: metrics::MetricsReporter,
+        forwarding_channel: mpsc::UnboundedSender<(SocketAddr, Vec<u8>)>,
     ) {
-        let mut buf = [0u8; sphinx::PACKET_SIZE];
+        info!("Starting socket listener...");
+        // this is the only location where our private key is going to be copied
+        // it will be held in memory owned by `MixNode` and inside an Arc of `PacketProcessor`
+        let packet_processor =
+            PacketProcessor::new(self.sphinx_keypair.private_key().clone(), metrics_reporter);
 
-        // In a loop, read data from the socket and write the data back.
-        loop {
-            match socket.read(&mut buf).await {
-                // socket closed
-                Ok(n) if n == 0 => {
-                    trace!("Remote connection closed.");
-                    return;
-                }
-                Ok(_) => {
-                    let fwd_data = match PacketProcessor::process_sphinx_data_packet(
-                        buf.as_ref(),
-                        processing_data.clone(),
-                    )
-                    .await
-                    {
-                        Ok(fwd_data) => fwd_data,
-                        Err(e) => {
-                            warn!("failed to process sphinx packet: {:?}", e);
-                            return;
-                        }
-                    };
-                    PacketProcessor::wait_and_forward(fwd_data).await;
-                }
-                Err(e) => {
-                    warn!("failed to read from socket; err = {:?}", e);
-                    return;
-                }
-            };
-
-            // Write the some data back
-            if let Err(e) = socket.write_all(b"foomp").await {
-                warn!("failed to write reply to socket; err = {:?}", e);
-                return;
-            }
-        }
+        listener::run_socket_listener(
+            self.runtime.handle(),
+            self.config.get_listening_address(),
+            packet_processor,
+            forwarding_channel,
+        );
     }
 
-    pub fn start(&self, config: node::Config) -> Result<(), Box<dyn std::error::Error>> {
-        // Create the runtime, probably later move it to MixNode itself?
-        let mut rt = Runtime::new()?;
+    fn start_packet_forwarder(&mut self) -> mpsc::UnboundedSender<(SocketAddr, Vec<u8>)> {
+        info!("Starting packet forwarder...");
 
-        let (received_tx, received_rx) = mpsc::channel(1024);
-        let (sent_tx, sent_rx) = mpsc::channel(1024);
+        // this can later be replaced with topology information
+        let initial_addresses = vec![];
+        self.runtime
+            .block_on(packet_forwarding::PacketForwarder::new(
+                initial_addresses,
+                self.config.get_packet_forwarding_initial_backoff(),
+                self.config.get_packet_forwarding_maximum_backoff(),
+            ))
+            .start(self.runtime.handle())
+    }
 
-        let directory_cfg = directory_client::Config {
-            base_url: self.directory_server.clone(),
-        };
-        let pub_key_str = bs58::encode(&self.public_key.to_bytes().to_vec()).into_string();
+    pub fn run(&mut self) {
+        let forwarding_channel = self.start_packet_forwarder();
+        let metrics_reporter = self.start_metrics_reporter();
+        self.start_socket_listener(metrics_reporter, forwarding_channel);
+        self.start_presence_notifier();
 
-        rt.spawn({
-            let presence_notifier = presence::Notifier::new(&config);
-            presence_notifier.run()
-        });
+        if let Err(e) = self.runtime.block_on(tokio::signal::ctrl_c()) {
+            error!(
+                "There was an error while capturing SIGINT - {:?}. We will terminate regardless",
+                e
+            );
+        }
 
-        let metrics = MetricsReporter::new().add_arc_mutex();
-        rt.spawn(MetricsReporter::run_received_metrics_control(
-            metrics.clone(),
-            received_rx,
-        ));
-        rt.spawn(MetricsReporter::run_sent_metrics_control(
-            metrics.clone(),
-            sent_rx,
-        ));
-        rt.spawn(MetricsReporter::run_metrics_sender(
-            metrics,
-            directory_cfg,
-            pub_key_str,
-        ));
-
-        // Spawn the root task
-        rt.block_on(async {
-            let mut listener = tokio::net::TcpListener::bind(self.network_address).await?;
-            let processing_data =
-                ProcessingData::new(self.secret_key, received_tx, sent_tx).add_arc_mutex();
-
-            loop {
-                let (socket, _) = listener.accept().await?;
-
-                let thread_processing_data = processing_data.clone();
-                tokio::spawn(async move {
-                    MixNode::process_socket_connection(socket, thread_processing_data).await;
-                });
-            }
-        })
+        println!(
+            "Received SIGINT - the mixnode will terminate now (threads are not YET nicely stopped)"
+        );
     }
 }

mixnode/src/node/packet_forwarding.rs

@@ -0,0 +1,48 @@
+use futures::channel::mpsc;
+use futures::StreamExt;
+use log::*;
+use std::net::SocketAddr;
+use std::time::Duration;
+use tokio::runtime::Handle;
+
+pub(crate) struct PacketForwarder<'a> {
+    tcp_client: multi_tcp_client::Client<'a>,
+    conn_tx: mpsc::UnboundedSender<(SocketAddr, Vec<u8>)>,
+    conn_rx: mpsc::UnboundedReceiver<(SocketAddr, Vec<u8>)>,
+}
+
+impl PacketForwarder<'static> {
+    pub(crate) async fn new(
+        initial_endpoints: Vec<SocketAddr>,
+        initial_reconnection_backoff: Duration,
+        maximum_reconnection_backoff: Duration,
+    ) -> PacketForwarder<'static> {
+        let tcp_client_config = multi_tcp_client::Config::new(
+            initial_endpoints,
+            initial_reconnection_backoff,
+            maximum_reconnection_backoff,
+        );
+
+        let (conn_tx, conn_rx) = mpsc::unbounded();
+
+        PacketForwarder {
+            tcp_client: multi_tcp_client::Client::new(tcp_client_config).await,
+            conn_tx,
+            conn_rx,
+        }
+    }
+
+    pub(crate) fn start(mut self, handle: &Handle) -> mpsc::UnboundedSender<(SocketAddr, Vec<u8>)> {
+        // TODO: what to do with the lost JoinHandle?
+        let sender_channel = self.conn_tx.clone();
+        handle.spawn(async move {
+            while let Some((address, packet)) = self.conn_rx.next().await {
+                match self.tcp_client.send(address, &packet).await {
+                    Err(e) => warn!("Failed to forward packet to {:?} - {:?}", address, e),
+                    Ok(_) => trace!("Forwarded packet to {:?}", address),
+                }
+            }
+        });
+        sender_channel
+    }
+}

mixnode/src/node/packet_processing.rs

@@ -0,0 +1,109 @@
+use crate::node::metrics;
+use addressing::AddressTypeError;
+use crypto::encryption;
+use log::*;
+use sphinx::header::delays::Delay as SphinxDelay;
+use sphinx::route::NodeAddressBytes;
+use sphinx::{ProcessedPacket, SphinxPacket};
+use std::net::SocketAddr;
+use std::ops::Deref;
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub enum MixProcessingError {
+    SphinxRecoveryError,
+    ReceivedFinalHopError,
+    SphinxProcessingError,
+    InvalidHopAddress,
+}
+
+pub enum MixProcessingResult {
+    ForwardHop(SocketAddr, Vec<u8>),
+    #[allow(dead_code)]
+    LoopMessage,
+}
+
+impl From<sphinx::ProcessingError> for MixProcessingError {
+    // for time being just have a single error instance for all possible results of sphinx::ProcessingError
+    fn from(_: sphinx::ProcessingError) -> Self {
+        use MixProcessingError::*;
+
+        SphinxRecoveryError
+    }
+}
+
+impl From<addressing::AddressTypeError> for MixProcessingError {
+    fn from(_: AddressTypeError) -> Self {
+        use MixProcessingError::*;
+
+        InvalidHopAddress
+    }
+}
+
+// PacketProcessor contains all data required to correctly unwrap and forward sphinx packets
+#[derive(Clone)]
+pub struct PacketProcessor {
+    secret_key: Arc<encryption::PrivateKey>,
+    metrics_reporter: metrics::MetricsReporter,
+}
+
+impl PacketProcessor {
+    pub(crate) fn new(
+        secret_key: encryption::PrivateKey,
+        metrics_reporter: metrics::MetricsReporter,
+    ) -> Self {
+        PacketProcessor {
+            secret_key: Arc::new(secret_key),
+            metrics_reporter,
+        }
+    }
+
+    pub(crate) fn report_sent(&self, addr: SocketAddr) {
+        self.metrics_reporter.report_sent(addr.to_string())
+    }
+
+    async fn process_forward_hop(
+        &self,
+        packet: SphinxPacket,
+        forward_address: NodeAddressBytes,
+        delay: SphinxDelay,
+    ) -> Result<MixProcessingResult, MixProcessingError> {
+        let next_hop_address =
+            addressing::socket_address_from_encoded_bytes(forward_address.to_bytes())?;
+
+        // Delay packet for as long as required
+        tokio::time::delay_for(delay.to_duration()).await;
+
+        Ok(MixProcessingResult::ForwardHop(
+            next_hop_address,
+            packet.to_bytes(),
+        ))
+    }
+
+    pub(crate) async fn process_sphinx_packet(
+        &self,
+        raw_packet_data: [u8; sphinx::PACKET_SIZE],
+    ) -> Result<MixProcessingResult, MixProcessingError> {
+        // we received something resembling a sphinx packet, report it!
+        self.metrics_reporter.report_received();
+
+        let packet = SphinxPacket::from_bytes(&raw_packet_data)?;
+
+        match packet.process(self.secret_key.deref().inner()) {
+            Ok(ProcessedPacket::ProcessedPacketForwardHop(packet, address, delay)) => {
+                self.process_forward_hop(packet, address, delay).await
+            }
+            Ok(ProcessedPacket::ProcessedPacketFinalHop(_, _, _)) => {
+                warn!("Received a loop cover message that we haven't implemented yet!");
+                Err(MixProcessingError::ReceivedFinalHopError)
+            }
+            Err(e) => {
+                warn!("Failed to unwrap Sphinx packet: {:?}", e);
+                Err(MixProcessingError::SphinxProcessingError)
+            }
+        }
+    }
+}
+
+// TODO: the test that definitely needs to be written is as follows:
+// we are stuck trying to write to mix A, can we still forward just fine to mix B?

mixnode/src/node/presence.rs

@@ -1,47 +1,77 @@
-use crate::node;
+use crate::built_info;
 use directory_client::presence::mixnodes::MixNodePresence;
 use directory_client::requests::presence_mixnodes_post::PresenceMixNodesPoster;
 use directory_client::DirectoryClient;
 use log::{debug, error};
 use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+pub struct NotifierConfig {
+    directory_server: String,
+    announce_host: String,
+    pub_key_string: String,
+    layer: u64,
+    sending_delay: Duration,
+}
+
+impl NotifierConfig {
+    pub fn new(
+        directory_server: String,
+        announce_host: String,
+        pub_key_string: String,
+        layer: u64,
+        sending_delay: Duration,
+    ) -> Self {
+        NotifierConfig {
+            directory_server,
+            announce_host,
+            pub_key_string,
+            layer,
+            sending_delay,
+        }
+    }
+}
 
 pub struct Notifier {
-    pub net_client: directory_client::Client,
+    net_client: directory_client::Client,
     presence: MixNodePresence,
+    sending_delay: Duration,
 }
 
 impl Notifier {
-    pub fn new(node_config: &node::Config) -> Notifier {
-        let config = directory_client::Config {
-            base_url: node_config.directory_server.clone(),
+    pub fn new(config: NotifierConfig) -> Notifier {
+        let directory_client_cfg = directory_client::Config {
+            base_url: config.directory_server,
         };
-        let net_client = directory_client::Client::new(config);
+        let net_client = directory_client::Client::new(directory_client_cfg);
         let presence = MixNodePresence {
-            host: node_config.announce_address.clone(),
-            pub_key: node_config.public_key_string(),
-            layer: node_config.layer as u64,
+            host: config.announce_host,
+            pub_key: config.pub_key_string,
+            layer: config.layer,
             last_seen: 0,
-            version: env!("CARGO_PKG_VERSION").to_string(),
+            version: built_info::PKG_VERSION.to_string(),
         };
         Notifier {
             net_client,
             presence,
+            sending_delay: config.sending_delay,
         }
     }
 
-    pub fn notify(&self) {
+    fn notify(&self) {
         match self.net_client.presence_mix_nodes_post.post(&self.presence) {
             Err(err) => error!("failed to send presence - {:?}", err),
             Ok(_) => debug!("sent presence information"),
         }
     }
 
-    pub async fn run(self) {
-        let delay_duration = Duration::from_secs(5);
-
-        loop {
-            self.notify();
-            tokio::time::delay_for(delay_duration).await;
-        }
+    pub fn start(self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            loop {
+                self.notify();
+                tokio::time::delay_for(self.sending_delay).await;
+            }
+        })
     }
 }

mixnode/src/node/runner.rs

@@ -1,87 +0,0 @@
-use crate::banner;
-use crate::node;
-use crate::node::MixNode;
-use clap::ArgMatches;
-use std::net::ToSocketAddrs;
-
-fn print_binding_warning(address: &str) {
-    println!("\n##### WARNING #####");
-    println!(
-        "\nYou are trying to bind to {} - you might not be accessible to other nodes",
-        address
-    );
-    println!("\n##### WARNING #####\n");
-}
-
-pub fn start(matches: &ArgMatches) {
-    println!("{}", banner());
-    println!("Starting mixnode...");
-
-    let config = new_config(matches);
-    println!("Public key: {}", config.public_key_string());
-    println!("Directory server: {}", config.directory_server);
-    println!(
-        "Listening for incoming packets on {}",
-        config.socket_address
-    );
-    println!(
-        "Announcing the following socket address: {}",
-        config.announce_address
-    );
-
-    let mix = MixNode::new(&config);
-    mix.start(config).unwrap();
-}
-
-fn new_config(matches: &ArgMatches) -> node::Config {
-    let host = matches.value_of("host").unwrap();
-    if host == "localhost" || host == "127.0.0.1" || host == "0.0.0.0" {
-        print_binding_warning(host);
-    }
-
-    let port = match matches.value_of("port").unwrap_or("1789").parse::<u16>() {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid port value provided - {:?}", err),
-    };
-
-    let layer = match matches.value_of("layer").unwrap().parse::<usize>() {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid layer value provided - {:?}", err),
-    };
-
-    let socket_address = (host, port)
-        .to_socket_addrs()
-        .expect("Failed to combine host and port")
-        .next()
-        .expect("Failed to extract the socket address from the iterator");
-
-    let announce_host = matches.value_of("announce_host").unwrap_or(host);
-    let announce_port = matches
-        .value_of("announce_port")
-        .map(|port| port.parse::<u16>().unwrap())
-        .unwrap_or(port);
-
-    let _ = (announce_host, announce_port)
-        .to_socket_addrs()
-        .expect("Failed to combine announce host and port")
-        .next()
-        .expect("Failed to extract the announce socket address from the iterator");
-
-    let announce_address = format!("{}:{}", announce_host, announce_port);
-
-    let (secret_key, public_key) = sphinx::crypto::keygen();
-
-    let directory_server = matches
-        .value_of("directory")
-        .unwrap_or("https://directory.nymtech.net")
-        .to_string();
-
-    node::Config {
-        directory_server,
-        layer,
-        public_key,
-        socket_address,
-        announce_address,
-        secret_key,
-    }
-}

nym-client/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 build = "build.rs"
 name = "nym-client"
-version = "0.4.1"
+version = "0.5.0-rc.1"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 edition = "2018"
 
@@ -25,25 +25,31 @@ reqwest = "0.9.22"
 serde = { version = "1.0.104", features = ["derive"] }
 serde_json = "1.0.44"
 tokio = { version = "0.2", features = ["full"] }
-tungstenite = "0.9.2"
+tokio-tungstenite = "0.10.1"
 
 ## internal
 addressing = {path = "../common/addressing" }
+config = {path = "../common/config"}
 crypto = {path = "../common/crypto"}
 directory-client = { path = "../common/clients/directory-client" }
 healthcheck = { path = "../common/healthcheck" }
 mix-client = { path = "../common/clients/mix-client" }
+multi-tcp-client = { path = "../common/clients/multi-tcp-client" }
 pemstore = {path = "../common/pemstore"}
 provider-client = { path = "../common/clients/provider-client" }
 sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" }
 topology = {path = "../common/topology" }
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 # sphinx = { path = "../../sphinx"}
 
-# putting this explicitly below everything and most likely, the next time we look into it, it will already have a proper release
-tokio-tungstenite = { git = "https://github.com/snapview/tokio-tungstenite", rev="308d9680c0e59dd1e8651659a775c05df937934e" }
-
 [build-dependencies]
 built = "0.3.2"
+
+[dev-dependencies]
+tempfile = "3.1.0"
+
+[features]
+qa = []
+local = []

nym-client/src/client/cover_traffic_stream.rs

@@ -1,37 +1,86 @@
-use crate::client::mix_traffic::MixMessage;
-use crate::client::topology_control::TopologyInnerRef;
-use crate::client::LOOP_COVER_AVERAGE_DELAY;
-use futures::channel::mpsc;
-use log::{error, info, trace, warn};
+use crate::client::mix_traffic::{MixMessage, MixMessageSender};
+use crate::client::topology_control::TopologyAccessor;
+use futures::task::{Context, Poll};
+use futures::{Future, Stream, StreamExt};
+use log::*;
 use sphinx::route::Destination;
+use std::pin::Pin;
 use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+use tokio::time;
 use topology::NymTopology;
 
-pub(crate) async fn start_loop_cover_traffic_stream<T: NymTopology>(
-    tx: mpsc::UnboundedSender<MixMessage>,
+pub(crate) struct LoopCoverTrafficStream<T: NymTopology> {
+    average_packet_delay: Duration,
+    average_cover_message_sending_delay: Duration,
+    next_delay: time::Delay,
+    mix_tx: MixMessageSender,
     our_info: Destination,
-    topology_ctrl_ref: TopologyInnerRef<T>,
-) {
-    info!("Starting loop cover traffic stream");
-    loop {
-        trace!("next cover message!");
-        let delay = mix_client::poisson::sample(LOOP_COVER_AVERAGE_DELAY);
-        let delay_duration = Duration::from_secs_f64(delay);
-        tokio::time::delay_for(delay_duration).await;
+    topology_access: TopologyAccessor<T>,
+}
 
-        let read_lock = topology_ctrl_ref.read().await;
-        let topology = match read_lock.topology.as_ref() {
-            None => {
-                warn!("No valid topology detected - won't send any loop cover message this time");
-                continue;
-            }
-            Some(topology) => topology,
+impl<T: NymTopology> Stream for LoopCoverTrafficStream<T> {
+    // Item is only used to indicate we should create a new message rather than actual cover message
+    // reason being to not introduce unnecessary complexity by having to keep state of topology
+    // mutex when trying to acquire it. So right now the Stream trait serves as a glorified timer.
+    // Perhaps this should be changed in the future.
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        // it is not yet time to return a message
+        if Pin::new(&mut self.next_delay).poll(cx).is_pending() {
+            return Poll::Pending;
         };
 
-        let cover_message = match mix_client::packet::loop_cover_message(
-            our_info.address,
-            our_info.identifier,
-            topology,
+        // we know it's time to send a message, so let's prepare delay for the next one
+        // Get the `now` by looking at the current `delay` deadline
+        let now = self.next_delay.deadline();
+        let next_poisson_delay =
+            mix_client::poisson::sample(self.average_cover_message_sending_delay);
+
+        // The next interval value is `next_poisson_delay` after the one that just
+        // yielded.
+        let next = now + next_poisson_delay;
+        self.next_delay.reset(next);
+
+        Poll::Ready(Some(()))
+    }
+}
+
+impl<T: 'static + NymTopology> LoopCoverTrafficStream<T> {
+    pub(crate) fn new(
+        mix_tx: MixMessageSender,
+        our_info: Destination,
+        topology_access: TopologyAccessor<T>,
+        average_cover_message_sending_delay: time::Duration,
+        average_packet_delay: time::Duration,
+    ) -> Self {
+        LoopCoverTrafficStream {
+            average_packet_delay,
+            average_cover_message_sending_delay,
+            next_delay: time::delay_for(Default::default()),
+            mix_tx,
+            our_info,
+            topology_access,
+        }
+    }
+
+    async fn on_new_message(&mut self) {
+        trace!("next cover message!");
+        let route = match self.topology_access.random_route().await {
+            None => {
+                warn!("No valid topology detected - won't send any loop cover message this time");
+                return;
+            }
+            Some(route) => route,
+        };
+
+        let cover_message = match mix_client::packet::loop_cover_message_route(
+            self.our_info.address.clone(),
+            self.our_info.identifier,
+            route,
+            self.average_packet_delay,
         ) {
             Ok(message) => message,
             Err(err) => {
@@ -39,7 +88,7 @@ pub(crate) async fn start_loop_cover_traffic_stream<T: NymTopology>(
                     "Somehow we managed to create an invalid cover message - {:?}",
                     err
                 );
-                continue;
+                return;
             }
         };
 
@@ -47,7 +96,25 @@ pub(crate) async fn start_loop_cover_traffic_stream<T: NymTopology>(
         // - we run out of memory
         // - the receiver channel is closed
         // in either case there's no recovery and we can only panic
-        tx.unbounded_send(MixMessage::new(cover_message.0, cover_message.1))
+        self.mix_tx
+            .unbounded_send(MixMessage::new(cover_message.0, cover_message.1))
             .unwrap();
     }
+
+    async fn run(&mut self) {
+        // we should set initial delay only when we actually start the stream
+        self.next_delay = time::delay_for(mix_client::poisson::sample(
+            self.average_cover_message_sending_delay,
+        ));
+
+        while let Some(_) = self.next().await {
+            self.on_new_message().await;
+        }
+    }
+
+    pub(crate) fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            self.run().await;
+        })
+    }
 }

nym-client/src/client/mix_traffic.rs

@@ -1,10 +1,15 @@
 use futures::channel::mpsc;
 use futures::StreamExt;
-use log::{debug, error, info, trace};
+use log::*;
 use sphinx::SphinxPacket;
 use std::net::SocketAddr;
+use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
 
 pub(crate) struct MixMessage(SocketAddr, SphinxPacket);
+pub(crate) type MixMessageSender = mpsc::UnboundedSender<MixMessage>;
+pub(crate) type MixMessageReceiver = mpsc::UnboundedReceiver<MixMessage>;
 
 impl MixMessage {
     pub(crate) fn new(address: SocketAddr, packet: SphinxPacket) -> Self {
@@ -12,26 +17,57 @@ impl MixMessage {
     }
 }
 
-pub(crate) struct MixTrafficController;
+// TODO: put our TCP client here
+pub(crate) struct MixTrafficController<'a> {
+    tcp_client: multi_tcp_client::Client<'a>,
+    mix_rx: MixMessageReceiver,
+}
 
-impl MixTrafficController {
-    pub(crate) async fn run(mut rx: mpsc::UnboundedReceiver<MixMessage>) {
-        info!("Mix Traffic Controller started!");
-        let mix_client = mix_client::MixClient::new();
-        while let Some(mix_message) = rx.next().await {
-            debug!("Got a mix_message for {:?}", mix_message.0);
-            let send_res = mix_client.send(mix_message.1, mix_message.0).await;
-            match send_res {
-                Ok(_) => {
-                    trace!("sent a mix message");
-                }
-                // TODO: should there be some kind of threshold of failed messages
-                // that if reached, the application blows?
-                Err(e) => error!(
-                    "We failed to send the message to {} :( - {:?}",
-                    mix_message.0, e
-                ),
-            };
+impl MixTrafficController<'static> {
+    pub(crate) async fn new(
+        initial_endpoints: Vec<SocketAddr>,
+        initial_reconnection_backoff: Duration,
+        maximum_reconnection_backoff: Duration,
+        mix_rx: MixMessageReceiver,
+    ) -> Self {
+        let tcp_client_config = multi_tcp_client::Config::new(
+            initial_endpoints,
+            initial_reconnection_backoff,
+            maximum_reconnection_backoff,
+        );
+
+        MixTrafficController {
+            tcp_client: multi_tcp_client::Client::new(tcp_client_config).await,
+            mix_rx,
         }
     }
+
+    async fn on_message(&mut self, mix_message: MixMessage) {
+        debug!("Got a mix_message for {:?}", mix_message.0);
+        match self
+            .tcp_client
+            .send(mix_message.0, &mix_message.1.to_bytes())
+            .await
+        {
+            Ok(_) => trace!("sent a mix message"),
+            // TODO: should there be some kind of threshold of failed messages
+            // that if reached, the application blows?
+            Err(e) => error!(
+                "We failed to send the packet to {} - {:?}",
+                mix_message.0, e
+            ),
+        };
+    }
+
+    pub(crate) async fn run(&mut self) {
+        while let Some(mix_message) = self.mix_rx.next().await {
+            self.on_message(mix_message).await;
+        }
+    }
+
+    pub(crate) fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            self.run().await;
+        })
+    }
 }

nym-client/src/client/mod.rs

@@ -1,16 +1,23 @@
-use crate::client::mix_traffic::MixTrafficController;
-use crate::client::received_buffer::ReceivedMessagesBuffer;
-use crate::client::topology_control::TopologyInnerRef;
+use crate::client::cover_traffic_stream::LoopCoverTrafficStream;
+use crate::client::mix_traffic::{MixMessageReceiver, MixMessageSender, MixTrafficController};
+use crate::client::provider_poller::{PolledMessagesReceiver, PolledMessagesSender};
+use crate::client::received_buffer::{
+    ReceivedBufferRequestReceiver, ReceivedBufferRequestSender, ReceivedMessagesBufferController,
+};
+use crate::client::topology_control::{
+    TopologyAccessor, TopologyRefresher, TopologyRefresherConfig,
+};
+use crate::config::persistence::pathfinder::ClientPathfinder;
+use crate::config::{Config, SocketType};
 use crate::sockets::tcp;
 use crate::sockets::ws;
-use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey};
-use directory_client::presence::Topology;
+use crypto::identity::MixIdentityKeyPair;
+use directory_client::presence;
 use futures::channel::mpsc;
-use futures::join;
 use log::*;
+use pemstore::pemstore::PemStore;
 use sfw_provider_requests::AuthToken;
 use sphinx::route::Destination;
-use std::marker::PhantomData;
 use std::net::SocketAddr;
 use tokio::runtime::Runtime;
 use topology::NymTopology;
@@ -19,92 +26,260 @@ mod cover_traffic_stream;
 mod mix_traffic;
 mod provider_poller;
 mod real_traffic_stream;
-pub mod received_buffer;
-pub mod topology_control;
+pub(crate) mod received_buffer;
+pub(crate) mod topology_control;
 
-// TODO: all of those constants should probably be moved to config file
-const LOOP_COVER_AVERAGE_DELAY: f64 = 0.5;
-// seconds
-const MESSAGE_SENDING_AVERAGE_DELAY: f64 = 0.5;
-//  seconds;
-const FETCH_MESSAGES_DELAY: f64 = 1.0; // seconds;
+pub(crate) type InputMessageSender = mpsc::UnboundedSender<InputMessage>;
+pub(crate) type InputMessageReceiver = mpsc::UnboundedReceiver<InputMessage>;
 
-const TOPOLOGY_REFRESH_RATE: f64 = 10.0; // seconds
-
-pub enum SocketType {
-    TCP,
-    WebSocket,
-    None,
-}
-
-pub struct NymClient<IDPair, Priv, Pub>
-where
-    IDPair: MixnetIdentityKeyPair<Priv, Pub> + Send + Sync,
-    Priv: MixnetIdentityPrivateKey + Send + Sync,
-    Pub: MixnetIdentityPublicKey + Send + Sync,
-{
-    keypair: IDPair,
+pub struct NymClient {
+    runtime: Runtime,
+    config: Config,
+    identity_keypair: MixIdentityKeyPair,
 
     // to be used by "send" function or socket, etc
-    pub input_tx: mpsc::UnboundedSender<InputMessage>,
-
-    input_rx: mpsc::UnboundedReceiver<InputMessage>,
-    socket_listening_address: SocketAddr,
-    directory: String,
-    auth_token: Option<AuthToken>,
-    socket_type: SocketType,
-
-    _phantom_private: PhantomData<Priv>,
-    _phantom_public: PhantomData<Pub>,
+    input_tx: Option<InputMessageSender>,
 }
 
 #[derive(Debug)]
-pub struct InputMessage(pub Destination, pub Vec<u8>);
+// TODO: make fields private
+pub(crate) struct InputMessage(pub Destination, pub Vec<u8>);
 
-impl<IDPair: 'static, Priv: 'static, Pub: 'static> NymClient<IDPair, Priv, Pub>
-where
-    IDPair: MixnetIdentityKeyPair<Priv, Pub> + Send + Sync,
-    Priv: MixnetIdentityPrivateKey + Send + Sync,
-    Pub: MixnetIdentityPublicKey + Send + Sync,
-{
-    pub fn new(
-        keypair: IDPair,
-        socket_listening_address: SocketAddr,
-        directory: String,
-        auth_token: Option<AuthToken>,
-        socket_type: SocketType,
-    ) -> Self {
-        let (input_tx, input_rx) = mpsc::unbounded::<InputMessage>();
+impl NymClient {
+    fn load_identity_keys(config_file: &Config) -> MixIdentityKeyPair {
+        let identity_keypair = PemStore::new(ClientPathfinder::new_from_config(&config_file))
+            .read_identity()
+            .expect("Failed to read stored identity key files");
+        println!(
+            "Public identity key: {}\nFor time being, it is identical to address",
+            identity_keypair.public_key.to_base58_string()
+        );
+        identity_keypair
+    }
+
+    pub fn new(config: Config) -> Self {
+        let identity_keypair = Self::load_identity_keys(&config);
 
         NymClient {
-            keypair,
-            input_tx,
-            input_rx,
-            socket_listening_address,
-            directory,
-            auth_token,
-            socket_type,
-            _phantom_private: PhantomData,
-            _phantom_public: PhantomData,
+            runtime: Runtime::new().unwrap(),
+            config,
+            identity_keypair,
+            input_tx: None,
         }
     }
 
+    pub fn as_mix_destination(&self) -> Destination {
+        Destination::new(
+            self.identity_keypair.public_key().derive_address(),
+            // TODO: what with SURBs?
+            Default::default(),
+        )
+    }
+
     async fn get_provider_socket_address<T: NymTopology>(
-        &self,
-        topology_ctrl_ref: TopologyInnerRef<T>,
+        provider_id: String,
+        mut topology_accessor: TopologyAccessor<T>,
     ) -> SocketAddr {
-        // this is temporary and assumes there exists only a single provider.
-        topology_ctrl_ref.read().await.topology.as_ref().unwrap()
+        topology_accessor.get_current_topology_clone().await.as_ref().expect("The current network topoloy is empty - are you using correct directory server?")
             .providers()
-            .first()
-            .expect("Could not get a provider from the initial network topology, are you using the right directory server?")
+            .iter()
+            .find(|provider| provider.pub_key == provider_id)
+            .unwrap_or_else( || panic!("Could not find provider with id {:?} - are you sure it is still online? Perhaps try to run `nym-client init` again to obtain a new provider", provider_id))
             .client_listener
     }
 
-    pub fn start(self) -> Result<(), Box<dyn std::error::Error>> {
-        info!("Starting nym client");
-        let mut rt = Runtime::new()?;
+    // future constantly pumping loop cover traffic at some specified average rate
+    // the pumped traffic goes to the MixTrafficController
+    fn start_cover_traffic_stream<T: 'static + NymTopology>(
+        &self,
+        topology_accessor: TopologyAccessor<T>,
+        mix_tx: MixMessageSender,
+    ) {
+        info!("Starting loop cover traffic stream...");
+        // we need to explicitly enter runtime due to "next_delay: time::delay_for(Default::default())"
+        // set in the constructor which HAS TO be called within context of a tokio runtime
+        self.runtime
+            .enter(|| {
+                LoopCoverTrafficStream::new(
+                    mix_tx,
+                    self.as_mix_destination(),
+                    topology_accessor,
+                    self.config.get_loop_cover_traffic_average_delay(),
+                    self.config.get_average_packet_delay(),
+                )
+            })
+            .start(self.runtime.handle());
+    }
 
+    fn start_real_traffic_stream<T: 'static + NymTopology>(
+        &self,
+        topology_accessor: TopologyAccessor<T>,
+        mix_tx: MixMessageSender,
+        input_rx: InputMessageReceiver,
+    ) {
+        info!("Starting real traffic stream...");
+        // we need to explicitly enter runtime due to "next_delay: time::delay_for(Default::default())"
+        // set in the constructor which HAS TO be called within context of a tokio runtime
+        self.runtime
+            .enter(|| {
+                real_traffic_stream::OutQueueControl::new(
+                    mix_tx,
+                    input_rx,
+                    self.as_mix_destination(),
+                    topology_accessor,
+                    self.config.get_average_packet_delay(),
+                    self.config.get_message_sending_average_delay(),
+                )
+            })
+            .start(self.runtime.handle());
+    }
+
+    // buffer controlling all messages fetched from provider
+    // required so that other components would be able to use them (say the websocket)
+    fn start_received_messages_buffer_controller(
+        &self,
+        query_receiver: ReceivedBufferRequestReceiver,
+        poller_receiver: PolledMessagesReceiver,
+    ) {
+        info!("Starting 'received messages buffer controller'...");
+        ReceivedMessagesBufferController::new(query_receiver, poller_receiver)
+            .start(self.runtime.handle())
+    }
+
+    // future constantly trying to fetch any received messages from the provider
+    // the received messages are sent to ReceivedMessagesBuffer to be available to rest of the system
+    fn start_provider_poller<T: NymTopology>(
+        &mut self,
+        topology_accessor: TopologyAccessor<T>,
+        poller_input_tx: PolledMessagesSender,
+    ) {
+        info!("Starting provider poller...");
+        // we already have our provider written in the config
+        let provider_id = self.config.get_provider_id();
+
+        let provider_client_listener_address = self.runtime.block_on(
+            Self::get_provider_socket_address(provider_id, topology_accessor),
+        );
+
+        let mut provider_poller = provider_poller::ProviderPoller::new(
+            poller_input_tx,
+            provider_client_listener_address,
+            self.identity_keypair.public_key().derive_address(),
+            self.config
+                .get_provider_auth_token()
+                .map(|str_token| AuthToken::try_from_base58_string(str_token).ok())
+                .unwrap_or(None),
+            self.config.get_fetch_message_delay(),
+        );
+
+        if !provider_poller.is_registered() {
+            info!("Trying to perform initial provider registration...");
+            self.runtime
+                .block_on(provider_poller.perform_initial_registration())
+                .expect("Failed to perform initial provider registration");
+        }
+        provider_poller.start(self.runtime.handle());
+    }
+
+    // future responsible for periodically polling directory server and updating
+    // the current global view of topology
+    fn start_topology_refresher<T: 'static + NymTopology>(
+        &mut self,
+        topology_accessor: TopologyAccessor<T>,
+    ) {
+        let healthcheck_keys = MixIdentityKeyPair::new();
+
+        let topology_refresher_config = TopologyRefresherConfig::new(
+            self.config.get_directory_server(),
+            self.config.get_topology_refresh_rate(),
+            healthcheck_keys,
+            self.config.get_topology_resolution_timeout(),
+            self.config.get_number_of_healthcheck_test_packets() as usize,
+            self.config.get_node_score_threshold(),
+        );
+        let mut topology_refresher =
+            TopologyRefresher::new(topology_refresher_config, topology_accessor);
+        // before returning, block entire runtime to refresh the current network view so that any
+        // components depending on topology would see a non-empty view
+        info!("Obtaining initial network topology...");
+        self.runtime.block_on(topology_refresher.refresh());
+        info!("Starting topology refresher...");
+        topology_refresher.start(self.runtime.handle());
+    }
+
+    // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
+    fn start_mix_traffic_controller(&mut self, mix_rx: MixMessageReceiver) {
+        info!("Starting mix trafic controller...");
+        // TODO: possible optimisation: set the initial endpoints to all known mixes from layer 1
+        let initial_mix_endpoints = Vec::new();
+        self.runtime
+            .block_on(MixTrafficController::new(
+                initial_mix_endpoints,
+                self.config.get_packet_forwarding_initial_backoff(),
+                self.config.get_packet_forwarding_maximum_backoff(),
+                mix_rx,
+            ))
+            .start(self.runtime.handle());
+    }
+
+    fn start_socket_listener<T: 'static + NymTopology>(
+        &self,
+        topology_accessor: TopologyAccessor<T>,
+        received_messages_buffer_output_tx: ReceivedBufferRequestSender,
+        input_tx: InputMessageSender,
+    ) {
+        match self.config.get_socket_type() {
+            SocketType::WebSocket => {
+                ws::start_websocket(
+                    self.runtime.handle(),
+                    self.config.get_listening_port(),
+                    input_tx,
+                    received_messages_buffer_output_tx,
+                    self.identity_keypair.public_key().derive_address(),
+                    topology_accessor,
+                );
+            }
+            SocketType::TCP => {
+                tcp::start_tcpsocket(
+                    self.runtime.handle(),
+                    self.config.get_listening_port(),
+                    input_tx,
+                    received_messages_buffer_output_tx,
+                    self.identity_keypair.public_key().derive_address(),
+                    topology_accessor,
+                );
+            }
+            SocketType::None => (),
+        }
+    }
+
+    /// EXPERIMENTAL DIRECT RUST API
+    /// It's entirely untested and there are absolutely no guarantees about it
+    pub fn send_message(&self, destination: Destination, message: Vec<u8>) {
+        self.input_tx
+            .as_ref()
+            .expect("start method was not called before!")
+            .unbounded_send(InputMessage(destination, message))
+            .unwrap()
+    }
+
+    /// blocking version of `start` method. Will run forever (or until SIGINT is sent)
+    pub fn run_forever(&mut self) {
+        self.start();
+        if let Err(e) = self.runtime.block_on(tokio::signal::ctrl_c()) {
+            error!(
+                "There was an error while capturing SIGINT - {:?}. We will terminate regardless",
+                e
+            );
+        }
+
+        println!(
+            "Received SIGINT - the mixnode will terminate now (threads are not YET nicely stopped)"
+        );
+    }
+
+    pub fn start(&mut self) {
+        info!("Starting nym client");
         // channels for inter-component communication
 
         // mix_tx is the transmitter for any component generating sphinx packets that are to be sent to the mixnet
@@ -122,128 +297,27 @@ where
         let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) =
             mpsc::unbounded();
 
-        let self_address = self.keypair.public_key().derive_address();
+        // channels responsible for controlling real messages
+        let (input_tx, input_rx) = mpsc::unbounded::<InputMessage>();
 
-        // generate same type of keys we have as our identity
-        let healthcheck_keys = IDPair::new();
-
-        // TODO: when we switch to our graph topology, we need to remember to change 'Topology' type
-        let topology_controller =
-            rt.block_on(topology_control::TopologyControl::<Topology, _, _, _>::new(
-                self.directory.clone(),
-                TOPOLOGY_REFRESH_RATE,
-                healthcheck_keys,
-            ));
-
-        let provider_client_listener_address =
-            rt.block_on(self.get_provider_socket_address(topology_controller.get_inner_ref()));
-
-        let mut provider_poller = provider_poller::ProviderPoller::new(
-            poller_input_tx,
-            provider_client_listener_address,
-            self_address,
-            self.auth_token,
+        // TODO: when we switch to our graph topology, we need to remember to change 'presence::Topology' type
+        let shared_topology_accessor = TopologyAccessor::<presence::Topology>::new();
+        // the components are started in very specific order. Unless you know what you are doing,
+        // do not change that.
+        self.start_topology_refresher(shared_topology_accessor.clone());
+        self.start_received_messages_buffer_controller(
+            received_messages_buffer_output_rx,
+            poller_input_rx,
         );
-
-        // registration
-        if let Err(err) = rt.block_on(provider_poller.perform_initial_registration()) {
-            panic!("Failed to perform initial registration: {:?}", err);
-        };
-
-        // setup all of futures for the components running on the client
-
-        // buffer controlling all messages fetched from provider
-        // required so that other components would be able to use them (say the websocket)
-        let received_messages_buffer_controllers_future = rt.spawn(
-            ReceivedMessagesBuffer::new()
-                .start_controllers(poller_input_rx, received_messages_buffer_output_rx),
+        self.start_provider_poller(shared_topology_accessor.clone(), poller_input_tx);
+        self.start_mix_traffic_controller(mix_rx);
+        self.start_cover_traffic_stream(shared_topology_accessor.clone(), mix_tx.clone());
+        self.start_real_traffic_stream(shared_topology_accessor.clone(), mix_tx, input_rx);
+        self.start_socket_listener(
+            shared_topology_accessor,
+            received_messages_buffer_output_tx,
+            input_tx.clone(),
         );
-
-        // controller for sending sphinx packets to mixnet (either real traffic or cover traffic)
-        let mix_traffic_future = rt.spawn(MixTrafficController::run(mix_rx));
-
-        // future constantly pumping loop cover traffic at some specified average rate
-        // the pumped traffic goes to the MixTrafficController
-        let loop_cover_traffic_future =
-            rt.spawn(cover_traffic_stream::start_loop_cover_traffic_stream(
-                mix_tx.clone(),
-                Destination::new(self_address, Default::default()),
-                topology_controller.get_inner_ref(),
-            ));
-
-        // cloning arguments required by OutQueueControl; required due to move
-        let input_rx = self.input_rx;
-        let topology_ref = topology_controller.get_inner_ref();
-
-        // future constantly pumping traffic at some specified average rate
-        // if a real message is available on 'input_rx' that might have been received from say
-        // the websocket, the real message is used, otherwise a loop cover message is generated
-        // the pumped traffic goes to the MixTrafficController
-        let out_queue_control_future = rt.spawn(async move {
-            real_traffic_stream::OutQueueControl::new(
-                mix_tx,
-                input_rx,
-                Destination::new(self_address, Default::default()),
-                topology_ref,
-            )
-            .run_out_queue_control()
-            .await
-        });
-
-        // future constantly trying to fetch any received messages from the provider
-        // the received messages are sent to ReceivedMessagesBuffer to be available to rest of the system
-        let provider_polling_future = rt.spawn(provider_poller.start_provider_polling());
-
-        // a temporary workaround for starting socket listener of specified type
-        // in the future the actual socket handler should start THIS client instead
-        match self.socket_type {
-            SocketType::WebSocket => {
-                rt.spawn(ws::start_websocket(
-                    self.socket_listening_address,
-                    self.input_tx,
-                    received_messages_buffer_output_tx,
-                    self_address,
-                    topology_controller.get_inner_ref(),
-                ));
-            }
-            SocketType::TCP => {
-                rt.spawn(tcp::start_tcpsocket(
-                    self.socket_listening_address,
-                    self.input_tx,
-                    received_messages_buffer_output_tx,
-                    self_address,
-                    topology_controller.get_inner_ref(),
-                ));
-            }
-            SocketType::None => (),
-        }
-
-        // future responsible for periodically polling directory server and updating
-        // the current global view of topology
-        let topology_refresher_future = rt.spawn(topology_controller.run_refresher());
-
-        rt.block_on(async {
-            let future_results = join!(
-                received_messages_buffer_controllers_future,
-                mix_traffic_future,
-                loop_cover_traffic_future,
-                out_queue_control_future,
-                provider_polling_future,
-                topology_refresher_future,
-            );
-
-            assert!(
-                future_results.0.is_ok()
-                    && future_results.1.is_ok()
-                    && future_results.2.is_ok()
-                    && future_results.3.is_ok()
-                    && future_results.4.is_ok()
-                    && future_results.5.is_ok()
-            );
-        });
-
-        // this line in theory should never be reached as the runtime should be permanently blocked on traffic senders
-        error!("The client went kaput...");
-        Ok(())
+        self.input_tx = Some(input_tx);
     }
 }

nym-client/src/client/provider_poller.rs

@@ -1,13 +1,18 @@
-use crate::client::FETCH_MESSAGES_DELAY;
 use futures::channel::mpsc;
-use log::{debug, error, info, trace, warn};
+use log::*;
 use provider_client::ProviderClientError;
 use sfw_provider_requests::AuthToken;
 use sphinx::route::DestinationAddressBytes;
 use std::net::SocketAddr;
-use std::time::Duration;
+use std::time;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+pub(crate) type PolledMessagesSender = mpsc::UnboundedSender<Vec<Vec<u8>>>;
+pub(crate) type PolledMessagesReceiver = mpsc::UnboundedReceiver<Vec<Vec<u8>>>;
 
 pub(crate) struct ProviderPoller {
+    polling_rate: time::Duration,
     provider_client: provider_client::ProviderClient,
     poller_tx: mpsc::UnboundedSender<Vec<Vec<u8>>>,
 }
@@ -18,6 +23,7 @@ impl ProviderPoller {
         provider_client_listener_address: SocketAddr,
         client_address: DestinationAddressBytes,
         auth_token: Option<AuthToken>,
+        polling_rate: time::Duration,
     ) -> Self {
         ProviderPoller {
             provider_client: provider_client::ProviderClient::new(
@@ -26,14 +32,19 @@ impl ProviderPoller {
                 auth_token,
             ),
             poller_tx,
+            polling_rate,
         }
     }
 
+    pub(crate) fn is_registered(&self) -> bool {
+        self.provider_client.is_registered()
+    }
+
     // This method is only temporary until registration is moved to `client init`
     pub(crate) async fn perform_initial_registration(&mut self) -> Result<(), ProviderClientError> {
         debug!("performing initial provider registration");
 
-        if !self.provider_client.is_registered() {
+        if !self.is_registered() {
             let auth_token = match self.provider_client.register().await {
                 // in this particular case we can ignore this error
                 Err(ProviderClientError::ClientAlreadyRegisteredError) => return Ok(()),
@@ -43,20 +54,17 @@ impl ProviderPoller {
 
             self.provider_client.update_token(auth_token)
         } else {
-            warn!("did not perform registration - we were already registered")
+            warn!("did not perform provider registration - we were already registered")
         }
 
         Ok(())
     }
 
     pub(crate) async fn start_provider_polling(self) {
-        info!("Starting provider poller");
-
         let loop_message = &mix_client::packet::LOOP_COVER_MESSAGE_PAYLOAD.to_vec();
         let dummy_message = &sfw_provider_requests::DUMMY_MESSAGE_CONTENT.to_vec();
 
-        let delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY);
-        let extended_delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY * 10.0);
+        let extended_delay_duration = self.polling_rate * 10;
 
         loop {
             debug!("Polling provider...");
@@ -82,7 +90,11 @@ impl ProviderPoller {
             // in either case there's no recovery and we can only panic
             self.poller_tx.unbounded_send(good_messages).unwrap();
 
-            tokio::time::delay_for(delay_duration).await;
+            tokio::time::delay_for(self.polling_rate).await;
         }
     }
+
+    pub(crate) fn start(self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move { self.start_provider_polling().await })
+    }
 }

nym-client/src/client/real_traffic_stream.rs

@@ -1,6 +1,6 @@
 use crate::client::mix_traffic::MixMessage;
-use crate::client::topology_control::TopologyInnerRef;
-use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY};
+use crate::client::topology_control::TopologyAccessor;
+use crate::client::InputMessage;
 use futures::channel::mpsc;
 use futures::task::{Context, Poll};
 use futures::{Future, Stream, StreamExt};
@@ -8,18 +8,19 @@ use log::{error, info, trace, warn};
 use sphinx::route::Destination;
 use std::pin::Pin;
 use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
 use tokio::time;
 use topology::NymTopology;
 
-// have a rather low value for test sake
-const AVERAGE_PACKET_DELAY: f64 = 0.1;
-
 pub(crate) struct OutQueueControl<T: NymTopology> {
-    delay: time::Delay,
+    average_packet_delay: Duration,
+    average_message_sending_delay: Duration,
+    next_delay: time::Delay,
     mix_tx: mpsc::UnboundedSender<MixMessage>,
     input_rx: mpsc::UnboundedReceiver<InputMessage>,
     our_info: Destination,
-    topology_ctrl_ref: TopologyInnerRef<T>,
+    topology_access: TopologyAccessor<T>,
 }
 
 pub(crate) enum StreamMessage {
@@ -32,21 +33,19 @@ impl<T: NymTopology> Stream for OutQueueControl<T> {
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
         // it is not yet time to return a message
-        if Pin::new(&mut self.delay).poll(cx).is_pending() {
+        if Pin::new(&mut self.next_delay).poll(cx).is_pending() {
             return Poll::Pending;
         };
 
         // we know it's time to send a message, so let's prepare delay for the next one
         // Get the `now` by looking at the current `delay` deadline
-        let now = self.delay.deadline();
-
-        let next_poisson_delay =
-            Duration::from_secs_f64(mix_client::poisson::sample(MESSAGE_SENDING_AVERAGE_DELAY));
+        let now = self.next_delay.deadline();
+        let next_poisson_delay = mix_client::poisson::sample(self.average_message_sending_delay);
 
         // The next interval value is `next_poisson_delay` after the one that just
         // yielded.
         let next = now + next_poisson_delay;
-        self.delay.reset(next);
+        self.next_delay.reset(next);
 
         // decide what kind of message to send
         match Pin::new(&mut self.input_rx).poll_next(cx) {
@@ -63,70 +62,86 @@ impl<T: NymTopology> Stream for OutQueueControl<T> {
     }
 }
 
-impl<T: NymTopology> OutQueueControl<T> {
+impl<T: 'static + NymTopology> OutQueueControl<T> {
     pub(crate) fn new(
         mix_tx: mpsc::UnboundedSender<MixMessage>,
         input_rx: mpsc::UnboundedReceiver<InputMessage>,
         our_info: Destination,
-        topology: TopologyInnerRef<T>,
+        topology_access: TopologyAccessor<T>,
+        average_packet_delay: Duration,
+        average_message_sending_delay: Duration,
     ) -> Self {
-        let initial_delay = time::delay_for(Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY));
         OutQueueControl {
-            delay: initial_delay,
+            average_packet_delay,
+            average_message_sending_delay,
+            next_delay: time::delay_for(Default::default()),
             mix_tx,
             input_rx,
             our_info,
-            topology_ctrl_ref: topology,
+            topology_access,
         }
     }
 
+    async fn on_message(&mut self, next_message: StreamMessage) {
+        trace!("created new message");
+        let route = match self.topology_access.random_route().await {
+            None => {
+                warn!("No valid topology detected - won't send any real or loop message this time");
+                // TODO: this creates a potential problem: we can lose real messages if we were
+                // unable to get topology, perhaps we should store them in some buffer?
+                return;
+            }
+            Some(route) => route,
+        };
+
+        let next_packet = match next_message {
+            StreamMessage::Cover => mix_client::packet::loop_cover_message_route(
+                self.our_info.address.clone(),
+                self.our_info.identifier,
+                route,
+                self.average_packet_delay,
+            ),
+            StreamMessage::Real(real_message) => mix_client::packet::encapsulate_message_route(
+                real_message.0,
+                real_message.1,
+                route,
+                self.average_packet_delay,
+            ),
+        };
+
+        let next_packet = match next_packet {
+            Ok(message) => message,
+            Err(err) => {
+                error!(
+                    "Somehow we managed to create an invalid traffic message - {:?}",
+                    err
+                );
+                return;
+            }
+        };
+
+        // if this one fails, there's no retrying because it means that either:
+        // - we run out of memory
+        // - the receiver channel is closed
+        // in either case there's no recovery and we can only panic
+        self.mix_tx
+            .unbounded_send(MixMessage::new(next_packet.0, next_packet.1))
+            .unwrap();
+    }
+
     pub(crate) async fn run_out_queue_control(mut self) {
+        // we should set initial delay only when we actually start the stream
+        self.next_delay = time::delay_for(mix_client::poisson::sample(
+            self.average_message_sending_delay,
+        ));
+
         info!("starting out queue controller");
         while let Some(next_message) = self.next().await {
-            trace!("created new message");
-            let read_lock = self.topology_ctrl_ref.read().await;
-            let topology = match read_lock.topology.as_ref() {
-                None => {
-                    warn!(
-                        "No valid topology detected - won't send any loop cover message this time"
-                    );
-                    continue;
-                }
-                Some(topology) => topology,
-            };
-
-            let next_packet = match next_message {
-                StreamMessage::Cover => mix_client::packet::loop_cover_message(
-                    self.our_info.address,
-                    self.our_info.identifier,
-                    topology,
-                ),
-                StreamMessage::Real(real_message) => mix_client::packet::encapsulate_message(
-                    real_message.0,
-                    real_message.1,
-                    topology,
-                    AVERAGE_PACKET_DELAY,
-                ),
-            };
-
-            let next_packet = match next_packet {
-                Ok(message) => message,
-                Err(err) => {
-                    error!(
-                        "Somehow we managed to create an invalid traffic message - {:?}",
-                        err
-                    );
-                    continue;
-                }
-            };
-
-            // if this one fails, there's no retrying because it means that either:
-            // - we run out of memory
-            // - the receiver channel is closed
-            // in either case there's no recovery and we can only panic
-            self.mix_tx
-                .unbounded_send(MixMessage::new(next_packet.0, next_packet.1))
-                .unwrap();
+            self.on_message(next_message).await;
         }
     }
+
+    pub(crate) fn start(self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move { self.run_out_queue_control().await })
+    }
 }

nym-client/src/client/received_buffer.rs

@@ -1,89 +1,124 @@
+use crate::client::provider_poller::PolledMessagesReceiver;
 use futures::channel::{mpsc, oneshot};
-use futures::lock::Mutex as FMutex;
+use futures::lock::Mutex;
 use futures::StreamExt;
-use log::{error, info, trace};
+use log::*;
 use std::sync::Arc;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
 
-pub type BufferResponse = oneshot::Sender<Vec<Vec<u8>>>;
+pub(crate) type ReceivedBufferResponse = oneshot::Sender<Vec<Vec<u8>>>;
+pub(crate) type ReceivedBufferRequestSender = mpsc::UnboundedSender<ReceivedBufferResponse>;
+pub(crate) type ReceivedBufferRequestReceiver = mpsc::UnboundedReceiver<ReceivedBufferResponse>;
 
-pub(crate) struct ReceivedMessagesBuffer {
-    inner: Arc<FMutex<Inner>>,
-}
-
-impl ReceivedMessagesBuffer {
-    pub(crate) fn new() -> Self {
-        ReceivedMessagesBuffer {
-            inner: Arc::new(FMutex::new(Inner::new())),
-        }
-    }
-
-    pub(crate) async fn start_controllers(
-        self,
-        poller_rx: mpsc::UnboundedReceiver<Vec<Vec<u8>>>, // to receive new messages
-        query_receiver: mpsc::UnboundedReceiver<BufferResponse>, // to receive requests to acquire all stored messages
-    ) {
-        let input_controller_future = tokio::spawn(Self::run_poller_input_controller(
-            self.inner.clone(),
-            poller_rx,
-        ));
-        let output_controller_future = tokio::spawn(Self::run_query_output_controller(
-            self.inner,
-            query_receiver,
-        ));
-
-        futures::future::select(input_controller_future, output_controller_future).await;
-        panic!("One of the received buffer controllers failed!")
-    }
-
-    pub(crate) async fn run_poller_input_controller(
-        buf: Arc<FMutex<Inner>>,
-        mut poller_rx: mpsc::UnboundedReceiver<Vec<Vec<u8>>>,
-    ) {
-        info!("Started Received Messages Buffer Input Controller");
-
-        while let Some(new_messages) = poller_rx.next().await {
-            Inner::add_new_messages(&*buf, new_messages).await;
-        }
-    }
-
-    pub(crate) async fn run_query_output_controller(
-        buf: Arc<FMutex<Inner>>,
-        mut query_receiver: mpsc::UnboundedReceiver<BufferResponse>,
-    ) {
-        info!("Started Received Messages Buffer Output Controller");
-
-        while let Some(request) = query_receiver.next().await {
-            let messages = Inner::acquire_and_empty(&*buf).await;
-            if let Err(failed_messages) = request.send(messages) {
-                error!(
-                    "Failed to send the messages to the requester. Adding them back to the buffer"
-                );
-                Inner::add_new_messages(&*buf, failed_messages).await;
-            }
-        }
-    }
-}
-
-pub(crate) struct Inner {
+struct ReceivedMessagesBufferInner {
     messages: Vec<Vec<u8>>,
 }
 
-impl Inner {
+#[derive(Debug, Clone)]
+// Note: you should NEVER create more than a single instance of this using 'new()'.
+// You should always use .clone() to create additional instances
+struct ReceivedMessagesBuffer {
+    inner: Arc<Mutex<ReceivedMessagesBufferInner>>,
+}
+
+impl ReceivedMessagesBuffer {
     fn new() -> Self {
-        Inner {
-            messages: Vec::new(),
+        ReceivedMessagesBuffer {
+            inner: Arc::new(Mutex::new(ReceivedMessagesBufferInner {
+                messages: Vec::new(),
+            })),
         }
     }
 
-    async fn add_new_messages(buf: &FMutex<Self>, msgs: Vec<Vec<u8>>) {
+    async fn add_new_messages(&mut self, msgs: Vec<Vec<u8>>) {
         trace!("Adding new messages to the buffer! {:?}", msgs);
-        let mut unlocked = buf.lock().await;
-        unlocked.messages.extend(msgs);
+        self.inner.lock().await.messages.extend(msgs)
     }
 
-    async fn acquire_and_empty(buf: &FMutex<Self>) -> Vec<Vec<u8>> {
+    async fn acquire_and_empty(&mut self) -> Vec<Vec<u8>> {
         trace!("Emptying the buffer and returning all messages");
-        let mut unlocked = buf.lock().await;
-        std::mem::replace(&mut unlocked.messages, Vec::new())
+        let mut mutex_guard = self.inner.lock().await;
+        std::mem::replace(&mut mutex_guard.messages, Vec::new())
+    }
+}
+
+struct RequestReceiver {
+    received_buffer: ReceivedMessagesBuffer,
+    query_receiver: ReceivedBufferRequestReceiver,
+}
+
+impl RequestReceiver {
+    fn new(
+        received_buffer: ReceivedMessagesBuffer,
+        query_receiver: ReceivedBufferRequestReceiver,
+    ) -> Self {
+        RequestReceiver {
+            received_buffer,
+            query_receiver,
+        }
+    }
+
+    fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            while let Some(request) = self.query_receiver.next().await {
+                let messages = self.received_buffer.acquire_and_empty().await;
+                if let Err(failed_messages) = request.send(messages) {
+                    error!(
+                        "Failed to send the messages to the requester. Adding them back to the buffer"
+                    );
+                    self.received_buffer.add_new_messages(failed_messages).await;
+                }
+            }
+        })
+    }
+}
+
+struct MessageReceiver {
+    received_buffer: ReceivedMessagesBuffer,
+    poller_receiver: PolledMessagesReceiver,
+}
+
+impl MessageReceiver {
+    fn new(
+        received_buffer: ReceivedMessagesBuffer,
+        poller_receiver: PolledMessagesReceiver,
+    ) -> Self {
+        MessageReceiver {
+            received_buffer,
+            poller_receiver,
+        }
+    }
+    fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            while let Some(new_messages) = self.poller_receiver.next().await {
+                self.received_buffer.add_new_messages(new_messages).await;
+            }
+        })
+    }
+}
+
+pub(crate) struct ReceivedMessagesBufferController {
+    messsage_receiver: MessageReceiver,
+    request_receiver: RequestReceiver,
+}
+
+impl ReceivedMessagesBufferController {
+    pub(crate) fn new(
+        query_receiver: ReceivedBufferRequestReceiver,
+        poller_receiver: PolledMessagesReceiver,
+    ) -> Self {
+        let received_buffer = ReceivedMessagesBuffer::new();
+
+        ReceivedMessagesBufferController {
+            messsage_receiver: MessageReceiver::new(received_buffer.clone(), poller_receiver),
+            request_receiver: RequestReceiver::new(received_buffer, query_receiver),
+        }
+    }
+
+    pub(crate) fn start(self, handle: &Handle) {
+        // TODO: should we do anything with JoinHandle(s) returned by start methods?
+        self.messsage_receiver.start(handle);
+        self.request_receiver.start(handle);
     }
 }

nym-client/src/client/topology_control.rs

@@ -1,28 +1,67 @@
 use crate::built_info;
-use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey};
+use crypto::identity::MixIdentityKeyPair;
+use futures::lock::Mutex;
 use healthcheck::HealthChecker;
-use log::{error, info, trace, warn};
+use log::*;
 use std::sync::Arc;
+use std::time;
 use std::time::Duration;
-use tokio::sync::RwLock as FRwLock;
+use tokio::runtime::Handle;
+// use tokio::sync::RwLock;
+use tokio::task::JoinHandle;
 use topology::NymTopology;
 
-const NODE_HEALTH_THRESHOLD: f64 = 0.0;
+struct TopologyAccessorInner<T: NymTopology>(Option<T>);
 
-// auxiliary type for ease of use
-pub type TopologyInnerRef<T> = Arc<FRwLock<Inner<T>>>;
+impl<T: NymTopology> TopologyAccessorInner<T> {
+    fn new() -> Self {
+        TopologyAccessorInner(None)
+    }
 
-pub(crate) struct TopologyControl<T, IDPair, Priv, Pub>
-where
-    T: NymTopology,
-    IDPair: MixnetIdentityKeyPair<Priv, Pub>,
-    Priv: MixnetIdentityPrivateKey,
-    Pub: MixnetIdentityPublicKey,
-{
-    directory_server: String,
-    inner: Arc<FRwLock<Inner<T>>>,
-    health_checker: HealthChecker<IDPair, Priv, Pub>,
-    refresh_rate: f64,
+    fn update(&mut self, new: Option<T>) {
+        self.0 = new;
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct TopologyAccessor<T: NymTopology> {
+    // TODO: this requires some actual benchmarking to determine if obtaining mutex is not going
+    // to cause some bottlenecking and whether perhaps RwLock would be better
+    inner: Arc<Mutex<TopologyAccessorInner<T>>>,
+}
+
+impl<T: NymTopology> TopologyAccessor<T> {
+    pub(crate) fn new() -> Self {
+        TopologyAccessor {
+            inner: Arc::new(Mutex::new(TopologyAccessorInner::new())),
+        }
+    }
+
+    async fn update_global_topology(&mut self, new_topology: Option<T>) {
+        self.inner.lock().await.update(new_topology);
+    }
+
+    // Unless you absolutely need the entire topology, use `random_route` instead
+    pub(crate) async fn get_current_topology_clone(&mut self) -> Option<T> {
+        self.inner.lock().await.0.clone()
+    }
+
+    // this is a rather temporary solution as each client will have an associated provider
+    // currently that is not implemented yet and there only exists one provider in the network
+    pub(crate) async fn random_route(&mut self) -> Option<Vec<sphinx::route::Node>> {
+        match &self.inner.lock().await.0 {
+            None => None,
+            Some(ref topology) => {
+                let mut providers = topology.providers();
+                if providers.is_empty() {
+                    return None;
+                }
+                // unwrap is fine here as we asserted there is at least single provider
+                let provider = providers.pop().unwrap().into();
+                topology.route_to(provider).ok()
+            }
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -31,41 +70,62 @@ enum TopologyError {
     NoValidPathsError,
 }
 
-impl<T, IDPair, Priv, Pub> TopologyControl<T, IDPair, Priv, Pub>
-where
-    T: NymTopology,
-    IDPair: MixnetIdentityKeyPair<Priv, Pub>,
-    Priv: MixnetIdentityPrivateKey,
-    Pub: MixnetIdentityPublicKey,
-{
-    pub(crate) async fn new(
-        directory_server: String,
-        refresh_rate: f64,
-        identity_keypair: IDPair,
-    ) -> Self {
-        // this is a temporary solution as the healthcheck will eventually be moved to validators
-        let health_checker = healthcheck::HealthChecker::new(5.0, 2, identity_keypair);
+pub(crate) struct TopologyRefresherConfig {
+    directory_server: String,
+    refresh_rate: time::Duration,
+    identity_keypair: MixIdentityKeyPair,
+    resolution_timeout: time::Duration,
+    number_test_packets: usize,
+    node_score_threshold: f64,
+}
 
-        let mut topology_control = TopologyControl {
+impl TopologyRefresherConfig {
+    pub(crate) fn new(
+        directory_server: String,
+        refresh_rate: time::Duration,
+        identity_keypair: MixIdentityKeyPair,
+        resolution_timeout: time::Duration,
+        number_test_packets: usize,
+        node_score_threshold: f64,
+    ) -> Self {
+        TopologyRefresherConfig {
             directory_server,
             refresh_rate,
-            inner: Arc::new(FRwLock::new(Inner::new(None))),
+            identity_keypair,
+            resolution_timeout,
+            number_test_packets,
+            node_score_threshold,
+        }
+    }
+}
+
+pub(crate) struct TopologyRefresher<T: NymTopology> {
+    directory_server: String,
+    topology_accessor: TopologyAccessor<T>,
+    health_checker: HealthChecker,
+    refresh_rate: Duration,
+    node_score_threshold: f64,
+}
+
+impl<T: 'static + NymTopology> TopologyRefresher<T> {
+    pub(crate) fn new(
+        cfg: TopologyRefresherConfig,
+        topology_accessor: TopologyAccessor<T>,
+    ) -> Self {
+        // this is a temporary solution as the healthcheck will eventually be moved to validators
+        let health_checker = healthcheck::HealthChecker::new(
+            cfg.resolution_timeout,
+            cfg.number_test_packets,
+            cfg.identity_keypair,
+        );
+
+        TopologyRefresher {
+            directory_server: cfg.directory_server,
+            topology_accessor,
             health_checker,
-        };
-
-        // best effort approach to try to get a valid topology after call to 'new'
-        let initial_topology = match topology_control.get_current_compatible_topology().await {
-            Ok(topology) => Some(topology),
-            Err(err) => {
-                error!("Initial topology is invalid - {:?}. Right now it will be impossible to send any packets through the mixnet!", err);
-                None
-            }
-        };
-        topology_control
-            .update_global_topology(initial_topology)
-            .await;
-
-        topology_control
+            refresh_rate: cfg.refresh_rate,
+            node_score_threshold: cfg.node_score_threshold,
+        }
     }
 
     async fn get_current_compatible_topology(&self) -> Result<T, TopologyError> {
@@ -89,7 +149,7 @@ where
         };
 
         let healthy_topology = healthcheck_scores
-            .filter_topology_by_score(&version_filtered_topology, NODE_HEALTH_THRESHOLD);
+            .filter_topology_by_score(&version_filtered_topology, self.node_score_threshold);
 
         // make sure you can still send a packet through the network:
         if !healthy_topology.can_construct_path_through() {
@@ -99,43 +159,27 @@ where
         Ok(healthy_topology)
     }
 
-    pub(crate) fn get_inner_ref(&self) -> Arc<FRwLock<Inner<T>>> {
-        self.inner.clone()
+    pub(crate) async fn refresh(&mut self) {
+        trace!("Refreshing the topology");
+        let new_topology = match self.get_current_compatible_topology().await {
+            Ok(topology) => Some(topology),
+            Err(err) => {
+                warn!("the obtained topology seems to be invalid - {:?}, it will be impossible to send packets through", err);
+                None
+            }
+        };
+
+        self.topology_accessor
+            .update_global_topology(new_topology)
+            .await;
     }
 
-    async fn update_global_topology(&mut self, new_topology: Option<T>) {
-        // acquire write lock
-        let mut write_lock = self.inner.write().await;
-        write_lock.topology = new_topology;
-    }
-
-    pub(crate) async fn run_refresher(mut self) {
-        info!("Starting topology refresher");
-        let delay_duration = Duration::from_secs_f64(self.refresh_rate);
-        loop {
-            trace!("Refreshing the topology");
-            let new_topology_res = self.get_current_compatible_topology().await;
-
-            let new_topology = match new_topology_res {
-                Ok(topology) => Some(topology),
-                Err(err) => {
-                    warn!("the obtained topology seems to be invalid - {:?}, it will be impossible to send packets through", err);
-                    None
-                }
-            };
-
-            self.update_global_topology(new_topology).await;
-            tokio::time::delay_for(delay_duration).await;
-        }
-    }
-}
-
-pub struct Inner<T: NymTopology> {
-    pub topology: Option<T>,
-}
-
-impl<T: NymTopology> Inner<T> {
-    fn new(topology: Option<T>) -> Self {
-        Inner { topology }
+    pub(crate) fn start(mut self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            loop {
+                self.refresh().await;
+                tokio::time::delay_for(self.refresh_rate).await;
+            }
+        })
     }
 }

nym-client/src/commands/init.rs

@@ -1,18 +1,146 @@
-use crate::config::persistance::pathfinder::ClientPathfinder;
-use clap::ArgMatches;
-use crypto::identity::MixnetIdentityKeyPair;
+use crate::built_info;
+use crate::commands::override_config;
+use crate::config::persistence::pathfinder::ClientPathfinder;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+use crypto::identity::MixIdentityKeyPair;
+use directory_client::presence::Topology;
 use pemstore::pemstore::PemStore;
+use sfw_provider_requests::AuthToken;
+use sphinx::route::DestinationAddressBytes;
+use topology::provider::Node;
+use topology::NymTopology;
+
+pub fn command_args<'a, 'b>() -> clap::App<'a, 'b> {
+    App::new("init")
+        .about("Initialise a Nym client. Do this first!")
+        .arg(Arg::with_name("id")
+            .long("id")
+            .help("Id of the nym-mixnet-client we want to create config for.")
+            .takes_value(true)
+            .required(true)
+        )
+        .arg(Arg::with_name("provider")
+            .long("provider")
+            .help("Id of the provider we have preference to connect to. If left empty, a random provider will be chosen.")
+            .takes_value(true)
+        )
+        .arg(Arg::with_name("directory")
+                 .long("directory")
+                 .help("Address of the directory server the client is getting topology from")
+                 .takes_value(true),
+        )
+        .arg(Arg::with_name("socket-type")
+            .long("socket-type")
+            .help("Type of socket to use (TCP, WebSocket or None) in all subsequent runs")
+            .takes_value(true)
+        )
+        .arg(Arg::with_name("port")
+            .short("p")
+            .long("port")
+            .help("Port for the socket (if applicable) to listen on in all subsequent runs")
+            .takes_value(true)
+        )
+}
+
+async fn try_provider_registrations(
+    providers: Vec<Node>,
+    our_address: DestinationAddressBytes,
+) -> Option<(String, AuthToken)> {
+    // since the order of providers is non-deterministic we can just try to get a first 'working' provider
+    for provider in providers {
+        let provider_client = provider_client::ProviderClient::new(
+            provider.client_listener,
+            our_address.clone(),
+            None,
+        );
+        let auth_token = provider_client.register().await;
+        if let Ok(token) = auth_token {
+            return Some((provider.pub_key, token));
+        }
+    }
+    None
+}
+
+// in the long run this will be provider specific and only really applicable to a
+// relatively small subset of all providers
+async fn choose_provider(
+    directory_server: String,
+    our_address: DestinationAddressBytes,
+) -> (String, AuthToken) {
+    // TODO: once we change to graph topology this here will need to be updated!
+    let topology = Topology::new(directory_server.clone());
+    let version_filtered_topology = topology.filter_node_versions(
+        built_info::PKG_VERSION,
+        built_info::PKG_VERSION,
+        built_info::PKG_VERSION,
+    );
+    // don't care about health of the networks as mixes can go up and down any time,
+    // but DO care about providers
+    let providers = version_filtered_topology.providers();
+
+    // try to perform registration so that we wouldn't need to do it at startup
+    // + at the same time we'll know if we can actually talk with that provider
+    let registration_result = try_provider_registrations(providers, our_address).await;
+    match registration_result {
+        None => {
+            // while technically there's no issue client-side, it will be impossible to execute
+            // `nym-client run` as no provider is available so it might be best to not finalize
+            // the init and rely on users trying to init another time?
+            panic!(
+                "Currently there are no valid providers available on the network ({}). \
+                 Please try to run `init` again at later time or change your directory server",
+                directory_server
+            )
+        }
+        Some((provider_id, auth_token)) => (provider_id, auth_token),
+    }
+}
 
 pub fn execute(matches: &ArgMatches) {
     println!("Initialising client...");
 
-    let id = matches.value_of("id").unwrap().to_string(); // required for now
-    let pathfinder = ClientPathfinder::new(id);
+    let id = matches.value_of("id").unwrap(); // required for now
+    let mut config = crate::config::Config::new(id);
 
-    println!("Writing keypairs to {:?}...", pathfinder.config_dir);
-    let mix_keys = crypto::identity::DummyMixIdentityKeyPair::new();
+    config = override_config(config, matches);
+
+    let mix_identity_keys = MixIdentityKeyPair::new();
+
+    // if there is no provider chosen, get a random-ish one from the topology
+    if config.get_provider_id().is_empty() {
+        let our_address = mix_identity_keys.public_key().derive_address();
+        // TODO: is there perhaps a way to make it work without having to spawn entire runtime?
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let (provider_id, auth_token) =
+            rt.block_on(choose_provider(config.get_directory_server(), our_address));
+        config = config
+            .with_provider_id(provider_id)
+            .with_provider_auth_token(auth_token);
+    }
+
+    let pathfinder = ClientPathfinder::new_from_config(&config);
     let pem_store = PemStore::new(pathfinder);
-    pem_store.write_identity(mix_keys).unwrap();
+    pem_store
+        .write_identity(mix_identity_keys)
+        .expect("Failed to save identity keys");
+    println!("Saved mixnet identity keypair");
 
+    let config_save_location = config.get_config_file_save_location();
+    config
+        .save_to_file(None)
+        .expect("Failed to save the config file");
+    println!("Saved configuration file to {:?}", config_save_location);
+
+    println!(
+        "Unless overridden in all `nym-client run` we will be talking to the following provider: {}...",
+        config.get_provider_id(),
+    );
+    if config.get_provider_auth_token().is_some() {
+        println!(
+            "using optional AuthToken: {:?}",
+            config.get_provider_auth_token().unwrap()
+        )
+    }
     println!("Client configuration completed.\n\n\n")
 }

nym-client/src/commands/mod.rs

@@ -1,3 +1,29 @@
+use crate::config::{Config, SocketType};
+use clap::ArgMatches;
+
 pub mod init;
-pub mod tcpsocket;
-pub mod websocket;
+pub mod run;
+
+pub(crate) fn override_config(mut config: Config, matches: &ArgMatches) -> Config {
+    if let Some(directory) = matches.value_of("directory") {
+        config = config.with_custom_directory(directory);
+    }
+
+    if let Some(provider_id) = matches.value_of("provider") {
+        config = config.with_provider_id(provider_id);
+    }
+
+    if let Some(socket_type) = matches.value_of("socket-type") {
+        config = config.with_socket(SocketType::from_string(socket_type));
+    }
+
+    if let Some(port) = matches.value_of("port").map(|port| port.parse::<u16>()) {
+        if let Err(err) = port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_port(port.unwrap());
+    }
+
+    config
+}

nym-client/src/commands/run.rs

@@ -0,0 +1,54 @@
+use crate::client::NymClient;
+use crate::commands::override_config;
+use crate::config::Config;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+
+pub fn command_args<'a, 'b>() -> clap::App<'a, 'b> {
+    App::new("run")
+        .about("Run the Nym client with provided configuration client optionally overriding set parameters")
+        .arg(Arg::with_name("id")
+            .long("id")
+            .help("Id of the nym-mixnet-client we want to run.")
+            .takes_value(true)
+            .required(true)
+        )
+        // the rest of arguments are optional, they are used to override settings in config file
+        .arg(Arg::with_name("config")
+            .long("config")
+            .help("Custom path to the nym-mixnet-client configuration file")
+            .takes_value(true)
+        )
+        .arg(Arg::with_name("directory")
+                 .long("directory")
+                 .help("Address of the directory server the client is getting topology from")
+                 .takes_value(true),
+        )
+        .arg(Arg::with_name("provider")
+            .long("provider")
+            .help("Id of the provider we want to connect to. If overridden, it is user's responsibility to ensure prior registration happened")
+            .takes_value(true)
+        )
+        .arg(Arg::with_name("socket-type")
+            .long("socket-type")
+            .help("Type of socket to use (TCP, WebSocket or None)")
+            .takes_value(true)
+        )
+        .arg(Arg::with_name("port")
+            .short("p")
+            .long("port")
+            .help("Port for the socket (if applicable) to listen on")
+            .takes_value(true)
+        )
+}
+
+pub fn execute(matches: &ArgMatches) {
+    let id = matches.value_of("id").unwrap();
+
+    let mut config =
+        Config::load_from_file(matches.value_of("config").map(|path| path.into()), Some(id))
+            .expect("Failed to load config file");
+
+    config = override_config(config, matches);
+    NymClient::new(config).run_forever();
+}

nym-client/src/commands/tcpsocket.rs

@@ -1,47 +0,0 @@
-use crate::client::{NymClient, SocketType};
-use crate::config::persistance::pathfinder::ClientPathfinder;
-use clap::ArgMatches;
-use crypto::identity::DummyMixIdentityKeyPair;
-use pemstore::pemstore::PemStore;
-use std::net::ToSocketAddrs;
-
-pub fn execute(matches: &ArgMatches) {
-    let id = matches.value_of("id").unwrap().to_string();
-    let port = match matches.value_of("port").unwrap_or("9001").parse::<u16>() {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid port value provided - {:?}", err),
-    };
-
-    let directory_server = matches
-        .value_of("directory")
-        .unwrap_or("https://directory.nymtech.net")
-        .to_string();
-
-    println!("Starting TCP socket on port: {:?}", port);
-    println!("Listening for messages...");
-
-    let socket_address = ("127.0.0.1", port)
-        .to_socket_addrs()
-        .expect("Failed to combine host and port")
-        .next()
-        .expect("Failed to extract the socket address from the iterator");
-
-    // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type?
-    let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id))
-        .read_identity()
-        .unwrap();
-    // TODO: reading auth_token from disk (if exists);
-
-    println!("Public key: {}", keypair.public_key.to_base58_string());
-
-    let auth_token = None;
-    let client = NymClient::new(
-        keypair,
-        socket_address.clone(),
-        directory_server,
-        auth_token,
-        SocketType::TCP,
-    );
-
-    client.start().unwrap();
-}

nym-client/src/commands/websocket.rs

@@ -1,48 +0,0 @@
-use crate::client::{NymClient, SocketType};
-use crate::config::persistance::pathfinder::ClientPathfinder;
-use clap::ArgMatches;
-use crypto::identity::DummyMixIdentityKeyPair;
-use pemstore::pemstore::PemStore;
-use std::net::ToSocketAddrs;
-
-pub fn execute(matches: &ArgMatches) {
-    let id = matches.value_of("id").unwrap().to_string();
-    let port = match matches.value_of("port").unwrap_or("9001").parse::<u16>() {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid port value provided - {:?}", err),
-    };
-
-    let directory_server = matches
-        .value_of("directory")
-        .unwrap_or("https://directory.nymtech.net")
-        .to_string();
-
-    println!("Starting websocket on port: {:?}", port);
-    println!("Listening for messages...");
-
-    let socket_address = ("127.0.0.1", port)
-        .to_socket_addrs()
-        .expect("Failed to combine host and port")
-        .next()
-        .expect("Failed to extract the socket address from the iterator");
-
-    // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type?
-    let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id))
-        .read_identity()
-        .unwrap();
-
-    // TODO: reading auth_token from disk (if exists);
-
-    println!("Public key: {}", keypair.public_key.to_base58_string());
-
-    let auth_token = None;
-    let client = NymClient::new(
-        keypair,
-        socket_address,
-        directory_server,
-        auth_token,
-        SocketType::WebSocket,
-    );
-
-    client.start().unwrap();
-}

nym-client/src/config/mod.rs

@@ -1 +1,417 @@
-pub mod persistance;
+use crate::config::template::config_template;
+use config::NymConfig;
+use serde::{Deserialize, Deserializer, Serialize};
+use std::path::PathBuf;
+use std::time;
+
+pub mod persistence;
+mod template;
+
+// 'CLIENT'
+const DEFAULT_LISTENING_PORT: u16 = 9001;
+
+// 'DEBUG'
+// where applicable, the below are defined in milliseconds
+const DEFAULT_LOOP_COVER_STREAM_AVERAGE_DELAY: u64 = 1000;
+const DEFAULT_MESSAGE_STREAM_AVERAGE_DELAY: u64 = 500;
+const DEFAULT_AVERAGE_PACKET_DELAY: u64 = 200;
+const DEFAULT_FETCH_MESSAGES_DELAY: u64 = 1000;
+const DEFAULT_TOPOLOGY_REFRESH_RATE: u64 = 10_000;
+const DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT: u64 = 5_000;
+const DEFAULT_PACKET_FORWARDING_INITIAL_BACKOFF: u64 = 10_000; // 10s
+const DEFAULT_PACKET_FORWARDING_MAXIMUM_BACKOFF: u64 = 300_000; // 5min
+
+const DEFAULT_NUMBER_OF_HEALTHCHECK_TEST_PACKETS: u64 = 2;
+const DEFAULT_NODE_SCORE_THRESHOLD: f64 = 0.0;
+
+#[derive(Debug, Deserialize, PartialEq, Serialize, Clone, Copy)]
+#[serde(deny_unknown_fields)]
+pub enum SocketType {
+    TCP,
+    WebSocket,
+    None,
+}
+
+impl SocketType {
+    pub fn from_string<S: Into<String>>(val: S) -> Self {
+        let mut upper = val.into();
+        upper.make_ascii_uppercase();
+        match upper.as_ref() {
+            "TCP" => SocketType::TCP,
+            "WEBSOCKET" | "WS" => SocketType::WebSocket,
+            _ => SocketType::None,
+        }
+    }
+}
+
+#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Config {
+    client: Client,
+    socket: Socket,
+
+    #[serde(default)]
+    logging: Logging,
+    #[serde(default)]
+    debug: Debug,
+}
+
+impl NymConfig for Config {
+    fn template() -> &'static str {
+        config_template()
+    }
+
+    fn config_file_name() -> String {
+        "config.toml".to_string()
+    }
+
+    fn default_root_directory() -> PathBuf {
+        dirs::home_dir()
+            .expect("Failed to evaluate $HOME value")
+            .join(".nym")
+            .join("clients")
+    }
+
+    fn root_directory(&self) -> PathBuf {
+        self.client.nym_root_directory.clone()
+    }
+
+    fn config_directory(&self) -> PathBuf {
+        self.client
+            .nym_root_directory
+            .join(&self.client.id)
+            .join("config")
+    }
+
+    fn data_directory(&self) -> PathBuf {
+        self.client
+            .nym_root_directory
+            .join(&self.client.id)
+            .join("data")
+    }
+}
+
+impl Config {
+    pub fn new<S: Into<String>>(id: S) -> Self {
+        Config::default().with_id(id)
+    }
+
+    // builder methods
+    pub fn with_id<S: Into<String>>(mut self, id: S) -> Self {
+        let id = id.into();
+        if self.client.private_identity_key_file.as_os_str().is_empty() {
+            self.client.private_identity_key_file =
+                self::Client::default_private_identity_key_file(&id);
+        }
+        if self.client.public_identity_key_file.as_os_str().is_empty() {
+            self.client.public_identity_key_file =
+                self::Client::default_public_identity_key_file(&id);
+        }
+        self.client.id = id;
+        self
+    }
+
+    pub fn with_provider_id<S: Into<String>>(mut self, id: S) -> Self {
+        self.client.provider_id = id.into();
+        self
+    }
+
+    pub fn with_provider_auth_token<S: Into<String>>(mut self, token: S) -> Self {
+        self.client.provider_authtoken = Some(token.into());
+        self
+    }
+
+    pub fn with_custom_directory<S: Into<String>>(mut self, directory_server: S) -> Self {
+        self.client.directory_server = directory_server.into();
+        self
+    }
+
+    pub fn with_socket(mut self, socket_type: SocketType) -> Self {
+        self.socket.socket_type = socket_type;
+        self
+    }
+
+    pub fn with_port(mut self, port: u16) -> Self {
+        self.socket.listening_port = port;
+        self
+    }
+
+    // getters
+    pub fn get_config_file_save_location(&self) -> PathBuf {
+        self.config_directory().join(Self::config_file_name())
+    }
+
+    pub fn get_private_identity_key_file(&self) -> PathBuf {
+        self.client.private_identity_key_file.clone()
+    }
+
+    pub fn get_public_identity_key_file(&self) -> PathBuf {
+        self.client.public_identity_key_file.clone()
+    }
+
+    pub fn get_directory_server(&self) -> String {
+        self.client.directory_server.clone()
+    }
+
+    pub fn get_provider_id(&self) -> String {
+        self.client.provider_id.clone()
+    }
+
+    pub fn get_provider_auth_token(&self) -> Option<String> {
+        self.client.provider_authtoken.clone()
+    }
+
+    pub fn get_socket_type(&self) -> SocketType {
+        self.socket.socket_type
+    }
+
+    pub fn get_listening_port(&self) -> u16 {
+        self.socket.listening_port
+    }
+
+    // Debug getters
+    pub fn get_average_packet_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.average_packet_delay)
+    }
+
+    pub fn get_loop_cover_traffic_average_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.loop_cover_traffic_average_delay)
+    }
+
+    pub fn get_fetch_message_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.fetch_message_delay)
+    }
+
+    pub fn get_message_sending_average_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.message_sending_average_delay)
+    }
+
+    pub fn get_rate_compliant_cover_messages_disabled(&self) -> bool {
+        self.debug.rate_compliant_cover_messages_disabled
+    }
+
+    pub fn get_topology_refresh_rate(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.topology_refresh_rate)
+    }
+
+    pub fn get_topology_resolution_timeout(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.topology_resolution_timeout)
+    }
+
+    pub fn get_number_of_healthcheck_test_packets(&self) -> u64 {
+        self.debug.number_of_healthcheck_test_packets
+    }
+
+    pub fn get_node_score_threshold(&self) -> f64 {
+        self.debug.node_score_threshold
+    }
+
+    pub fn get_packet_forwarding_initial_backoff(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.packet_forwarding_initial_backoff)
+    }
+
+    pub fn get_packet_forwarding_maximum_backoff(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.packet_forwarding_maximum_backoff)
+    }
+}
+
+fn de_option_string<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s = String::deserialize(deserializer)?;
+    if s.is_empty() {
+        Ok(None)
+    } else {
+        Ok(Some(s))
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Client {
+    /// ID specifies the human readable ID of this particular client.
+    id: String,
+
+    /// URL to the directory server.
+    directory_server: String,
+
+    /// Path to file containing private identity key.
+    private_identity_key_file: PathBuf,
+
+    /// Path to file containing public identity key.
+    public_identity_key_file: PathBuf,
+
+    /// provider_id specifies ID of the provider to which the client should send messages.
+    /// If initially omitted, a random provider will be chosen from the available topology.
+    provider_id: String,
+
+    /// A provider specific, optional, base58 stringified authentication token used for
+    /// communication with particular provider.
+    #[serde(deserialize_with = "de_option_string")]
+    provider_authtoken: Option<String>,
+
+    /// nym_home_directory specifies absolute path to the home nym Clients directory.
+    /// It is expected to use default value and hence .toml file should not redefine this field.
+    nym_root_directory: PathBuf,
+}
+
+impl Default for Client {
+    fn default() -> Self {
+        // there must be explicit checks for whether id is not empty later
+        Client {
+            id: "".to_string(),
+            directory_server: Self::default_directory_server(),
+            private_identity_key_file: Default::default(),
+            public_identity_key_file: Default::default(),
+            provider_id: "".to_string(),
+            provider_authtoken: None,
+            nym_root_directory: Config::default_root_directory(),
+        }
+    }
+}
+
+impl Client {
+    fn default_directory_server() -> String {
+        #[cfg(feature = "qa")]
+        return "https://qa-directory.nymtech.net".to_string();
+        #[cfg(feature = "local")]
+        return "http://localhost:8080".to_string();
+
+        "https://directory.nymtech.net".to_string()
+    }
+
+    fn default_private_identity_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("private_identity.pem")
+    }
+
+    fn default_public_identity_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("public_identity.pem")
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Socket {
+    socket_type: SocketType,
+    listening_port: u16,
+}
+
+impl Default for Socket {
+    fn default() -> Self {
+        Socket {
+            socket_type: SocketType::None,
+            listening_port: DEFAULT_LISTENING_PORT,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Logging {}
+
+impl Default for Logging {
+    fn default() -> Self {
+        Logging {}
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Debug {
+    /// The parameter of Poisson distribution determining how long, on average,
+    /// sent packet is going to be delayed at any given mix node.
+    /// So for a packet going through three mix nodes, on average, it will take three times this value
+    /// until the packet reaches its destination.
+    /// The provided value is interpreted as milliseconds.
+    average_packet_delay: u64,
+
+    /// The parameter of Poisson distribution determining how long, on average,
+    /// it is going to take for another loop cover traffic message to be sent.
+    /// The provided value is interpreted as milliseconds.
+    loop_cover_traffic_average_delay: u64,
+
+    /// The uniform delay every which clients are querying the providers for received packets.
+    /// The provided value is interpreted as milliseconds.
+    fetch_message_delay: u64,
+
+    /// The parameter of Poisson distribution determining how long, on average,
+    /// it is going to take another 'real traffic stream' message to be sent.
+    /// If no real packets are available and cover traffic is enabled,
+    /// a loop cover message is sent instead in order to preserve the rate.
+    /// The provided value is interpreted as milliseconds.
+    message_sending_average_delay: u64,
+
+    /// Whether loop cover messages should be sent to respect message_sending_rate.
+    /// In the case of it being disabled and not having enough real traffic
+    /// waiting to be sent the actual sending rate is going be lower than the desired value
+    /// thus decreasing the anonymity.
+    rate_compliant_cover_messages_disabled: bool,
+
+    /// The uniform delay every which clients are querying the directory server
+    /// to try to obtain a compatible network topology to send sphinx packets through.
+    /// The provided value is interpreted as milliseconds.
+    topology_refresh_rate: u64,
+
+    /// During topology refresh, test packets are sent through every single possible network
+    /// path. This timeout determines waiting period until it is decided that the packet
+    /// did not reach its destination.
+    /// The provided value is interpreted as milliseconds.
+    topology_resolution_timeout: u64,
+
+    /// How many packets should be sent through each path during the healthcheck
+    number_of_healthcheck_test_packets: u64,
+
+    /// In the current healthcheck implementation, threshold indicating percentage of packets
+    /// node received during healthcheck. Node's score must be above that value to be
+    /// considered healthy.
+    node_score_threshold: f64,
+
+    /// Initial value of an exponential backoff to reconnect to dropped TCP connection when
+    /// forwarding sphinx packets.
+    /// The provided value is interpreted as milliseconds.
+    packet_forwarding_initial_backoff: u64,
+
+    /// Maximum value of an exponential backoff to reconnect to dropped TCP connection when
+    /// forwarding sphinx packets.
+    /// The provided value is interpreted as milliseconds.
+    packet_forwarding_maximum_backoff: u64,
+}
+
+impl Default for Debug {
+    fn default() -> Self {
+        Debug {
+            average_packet_delay: DEFAULT_AVERAGE_PACKET_DELAY,
+            loop_cover_traffic_average_delay: DEFAULT_LOOP_COVER_STREAM_AVERAGE_DELAY,
+            fetch_message_delay: DEFAULT_FETCH_MESSAGES_DELAY,
+            message_sending_average_delay: DEFAULT_MESSAGE_STREAM_AVERAGE_DELAY,
+            rate_compliant_cover_messages_disabled: false,
+            topology_refresh_rate: DEFAULT_TOPOLOGY_REFRESH_RATE,
+            topology_resolution_timeout: DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT,
+            number_of_healthcheck_test_packets: DEFAULT_NUMBER_OF_HEALTHCHECK_TEST_PACKETS,
+            node_score_threshold: DEFAULT_NODE_SCORE_THRESHOLD,
+            packet_forwarding_initial_backoff: DEFAULT_PACKET_FORWARDING_INITIAL_BACKOFF,
+            packet_forwarding_maximum_backoff: DEFAULT_PACKET_FORWARDING_MAXIMUM_BACKOFF,
+        }
+    }
+}
+
+#[cfg(test)]
+mod client_config {
+    use super::*;
+
+    #[test]
+    fn after_saving_default_config_the_loaded_one_is_identical() {
+        // need to figure out how to do something similar but without touching the disk
+        // or the file system at all...
+        let temp_location = tempfile::tempdir().unwrap().path().join("config.toml");
+        let default_config = Config::default().with_id("foomp".to_string());
+        default_config
+            .save_to_file(Some(temp_location.clone()))
+            .unwrap();
+
+        let loaded_config = Config::load_from_file(Some(temp_location), None).unwrap();
+
+        assert_eq!(default_config, loaded_config);
+    }
+}

nym-client/src/config/persistence/mod.rs

@@ -0,0 +1 @@
+pub mod pathfinder;

nym-client/src/config/persistence/pathfinder.rs

@@ -1,6 +1,8 @@
+use crate::config::Config;
 use pemstore::pathfinder::PathFinder;
 use std::path::PathBuf;
 
+#[derive(Debug)]
 pub struct ClientPathfinder {
     pub config_dir: PathBuf,
     pub private_mix_key: PathBuf,
@@ -19,6 +21,14 @@ impl ClientPathfinder {
             public_mix_key,
         }
     }
+
+    pub fn new_from_config(config: &Config) -> Self {
+        ClientPathfinder {
+            config_dir: config.get_config_file_save_location(),
+            private_mix_key: config.get_private_identity_key_file(),
+            public_mix_key: config.get_public_identity_key_file(),
+        }
+    }
 }
 
 impl PathFinder for ClientPathfinder {

nym-client/src/config/template.rs

@@ -0,0 +1,123 @@
+pub(crate) fn config_template() -> &'static str {
+    // While using normal toml marshalling would have been way simpler with less overhead,
+    // I think it's useful to have comments attached to the saved config file to explain behaviour of
+    // particular fields.
+    // Note: any changes to the template must be reflected in the appropriate structs in mod.rs.
+    r#"
+# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+##### main base client config options #####
+
+[client]
+# Human readable ID of this particular client.
+id = "{{ client.id }}"
+
+# URL to the directory server.
+directory_server = "{{ client.directory_server }}"
+
+# Path to file containing private identity key.
+private_identity_key_file = "{{ client.private_identity_key_file }}"
+
+# Path to file containing public identity key.
+public_identity_key_file = "{{ client.public_identity_key_file }}"
+
+##### additional client config options #####
+
+# ID of the provider from which the client should be fetching messages.
+provider_id = "{{ client.provider_id }}"
+
+# A provider specific, optional, base58 stringified authentication token used for 
+# communication with particular provider.
+provider_authtoken = "{{ client.provider_authtoken }}"
+    
+##### advanced configuration options #####
+
+# Absolute path to the home Nym Clients directory.
+nym_root_directory = "{{ client.nym_root_directory }}"
+
+
+##### socket config options #####
+
+[socket]
+
+# allowed values are 'TCP', 'WebSocket' or 'None'
+socket_type = "{{ socket.socket_type }}"
+
+# if applicable (for the case of 'TCP' or 'WebSocket'), the port on which the client
+# will be listening for incoming requests
+listening_port = {{ socket.listening_port }}
+
+
+##### logging configuration options #####
+
+[logging]
+
+# TODO
+
+
+##### debug configuration options #####
+# The following options should not be modified unless you know EXACTLY what you are doing
+# as if set incorrectly, they may impact your anonymity.
+
+[debug]
+
+# The parameter of Poisson distribution determining how long, on average,
+# sent packet is going to be delayed at any given mix node.
+# So for a packet going through three mix nodes, on average, it will take three times this value
+# until the packet reaches its destination.
+# The provided value is interpreted as milliseconds.
+average_packet_delay = {{ debug.average_packet_delay }}
+
+# The parameter of Poisson distribution determining how long, on average,
+# it is going to take for another loop cover traffic message to be sent.
+# The provided value is interpreted as milliseconds.
+loop_cover_traffic_average_delay = {{ debug.loop_cover_traffic_average_delay }}
+
+# The uniform delay every which clients are querying the providers for received packets.
+# The provided value is interpreted as milliseconds.
+fetch_message_delay = {{ debug.fetch_message_delay }}
+
+# The parameter of Poisson distribution determining how long, on average,
+# it is going to take another 'real traffic stream' message to be sent.
+# If no real packets are available and cover traffic is enabled,
+# a loop cover message is sent instead in order to preserve the rate.
+# The provided value is interpreted as milliseconds.
+message_sending_average_delay = {{ debug.message_sending_average_delay }}
+
+# Whether loop cover messages should be sent to respect message_sending_rate.
+# In the case of it being disabled and not having enough real traffic
+# waiting to be sent the actual sending rate is going be lower than the desired value
+# thus decreasing the anonymity.
+rate_compliant_cover_messages_disabled = {{ debug.rate_compliant_cover_messages_disabled }}
+
+# The uniform delay every which clients are querying the directory server
+# to try to obtain a compatible network topology to send sphinx packets through.
+# The provided value is interpreted as milliseconds.
+topology_refresh_rate = {{ debug.topology_refresh_rate }}
+
+# During topology refresh, test packets are sent through every single possible network
+# path. This timeout determines waiting period until it is decided that the packet
+# did not reach its destination.
+# The provided value is interpreted as milliseconds.
+topology_resolution_timeout = {{ debug.topology_resolution_timeout }}
+  
+# How many packets should be sent through each path during the healthcheck
+number_of_healthcheck_test_packets = {{ debug.number_of_healthcheck_test_packets }}
+
+# In the current healthcheck implementation, threshold indicating percentage of packets 
+# node received during healthcheck. Node's score must be above that value to be 
+# considered healthy.
+node_score_threshold = {{ debug.node_score_threshold }}
+
+# Initial value of an exponential backoff to reconnect to dropped TCP connection when
+# forwarding sphinx packets.
+# The provided value is interpreted as milliseconds.
+packet_forwarding_initial_backoff = {{ debug.packet_forwarding_initial_backoff }}
+
+# Maximum value of an exponential backoff to reconnect to dropped TCP connection when
+# forwarding sphinx packets.
+# The provided value is interpreted as milliseconds.
+packet_forwarding_maximum_backoff = {{ debug.packet_forwarding_maximum_backoff }}    
+"#
+}

nym-client/src/main.rs

@@ -1,4 +1,4 @@
-use clap::{App, Arg, ArgMatches, SubCommand};
+use clap::{App, ArgMatches};
 
 pub mod built_info;
 pub mod client;
@@ -10,72 +10,14 @@ fn main() {
     dotenv::dotenv().ok();
     pretty_env_logger::init();
 
+    println!("{}", banner());
+
     let arg_matches = App::new("Nym Client")
         .version(built_info::PKG_VERSION)
         .author("Nymtech")
         .about("Implementation of the Nym Client")
-        .subcommand(
-            SubCommand::with_name("init")
-                .about("Initialise a Nym client. Do this first!")
-                .arg(Arg::with_name("id")
-                    .long("id")
-                    .help("Id of the nym-mixnet-client we want to create config for.")
-                    .takes_value(true)
-                    .required(true)
-                )
-                .arg(Arg::with_name("provider")
-                    .long("provider")
-                    .help("Id of the provider we have preference to connect to. If left empty, a random provider will be chosen.")
-                    .takes_value(true)
-                )
-        )
-        .subcommand(
-            SubCommand::with_name("tcpsocket")
-                .about("Run Nym client that listens for bytes on a TCP socket")
-                .arg(
-                    Arg::with_name("port")
-                        .short("p")
-                        .long("port")
-                        .help("Port for TCP socket to listen on")
-                        .takes_value(true)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("directory")
-                        .long("directory")
-                        .help("Address of the directory server the client is getting topology from")
-                        .takes_value(true),
-                )
-                .arg(Arg::with_name("id")
-                    .long("id")
-                    .help("Id of the nym-mixnet-client we want to run.")
-                    .takes_value(true)
-                    .required(true)
-                )
-        )
-        .subcommand(
-            SubCommand::with_name("websocket")
-                .about("Run Nym client that listens on a websocket")
-                .arg(
-                    Arg::with_name("port")
-                        .short("p")
-                        .long("port")
-                        .help("Port for websocket to listen on")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("directory")
-                        .long("directory")
-                        .help("Address of the directory server the client is getting topology from")
-                        .takes_value(true),
-                )
-                .arg(Arg::with_name("id")
-                    .long("id")
-                    .help("Id of the nym-mixnet-client we want to run.")
-                    .takes_value(true)
-                    .required(true)
-                )
-        )
+        .subcommand(commands::init::command_args())
+        .subcommand(commands::run::command_args())
         .get_matches();
 
     execute(arg_matches);
@@ -83,26 +25,14 @@ fn main() {
 
 fn execute(matches: ArgMatches) {
     match matches.subcommand() {
-        ("init", Some(m)) => {
-            println!("{}", banner());
-            commands::init::execute(m);
-        }
-        ("tcpsocket", Some(m)) => {
-            println!("{}", banner());
-            commands::tcpsocket::execute(m);
-        }
-        ("websocket", Some(m)) => {
-            println!("{}", banner());
-            commands::websocket::execute(m);
-        }
-        _ => {
-            println!("{}", usage());
-        }
+        ("init", Some(m)) => commands::init::execute(m),
+        ("run", Some(m)) => commands::run::execute(m),
+        _ => println!("{}", usage()),
     }
 }
 
-fn usage() -> String {
-    banner() + "usage: --help to see available options.\n\n"
+fn usage() -> &'static str {
+    "usage: --help to see available options.\n\n"
 }
 
 fn banner() -> String {

nym-client/src/sockets/tcp.rs

@@ -1,5 +1,5 @@
-use crate::client::received_buffer::BufferResponse;
-use crate::client::topology_control::TopologyInnerRef;
+use crate::client::received_buffer::ReceivedBufferResponse;
+use crate::client::topology_control::TopologyAccessor;
 use crate::client::InputMessage;
 use futures::channel::{mpsc, oneshot};
 use futures::future::FutureExt;
@@ -11,6 +11,8 @@ use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
 use tokio::prelude::*;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
 use topology::NymTopology;
 
 const SEND_REQUEST_PREFIX: u8 = 1;
@@ -84,7 +86,7 @@ fn parse_send_request(data: &[u8]) -> Result<ClientRequest, TCPSocketError> {
 
     Ok(ClientRequest::Send {
         message,
-        recipient_address,
+        recipient_address: DestinationAddressBytes::from_bytes(recipient_address),
     })
 }
 
@@ -101,8 +103,7 @@ impl ClientRequest {
                     "too long message. Sent {} bytes while the maximum is {}",
                     msg.len(),
                     sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH
-                )
-                .to_string(),
+                ),
             };
         }
         let dummy_surb = [0; 16];
@@ -111,7 +112,9 @@ impl ClientRequest {
         ServerResponse::Send
     }
 
-    async fn handle_fetch(mut msg_query: mpsc::UnboundedSender<BufferResponse>) -> ServerResponse {
+    async fn handle_fetch(
+        mut msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
+    ) -> ServerResponse {
         trace!("handle_fetch called");
         let (res_tx, res_rx) = oneshot::channel();
         if msg_query.send(res_tx).await.is_err() {
@@ -125,7 +128,7 @@ impl ClientRequest {
             Err(e) => {
                 warn!("Failed to fetch client messages - {:?}", e);
                 return ServerResponse::Error {
-                    message: format!("Server failed to receive messages").to_string(),
+                    message: "Server failed to receive messages".to_string(),
                 };
             }
         };
@@ -134,9 +137,10 @@ impl ClientRequest {
         ServerResponse::Fetch { messages }
     }
 
-    async fn handle_get_clients<T: NymTopology>(topology: &TopologyInnerRef<T>) -> ServerResponse {
-        let topology_data = &topology.read().await.topology;
-        match topology_data {
+    async fn handle_get_clients<T: NymTopology>(
+        mut topology_accessor: TopologyAccessor<T>,
+    ) -> ServerResponse {
+        match topology_accessor.get_current_topology_clone().await {
             Some(topology) => {
                 let clients = topology
                     .providers()
@@ -154,7 +158,7 @@ impl ClientRequest {
 
     async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse {
         ServerResponse::OwnDetails {
-            address: self_address_bytes.to_vec(),
+            address: self_address_bytes.to_bytes().to_vec(),
         }
     }
 }
@@ -229,7 +233,7 @@ async fn handle_connection<T: NymTopology>(
         }
         ClientRequest::Fetch => ClientRequest::handle_fetch(request_handling_data.msg_query).await,
         ClientRequest::GetClients => {
-            ClientRequest::handle_get_clients(&request_handling_data.topology).await
+            ClientRequest::handle_get_clients(request_handling_data.topology_accessor).await
         }
         ClientRequest::OwnDetails => {
             ClientRequest::handle_own_details(request_handling_data.self_address).await
@@ -241,17 +245,17 @@ async fn handle_connection<T: NymTopology>(
 
 struct RequestHandlingData<T: NymTopology> {
     msg_input: mpsc::UnboundedSender<InputMessage>,
-    msg_query: mpsc::UnboundedSender<BufferResponse>,
+    msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
+    topology_accessor: TopologyAccessor<T>,
 }
 
 async fn accept_connection<T: 'static + NymTopology>(
     mut socket: tokio::net::TcpStream,
     msg_input: mpsc::UnboundedSender<InputMessage>,
-    msg_query: mpsc::UnboundedSender<BufferResponse>,
+    msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
+    topology_accessor: TopologyAccessor<T>,
 ) {
     let address = socket
         .peer_addr()
@@ -272,7 +276,7 @@ async fn accept_connection<T: 'static + NymTopology>(
             }
             Ok(n) => {
                 let request_handling_data = RequestHandlingData {
-                    topology: topology.clone(),
+                    topology_accessor: topology_accessor.clone(),
                     msg_input: msg_input.clone(),
                     msg_query: msg_query.clone(),
                     self_address: self_address.clone(),
@@ -296,14 +300,16 @@ async fn accept_connection<T: 'static + NymTopology>(
     }
 }
 
-pub async fn start_tcpsocket<T: 'static + NymTopology>(
-    address: SocketAddr,
+pub(crate) async fn run_tcpsocket<T: 'static + NymTopology>(
+    listening_port: u16,
     message_tx: mpsc::UnboundedSender<InputMessage>,
-    received_messages_query_tx: mpsc::UnboundedSender<BufferResponse>,
+    received_messages_query_tx: mpsc::UnboundedSender<ReceivedBufferResponse>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
-) -> Result<(), TCPSocketError> {
-    let mut listener = tokio::net::TcpListener::bind(address).await?;
+    topology_accessor: TopologyAccessor<T>,
+) {
+    let address = SocketAddr::new("127.0.0.1".parse().unwrap(), listening_port);
+    info!("Starting tcp socket listener at {:?}", address);
+    let mut listener = tokio::net::TcpListener::bind(address).await.unwrap();
 
     while let Ok((stream, _)) = listener.accept().await {
         // it's fine to be cloning the channel on all new connection, because in principle
@@ -312,11 +318,28 @@ pub async fn start_tcpsocket<T: 'static + NymTopology>(
             stream,
             message_tx.clone(),
             received_messages_query_tx.clone(),
-            self_address,
-            topology.clone(),
+            self_address.clone(),
+            topology_accessor.clone(),
         ));
     }
-
-    error!("The tcpsocket went kaput...");
-    Ok(())
+}
+
+pub(crate) fn start_tcpsocket<T: 'static + NymTopology>(
+    handle: &Handle,
+    listening_port: u16,
+    message_tx: mpsc::UnboundedSender<InputMessage>,
+    received_messages_query_tx: mpsc::UnboundedSender<ReceivedBufferResponse>,
+    self_address: DestinationAddressBytes,
+    topology_accessor: TopologyAccessor<T>,
+) -> JoinHandle<()> {
+    handle.spawn(async move {
+        run_tcpsocket(
+            listening_port,
+            message_tx,
+            received_messages_query_tx,
+            self_address,
+            topology_accessor,
+        )
+        .await;
+    })
 }

nym-client/src/sockets/ws.rs

@@ -1,5 +1,5 @@
-use crate::client::received_buffer::BufferResponse;
-use crate::client::topology_control::TopologyInnerRef;
+use crate::client::received_buffer::ReceivedBufferResponse;
+use crate::client::topology_control::TopologyAccessor;
 use crate::client::InputMessage;
 use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender};
 use futures::channel::{mpsc, oneshot};
@@ -12,17 +12,19 @@ use sphinx::route::{Destination, DestinationAddressBytes};
 use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
+use tokio_tungstenite::tungstenite::protocol::{CloseFrame, Message};
 use topology::NymTopology;
-use tungstenite::protocol::frame::coding::CloseCode;
-use tungstenite::protocol::{CloseFrame, Message};
 
 struct Connection<T: NymTopology> {
     address: SocketAddr,
     msg_input: mpsc::UnboundedSender<InputMessage>,
-    msg_query: mpsc::UnboundedSender<BufferResponse>,
+    msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
     rx: UnboundedReceiver<Message>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
+    topology_accessor: TopologyAccessor<T>,
     tx: UnboundedSender<Message>,
 }
 
@@ -49,8 +51,12 @@ impl<T: NymTopology> Connection<T> {
                 ClientRequest::handle_send(message, recipient_address, self.msg_input.clone()).await
             }
             ClientRequest::Fetch => ClientRequest::handle_fetch(self.msg_query.clone()).await,
-            ClientRequest::GetClients => ClientRequest::handle_get_clients(&self.topology).await,
-            ClientRequest::OwnDetails => ClientRequest::handle_own_details(self.self_address).await,
+            ClientRequest::GetClients => {
+                ClientRequest::handle_get_clients(self.topology_accessor.clone()).await
+            }
+            ClientRequest::OwnDetails => {
+                ClientRequest::handle_own_details(self.self_address.clone()).await
+            }
         }
     }
 
@@ -194,8 +200,7 @@ impl ClientRequest {
                     "message too long. Sent {} bytes, but the maximum is {}",
                     message_bytes.len(),
                     sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH
-                )
-                .to_string(),
+                ),
             };
         }
 
@@ -219,13 +224,18 @@ impl ClientRequest {
 
         let dummy_surb = [0; 16];
 
-        let input_msg = InputMessage(Destination::new(address, dummy_surb), message_bytes);
+        let input_msg = InputMessage(
+            Destination::new(DestinationAddressBytes::from_bytes(address), dummy_surb),
+            message_bytes,
+        );
         input_tx.send(input_msg).await.unwrap();
 
         ServerResponse::Send
     }
 
-    async fn handle_fetch(mut msg_query: mpsc::UnboundedSender<BufferResponse>) -> ServerResponse {
+    async fn handle_fetch(
+        mut msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
+    ) -> ServerResponse {
         let (res_tx, res_rx) = oneshot::channel();
         if msg_query.send(res_tx).await.is_err() {
             warn!("Failed to handle_fetch. msg_query.send() is an error.");
@@ -239,7 +249,7 @@ impl ClientRequest {
             Err(e) => {
                 warn!("Failed to fetch client messages - {:?}", e);
                 return ServerResponse::Error {
-                    message: format!("Server failed to receive messages").to_string(),
+                    message: "Server failed to receive messages".to_string(),
                 };
             }
         };
@@ -261,9 +271,10 @@ impl ClientRequest {
         ServerResponse::Fetch { messages }
     }
 
-    async fn handle_get_clients<T: NymTopology>(topology: &TopologyInnerRef<T>) -> ServerResponse {
-        let topology_data = &topology.read().await.topology;
-        match topology_data {
+    async fn handle_get_clients<T: NymTopology>(
+        mut topology_accessor: TopologyAccessor<T>,
+    ) -> ServerResponse {
+        match topology_accessor.get_current_topology_clone().await {
             Some(topology) => {
                 let clients = topology
                     .providers()
@@ -280,7 +291,7 @@ impl ClientRequest {
     }
 
     async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse {
-        let self_address = bs58::encode(&self_address_bytes).into_string();
+        let self_address = self_address_bytes.to_base58_string();
         ServerResponse::OwnDetails {
             address: self_address,
         }
@@ -309,18 +320,22 @@ impl Into<Message> for ServerResponse {
 async fn accept_connection<T: 'static + NymTopology>(
     stream: tokio::net::TcpStream,
     msg_input: mpsc::UnboundedSender<InputMessage>,
-    msg_query: mpsc::UnboundedSender<BufferResponse>,
+    msg_query: mpsc::UnboundedSender<ReceivedBufferResponse>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
+    topology_accessor: TopologyAccessor<T>,
 ) {
     let address = stream
         .peer_addr()
         .expect("connected streams should have a peer address");
     debug!("Peer address: {}", address);
 
-    let mut ws_stream = tokio_tungstenite::accept_async(stream)
-        .await
-        .expect("Error during the websocket handshake occurred");
+    let mut ws_stream = match tokio_tungstenite::accept_async(stream).await {
+        Ok(ws_stream) => ws_stream,
+        Err(e) => {
+            error!("Error during the websocket handshake occurred - {}", e);
+            return;
+        }
+    };
 
     // Create a channel for our stream, which other sockets will use to
     // send us messages. Then register our address with the stream to send
@@ -331,7 +346,7 @@ async fn accept_connection<T: 'static + NymTopology>(
         address,
         rx: msg_rx,
         tx: response_tx,
-        topology,
+        topology_accessor,
         msg_input,
         msg_query,
         self_address,
@@ -349,10 +364,7 @@ async fn accept_connection<T: 'static + NymTopology>(
             }
         };
 
-        let mut should_close = false;
-        if message.is_close() {
-            should_close = true;
-        }
+        let should_close = message.is_close();
 
         if let Err(err) = msg_tx.unbounded_send(message) {
             error!(
@@ -386,14 +398,16 @@ async fn accept_connection<T: 'static + NymTopology>(
     }
 }
 
-pub async fn start_websocket<T: 'static + NymTopology>(
-    address: SocketAddr,
+pub(crate) async fn run_websocket<T: 'static + NymTopology>(
+    listening_port: u16,
     message_tx: mpsc::UnboundedSender<InputMessage>,
-    received_messages_query_tx: mpsc::UnboundedSender<BufferResponse>,
+    received_messages_query_tx: mpsc::UnboundedSender<ReceivedBufferResponse>,
     self_address: DestinationAddressBytes,
-    topology: TopologyInnerRef<T>,
-) -> Result<(), WebSocketError> {
-    let mut listener = tokio::net::TcpListener::bind(address).await?;
+    topology_accessor: TopologyAccessor<T>,
+) {
+    let address = SocketAddr::new("127.0.0.1".parse().unwrap(), listening_port);
+    info!("Starting websocket listener at {:?}", address);
+    let mut listener = tokio::net::TcpListener::bind(address).await.unwrap();
 
     while let Ok((stream, _)) = listener.accept().await {
         // it's fine to be cloning the channel on all new connection, because in principle
@@ -402,11 +416,28 @@ pub async fn start_websocket<T: 'static + NymTopology>(
             stream,
             message_tx.clone(),
             received_messages_query_tx.clone(),
-            self_address,
-            topology.clone(),
+            self_address.clone(),
+            topology_accessor.clone(),
         ));
     }
-
-    error!("The websocket went kaput...");
-    Ok(())
+}
+
+pub(crate) fn start_websocket<T: 'static + NymTopology>(
+    handle: &Handle,
+    listening_port: u16,
+    message_tx: mpsc::UnboundedSender<InputMessage>,
+    received_messages_query_tx: mpsc::UnboundedSender<ReceivedBufferResponse>,
+    self_address: DestinationAddressBytes,
+    topology_accessor: TopologyAccessor<T>,
+) -> JoinHandle<()> {
+    handle.spawn(async move {
+        run_websocket(
+            listening_port,
+            message_tx,
+            received_messages_query_tx,
+            self_address,
+            topology_accessor,
+        )
+        .await;
+    })
 }

sample-configs/validator-config.toml.sample

@@ -1,7 +1,7 @@
 [healthcheck]
 
 #directory-server = "http://localhost:8080"
-directory-server = "https://qa-directory.nymtech.net"
+directory-server = "https://directory.nymtech.net"
 interval = 10.0
 test-packets-per-node = 2 # in seconds
 resolution-timeout = 5 # in seconds

scripts/run_local_network.sh

@@ -21,24 +21,32 @@ killall nym-mixnode
 killall nym-sfw-provider
 
 echo "Press CTRL-C to stop."
+echo "Make sure you have started nym-directory !"
 
-cargo build
+cargo build --release --manifest-path nym-client/Cargo.toml --features=local
+cargo build --release --manifest-path mixnode/Cargo.toml --features=local
+cargo build --release --manifest-path sfw-provider/Cargo.toml --features=local
 
 MAX_LAYERS=3
 NUMMIXES=${1:-3} # Set $NUMMIXES to default of 3, but allow the user to set other values if desired
 
+
+$PWD/target/release/nym-sfw-provider init --id provider-local --clients-host 127.0.0.1 --mix-host 127.0.0.1 --mix-port 4000 --mix-announce-port 4000
+$PWD/target/release/nym-sfw-provider run --id provider-local &
+
+sleep 1
+
 for (( j=0; j<$NUMMIXES; j++ ))
 
 # Note: to disable logging (or direct it to another output) modify the constant on top of mixnode or provider;
 # Will make it later either configurable by flags or config file.
 do
     let layer=j%MAX_LAYERS+1
-    $PWD/target/debug/nym-mixnode run --port $((9980+$j)) --host "localhost" --layer $layer --directory http://localhost:8080 &
+    $PWD/target/release/nym-mixnode init --id mix-local$j --host 127.0.0.1 --port $((9980+$j)) --layer $layer --announce-host 127.0.0.1:$((9980+$j))
+    $PWD/target/release/nym-mixnode run --id mix-local$j &
     sleep 1
 done
 
-sleep 1
-$PWD/target/debug/nym-sfw-provider run --clientHost "localhost" --mixHost "localhost" --mixPort 9997 --clientPort 9998 --directory http://localhost:8080
 
 # trap call ctrl_c()
 trap ctrl_c SIGINT SIGTERM SIGTSTP

sfw-provider/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 build = "build.rs"
 name = "nym-sfw-provider"
-version = "0.4.1"
+version = "0.5.0-rc.1"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 edition = "2018"
 
@@ -11,6 +11,7 @@ edition = "2018"
 bs58 = "0.3.0"
 clap = "2.33.0"
 curve25519-dalek = "1.2.3"
+dirs = "2.0.2"
 dotenv = "0.15.0"
 futures = "0.3.1"
 log = "0.4"
@@ -24,11 +25,20 @@ hmac = "0.7.1"
 
 ## internal
 crypto = {path = "../common/crypto"}
+config = {path = "../common/config"}
 directory-client = { path = "../common/clients/directory-client" }
+pemstore = {path = "../common/pemstore"}
 sfw-provider-requests = { path = "./sfw-provider-requests" }
 
 ## will be moved to proper dependencies once released
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }
 
 [build-dependencies]
 built = "0.3.2"
+
+[dev-dependencies]
+tempfile = "3.1.0"
+
+[features]
+qa = []
+local = []

sfw-provider/sfw-provider-requests/Cargo.toml

@@ -7,4 +7,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" }
+bs58 = "0.3.0"
+sphinx = { git = "https://github.com/nymtech/sphinx", rev="23f9c89b257ee0936e70afd682e9ed6a62e89eee" }

sfw-provider/sfw-provider-requests/src/lib.rs

@@ -4,4 +4,113 @@ pub mod responses;
 pub const DUMMY_MESSAGE_CONTENT: &[u8] =
     b"[DUMMY MESSAGE] Wanting something does not give you the right to have it.";
 
-pub type AuthToken = [u8; 32];
+// To be renamed to 'AuthToken' once it is safe to replace it
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub struct AuthToken([u8; 32]);
+
+#[derive(Debug)]
+pub enum AuthTokenConversionError {
+    InvalidStringError,
+    StringOfInvalidLengthError,
+}
+
+impl AuthToken {
+    pub fn from_bytes(bytes: [u8; 32]) -> Self {
+        AuthToken(bytes)
+    }
+
+    pub fn to_bytes(&self) -> [u8; 32] {
+        self.0
+    }
+
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.0
+    }
+
+    pub fn try_from_base58_string<S: Into<String>>(
+        val: S,
+    ) -> Result<Self, AuthTokenConversionError> {
+        let decoded = match bs58::decode(val.into()).into_vec() {
+            Ok(decoded) => decoded,
+            Err(_) => return Err(AuthTokenConversionError::InvalidStringError),
+        };
+
+        if decoded.len() != 32 {
+            return Err(AuthTokenConversionError::StringOfInvalidLengthError);
+        }
+
+        let mut token = [0u8; 32];
+        token.copy_from_slice(&decoded[..]);
+        Ok(AuthToken(token))
+    }
+
+    pub fn to_base58_string(&self) -> String {
+        bs58::encode(self.0).into_string()
+    }
+}
+
+impl Into<String> for AuthToken {
+    fn into(self) -> String {
+        self.to_base58_string()
+    }
+}
+
+#[cfg(test)]
+mod auth_token_conversion {
+    use super::*;
+
+    #[test]
+    fn it_is_possible_to_recover_it_from_valid_b58_string() {
+        let auth_token = AuthToken([42; 32]);
+        let auth_token_string = auth_token.to_base58_string();
+        assert_eq!(
+            auth_token,
+            AuthToken::try_from_base58_string(auth_token_string).unwrap()
+        )
+    }
+
+    #[test]
+    fn it_is_possible_to_recover_it_from_valid_b58_str_ref() {
+        let auth_token = AuthToken([42; 32]);
+        let auth_token_string = auth_token.to_base58_string();
+        let auth_token_str_ref: &str = &auth_token_string;
+        assert_eq!(
+            auth_token,
+            AuthToken::try_from_base58_string(auth_token_str_ref).unwrap()
+        )
+    }
+
+    #[test]
+    fn it_returns_error_on_b58_with_invalid_characters() {
+        let auth_token = AuthToken([42; 32]);
+        let auth_token_string = auth_token.to_base58_string();
+
+        let mut chars = auth_token_string.chars();
+        let _consumed_first_char = chars.next().unwrap();
+
+        let invalid_chars_token = "=".to_string() + chars.as_str();
+        assert!(AuthToken::try_from_base58_string(invalid_chars_token).is_err())
+    }
+
+    #[test]
+    fn it_returns_error_on_too_long_b58_string() {
+        let auth_token = AuthToken([42; 32]);
+        let mut auth_token_string = auth_token.to_base58_string();
+        auth_token_string.push('f');
+
+        assert!(AuthToken::try_from_base58_string(auth_token_string).is_err())
+    }
+
+    #[test]
+    fn it_returns_error_on_too_short_b58_string() {
+        let auth_token = AuthToken([42; 32]);
+        let auth_token_string = auth_token.to_base58_string();
+
+        let mut chars = auth_token_string.chars();
+        let _consumed_first_char = chars.next().unwrap();
+        let _consumed_second_char = chars.next().unwrap();
+        let invalid_chars_token = chars.as_str();
+
+        assert!(AuthToken::try_from_base58_string(invalid_chars_token).is_err())
+    }
+}

sfw-provider/sfw-provider-requests/src/requests.rs

@@ -79,8 +79,8 @@ impl ProviderRequest for PullRequest {
         Self::get_prefix()
             .to_vec()
             .into_iter()
-            .chain(self.destination_address.iter().cloned())
-            .chain(self.auth_token.iter().cloned())
+            .chain(self.destination_address.to_bytes().iter().cloned())
+            .chain(self.auth_token.0.iter().cloned())
             .collect()
     }
 
@@ -102,8 +102,8 @@ impl ProviderRequest for PullRequest {
         auth_token.copy_from_slice(&bytes[34..]);
 
         Ok(PullRequest {
-            auth_token,
-            destination_address,
+            auth_token: AuthToken::from_bytes(auth_token),
+            destination_address: DestinationAddressBytes::from_bytes(destination_address),
         })
     }
 }
@@ -130,7 +130,7 @@ impl ProviderRequest for RegisterRequest {
         Self::get_prefix()
             .to_vec()
             .into_iter()
-            .chain(self.destination_address.iter().cloned())
+            .chain(self.destination_address.to_bytes().iter().cloned())
             .collect()
     }
 
@@ -149,7 +149,7 @@ impl ProviderRequest for RegisterRequest {
         destination_address.copy_from_slice(&bytes[2..]);
 
         Ok(RegisterRequest {
-            destination_address,
+            destination_address: DestinationAddressBytes::from_bytes(destination_address),
         })
     }
 }
@@ -160,34 +160,34 @@ mod creating_pull_request {
 
     #[test]
     fn it_is_possible_to_recover_it_from_bytes() {
-        let address = [
+        let address = DestinationAddressBytes::from_bytes([
             1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             0, 1, 2,
-        ];
+        ]);
         let auth_token = [1u8; 32];
-        let pull_request = PullRequest::new(address, auth_token);
+        let pull_request = PullRequest::new(address.clone(), AuthToken(auth_token));
         let bytes = pull_request.to_bytes();
 
         let recovered = PullRequest::from_bytes(&bytes).unwrap();
         assert_eq!(address, recovered.destination_address);
-        assert_eq!(auth_token, recovered.auth_token);
+        assert_eq!(AuthToken(auth_token), recovered.auth_token);
     }
 
     #[test]
     fn it_is_possible_to_recover_it_from_bytes_with_enum_wrapper() {
-        let address = [
+        let address = DestinationAddressBytes::from_bytes([
             1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             0, 1, 2,
-        ];
+        ]);
         let auth_token = [1u8; 32];
-        let pull_request = PullRequest::new(address, auth_token);
+        let pull_request = PullRequest::new(address.clone(), AuthToken(auth_token));
         let bytes = pull_request.to_bytes();
 
         let recovered = ProviderRequests::from_bytes(&bytes).unwrap();
         match recovered {
             ProviderRequests::PullMessages(req) => {
                 assert_eq!(address, req.destination_address);
-                assert_eq!(auth_token, req.auth_token);
+                assert_eq!(AuthToken(auth_token), req.auth_token);
             }
             _ => panic!("expected to recover pull request!"),
         }
@@ -200,11 +200,11 @@ mod creating_register_request {
 
     #[test]
     fn it_is_possible_to_recover_it_from_bytes() {
-        let address = [
+        let address = DestinationAddressBytes::from_bytes([
             1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             0, 1, 2,
-        ];
-        let register_request = RegisterRequest::new(address);
+        ]);
+        let register_request = RegisterRequest::new(address.clone());
         let bytes = register_request.to_bytes();
 
         let recovered = RegisterRequest::from_bytes(&bytes).unwrap();
@@ -213,11 +213,11 @@ mod creating_register_request {
 
     #[test]
     fn it_is_possible_to_recover_it_from_bytes_with_enum_wrapper() {
-        let address = [
+        let address = DestinationAddressBytes::from_bytes([
             1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
             0, 1, 2,
-        ];
-        let register_request = RegisterRequest::new(address);
+        ]);
+        let register_request = RegisterRequest::new(address.clone());
         let bytes = register_request.to_bytes();
 
         let recovered = ProviderRequests::from_bytes(&bytes).unwrap();

sfw-provider/sfw-provider-requests/src/responses.rs

@@ -26,6 +26,10 @@ pub struct RegisterResponse {
     pub auth_token: AuthToken,
 }
 
+pub struct ErrorResponse {
+    pub message: String,
+}
+
 impl PullResponse {
     pub fn new(messages: Vec<Vec<u8>>) -> Self {
         PullResponse { messages }
@@ -38,6 +42,14 @@ impl RegisterResponse {
     }
 }
 
+impl ErrorResponse {
+    pub fn new<S: Into<String>>(message: S) -> Self {
+        ErrorResponse {
+            message: message.into(),
+        }
+    }
+}
+
 // TODO: This should go into some kind of utils module/crate
 fn read_be_u16(input: &mut &[u8]) -> u16 {
     let (int_bytes, rest) = input.split_at(std::mem::size_of::<u16>());
@@ -72,7 +84,7 @@ impl ProviderResponse for PullResponse {
             return Err(ProviderResponseError::UnmarshalErrorInvalidLength);
         }
 
-        let mut bytes_copy = bytes.clone();
+        let mut bytes_copy = bytes;
         let num_msgs = read_be_u16(&mut bytes_copy);
 
         // can we read all lengths of messages?
@@ -109,7 +121,7 @@ impl ProviderResponse for PullResponse {
 
 impl ProviderResponse for RegisterResponse {
     fn to_bytes(&self) -> Vec<u8> {
-        self.auth_token.to_vec()
+        self.auth_token.0.to_vec()
     }
 
     fn from_bytes(bytes: &[u8]) -> Result<Self, ProviderResponseError> {
@@ -117,13 +129,28 @@ impl ProviderResponse for RegisterResponse {
             32 => {
                 let mut auth_token = [0u8; 32];
                 auth_token.copy_from_slice(&bytes[..32]);
-                Ok(RegisterResponse { auth_token })
+                Ok(RegisterResponse {
+                    auth_token: AuthToken(auth_token),
+                })
             }
             _ => Err(ProviderResponseError::UnmarshalErrorInvalidLength),
         }
     }
 }
 
+impl ProviderResponse for ErrorResponse {
+    fn to_bytes(&self) -> Vec<u8> {
+        self.message.clone().into_bytes()
+    }
+
+    fn from_bytes(bytes: &[u8]) -> Result<Self, ProviderResponseError> {
+        match String::from_utf8(bytes.to_vec()) {
+            Err(_) => Err(ProviderResponseError::UnmarshalError),
+            Ok(message) => Ok(ErrorResponse { message }),
+        }
+    }
+}
+
 #[cfg(test)]
 mod creating_pull_response {
     use super::*;

sfw-provider/src/built_info.rs

@@ -0,0 +1,2 @@
+// The file has been placed there by the build script.
+include!(concat!(env!("OUT_DIR"), "/built.rs"));

sfw-provider/src/commands/init.rs

@@ -0,0 +1,111 @@
+use crate::commands::override_config;
+use crate::config::persistence::pathfinder::ProviderPathfinder;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+use crypto::encryption;
+use pemstore::pemstore::PemStore;
+
+pub fn command_args<'a, 'b>() -> clap::App<'a, 'b> {
+    App::new("init")
+        .about("Initialise the store and forward service provider")
+        .arg(
+            Arg::with_name("id")
+                .long("id")
+                .help("Id of the sfw-provider we want to create config for.")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("mix-host")
+                .long("mix-host")
+                .help("The custom host on which the service provider will be running for receiving sphinx packets")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("mix-port")
+                .long("mix-port")
+                .help("The port on which the service provider will be listening for sphinx packets")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("clients-host")
+                .long("clients-host")
+                .help("The custom host on which the service provider will be running for receiving clients sfw-provider-requests")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("clients-port")
+                .long("clients-port")
+                .help("The port on which the service provider will be listening for clients sfw-provider-requests")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("mix-announce-host")
+                .long("mix-announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("mix-announce-port")
+                .long("mix-announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("clients-announce-host")
+                .long("clients-announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("clients-announce-port")
+                .long("clients-announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("inboxes")
+                .long("inboxes")
+                .help("Directory with inboxes where all packets for the clients are stored")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("clients-ledger")
+                .long("clients-ledger")
+                .help("[UNIMPLEMENTED] Ledger file containing registered clients")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("directory")
+                .long("directory")
+                .help("Address of the directory server the node is sending presence data to")
+                .takes_value(true),
+        )
+}
+
+pub fn execute(matches: &ArgMatches) {
+    let id = matches.value_of("id").unwrap();
+    println!("Initialising sfw service provider {}...", id);
+
+    let mut config = crate::config::Config::new(id);
+
+    config = override_config(config, matches);
+
+    let sphinx_keys = encryption::KeyPair::new();
+    let pathfinder = ProviderPathfinder::new_from_config(&config);
+    let pem_store = PemStore::new(pathfinder);
+    pem_store
+        .write_encryption_keys(sphinx_keys)
+        .expect("Failed to save sphinx keys");
+    println!("Saved mixnet sphinx keypair");
+
+    let config_save_location = config.get_config_file_save_location();
+    config
+        .save_to_file(None)
+        .expect("Failed to save the config file");
+    println!("Saved configuration file to {:?}", config_save_location);
+
+    println!("Service provider configuration completed.\n\n\n")
+}

sfw-provider/src/commands/mod.rs

@@ -0,0 +1,78 @@
+use crate::config::Config;
+use clap::ArgMatches;
+
+pub mod init;
+pub mod run;
+
+pub(crate) fn override_config(mut config: Config, matches: &ArgMatches) -> Config {
+    if let Some(mix_host) = matches.value_of("mix-host") {
+        config = config.with_mix_listening_host(mix_host);
+    }
+
+    if let Some(mix_port) = matches.value_of("mix-port").map(|port| port.parse::<u16>()) {
+        if let Err(err) = mix_port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_mix_listening_port(mix_port.unwrap());
+    }
+
+    if let Some(clients_host) = matches.value_of("clients-host") {
+        config = config.with_clients_listening_host(clients_host);
+    }
+
+    if let Some(clients_port) = matches
+        .value_of("clients-port")
+        .map(|port| port.parse::<u16>())
+    {
+        if let Err(err) = clients_port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_clients_listening_port(clients_port.unwrap());
+    }
+
+    if let Some(mix_announce_host) = matches.value_of("mix-announce-host") {
+        config = config.with_mix_announce_host(mix_announce_host);
+    }
+
+    if let Some(mix_announce_port) = matches
+        .value_of("mix-announce-port")
+        .map(|port| port.parse::<u16>())
+    {
+        if let Err(err) = mix_announce_port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_mix_announce_port(mix_announce_port.unwrap());
+    }
+
+    if let Some(clients_announce_host) = matches.value_of("clients-announce-host") {
+        config = config.with_clients_announce_host(clients_announce_host);
+    }
+
+    if let Some(clients_announce_port) = matches
+        .value_of("clients-announce-port")
+        .map(|port| port.parse::<u16>())
+    {
+        if let Err(err) = clients_announce_port {
+            // if port was overridden, it must be parsable
+            panic!("Invalid port value provided - {:?}", err);
+        }
+        config = config.with_clients_announce_port(clients_announce_port.unwrap());
+    }
+
+    if let Some(directory) = matches.value_of("directory") {
+        config = config.with_custom_directory(directory);
+    }
+
+    if let Some(inboxes_dir) = matches.value_of("inboxes") {
+        config = config.with_custom_clients_inboxes(inboxes_dir);
+    }
+
+    if let Some(clients_ledger) = matches.value_of("clients-ledger") {
+        config = config.with_custom_clients_ledger(clients_ledger);
+    }
+
+    config
+}

sfw-provider/src/commands/run.rs

@@ -0,0 +1,161 @@
+use crate::commands::override_config;
+use crate::config::Config;
+use crate::provider::ServiceProvider;
+use clap::{App, Arg, ArgMatches};
+use config::NymConfig;
+
+pub fn command_args<'a, 'b>() -> clap::App<'a, 'b> {
+    App::new("run")
+        .about("Starts the service provider")
+        .arg(
+            Arg::with_name("id")
+                .long("id")
+                .help("Id of the nym-mixnode we want to run")
+                .takes_value(true)
+                .required(true),
+        )
+        // the rest of arguments are optional, they are used to override settings in config file
+        .arg(
+            Arg::with_name("config")
+                .long("config")
+                .help("Custom path to the nym-provider configuration file")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("mix-host")
+                .long("mix-host")
+                .help("The custom host on which the service provider will be running for receiving sphinx packets")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("mix-port")
+                .long("mix-port")
+                .help("The port on which the service provider will be listening for sphinx packets")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("clients-host")
+                .long("clients-host")
+                .help("The custom host on which the service provider will be running for receiving clients sfw-provider-requests")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("clients-port")
+                .long("clients-port")
+                .help("The port on which the service provider will be listening for clients sfw-provider-requests")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("mix-announce-host")
+                .long("mix-announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("mix-announce-port")
+                .long("mix-announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("clients-announce-host")
+                .long("clients-announce-host")
+                .help("The host that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("clients-announce-port")
+                .long("clients-announce-port")
+                .help("The port that will be reported to the directory server")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("inboxes")
+                .long("inboxes")
+                .help("Directory with inboxes where all packets for the clients are stored")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("clients-ledger")
+                .long("clients-ledger")
+                .help("[UNIMPLEMENTED] Ledger file containing registered clients")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("directory")
+                .long("directory")
+                .help("Address of the directory server the node is sending presence data to")
+                .takes_value(true),
+        )
+}
+
+fn show_binding_warning(address: String) {
+    println!("\n##### WARNING #####");
+    println!(
+        "\nYou are trying to bind to {} - you might not be accessible to other nodes\n\
+         You can ignore this warning if you're running setup on a local network \n\
+         or have set a custom 'announce-host'",
+        address
+    );
+    println!("\n##### WARNING #####\n");
+}
+
+fn special_addresses() -> Vec<&'static str> {
+    vec!["localhost", "127.0.0.1", "0.0.0.0", "::1", "[::1]"]
+}
+
+pub fn execute(matches: &ArgMatches) {
+    let id = matches.value_of("id").unwrap();
+
+    println!("Starting sfw-provider {}...", id);
+
+    let mut config =
+        Config::load_from_file(matches.value_of("config").map(|path| path.into()), Some(id))
+            .expect("Failed to load config file");
+
+    config = override_config(config, matches);
+
+    let mix_listening_ip_string = config.get_mix_listening_address().ip().to_string();
+    if special_addresses().contains(&mix_listening_ip_string.as_ref()) {
+        show_binding_warning(mix_listening_ip_string);
+    }
+
+    let clients_listening_ip_string = config.get_clients_listening_address().ip().to_string();
+    if special_addresses().contains(&clients_listening_ip_string.as_ref()) {
+        show_binding_warning(clients_listening_ip_string);
+    }
+
+    println!(
+        "Directory server [presence]: {}",
+        config.get_presence_directory_server()
+    );
+
+    println!(
+        "Listening for incoming sphinx packets on {}",
+        config.get_mix_listening_address()
+    );
+    println!(
+        "Announcing the following socket address for sphinx packets: {}",
+        config.get_mix_announce_address()
+    );
+
+    println!(
+        "Listening for incoming clients packets on {}",
+        config.get_clients_listening_address()
+    );
+    println!(
+        "Announcing the following socket address for clients packets: {}",
+        config.get_clients_announce_address()
+    );
+
+    println!(
+        "Inboxes directory is: {:?}",
+        config.get_clients_inboxes_dir()
+    );
+    println!(
+        "[UNIMPLEMENTED] Registered client ledger is: {:?}",
+        config.get_clients_ledger_path()
+    );
+
+    ServiceProvider::new(config).run();
+}

sfw-provider/src/config/mod.rs

@@ -0,0 +1,497 @@
+use crate::config::template::config_template;
+use config::NymConfig;
+use log::*;
+use serde::{Deserialize, Serialize};
+use std::net::{IpAddr, SocketAddr};
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::time;
+
+pub mod persistence;
+mod template;
+
+// 'PROVIDER'
+const DEFAULT_MIX_LISTENING_PORT: u16 = 1789;
+const DEFAULT_CLIENT_LISTENING_PORT: u16 = 9000;
+// 'DEBUG'
+// where applicable, the below are defined in milliseconds
+const DEFAULT_PRESENCE_SENDING_DELAY: u64 = 3000;
+
+const DEFAULT_STORED_MESSAGE_FILENAME_LENGTH: u16 = 16;
+const DEFAULT_MESSAGE_RETRIEVAL_LIMIT: u16 = 5;
+
+#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Config {
+    provider: Provider,
+
+    mixnet_endpoint: MixnetEndpoint,
+
+    clients_endpoint: ClientsEndpoint,
+
+    #[serde(default)]
+    logging: Logging,
+    #[serde(default)]
+    debug: Debug,
+}
+
+impl NymConfig for Config {
+    fn template() -> &'static str {
+        config_template()
+    }
+
+    fn config_file_name() -> String {
+        "config.toml".to_string()
+    }
+
+    fn default_root_directory() -> PathBuf {
+        dirs::home_dir()
+            .expect("Failed to evaluate $HOME value")
+            .join(".nym")
+            .join("sfw-providers")
+    }
+
+    fn root_directory(&self) -> PathBuf {
+        self.provider.nym_root_directory.clone()
+    }
+
+    fn config_directory(&self) -> PathBuf {
+        self.provider
+            .nym_root_directory
+            .join(&self.provider.id)
+            .join("config")
+    }
+
+    fn data_directory(&self) -> PathBuf {
+        self.provider
+            .nym_root_directory
+            .join(&self.provider.id)
+            .join("data")
+    }
+}
+
+impl Config {
+    pub fn new<S: Into<String>>(id: S) -> Self {
+        Config::default().with_id(id)
+    }
+
+    // builder methods
+    pub fn with_id<S: Into<String>>(mut self, id: S) -> Self {
+        let id = id.into();
+        if self.provider.private_sphinx_key_file.as_os_str().is_empty() {
+            self.provider.private_sphinx_key_file =
+                self::Provider::default_private_sphinx_key_file(&id);
+        }
+        if self.provider.public_sphinx_key_file.as_os_str().is_empty() {
+            self.provider.public_sphinx_key_file =
+                self::Provider::default_public_sphinx_key_file(&id);
+        }
+        if self
+            .clients_endpoint
+            .inboxes_directory
+            .as_os_str()
+            .is_empty()
+        {
+            self.clients_endpoint.inboxes_directory =
+                self::ClientsEndpoint::default_inboxes_directory(&id);
+        }
+        if self.clients_endpoint.ledger_path.as_os_str().is_empty() {
+            self.clients_endpoint.ledger_path = self::ClientsEndpoint::default_ledger_path(&id);
+        }
+
+        self.provider.id = id;
+        self
+    }
+
+    pub fn with_custom_directory<S: Into<String>>(mut self, directory_server: S) -> Self {
+        self.debug.presence_directory_server = directory_server.into();
+        self
+    }
+
+    pub fn with_mix_listening_host<S: Into<String>>(mut self, host: S) -> Self {
+        // see if the provided `host` is just an ip address or ip:port
+        let host = host.into();
+
+        // is it ip:port?
+        match SocketAddr::from_str(host.as_ref()) {
+            Ok(socket_addr) => {
+                self.mixnet_endpoint.listening_address = socket_addr;
+                self
+            }
+            // try just for ip
+            Err(_) => match IpAddr::from_str(host.as_ref()) {
+                Ok(ip_addr) => {
+                    self.mixnet_endpoint.listening_address.set_ip(ip_addr);
+                    self
+                }
+                Err(_) => {
+                    error!(
+                        "failed to make any changes to config - invalid host {}",
+                        host
+                    );
+                    self
+                }
+            },
+        }
+    }
+
+    pub fn with_mix_listening_port(mut self, port: u16) -> Self {
+        self.mixnet_endpoint.listening_address.set_port(port);
+        self
+    }
+
+    pub fn with_mix_announce_host<S: Into<String>>(mut self, host: S) -> Self {
+        // this is slightly more complicated as we store announce information as String,
+        // since it might not necessarily be a valid SocketAddr (say `nymtech.net:8080` is a valid
+        // announce address, yet invalid SocketAddr`
+
+        // first lets see if we received host:port or just host part of an address
+        let host = host.into();
+        let split_host: Vec<_> = host.split(':').collect();
+        match split_host.len() {
+            1 => {
+                // we provided only 'host' part so we are going to reuse existing port
+                self.mixnet_endpoint.announce_address =
+                    format!("{}:{}", host, self.mixnet_endpoint.listening_address.port());
+                self
+            }
+            2 => {
+                // we provided 'host:port' so just put the whole thing there
+                self.mixnet_endpoint.announce_address = host;
+                self
+            }
+            _ => {
+                // we provided something completely invalid, so don't try to parse it
+                error!(
+                    "failed to make any changes to config - invalid announce host {}",
+                    host
+                );
+                self
+            }
+        }
+    }
+
+    pub fn with_mix_announce_port(mut self, port: u16) -> Self {
+        let current_host: Vec<_> = self.mixnet_endpoint.announce_address.split(':').collect();
+        debug_assert_eq!(current_host.len(), 2);
+        self.mixnet_endpoint.announce_address = format!("{}:{}", current_host[0], port);
+        self
+    }
+
+    pub fn with_clients_listening_host<S: Into<String>>(mut self, host: S) -> Self {
+        // see if the provided `host` is just an ip address or ip:port
+        let host = host.into();
+
+        // is it ip:port?
+        match SocketAddr::from_str(host.as_ref()) {
+            Ok(socket_addr) => {
+                self.clients_endpoint.listening_address = socket_addr;
+                self
+            }
+            // try just for ip
+            Err(_) => match IpAddr::from_str(host.as_ref()) {
+                Ok(ip_addr) => {
+                    self.clients_endpoint.listening_address.set_ip(ip_addr);
+                    self
+                }
+                Err(_) => {
+                    error!(
+                        "failed to make any changes to config - invalid host {}",
+                        host
+                    );
+                    self
+                }
+            },
+        }
+    }
+
+    pub fn with_clients_listening_port(mut self, port: u16) -> Self {
+        self.clients_endpoint.listening_address.set_port(port);
+        self
+    }
+
+    pub fn with_clients_announce_host<S: Into<String>>(mut self, host: S) -> Self {
+        // this is slightly more complicated as we store announce information as String,
+        // since it might not necessarily be a valid SocketAddr (say `nymtech.net:8080` is a valid
+        // announce address, yet invalid SocketAddr`
+
+        // first lets see if we received host:port or just host part of an address
+        let host = host.into();
+        let split_host: Vec<_> = host.split(':').collect();
+        match split_host.len() {
+            1 => {
+                // we provided only 'host' part so we are going to reuse existing port
+                self.clients_endpoint.announce_address = format!(
+                    "{}:{}",
+                    host,
+                    self.clients_endpoint.listening_address.port()
+                );
+                self
+            }
+            2 => {
+                // we provided 'host:port' so just put the whole thing there
+                self.clients_endpoint.announce_address = host;
+                self
+            }
+            _ => {
+                // we provided something completely invalid, so don't try to parse it
+                error!(
+                    "failed to make any changes to config - invalid announce host {}",
+                    host
+                );
+                self
+            }
+        }
+    }
+
+    pub fn with_clients_announce_port(mut self, port: u16) -> Self {
+        let current_host: Vec<_> = self.clients_endpoint.announce_address.split(':').collect();
+        debug_assert_eq!(current_host.len(), 2);
+        self.clients_endpoint.announce_address = format!("{}:{}", current_host[0], port);
+        self
+    }
+
+    pub fn with_custom_clients_inboxes<S: Into<String>>(mut self, inboxes_dir: S) -> Self {
+        self.clients_endpoint.inboxes_directory = PathBuf::from(inboxes_dir.into());
+        self
+    }
+
+    pub fn with_custom_clients_ledger<S: Into<String>>(mut self, ledger_path: S) -> Self {
+        self.clients_endpoint.ledger_path = PathBuf::from(ledger_path.into());
+        self
+    }
+
+    // getters
+    pub fn get_config_file_save_location(&self) -> PathBuf {
+        self.config_directory().join(Self::config_file_name())
+    }
+
+    pub fn get_private_sphinx_key_file(&self) -> PathBuf {
+        self.provider.private_sphinx_key_file.clone()
+    }
+
+    pub fn get_public_sphinx_key_file(&self) -> PathBuf {
+        self.provider.public_sphinx_key_file.clone()
+    }
+
+    pub fn get_presence_directory_server(&self) -> String {
+        self.debug.presence_directory_server.clone()
+    }
+
+    pub fn get_presence_sending_delay(&self) -> time::Duration {
+        time::Duration::from_millis(self.debug.presence_sending_delay)
+    }
+
+    pub fn get_mix_listening_address(&self) -> SocketAddr {
+        self.mixnet_endpoint.listening_address
+    }
+
+    pub fn get_mix_announce_address(&self) -> String {
+        self.mixnet_endpoint.announce_address.clone()
+    }
+
+    pub fn get_clients_listening_address(&self) -> SocketAddr {
+        self.clients_endpoint.listening_address
+    }
+
+    pub fn get_clients_announce_address(&self) -> String {
+        self.clients_endpoint.announce_address.clone()
+    }
+
+    pub fn get_clients_inboxes_dir(&self) -> PathBuf {
+        self.clients_endpoint.inboxes_directory.clone()
+    }
+
+    pub fn get_clients_ledger_path(&self) -> PathBuf {
+        self.clients_endpoint.ledger_path.clone()
+    }
+
+    pub fn get_message_retrieval_limit(&self) -> u16 {
+        self.debug.message_retrieval_limit
+    }
+
+    pub fn get_stored_messages_filename_length(&self) -> u16 {
+        self.debug.stored_messages_filename_length
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Provider {
+    /// ID specifies the human readable ID of this particular provider.
+    id: String,
+
+    /// Path to file containing private sphinx key.
+    private_sphinx_key_file: PathBuf,
+
+    /// Path to file containing public sphinx key.
+    public_sphinx_key_file: PathBuf,
+
+    /// nym_home_directory specifies absolute path to the home nym service providers directory.
+    /// It is expected to use default value and hence .toml file should not redefine this field.
+    nym_root_directory: PathBuf,
+}
+
+impl Provider {
+    fn default_private_sphinx_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("private_sphinx.pem")
+    }
+
+    fn default_public_sphinx_key_file(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("public_sphinx.pem")
+    }
+}
+
+impl Default for Provider {
+    fn default() -> Self {
+        Provider {
+            id: "".to_string(),
+            private_sphinx_key_file: Default::default(),
+            public_sphinx_key_file: Default::default(),
+            nym_root_directory: Config::default_root_directory(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct MixnetEndpoint {
+    /// Socket address to which this service provider will bind to
+    /// and will be listening for sphinx packets coming from the mixnet.
+    listening_address: SocketAddr,
+
+    /// Optional address announced to the directory server for the clients to connect to.
+    /// It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+    /// later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+    /// Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+    /// are valid announce addresses, while the later will default to whatever port is used for
+    /// `listening_address`.
+    announce_address: String,
+}
+
+impl Default for MixnetEndpoint {
+    fn default() -> Self {
+        MixnetEndpoint {
+            listening_address: format!("0.0.0.0:{}", DEFAULT_MIX_LISTENING_PORT)
+                .parse()
+                .unwrap(),
+            announce_address: format!("127.0.0.1:{}", DEFAULT_MIX_LISTENING_PORT),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct ClientsEndpoint {
+    /// Socket address to which this service provider will bind to
+    /// and will be listening for data packets coming from the clients.
+    listening_address: SocketAddr,
+
+    /// Optional address announced to the directory server for the clients to connect to.
+    /// It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+    /// later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+    /// Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+    /// are valid announce addresses, while the later will default to whatever port is used for
+    /// `listening_address`.
+    announce_address: String,
+
+    /// Path to the directory with clients inboxes containing messages stored for them.
+    inboxes_directory: PathBuf,
+
+    /// [TODO: implement its storage] Full path to a file containing mapping of
+    /// client addresses to their access tokens.
+    ledger_path: PathBuf,
+}
+
+impl ClientsEndpoint {
+    fn default_inboxes_directory(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("inboxes")
+    }
+
+    fn default_ledger_path(id: &str) -> PathBuf {
+        Config::default_data_directory(Some(id)).join("client_ledger.dat")
+    }
+}
+
+impl Default for ClientsEndpoint {
+    fn default() -> Self {
+        ClientsEndpoint {
+            listening_address: format!("0.0.0.0:{}", DEFAULT_CLIENT_LISTENING_PORT)
+                .parse()
+                .unwrap(),
+            announce_address: format!("127.0.0.1:{}", DEFAULT_CLIENT_LISTENING_PORT),
+            inboxes_directory: Default::default(),
+            ledger_path: Default::default(),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Logging {}
+
+impl Default for Logging {
+    fn default() -> Self {
+        Logging {}
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Debug {
+    /// Directory server to which the server will be reporting their presence data.
+    presence_directory_server: String,
+
+    /// Delay between each subsequent presence data being sent.
+    presence_sending_delay: u64,
+
+    /// Length of filenames for new client messages.
+    stored_messages_filename_length: u16,
+
+    /// number of messages client gets on each request
+    /// if there are no real messages, dummy ones are create to always return  
+    /// `message_retrieval_limit` total messages
+    message_retrieval_limit: u16,
+}
+
+impl Debug {
+    fn default_directory_server() -> String {
+        #[cfg(feature = "qa")]
+        return "https://qa-directory.nymtech.net".to_string();
+        #[cfg(feature = "local")]
+        return "http://localhost:8080".to_string();
+
+        "https://directory.nymtech.net".to_string()
+    }
+}
+
+impl Default for Debug {
+    fn default() -> Self {
+        Debug {
+            presence_directory_server: Self::default_directory_server(),
+            presence_sending_delay: DEFAULT_PRESENCE_SENDING_DELAY,
+            stored_messages_filename_length: DEFAULT_STORED_MESSAGE_FILENAME_LENGTH,
+            message_retrieval_limit: DEFAULT_MESSAGE_RETRIEVAL_LIMIT,
+        }
+    }
+}
+
+#[cfg(test)]
+mod provider_config {
+    use super::*;
+
+    #[test]
+    fn after_saving_default_config_the_loaded_one_is_identical() {
+        // need to figure out how to do something similar but without touching the disk
+        // or the file system at all...
+        let temp_location = tempfile::tempdir().unwrap().path().join("config.toml");
+        let default_config = Config::default().with_id("foomp".to_string());
+        default_config
+            .save_to_file(Some(temp_location.clone()))
+            .unwrap();
+
+        let loaded_config = Config::load_from_file(Some(temp_location), None).unwrap();
+
+        assert_eq!(default_config, loaded_config);
+    }
+}

sfw-provider/src/config/persistence/mod.rs

@@ -0,0 +1 @@
+pub mod pathfinder;

sfw-provider/src/config/persistence/pathfinder.rs

@@ -0,0 +1,44 @@
+use crate::config::Config;
+use pemstore::pathfinder::PathFinder;
+use std::path::PathBuf;
+
+#[derive(Debug)]
+pub struct ProviderPathfinder {
+    pub config_dir: PathBuf,
+    pub private_sphinx_key: PathBuf,
+    pub public_sphinx_key: PathBuf,
+}
+
+impl ProviderPathfinder {
+    pub fn new_from_config(config: &Config) -> Self {
+        ProviderPathfinder {
+            config_dir: config.get_config_file_save_location(),
+            private_sphinx_key: config.get_private_sphinx_key_file(),
+            public_sphinx_key: config.get_public_sphinx_key_file(),
+        }
+    }
+}
+
+impl PathFinder for ProviderPathfinder {
+    fn config_dir(&self) -> PathBuf {
+        self.config_dir.clone()
+    }
+
+    fn private_identity_key(&self) -> PathBuf {
+        // TEMPORARILY USE SAME KEYS AS ENCRYPTION
+        self.private_sphinx_key.clone()
+    }
+
+    fn public_identity_key(&self) -> PathBuf {
+        // TEMPORARILY USE SAME KEYS AS ENCRYPTION
+        self.public_sphinx_key.clone()
+    }
+
+    fn private_encryption_key(&self) -> Option<PathBuf> {
+        Some(self.private_sphinx_key.clone())
+    }
+
+    fn public_encryption_key(&self) -> Option<PathBuf> {
+        Some(self.public_sphinx_key.clone())
+    }
+}

sfw-provider/src/config/template.rs

@@ -0,0 +1,94 @@
+pub(crate) fn config_template() -> &'static str {
+    // While using normal toml marshalling would have been way simpler with less overhead,
+    // I think it's useful to have comments attached to the saved config file to explain behaviour of
+    // particular fields.
+    // Note: any changes to the template must be reflected in the appropriate structs in mod.rs.
+    r#"
+# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+##### main base mixnode config options #####
+
+[provider]
+# Human readable ID of this particular service provider.
+id = "{{ provider.id }}"
+
+# Path to file containing private sphinx key.
+private_sphinx_key_file = "{{ provider.private_sphinx_key_file }}"
+
+# Path to file containing public sphinx key.
+public_sphinx_key_file = "{{ provider.public_sphinx_key_file }}"
+
+# nym_home_directory specifies absolute path to the home nym service providers directory.
+# It is expected to use default value and hence .toml file should not redefine this field.
+nym_root_directory = "{{ provider.nym_root_directory }}"
+    
+
+##### Mixnet endpoint config options #####
+
+[mixnet_endpoint]
+# Socket address to which this service provider will bind to
+# and will be listening for sphinx packets coming from the mixnet.
+listening_address = "{{ mixnet_endpoint.listening_address }}"
+
+# Optional address announced to the directory server for the clients to connect to.
+# It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+# later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+# Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+# are valid announce addresses, while the later will default to whatever port is used for
+# `listening_address`.
+announce_address = "{{ mixnet_endpoint.announce_address }}"
+
+
+#### Clients endpoint config options #####
+
+[clients_endpoint]
+# Socket address to which this service provider will bind to
+# and will be listening for sphinx packets coming from the mixnet.
+listening_address = "{{ clients_endpoint.listening_address }}"
+
+# Optional address announced to the directory server for the clients to connect to.
+# It is useful, say, in NAT scenarios or wanting to more easily update actual IP address
+# later on by using name resolvable with a DNS query, such as `nymtech.net:8080`.
+# Additionally a custom port can be provided, so both `nymtech.net:8080` and `nymtech.net`
+# are valid announce addresses, while the later will default to whatever port is used for
+# `listening_address`.
+announce_address = "{{ clients_endpoint.announce_address }}"
+
+# Path to the directory with clients inboxes containing messages stored for them.
+inboxes_directory = "{{ clients_endpoint.inboxes_directory }}"
+
+# [TODO: implement its storage] Full path to a file containing mapping of
+# client addresses to their access tokens.
+ledger_path = "{{ clients_endpoint.ledger_path }}"
+
+
+##### logging configuration options #####
+
+[logging]
+
+# TODO
+
+
+##### debug configuration options #####
+# The following options should not be modified unless you know EXACTLY what you are doing
+# as if set incorrectly, they may impact your anonymity.
+
+[debug]
+
+# Directory server to which the server will be reporting their presence data.
+presence_directory_server = "{{ debug.presence_directory_server}}"
+
+# Delay between each subsequent presence data being sent.
+presence_sending_delay = {{ debug.presence_sending_delay }}
+
+# Length of filenames for new client messages.
+stored_messages_filename_length = {{ debug.stored_messages_filename_length }}
+
+# number of messages client gets on each request
+# if there are no real messages, dummy ones are create to always return  
+# `message_retrieval_limit` total messages
+message_retrieval_limit = {{ debug.message_retrieval_limit }}
+
+"#
+}

sfw-provider/src/main.rs

@@ -1,178 +1,32 @@
-use crate::provider::ServiceProvider;
-use clap::{App, Arg, ArgMatches, SubCommand};
-use crypto::identity::MixnetIdentityKeyPair;
-use log::error;
-use std::net::ToSocketAddrs;
-use std::path::PathBuf;
-use std::process;
+use clap::{App, ArgMatches};
 
+pub mod built_info;
+mod commands;
+mod config;
 pub mod provider;
 
 fn main() {
     dotenv::dotenv().ok();
     pretty_env_logger::init();
 
+    println!("{}", banner());
+
     let arg_matches = App::new("Nym Service Provider")
         .version(built_info::PKG_VERSION)
         .author("Nymtech")
         .about("Implementation of the Loopix-based Service Provider")
-        .subcommand(
-            SubCommand::with_name("run")
-                .about("Starts the service provider")
-                .arg(
-                    Arg::with_name("mixHost")
-                        .long("mixHost")
-                        .help("The custom host on which the service provider will be running for receiving sphinx packets")
-                        .takes_value(true)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("mixPort")
-                        .long("mixPort")
-                        .help("The port on which the service provider will be listening for sphinx packets")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("clientHost")
-                        .long("clientHost")
-                        .help("The custom host on which the service provider will be running for receiving client sfw-provider-requests")
-                        .takes_value(true)
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("clientPort")
-                        .long("clientPort")
-                        .help("The port on which the service provider will be listening for client sfw-provider-requests")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("storeDir")
-                        .short("s")
-                        .long("storeDir")
-                        .help("Directory storing all packets for the clients")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("registeredLedger")
-                        .short("r")
-                        .long("registeredLedger")
-                        .help("Directory of the ledger of registered clients")
-                        .takes_value(true)
-                )
-                .arg(
-                    Arg::with_name("directory")
-                        .long("directory")
-                        .help("Address of the directory server the node is sending presence and metrics to")
-                        .takes_value(true),
-                ),
-        )
+        .subcommand(commands::init::command_args())
+        .subcommand(commands::run::command_args())
         .get_matches();
 
-    if let Err(e) = execute(arg_matches) {
-        error!("{}", e);
-        process::exit(1);
-    }
+    execute(arg_matches);
 }
 
-fn print_binding_warning(address: &str) {
-    println!("\n##### WARNING #####");
-    println!(
-        "\nYou are trying to bind to {} - you might not be accessible to other nodes",
-        address
-    );
-    println!("\n##### WARNING #####\n");
-}
-
-fn run(matches: &ArgMatches) {
-    println!("{}", banner());
-    let config = new_config(matches);
-    let provider = ServiceProvider::new(config);
-
-    provider.start().unwrap()
-}
-
-fn new_config(matches: &ArgMatches) -> provider::Config {
-    let directory_server = matches
-        .value_of("directory")
-        .unwrap_or("https://directory.nymtech.net")
-        .to_string();
-
-    let mix_host = matches.value_of("mixHost").unwrap();
-    let mix_port = match matches.value_of("mixPort").unwrap_or("8085").parse::<u16>() {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid mix host port value provided - {:?}", err),
-    };
-
-    let client_host = matches.value_of("clientHost").unwrap();
-    let client_port = match matches
-        .value_of("clientPort")
-        .unwrap_or("9000")
-        .parse::<u16>()
-    {
-        Ok(n) => n,
-        Err(err) => panic!("Invalid client port value provided - {:?}", err),
-    };
-
-    if mix_host == "localhost" || mix_host == "127.0.0.1" || mix_host == "0.0.0.0" {
-        print_binding_warning(mix_host);
-    }
-
-    if client_host == "localhost" || client_host == "127.0.0.1" || client_host == "0.0.0.0" {
-        print_binding_warning(client_host);
-    }
-
-    let key_pair = crypto::identity::DummyMixIdentityKeyPair::new(); // TODO: persist this so keypairs don't change every restart
-    let store_dir = PathBuf::from(
-        matches
-            .value_of("storeDir")
-            .unwrap_or("/tmp/nym-provider/inboxes"),
-    );
-    let registered_client_ledger_dir = PathBuf::from(
-        matches
-            .value_of("registeredLedger")
-            .unwrap_or("/tmp/nym-provider/registered_clients"),
-    );
-
-    println!("store_dir is: {:?}", store_dir);
-    println!(
-        "registered_client_ledger_dir is: {:?}",
-        registered_client_ledger_dir
-    );
-
-    let mix_socket_address = (mix_host, mix_port)
-        .to_socket_addrs()
-        .expect("Failed to combine host and port")
-        .next()
-        .expect("Failed to extract the socket address from the iterator");
-
-    let client_socket_address = (client_host, client_port)
-        .to_socket_addrs()
-        .expect("Failed to combine host and port")
-        .next()
-        .expect("Failed to extract the socket address from the iterator");
-
-    println!("Listening for mixnet packets on {}", mix_socket_address);
-    println!("Listening for client requests on {}", client_socket_address);
-
-    provider::Config {
-        mix_socket_address,
-        directory_server,
-        public_key: key_pair.public_key,
-        client_socket_address,
-        secret_key: key_pair.private_key,
-        store_dir: PathBuf::from(store_dir),
-    }
-}
-
-pub mod built_info {
-    // The file has been placed there by the build script.
-    include!(concat!(env!("OUT_DIR"), "/built.rs"));
-}
-
-fn execute(matches: ArgMatches) -> Result<(), String> {
+fn execute(matches: ArgMatches) {
     match matches.subcommand() {
-        ("run", Some(m)) => Ok(run(m)),
-        _ => Err(usage()),
+        ("init", Some(m)) => commands::init::execute(m),
+        ("run", Some(m)) => commands::run::execute(m),
+        _ => println!("{}", usage()),
     }
 }
 

sfw-provider/src/provider/client_handling/ledger.rs

@@ -0,0 +1,74 @@
+use directory_client::presence::providers::MixProviderClient;
+use futures::lock::Mutex;
+use sfw_provider_requests::AuthToken;
+use sphinx::route::DestinationAddressBytes;
+use std::collections::HashMap;
+use std::io;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+// Note: you should NEVER create more than a single instance of this using 'new()'.
+// You should always use .clone() to create additional instances
+pub struct ClientLedger {
+    inner: Arc<Mutex<ClientLedgerInner>>,
+}
+
+impl ClientLedger {
+    pub(crate) fn new() -> Self {
+        ClientLedger {
+            inner: Arc::new(Mutex::new(ClientLedgerInner(HashMap::new()))),
+        }
+    }
+
+    pub(crate) async fn verify_token(
+        &self,
+        auth_token: &AuthToken,
+        client_address: &DestinationAddressBytes,
+    ) -> bool {
+        match self.inner.lock().await.0.get(client_address) {
+            None => false,
+            Some(expected_token) => expected_token == auth_token,
+        }
+    }
+
+    pub(crate) async fn insert_token(
+        &mut self,
+        auth_token: AuthToken,
+        client_address: DestinationAddressBytes,
+    ) -> Option<AuthToken> {
+        self.inner.lock().await.0.insert(client_address, auth_token)
+    }
+
+    pub(crate) async fn remove_token(
+        &mut self,
+        client_address: &DestinationAddressBytes,
+    ) -> Option<AuthToken> {
+        self.inner.lock().await.0.remove(client_address)
+    }
+
+    pub(crate) async fn current_clients(&self) -> Vec<MixProviderClient> {
+        self.inner
+            .lock()
+            .await
+            .0
+            .keys()
+            .map(|client_address| client_address.to_base58_string())
+            .map(|pub_key| MixProviderClient { pub_key })
+            .collect()
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn load(_file: PathBuf) -> Self {
+        // TODO: actual loading,
+        // temporarily just create a new one
+        Self::new()
+    }
+
+    #[allow(dead_code)]
+    pub(crate) async fn save(&self, _file: PathBuf) -> io::Result<()> {
+        unimplemented!()
+    }
+}
+
+struct ClientLedgerInner(HashMap<DestinationAddressBytes, AuthToken>);

sfw-provider/src/provider/client_handling/listener.rs

@@ -0,0 +1,95 @@
+use crate::provider::client_handling::request_processing::{
+    ClientProcessingResult, RequestProcessor,
+};
+use log::*;
+use sfw_provider_requests::responses::{
+    ErrorResponse, ProviderResponse, PullResponse, RegisterResponse,
+};
+use std::io;
+use std::net::SocketAddr;
+use tokio::prelude::*;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+async fn process_request(
+    socket: &mut tokio::net::TcpStream,
+    packet_data: &[u8],
+    request_processor: &mut RequestProcessor,
+) {
+    match request_processor.process_client_request(packet_data).await {
+        Err(e) => {
+            warn!("We failed to process client request - {:?}", e);
+            let response_bytes = ErrorResponse::new(format!("{:?}", e)).to_bytes();
+            if let Err(e) = socket.write_all(&response_bytes).await {
+                debug!("Failed to write response to the client - {:?}", e);
+            }
+        }
+        Ok(res) => match res {
+            ClientProcessingResult::RegisterResponse(auth_token) => {
+                let response_bytes = RegisterResponse::new(auth_token).to_bytes();
+                if let Err(e) = socket.write_all(&response_bytes).await {
+                    debug!("Failed to write response to the client - {:?}", e);
+                }
+            }
+            ClientProcessingResult::PullResponse(retrieved_messages) => {
+                let (messages, paths): (Vec<_>, Vec<_>) = retrieved_messages
+                    .into_iter()
+                    .map(|c| c.into_tuple())
+                    .unzip();
+                let response_bytes = PullResponse::new(messages).to_bytes();
+                if socket.write_all(&response_bytes).await.is_ok() {
+                    // only delete stored messages if we managed to actually send the response
+                    if let Err(e) = request_processor.delete_sent_messages(paths).await {
+                        error!("Somehow failed to delete stored messages! - {:?}", e);
+                    }
+                }
+            }
+        },
+    }
+}
+
+async fn process_socket_connection(
+    mut socket: tokio::net::TcpStream,
+    mut request_processor: RequestProcessor,
+) {
+    let mut buf = [0u8; 1024];
+    loop {
+        match socket.read(&mut buf).await {
+            // socket closed
+            Ok(n) if n == 0 => {
+                trace!("Remote connection closed.");
+                return;
+            }
+            // in here we do not really want to process multiple requests from the same
+            // client concurrently as then we might end up with really weird race conditions
+            // plus realistically it wouldn't really introduce any speed up
+            Ok(n) => process_request(&mut socket, &buf[0..n], &mut request_processor).await,
+            Err(e) => {
+                warn!(
+                    "failed to read from socket. Closing the connection; err = {:?}",
+                    e
+                );
+                return;
+            }
+        };
+    }
+}
+
+pub(crate) fn run_client_socket_listener(
+    handle: &Handle,
+    addr: SocketAddr,
+    request_processor: RequestProcessor,
+) -> JoinHandle<io::Result<()>> {
+    let handle_clone = handle.clone();
+    handle.spawn(async move {
+        let mut listener = tokio::net::TcpListener::bind(addr).await?;
+        loop {
+            let (socket, _) = listener.accept().await?;
+
+            let thread_request_processor = request_processor.clone();
+            handle_clone.spawn(async move {
+                process_socket_connection(socket, thread_request_processor).await;
+            });
+        }
+    })
+}

sfw-provider/src/provider/client_handling/mod.rs

@@ -1,274 +1,3 @@
-use crate::provider::storage::{ClientStorage, StoreError};
-use crate::provider::ClientLedger;
-use crypto::identity::{DummyMixIdentityPrivateKey, MixnetIdentityPrivateKey};
-use futures::lock::Mutex as FMutex;
-use hmac::{Hmac, Mac};
-use log::*;
-use sfw_provider_requests::requests::{
-    ProviderRequestError, ProviderRequests, PullRequest, RegisterRequest,
-};
-use sfw_provider_requests::responses::{ProviderResponse, PullResponse, RegisterResponse};
-use sfw_provider_requests::AuthToken;
-use sha2::Sha256;
-use std::io;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-type HmacSha256 = Hmac<Sha256>;
-
-#[derive(Debug)]
-pub enum ClientProcessingError {
-    ClientDoesntExistError,
-    StoreError,
-    InvalidRequest,
-    WrongToken,
-    IOError,
-}
-
-impl From<ProviderRequestError> for ClientProcessingError {
-    fn from(_: ProviderRequestError) -> Self {
-        use ClientProcessingError::*;
-
-        InvalidRequest
-    }
-}
-
-impl From<StoreError> for ClientProcessingError {
-    fn from(e: StoreError) -> Self {
-        match e {
-            StoreError::ClientDoesntExistError => ClientProcessingError::ClientDoesntExistError,
-            _ => ClientProcessingError::StoreError,
-        }
-    }
-}
-
-impl From<io::Error> for ClientProcessingError {
-    fn from(_: io::Error) -> Self {
-        use ClientProcessingError::*;
-
-        IOError
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct ClientProcessingData {
-    store_dir: PathBuf,
-    registered_clients_ledger: Arc<FMutex<ClientLedger>>,
-    secret_key: DummyMixIdentityPrivateKey,
-}
-
-impl ClientProcessingData {
-    pub(crate) fn new(
-        store_dir: PathBuf,
-        registered_clients_ledger: Arc<FMutex<ClientLedger>>,
-        secret_key: DummyMixIdentityPrivateKey,
-    ) -> Self {
-        ClientProcessingData {
-            store_dir,
-            registered_clients_ledger,
-            secret_key,
-        }
-    }
-
-    pub(crate) fn add_arc(self) -> Arc<Self> {
-        Arc::new(self)
-    }
-}
-
-pub(crate) struct ClientRequestProcessor;
-
-impl ClientRequestProcessor {
-    pub(crate) async fn process_client_request(
-        data: &[u8],
-        processing_data: Arc<ClientProcessingData>,
-    ) -> Result<Vec<u8>, ClientProcessingError> {
-        let client_request = ProviderRequests::from_bytes(&data)?;
-        trace!("Received the following request: {:?}", client_request);
-        match client_request {
-            ProviderRequests::Register(req) => Ok(ClientRequestProcessor::register_new_client(
-                req,
-                processing_data,
-            )
-            .await?
-            .to_bytes()),
-            ProviderRequests::PullMessages(req) => Ok(
-                ClientRequestProcessor::process_pull_messages_request(req, processing_data)
-                    .await?
-                    .to_bytes(),
-            ),
-        }
-    }
-
-    async fn process_pull_messages_request(
-        req: PullRequest,
-        processing_data: Arc<ClientProcessingData>,
-    ) -> Result<PullResponse, ClientProcessingError> {
-        // TODO: this lock is completely unnecessary as we're only reading the data.
-        // Wait for https://github.com/nymtech/nym-sfw-provider/issues/19 to resolve.
-        let unlocked_ledger = processing_data.registered_clients_ledger.lock().await;
-
-        if unlocked_ledger.has_token(req.auth_token) {
-            // drop the mutex so that we could do IO without blocking others wanting to get the lock
-            drop(unlocked_ledger);
-            let retrieved_messages = ClientStorage::retrieve_client_files(
-                req.destination_address,
-                processing_data.store_dir.as_path(),
-            )?;
-            Ok(PullResponse::new(retrieved_messages))
-        } else {
-            Err(ClientProcessingError::WrongToken)
-        }
-    }
-
-    async fn register_new_client(
-        req: RegisterRequest,
-        processing_data: Arc<ClientProcessingData>,
-    ) -> Result<RegisterResponse, ClientProcessingError> {
-        debug!(
-            "Processing register new client request: {:?}",
-            req.destination_address
-        );
-        let mut unlocked_ledger = processing_data.registered_clients_ledger.lock().await;
-
-        let auth_token = ClientRequestProcessor::generate_new_auth_token(
-            req.destination_address.to_vec(),
-            processing_data.secret_key,
-        );
-        if !unlocked_ledger.has_token(auth_token) {
-            unlocked_ledger.insert_token(auth_token, req.destination_address);
-            ClientRequestProcessor::create_storage_dir(
-                req.destination_address,
-                processing_data.store_dir.as_path(),
-            )?;
-        }
-        Ok(RegisterResponse::new(auth_token))
-    }
-
-    fn create_storage_dir(
-        client_address: sphinx::route::DestinationAddressBytes,
-        store_dir: &Path,
-    ) -> io::Result<()> {
-        let client_dir_name = bs58::encode(client_address).into_string();
-        let full_store_dir = store_dir.join(client_dir_name);
-        std::fs::create_dir_all(full_store_dir)
-    }
-
-    fn generate_new_auth_token(data: Vec<u8>, key: DummyMixIdentityPrivateKey) -> AuthToken {
-        // also note that `new_varkey` doesn't even have an execution branch returning an error
-        let mut auth_token_raw = HmacSha256::new_varkey(&key.to_bytes())
-            .expect("HMAC should be able take key of any size");
-        auth_token_raw.input(&data);
-        let mut auth_token = [0u8; 32];
-        auth_token.copy_from_slice(&auth_token_raw.result().code().to_vec());
-        auth_token
-    }
-}
-
-#[cfg(test)]
-mod register_new_client {
-    // use super::*;
-
-    // TODO: those tests require being called in async context. we need to research how to test this stuff...
-    //    #[test]
-    //    fn registers_new_auth_token_for_each_new_client() {
-    //        let req1 = RegisterRequest {
-    //            destination_address: [1u8; 32],
-    //        };
-    //        let registered_client_ledger = ClientLedger::new();
-    //        let store_dir = PathBuf::from("./foo/");
-    //        let key = Scalar::from_bytes_mod_order([1u8; 32]);
-    //        let client_processing_data = ClientProcessingData::new(store_dir, registered_client_ledger, key).add_arc_futures_mutex();
-    //
-    //
-    //        // need to do async....
-    //        client_processing_data.lock().await;
-    //        assert_eq!(0, registered_client_ledger.0.len());
-    //        ClientRequestProcessor::register_new_client(
-    //            req1,
-    //            client_processing_data.clone(),
-    //        );
-    //
-    //        assert_eq!(1, registered_client_ledger.0.len());
-    //
-    //        let req2 = RegisterRequest {
-    //            destination_address: [2u8; 32],
-    //        };
-    //        ClientRequestProcessor::register_new_client(
-    //            req2,
-    //            client_processing_data,
-    //        );
-    //        assert_eq!(2, registered_client_ledger.0.len());
-    //    }
-    //
-    //    #[test]
-    //    fn registers_given_token_only_once() {
-    //        let req1 = RegisterRequest {
-    //            destination_address: [1u8; 32],
-    //        };
-    //        let registered_client_ledger = ClientLedger::new();
-    //        let store_dir = PathBuf::from("./foo/");
-    //        let key = Scalar::from_bytes_mod_order([1u8; 32]);
-    //        let client_processing_data = ClientProcessingData::new(store_dir, registered_client_ledger, key).add_arc_futures_mutex();
-    //
-    //        ClientRequestProcessor::register_new_client(
-    //            req1,
-    //            client_processing_data.clone(),
-    //        );
-    //        let req2 = RegisterRequest {
-    //            destination_address: [1u8; 32],
-    //        };
-    //        ClientRequestProcessor::register_new_client(
-    //            req2,
-    //            client_processing_data.clone(),
-    //        );
-    //
-    //        client_processing_data.lock().await;
-    //
-    //        assert_eq!(1, registered_client_ledger.0.len())
-    //    }
-}
-
-#[cfg(test)]
-mod create_storage_dir {
-    use super::*;
-    use sphinx::route::DestinationAddressBytes;
-
-    #[test]
-    fn it_creates_a_correct_storage_directory() {
-        let client_address: DestinationAddressBytes = [1u8; 32];
-        let store_dir = Path::new("/tmp/");
-        ClientRequestProcessor::create_storage_dir(client_address, store_dir).unwrap();
-    }
-}
-
-#[cfg(test)]
-mod generating_new_auth_token {
-    use super::*;
-
-    #[test]
-    fn for_the_same_input_generates_the_same_auth_token() {
-        let data1 = vec![1u8; 55];
-        let data2 = vec![1u8; 55];
-        let key = DummyMixIdentityPrivateKey::from_bytes(&[1u8; 32]);
-        let token1 = ClientRequestProcessor::generate_new_auth_token(data1, key);
-        let token2 = ClientRequestProcessor::generate_new_auth_token(data2, key);
-        assert_eq!(token1, token2);
-    }
-
-    #[test]
-    fn for_different_inputs_generates_different_auth_tokens() {
-        let data1 = vec![1u8; 55];
-        let data2 = vec![2u8; 55];
-        let key = DummyMixIdentityPrivateKey::from_bytes(&[1u8; 32]);
-        let token1 = ClientRequestProcessor::generate_new_auth_token(data1, key);
-        let token2 = ClientRequestProcessor::generate_new_auth_token(data2, key);
-        assert_ne!(token1, token2);
-
-        let data1 = vec![1u8; 50];
-        let data2 = vec![2u8; 55];
-        let key = DummyMixIdentityPrivateKey::from_bytes(&[1u8; 32]);
-        let token1 = ClientRequestProcessor::generate_new_auth_token(data1, key);
-        let token2 = ClientRequestProcessor::generate_new_auth_token(data2, key);
-        assert_ne!(token1, token2);
-    }
-}
+pub(crate) mod ledger;
+pub(crate) mod listener;
+pub(crate) mod request_processing;

sfw-provider/src/provider/client_handling/request_processing.rs

@@ -0,0 +1,202 @@
+use crate::provider::client_handling::ledger::ClientLedger;
+use crate::provider::storage::{ClientFile, ClientStorage};
+use crypto::encryption;
+use hmac::{Hmac, Mac};
+use log::*;
+use sfw_provider_requests::requests::{
+    ProviderRequestError, ProviderRequests, PullRequest, RegisterRequest,
+};
+use sfw_provider_requests::AuthToken;
+use sha2::Sha256;
+use sphinx::route::DestinationAddressBytes;
+use std::io;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub enum ClientProcessingResult {
+    PullResponse(Vec<ClientFile>),
+    RegisterResponse(AuthToken),
+}
+
+#[derive(Debug)]
+pub enum ClientProcessingError {
+    InvalidRequest,
+    InvalidToken,
+    IOError(String),
+}
+
+impl From<ProviderRequestError> for ClientProcessingError {
+    fn from(_: ProviderRequestError) -> Self {
+        use ClientProcessingError::*;
+
+        InvalidRequest
+    }
+}
+
+impl From<io::Error> for ClientProcessingError {
+    fn from(e: io::Error) -> Self {
+        use ClientProcessingError::*;
+
+        IOError(e.to_string())
+    }
+}
+
+// PacketProcessor contains all data required to correctly process client requests
+#[derive(Clone)]
+pub struct RequestProcessor {
+    secret_key: Arc<encryption::PrivateKey>,
+    client_storage: ClientStorage,
+    client_ledger: ClientLedger,
+}
+
+impl RequestProcessor {
+    pub(crate) fn new(
+        secret_key: encryption::PrivateKey,
+        client_storage: ClientStorage,
+        client_ledger: ClientLedger,
+    ) -> Self {
+        RequestProcessor {
+            secret_key: Arc::new(secret_key),
+            client_storage,
+            client_ledger,
+        }
+    }
+
+    pub(crate) async fn process_client_request(
+        &mut self,
+        request_bytes: &[u8],
+    ) -> Result<ClientProcessingResult, ClientProcessingError> {
+        let client_request = ProviderRequests::from_bytes(request_bytes)?;
+        debug!("Received the following request: {:?}", client_request);
+        match client_request {
+            ProviderRequests::Register(req) => self.process_register_request(req).await,
+            ProviderRequests::PullMessages(req) => self.process_pull_request(req).await,
+        }
+    }
+
+    pub(crate) async fn process_register_request(
+        &mut self,
+        req: RegisterRequest,
+    ) -> Result<ClientProcessingResult, ClientProcessingError> {
+        debug!(
+            "Processing register new client request: {:?}",
+            req.destination_address.to_base58_string()
+        );
+
+        let auth_token = self.generate_new_auth_token(req.destination_address.clone());
+        if self
+            .client_ledger
+            .insert_token(auth_token.clone(), req.destination_address.clone())
+            .await
+            .is_some()
+        {
+            info!(
+                "Client {:?} was already registered before!",
+                req.destination_address.to_base58_string()
+            )
+        } else if let Err(e) = self
+            .client_storage
+            .create_storage_dir(req.destination_address.clone())
+            .await
+        {
+            error!("We failed to create inbox directory for the client -{:?}\nReverting issued token...", e);
+            // we must revert our changes if this operation failed
+            self.client_ledger
+                .remove_token(&req.destination_address)
+                .await;
+        }
+
+        Ok(ClientProcessingResult::RegisterResponse(auth_token))
+    }
+
+    fn generate_new_auth_token(&self, client_address: DestinationAddressBytes) -> AuthToken {
+        type HmacSha256 = Hmac<Sha256>;
+
+        // note that `new_varkey` doesn't even have an execution branch returning an error
+        // (true as of hmac 0.7.1)
+        let mut auth_token_raw = HmacSha256::new_varkey(&self.secret_key.to_bytes()).unwrap();
+        auth_token_raw.input(client_address.as_bytes());
+        let mut auth_token = [0u8; 32];
+        auth_token.copy_from_slice(auth_token_raw.result().code().as_slice());
+        AuthToken::from_bytes(auth_token)
+    }
+
+    pub(crate) async fn process_pull_request(
+        &self,
+        req: PullRequest,
+    ) -> Result<ClientProcessingResult, ClientProcessingError> {
+        if self
+            .client_ledger
+            .verify_token(&req.auth_token, &req.destination_address)
+            .await
+        {
+            let retrieved_messages = self
+                .client_storage
+                .retrieve_client_files(req.destination_address)
+                .await?;
+            return Ok(ClientProcessingResult::PullResponse(retrieved_messages));
+        }
+
+        Err(ClientProcessingError::InvalidToken)
+    }
+
+    pub(crate) async fn delete_sent_messages(&self, file_paths: Vec<PathBuf>) -> io::Result<()> {
+        self.client_storage.delete_files(file_paths).await
+    }
+}
+
+#[cfg(test)]
+mod generating_new_auth_token {
+    use super::*;
+
+    #[test]
+    fn for_the_same_input_generates_the_same_auth_token() {
+        let client_address1 = DestinationAddressBytes::from_bytes([1; 32]);
+        let client_address2 = DestinationAddressBytes::from_bytes([1; 32]);
+        let key = encryption::PrivateKey::from_bytes(&[2u8; 32]);
+
+        let request_processor = RequestProcessor {
+            secret_key: Arc::new(key),
+            client_storage: ClientStorage::new(3, 16, Default::default()),
+            client_ledger: ClientLedger::new(),
+        };
+
+        let token1 = request_processor.generate_new_auth_token(client_address1);
+        let token2 = request_processor.generate_new_auth_token(client_address2);
+        assert_eq!(token1, token2);
+    }
+
+    #[test]
+    fn for_different_inputs_generates_different_auth_tokens() {
+        let client_address1 = DestinationAddressBytes::from_bytes([1; 32]);
+        let client_address2 = DestinationAddressBytes::from_bytes([2; 32]);
+        let key1 = encryption::PrivateKey::from_bytes(&[3u8; 32]);
+        let key2 = encryption::PrivateKey::from_bytes(&[4u8; 32]);
+
+        let request_processor1 = RequestProcessor {
+            secret_key: Arc::new(key1),
+            client_storage: ClientStorage::new(3, 16, Default::default()),
+            client_ledger: ClientLedger::new(),
+        };
+
+        let request_processor2 = RequestProcessor {
+            secret_key: Arc::new(key2),
+            client_storage: ClientStorage::new(3, 16, Default::default()),
+            client_ledger: ClientLedger::new(),
+        };
+
+        let token1 = request_processor1.generate_new_auth_token(client_address1.clone());
+        let token2 = request_processor1.generate_new_auth_token(client_address2.clone());
+
+        let token3 = request_processor2.generate_new_auth_token(client_address1);
+        let token4 = request_processor2.generate_new_auth_token(client_address2);
+
+        assert_ne!(token1, token2);
+        assert_ne!(token1, token3);
+        assert_ne!(token1, token4);
+        assert_ne!(token2, token3);
+        assert_ne!(token2, token4);
+        assert_ne!(token3, token4);
+    }
+}

sfw-provider/src/provider/mix_handling/listener.rs

@@ -0,0 +1,75 @@
+use crate::provider::mix_handling::packet_processing::{MixProcessingResult, PacketProcessor};
+use log::*;
+use std::io;
+use std::net::SocketAddr;
+use tokio::prelude::*;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+async fn process_received_packet(
+    packet_data: [u8; sphinx::PACKET_SIZE],
+    packet_processor: PacketProcessor,
+) {
+    match packet_processor.process_sphinx_packet(packet_data).await {
+        Err(e) => debug!("We failed to process received sphinx packet - {:?}", e),
+        Ok(res) => match res {
+            MixProcessingResult::ForwardHop => {
+                error!("Somehow processed a forward hop message - those are not implemented for providers!")
+            }
+            MixProcessingResult::FinalHop => {
+                trace!("successfully processed [and stored] a final hop packet")
+            }
+        },
+    }
+}
+
+async fn process_socket_connection(
+    mut socket: tokio::net::TcpStream,
+    packet_processor: PacketProcessor,
+) {
+    let mut buf = [0u8; sphinx::PACKET_SIZE];
+    loop {
+        match socket.read(&mut buf).await {
+            // socket closed
+            Ok(n) if n == 0 => {
+                trace!("Remote connection closed.");
+                return;
+            }
+            Ok(n) => {
+                if n != sphinx::PACKET_SIZE {
+                    warn!("read data of different length than expected sphinx packet size - {} (expected {})", n, sphinx::PACKET_SIZE);
+                    continue;
+                }
+
+                // we must be able to handle multiple packets from same connection independently
+                tokio::spawn(process_received_packet(buf, packet_processor.clone()))
+            }
+            Err(e) => {
+                warn!(
+                    "failed to read from socket. Closing the connection; err = {:?}",
+                    e
+                );
+                return;
+            }
+        };
+    }
+}
+
+pub(crate) fn run_mix_socket_listener(
+    handle: &Handle,
+    addr: SocketAddr,
+    packet_processor: PacketProcessor,
+) -> JoinHandle<io::Result<()>> {
+    let handle_clone = handle.clone();
+    handle.spawn(async move {
+        let mut listener = tokio::net::TcpListener::bind(addr).await?;
+        loop {
+            let (socket, _) = listener.accept().await?;
+
+            let thread_packet_processor = packet_processor.clone();
+            handle_clone.spawn(async move {
+                process_socket_connection(socket, thread_packet_processor).await;
+            });
+        }
+    })
+}

sfw-provider/src/provider/mix_handling/mod.rs

@@ -1,95 +1,2 @@
-use crate::provider::storage::StoreData;
-use crypto::identity::DummyMixIdentityPrivateKey;
-use log::{error, warn};
-use sphinx::{ProcessedPacket, SphinxPacket};
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock};
-
-// TODO: this will probably need to be moved elsewhere I imagine
-// DUPLICATE WITH MIXNODE CODE!!!
-#[derive(Debug)]
-pub enum MixProcessingError {
-    FileIOFailure,
-    InvalidPayload,
-    NonMatchingRecipient,
-    ReceivedForwardHopError,
-    SphinxRecoveryError,
-    SphinxProcessingError,
-}
-
-impl From<sphinx::ProcessingError> for MixProcessingError {
-    // for time being just have a single error instance for all possible results of sphinx::ProcessingError
-    fn from(_: sphinx::ProcessingError) -> Self {
-        use MixProcessingError::*;
-
-        SphinxRecoveryError
-    }
-}
-
-impl From<std::io::Error> for MixProcessingError {
-    fn from(_: std::io::Error) -> Self {
-        use MixProcessingError::*;
-
-        FileIOFailure
-    }
-}
-
-// ProcessingData defines all data required to correctly unwrap sphinx packets
-#[derive(Debug, Clone)]
-pub(crate) struct MixProcessingData {
-    secret_key: DummyMixIdentityPrivateKey,
-    pub(crate) store_dir: PathBuf,
-}
-
-impl MixProcessingData {
-    pub(crate) fn new(secret_key: DummyMixIdentityPrivateKey, store_dir: PathBuf) -> Self {
-        MixProcessingData {
-            secret_key,
-            store_dir,
-        }
-    }
-
-    pub(crate) fn add_arc_rwlock(self) -> Arc<RwLock<Self>> {
-        Arc::new(RwLock::new(self))
-    }
-}
-
-pub(crate) struct MixPacketProcessor(());
-
-impl MixPacketProcessor {
-    pub fn process_sphinx_data_packet(
-        packet_data: &[u8],
-        processing_data: &RwLock<MixProcessingData>,
-    ) -> Result<StoreData, MixProcessingError> {
-        let packet = SphinxPacket::from_bytes(packet_data.to_vec())?;
-        let read_processing_data = match processing_data.read() {
-            Ok(guard) => guard,
-            Err(e) => {
-                error!("processing data lock was poisoned! - {:?}", e);
-                std::process::exit(1)
-            }
-        };
-        let (client_address, client_surb_id, payload) =
-            match packet.process(read_processing_data.secret_key.as_scalar()) {
-                Ok(ProcessedPacket::ProcessedPacketFinalHop(client_address, surb_id, payload)) => {
-                    (client_address, surb_id, payload)
-                }
-                Ok(_) => return Err(MixProcessingError::ReceivedForwardHopError),
-                Err(e) => {
-                    warn!("Error unwrapping Sphinx packet: {:?}", e);
-                    return Err(MixProcessingError::SphinxProcessingError);
-                }
-            };
-
-        // TODO: should provider try to be recovering plaintext? this would potentially make client retrieve messages of non-constant length,
-        // perhaps provider should be re-padding them on retrieval or storing full data?
-        let (payload_destination, message) = payload
-            .try_recover_destination_and_plaintext()
-            .ok_or_else(|| MixProcessingError::InvalidPayload)?;
-        if client_address != payload_destination {
-            return Err(MixProcessingError::NonMatchingRecipient);
-        }
-
-        Ok(StoreData::new(client_address, client_surb_id, message))
-    }
-}
+pub(crate) mod listener;
+pub(crate) mod packet_processing;

sfw-provider/src/provider/mix_handling/packet_processing.rs

@@ -0,0 +1,100 @@
+use crate::provider::storage::{ClientStorage, StoreData};
+use crypto::encryption;
+use log::*;
+use sphinx::payload::Payload;
+use sphinx::route::{DestinationAddressBytes, SURBIdentifier};
+use sphinx::{ProcessedPacket, SphinxPacket};
+use std::io;
+use std::ops::Deref;
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub enum MixProcessingError {
+    ReceivedForwardHopError,
+    NonMatchingRecipient,
+    InvalidPayload,
+    SphinxProcessingError,
+    IOError(String),
+}
+
+pub enum MixProcessingResult {
+    #[allow(dead_code)]
+    ForwardHop,
+    FinalHop,
+}
+
+impl From<sphinx::ProcessingError> for MixProcessingError {
+    // for time being just have a single error instance for all possible results of sphinx::ProcessingError
+    fn from(_: sphinx::ProcessingError) -> Self {
+        use MixProcessingError::*;
+
+        SphinxProcessingError
+    }
+}
+
+impl From<io::Error> for MixProcessingError {
+    fn from(e: io::Error) -> Self {
+        use MixProcessingError::*;
+
+        IOError(e.to_string())
+    }
+}
+
+// PacketProcessor contains all data required to correctly unwrap and store sphinx packets
+#[derive(Clone)]
+pub struct PacketProcessor {
+    secret_key: Arc<encryption::PrivateKey>,
+    client_store: ClientStorage,
+}
+
+impl PacketProcessor {
+    pub(crate) fn new(secret_key: encryption::PrivateKey, client_store: ClientStorage) -> Self {
+        PacketProcessor {
+            secret_key: Arc::new(secret_key),
+            client_store,
+        }
+    }
+
+    async fn process_final_hop(
+        &self,
+        client_address: DestinationAddressBytes,
+        surb_id: SURBIdentifier,
+        payload: Payload,
+    ) -> Result<MixProcessingResult, MixProcessingError> {
+        // TODO: should provider try to be recovering plaintext? this would potentially make client retrieve messages of non-constant length,
+        // perhaps provider should be re-padding them on retrieval or storing full data?
+        let (payload_destination, message) = payload
+            .try_recover_destination_and_plaintext()
+            .ok_or_else(|| MixProcessingError::InvalidPayload)?;
+        if client_address != payload_destination {
+            return Err(MixProcessingError::NonMatchingRecipient);
+        }
+
+        let store_data = StoreData::new(client_address, surb_id, message);
+        self.client_store.store_processed_data(store_data).await?;
+
+        Ok(MixProcessingResult::FinalHop)
+    }
+
+    pub(crate) async fn process_sphinx_packet(
+        &self,
+        raw_packet_data: [u8; sphinx::PACKET_SIZE],
+    ) -> Result<MixProcessingResult, MixProcessingError> {
+        let packet = SphinxPacket::from_bytes(&raw_packet_data)?;
+
+        match packet.process(self.secret_key.deref().inner()) {
+            Ok(ProcessedPacket::ProcessedPacketForwardHop(_, _, _)) => {
+                warn!("Received a forward hop message - those are not implemented for providers");
+                Err(MixProcessingError::ReceivedForwardHopError)
+            }
+            Ok(ProcessedPacket::ProcessedPacketFinalHop(client_address, surb_id, payload)) => {
+                self.process_final_hop(client_address, surb_id, payload)
+                    .await
+            }
+            Err(e) => {
+                warn!("Failed to unwrap Sphinx packet: {:?}", e);
+                Err(MixProcessingError::SphinxProcessingError)
+            }
+        }
+    }
+}

sfw-provider/src/provider/mod.rs

@@ -1,19 +1,10 @@
-use crate::provider::client_handling::{ClientProcessingData, ClientRequestProcessor};
-use crate::provider::mix_handling::{MixPacketProcessor, MixProcessingData};
+use crate::config::persistence::pathfinder::ProviderPathfinder;
+use crate::config::Config;
+use crate::provider::client_handling::ledger::ClientLedger;
 use crate::provider::storage::ClientStorage;
-use crypto::identity::{DummyMixIdentityPrivateKey, DummyMixIdentityPublicKey};
-use directory_client::presence::providers::MixProviderClient;
-use futures::io::Error;
-use futures::lock::Mutex as FMutex;
+use crypto::encryption;
 use log::*;
-use sfw_provider_requests::AuthToken;
-use sphinx::route::DestinationAddressBytes;
-use std::collections::HashMap;
-use std::net::{Shutdown, SocketAddr};
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::sync::RwLock;
-use tokio::prelude::*;
+use pemstore::pemstore::PemStore;
 use tokio::runtime::Runtime;
 
 mod client_handling;
@@ -21,300 +12,103 @@ mod mix_handling;
 pub mod presence;
 mod storage;
 
-// TODO: if we ever create config file, this should go there
-const STORED_MESSAGE_FILENAME_LENGTH: usize = 16;
-const MESSAGE_RETRIEVAL_LIMIT: usize = 5;
-
-pub struct Config {
-    pub client_socket_address: SocketAddr,
-    pub directory_server: String,
-    pub mix_socket_address: SocketAddr,
-    pub public_key: DummyMixIdentityPublicKey,
-    pub secret_key: DummyMixIdentityPrivateKey,
-    pub store_dir: PathBuf,
-}
-
-#[derive(Debug)]
-pub enum ProviderError {
-    TcpListenerBindingError,
-    TcpListenerConnectionError,
-    TcpListenerUnexpectedEof,
-
-    TcpListenerUnknownError,
-}
-
-impl From<io::Error> for ProviderError {
-    fn from(err: Error) -> Self {
-        use ProviderError::*;
-        match err.kind() {
-            io::ErrorKind::ConnectionRefused => TcpListenerConnectionError,
-            io::ErrorKind::ConnectionReset => TcpListenerConnectionError,
-            io::ErrorKind::ConnectionAborted => TcpListenerConnectionError,
-            io::ErrorKind::NotConnected => TcpListenerConnectionError,
-
-            io::ErrorKind::AddrInUse => TcpListenerBindingError,
-            io::ErrorKind::AddrNotAvailable => TcpListenerBindingError,
-            io::ErrorKind::UnexpectedEof => TcpListenerUnexpectedEof,
-            _ => TcpListenerUnknownError,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct ClientLedger(HashMap<AuthToken, DestinationAddressBytes>);
-
-impl ClientLedger {
-    fn new() -> Self {
-        ClientLedger(HashMap::new())
-    }
-
-    fn add_arc_futures_mutex(self) -> Arc<FMutex<Self>> {
-        Arc::new(FMutex::new(self))
-    }
-
-    fn has_token(&self, auth_token: AuthToken) -> bool {
-        return self.0.contains_key(&auth_token);
-    }
-
-    fn insert_token(
-        &mut self,
-        auth_token: AuthToken,
-        client_address: DestinationAddressBytes,
-    ) -> Option<DestinationAddressBytes> {
-        self.0.insert(auth_token, client_address)
-    }
-
-    fn current_clients(&self) -> Vec<MixProviderClient> {
-        self.0
-            .iter()
-            .map(|(_, v)| bs58::encode(v).into_string())
-            .map(|pub_key| MixProviderClient { pub_key })
-            .collect()
-    }
-
-    #[allow(dead_code)]
-    fn load(_file: PathBuf) -> Self {
-        unimplemented!()
-    }
-}
-
 pub struct ServiceProvider {
-    directory_server: String,
-    mix_network_address: SocketAddr,
-    client_network_address: SocketAddr,
-    public_key: DummyMixIdentityPublicKey,
-    secret_key: DummyMixIdentityPrivateKey,
-    store_dir: PathBuf,
+    runtime: Runtime,
+    config: Config,
+    sphinx_keypair: encryption::KeyPair,
     registered_clients_ledger: ClientLedger,
 }
 
 impl ServiceProvider {
+    fn load_sphinx_keys(config_file: &Config) -> encryption::KeyPair {
+        let sphinx_keypair = PemStore::new(ProviderPathfinder::new_from_config(&config_file))
+            .read_encryption()
+            .expect("Failed to read stored sphinx key files");
+        println!(
+            "Public encryption key: {}\nFor time being, it is identical to identity keys",
+            sphinx_keypair.public_key().to_base58_string()
+        );
+        sphinx_keypair
+    }
+
     pub fn new(config: Config) -> Self {
+        let sphinx_keypair = Self::load_sphinx_keys(&config);
+        let registered_clients_ledger = ClientLedger::load(config.get_clients_ledger_path());
         ServiceProvider {
-            mix_network_address: config.mix_socket_address,
-            client_network_address: config.client_socket_address,
-            secret_key: config.secret_key,
-            public_key: config.public_key,
-            store_dir: PathBuf::from(config.store_dir.clone()),
-            // TODO: load initial ledger from file
-            registered_clients_ledger: ClientLedger::new(),
-            directory_server: config.directory_server.clone(),
+            runtime: Runtime::new().unwrap(),
+            config,
+            sphinx_keypair,
+            registered_clients_ledger,
         }
     }
 
-    async fn process_mixnet_socket_connection(
-        mut socket: tokio::net::TcpStream,
-        processing_data: Arc<RwLock<MixProcessingData>>,
-    ) {
-        let mut buf = [0u8; sphinx::PACKET_SIZE];
-
-        // In a loop, read data from the socket and write the data back.
-        loop {
-            match socket.read(&mut buf).await {
-                // socket closed
-                Ok(n) if n == 0 => {
-                    trace!("Remote connection closed.");
-                    return;
-                }
-                Ok(_) => {
-                    let store_data = match MixPacketProcessor::process_sphinx_data_packet(
-                        buf.as_ref(),
-                        processing_data.as_ref(),
-                    ) {
-                        Ok(sd) => sd,
-                        Err(e) => {
-                            warn!("failed to process sphinx packet; err = {:?}", e);
-                            return;
-                        }
-                    };
-                    let processing_data_lock = match processing_data.read() {
-                        Ok(guard) => guard,
-                        Err(e) => {
-                            error!("processing data lock was poisoned! - {:?}", e);
-                            std::process::exit(1)
-                        }
-                    };
-                    ClientStorage::store_processed_data(
-                        store_data,
-                        processing_data_lock.store_dir.as_path(),
-                    )
-                    .unwrap_or_else(|e| {
-                        error!("failed to store processed sphinx message; err = {:?}", e);
-                        return;
-                    });
-                }
-                Err(e) => {
-                    warn!("failed to read from socket; err = {:?}", e);
-                    return;
-                }
-            };
-
-            // Write the some data back
-            if let Err(e) = socket.write_all(b"foomp").await {
-                warn!("failed to write reply to socket; err = {:?}", e);
-                return;
-            }
-        }
+    fn start_presence_notifier(&self) {
+        info!("Starting presence notifier...");
+        let notifier_config = presence::NotifierConfig::new(
+            self.config.get_presence_directory_server(),
+            self.config.get_mix_announce_address(),
+            self.config.get_clients_announce_address(),
+            self.sphinx_keypair.public_key().to_base58_string(),
+            self.config.get_presence_sending_delay(),
+        );
+        presence::Notifier::new(notifier_config, self.registered_clients_ledger.clone())
+            .start(self.runtime.handle());
     }
 
-    async fn send_response(mut socket: tokio::net::TcpStream, data: &[u8]) {
-        if let Err(e) = socket.write_all(data).await {
-            warn!("failed to write reply to socket; err = {:?}", e)
-        }
-        if let Err(e) = socket.shutdown(Shutdown::Write) {
-            warn!("failed to close write part of the socket; err = {:?}", e)
-        }
-    }
-
-    // TODO: FIGURE OUT HOW TO SET READ_DEADLINES IN TOKIO
-    async fn process_client_socket_connection(
-        mut socket: tokio::net::TcpStream,
-        processing_data: Arc<ClientProcessingData>,
-    ) {
-        let mut buf = [0; 1024];
-
-        // TODO: restore the for loop once we go back to persistent tcp socket connection
-        let response = match socket.read(&mut buf).await {
-            // socket closed
-            Ok(n) if n == 0 => {
-                trace!("Remote connection closed.");
-                Err(())
-            }
-            Ok(n) => {
-                match ClientRequestProcessor::process_client_request(
-                    buf[..n].as_ref(),
-                    processing_data,
-                )
-                .await
-                {
-                    Err(e) => {
-                        warn!("failed to process client request; err = {:?}", e);
-                        Err(())
-                    }
-                    Ok(res) => Ok(res),
-                }
-            }
-            Err(e) => {
-                warn!("failed to read from socket; err = {:?}", e);
-                Err(())
-            }
-        };
-
-        if let Err(e) = socket.shutdown(Shutdown::Read) {
-            warn!("failed to close read part of the socket; err = {:?}", e)
-        }
-
-        match response {
-            Ok(res) => {
-                ServiceProvider::send_response(socket, &res).await;
-            }
-            _ => {
-                ServiceProvider::send_response(socket, b"bad foomp").await;
-            }
-        }
-    }
-
-    async fn start_mixnet_listening(
-        address: SocketAddr,
-        secret_key: DummyMixIdentityPrivateKey,
-        store_dir: PathBuf,
-    ) -> Result<(), ProviderError> {
-        let mut listener = tokio::net::TcpListener::bind(address).await?;
-        let processing_data = MixProcessingData::new(secret_key, store_dir).add_arc_rwlock();
-
-        loop {
-            let (socket, _) = listener.accept().await?;
-            // do note that the underlying data is NOT copied here; arc is incremented and lock is shared
-            // (if I understand it all correctly)
-            let thread_processing_data = processing_data.clone();
-            tokio::spawn(async move {
-                ServiceProvider::process_mixnet_socket_connection(socket, thread_processing_data)
-                    .await
-            });
-        }
-    }
-
-    async fn start_client_listening(
-        address: SocketAddr,
-        store_dir: PathBuf,
-        client_ledger: Arc<FMutex<ClientLedger>>,
-        secret_key: DummyMixIdentityPrivateKey,
-    ) -> Result<(), ProviderError> {
-        let mut listener = tokio::net::TcpListener::bind(address).await?;
-        let processing_data =
-            ClientProcessingData::new(store_dir, client_ledger, secret_key).add_arc();
-
-        loop {
-            let (socket, _) = listener.accept().await?;
-            // do note that the underlying data is NOT copied here; arc is incremented and lock is shared
-            // (if I understand it all correctly)
-            let thread_processing_data = processing_data.clone();
-            tokio::spawn(async move {
-                ServiceProvider::process_client_socket_connection(socket, thread_processing_data)
-                    .await
-            });
-        }
-    }
-
-    // Note: this now consumes the provider
-    pub fn start(self) -> Result<(), Box<dyn std::error::Error>> {
-        // Create the runtime, probably later move it to Provider struct itself?
-        // TODO: figure out the difference between Runtime and Handle
-        let mut rt = Runtime::new()?;
-        //        let mut h = rt.handle();
-
-        let initial_client_ledger = self.registered_clients_ledger;
-        let thread_shareable_ledger = initial_client_ledger.add_arc_futures_mutex();
-
-        let presence_notifier = presence::Notifier::new(
-            self.directory_server,
-            self.client_network_address.clone(),
-            self.mix_network_address.clone(),
-            self.public_key,
-            thread_shareable_ledger.clone(),
+    fn start_mix_socket_listener(&self, client_storage: ClientStorage) {
+        info!("Starting mix socket listener...");
+        let packet_processor = mix_handling::packet_processing::PacketProcessor::new(
+            self.sphinx_keypair.private_key().clone(),
+            client_storage,
         );
 
-        let presence_future = rt.spawn(presence_notifier.run());
-        let mix_future = rt.spawn(ServiceProvider::start_mixnet_listening(
-            self.mix_network_address,
-            self.secret_key.clone(),
-            self.store_dir.clone(),
-        ));
-        let client_future = rt.spawn(ServiceProvider::start_client_listening(
-            self.client_network_address,
-            self.store_dir.clone(),
-            thread_shareable_ledger,
-            self.secret_key,
-        ));
-        // Spawn the root task
-        rt.block_on(async {
-            let future_results =
-                futures::future::join3(mix_future, client_future, presence_future).await;
-            assert!(future_results.0.is_ok() && future_results.1.is_ok());
-        });
+        mix_handling::listener::run_mix_socket_listener(
+            self.runtime.handle(),
+            self.config.get_mix_listening_address(),
+            packet_processor,
+        );
+    }
 
-        // this line in theory should never be reached as the runtime should be permanently blocked on listeners
-        error!("The server went kaput...");
-        Ok(())
+    fn start_client_socket_listener(&self, client_storage: ClientStorage) {
+        info!("Starting client socket listener...");
+        let packet_processor = client_handling::request_processing::RequestProcessor::new(
+            self.sphinx_keypair.private_key().clone(),
+            client_storage,
+            self.registered_clients_ledger.clone(),
+        );
+
+        client_handling::listener::run_client_socket_listener(
+            self.runtime.handle(),
+            self.config.get_clients_listening_address(),
+            packet_processor,
+        );
+    }
+
+    pub fn run(&mut self) {
+        // A possible future optimisation, depending on bottlenecks and resource usage:
+        // considering, presumably, there will be more mix packets received than client requests:
+        // create 2 separate runtimes - one with bigger threadpool dedicated solely for
+        // the mix handling and the other one for the rest of tasks
+
+        let client_storage = ClientStorage::new(
+            self.config.get_message_retrieval_limit() as usize,
+            self.config.get_stored_messages_filename_length(),
+            self.config.get_clients_inboxes_dir(),
+        );
+
+        self.start_presence_notifier();
+        self.start_mix_socket_listener(client_storage.clone());
+        self.start_client_socket_listener(client_storage);
+
+        if let Err(e) = self.runtime.block_on(tokio::signal::ctrl_c()) {
+            error!(
+                "There was an error while capturing SIGINT - {:?}. We will terminate regardless",
+                e
+            );
+        }
+
+        println!(
+            "Received SIGINT - the provider will terminate now (threads are not YET nicely stopped)"
+        );
     }
 }

sfw-provider/src/provider/presence.rs

@@ -1,54 +1,73 @@
+use crate::built_info;
 use crate::provider::ClientLedger;
-use crypto::identity::DummyMixIdentityPublicKey;
 use directory_client::presence::providers::MixProviderPresence;
 use directory_client::requests::presence_providers_post::PresenceMixProviderPoster;
 use directory_client::DirectoryClient;
-use futures::lock::Mutex as FMutex;
 use log::{debug, error};
-use std::net::SocketAddr;
-use std::sync::Arc;
 use std::time::Duration;
+use tokio::runtime::Handle;
+use tokio::task::JoinHandle;
+
+pub struct NotifierConfig {
+    directory_server: String,
+    mix_announce_host: String,
+    clients_announce_host: String,
+    pub_key_string: String,
+    sending_delay: Duration,
+}
+
+impl NotifierConfig {
+    pub fn new(
+        directory_server: String,
+        mix_announce_host: String,
+        clients_announce_host: String,
+        pub_key_string: String,
+        sending_delay: Duration,
+    ) -> Self {
+        NotifierConfig {
+            directory_server,
+            mix_announce_host,
+            clients_announce_host,
+            pub_key_string,
+            sending_delay,
+        }
+    }
+}
 
 pub struct Notifier {
-    pub net_client: directory_client::Client,
-    client_ledger: Arc<FMutex<ClientLedger>>,
+    net_client: directory_client::Client,
+    client_ledger: ClientLedger,
+    sending_delay: Duration,
     client_listener: String,
     mixnet_listener: String,
-    pub_key: String,
+    pub_key_string: String,
 }
 
 impl Notifier {
-    pub fn new(
-        directory_server_address: String,
-        client_listener: SocketAddr,
-        mixnet_listener: SocketAddr,
-        pub_key: DummyMixIdentityPublicKey,
-        client_ledger: Arc<FMutex<ClientLedger>>,
-    ) -> Notifier {
-        let directory_config = directory_client::Config {
-            base_url: directory_server_address,
+    pub fn new(config: NotifierConfig, client_ledger: ClientLedger) -> Notifier {
+        let directory_client_cfg = directory_client::Config {
+            base_url: config.directory_server,
         };
-        let net_client = directory_client::Client::new(directory_config);
+        let net_client = directory_client::Client::new(directory_client_cfg);
 
         Notifier {
-            net_client,
-            client_listener: client_listener.to_string(),
-            mixnet_listener: mixnet_listener.to_string(),
-            pub_key: pub_key.to_base58_string(),
             client_ledger,
+            net_client,
+            client_listener: config.clients_announce_host,
+            mixnet_listener: config.mix_announce_host,
+            pub_key_string: config.pub_key_string,
+            sending_delay: config.sending_delay,
         }
     }
 
     async fn make_presence(&self) -> MixProviderPresence {
-        let unlocked_ledger = self.client_ledger.lock().await;
-
         MixProviderPresence {
             client_listener: self.client_listener.clone(),
             mixnet_listener: self.mixnet_listener.clone(),
-            pub_key: self.pub_key.clone(),
-            registered_clients: unlocked_ledger.current_clients(),
+            pub_key: self.pub_key_string.clone(),
+            registered_clients: self.client_ledger.current_clients().await,
             last_seen: 0,
-            version: env!("CARGO_PKG_VERSION").to_string(),
+            version: built_info::PKG_VERSION.to_string(),
         }
     }
 
@@ -59,12 +78,13 @@ impl Notifier {
         }
     }
 
-    pub async fn run(self) {
-        loop {
-            let presence = self.make_presence().await;
-            self.notify(presence);
-            let delay_duration = Duration::from_secs(5);
-            tokio::time::delay_for(delay_duration).await;
-        }
+    pub fn start(self, handle: &Handle) -> JoinHandle<()> {
+        handle.spawn(async move {
+            loop {
+                let presence = self.make_presence().await;
+                self.notify(presence);
+                tokio::time::delay_for(self.sending_delay).await;
+            }
+        })
     }
 }

sfw-provider/src/provider/storage/mod.rs

@@ -1,23 +1,36 @@
-use crate::provider::{MESSAGE_RETRIEVAL_LIMIT, STORED_MESSAGE_FILENAME_LENGTH};
+use futures::lock::Mutex;
+use futures::StreamExt;
 use log::*;
 use rand::Rng;
 use sfw_provider_requests::DUMMY_MESSAGE_CONTENT;
 use sphinx::route::{DestinationAddressBytes, SURBIdentifier};
-use std::fs::File;
 use std::io;
-use std::io::Write;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::fs;
+use tokio::fs::File;
+use tokio::prelude::*;
 
-pub enum StoreError {
-    ClientDoesntExistError,
-    FileIOFailure,
+fn dummy_message() -> ClientFile {
+    ClientFile {
+        content: DUMMY_MESSAGE_CONTENT.to_vec(),
+        path: Default::default(),
+    }
 }
 
-impl From<std::io::Error> for StoreError {
-    fn from(_: std::io::Error) -> Self {
-        use StoreError::*;
+#[derive(Clone, Debug)]
+pub struct ClientFile {
+    content: Vec<u8>,
+    path: PathBuf,
+}
 
-        FileIOFailure
+impl ClientFile {
+    fn new(content: Vec<u8>, path: PathBuf) -> Self {
+        ClientFile { content, path }
+    }
+
+    pub(crate) fn into_tuple(self) -> (Vec<u8>, PathBuf) {
+        (self.content, self.path)
     }
 }
 
@@ -42,27 +55,64 @@ impl StoreData {
     }
 }
 
-// TODO: replace with database
-pub struct ClientStorage(());
+// TODO: replace with proper database...
+// Note: you should NEVER create more than a single instance of this using 'new()'.
+// You should always use .clone() to create additional instances
+#[derive(Clone, Debug)]
+pub struct ClientStorage {
+    inner: Arc<Mutex<ClientStorageInner>>,
+}
 
-// TODO: change it to some generic implementation to inject fs
+// even though the data inside is extremely cheap to copy, we have to have a single mutex,
+// so might as well store the data behind it
+pub struct ClientStorageInner {
+    message_retrieval_limit: usize,
+    filename_length: u16,
+    main_store_path_dir: PathBuf,
+}
+
+// TODO: change it to some generic implementation to inject fs (or even better - proper database)
 impl ClientStorage {
-    pub(crate) fn generate_random_file_name() -> String {
+    pub(crate) fn new(message_limit: usize, filename_len: u16, main_store_dir: PathBuf) -> Self {
+        ClientStorage {
+            inner: Arc::new(Mutex::new(ClientStorageInner {
+                message_retrieval_limit: message_limit,
+                filename_length: filename_len,
+                main_store_path_dir: main_store_dir,
+            })),
+        }
+    }
+
+    // TODO: does this method really require locking?
+    // The worst that can happen is client sending 2 requests: to pull messages and register
+    // if register does not lock, then under specific timing pull messages will fail,
+    // but can simply be retried with no issues
+    pub(crate) async fn create_storage_dir(
+        &self,
+        client_address: DestinationAddressBytes,
+    ) -> io::Result<()> {
+        let inner_data = self.inner.lock().await;
+
+        let client_dir_name = client_address.to_base58_string();
+        let full_store_dir = inner_data.main_store_path_dir.join(client_dir_name);
+        fs::create_dir_all(full_store_dir).await
+    }
+
+    pub(crate) fn generate_random_file_name(length: usize) -> String {
         rand::thread_rng()
             .sample_iter(&rand::distributions::Alphanumeric)
-            .take(STORED_MESSAGE_FILENAME_LENGTH)
+            .take(length)
             .collect::<String>()
     }
 
-    fn dummy_message() -> Vec<u8> {
-        // TODO: should it be padded to constant length?
-        DUMMY_MESSAGE_CONTENT.to_vec()
-    }
+    pub(crate) async fn store_processed_data(&self, store_data: StoreData) -> io::Result<()> {
+        let inner_data = self.inner.lock().await;
 
-    pub fn store_processed_data(store_data: StoreData, store_dir: &Path) -> io::Result<()> {
-        let client_dir_name = bs58::encode(store_data.client_address).into_string();
-        let full_store_dir = store_dir.join(client_dir_name);
-        let full_store_path = full_store_dir.join(ClientStorage::generate_random_file_name());
+        let client_dir_name = store_data.client_address.to_base58_string();
+        let full_store_dir = inner_data.main_store_path_dir.join(client_dir_name);
+        let full_store_path = full_store_dir.join(Self::generate_random_file_name(
+            inner_data.filename_length as usize,
+        ));
         debug!(
             "going to store: {:?} in file: {:?}",
             store_data.message, full_store_path
@@ -70,43 +120,62 @@ impl ClientStorage {
 
         // TODO: what to do with surbIDs??
 
-        // we can use normal io here, no need for tokio as it's all happening in one thread per connection
-        let mut file = File::create(full_store_path)?;
-        file.write_all(store_data.message.as_ref())?;
-
-        Ok(())
+        let mut file = File::create(full_store_path).await?;
+        file.write_all(store_data.message.as_ref()).await
     }
 
-    pub fn retrieve_client_files(
+    pub(crate) async fn retrieve_client_files(
+        &self,
         client_address: DestinationAddressBytes,
-        store_dir: &Path,
-    ) -> Result<Vec<Vec<u8>>, StoreError> {
-        let client_dir_name = bs58::encode(client_address).into_string();
-        let full_store_dir = store_dir.join(client_dir_name);
+    ) -> io::Result<Vec<ClientFile>> {
+        let inner_data = self.inner.lock().await;
+
+        let client_dir_name = client_address.to_base58_string();
+        let full_store_dir = inner_data.main_store_path_dir.join(client_dir_name);
 
         trace!("going to lookup: {:?}!", full_store_dir);
         if !full_store_dir.exists() {
-            return Err(StoreError::ClientDoesntExistError);
+            return Err(io::Error::new(
+                io::ErrorKind::NotFound,
+                "Target client does not exist",
+            ));
         }
 
-        let msgs: Vec<_> = std::fs::read_dir(full_store_dir)?
-            .filter_map(|entry| entry.ok())
-            .filter(|entry| ClientStorage::is_valid_file(entry))
-            .map(|entry| {
-                // Not yet sure how to exactly get rid of those unwraps
-                let content = std::fs::read(entry.path()).unwrap();
-                ClientStorage::delete_file(entry.path()).unwrap();
-                content
-            }) // TODO: THIS MAP IS UNSAFE (BOTH FOR READING AND DELETING)!! - in the case where there are multiple requests from the same client being processed in parallel
-            .chain(std::iter::repeat(ClientStorage::dummy_message()))
-            .take(MESSAGE_RETRIEVAL_LIMIT)
-            .collect();
+        let mut msgs = Vec::new();
+        let mut read_dir = fs::read_dir(full_store_dir).await?;
+
+        while let Some(dir_entry) = read_dir.next().await {
+            if let Ok(dir_entry) = dir_entry {
+                if !Self::is_valid_file(&dir_entry).await {
+                    continue;
+                }
+                // Do not delete the file itself here!
+                // Only do it after client has received it
+                let client_file =
+                    ClientFile::new(fs::read(dir_entry.path()).await?, dir_entry.path());
+                msgs.push(client_file)
+            }
+            if msgs.len() == inner_data.message_retrieval_limit {
+                break;
+            }
+        }
+
+        let dummy_message = dummy_message();
+
+        // make sure we always return as many messages as we need
+        if msgs.len() != inner_data.message_retrieval_limit as usize {
+            msgs = msgs
+                .into_iter()
+                .chain(std::iter::repeat(dummy_message))
+                .take(inner_data.message_retrieval_limit)
+                .collect();
+        }
 
         Ok(msgs)
     }
 
-    fn is_valid_file(entry: &std::fs::DirEntry) -> bool {
-        let metadata = match entry.metadata() {
+    async fn is_valid_file(entry: &fs::DirEntry) -> bool {
+        let metadata = match entry.metadata().await {
             Ok(meta) => meta,
             Err(e) => {
                 error!(
@@ -129,11 +198,18 @@ impl ClientStorage {
         is_file
     }
 
-    // TODO: THIS NEEDS A LOCKING MECHANISM!!! (or a db layer on top - basically 'ClientStorage' on steroids)
-    // TODO 2: This should only be called AFTER we sent the reply. Because if client's connection failed after sending request
-    // the messages would be deleted but he wouldn't have received them
-    fn delete_file(path: PathBuf) -> io::Result<()> {
-        trace!("Here {:?} will be deleted!", path);
-        std::fs::remove_file(path) // another argument for db layer -> remove_file is NOT guaranteed to immediately get rid of the file
+    pub(crate) async fn delete_files(&self, file_paths: Vec<PathBuf>) -> io::Result<()> {
+        let dummy_message = dummy_message();
+        let _guard = self.inner.lock().await;
+
+        for file_path in file_paths {
+            if file_path == dummy_message.path {
+                continue;
+            }
+            if let Err(e) = fs::remove_file(file_path).await {
+                error!("Failed to delete client message! - {:?}", e)
+            }
+        }
+        Ok(())
     }
 }

validator/Cargo.toml

@@ -1,26 +1,28 @@
 [package]
 build = "build.rs"
 name = "nym-validator"
-version = "0.4.1"
+version = "0.5.0-rc.1"
 authors = ["Jedrzej Stuczynski <andrew@nymtech.net>"]
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+abci = "0.6.4"
+byteorder = "1.3.2"
 clap = "2.33.0"
+dirs = "2.0.2"
 # Read notes https://crates.io/crates/dotenv - tl;dr: don't use in production, set environmental variables properly.
 dotenv = "0.15.0"
 futures = "0.3.1"
 log = "0.4"
 pretty_env_logger = "0.3"
 serde = "1.0.104"
-serde_derive = "1.0.104"
 tokio = { version = "0.2", features = ["full"] }
-toml = "0.5.5"
 
 ## internal
 crypto = {path = "../common/crypto"}
+config = {path = "../common/config"}
 directory-client = { path = "../common/clients/directory-client" }
 healthcheck = {path = "../common/healthcheck" }
 topology = {path = "../common/topology"}
@@ -29,3 +31,8 @@ topology = {path = "../common/topology"}
 built = "0.3.2"
 
 [dev-dependencies]
+tempfile = "3.1.0"
+
+[features]
+qa = []
+local = []

validator/README.md

@@ -0,0 +1,21 @@
+Nym Validator
+=============
+
+The Nym Validator has several jobs: 
+
+* use Tendermint (v0.33.0) to maintain a total global ordering of incoming transactions
+* track quality of service for mixnet nodes (mixmining)
+* generate Coconut credentials and ensure they're not double spent
+* maintain a decentralized directory of all Nym nodes that have staked into the system
+  
+Some of these functions may be moved away to their own node types in the future, for example to increase scalability or performance. At the moment, we'd like to keep deployments simple, so they're all in the validator node.
+
+Running the validator on your local machine
+-------------------------------------------
+
+1. Download and install [Tendermint 0.32.7](https://github.com/tendermint/tendermint/releases/tag/v0.32.7)
+2. `tendermint init` sets up Tendermint for use
+3. `tendermint node` runs Tendermint. You'll get errors until you run the Nym validator, this is normal :).
+4. `cp sample-configs/validator-config.toml.sample sample-configs/validator-config.toml`
+5. `cargo run -- run --config ../sample-configs/validator-config.toml` builds the Nym Validator and runs it
+