From 9f69cc92fef05915fc720fb63cf2dd58ee1c3f03 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:04:20 +0000 Subject: [PATCH 001/105] Moved MixTrafficController to separate module --- nym-client/src/client/mix_traffic.rs | 38 ++++++++++++++++++++ nym-client/src/{clients => client}/mod.rs | 43 +++-------------------- nym-client/src/commands/tcpsocket.rs | 2 +- nym-client/src/commands/websocket.rs | 2 +- nym-client/src/lib.rs | 2 +- nym-client/src/main.rs | 2 +- nym-client/src/persistence/pathfinder.rs | 2 +- nym-client/src/sockets/tcp.rs | 10 +++--- nym-client/src/sockets/ws.rs | 4 +-- 9 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 nym-client/src/client/mix_traffic.rs rename nym-client/src/{clients => client}/mod.rs (89%) diff --git a/nym-client/src/client/mix_traffic.rs b/nym-client/src/client/mix_traffic.rs new file mode 100644 index 0000000000..ef43645311 --- /dev/null +++ b/nym-client/src/client/mix_traffic.rs @@ -0,0 +1,38 @@ +use futures::channel::mpsc; +use futures::StreamExt; +use log::*; +use sphinx::SphinxPacket; +use std::io; +use std::io::prelude::*; +use std::net::SocketAddr; + +pub(crate) struct MixMessage(SocketAddr, SphinxPacket); + +impl MixMessage { + pub(crate) fn new(address: SocketAddr, packet: SphinxPacket) -> Self { + MixMessage(address, packet) + } +} + +pub(crate) struct MixTrafficController; + +impl MixTrafficController { + // this was way more difficult to implement than what this code may suggest... + pub(crate) async fn run(mut rx: mpsc::UnboundedReceiver) { + let mix_client = mix_client::MixClient::new(); + while let Some(mix_message) = rx.next().await { + info!( + "[MIX TRAFFIC CONTROL] - got a mix_message for {:?}", + mix_message.0 + ); + let send_res = mix_client.send(mix_message.1, mix_message.0).await; + match send_res { + Ok(_) => { + print!("."); + io::stdout().flush().ok().expect("Could not flush stdout"); + } + Err(e) => error!("We failed to send the message :( - {:?}", e), + }; + } + } +} diff --git a/nym-client/src/clients/mod.rs b/nym-client/src/client/mod.rs similarity index 89% rename from nym-client/src/clients/mod.rs rename to nym-client/src/client/mod.rs index 06d3c79ff1..bfd9e7198d 100644 --- a/nym-client/src/clients/mod.rs +++ b/nym-client/src/client/mod.rs @@ -1,4 +1,5 @@ use crate::built_info; +use crate::client::mix_traffic::{MixMessage, MixTrafficController}; use crate::sockets::tcp; use crate::sockets::ws; use crate::utils; @@ -11,54 +12,20 @@ use futures::{SinkExt, StreamExt}; use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::{Destination, DestinationAddressBytes}; -use sphinx::SphinxPacket; -use std::io; -use std::io::prelude::*; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; use topology::NymTopology; +mod mix_traffic; + const LOOP_COVER_AVERAGE_DELAY: f64 = 0.5; // seconds const MESSAGE_SENDING_AVERAGE_DELAY: f64 = 0.5; // seconds; const FETCH_MESSAGES_DELAY: f64 = 1.0; // seconds; -// provider-poller sends polls service provider; receives messages -// provider-poller sends (TX) to ReceivedBufferController (RX) -// ReceivedBufferController sends (TX) to ... ??Client?? -// outQueueController sends (TX) to TrafficStreamController (RX) -// TrafficStreamController sends messages to mixnet -// ... ??Client?? sends (TX) to outQueueController (RX) -// Loop cover traffic stream just sends messages to mixnet without any channel communication - -struct MixMessage(SocketAddr, SphinxPacket); - -struct MixTrafficController; - -impl MixTrafficController { - // this was way more difficult to implement than what this code may suggest... - async fn run(mut rx: mpsc::UnboundedReceiver) { - let mix_client = mix_client::MixClient::new(); - while let Some(mix_message) = rx.next().await { - info!( - "[MIX TRAFFIC CONTROL] - got a mix_message for {:?}", - mix_message.0 - ); - let send_res = mix_client.send(mix_message.1, mix_message.0).await; - match send_res { - Ok(_) => { - print!("."); - io::stdout().flush().ok().expect("Could not flush stdout"); - } - Err(e) => error!("We failed to send the message :( - {:?}", e), - }; - } - } -} - pub type BufferResponse = oneshot::Sender>>; struct ReceivedMessagesBuffer { @@ -163,7 +130,7 @@ impl NymClient { tokio::time::delay_for(delay_duration).await; let cover_message = utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology); - tx.send(MixMessage(cover_message.0, cover_message.1)) + tx.send(MixMessage::new(cover_message.0, cover_message.1)) .await .unwrap(); } @@ -197,7 +164,7 @@ impl NymClient { }; mix_tx - .send(MixMessage(traffic_message.0, traffic_message.1)) + .send(MixMessage::new(traffic_message.0, traffic_message.1)) .await .unwrap(); diff --git a/nym-client/src/commands/tcpsocket.rs b/nym-client/src/commands/tcpsocket.rs index 3ed3662c7b..053981d446 100644 --- a/nym-client/src/commands/tcpsocket.rs +++ b/nym-client/src/commands/tcpsocket.rs @@ -1,4 +1,4 @@ -use crate::clients::{NymClient, SocketType}; +use crate::client::{NymClient, SocketType}; use crate::persistence::pemstore; use clap::ArgMatches; diff --git a/nym-client/src/commands/websocket.rs b/nym-client/src/commands/websocket.rs index 99dbc1ecc6..0a66193a6f 100644 --- a/nym-client/src/commands/websocket.rs +++ b/nym-client/src/commands/websocket.rs @@ -1,4 +1,4 @@ -use crate::clients::{NymClient, SocketType}; +use crate::client::{NymClient, SocketType}; use crate::persistence::pemstore; use clap::ArgMatches; diff --git a/nym-client/src/lib.rs b/nym-client/src/lib.rs index 5fa9078897..a7760d6e36 100644 --- a/nym-client/src/lib.rs +++ b/nym-client/src/lib.rs @@ -1,7 +1,7 @@ #![recursion_limit = "256"] pub mod built_info; -pub mod clients; +pub mod client; pub mod commands; pub mod persistence; pub mod sockets; diff --git a/nym-client/src/main.rs b/nym-client/src/main.rs index 8455bc7287..64daf4e515 100644 --- a/nym-client/src/main.rs +++ b/nym-client/src/main.rs @@ -3,7 +3,7 @@ use clap::{App, Arg, ArgMatches, SubCommand}; pub mod built_info; -pub mod clients; +pub mod client; mod commands; mod persistence; mod sockets; diff --git a/nym-client/src/persistence/pathfinder.rs b/nym-client/src/persistence/pathfinder.rs index 08110b9ae8..3e5d82bb78 100644 --- a/nym-client/src/persistence/pathfinder.rs +++ b/nym-client/src/persistence/pathfinder.rs @@ -9,7 +9,7 @@ pub struct Pathfinder { impl Pathfinder { pub fn new(id: String) -> Pathfinder { let os_config_dir = dirs::config_dir().unwrap(); // grabs the OS default config dir - let config_dir = os_config_dir.join("nym").join("clients").join(id); + let config_dir = os_config_dir.join("nym").join("client").join(id); let private_mix_key = config_dir.join("private.pem"); let public_mix_key = config_dir.join("public.pem"); Pathfinder { diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index 66d5a3dbf6..a6cf018bec 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -1,5 +1,5 @@ -use crate::clients::BufferResponse; -use crate::clients::InputMessage; +use crate::client::BufferResponse; +use crate::client::InputMessage; use directory_client::presence::Topology; use futures::channel::{mpsc, oneshot}; use futures::future::FutureExt; @@ -130,7 +130,7 @@ impl ClientRequest { } async fn handle_get_clients(topology: &Topology) -> ServerResponse { - println!("get clients handle"); + println!("get client handle"); let clients = topology .mix_provider_nodes .iter() @@ -189,8 +189,8 @@ fn encode_fetched_messages(messages: Vec>) -> Vec { } fn encode_list_of_clients(clients: Vec>) -> Vec { - println!("clients: {:?}", clients); - // we can just concat all clients since all of them got to be 32 bytes long + println!("client: {:?}", clients); + // we can just concat all client since all of them got to be 32 bytes long // (if not, then we have bigger problem somewhere up the line) // converts [[1,2,3],[4,5,6],...] into [1,2,3,4,5,6,...] diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index ab01fca011..685964c898 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -1,5 +1,5 @@ -use crate::clients::BufferResponse; -use crate::clients::InputMessage; +use crate::client::BufferResponse; +use crate::client::InputMessage; use directory_client::presence::Topology; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::{mpsc, oneshot}; From abd88c0c01498ce1cfee765da02d65345da732e9 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:07:22 +0000 Subject: [PATCH 002/105] Fixed accidentally broken pathfinder --- nym-client/src/persistence/pathfinder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nym-client/src/persistence/pathfinder.rs b/nym-client/src/persistence/pathfinder.rs index 3e5d82bb78..08110b9ae8 100644 --- a/nym-client/src/persistence/pathfinder.rs +++ b/nym-client/src/persistence/pathfinder.rs @@ -9,7 +9,7 @@ pub struct Pathfinder { impl Pathfinder { pub fn new(id: String) -> Pathfinder { let os_config_dir = dirs::config_dir().unwrap(); // grabs the OS default config dir - let config_dir = os_config_dir.join("nym").join("client").join(id); + let config_dir = os_config_dir.join("nym").join("clients").join(id); let private_mix_key = config_dir.join("private.pem"); let public_mix_key = config_dir.join("public.pem"); Pathfinder { From 6625f6f7fee562db1c92d467f035ebbba59aa576 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:16:43 +0000 Subject: [PATCH 003/105] Slightly decreases the aggressive logging in mix traffic controller --- nym-client/src/client/mix_traffic.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/nym-client/src/client/mix_traffic.rs b/nym-client/src/client/mix_traffic.rs index ef43645311..bfcd3ddfc6 100644 --- a/nym-client/src/client/mix_traffic.rs +++ b/nym-client/src/client/mix_traffic.rs @@ -1,9 +1,7 @@ use futures::channel::mpsc; use futures::StreamExt; -use log::*; +use log::{debug, error, info, trace}; use sphinx::SphinxPacket; -use std::io; -use std::io::prelude::*; use std::net::SocketAddr; pub(crate) struct MixMessage(SocketAddr, SphinxPacket); @@ -17,21 +15,22 @@ impl MixMessage { pub(crate) struct MixTrafficController; impl MixTrafficController { - // this was way more difficult to implement than what this code may suggest... pub(crate) async fn run(mut rx: mpsc::UnboundedReceiver) { + info!("Mix Traffic Controller started!"); let mix_client = mix_client::MixClient::new(); while let Some(mix_message) = rx.next().await { - info!( - "[MIX TRAFFIC CONTROL] - got a mix_message for {:?}", - mix_message.0 - ); + debug!("Got a mix_message for {:?}", mix_message.0); let send_res = mix_client.send(mix_message.1, mix_message.0).await; match send_res { Ok(_) => { - print!("."); - io::stdout().flush().ok().expect("Could not flush stdout"); + trace!("sent a mix message"); } - Err(e) => error!("We failed to send the message :( - {:?}", e), + // TODO: should there be some kind of threshold of failed messages + // that if reached, the application blows? + Err(e) => error!( + "We failed to send the message to {} :( - {:?}", + mix_message.0, e + ), }; } } From e57cb4f56212c5caa955b050588bc75c9d0275b9 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:21:46 +0000 Subject: [PATCH 004/105] Release builds should no longer silently fail - everything will be immediately stopped on any panic --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index d03d085fa8..42a75a2b77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,6 @@ +[profile.release] +panic = "abort" + [workspace] members = [ From e9e5c83ab358737f46c96b82678fbcbac2fb8ab2 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:33:26 +0000 Subject: [PATCH 005/105] Moved ReceivedMessagesBuffer to seperate module --- nym-client/src/client/mod.rs | 56 ++---------------------- nym-client/src/client/received_buffer.rs | 55 +++++++++++++++++++++++ nym-client/src/sockets/tcp.rs | 2 +- nym-client/src/sockets/ws.rs | 2 +- 4 files changed, 60 insertions(+), 55 deletions(-) create mode 100644 nym-client/src/client/received_buffer.rs diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index bfd9e7198d..376e3a102c 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -4,21 +4,21 @@ use crate::sockets::tcp; use crate::sockets::ws; use crate::utils; use directory_client::presence::Topology; -use futures::channel::{mpsc, oneshot}; +use futures::channel::mpsc; use futures::join; -use futures::lock::Mutex as FMutex; use futures::select; use futures::{SinkExt, StreamExt}; use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::{Destination, DestinationAddressBytes}; use std::net::SocketAddr; -use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; use topology::NymTopology; +use crate::client::received_buffer::ReceivedMessagesBuffer; mod mix_traffic; +pub mod received_buffer; const LOOP_COVER_AVERAGE_DELAY: f64 = 0.5; // seconds @@ -26,56 +26,6 @@ const MESSAGE_SENDING_AVERAGE_DELAY: f64 = 0.5; // seconds; const FETCH_MESSAGES_DELAY: f64 = 1.0; // seconds; -pub type BufferResponse = oneshot::Sender>>; - -struct ReceivedMessagesBuffer { - messages: Vec>, -} - -impl ReceivedMessagesBuffer { - fn add_arc_futures_mutex(self) -> Arc> { - Arc::new(FMutex::new(self)) - } - - fn new() -> Self { - ReceivedMessagesBuffer { - messages: Vec::new(), - } - } - - async fn add_new_messages(buf: Arc>, msgs: Vec>) { - info!("Adding new messages to the buffer! {:?}", msgs); - let mut unlocked = buf.lock().await; - unlocked.messages.extend(msgs); - } - - async fn run_poller_input_controller( - buf: Arc>, - mut poller_rx: mpsc::UnboundedReceiver>>, - ) { - while let Some(new_messages) = poller_rx.next().await { - ReceivedMessagesBuffer::add_new_messages(buf.clone(), new_messages).await; - } - } - - async fn acquire_and_empty(buf: Arc>) -> Vec> { - let mut unlocked = buf.lock().await; - std::mem::replace(&mut unlocked.messages, Vec::new()) - } - - async fn run_query_output_controller( - buf: Arc>, - mut query_receiver: mpsc::UnboundedReceiver, - ) { - while let Some(request) = query_receiver.next().await { - let messages = ReceivedMessagesBuffer::acquire_and_empty(buf.clone()).await; - // if this fails, the whole application needs to blow - // because currently only this thread would fail - request.send(messages).unwrap(); - } - } -} - pub enum SocketType { TCP, WebSocket, diff --git a/nym-client/src/client/received_buffer.rs b/nym-client/src/client/received_buffer.rs new file mode 100644 index 0000000000..4c6d460290 --- /dev/null +++ b/nym-client/src/client/received_buffer.rs @@ -0,0 +1,55 @@ +use futures::channel::{mpsc, oneshot}; +use futures::lock::Mutex as FMutex; +use futures::StreamExt; +use log::info; +use std::sync::Arc; + +pub type BufferResponse = oneshot::Sender>>; + +pub(crate) struct ReceivedMessagesBuffer { + messages: Vec>, +} + +impl ReceivedMessagesBuffer { + pub(crate) fn new() -> Self { + ReceivedMessagesBuffer { + messages: Vec::new(), + } + } + + pub(crate) fn add_arc_futures_mutex(self) -> Arc> { + Arc::new(FMutex::new(self)) + } + + pub(crate) async fn add_new_messages(buf: Arc>, msgs: Vec>) { + info!("Adding new messages to the buffer! {:?}", msgs); + let mut unlocked = buf.lock().await; + unlocked.messages.extend(msgs); + } + + pub(crate) async fn run_poller_input_controller( + buf: Arc>, + mut poller_rx: mpsc::UnboundedReceiver>>, + ) { + while let Some(new_messages) = poller_rx.next().await { + ReceivedMessagesBuffer::add_new_messages(buf.clone(), new_messages).await; + } + } + + pub(crate) async fn acquire_and_empty(buf: Arc>) -> Vec> { + let mut unlocked = buf.lock().await; + std::mem::replace(&mut unlocked.messages, Vec::new()) + } + + pub(crate) async fn run_query_output_controller( + buf: Arc>, + mut query_receiver: mpsc::UnboundedReceiver, + ) { + while let Some(request) = query_receiver.next().await { + let messages = ReceivedMessagesBuffer::acquire_and_empty(buf.clone()).await; + // if this fails, the whole application needs to blow + // because currently only this thread would fail + request.send(messages).unwrap(); + } + } +} diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index a6cf018bec..c063c7e4b8 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -1,4 +1,4 @@ -use crate::client::BufferResponse; +use crate::client::received_buffer::BufferResponse; use crate::client::InputMessage; use directory_client::presence::Topology; use futures::channel::{mpsc, oneshot}; diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 685964c898..14aacdcb5d 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -1,4 +1,4 @@ -use crate::client::BufferResponse; +use crate::client::received_buffer::BufferResponse; use crate::client::InputMessage; use directory_client::presence::Topology; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; From af6a8f5546a4e12eda05708dca61d6e6f5195063 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:49:56 +0000 Subject: [PATCH 006/105] Better logging for ibid. --- nym-client/src/client/received_buffer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nym-client/src/client/received_buffer.rs b/nym-client/src/client/received_buffer.rs index 4c6d460290..40808bf66e 100644 --- a/nym-client/src/client/received_buffer.rs +++ b/nym-client/src/client/received_buffer.rs @@ -1,7 +1,7 @@ use futures::channel::{mpsc, oneshot}; use futures::lock::Mutex as FMutex; use futures::StreamExt; -use log::info; +use log::{error, info, trace}; use std::sync::Arc; pub type BufferResponse = oneshot::Sender>>; @@ -22,7 +22,7 @@ impl ReceivedMessagesBuffer { } pub(crate) async fn add_new_messages(buf: Arc>, msgs: Vec>) { - info!("Adding new messages to the buffer! {:?}", msgs); + trace!("Adding new messages to the buffer! {:?}", msgs); let mut unlocked = buf.lock().await; unlocked.messages.extend(msgs); } @@ -31,12 +31,14 @@ impl ReceivedMessagesBuffer { buf: Arc>, mut poller_rx: mpsc::UnboundedReceiver>>, ) { + info!("Started Received Messages Buffer Input Controller"); while let Some(new_messages) = poller_rx.next().await { ReceivedMessagesBuffer::add_new_messages(buf.clone(), new_messages).await; } } pub(crate) async fn acquire_and_empty(buf: Arc>) -> Vec> { + trace!("Emptying the buffer and returning all messages"); let mut unlocked = buf.lock().await; std::mem::replace(&mut unlocked.messages, Vec::new()) } @@ -45,6 +47,8 @@ impl ReceivedMessagesBuffer { buf: Arc>, mut query_receiver: mpsc::UnboundedReceiver, ) { + info!("Started Received Messages Buffer Output Controller"); + while let Some(request) = query_receiver.next().await { let messages = ReceivedMessagesBuffer::acquire_and_empty(buf.clone()).await; // if this fails, the whole application needs to blow From e4c7d211b5acea584a55a6fa60b33cbcb7b935c0 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 10:50:34 +0000 Subject: [PATCH 007/105] Better handling of when messages failed to be sent to the requester Now rather than causing a panic, they will simply be readded back to the buffer --- nym-client/src/client/received_buffer.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nym-client/src/client/received_buffer.rs b/nym-client/src/client/received_buffer.rs index 40808bf66e..3dcb0627f7 100644 --- a/nym-client/src/client/received_buffer.rs +++ b/nym-client/src/client/received_buffer.rs @@ -51,9 +51,12 @@ impl ReceivedMessagesBuffer { while let Some(request) = query_receiver.next().await { let messages = ReceivedMessagesBuffer::acquire_and_empty(buf.clone()).await; - // if this fails, the whole application needs to blow - // because currently only this thread would fail - request.send(messages).unwrap(); + if let Err(failed_messages) = request.send(messages) { + error!( + "Failed to send the messages to the requester. Adding them back to the buffer" + ); + ReceivedMessagesBuffer::add_new_messages(buf.clone(), failed_messages).await; + } } } } From ea383871ab6690be230b0dd10dc893a545daf96f Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 12:02:21 +0000 Subject: [PATCH 008/105] ReceivedBuffer is now responsible for starting both controllers --- nym-client/src/client/mod.rs | 22 +++----- nym-client/src/client/received_buffer.rs | 67 +++++++++++++++++------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 376e3a102c..0be475c82e 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -1,5 +1,6 @@ use crate::built_info; use crate::client::mix_traffic::{MixMessage, MixTrafficController}; +use crate::client::received_buffer::ReceivedMessagesBuffer; use crate::sockets::tcp; use crate::sockets::ws; use crate::utils; @@ -15,7 +16,6 @@ use std::net::SocketAddr; use std::time::Duration; use tokio::runtime::Runtime; use topology::NymTopology; -use crate::client::received_buffer::ReceivedMessagesBuffer; mod mix_traffic; pub mod received_buffer; @@ -223,18 +223,10 @@ impl NymClient { let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); - let received_messages_buffer = ReceivedMessagesBuffer::new().add_arc_futures_mutex(); - - let received_messages_buffer_input_controller_future = - rt.spawn(ReceivedMessagesBuffer::run_poller_input_controller( - received_messages_buffer.clone(), - poller_input_rx, - )); - let received_messages_buffer_output_controller_future = - rt.spawn(ReceivedMessagesBuffer::run_query_output_controller( - received_messages_buffer, - received_messages_buffer_output_rx, - )); + let received_messages_buffer_controllers_future = rt.spawn( + ReceivedMessagesBuffer::new() + .start_controllers(poller_input_rx, received_messages_buffer_output_rx), + ); let mix_traffic_future = rt.spawn(MixTrafficController::run(mix_rx)); let loop_cover_traffic_future = rt.spawn(NymClient::start_loop_cover_traffic_stream( @@ -279,8 +271,7 @@ impl NymClient { rt.block_on(async { let future_results = join!( - received_messages_buffer_input_controller_future, - received_messages_buffer_output_controller_future, + received_messages_buffer_controllers_future, mix_traffic_future, loop_cover_traffic_future, out_queue_control_future, @@ -293,7 +284,6 @@ impl NymClient { && future_results.2.is_ok() && future_results.3.is_ok() && future_results.4.is_ok() - && future_results.5.is_ok() ); }); diff --git a/nym-client/src/client/received_buffer.rs b/nym-client/src/client/received_buffer.rs index 3dcb0627f7..a6df580c9f 100644 --- a/nym-client/src/client/received_buffer.rs +++ b/nym-client/src/client/received_buffer.rs @@ -7,56 +7,83 @@ use std::sync::Arc; pub type BufferResponse = oneshot::Sender>>; pub(crate) struct ReceivedMessagesBuffer { - messages: Vec>, + inner: Arc>, } impl ReceivedMessagesBuffer { pub(crate) fn new() -> Self { ReceivedMessagesBuffer { - messages: Vec::new(), + inner: Arc::new(FMutex::new(Inner::new())), } } - pub(crate) fn add_arc_futures_mutex(self) -> Arc> { - Arc::new(FMutex::new(self)) - } + pub(crate) async fn start_controllers( + self, + poller_rx: mpsc::UnboundedReceiver>>, // to receive new messages + query_receiver: mpsc::UnboundedReceiver, // to receive requests to acquire all stored messages + ) { + let input_controller_future = tokio::spawn(Self::run_poller_input_controller( + self.inner.clone(), + poller_rx, + )); + let output_controller_future = tokio::spawn(Self::run_query_output_controller( + self.inner, + query_receiver, + )); - pub(crate) async fn add_new_messages(buf: Arc>, msgs: Vec>) { - trace!("Adding new messages to the buffer! {:?}", msgs); - let mut unlocked = buf.lock().await; - unlocked.messages.extend(msgs); + futures::future::select(input_controller_future, output_controller_future).await; + panic!("One of the received buffer controllers failed!") } pub(crate) async fn run_poller_input_controller( - buf: Arc>, + buf: Arc>, mut poller_rx: mpsc::UnboundedReceiver>>, ) { info!("Started Received Messages Buffer Input Controller"); + while let Some(new_messages) = poller_rx.next().await { - ReceivedMessagesBuffer::add_new_messages(buf.clone(), new_messages).await; + Inner::add_new_messages(&*buf, new_messages).await; } } - pub(crate) async fn acquire_and_empty(buf: Arc>) -> Vec> { - trace!("Emptying the buffer and returning all messages"); - let mut unlocked = buf.lock().await; - std::mem::replace(&mut unlocked.messages, Vec::new()) - } - pub(crate) async fn run_query_output_controller( - buf: Arc>, + buf: Arc>, mut query_receiver: mpsc::UnboundedReceiver, ) { info!("Started Received Messages Buffer Output Controller"); while let Some(request) = query_receiver.next().await { - let messages = ReceivedMessagesBuffer::acquire_and_empty(buf.clone()).await; + let messages = Inner::acquire_and_empty(&*buf).await; if let Err(failed_messages) = request.send(messages) { error!( "Failed to send the messages to the requester. Adding them back to the buffer" ); - ReceivedMessagesBuffer::add_new_messages(buf.clone(), failed_messages).await; + Inner::add_new_messages(&*buf, failed_messages).await; } } } } + +pub(crate) struct Inner { + messages: Vec>, +} + +impl Inner { + fn new() -> Self { + Inner { + messages: Vec::new(), + } + } + + async fn add_new_messages(buf: &FMutex, msgs: Vec>) { + trace!("Adding new messages to the buffer! {:?}", msgs); + let mut unlocked = buf.lock().await; + unlocked.messages.extend(msgs); + } + + async fn acquire_and_empty(buf: &FMutex) -> Vec> { + trace!("Emptying the buffer and returning all messages"); + let mut unlocked = buf.lock().await; + std::mem::replace(&mut unlocked.messages, Vec::new()) + } +} From 2af18a85e551160d35342fbe71d7061892a28b90 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 12:26:10 +0000 Subject: [PATCH 009/105] Added "is_registered" method to provider client --- common/clients/provider-client/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index 643770c1dc..a3faf05617 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -111,4 +111,8 @@ impl ProviderClient { Ok(parsed_response.auth_token) } + + pub fn is_registered(&self) -> bool { + self.auth_token.is_some() + } } From 7889ab4778f0e8f7802843882485e470b210e5f9 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 13:07:51 +0000 Subject: [PATCH 010/105] Removed redundant logging statement --- common/clients/provider-client/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index a3faf05617..5ed8a8a4f0 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -92,7 +92,6 @@ impl ProviderClient { let bytes = pull_request.to_bytes(); let response = self.send_request(bytes).await?; - info!("Received the following response: {:?}", response); let parsed_response = PullResponse::from_bytes(&response)?; Ok(parsed_response.messages) From 06b96bd2c1755cb8c85a0bb209c1e6ee62aad8ec Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 13:08:26 +0000 Subject: [PATCH 011/105] Moved provider-polling related functionalities to new module --- nym-client/src/client/mod.rs | 55 ++++----------- nym-client/src/client/provider_poller.rs | 89 ++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 41 deletions(-) create mode 100644 nym-client/src/client/provider_poller.rs diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 0be475c82e..ae69b29795 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -18,8 +18,10 @@ use tokio::runtime::Runtime; use topology::NymTopology; mod mix_traffic; +mod provider_poller; pub mod received_buffer; +// TODO: all of those constants should probably be moved to config file const LOOP_COVER_AVERAGE_DELAY: f64 = 0.5; // seconds const MESSAGE_SENDING_AVERAGE_DELAY: f64 = 0.5; @@ -123,26 +125,6 @@ impl NymClient { } } - async fn start_provider_polling( - provider_client: provider_client::ProviderClient, - mut poller_tx: mpsc::UnboundedSender>>, - ) { - let loop_message = &utils::sphinx::LOOP_COVER_MESSAGE_PAYLOAD.to_vec(); - let dummy_message = &sfw_provider_requests::DUMMY_MESSAGE_CONTENT.to_vec(); - loop { - let delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY); - tokio::time::delay_for(delay_duration).await; - info!("[FETCH MSG] - Polling provider..."); - let messages = provider_client.retrieve_messages().await.unwrap(); - let good_messages = messages - .into_iter() - .filter(|message| message != loop_message && message != dummy_message) - .collect(); - // if any of those fails, whole application should blow... - poller_tx.send(good_messages).await.unwrap(); - } - } - pub fn start(self) -> Result<(), Box> { let score_threshold = 0.0; println!("Starting nym client"); @@ -192,6 +174,12 @@ impl NymClient { panic!("No valid path exists in the topology"); } + // channels for intercomponent communication + let (mix_tx, mix_rx) = mpsc::unbounded(); + let (poller_input_tx, poller_input_rx) = mpsc::unbounded(); + let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = + mpsc::unbounded(); + // this is temporary and assumes there exists only a single provider. let provider_client_listener_address: SocketAddr = versioned_healthy_topology .get_mix_provider_nodes() @@ -199,29 +187,17 @@ impl NymClient { .expect("Could not get a provider from the supplied network topology, are you using the right directory server?") .client_listener; - let mut provider_client = provider_client::ProviderClient::new( + let mut provider_poller = provider_poller::ProviderPoller::new( + poller_input_tx, provider_client_listener_address, self.address, self.auth_token, ); // registration - rt.block_on(async { - match self.auth_token { - None => { - let auth_token = provider_client.register().await.unwrap(); - provider_client.update_token(auth_token); - info!("Obtained new token! - {:?}", auth_token); - } - Some(token) => println!("Already got the token! - {:?}", token), - } - }); - - // channels for intercomponent communication - let (mix_tx, mix_rx) = mpsc::unbounded(); - let (poller_input_tx, poller_input_rx) = mpsc::unbounded(); - let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = - mpsc::unbounded(); + if let Err(err) = rt.block_on(provider_poller.perform_initial_registration()) { + panic!("Failed to perform initial registration: {:?}", err); + }; let received_messages_buffer_controllers_future = rt.spawn( ReceivedMessagesBuffer::new() @@ -242,10 +218,7 @@ impl NymClient { versioned_healthy_topology.clone(), )); - let provider_polling_future = rt.spawn(NymClient::start_provider_polling( - provider_client, - poller_input_tx, - )); + let provider_polling_future = rt.spawn(provider_poller.start_provider_polling()); match self.socket_type { SocketType::WebSocket => { diff --git a/nym-client/src/client/provider_poller.rs b/nym-client/src/client/provider_poller.rs new file mode 100644 index 0000000000..f55b11cde0 --- /dev/null +++ b/nym-client/src/client/provider_poller.rs @@ -0,0 +1,89 @@ +use crate::client::FETCH_MESSAGES_DELAY; +use crate::utils; +use futures::channel::mpsc; +use log::{debug, error, info, trace, warn}; +use provider_client::ProviderClientError; +use sfw_provider_requests::AuthToken; +use sphinx::route::DestinationAddressBytes; +use std::net::SocketAddr; +use std::time::Duration; + +pub(crate) struct ProviderPoller { + provider_client: provider_client::ProviderClient, + poller_tx: mpsc::UnboundedSender>>, +} + +impl ProviderPoller { + pub(crate) fn new( + poller_tx: mpsc::UnboundedSender>>, + provider_client_listener_address: SocketAddr, + client_address: DestinationAddressBytes, + auth_token: Option, + ) -> Self { + ProviderPoller { + provider_client: provider_client::ProviderClient::new( + provider_client_listener_address, + client_address, + auth_token, + ), + poller_tx, + } + } + + // This method is only temporary until registration is moved to `client init` + pub(crate) async fn perform_initial_registration(&mut self) -> Result<(), ProviderClientError> { + debug!("performing initial provider registration"); + + if !self.provider_client.is_registered() { + let auth_token = match self.provider_client.register().await { + // in this particular case we can ignore this error + Err(ProviderClientError::ClientAlreadyRegisteredError) => return Ok(()), + Err(err) => return Err(err), + Ok(token) => token, + }; + + self.provider_client.update_token(auth_token) + } else { + warn!("did not perform registration - we were already registered") + } + + Ok(()) + } + + pub(crate) async fn start_provider_polling(mut self) { + info!("Starting provider poller"); + + let loop_message = &utils::sphinx::LOOP_COVER_MESSAGE_PAYLOAD.to_vec(); + let dummy_message = &sfw_provider_requests::DUMMY_MESSAGE_CONTENT.to_vec(); + + let delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY); + loop { + debug!("Polling provider..."); + + let messages = match self.provider_client.retrieve_messages().await { + Ok(messages) => messages, + Err(err) => { + let extended_delay_duration = + Duration::from_secs_f64(FETCH_MESSAGES_DELAY * 10.0); + error!("Failed to query the provider for messages... Going to wait {:?} before retrying", extended_delay_duration); + tokio::time::delay_for(extended_delay_duration).await; + continue; + } + }; + + let good_messages = messages + .into_iter() + .filter(|message| message != loop_message && message != dummy_message) + .collect(); + trace!("Obtained the following messages: {:?}", good_messages); + + // if this one fails, there's no retrying because it means that either: + // - we run out of memory + // - the receiver channel is closed + // in either case there's no recovery and we can only panic + self.poller_tx.unbounded_send(good_messages).unwrap(); + + tokio::time::delay_for(delay_duration).await; + } + } +} From f1debb91c6ec6199825b616d21b041aa31480ae0 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 13:29:59 +0000 Subject: [PATCH 012/105] Obtaining initial compatible topology in separate method --- nym-client/src/client/mod.rs | 51 +++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index ae69b29795..71d9928544 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -12,6 +12,7 @@ use futures::{SinkExt, StreamExt}; use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::{Destination, DestinationAddressBytes}; +use std::error::Error; use std::net::SocketAddr; use std::time::Duration; use tokio::runtime::Runtime; @@ -46,6 +47,13 @@ pub struct NymClient { socket_type: SocketType, } +// TODO: this will be moved into module responsible for refreshing topology +#[derive(Debug)] +enum TopologyError { + HealthCheckError, + NoValidPathsError, +} + #[derive(Debug)] pub struct InputMessage(pub Destination, pub Vec); @@ -125,12 +133,11 @@ impl NymClient { } } - pub fn start(self) -> Result<(), Box> { + // TODO: this will be moved into module responsible for refreshing topology + async fn get_compatible_topology(&self) -> Result { let score_threshold = 0.0; - println!("Starting nym client"); - let mut rt = Runtime::new()?; + info!("Trying to obtain valid, healthy, topology"); - println!("Trying to obtain valid, healthy, topology"); let full_topology = Topology::new(self.directory.clone()); // run a healthcheck to determine healthy-ish nodes: @@ -143,15 +150,11 @@ impl NymClient { num_test_packets: 2, }; let healthcheck = healthcheck::HealthChecker::new(healthcheck_config); - let healthcheck_result = rt.block_on(healthcheck.do_check()); + let healthcheck_result = healthcheck.do_check().await; let healthcheck_scores = match healthcheck_result { Err(err) => { - error!( - "failed to perform healthcheck to determine healthy topology - {:?}", - err - ); - return Err(Box::new(err)); + return Err(TopologyError::HealthCheckError); } Ok(scores) => scores, }; @@ -169,19 +172,31 @@ impl NymClient { // make sure you can still send a packet through the network: if !versioned_healthy_topology.can_construct_path_through() { - error!("No valid path exists in the topology"); - // TODO: replace panic with proper return type - panic!("No valid path exists in the topology"); + return Err(TopologyError::NoValidPathsError); } + Ok(versioned_healthy_topology) + } + + pub fn start(self) -> Result<(), Box> { + info!("Starting nym client"); + let mut rt = Runtime::new()?; + // channels for intercomponent communication let (mix_tx, mix_rx) = mpsc::unbounded(); let (poller_input_tx, poller_input_rx) = mpsc::unbounded(); let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); + let initial_topology = match rt.block_on(self.get_compatible_topology()) { + Ok(topology) => topology, + Err(err) => { + panic!("Failed to obtain initial network topology: {:?}", err); + } + }; + // this is temporary and assumes there exists only a single provider. - let provider_client_listener_address: SocketAddr = versioned_healthy_topology + let provider_client_listener_address: SocketAddr = initial_topology .get_mix_provider_nodes() .first() .expect("Could not get a provider from the supplied network topology, are you using the right directory server?") @@ -208,14 +223,14 @@ impl NymClient { let loop_cover_traffic_future = rt.spawn(NymClient::start_loop_cover_traffic_stream( mix_tx.clone(), Destination::new(self.address, Default::default()), - versioned_healthy_topology.clone(), + initial_topology.clone(), )); let out_queue_control_future = rt.spawn(NymClient::control_out_queue( mix_tx, self.input_rx, Destination::new(self.address, Default::default()), - versioned_healthy_topology.clone(), + initial_topology.clone(), )); let provider_polling_future = rt.spawn(provider_poller.start_provider_polling()); @@ -227,7 +242,7 @@ impl NymClient { self.input_tx, received_messages_buffer_output_tx, self.address, - versioned_healthy_topology, + initial_topology, )); } SocketType::TCP => { @@ -236,7 +251,7 @@ impl NymClient { self.input_tx, received_messages_buffer_output_tx, self.address, - versioned_healthy_topology, + initial_topology, )); } SocketType::None => (), From a57708d9362c552964003504f5bbeb632dd1e4dd Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 14:21:41 +0000 Subject: [PATCH 013/105] Separate module for cover traffic stream --- nym-client/src/client/cover_traffic_stream.rs | 33 +++++++++++++++++++ nym-client/src/client/mod.rs | 31 ++++------------- 2 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 nym-client/src/client/cover_traffic_stream.rs diff --git a/nym-client/src/client/cover_traffic_stream.rs b/nym-client/src/client/cover_traffic_stream.rs new file mode 100644 index 0000000000..3405cc2cf9 --- /dev/null +++ b/nym-client/src/client/cover_traffic_stream.rs @@ -0,0 +1,33 @@ +use crate::client::mix_traffic::MixMessage; +use crate::client::LOOP_COVER_AVERAGE_DELAY; +use crate::utils; +use futures::channel::mpsc; +use log::{info, trace}; +use sphinx::route::Destination; +use std::time::Duration; +use topology::NymTopology; + +pub(crate) async fn start_loop_cover_traffic_stream( + tx: mpsc::UnboundedSender, + our_info: Destination, + topology: T, +) where + T: NymTopology, +{ + info!("Starting loop cover traffic stream"); + loop { + trace!("next cover message!"); + let delay = utils::poisson::sample(LOOP_COVER_AVERAGE_DELAY); + let delay_duration = Duration::from_secs_f64(delay); + tokio::time::delay_for(delay_duration).await; + let cover_message = + utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology); + + // if this one fails, there's no retrying because it means that either: + // - we run out of memory + // - the receiver channel is closed + // in either case there's no recovery and we can only panic + tx.unbounded_send(MixMessage::new(cover_message.0, cover_message.1)) + .unwrap(); + } +} diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 71d9928544..d992e0e110 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -12,12 +12,12 @@ use futures::{SinkExt, StreamExt}; use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::{Destination, DestinationAddressBytes}; -use std::error::Error; use std::net::SocketAddr; use std::time::Duration; use tokio::runtime::Runtime; use topology::NymTopology; +mod cover_traffic_stream; mod mix_traffic; mod provider_poller; pub mod received_buffer; @@ -78,24 +78,6 @@ impl NymClient { } } - async fn start_loop_cover_traffic_stream( - mut tx: mpsc::UnboundedSender, - our_info: Destination, - topology: Topology, - ) { - loop { - info!("[LOOP COVER TRAFFIC STREAM] - next cover message!"); - let delay = utils::poisson::sample(LOOP_COVER_AVERAGE_DELAY); - let delay_duration = Duration::from_secs_f64(delay); - tokio::time::delay_for(delay_duration).await; - let cover_message = - utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology); - tx.send(MixMessage::new(cover_message.0, cover_message.1)) - .await - .unwrap(); - } - } - async fn control_out_queue( mut mix_tx: mpsc::UnboundedSender, mut input_rx: mpsc::UnboundedReceiver, @@ -220,11 +202,12 @@ impl NymClient { ); let mix_traffic_future = rt.spawn(MixTrafficController::run(mix_rx)); - let loop_cover_traffic_future = rt.spawn(NymClient::start_loop_cover_traffic_stream( - mix_tx.clone(), - Destination::new(self.address, Default::default()), - initial_topology.clone(), - )); + let loop_cover_traffic_future = + rt.spawn(cover_traffic_stream::start_loop_cover_traffic_stream( + mix_tx.clone(), + Destination::new(self.address, Default::default()), + initial_topology.clone(), + )); let out_queue_control_future = rt.spawn(NymClient::control_out_queue( mix_tx, From fe48c11cd6ffe157e66d43e22f9afad78958212c Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 14:29:56 +0000 Subject: [PATCH 014/105] Moved real traffic controller to separate module --- nym-client/src/client/mod.rs | 50 +++---------------- nym-client/src/client/real_traffic_stream.rs | 51 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 44 deletions(-) create mode 100644 nym-client/src/client/real_traffic_stream.rs diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index d992e0e110..3ee95debd7 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -1,25 +1,22 @@ use crate::built_info; -use crate::client::mix_traffic::{MixMessage, MixTrafficController}; +use crate::client::mix_traffic::MixTrafficController; use crate::client::received_buffer::ReceivedMessagesBuffer; use crate::sockets::tcp; use crate::sockets::ws; -use crate::utils; use directory_client::presence::Topology; use futures::channel::mpsc; use futures::join; -use futures::select; -use futures::{SinkExt, StreamExt}; use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::{Destination, DestinationAddressBytes}; use std::net::SocketAddr; -use std::time::Duration; use tokio::runtime::Runtime; use topology::NymTopology; mod cover_traffic_stream; mod mix_traffic; mod provider_poller; +mod real_traffic_stream; pub mod received_buffer; // TODO: all of those constants should probably be moved to config file @@ -38,8 +35,10 @@ pub enum SocketType { pub struct NymClient { // to be replaced by something else I guess address: DestinationAddressBytes, - pub input_tx: mpsc::UnboundedSender, + // to be used by "send" function or socket, etc + pub input_tx: mpsc::UnboundedSender, + input_rx: mpsc::UnboundedReceiver, socket_listening_address: SocketAddr, directory: String, @@ -78,43 +77,6 @@ impl NymClient { } } - async fn control_out_queue( - mut mix_tx: mpsc::UnboundedSender, - mut input_rx: mpsc::UnboundedReceiver, - our_info: Destination, - topology: Topology, - ) { - loop { - info!("[OUT QUEUE] here I will be sending real traffic (or loop cover if nothing is available)"); - // TODO: consider replacing select macro with our own proper future definition with polling - let traffic_message = select! { - real_message = input_rx.next() => { - info!("[OUT QUEUE] - we got a real message!"); - if real_message.is_none() { - error!("Unexpected 'None' real message!"); - std::process::exit(1); - } - let real_message = real_message.unwrap(); - println!("real: {:?}", real_message); - utils::sphinx::encapsulate_message(real_message.0, real_message.1, &topology) - }, - - default => { - info!("[OUT QUEUE] - no real message - going to send extra loop cover"); - utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology) - } - }; - - mix_tx - .send(MixMessage::new(traffic_message.0, traffic_message.1)) - .await - .unwrap(); - - let delay_duration = Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY); - tokio::time::delay_for(delay_duration).await; - } - } - // TODO: this will be moved into module responsible for refreshing topology async fn get_compatible_topology(&self) -> Result { let score_threshold = 0.0; @@ -209,7 +171,7 @@ impl NymClient { initial_topology.clone(), )); - let out_queue_control_future = rt.spawn(NymClient::control_out_queue( + let out_queue_control_future = rt.spawn(real_traffic_stream::control_out_queue( mix_tx, self.input_rx, Destination::new(self.address, Default::default()), diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs new file mode 100644 index 0000000000..fa1d3efdc9 --- /dev/null +++ b/nym-client/src/client/real_traffic_stream.rs @@ -0,0 +1,51 @@ +use crate::client::mix_traffic::MixMessage; +use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY}; +use crate::utils; +use futures::channel::mpsc; +use futures::{select, StreamExt}; +use log::{debug, error, info, trace}; +use sphinx::route::Destination; +use std::time::Duration; +use topology::NymTopology; + +pub(crate) async fn control_out_queue( + mix_tx: mpsc::UnboundedSender, + mut input_rx: mpsc::UnboundedReceiver, + our_info: Destination, + topology: T, +) where + T: NymTopology, +{ + info!("Starting out queue controller where real traffic (or loop cover if nothing is available) will be sent"); + loop { + // TODO: consider replacing select macro with our own proper future definition with polling + let traffic_message = select! { + real_message = input_rx.next() => { + debug!("we got a real message!"); + if real_message.is_none() { + error!("Unexpected 'None' real message!"); + std::process::exit(1); + } + let real_message = real_message.unwrap(); + trace!("real message: {:?}", real_message); + utils::sphinx::encapsulate_message(real_message.0, real_message.1, &topology) + }, + + default => { + debug!("no real message - going to send extra loop cover"); + utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology) + } + }; + + // if this one fails, there's no retrying because it means that either: + // - we run out of memory + // - the receiver channel is closed + // in either case there's no recovery and we can only panic + mix_tx + .unbounded_send(MixMessage::new(traffic_message.0, traffic_message.1)) + .unwrap(); + + let delay_duration = Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY); + tokio::time::delay_for(delay_duration).await; + } +} From ee812a86c6c34fda62ad9f77b1a6ceb6c02883aa Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 14:30:17 +0000 Subject: [PATCH 015/105] Moved definition of extended duration to before the loop --- nym-client/src/client/provider_poller.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nym-client/src/client/provider_poller.rs b/nym-client/src/client/provider_poller.rs index f55b11cde0..20d88a049d 100644 --- a/nym-client/src/client/provider_poller.rs +++ b/nym-client/src/client/provider_poller.rs @@ -57,18 +57,18 @@ impl ProviderPoller { let dummy_message = &sfw_provider_requests::DUMMY_MESSAGE_CONTENT.to_vec(); let delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY); + let extended_delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY * 10.0); + loop { debug!("Polling provider..."); let messages = match self.provider_client.retrieve_messages().await { - Ok(messages) => messages, Err(err) => { - let extended_delay_duration = - Duration::from_secs_f64(FETCH_MESSAGES_DELAY * 10.0); error!("Failed to query the provider for messages... Going to wait {:?} before retrying", extended_delay_duration); tokio::time::delay_for(extended_delay_duration).await; continue; } + Ok(messages) => messages, }; let good_messages = messages From 641dc78903a6cb9a83d6159c0cf7060ec413e7b3 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:07:21 +0000 Subject: [PATCH 016/105] Removed too aggressive logging --- common/clients/mix-client/src/lib.rs | 3 +-- common/clients/provider-client/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/clients/mix-client/src/lib.rs b/common/clients/mix-client/src/lib.rs index 431623178a..bbdfa9bc0b 100644 --- a/common/clients/mix-client/src/lib.rs +++ b/common/clients/mix-client/src/lib.rs @@ -17,8 +17,7 @@ impl MixClient { mix_addr: SocketAddr, ) -> Result<(), Box> { let bytes = packet.to_bytes(); - - info!("socket addr: {:?}", mix_addr); + debug!("Sending to the following address: {:?}", mix_addr); let mut stream = tokio::net::TcpStream::connect(mix_addr).await?; stream.write_all(&bytes[..]).await?; diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index 5ed8a8a4f0..1200a18fcd 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -65,7 +65,7 @@ impl ProviderClient { pub async fn send_request(&self, bytes: Vec) -> Result, ProviderClientError> { let mut socket = tokio::net::TcpStream::connect(self.provider_network_address).await?; - info!("keep alive: {:?}", socket.keepalive()); + socket.set_keepalive(Some(Duration::from_secs(2))).unwrap(); socket.write_all(&bytes[..]).await?; if let Err(_e) = socket.shutdown(Shutdown::Write) { From c92177402d53b8f6832ba16e2a0679e9b4f0166f Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:08:19 +0000 Subject: [PATCH 017/105] Implemented OutQueueControl as a proper Stream to get rid of the select! macro --- nym-client/src/client/real_traffic_stream.rs | 114 +++++++++++++------ 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index fa1d3efdc9..cc00ea7ef0 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -1,51 +1,95 @@ use crate::client::mix_traffic::MixMessage; use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY}; use crate::utils; +use directory_client::presence::Topology; use futures::channel::mpsc; -use futures::{select, StreamExt}; -use log::{debug, error, info, trace}; +use futures::task::{Context, Poll}; +use futures::{select, Stream, StreamExt}; +use log::{debug, error, info, trace, warn}; use sphinx::route::Destination; +use sphinx::SphinxPacket; +use std::net::SocketAddr; +use std::pin::Pin; use std::time::Duration; +use tokio::time; use topology::NymTopology; -pub(crate) async fn control_out_queue( +pub(crate) struct OutQueueControl { + interval: time::Interval, mix_tx: mpsc::UnboundedSender, - mut input_rx: mpsc::UnboundedReceiver, + input_rx: mpsc::UnboundedReceiver, our_info: Destination, - topology: T, -) where - T: NymTopology, -{ - info!("Starting out queue controller where real traffic (or loop cover if nothing is available) will be sent"); - loop { - // TODO: consider replacing select macro with our own proper future definition with polling - let traffic_message = select! { - real_message = input_rx.next() => { - debug!("we got a real message!"); - if real_message.is_none() { - error!("Unexpected 'None' real message!"); - std::process::exit(1); - } - let real_message = real_message.unwrap(); - trace!("real message: {:?}", real_message); - utils::sphinx::encapsulate_message(real_message.0, real_message.1, &topology) - }, - default => { - debug!("no real message - going to send extra loop cover"); - utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology) - } + // due to pinning, DerefMut trait, futures, etc its way easier to + // just have concrete implementation here rather than generic NymTopology + // considering that it will be replaced with refreshing topology within few days anyway + topology: Topology, +} + +impl Stream for OutQueueControl { + type Item = (SocketAddr, SphinxPacket); + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // it is not yet time to return a message + if Stream::poll_next(Pin::new(&mut self.interval), cx).is_pending() { + return Poll::Pending; }; - // if this one fails, there's no retrying because it means that either: - // - we run out of memory - // - the receiver channel is closed - // in either case there's no recovery and we can only panic - mix_tx - .unbounded_send(MixMessage::new(traffic_message.0, traffic_message.1)) - .unwrap(); + match Stream::poll_next(Pin::new(&mut self.input_rx), cx) { + // in the case our real message channel stream was closed, we should also indicate we are closed + // (and whoever is using the stream should panic) + Poll::Ready(None) => Poll::Ready(None), - let delay_duration = Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY); - tokio::time::delay_for(delay_duration).await; + // if there's an actual message - return it + Poll::Ready(Some(real_message)) => { + trace!("real message"); + Poll::Ready(Some(utils::sphinx::encapsulate_message( + real_message.0, + real_message.1, + &self.topology, + ))) + } + + // otherwise construct a dummy one + _ => { + trace!("loop cover message"); + Poll::Ready(Some(utils::sphinx::loop_cover_message( + self.our_info.address, + self.our_info.identifier, + &self.topology, + ))) + } + } + } +} + +impl OutQueueControl { + pub(crate) fn new( + mix_tx: mpsc::UnboundedSender, + input_rx: mpsc::UnboundedReceiver, + our_info: Destination, + topology: Topology, + ) -> Self { + OutQueueControl { + interval: time::interval(Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY)), + mix_tx, + input_rx, + our_info, + topology, + } + } + + pub(crate) async fn run_out_queue_control(mut self) { + info!("starting out queue controller"); + while let Some(next_message) = self.next().await { + debug!("created new message"); + // if this one fails, there's no retrying because it means that either: + // - we run out of memory + // - the receiver channel is closed + // in either case there's no recovery and we can only panic + self.mix_tx + .unbounded_send(MixMessage::new(next_message.0, next_message.1)) + .unwrap(); + } } } From 73438b672a8fd2da72f2282ddbdcc68fa91cb8e3 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:09:12 +0000 Subject: [PATCH 018/105] Change in how OutQueueControl is spawned - not entirely sure at this point why its required --- nym-client/src/client/mod.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 3ee95debd7..786c36e7bc 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -171,13 +171,20 @@ impl NymClient { initial_topology.clone(), )); - let out_queue_control_future = rt.spawn(real_traffic_stream::control_out_queue( - mix_tx, - self.input_rx, - Destination::new(self.address, Default::default()), - initial_topology.clone(), - )); + let topology_clone = initial_topology.clone(); + let self_address = self.address; + let input_rx = self.input_rx; + let out_queue_control_future = rt.spawn(async move { + real_traffic_stream::OutQueueControl::new( + mix_tx, + input_rx, + Destination::new(self_address, Default::default()), + topology_clone, + ) + .run_out_queue_control() + .await + }); let provider_polling_future = rt.spawn(provider_poller.start_provider_polling()); match self.socket_type { From 2e817a9a6b401409bcf88803cdbfb0560ef686b2 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:09:28 +0000 Subject: [PATCH 019/105] Comments explaining purposes of different futures --- nym-client/src/client/mod.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 786c36e7bc..b02ba0bd0b 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -126,12 +126,13 @@ impl NymClient { info!("Starting nym client"); let mut rt = Runtime::new()?; - // channels for intercomponent communication + // channels for inter-component communication let (mix_tx, mix_rx) = mpsc::unbounded(); let (poller_input_tx, poller_input_rx) = mpsc::unbounded(); let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); + // get initial topology; already filtered by health and version let initial_topology = match rt.block_on(self.get_compatible_topology()) { Ok(topology) => topology, Err(err) => { @@ -158,12 +159,20 @@ impl NymClient { panic!("Failed to perform initial registration: {:?}", err); }; + // setup all of futures for the components running on the client + + // buffer controlling all messages fetched from provider + // required so that other components would be able to use them (say the websocket) let received_messages_buffer_controllers_future = rt.spawn( ReceivedMessagesBuffer::new() .start_controllers(poller_input_rx, received_messages_buffer_output_rx), ); + // controller for sending sphinx packets to mixnet (either real traffic or cover traffic) let mix_traffic_future = rt.spawn(MixTrafficController::run(mix_rx)); + + // future constantly pumping loop cover traffic at some specified average rate + // the pumped traffic goes to the MixTrafficController let loop_cover_traffic_future = rt.spawn(cover_traffic_stream::start_loop_cover_traffic_stream( mix_tx.clone(), @@ -171,10 +180,15 @@ impl NymClient { initial_topology.clone(), )); + // cloning arguments required by OutQueueControl; required due to move let topology_clone = initial_topology.clone(); let self_address = self.address; let input_rx = self.input_rx; + // future constantly pumping traffic at some specified average rate + // if a real message is available on 'input_rx' that might have been received from say + // the websocket, the real message is used, otherwise a loop cover message is generated + // the pumped traffic goes to the MixTrafficController let out_queue_control_future = rt.spawn(async move { real_traffic_stream::OutQueueControl::new( mix_tx, @@ -185,8 +199,13 @@ impl NymClient { .run_out_queue_control() .await }); + + // future constantly trying to fetch any received messages from the provider + // the received messages are sent to ReceivedMessagesBuffer to be available to rest of the system let provider_polling_future = rt.spawn(provider_poller.start_provider_polling()); + // a temporary workaround for starting socket listener of specified type + // in the future the actual socket handler should start THIS client instead match self.socket_type { SocketType::WebSocket => { rt.spawn(ws::start_websocket( From 551d0eb161e59494bb109c2db4591841c8eecfdb Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:16:22 +0000 Subject: [PATCH 020/105] More comments regarding uses of specific channels --- nym-client/src/client/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index b02ba0bd0b..d443d323d1 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -127,8 +127,19 @@ impl NymClient { let mut rt = Runtime::new()?; // channels for inter-component communication + + // mix_tx is the transmitter for any component generating sphinx packets that are to be sent to the mixnet + // they are used by cover traffic stream and real traffic stream + // mix_rx is the receiver used by MixTrafficController that sends the actual traffic let (mix_tx, mix_rx) = mpsc::unbounded(); + + // poller_input_tx is the transmitter of messages fetched from the provider - used by ProviderPoller + // poller_input_rx is the receiver for said messages - used by ReceivedMessagesBuffer let (poller_input_tx, poller_input_rx) = mpsc::unbounded(); + + // received_messages_buffer_output_tx is the transmitter for *REQUESTS* for messages contained in ReceivedMessagesBuffer - used by sockets + // the requests contain a oneshot channel to send a reply on + // received_messages_buffer_output_rx is the received for the said requests - used by ReceivedMessagesBuffer let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); From ac988c62e99d91b394ce12f4e33dd0bfc752033d Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:21:02 +0000 Subject: [PATCH 021/105] Removed 'toml' from client-persistence as it will be module-specific --- nym-client/src/persistence/toml.rs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 nym-client/src/persistence/toml.rs diff --git a/nym-client/src/persistence/toml.rs b/nym-client/src/persistence/toml.rs deleted file mode 100644 index f4f5578ad0..0000000000 --- a/nym-client/src/persistence/toml.rs +++ /dev/null @@ -1 +0,0 @@ -// TODO: we can put all the TOML config templating code in here once we get to that. From 55e473d04ed8f4d109dc256b930653cdad60b365 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:46:34 +0000 Subject: [PATCH 022/105] Moved Pemstore as separate crate + defined pathfinder as trait --- Cargo.lock | 9 ++++++++ Cargo.toml | 1 + common/pemstore/Cargo.toml | 13 +++++++++++ .../mod.rs => common/pemstore/src/lib.rs | 0 common/pemstore/src/pathfinder.rs | 15 ++++++++++++ .../pemstore/src}/pemstore.rs | 19 ++++----------- nym-client/Cargo.toml | 1 + nym-client/src/commands/init.rs | 6 ++--- nym-client/src/commands/tcpsocket.rs | 9 ++++---- nym-client/src/commands/websocket.rs | 10 ++++---- nym-client/src/config/mod.rs | 1 + nym-client/src/config/persistance/mod.rs | 1 + .../persistance}/pathfinder.rs | 23 +++++++++++++++---- nym-client/src/main.rs | 2 +- 14 files changed, 80 insertions(+), 30 deletions(-) create mode 100644 common/pemstore/Cargo.toml rename nym-client/src/persistence/mod.rs => common/pemstore/src/lib.rs (100%) create mode 100644 common/pemstore/src/pathfinder.rs rename {nym-client/src/persistence => common/pemstore/src}/pemstore.rs (84%) create mode 100644 nym-client/src/config/mod.rs create mode 100644 nym-client/src/config/persistance/mod.rs rename nym-client/src/{persistence => config/persistance}/pathfinder.rs (52%) diff --git a/Cargo.lock b/Cargo.lock index 6aa1d181a4..3e9d9da323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1356,6 +1356,7 @@ dependencies = [ "log", "mix-client", "pem", + "pemstore", "provider-client", "rand 0.7.2", "rand_distr", @@ -1502,6 +1503,14 @@ dependencies = [ "regex", ] +[[package]] +name = "pemstore" +version = "0.1.0" +dependencies = [ + "crypto", + "pem", +] + [[package]] name = "percent-encoding" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index d03d085fa8..e233914c0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "common/addressing", "common/crypto", "common/healthcheck", + "common/pemstore", "common/topology", "mixnode", "nym-client", diff --git a/common/pemstore/Cargo.toml b/common/pemstore/Cargo.toml new file mode 100644 index 0000000000..2774ec2ccd --- /dev/null +++ b/common/pemstore/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "pemstore" +version = "0.1.0" +authors = ["Jedrzej Stuczynski "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +pem = "0.7.0" + +## internal +crypto = {path = "../crypto"} diff --git a/nym-client/src/persistence/mod.rs b/common/pemstore/src/lib.rs similarity index 100% rename from nym-client/src/persistence/mod.rs rename to common/pemstore/src/lib.rs diff --git a/common/pemstore/src/pathfinder.rs b/common/pemstore/src/pathfinder.rs new file mode 100644 index 0000000000..8633f15103 --- /dev/null +++ b/common/pemstore/src/pathfinder.rs @@ -0,0 +1,15 @@ +use std::path::PathBuf; + +pub trait PathFinder { + fn config_dir(&self) -> PathBuf; + fn private_identity_key(&self) -> PathBuf; + fn public_identity_key(&self) -> PathBuf; + + // Optional: + fn private_encryption_key(&self) -> Option { + None + } + fn public_encryption_key(&self) -> Option { + None + } +} diff --git a/nym-client/src/persistence/pemstore.rs b/common/pemstore/src/pemstore.rs similarity index 84% rename from nym-client/src/persistence/pemstore.rs rename to common/pemstore/src/pemstore.rs index 19f6bb07df..d16f8574fa 100644 --- a/nym-client/src/persistence/pemstore.rs +++ b/common/pemstore/src/pemstore.rs @@ -1,18 +1,9 @@ -use crate::persistence::pathfinder::Pathfinder; +use crate::pathfinder::PathFinder; use pem::{encode, parse, Pem}; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; -pub fn read_mix_identity_keypair_from_disk( - id: String, -) -> crypto::identity::DummyMixIdentityKeyPair { - let pathfinder = Pathfinder::new(id); - let pem_store = PemStore::new(pathfinder); - let keypair = pem_store.read_identity(); - keypair -} - #[allow(dead_code)] pub fn read_mix_encryption_keypair_from_disk(_id: String) -> crypto::encryption::x25519::KeyPair { unimplemented!() @@ -25,11 +16,11 @@ pub struct PemStore { } impl PemStore { - pub fn new(pathfinder: Pathfinder) -> PemStore { + pub fn new(pathfinder: P) -> PemStore { PemStore { - config_dir: pathfinder.config_dir, - private_mix_key: pathfinder.private_mix_key, - public_mix_key: pathfinder.public_mix_key, + config_dir: pathfinder.config_dir(), + private_mix_key: pathfinder.private_identity_key(), + public_mix_key: pathfinder.public_identity_key(), } } diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index f336d11b5a..86e3073c6e 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -35,6 +35,7 @@ crypto = {path = "../common/crypto"} directory-client = { path = "../common/clients/directory-client" } healthcheck = { path = "../common/healthcheck" } mix-client = { path = "../common/clients/mix-client" } +pemstore = {path = "../common/pemstore"} provider-client = { path = "../common/clients/provider-client" } sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" } topology = {path = "../common/topology" } diff --git a/nym-client/src/commands/init.rs b/nym-client/src/commands/init.rs index f6d675ea02..4ac9c47db4 100644 --- a/nym-client/src/commands/init.rs +++ b/nym-client/src/commands/init.rs @@ -1,13 +1,13 @@ -use crate::persistence::pathfinder::Pathfinder; -use crate::persistence::pemstore::PemStore; +use crate::config::persistance::pathfinder::ClientPathfinder; use clap::ArgMatches; use crypto::identity::MixnetIdentityKeyPair; +use pemstore::pemstore::PemStore; pub fn execute(matches: &ArgMatches) { println!("Initialising client..."); let id = matches.value_of("id").unwrap().to_string(); // required for now - let pathfinder = Pathfinder::new(id); + let pathfinder = ClientPathfinder::new(id); println!("Writing keypairs to {:?}...", pathfinder.config_dir); let mix_keys = crypto::identity::DummyMixIdentityKeyPair::new(); diff --git a/nym-client/src/commands/tcpsocket.rs b/nym-client/src/commands/tcpsocket.rs index 053981d446..81e0ef72d5 100644 --- a/nym-client/src/commands/tcpsocket.rs +++ b/nym-client/src/commands/tcpsocket.rs @@ -1,8 +1,8 @@ use crate::client::{NymClient, SocketType}; -use crate::persistence::pemstore; - +use crate::config::persistance::pathfinder::ClientPathfinder; use clap::ArgMatches; -use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use pemstore::pemstore::PemStore; use std::net::ToSocketAddrs; pub fn execute(matches: &ArgMatches) { @@ -26,7 +26,8 @@ pub fn execute(matches: &ArgMatches) { .next() .expect("Failed to extract the socket address from the iterator"); - let keypair = pemstore::read_mix_identity_keypair_from_disk(id); + // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type? + let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)).read_identity(); // TODO: reading auth_token from disk (if exists); println!("Public key: {}", keypair.public_key.to_b64_string()); diff --git a/nym-client/src/commands/websocket.rs b/nym-client/src/commands/websocket.rs index 0a66193a6f..644e023306 100644 --- a/nym-client/src/commands/websocket.rs +++ b/nym-client/src/commands/websocket.rs @@ -1,8 +1,8 @@ use crate::client::{NymClient, SocketType}; -use crate::persistence::pemstore; - +use crate::config::persistance::pathfinder::ClientPathfinder; use clap::ArgMatches; -use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use pemstore::pemstore::PemStore; use std::net::ToSocketAddrs; pub fn execute(matches: &ArgMatches) { @@ -26,7 +26,9 @@ pub fn execute(matches: &ArgMatches) { .next() .expect("Failed to extract the socket address from the iterator"); - let keypair = pemstore::read_mix_identity_keypair_from_disk(id); + // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type? + let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)).read_identity(); + // TODO: reading auth_token from disk (if exists); println!("Public key: {}", keypair.public_key.to_b64_string()); diff --git a/nym-client/src/config/mod.rs b/nym-client/src/config/mod.rs new file mode 100644 index 0000000000..4bf0f95209 --- /dev/null +++ b/nym-client/src/config/mod.rs @@ -0,0 +1 @@ +pub mod persistance; diff --git a/nym-client/src/config/persistance/mod.rs b/nym-client/src/config/persistance/mod.rs new file mode 100644 index 0000000000..5692e9bc8b --- /dev/null +++ b/nym-client/src/config/persistance/mod.rs @@ -0,0 +1 @@ +pub mod pathfinder; diff --git a/nym-client/src/persistence/pathfinder.rs b/nym-client/src/config/persistance/pathfinder.rs similarity index 52% rename from nym-client/src/persistence/pathfinder.rs rename to nym-client/src/config/persistance/pathfinder.rs index 08110b9ae8..c71aab5b84 100644 --- a/nym-client/src/persistence/pathfinder.rs +++ b/nym-client/src/config/persistance/pathfinder.rs @@ -1,21 +1,36 @@ +use pemstore::pathfinder::PathFinder; use std::path::PathBuf; -pub struct Pathfinder { +pub struct ClientPathfinder { pub config_dir: PathBuf, pub private_mix_key: PathBuf, pub public_mix_key: PathBuf, } -impl Pathfinder { - pub fn new(id: String) -> Pathfinder { +impl ClientPathfinder { + pub fn new(id: String) -> Self { let os_config_dir = dirs::config_dir().unwrap(); // grabs the OS default config dir let config_dir = os_config_dir.join("nym").join("clients").join(id); let private_mix_key = config_dir.join("private.pem"); let public_mix_key = config_dir.join("public.pem"); - Pathfinder { + ClientPathfinder { config_dir, private_mix_key, public_mix_key, } } } + +impl PathFinder for ClientPathfinder { + fn config_dir(&self) -> PathBuf { + self.config_dir.clone() + } + + fn private_identity_key(&self) -> PathBuf { + self.private_mix_key.clone() + } + + fn public_identity_key(&self) -> PathBuf { + self.public_mix_key.clone() + } +} diff --git a/nym-client/src/main.rs b/nym-client/src/main.rs index 64daf4e515..d1e12a85d4 100644 --- a/nym-client/src/main.rs +++ b/nym-client/src/main.rs @@ -5,7 +5,7 @@ use clap::{App, Arg, ArgMatches, SubCommand}; pub mod built_info; pub mod client; mod commands; -mod persistence; +pub mod config; mod sockets; pub mod utils; From da575f365790d0cb61041e5f104c04323e416189 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:49:48 +0000 Subject: [PATCH 023/105] ibid --- nym-client/src/lib.rs | 2 +- nym-client/src/utils/bytes.rs | 63 ----------------------------------- 2 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 nym-client/src/utils/bytes.rs diff --git a/nym-client/src/lib.rs b/nym-client/src/lib.rs index a7760d6e36..a79f75bf0c 100644 --- a/nym-client/src/lib.rs +++ b/nym-client/src/lib.rs @@ -1,8 +1,8 @@ #![recursion_limit = "256"] pub mod built_info; +pub mod config; pub mod client; pub mod commands; -pub mod persistence; pub mod sockets; pub mod utils; diff --git a/nym-client/src/utils/bytes.rs b/nym-client/src/utils/bytes.rs deleted file mode 100644 index 6079fa8385..0000000000 --- a/nym-client/src/utils/bytes.rs +++ /dev/null @@ -1,63 +0,0 @@ -pub fn zero_pad_to_32(mut bytes: Vec) -> [u8; 32] { - assert!(bytes.len() <= 32); - if bytes.len() != 32 { - bytes.resize(32, 0); - } - let mut padded_bytes = [0; 32]; - padded_bytes.copy_from_slice(&bytes[..]); - padded_bytes -} - -#[cfg(test)] -mod zero_padding_to_32_bytes { - use super::*; - - #[cfg(test)] - mod with_empty_input { - use super::*; - - #[test] - fn it_returns_32_zeros() { - let input = vec![]; - let result = zero_pad_to_32(input); - assert_eq!([0u8; 32], result); - } - } - - #[cfg(test)] - mod with_all_bytes_set_to_1 { - use super::*; - #[test] - fn it_returns_32_ones() { - let input = vec![1u8; 32]; - let result = zero_pad_to_32(input); - assert_eq!([1u8; 32], result); - } - } - - #[cfg(test)] - mod with_3_bytes_set { - use super::*; - #[test] - fn it_returns_input_zero_padded_to_32_bytes() { - let input = vec![1u8; 3]; - let result = zero_pad_to_32(input); - let expected_content = vec![ - 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, - ]; - assert_eq!(expected_content, result.to_vec()); - } - } - - #[cfg(test)] - mod with_oversized_input { - use super::*; - #[test] - #[should_panic] - fn it_panics() { - let input = vec![1u8; 33]; - zero_pad_to_32(input); - } - } -} From 98610e20dcdfa32a056686757bb4cffea376dd22 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 17:50:07 +0000 Subject: [PATCH 024/105] Removed unused utils/bytes module --- nym-client/src/utils/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nym-client/src/utils/mod.rs b/nym-client/src/utils/mod.rs index d213166bdc..3e2ed40a1b 100644 --- a/nym-client/src/utils/mod.rs +++ b/nym-client/src/utils/mod.rs @@ -1,3 +1,2 @@ -pub mod bytes; pub mod poisson; pub mod sphinx; From b4c15eae12526454bc2a24736cb03b256a1df43a Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Tue, 21 Jan 2020 18:02:35 +0000 Subject: [PATCH 025/105] Sending real traffic at poisson rate rather than at uniform rate --- nym-client/src/client/real_traffic_stream.rs | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index cc00ea7ef0..8750df7727 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -4,7 +4,7 @@ use crate::utils; use directory_client::presence::Topology; use futures::channel::mpsc; use futures::task::{Context, Poll}; -use futures::{select, Stream, StreamExt}; +use futures::{select, Future, Stream, StreamExt}; use log::{debug, error, info, trace, warn}; use sphinx::route::Destination; use sphinx::SphinxPacket; @@ -15,7 +15,7 @@ use tokio::time; use topology::NymTopology; pub(crate) struct OutQueueControl { - interval: time::Interval, + delay: time::Delay, mix_tx: mpsc::UnboundedSender, input_rx: mpsc::UnboundedReceiver, our_info: Destination, @@ -31,10 +31,23 @@ impl Stream for OutQueueControl { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // it is not yet time to return a message - if Stream::poll_next(Pin::new(&mut self.interval), cx).is_pending() { + if Future::poll(Pin::new(&mut self.delay), cx).is_pending() { return Poll::Pending; }; + // we know it's time to send a message, so let's prepare delay for the next one + // Get the `now` by looking at the current `delay` deadline + let now = self.delay.deadline(); + + let next_poisson_delay = + Duration::from_secs_f64(utils::poisson::sample(MESSAGE_SENDING_AVERAGE_DELAY)); + + // The next interval value is `next_poisson_delay` after the one that just + // yielded. + let next = now + next_poisson_delay; + self.delay.reset(next); + + // decide what kind of message to send match Stream::poll_next(Pin::new(&mut self.input_rx), cx) { // in the case our real message channel stream was closed, we should also indicate we are closed // (and whoever is using the stream should panic) @@ -70,8 +83,9 @@ impl OutQueueControl { our_info: Destination, topology: Topology, ) -> Self { + let initial_delay = time::delay_for(Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY)); OutQueueControl { - interval: time::interval(Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY)), + delay: initial_delay, mix_tx, input_rx, our_info, From 2364fbdc093869518d23187e5fee0171f3e05a2d Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Wed, 22 Jan 2020 10:50:03 +0000 Subject: [PATCH 026/105] Moved sphinx packet encapsulation and poisson delay sampling to mix client --- Cargo.lock | 6 ++++-- common/clients/mix-client/Cargo.toml | 7 +++++++ common/clients/mix-client/src/lib.rs | 3 +++ .../sphinx.rs => common/clients/mix-client/src/packet.rs | 8 ++++---- .../utils => common/clients/mix-client/src}/poisson.rs | 0 nym-client/src/lib.rs | 9 +++------ nym-client/src/main.rs | 3 --- nym-client/src/utils/mod.rs | 2 -- 8 files changed, 21 insertions(+), 17 deletions(-) rename nym-client/src/utils/sphinx.rs => common/clients/mix-client/src/packet.rs (89%) rename {nym-client/src/utils => common/clients/mix-client/src}/poisson.rs (100%) delete mode 100644 nym-client/src/utils/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 3e9d9da323..66d9693e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1256,9 +1256,13 @@ dependencies = [ name = "mix-client" version = "0.1.0" dependencies = [ + "addressing", "log", + "rand 0.7.2", + "rand_distr", "sphinx", "tokio 0.2.9", + "topology", ] [[package]] @@ -1358,8 +1362,6 @@ dependencies = [ "pem", "pemstore", "provider-client", - "rand 0.7.2", - "rand_distr", "reqwest", "serde", "serde_json", diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index 89b497375d..362fad3f0a 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -8,7 +8,14 @@ edition = "2018" [dependencies] log = "0.4.8" +rand = "0.7.2" +rand_distr = "0.2.2" tokio = { version = "0.2", features = ["full"] } +## internal +addressing = {path = "../../addressing"} +topology = {path = "../../topology"} + ## will be moved to proper dependencies once released sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } + diff --git a/common/clients/mix-client/src/lib.rs b/common/clients/mix-client/src/lib.rs index bbdfa9bc0b..fd883a54a4 100644 --- a/common/clients/mix-client/src/lib.rs +++ b/common/clients/mix-client/src/lib.rs @@ -3,6 +3,9 @@ use sphinx::SphinxPacket; use std::net::SocketAddr; use tokio::prelude::*; +pub mod packet; +pub mod poisson; + pub struct MixClient {} impl MixClient { diff --git a/nym-client/src/utils/sphinx.rs b/common/clients/mix-client/src/packet.rs similarity index 89% rename from nym-client/src/utils/sphinx.rs rename to common/clients/mix-client/src/packet.rs index 0f3e69f36b..356d2bbd2e 100644 --- a/nym-client/src/utils/sphinx.rs +++ b/common/clients/mix-client/src/packet.rs @@ -5,6 +5,7 @@ use std::net::SocketAddr; use topology::NymTopology; pub const LOOP_COVER_MESSAGE_PAYLOAD: &[u8] = b"The cake is a lie!"; +pub const LOOP_COVER_MESSAGE_AVERAGE_DELAY: f64 = 2.0; pub fn loop_cover_message( our_address: DestinationAddressBytes, @@ -13,21 +14,20 @@ pub fn loop_cover_message( ) -> (SocketAddr, SphinxPacket) { let destination = Destination::new(our_address, surb_id); - encapsulate_message(destination, LOOP_COVER_MESSAGE_PAYLOAD.to_vec(), topology) + encapsulate_message(destination, LOOP_COVER_MESSAGE_PAYLOAD.to_vec(), topology, LOOP_COVER_MESSAGE_AVERAGE_DELAY) } pub fn encapsulate_message( recipient: Destination, message: Vec, topology: &T, + average_delay: f64, ) -> (SocketAddr, SphinxPacket) { let mut providers = topology.get_mix_provider_nodes(); let provider = providers.pop().unwrap().into(); let route = topology.route_to(provider).unwrap(); - - // Set average packet delay to an arbitrary but at least not super-slow value for testing. - let average_delay = 0.1; + let delays = sphinx::header::delays::generate(route.len(), average_delay); // build the packet diff --git a/nym-client/src/utils/poisson.rs b/common/clients/mix-client/src/poisson.rs similarity index 100% rename from nym-client/src/utils/poisson.rs rename to common/clients/mix-client/src/poisson.rs diff --git a/nym-client/src/lib.rs b/nym-client/src/lib.rs index a79f75bf0c..477c106b34 100644 --- a/nym-client/src/lib.rs +++ b/nym-client/src/lib.rs @@ -1,8 +1,5 @@ -#![recursion_limit = "256"] - pub mod built_info; -pub mod config; pub mod client; -pub mod commands; -pub mod sockets; -pub mod utils; +mod commands; +pub mod config; +mod sockets; diff --git a/nym-client/src/main.rs b/nym-client/src/main.rs index d1e12a85d4..a3f20596b0 100644 --- a/nym-client/src/main.rs +++ b/nym-client/src/main.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "256"] - use clap::{App, Arg, ArgMatches, SubCommand}; pub mod built_info; @@ -7,7 +5,6 @@ pub mod client; mod commands; pub mod config; mod sockets; -pub mod utils; fn main() { env_logger::init(); diff --git a/nym-client/src/utils/mod.rs b/nym-client/src/utils/mod.rs deleted file mode 100644 index 3e2ed40a1b..0000000000 --- a/nym-client/src/utils/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod poisson; -pub mod sphinx; From 7045db7d70dc926cd86f1316755d2db1e01a1d62 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Wed, 22 Jan 2020 10:50:22 +0000 Subject: [PATCH 027/105] ibid. --- common/clients/mix-client/src/packet.rs | 9 +++++++-- nym-client/Cargo.toml | 2 -- nym-client/src/client/cover_traffic_stream.rs | 10 ++++++---- nym-client/src/client/provider_poller.rs | 5 ++--- nym-client/src/client/real_traffic_stream.rs | 16 +++++++++------- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/common/clients/mix-client/src/packet.rs b/common/clients/mix-client/src/packet.rs index 356d2bbd2e..cc1a027cb6 100644 --- a/common/clients/mix-client/src/packet.rs +++ b/common/clients/mix-client/src/packet.rs @@ -14,7 +14,12 @@ pub fn loop_cover_message( ) -> (SocketAddr, SphinxPacket) { let destination = Destination::new(our_address, surb_id); - encapsulate_message(destination, LOOP_COVER_MESSAGE_PAYLOAD.to_vec(), topology, LOOP_COVER_MESSAGE_AVERAGE_DELAY) + encapsulate_message( + destination, + LOOP_COVER_MESSAGE_PAYLOAD.to_vec(), + topology, + LOOP_COVER_MESSAGE_AVERAGE_DELAY, + ) } pub fn encapsulate_message( @@ -27,7 +32,7 @@ pub fn encapsulate_message( let provider = providers.pop().unwrap().into(); let route = topology.route_to(provider).unwrap(); - + let delays = sphinx::header::delays::generate(route.len(), average_delay); // build the packet diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 86e3073c6e..d80d6fcdde 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -21,8 +21,6 @@ futures = "0.3.1" hex = "0.4.0" log = "0.4.8" pem = "0.7.0" -rand = "0.7.2" -rand_distr = "0.2.2" reqwest = "0.9.22" serde = { version = "1.0.104", features = ["derive"] } serde_json = "1.0.44" diff --git a/nym-client/src/client/cover_traffic_stream.rs b/nym-client/src/client/cover_traffic_stream.rs index 3405cc2cf9..82f904eec7 100644 --- a/nym-client/src/client/cover_traffic_stream.rs +++ b/nym-client/src/client/cover_traffic_stream.rs @@ -1,6 +1,5 @@ use crate::client::mix_traffic::MixMessage; use crate::client::LOOP_COVER_AVERAGE_DELAY; -use crate::utils; use futures::channel::mpsc; use log::{info, trace}; use sphinx::route::Destination; @@ -17,11 +16,14 @@ pub(crate) async fn start_loop_cover_traffic_stream( info!("Starting loop cover traffic stream"); loop { trace!("next cover message!"); - let delay = utils::poisson::sample(LOOP_COVER_AVERAGE_DELAY); + let delay = mix_client::poisson::sample(LOOP_COVER_AVERAGE_DELAY); let delay_duration = Duration::from_secs_f64(delay); tokio::time::delay_for(delay_duration).await; - let cover_message = - utils::sphinx::loop_cover_message(our_info.address, our_info.identifier, &topology); + let cover_message = mix_client::packet::loop_cover_message( + our_info.address, + our_info.identifier, + &topology, + ); // if this one fails, there's no retrying because it means that either: // - we run out of memory diff --git a/nym-client/src/client/provider_poller.rs b/nym-client/src/client/provider_poller.rs index 20d88a049d..aa3924f991 100644 --- a/nym-client/src/client/provider_poller.rs +++ b/nym-client/src/client/provider_poller.rs @@ -1,5 +1,4 @@ use crate::client::FETCH_MESSAGES_DELAY; -use crate::utils; use futures::channel::mpsc; use log::{debug, error, info, trace, warn}; use provider_client::ProviderClientError; @@ -53,7 +52,7 @@ impl ProviderPoller { pub(crate) async fn start_provider_polling(mut self) { info!("Starting provider poller"); - let loop_message = &utils::sphinx::LOOP_COVER_MESSAGE_PAYLOAD.to_vec(); + let loop_message = &mix_client::packet::LOOP_COVER_MESSAGE_PAYLOAD.to_vec(); let dummy_message = &sfw_provider_requests::DUMMY_MESSAGE_CONTENT.to_vec(); let delay_duration = Duration::from_secs_f64(FETCH_MESSAGES_DELAY); @@ -64,7 +63,7 @@ impl ProviderPoller { let messages = match self.provider_client.retrieve_messages().await { Err(err) => { - error!("Failed to query the provider for messages... Going to wait {:?} before retrying", extended_delay_duration); + error!("Failed to query the provider for messages: {:?}, ... Going to wait {:?} before retrying", err, extended_delay_duration); tokio::time::delay_for(extended_delay_duration).await; continue; } diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index 8750df7727..f89257747b 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -1,18 +1,19 @@ use crate::client::mix_traffic::MixMessage; use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY}; -use crate::utils; use directory_client::presence::Topology; use futures::channel::mpsc; use futures::task::{Context, Poll}; -use futures::{select, Future, Stream, StreamExt}; -use log::{debug, error, info, trace, warn}; +use futures::{Future, Stream, StreamExt}; +use log::{debug, info, trace}; use sphinx::route::Destination; use sphinx::SphinxPacket; use std::net::SocketAddr; use std::pin::Pin; use std::time::Duration; use tokio::time; -use topology::NymTopology; + +// have a rather low value for test sake +const AVERAGE_PACKET_DELAY: f64 = 0.1; pub(crate) struct OutQueueControl { delay: time::Delay, @@ -40,7 +41,7 @@ impl Stream for OutQueueControl { let now = self.delay.deadline(); let next_poisson_delay = - Duration::from_secs_f64(utils::poisson::sample(MESSAGE_SENDING_AVERAGE_DELAY)); + Duration::from_secs_f64(mix_client::poisson::sample(MESSAGE_SENDING_AVERAGE_DELAY)); // The next interval value is `next_poisson_delay` after the one that just // yielded. @@ -56,17 +57,18 @@ impl Stream for OutQueueControl { // if there's an actual message - return it Poll::Ready(Some(real_message)) => { trace!("real message"); - Poll::Ready(Some(utils::sphinx::encapsulate_message( + Poll::Ready(Some(mix_client::packet::encapsulate_message( real_message.0, real_message.1, &self.topology, + AVERAGE_PACKET_DELAY, ))) } // otherwise construct a dummy one _ => { trace!("loop cover message"); - Poll::Ready(Some(utils::sphinx::loop_cover_message( + Poll::Ready(Some(mix_client::packet::loop_cover_message( self.our_info.address, self.our_info.identifier, &self.topology, From 974ac5ef655840d9976af0c071e256b4b1534ad3 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Wed, 22 Jan 2020 10:52:30 +0000 Subject: [PATCH 028/105] compiler warnings --- nym-client/src/client/mod.rs | 1 + nym-client/src/client/provider_poller.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index d443d323d1..c9685af0b9 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -98,6 +98,7 @@ impl NymClient { let healthcheck_scores = match healthcheck_result { Err(err) => { + error!("Error while performing the healtcheck: {:?}", err); return Err(TopologyError::HealthCheckError); } Ok(scores) => scores, diff --git a/nym-client/src/client/provider_poller.rs b/nym-client/src/client/provider_poller.rs index aa3924f991..71ccd41fdd 100644 --- a/nym-client/src/client/provider_poller.rs +++ b/nym-client/src/client/provider_poller.rs @@ -49,7 +49,7 @@ impl ProviderPoller { Ok(()) } - pub(crate) async fn start_provider_polling(mut self) { + pub(crate) async fn start_provider_polling(self) { info!("Starting provider poller"); let loop_message = &mix_client::packet::LOOP_COVER_MESSAGE_PAYLOAD.to_vec(); From 566e17719c8e8691a0da7e7bb76488ccec0fe5ee Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Wed, 22 Jan 2020 12:22:05 +0000 Subject: [PATCH 029/105] added PartialEq restriction on NymTopology for convenience sake --- common/topology/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index dd29508ef2..0cd511de3f 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -83,7 +83,7 @@ pub enum NymTopologyError { MissingLayerError(Vec), } -pub trait NymTopology: Sized { +pub trait NymTopology: Sized + PartialEq { fn new(directory_server: String) -> Self; fn new_from_nodes( mix_nodes: Vec, From 91eb03fbb6b9959946f3c1242c325083741a2205 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Wed, 22 Jan 2020 12:22:15 +0000 Subject: [PATCH 030/105] Derived PartialEq on Topology --- common/clients/directory-client/src/presence.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index f20ed9f914..b333e9fce9 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -6,7 +6,7 @@ use std::io; use std::net::ToSocketAddrs; use topology::{CocoNode, MixNode, MixProviderNode, NymTopology}; -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct CocoPresence { pub host: String, @@ -37,7 +37,7 @@ impl From for CocoPresence { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct MixNodePresence { pub host: String, @@ -81,7 +81,7 @@ impl From for MixNodePresence { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct MixProviderPresence { pub client_listener: String, @@ -126,7 +126,7 @@ impl From for MixProviderPresence { } } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct MixProviderClient { pub pub_key: String, @@ -149,7 +149,7 @@ impl From for MixProviderClient { } // Topology shows us the current state of the overall Nym network -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Topology { pub coco_nodes: Vec, From f785637cd5c0a14ce0e80583e48c89c8fa146205 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 12:57:35 +0000 Subject: [PATCH 031/105] provider-client: removing unused commands module --- common/clients/provider-client/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index 1200a18fcd..9b4ca131e4 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -1,5 +1,4 @@ use futures::io::Error; -use log::info; use sfw_provider_requests::requests::{ProviderRequest, PullRequest, RegisterRequest}; use sfw_provider_requests::responses::{ ProviderResponse, ProviderResponseError, PullResponse, RegisterResponse, From d34907eccf5c1e9a9cde152b5756f5e04e64f1ca Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 12:57:41 +0000 Subject: [PATCH 032/105] ibid --- nym-client/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nym-client/src/lib.rs b/nym-client/src/lib.rs index 477c106b34..b264899dca 100644 --- a/nym-client/src/lib.rs +++ b/nym-client/src/lib.rs @@ -1,5 +1,4 @@ pub mod built_info; pub mod client; -mod commands; pub mod config; mod sockets; From 44116e49074b398a652671bf829267fa53738fbd Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 16:17:35 +0000 Subject: [PATCH 033/105] common: adding a version-checker crate --- Cargo.lock | 8 +++++ common/version-checker/.gitignore | 3 ++ common/version-checker/Cargo.toml | 10 ++++++ common/version-checker/src/lib.rs | 51 +++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 common/version-checker/.gitignore create mode 100644 common/version-checker/Cargo.toml create mode 100644 common/version-checker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 66d9693e54..5aec0f39d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2422,6 +2422,7 @@ dependencies = [ "rand 0.7.2", "serde", "sphinx", + "version-checker", ] [[package]] @@ -2557,6 +2558,13 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +[[package]] +name = "version-checker" +version = "0.1.0" +dependencies = [ + "semver", +] + [[package]] name = "version_check" version = "0.1.5" diff --git a/common/version-checker/.gitignore b/common/version-checker/.gitignore new file mode 100644 index 0000000000..693699042b --- /dev/null +++ b/common/version-checker/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/common/version-checker/Cargo.toml b/common/version-checker/Cargo.toml new file mode 100644 index 0000000000..80c97fb349 --- /dev/null +++ b/common/version-checker/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "version-checker" +version = "0.1.0" +authors = ["Dave Hrycyszyn "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +semver = "0.9.0" \ No newline at end of file diff --git a/common/version-checker/src/lib.rs b/common/version-checker/src/lib.rs new file mode 100644 index 0000000000..d6ae38d19e --- /dev/null +++ b/common/version-checker/src/lib.rs @@ -0,0 +1,51 @@ +use semver::Version; +use semver::VersionReq; + +/// Checks whether given `version` is compatible with a given semantic +/// version requirement `req` according to major-minor semver rules. +/// The semantic version requirement can be passed as a full, concrete version +/// number, because that's what we'll have in our Cargo.toml files (e.g. 0.3.2). +/// The patch number in the requirement will be dropped and replaced with a wildcard (0.3.*) as all minor versions should be compatible with each other. +pub fn is_compatible(version: &str, req: &str) -> bool { + let version = Version::parse(version).unwrap(); + let tmp = Version::parse(req).unwrap(); + let wildcard = format!("{}.{}.*", tmp.major, tmp.minor).to_string(); + let semver_requirement = VersionReq::parse(&wildcard).unwrap(); + + semver_requirement.matches(&version) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_0_3_0_is_compatible_with_requirement_0_3_x() { + assert!(is_compatible("0.3.0", "0.3.2")); + } + + #[test] + fn version_0_3_1_is_compatible_with_minimum_requirement_0_3_x() { + assert!(is_compatible("0.3.1", "0.3.2")); + } + + #[test] + fn version_0_3_2_is_compatible_with_minimum_requirement_0_3_x() { + assert!(is_compatible("0.3.2", "0.3.0")); + } + + #[test] + fn version_0_2_0_is_not_compatible_with_requirement_0_3_x() { + assert!(!is_compatible("0.2.0", "0.3.2")); + } + + #[test] + fn version_0_4_0_is_not_compatible_with_requirement_0_3_x() { + assert!(!is_compatible("0.4.0", "0.3.2")); + } + + #[test] + fn version_1_3_2_is_not_compatible_with_requirement_0_3_x() { + assert!(!is_compatible("1.3.2", "0.3.2")); + } +} From 65f5683b12050f4b99f678f34c51acf38ad4fc78 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 16:18:00 +0000 Subject: [PATCH 034/105] topology: checking for compatible semver between nodes and clients --- common/topology/Cargo.toml | 1 + common/topology/src/lib.rs | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index 628672da6e..7015348586 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -16,6 +16,7 @@ serde = { version = "1.0.104", features = ["derive"] } ## internal addressing = {path = "../addressing"} +version-checker = {path = "../version-checker" } ## will be moved to proper dependencies once released sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index dd29508ef2..8fadc5c2ab 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -6,6 +6,7 @@ use sphinx::route::{Node as SphinxNode, NodeAddressBytes}; use std::cmp::max; use std::collections::HashMap; use std::net::SocketAddr; +use version_checker; #[derive(Debug, Clone)] pub struct MixNode { @@ -167,27 +168,33 @@ pub trait NymTopology: Sized { fn filter_node_versions( &self, - mix_version: &str, - provider_version: &str, - coco_version: &str, + expected_mix_version: &str, + expected_provider_version: &str, + expected_coco_version: &str, ) -> Self { let filtered_mixes = self .get_mix_nodes() .iter() + .filter(|mix_node| { + version_checker::is_compatible(&mix_node.version, expected_mix_version) + }) .cloned() - .filter(|mix_node| mix_node.version == mix_version) .collect(); let filtered_providers = self .get_mix_provider_nodes() .iter() + .filter(|provider_node| { + version_checker::is_compatible(&provider_node.version, expected_provider_version) + }) .cloned() - .filter(|provider_node| provider_node.version == provider_version) .collect(); let filtered_coco_nodes = self .get_coco_nodes() .iter() + .filter(|coco_node| { + version_checker::is_compatible(&coco_node.version, expected_coco_version) + }) .cloned() - .filter(|coco_node| coco_node.version == coco_version) .collect(); Self::new_from_nodes(filtered_mixes, filtered_providers, filtered_coco_nodes) From 09cb17e6aba8b762c22d805e05f6897c7f0d9d2d Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 16:29:59 +0000 Subject: [PATCH 035/105] all: using pretty_env_logger pretty much everywhere --- Cargo.lock | 21 +++++++++++++++++++++ common/addressing/Cargo.toml | 2 ++ common/clients/directory-client/Cargo.toml | 2 ++ common/clients/mix-client/Cargo.toml | 1 + common/clients/provider-client/Cargo.toml | 1 + common/clients/validator-client/Cargo.toml | 2 ++ common/crypto/Cargo.toml | 2 ++ common/healthcheck/Cargo.toml | 1 + common/pemstore/Cargo.toml | 2 ++ common/topology/Cargo.toml | 3 ++- mixnode/Cargo.toml | 3 ++- nym-client/Cargo.toml | 4 ++-- nym-client/src/main.rs | 2 +- sfw-provider/Cargo.toml | 3 ++- validator/Cargo.toml | 4 ++-- 15 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66d9693e54..9930dc746b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,6 +3,10 @@ [[package]] name = "addressing" version = "0.1.0" +dependencies = [ + "log", + "pretty_env_logger", +] [[package]] name = "adler32" @@ -458,6 +462,8 @@ version = "0.1.0" dependencies = [ "base64 0.11.0", "curve25519-dalek", + "log", + "pretty_env_logger", "rand 0.7.2", "rand_os", ] @@ -514,7 +520,9 @@ dependencies = [ name = "directory-client" version = "0.1.0" dependencies = [ + "log", "mockito", + "pretty_env_logger", "reqwest", "serde", "topology", @@ -856,6 +864,7 @@ dependencies = [ "itertools", "log", "mix-client", + "pretty_env_logger", "provider-client", "serde", "serde_derive", @@ -1258,6 +1267,7 @@ version = "0.1.0" dependencies = [ "addressing", "log", + "pretty_env_logger", "rand 0.7.2", "rand_distr", "sphinx", @@ -1361,6 +1371,7 @@ dependencies = [ "mix-client", "pem", "pemstore", + "pretty_env_logger", "provider-client", "reqwest", "serde", @@ -1385,6 +1396,7 @@ dependencies = [ "directory-client", "futures 0.3.1", "log", + "pretty_env_logger", "sphinx", "tokio 0.2.9", ] @@ -1403,6 +1415,7 @@ dependencies = [ "hex", "hmac", "log", + "pretty_env_logger", "rand 0.7.2", "serde", "serde_json", @@ -1510,7 +1523,9 @@ name = "pemstore" version = "0.1.0" dependencies = [ "crypto", + "log", "pem", + "pretty_env_logger", ] [[package]] @@ -1612,6 +1627,7 @@ version = "0.1.0" dependencies = [ "futures 0.3.1", "log", + "pretty_env_logger", "sfw-provider-requests", "sphinx", "tokio 0.2.9", @@ -2419,6 +2435,7 @@ dependencies = [ "curve25519-dalek", "itertools", "log", + "pretty_env_logger", "rand 0.7.2", "serde", "sphinx", @@ -2544,6 +2561,10 @@ dependencies = [ [[package]] name = "validator-client" version = "0.1.0" +dependencies = [ + "log", + "pretty_env_logger", +] [[package]] name = "vcpkg" diff --git a/common/addressing/Cargo.toml b/common/addressing/Cargo.toml index 3ffea36a3a..304a25dbc9 100644 --- a/common/addressing/Cargo.toml +++ b/common/addressing/Cargo.toml @@ -7,3 +7,5 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +log = "0.4" +pretty_env_logger = "0.3" \ No newline at end of file diff --git a/common/clients/directory-client/Cargo.toml b/common/clients/directory-client/Cargo.toml index 019fd4700a..a9e30636e0 100644 --- a/common/clients/directory-client/Cargo.toml +++ b/common/clients/directory-client/Cargo.toml @@ -7,6 +7,8 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +log = "0.4" +pretty_env_logger = "0.3" reqwest = "0.9.22" serde = { version = "1.0.104", features = ["derive"] } diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index 362fad3f0a..a7ef6e35c8 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] log = "0.4.8" +pretty_env_logger = "0.3" rand = "0.7.2" rand_distr = "0.2.2" tokio = { version = "0.2", features = ["full"] } diff --git a/common/clients/provider-client/Cargo.toml b/common/clients/provider-client/Cargo.toml index 602f7d2725..450a76cb2b 100644 --- a/common/clients/provider-client/Cargo.toml +++ b/common/clients/provider-client/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [dependencies] futures = "0.3.1" log = "0.4.8" +pretty_env_logger = "0.3" tokio = { version = "0.2", features = ["full"] } ## internal diff --git a/common/clients/validator-client/Cargo.toml b/common/clients/validator-client/Cargo.toml index 1d633f29ba..d9e12aefc7 100644 --- a/common/clients/validator-client/Cargo.toml +++ b/common/clients/validator-client/Cargo.toml @@ -7,3 +7,5 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +log = "0.4" +pretty_env_logger = "0.3" \ No newline at end of file diff --git a/common/crypto/Cargo.toml b/common/crypto/Cargo.toml index 48494e850b..23790a23c8 100644 --- a/common/crypto/Cargo.toml +++ b/common/crypto/Cargo.toml @@ -9,5 +9,7 @@ edition = "2018" [dependencies] base64 = "0.11.0" curve25519-dalek = "1.2.3" +log = "0.4" +pretty_env_logger = "0.3" rand = "0.7.2" rand_os = "0.1" diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index eb82f95fd5..918efa0df9 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" futures = "0.3.1" itertools = "0.8.2" log = "0.4.8" +pretty_env_logger = "0.3" serde = "1.0.104" serde_derive = "1.0.104" tokio = { version = "0.2", features = ["full"] } diff --git a/common/pemstore/Cargo.toml b/common/pemstore/Cargo.toml index 2774ec2ccd..d66ba1aff5 100644 --- a/common/pemstore/Cargo.toml +++ b/common/pemstore/Cargo.toml @@ -7,7 +7,9 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +log = "0.4" pem = "0.7.0" +pretty_env_logger = "0.3" ## internal crypto = {path = "../crypto"} diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index 628672da6e..eab611a86a 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -10,7 +10,8 @@ edition = "2018" base64 = "0.11.0" curve25519-dalek = "1.2.3" itertools = "0.8.2" -log = "0.4.8" +log = "0.4" +pretty_env_logger = "0.3" rand = "0.7.2" serde = { version = "1.0.104", features = ["derive"] } diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index 45d800f1d8..602d119bad 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -12,7 +12,8 @@ base64 = "0.11.0" clap = "2.33.0" curve25519-dalek = "1.2.3" futures = "0.3.1" -log = "0.4.8" +log = "0.4" +pretty_env_logger = "0.3" tokio = { version = "0.2", features = ["full"] } ## internal diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index d80d6fcdde..782b68d31e 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -16,11 +16,11 @@ base64 = "0.11.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dirs = "2.0.2" -env_logger = "0.7.1" futures = "0.3.1" hex = "0.4.0" -log = "0.4.8" +log = "0.4" pem = "0.7.0" +pretty_env_logger = "0.3" reqwest = "0.9.22" serde = { version = "1.0.104", features = ["derive"] } serde_json = "1.0.44" diff --git a/nym-client/src/main.rs b/nym-client/src/main.rs index a3f20596b0..5dd19e9a7c 100644 --- a/nym-client/src/main.rs +++ b/nym-client/src/main.rs @@ -7,7 +7,7 @@ pub mod config; mod sockets; fn main() { - env_logger::init(); + pretty_env_logger::init(); let arg_matches = App::new("Nym Client") .version(built_info::PKG_VERSION) diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index d76965a12f..1fe97babcf 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -13,7 +13,8 @@ clap = "2.33.0" curve25519-dalek = "1.2.3" hex = "0.4.0" futures = "0.3.1" -log = "0.4.8" +log = "0.4" +pretty_env_logger = "0.3" rand = "0.7.2" tokio = { version = "0.2.4", features = ["full"] } sha2 = "0.8.0" diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 2b97dcb8f5..ead9eade21 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -12,8 +12,8 @@ clap = "2.33.0" # Read notes https://crates.io/crates/dotenv - tl;dr: don't use in production, set environmental variables properly. dotenv = "0.15.0" futures = "0.3.1" -log = "0.4.8" -pretty_env_logger = "0.3.1" +log = "0.4" +pretty_env_logger = "0.3" serde = "1.0.104" serde_derive = "1.0.104" tokio = { version = "0.2", features = ["full"] } From c592ac66551376e22547e48640e512f5318f7323 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 16:30:20 +0000 Subject: [PATCH 036/105] project: removing env_logger from the Cargo project file --- Cargo.lock | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9930dc746b..29518de692 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -590,19 +590,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "error-chain" version = "0.12.1" @@ -1363,7 +1350,6 @@ dependencies = [ "curve25519-dalek", "directory-client", "dirs", - "env_logger 0.7.1", "futures 0.3.1", "healthcheck", "hex", @@ -1591,7 +1577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "717ee476b1690853d222af4634056d830b5197ffd747726a9a1eee6da9f49074" dependencies = [ "chrono", - "env_logger 0.6.2", + "env_logger", "log", ] From ac61196fc65a44736a9c5bfc55e33b693383a375 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 16:34:42 +0000 Subject: [PATCH 037/105] all: initializing logger at start --- mixnode/src/main.rs | 2 ++ sfw-provider/src/main.rs | 2 ++ validator/src/main.rs | 9 --------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/mixnode/src/main.rs b/mixnode/src/main.rs index ee05378043..6a466f67f2 100644 --- a/mixnode/src/main.rs +++ b/mixnode/src/main.rs @@ -5,6 +5,8 @@ mod mix_peer; mod node; fn main() { + pretty_env_logger::init(); + let arg_matches = App::new("Nym Mixnode") .version(built_info::PKG_VERSION) .author("Nymtech") diff --git a/sfw-provider/src/main.rs b/sfw-provider/src/main.rs index 9e55332ca1..e0efaa480c 100644 --- a/sfw-provider/src/main.rs +++ b/sfw-provider/src/main.rs @@ -8,6 +8,8 @@ use std::process; pub mod provider; fn main() { + pretty_env_logger::init(); + let arg_matches = App::new("Nym Service Provider") .version(built_info::PKG_VERSION) .author("Nymtech") diff --git a/validator/src/main.rs b/validator/src/main.rs index ea7b2679a2..8b8503cc91 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1,7 +1,6 @@ use crate::validator::config::Config; use crate::validator::Validator; use clap::{App, Arg, ArgMatches, SubCommand}; -use dotenv; use log::{error, trace}; use std::process; use toml; @@ -9,14 +8,6 @@ use toml; mod validator; fn main() { - // load environment variables from .env file - if dotenv::dotenv().is_err() { - eprint!("failed to read .env file - the logging is unlikely to work correctly") - } - - // if we want to log to file or use different logger, we'd need to replace it here. - // a better alternative, but way more complex would be `slog` crate - we should - // perhaps research it at some point. pretty_env_logger::init(); let arg_matches = App::new("Nym Validator") From 1e8423b3395c7f31d327d431271a5ce7b04ba4b6 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 17:22:03 +0000 Subject: [PATCH 038/105] Started to use logging instead of println --- .../clients/directory-client/src/presence.rs | 3 +- common/clients/provider-client/src/lib.rs | 11 +++---- mixnode/src/main.rs | 3 +- mixnode/src/node/mod.rs | 11 ++++--- nym-client/src/client/mod.rs | 2 +- nym-client/src/sockets/tcp.rs | 33 ++++++++----------- nym-client/src/sockets/ws.rs | 2 +- sfw-provider/src/main.rs | 3 +- .../src/provider/client_handling/mod.rs | 9 +++-- sfw-provider/src/provider/mod.rs | 27 ++++++++------- sfw-provider/src/provider/storage/mod.rs | 11 +++---- 11 files changed, 56 insertions(+), 59 deletions(-) diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index f20ed9f914..8be3c73b9e 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -1,5 +1,6 @@ use crate::requests::presence_topology_get::PresenceTopologyGetRequester; use crate::{Client, Config, DirectoryClient}; +use log::*; use serde::{Deserialize, Serialize}; use std::convert::TryInto; use std::io; @@ -159,7 +160,7 @@ pub struct Topology { impl NymTopology for Topology { fn new(directory_server: String) -> Self { - println!("Using directory server: {:?}", directory_server); + debug!("Using directory server: {:?}", directory_server); let directory_config = Config { base_url: directory_server, }; diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index 9b4ca131e4..befac332c9 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -1,4 +1,5 @@ use futures::io::Error; +use log::*; use sfw_provider_requests::requests::{ProviderRequest, PullRequest, RegisterRequest}; use sfw_provider_requests::responses::{ ProviderResponse, ProviderResponseError, PullResponse, RegisterResponse, @@ -67,16 +68,14 @@ impl ProviderClient { socket.set_keepalive(Some(Duration::from_secs(2))).unwrap(); socket.write_all(&bytes[..]).await?; - if let Err(_e) = socket.shutdown(Shutdown::Write) { - // TODO: make it a silent log once we have a proper logging library - // eprintln!("failed to close write part of the socket; err = {:?}", e) + if let Err(e) = socket.shutdown(Shutdown::Write) { + warn!("failed to close write part of the socket; err = {:?}", e) } let mut response = Vec::new(); socket.read_to_end(&mut response).await?; - if let Err(_e) = socket.shutdown(Shutdown::Read) { - // TODO: make it a silent log once we have a proper logging library - // eprintln!("failed to close read part of the socket; err = {:?}", e) + if let Err(e) = socket.shutdown(Shutdown::Read) { + warn!("failed to close read part of the socket; err = {:?}", e) } Ok(response) diff --git a/mixnode/src/main.rs b/mixnode/src/main.rs index 6a466f67f2..e677558d15 100644 --- a/mixnode/src/main.rs +++ b/mixnode/src/main.rs @@ -1,4 +1,5 @@ use clap::{App, Arg, ArgMatches, SubCommand}; +use log::*; use std::process; mod mix_peer; @@ -56,7 +57,7 @@ fn main() { .get_matches(); if let Err(e) = execute(arg_matches) { - println!("{}", e); + error!("{}", e); process::exit(1); } } diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index 99f4e9646e..1ae4b6e65d 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -6,6 +6,7 @@ use curve25519_dalek::scalar::Scalar; use futures::channel::mpsc; use futures::lock::Mutex; use futures::SinkExt; +use log::*; use sphinx::header::delays::Delay as SphinxDelay; use sphinx::{ProcessedPacket, SphinxPacket}; use std::net::SocketAddr; @@ -140,7 +141,7 @@ impl PacketProcessor { .await .unwrap(); - println!("RECIPIENT: {:?}", forwarding_data.recipient); + trace!("RECIPIENT: {:?}", forwarding_data.recipient); match forwarding_data .recipient .send(forwarding_data.packet.to_bytes()) @@ -148,7 +149,7 @@ impl PacketProcessor { { Ok(()) => (), Err(e) => { - println!( + warn!( "failed to write bytes to next mix peer. err = {:?}", e.to_string() ); @@ -190,7 +191,7 @@ impl MixNode { match socket.read(&mut buf).await { // socket closed Ok(n) if n == 0 => { - println!("Remote connection closed."); + trace!("Remote connection closed."); return; } Ok(_) => { @@ -203,14 +204,14 @@ impl MixNode { PacketProcessor::wait_and_forward(fwd_data).await; } Err(e) => { - println!("failed to read from socket; err = {:?}", e); + warn!("failed to read from socket; err = {:?}", e); return; } }; // Write the some data back if let Err(e) = socket.write_all(b"foomp").await { - println!("failed to write reply to socket; err = {:?}", e); + warn!("failed to write reply to socket; err = {:?}", e); return; } } diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index c9685af0b9..8359716f71 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -259,7 +259,7 @@ impl NymClient { }); // this line in theory should never be reached as the runtime should be permanently blocked on traffic senders - eprintln!("The client went kaput..."); + error!("The client went kaput..."); Ok(()) } } diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index c063c7e4b8..709181a684 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -5,6 +5,7 @@ use futures::channel::{mpsc, oneshot}; use futures::future::FutureExt; use futures::io::Error; use futures::SinkExt; +use log::*; use sphinx::route::{Destination, DestinationAddressBytes}; use std::borrow::Borrow; use std::convert::TryFrom; @@ -94,26 +95,20 @@ impl ClientRequest { recipient_address: DestinationAddressBytes, mut input_tx: mpsc::UnboundedSender, ) -> ServerResponse { - println!( - "send handle. sending to: {:?}, msg: {:?}", - recipient_address, msg - ); + trace!("sending to: {:?}, msg: {:?}", recipient_address, msg); let dummy_surb = [0; 16]; let input_msg = InputMessage(Destination::new(recipient_address, dummy_surb), msg); - - println!("ALMOST ABOUT TO SOMEDAY SEND {:?}", input_msg); input_tx.send(input_msg).await.unwrap(); - ServerResponse::Send } async fn handle_fetch(mut msg_query: mpsc::UnboundedSender) -> ServerResponse { - println!("fetch handle"); + trace!("handle_fetch called"); let (res_tx, res_rx) = oneshot::channel(); if msg_query.send(res_tx).await.is_err() { - return ServerResponse::Error { - message: "Server failed to receive messages".to_string(), - }; + let e = "Nym-client TCP socket failed to receive messages".to_string(); + error!("{}", e); + return ServerResponse::Error { message: e }; } let messages = res_rx.map(|msg| msg).await; @@ -125,12 +120,11 @@ impl ClientRequest { } let messages = messages.unwrap(); - println!("fetched {} messages", messages.len()); + trace!("fetched {} messages", messages.len()); ServerResponse::Fetch { messages } } async fn handle_get_clients(topology: &Topology) -> ServerResponse { - println!("get client handle"); let clients = topology .mix_provider_nodes .iter() @@ -141,7 +135,6 @@ impl ClientRequest { } async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse { - println!("own details handle"); ServerResponse::OwnDetails { address: self_address_bytes.to_vec(), } @@ -189,7 +182,7 @@ fn encode_fetched_messages(messages: Vec>) -> Vec { } fn encode_list_of_clients(clients: Vec>) -> Vec { - println!("client: {:?}", clients); + debug!("client: {:?}", clients); // we can just concat all client since all of them got to be 32 bytes long // (if not, then we have bigger problem somewhere up the line) @@ -245,7 +238,7 @@ async fn accept_connection( let address = socket .peer_addr() .expect("connected streams should have a peer address"); - println!("Peer address: {}", address); + debug!("Peer address: {}", address); let topology = Arc::new(topology); @@ -258,7 +251,7 @@ async fn accept_connection( let response = match socket.read(&mut buf).await { // socket closed Ok(n) if n == 0 => { - println!("Remote connection closed."); + trace!("Remote connection closed."); return; } Ok(n) => { @@ -274,14 +267,14 @@ async fn accept_connection( } } Err(e) => { - eprintln!("failed to read from socket; err = {:?}", e); + warn!("failed to read from socket; err = {:?}", e); return; } }; let response_vec: Vec = response.into(); if let Err(e) = socket.write_all(&response_vec).await { - eprintln!("failed to write reply to socket; err = {:?}", e); + warn!("failed to write reply to socket; err = {:?}", e); return; } } @@ -308,6 +301,6 @@ pub async fn start_tcpsocket( )); } - eprintln!("The tcpsocket went kaput..."); + error!("The tcpsocket went kaput..."); Ok(()) } diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 14aacdcb5d..621c9ffe80 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -406,6 +406,6 @@ pub async fn start_websocket( )); } - eprintln!("The websocket went kaput..."); + error!("The websocket went kaput..."); Ok(()) } diff --git a/sfw-provider/src/main.rs b/sfw-provider/src/main.rs index e0efaa480c..4365d943d9 100644 --- a/sfw-provider/src/main.rs +++ b/sfw-provider/src/main.rs @@ -1,6 +1,7 @@ use crate::provider::ServiceProvider; use clap::{App, Arg, ArgMatches, SubCommand}; use crypto::identity::MixnetIdentityKeyPair; +use log::error; use std::net::ToSocketAddrs; use std::path::PathBuf; use std::process; @@ -67,7 +68,7 @@ fn main() { .get_matches(); if let Err(e) = execute(arg_matches) { - println!("{}", e); + error!("{}", e); process::exit(1); } } diff --git a/sfw-provider/src/provider/client_handling/mod.rs b/sfw-provider/src/provider/client_handling/mod.rs index e35540f682..b79c1962f7 100644 --- a/sfw-provider/src/provider/client_handling/mod.rs +++ b/sfw-provider/src/provider/client_handling/mod.rs @@ -3,6 +3,7 @@ use crate::provider::ClientLedger; use crypto::identity::{DummyMixIdentityPrivateKey, MixnetIdentityPrivateKey}; use futures::lock::Mutex as FMutex; use hmac::{Hmac, Mac}; +use log::*; use sfw_provider_requests::requests::{ ProviderRequestError, ProviderRequests, PullRequest, RegisterRequest, }; @@ -82,7 +83,7 @@ impl ClientRequestProcessor { processing_data: Arc, ) -> Result, ClientProcessingError> { let client_request = ProviderRequests::from_bytes(&data)?; - println!("Received the following request: {:?}", client_request); + trace!("Received the following request: {:?}", client_request); match client_request { ProviderRequests::Register(req) => Ok(ClientRequestProcessor::register_new_client( req, @@ -106,7 +107,6 @@ impl ClientRequestProcessor { // Wait for https://github.com/nymtech/nym-sfw-provider/issues/19 to resolve. let unlocked_ledger = processing_data.registered_clients_ledger.lock().await; - println!("Processing pull!"); if unlocked_ledger.has_token(req.auth_token) { // drop the mutex so that we could do IO without blocking others wanting to get the lock drop(unlocked_ledger); @@ -124,7 +124,10 @@ impl ClientRequestProcessor { req: RegisterRequest, processing_data: Arc, ) -> Result { - println!("Processing register new client request!"); + debug!( + "Processing register new client request: {:?}", + req.destination_address + ); let mut unlocked_ledger = processing_data.registered_clients_ledger.lock().await; let auth_token = ClientRequestProcessor::generate_new_auth_token( diff --git a/sfw-provider/src/provider/mod.rs b/sfw-provider/src/provider/mod.rs index 567bd9f850..bc32073da8 100644 --- a/sfw-provider/src/provider/mod.rs +++ b/sfw-provider/src/provider/mod.rs @@ -5,6 +5,7 @@ use crypto::identity::{DummyMixIdentityPrivateKey, DummyMixIdentityPublicKey}; use directory_client::presence::MixProviderClient; use futures::io::Error; use futures::lock::Mutex as FMutex; +use log::*; use sfw_provider_requests::AuthToken; use sphinx::route::DestinationAddressBytes; use std::collections::HashMap; @@ -132,7 +133,7 @@ impl ServiceProvider { match socket.read(&mut buf).await { // socket closed Ok(n) if n == 0 => { - println!("Remote connection closed."); + trace!("Remote connection closed."); return; } Ok(_) => { @@ -142,7 +143,7 @@ impl ServiceProvider { ) { Ok(sd) => sd, Err(e) => { - eprintln!("failed to process sphinx packet; err = {:?}", e); + warn!("failed to process sphinx packet; err = {:?}", e); return; } }; @@ -151,19 +152,19 @@ impl ServiceProvider { processing_data.read().unwrap().store_dir.as_path(), ) .unwrap_or_else(|e| { - eprintln!("failed to store processed sphinx message; err = {:?}", e); + error!("failed to store processed sphinx message; err = {:?}", e); return; }); } Err(e) => { - eprintln!("failed to read from socket; err = {:?}", e); + warn!("failed to read from socket; err = {:?}", e); return; } }; // Write the some data back if let Err(e) = socket.write_all(b"foomp").await { - eprintln!("failed to write reply to socket; err = {:?}", e); + warn!("failed to write reply to socket; err = {:?}", e); return; } } @@ -171,10 +172,10 @@ impl ServiceProvider { async fn send_response(mut socket: tokio::net::TcpStream, data: &[u8]) { if let Err(e) = socket.write_all(data).await { - eprintln!("failed to write reply to socket; err = {:?}", e) + warn!("failed to write reply to socket; err = {:?}", e) } if let Err(e) = socket.shutdown(Shutdown::Write) { - eprintln!("failed to close write part of the socket; err = {:?}", e) + warn!("failed to close write part of the socket; err = {:?}", e) } } @@ -189,7 +190,7 @@ impl ServiceProvider { let response = match socket.read(&mut buf).await { // socket closed Ok(n) if n == 0 => { - println!("Remote connection closed."); + trace!("Remote connection closed."); Err(()) } Ok(n) => { @@ -200,29 +201,27 @@ impl ServiceProvider { .await { Err(e) => { - eprintln!("failed to process client request; err = {:?}", e); + warn!("failed to process client request; err = {:?}", e); Err(()) } Ok(res) => Ok(res), } } Err(e) => { - eprintln!("failed to read from socket; err = {:?}", e); + warn!("failed to read from socket; err = {:?}", e); Err(()) } }; if let Err(e) = socket.shutdown(Shutdown::Read) { - eprintln!("failed to close read part of the socket; err = {:?}", e) + warn!("failed to close read part of the socket; err = {:?}", e) } match response { Ok(res) => { - println!("should send this response! {:?}", res); ServiceProvider::send_response(socket, &res).await; } _ => { - println!("we failed..."); ServiceProvider::send_response(socket, b"bad foomp").await; } } @@ -308,7 +307,7 @@ impl ServiceProvider { }); // this line in theory should never be reached as the runtime should be permanently blocked on listeners - eprintln!("The server went kaput..."); + error!("The server went kaput..."); Ok(()) } } diff --git a/sfw-provider/src/provider/storage/mod.rs b/sfw-provider/src/provider/storage/mod.rs index 5a0af2d639..fb7607a506 100644 --- a/sfw-provider/src/provider/storage/mod.rs +++ b/sfw-provider/src/provider/storage/mod.rs @@ -1,4 +1,5 @@ use crate::provider::{MESSAGE_RETRIEVAL_LIMIT, STORED_MESSAGE_FILENAME_LENGTH}; +use log::*; use rand::Rng; use sfw_provider_requests::DUMMY_MESSAGE_CONTENT; use sphinx::route::{DestinationAddressBytes, SURBIdentifier}; @@ -62,7 +63,7 @@ impl ClientStorage { let client_dir_name = hex::encode(store_data.client_address); let full_store_dir = store_dir.join(client_dir_name); let full_store_path = full_store_dir.join(ClientStorage::generate_random_file_name()); - println!( + debug!( "going to store: {:?} in file: {:?}", store_data.message, full_store_path ); @@ -83,7 +84,7 @@ impl ClientStorage { let client_dir_name = hex::encode(client_address); let full_store_dir = store_dir.join(client_dir_name); - println!("going to lookup: {:?}!", full_store_dir); + trace!("going to lookup: {:?}!", full_store_dir); if !full_store_dir.exists() { return Err(StoreError::ClientDoesntExistError); } @@ -93,7 +94,7 @@ impl ClientStorage { .filter(|entry| { let is_file = entry.metadata().unwrap().is_file(); if !is_file { - eprintln!( + error!( "potentially corrupted client inbox! - found a non-file - {:?}", entry.path() ); @@ -109,8 +110,6 @@ impl ClientStorage { .take(MESSAGE_RETRIEVAL_LIMIT) .collect(); - println!("retrieved the following data: {:?}", msgs); - Ok(msgs) } @@ -118,7 +117,7 @@ impl ClientStorage { // TODO 2: This should only be called AFTER we sent the reply. Because if client's connection failed after sending request // the messages would be deleted but he wouldn't have received them fn delete_file(path: PathBuf) -> io::Result<()> { - println!("Here {:?} will be deleted!", path); + trace!("Here {:?} will be deleted!", path); std::fs::remove_file(path) // another argument for db layer -> remove_file is NOT guaranteed to immediately get rid of the file } } From b5e2f9fca8c4be55bdee6180a2dbb4477b482c32 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Wed, 22 Jan 2020 17:27:30 +0000 Subject: [PATCH 039/105] all: adding dotenv to all binary crates --- Cargo.lock | 3 +++ mixnode/Cargo.toml | 1 + mixnode/src/main.rs | 1 + nym-client/Cargo.toml | 1 + nym-client/src/main.rs | 1 + sfw-provider/Cargo.toml | 1 + sfw-provider/src/main.rs | 1 + validator/src/main.rs | 1 + 8 files changed, 10 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 29518de692..8f66257ba3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,6 +1350,7 @@ dependencies = [ "curve25519-dalek", "directory-client", "dirs", + "dotenv", "futures 0.3.1", "healthcheck", "hex", @@ -1380,6 +1381,7 @@ dependencies = [ "clap", "curve25519-dalek", "directory-client", + "dotenv", "futures 0.3.1", "log", "pretty_env_logger", @@ -1397,6 +1399,7 @@ dependencies = [ "crypto", "curve25519-dalek", "directory-client", + "dotenv", "futures 0.3.1", "hex", "hmac", diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index 602d119bad..e538c329d9 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" base64 = "0.11.0" clap = "2.33.0" curve25519-dalek = "1.2.3" +dotenv = "0.15.0" futures = "0.3.1" log = "0.4" pretty_env_logger = "0.3" diff --git a/mixnode/src/main.rs b/mixnode/src/main.rs index e677558d15..f16f460dd3 100644 --- a/mixnode/src/main.rs +++ b/mixnode/src/main.rs @@ -6,6 +6,7 @@ mod mix_peer; mod node; fn main() { + dotenv::dotenv().ok(); pretty_env_logger::init(); let arg_matches = App::new("Nym Mixnode") diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 782b68d31e..51730b0dd1 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -16,6 +16,7 @@ base64 = "0.11.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dirs = "2.0.2" +dotenv = "0.15.0" futures = "0.3.1" hex = "0.4.0" log = "0.4" diff --git a/nym-client/src/main.rs b/nym-client/src/main.rs index 5dd19e9a7c..8160641828 100644 --- a/nym-client/src/main.rs +++ b/nym-client/src/main.rs @@ -7,6 +7,7 @@ pub mod config; mod sockets; fn main() { + dotenv::dotenv().ok(); pretty_env_logger::init(); let arg_matches = App::new("Nym Client") diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index 1fe97babcf..4c0fc7c132 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" base64 = "0.11.0" clap = "2.33.0" curve25519-dalek = "1.2.3" +dotenv = "0.15.0" hex = "0.4.0" futures = "0.3.1" log = "0.4" diff --git a/sfw-provider/src/main.rs b/sfw-provider/src/main.rs index 4365d943d9..6cd945a2c0 100644 --- a/sfw-provider/src/main.rs +++ b/sfw-provider/src/main.rs @@ -9,6 +9,7 @@ use std::process; pub mod provider; fn main() { + dotenv::dotenv().ok(); pretty_env_logger::init(); let arg_matches = App::new("Nym Service Provider") diff --git a/validator/src/main.rs b/validator/src/main.rs index 8b8503cc91..39d0d375f3 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -8,6 +8,7 @@ use toml; mod validator; fn main() { + dotenv::dotenv().ok(); pretty_env_logger::init(); let arg_matches = App::new("Nym Validator") From 753ecf6b6a026127a39e91f1fd96461b38816e07 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Thu, 23 Jan 2020 15:06:10 +0000 Subject: [PATCH 040/105] crypto: replaced base64 with bs58 --- Cargo.lock | 8 +++++++- common/crypto/Cargo.toml | 2 +- common/crypto/src/identity/mod.rs | 5 +++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 252451a01d..86b41fd545 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,6 +213,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bs58" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae" + [[package]] name = "built" version = "0.3.2" @@ -460,7 +466,7 @@ dependencies = [ name = "crypto" version = "0.1.0" dependencies = [ - "base64 0.11.0", + "bs58", "curve25519-dalek", "log", "pretty_env_logger", diff --git a/common/crypto/Cargo.toml b/common/crypto/Cargo.toml index 23790a23c8..5e4de60816 100644 --- a/common/crypto/Cargo.toml +++ b/common/crypto/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -base64 = "0.11.0" +bs58 = "0.3.0" curve25519-dalek = "1.2.3" log = "0.4" pretty_env_logger = "0.3" diff --git a/common/crypto/src/identity/mod.rs b/common/crypto/src/identity/mod.rs index 5953581d6b..2a580547bf 100644 --- a/common/crypto/src/identity/mod.rs +++ b/common/crypto/src/identity/mod.rs @@ -2,6 +2,7 @@ use crate::encryption::{ MixnetEncryptionKeyPair, MixnetEncryptionPrivateKey, MixnetEncryptionPublicKey, }; use crate::{encryption, PemStorable}; +use bs58; use curve25519_dalek::scalar::Scalar; pub trait MixnetIdentityKeyPair @@ -100,12 +101,12 @@ impl PemStorable for DummyMixIdentityPublicKey { impl DummyMixIdentityPublicKey { pub fn to_b64_string(&self) -> String { - base64::encode_config(&self.to_bytes(), base64::URL_SAFE) + bs58::encode(&self.to_bytes()).into_string() } #[allow(dead_code)] fn from_b64_string(val: String) -> Self { - Self::from_bytes(&base64::decode_config(&val, base64::URL_SAFE).unwrap()) + Self::from_bytes(&bs58::decode(&val).into_vec().unwrap()) } } From 2b9ed29ebd4b6eccbceee2c296d40f9f0e6928d5 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Thu, 23 Jan 2020 16:10:53 +0000 Subject: [PATCH 041/105] Minor smoothing of the README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d43dd40a96..82767c0049 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ This repository contains the full Nym platform, written in Rust. The platform is composed of multiple Rust crates. Top-level executable binary crates include: -* client - an executable which you can build into your own applications. Use it for interacting with Nym nodes. -* mixnode - the mixnode crate. +* mixnode - shuffles [Sphinx](https://github.com/nymtech/sphinx) packets together to provide privacy against network-level attackers. +* nym-client - an executable which you can build into your own applications. Use it for interacting with Nym nodes. * sfw-provider - a store-and-forward service provider. The provider acts sort of like a mailbox for mixnet messages. * validator - currently just starting development. Handles consensus ordering of transactions, mixmining, and coconut credential generation and validation. From dea363e8136478af17a83cd1b710fe71ffd8e2ed Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Thu, 23 Jan 2020 16:26:40 +0000 Subject: [PATCH 042/105] Minor docs formatting --- common/version-checker/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/version-checker/src/lib.rs b/common/version-checker/src/lib.rs index d6ae38d19e..48125479af 100644 --- a/common/version-checker/src/lib.rs +++ b/common/version-checker/src/lib.rs @@ -1,11 +1,11 @@ use semver::Version; use semver::VersionReq; -/// Checks whether given `version` is compatible with a given semantic -/// version requirement `req` according to major-minor semver rules. -/// The semantic version requirement can be passed as a full, concrete version -/// number, because that's what we'll have in our Cargo.toml files (e.g. 0.3.2). -/// The patch number in the requirement will be dropped and replaced with a wildcard (0.3.*) as all minor versions should be compatible with each other. +/// Checks whether given `version` is compatible with a given semantic version requirement `req` +/// according to major-minor semver rules. The semantic version requirement can be passed as a full, +/// concrete version number, because that's what we'll have in our Cargo.toml files (e.g. 0.3.2). +/// The patch number in the requirement gets dropped and replaced with a wildcard (0.3.*) as all +/// minor versions should be compatible with each other. pub fn is_compatible(version: &str, req: &str) -> bool { let version = Version::parse(version).unwrap(); let tmp = Version::parse(req).unwrap(); From e5b3a29c5f1b29bc0d9a7d172edf5f56465f7cae Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:53:02 +0000 Subject: [PATCH 043/105] Defined ability to compare two 'Topology' instances The comparison ignores 'last-seen' field or ordering of nodes and clients --- .../clients/directory-client/src/presence.rs | 155 +++++++++++++++++- 1 file changed, 149 insertions(+), 6 deletions(-) diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index b333e9fce9..f691bc9035 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -1,12 +1,19 @@ use crate::requests::presence_topology_get::PresenceTopologyGetRequester; use crate::{Client, Config, DirectoryClient}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::convert::TryInto; use std::io; use std::net::ToSocketAddrs; use topology::{CocoNode, MixNode, MixProviderNode, NymTopology}; -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +// special version of 'PartialEq' that does not care about last_seen field (where applicable) +// or order of elements in vectors +trait PresenceEq { + fn presence_eq(&self, other: &Self) -> bool; +} + +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct CocoPresence { pub host: String, @@ -37,7 +44,7 @@ impl From for CocoPresence { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct MixNodePresence { pub host: String, @@ -81,7 +88,7 @@ impl From for MixNodePresence { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct MixProviderPresence { pub client_listener: String, @@ -92,6 +99,32 @@ pub struct MixProviderPresence { pub version: String, } +impl PresenceEq for MixProviderPresence { + fn presence_eq(&self, other: &Self) -> bool { + if self.registered_clients.len() != other.registered_clients.len() { + return false; + } + + if self.client_listener != other.client_listener + || self.mixnet_listener != other.mixnet_listener + || self.pub_key != other.pub_key + || self.version != other.version + { + return false; + } + + let clients_self_set: HashSet<_> = + self.registered_clients.iter().map(|c| &c.pub_key).collect(); + let clients_other_set: HashSet<_> = other + .registered_clients + .iter() + .map(|c| &c.pub_key) + .collect(); + + clients_self_set == clients_other_set + } +} + impl Into for MixProviderPresence { fn into(self) -> topology::MixProviderNode { topology::MixProviderNode { @@ -126,7 +159,7 @@ impl From for MixProviderPresence { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct MixProviderClient { pub pub_key: String, @@ -148,8 +181,102 @@ impl From for MixProviderClient { } } +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration + let self_set: HashSet<_> = self + .iter() + .map(|c| (&c.host, &c.pub_key, &c.version)) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|c| (&c.host, &c.pub_key, &c.version)) + .collect(); + + self_set == other_set + } +} + +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration + let self_set: HashSet<_> = self + .iter() + .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) + .collect(); + + self_set == other_set + } +} + +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration. + // We also don't care about order of registered_clients + + // since we're going to be getting rid of this very soon anyway, just clone registered + // clients vector and sort it + + let self_set: HashSet<_> = self + .iter() + .map(|p| { + ( + &p.client_listener, + &p.mixnet_listener, + &p.pub_key, + &p.version, + p.registered_clients + .iter() + .cloned() + .map(|c| c.pub_key) + .collect::>() + .sort(), + ) + }) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|p| { + ( + &p.client_listener, + &p.mixnet_listener, + &p.pub_key, + &p.version, + p.registered_clients + .iter() + .cloned() + .map(|c| c.pub_key) + .collect::>() + .sort(), + ) + }) + .collect(); + + self_set == other_set + } +} + // Topology shows us the current state of the overall Nym network -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct Topology { pub coco_nodes: Vec, @@ -157,9 +284,25 @@ pub struct Topology { pub mix_provider_nodes: Vec, } +impl PartialEq for Topology { + // we need a custom implementation as the order of nodes does not matter + // also we do not care about 'last_seen' when comparing topologies + fn eq(&self, other: &Self) -> bool { + if !self.coco_nodes.presence_eq(&other.coco_nodes) + || !self.mix_nodes.presence_eq(&other.mix_nodes) + || !self + .mix_provider_nodes + .presence_eq(&other.mix_provider_nodes) + { + return false; + } + + true + } +} + impl NymTopology for Topology { fn new(directory_server: String) -> Self { - println!("Using directory server: {:?}", directory_server); let directory_config = Config { base_url: directory_server, }; From 52272d6119cbad53dac099f70113ca7a39ffc943 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:53:30 +0000 Subject: [PATCH 044/105] Added restriction on NymTopology to require Debug and Send + Sync --- common/topology/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 0cd511de3f..2833c0dd08 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -83,7 +83,7 @@ pub enum NymTopologyError { MissingLayerError(Vec), } -pub trait NymTopology: Sized + PartialEq { +pub trait NymTopology: Sized + PartialEq + std::fmt::Debug + Send + Sync { fn new(directory_server: String) -> Self; fn new_from_nodes( mix_nodes: Vec, From 8c26ef848a8b9d4e169777a657727e452d318274 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:54:16 +0000 Subject: [PATCH 045/105] Created a module responsible for periodically refreshing topology --- nym-client/src/client/topology_control.rs | 140 ++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 nym-client/src/client/topology_control.rs diff --git a/nym-client/src/client/topology_control.rs b/nym-client/src/client/topology_control.rs new file mode 100644 index 0000000000..8adf953dfb --- /dev/null +++ b/nym-client/src/client/topology_control.rs @@ -0,0 +1,140 @@ +use crate::built_info; +use log::{error, info, trace, warn}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::RwLock as FRwLock; +use topology::NymTopology; + +const NODE_HEALTH_THRESHOLD: f64 = 0.0; + +// auxiliary type for ease of use +pub type TopologyInnerRef = Arc>>; + +pub(crate) struct TopologyControl { + directory_server: String, + refresh_rate: f64, + inner: Arc>>, +} + +#[derive(Debug)] +enum TopologyError { + HealthCheckError, + NoValidPathsError, +} + +impl TopologyControl { + pub(crate) async fn new(directory_server: String, refresh_rate: f64) -> Self { + let initial_topology = match Self::get_current_compatible_topology(directory_server.clone()) + .await + { + Ok(topology) => Some(topology), + Err(err) => { + error!("Initial topology is invalid - {:?}. Right now it will be impossible to send any packets through the mixnet!", err); + None + } + }; + + TopologyControl { + directory_server, + refresh_rate, + inner: Arc::new(FRwLock::new(Inner::new(initial_topology))), + } + } + + async fn get_current_compatible_topology(directory_server: String) -> Result { + let full_topology = T::new(directory_server.clone()); + + // run a healthcheck to determine healthy-ish nodes: + // this is a temporary solution as the healthcheck will eventually be moved to validators + let healthcheck_config = healthcheck::config::HealthCheck { + directory_server, + // those are literally irrelevant when running single check + interval: 100000.0, + resolution_timeout: 5.0, + num_test_packets: 2, + }; + let healthcheck = healthcheck::HealthChecker::new(healthcheck_config); + let healthcheck_result = healthcheck.do_check().await; + + let healthcheck_scores = match healthcheck_result { + Err(err) => { + error!("Error while performing the healtcheck: {:?}", err); + return Err(TopologyError::HealthCheckError); + } + Ok(scores) => scores, + }; + + let healthy_topology = + healthcheck_scores.filter_topology_by_score(&full_topology, NODE_HEALTH_THRESHOLD); + + // for time being assume same versioning, i.e. if client is running X.Y.Z, + // we're expecting mixes, providers and coconodes to also be running X.Y.Z + let versioned_healthy_topology = + healthy_topology.filter_node_versions("0.3.2", "0.3.2", built_info::PKG_VERSION); + + // make sure you can still send a packet through the network: + if !versioned_healthy_topology.can_construct_path_through() { + return Err(TopologyError::NoValidPathsError); + } + + Ok(versioned_healthy_topology) + } + + pub(crate) fn get_inner_ref(&self) -> Arc>> { + self.inner.clone() + } + + async fn update_global_topology(&mut self, new_topology: Option) { + // acquire write lock + let mut write_lock = self.inner.write().await; + write_lock.topology = new_topology; + } + + async fn should_update_topology(&mut self, new_topology: &Option) -> bool { + let read_lock = self.inner.read().await; + match new_topology { + // if new topology is invalid, we MUST update to it as it is impossible to send packets through + None => true, + Some(new_topology) => match &read_lock.topology { + None => true, + Some(old_topology) => new_topology != old_topology, + }, + } + } + + pub(crate) async fn run_refresher(mut self) { + info!("Starting topology refresher"); + let delay_duration = Duration::from_secs_f64(self.refresh_rate); + loop { + trace!("Refreshing the topology"); + let new_topology_res = + Self::get_current_compatible_topology(self.directory_server.clone()).await; + + let new_topology = match new_topology_res { + Ok(topology) => Some(topology), + Err(err) => { + warn!("the obtained topology seems to be invalid - {:?}, it will be impossible to send packets through", err); + None + } + }; + + if self.should_update_topology(&new_topology).await { + info!("Detected changes in topology - updating global view!"); + + self.update_global_topology(new_topology).await; + } + + tokio::time::delay_for(delay_duration).await; + } + } +} + +pub struct Inner { + pub topology: Option, +} + +impl Inner { + fn new(topology: Option) -> Self { + Inner { topology } + } +} From 2079b8f9262b4e3d3dbd9481acbceeb31b4b5066 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:54:43 +0000 Subject: [PATCH 046/105] Said module used by cover traffic stream --- nym-client/src/client/cover_traffic_stream.rs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/nym-client/src/client/cover_traffic_stream.rs b/nym-client/src/client/cover_traffic_stream.rs index 82f904eec7..f1caedc740 100644 --- a/nym-client/src/client/cover_traffic_stream.rs +++ b/nym-client/src/client/cover_traffic_stream.rs @@ -1,29 +1,35 @@ use crate::client::mix_traffic::MixMessage; +use crate::client::topology_control::TopologyInnerRef; use crate::client::LOOP_COVER_AVERAGE_DELAY; use futures::channel::mpsc; -use log::{info, trace}; +use log::{info, trace, warn}; use sphinx::route::Destination; use std::time::Duration; use topology::NymTopology; -pub(crate) async fn start_loop_cover_traffic_stream( +pub(crate) async fn start_loop_cover_traffic_stream( tx: mpsc::UnboundedSender, our_info: Destination, - topology: T, -) where - T: NymTopology, -{ + topology_ctrl_ref: TopologyInnerRef, +) { info!("Starting loop cover traffic stream"); loop { trace!("next cover message!"); let delay = mix_client::poisson::sample(LOOP_COVER_AVERAGE_DELAY); let delay_duration = Duration::from_secs_f64(delay); tokio::time::delay_for(delay_duration).await; - let cover_message = mix_client::packet::loop_cover_message( - our_info.address, - our_info.identifier, - &topology, - ); + + let read_lock = topology_ctrl_ref.read().await; + let topology = read_lock.topology.as_ref(); + if topology.is_none() { + warn!("No valid topology detected - won't send any loop cover message this time"); + continue; + } + + let topology = topology.unwrap(); + + let cover_message = + mix_client::packet::loop_cover_message(our_info.address, our_info.identifier, topology); // if this one fails, there's no retrying because it means that either: // - we run out of memory From 35451a05de2760c6d2e125dbdc8bb57f37baa65d Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:55:11 +0000 Subject: [PATCH 047/105] And by real traffic stream + necessary adjustments --- nym-client/src/client/real_traffic_stream.rs | 81 +++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index f89257747b..66a80a2ebe 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -1,38 +1,38 @@ use crate::client::mix_traffic::MixMessage; +use crate::client::topology_control::TopologyInnerRef; use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY}; -use directory_client::presence::Topology; use futures::channel::mpsc; use futures::task::{Context, Poll}; use futures::{Future, Stream, StreamExt}; -use log::{debug, info, trace}; +use log::{info, trace, warn}; use sphinx::route::Destination; -use sphinx::SphinxPacket; -use std::net::SocketAddr; use std::pin::Pin; use std::time::Duration; use tokio::time; +use topology::NymTopology; // have a rather low value for test sake const AVERAGE_PACKET_DELAY: f64 = 0.1; -pub(crate) struct OutQueueControl { +pub(crate) struct OutQueueControl { delay: time::Delay, mix_tx: mpsc::UnboundedSender, input_rx: mpsc::UnboundedReceiver, our_info: Destination, - - // due to pinning, DerefMut trait, futures, etc its way easier to - // just have concrete implementation here rather than generic NymTopology - // considering that it will be replaced with refreshing topology within few days anyway - topology: Topology, + topology_ctrl_ref: TopologyInnerRef, } -impl Stream for OutQueueControl { - type Item = (SocketAddr, SphinxPacket); +pub(crate) enum StreamMessage { + Cover, + Real(InputMessage), +} + +impl Stream for OutQueueControl { + type Item = StreamMessage; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // it is not yet time to return a message - if Future::poll(Pin::new(&mut self.delay), cx).is_pending() { + if Pin::new(&mut self.delay).poll(cx).is_pending() { return Poll::Pending; }; @@ -49,41 +49,26 @@ impl Stream for OutQueueControl { self.delay.reset(next); // decide what kind of message to send - match Stream::poll_next(Pin::new(&mut self.input_rx), cx) { + match Pin::new(&mut self.input_rx).poll_next(cx) { // in the case our real message channel stream was closed, we should also indicate we are closed // (and whoever is using the stream should panic) Poll::Ready(None) => Poll::Ready(None), // if there's an actual message - return it - Poll::Ready(Some(real_message)) => { - trace!("real message"); - Poll::Ready(Some(mix_client::packet::encapsulate_message( - real_message.0, - real_message.1, - &self.topology, - AVERAGE_PACKET_DELAY, - ))) - } + Poll::Ready(Some(real_message)) => Poll::Ready(Some(StreamMessage::Real(real_message))), // otherwise construct a dummy one - _ => { - trace!("loop cover message"); - Poll::Ready(Some(mix_client::packet::loop_cover_message( - self.our_info.address, - self.our_info.identifier, - &self.topology, - ))) - } + Poll::Pending => Poll::Ready(Some(StreamMessage::Cover)), } } } -impl OutQueueControl { +impl OutQueueControl { pub(crate) fn new( mix_tx: mpsc::UnboundedSender, input_rx: mpsc::UnboundedReceiver, our_info: Destination, - topology: Topology, + topology: TopologyInnerRef, ) -> Self { let initial_delay = time::delay_for(Duration::from_secs_f64(MESSAGE_SENDING_AVERAGE_DELAY)); OutQueueControl { @@ -91,20 +76,44 @@ impl OutQueueControl { mix_tx, input_rx, our_info, - topology, + topology_ctrl_ref: topology, } } pub(crate) async fn run_out_queue_control(mut self) { info!("starting out queue controller"); while let Some(next_message) = self.next().await { - debug!("created new message"); + trace!("created new message"); + let read_lock = self.topology_ctrl_ref.read().await; + let topology = read_lock.topology.as_ref(); + + if topology.is_none() { + warn!("No valid topology detected - won't send any loop cover message this time"); + continue; + } + + let topology = topology.unwrap(); + + let next_packet = match next_message { + StreamMessage::Cover => mix_client::packet::loop_cover_message( + self.our_info.address, + self.our_info.identifier, + topology, + ), + StreamMessage::Real(real_message) => mix_client::packet::encapsulate_message( + real_message.0, + real_message.1, + topology, + AVERAGE_PACKET_DELAY, + ), + }; + // if this one fails, there's no retrying because it means that either: // - we run out of memory // - the receiver channel is closed // in either case there's no recovery and we can only panic self.mix_tx - .unbounded_send(MixMessage::new(next_message.0, next_message.1)) + .unbounded_send(MixMessage::new(next_packet.0, next_packet.1)) .unwrap(); } } From 6f69641443cb83af6f7d29919f761cdf020d404c Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:55:39 +0000 Subject: [PATCH 048/105] Made sockets more generic to work on any NymTopology object --- nym-client/src/sockets/tcp.rs | 52 +++++++++++++++++++---------------- nym-client/src/sockets/ws.rs | 50 +++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index c063c7e4b8..7270453a9e 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -1,17 +1,16 @@ use crate::client::received_buffer::BufferResponse; +use crate::client::topology_control::TopologyInnerRef; use crate::client::InputMessage; -use directory_client::presence::Topology; use futures::channel::{mpsc, oneshot}; use futures::future::FutureExt; use futures::io::Error; use futures::SinkExt; use sphinx::route::{Destination, DestinationAddressBytes}; -use std::borrow::Borrow; use std::convert::TryFrom; use std::io; use std::net::SocketAddr; -use std::sync::Arc; use tokio::prelude::*; +use topology::NymTopology; const SEND_REQUEST_PREFIX: u8 = 1; const FETCH_REQUEST_PREFIX: u8 = 2; @@ -129,15 +128,24 @@ impl ClientRequest { ServerResponse::Fetch { messages } } - async fn handle_get_clients(topology: &Topology) -> ServerResponse { - println!("get client handle"); - let clients = topology - .mix_provider_nodes - .iter() - .flat_map(|provider| provider.registered_clients.iter()) - .map(|client| base64::decode_config(&client.pub_key, base64::URL_SAFE).unwrap()) // TODO: this can potentially throw an error - .collect(); - ServerResponse::GetClients { clients } + async fn handle_get_clients(topology: &TopologyInnerRef) -> ServerResponse { + let topology_data = &topology.read().await.topology; + match topology_data { + Some(topology) => { + let clients = topology + .get_mix_provider_nodes() + .iter() + .flat_map(|provider| provider.registered_clients.iter()) + .filter_map(|client| { + base64::decode_config(&client.pub_key, base64::URL_SAFE).ok() + }) + .collect(); + ServerResponse::GetClients { clients } + } + None => ServerResponse::Error { + message: "Invalid network topology".to_string(), + }, + } } async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse { @@ -203,9 +211,9 @@ impl ServerResponse { } } -async fn handle_connection( +async fn handle_connection( data: &[u8], - request_handling_data: RequestHandlingData, + request_handling_data: RequestHandlingData, ) -> Result { let request = ClientRequest::try_from(data)?; let response = match request { @@ -218,7 +226,7 @@ async fn handle_connection( } ClientRequest::Fetch => ClientRequest::handle_fetch(request_handling_data.msg_query).await, ClientRequest::GetClients => { - ClientRequest::handle_get_clients(request_handling_data.topology.borrow()).await + ClientRequest::handle_get_clients(&request_handling_data.topology).await } ClientRequest::OwnDetails => { ClientRequest::handle_own_details(request_handling_data.self_address).await @@ -228,27 +236,25 @@ async fn handle_connection( Ok(response) } -struct RequestHandlingData { +struct RequestHandlingData { msg_input: mpsc::UnboundedSender, msg_query: mpsc::UnboundedSender, self_address: DestinationAddressBytes, - topology: Arc, + topology: TopologyInnerRef, } -async fn accept_connection( +async fn accept_connection( mut socket: tokio::net::TcpStream, msg_input: mpsc::UnboundedSender, msg_query: mpsc::UnboundedSender, self_address: DestinationAddressBytes, - topology: Topology, + topology: TopologyInnerRef, ) { let address = socket .peer_addr() .expect("connected streams should have a peer address"); println!("Peer address: {}", address); - let topology = Arc::new(topology); - let mut buf = [0u8; 2048]; // In a loop, read data from the socket and write the data back. @@ -287,12 +293,12 @@ async fn accept_connection( } } -pub async fn start_tcpsocket( +pub async fn start_tcpsocket( address: SocketAddr, message_tx: mpsc::UnboundedSender, received_messages_query_tx: mpsc::UnboundedSender, self_address: DestinationAddressBytes, - topology: Topology, + topology: TopologyInnerRef, ) -> Result<(), TCPSocketError> { let mut listener = tokio::net::TcpListener::bind(address).await?; diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 14aacdcb5d..10f9eb4020 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -1,6 +1,6 @@ use crate::client::received_buffer::BufferResponse; +use crate::client::topology_control::TopologyInnerRef; use crate::client::InputMessage; -use directory_client::presence::Topology; use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use futures::channel::{mpsc, oneshot}; use futures::future::FutureExt; @@ -12,20 +12,21 @@ use sphinx::route::{Destination, DestinationAddressBytes}; use std::convert::TryFrom; use std::io; use std::net::SocketAddr; +use topology::NymTopology; use tungstenite::protocol::frame::coding::CloseCode; use tungstenite::protocol::{CloseFrame, Message}; -struct Connection { +struct Connection { address: SocketAddr, msg_input: mpsc::UnboundedSender, msg_query: mpsc::UnboundedSender, rx: UnboundedReceiver, self_address: DestinationAddressBytes, - topology: Topology, + topology: TopologyInnerRef, tx: UnboundedSender, } -impl Connection { +impl Connection { async fn handle_text_message(&self, msg: String) -> ServerResponse { debug!("Handling text message request"); trace!("Content: {:?}", msg.clone()); @@ -48,9 +49,7 @@ impl Connection { ClientRequest::handle_send(message, recipient_address, self.msg_input.clone()).await } ClientRequest::Fetch => ClientRequest::handle_fetch(self.msg_query.clone()).await, - ClientRequest::GetClients => { - ClientRequest::handle_get_clients(self.topology.clone()).await - } + ClientRequest::GetClients => ClientRequest::handle_get_clients(&self.topology).await, ClientRequest::OwnDetails => ClientRequest::handle_own_details(self.self_address).await, } } @@ -212,7 +211,7 @@ impl ClientRequest { Err(e) => { return ServerResponse::Error { message: e.to_string(), - } + }; } Ok(hex) => hex, }; @@ -269,14 +268,22 @@ impl ClientRequest { ServerResponse::Fetch { messages } } - async fn handle_get_clients(topology: Topology) -> ServerResponse { - let clients = topology - .mix_provider_nodes - .into_iter() - .flat_map(|provider| provider.registered_clients.into_iter()) - .map(|client| client.pub_key) - .collect(); - ServerResponse::GetClients { clients } + async fn handle_get_clients(topology: &TopologyInnerRef) -> ServerResponse { + let topology_data = &topology.read().await.topology; + match topology_data { + Some(topology) => { + let clients = topology + .get_mix_provider_nodes() + .iter() + .flat_map(|provider| provider.registered_clients.iter()) + .map(|client| client.pub_key.clone()) + .collect(); + ServerResponse::GetClients { clients } + } + None => ServerResponse::Error { + message: "Invalid network topology".to_string(), + }, + } } async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse { @@ -306,14 +313,13 @@ impl Into for ServerResponse { } } -async fn accept_connection( +async fn accept_connection( stream: tokio::net::TcpStream, msg_input: mpsc::UnboundedSender, msg_query: mpsc::UnboundedSender, self_address: DestinationAddressBytes, - topology: Topology, + topology: TopologyInnerRef, ) { - warn!("accept_connection"); let address = stream .peer_addr() .expect("connected streams should have a peer address"); @@ -337,6 +343,8 @@ async fn accept_connection( msg_query, self_address, }; + + // TODO: make sure this actually doesn't leak memory... tokio::spawn(conn.handle()); while let Some(message) = ws_stream.next().await { @@ -385,12 +393,12 @@ async fn accept_connection( } } -pub async fn start_websocket( +pub async fn start_websocket( address: SocketAddr, message_tx: mpsc::UnboundedSender, received_messages_query_tx: mpsc::UnboundedSender, self_address: DestinationAddressBytes, - topology: Topology, + topology: TopologyInnerRef, ) -> Result<(), WebSocketError> { let mut listener = tokio::net::TcpListener::bind(address).await?; From 54c244e4eacab88602c1ed1b1925e08643ff3400 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 16:55:59 +0000 Subject: [PATCH 049/105] Everything put together in mod.rs --- nym-client/src/client/mod.rs | 102 +++++++++++------------------------ 1 file changed, 32 insertions(+), 70 deletions(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index c9685af0b9..01c12470da 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -1,6 +1,6 @@ -use crate::built_info; use crate::client::mix_traffic::MixTrafficController; use crate::client::received_buffer::ReceivedMessagesBuffer; +use crate::client::topology_control::TopologyInnerRef; use crate::sockets::tcp; use crate::sockets::ws; use directory_client::presence::Topology; @@ -18,6 +18,7 @@ mod mix_traffic; mod provider_poller; mod real_traffic_stream; pub mod received_buffer; +pub mod topology_control; // TODO: all of those constants should probably be moved to config file const LOOP_COVER_AVERAGE_DELAY: f64 = 0.5; @@ -26,6 +27,8 @@ const MESSAGE_SENDING_AVERAGE_DELAY: f64 = 0.5; // seconds; const FETCH_MESSAGES_DELAY: f64 = 1.0; // seconds; +const TOPOLOGY_REFRESH_RATE: f64 = 10.0; // seconds + pub enum SocketType { TCP, WebSocket, @@ -46,13 +49,6 @@ pub struct NymClient { socket_type: SocketType, } -// TODO: this will be moved into module responsible for refreshing topology -#[derive(Debug)] -enum TopologyError { - HealthCheckError, - NoValidPathsError, -} - #[derive(Debug)] pub struct InputMessage(pub Destination, pub Vec); @@ -77,50 +73,16 @@ impl NymClient { } } - // TODO: this will be moved into module responsible for refreshing topology - async fn get_compatible_topology(&self) -> Result { - let score_threshold = 0.0; - info!("Trying to obtain valid, healthy, topology"); - - let full_topology = Topology::new(self.directory.clone()); - - // run a healthcheck to determine healthy-ish nodes: - // this is a temporary solution as the healthcheck will eventually be moved to validators - let healthcheck_config = healthcheck::config::HealthCheck { - directory_server: self.directory.clone(), - // those are literally unrelevant when running single check - interval: 100000.0, - resolution_timeout: 5.0, - num_test_packets: 2, - }; - let healthcheck = healthcheck::HealthChecker::new(healthcheck_config); - let healthcheck_result = healthcheck.do_check().await; - - let healthcheck_scores = match healthcheck_result { - Err(err) => { - error!("Error while performing the healtcheck: {:?}", err); - return Err(TopologyError::HealthCheckError); - } - Ok(scores) => scores, - }; - - let healthy_topology = - healthcheck_scores.filter_topology_by_score(&full_topology, score_threshold); - - // for time being assume same versioning, i.e. if client is running X.Y.Z, - // we're expecting mixes, providers and coconodes to also be running X.Y.Z - let versioned_healthy_topology = healthy_topology.filter_node_versions( - built_info::PKG_VERSION, - built_info::PKG_VERSION, - built_info::PKG_VERSION, - ); - - // make sure you can still send a packet through the network: - if !versioned_healthy_topology.can_construct_path_through() { - return Err(TopologyError::NoValidPathsError); - } - - Ok(versioned_healthy_topology) + async fn get_provider_socket_address( + &self, + topology_ctrl_ref: TopologyInnerRef, + ) -> SocketAddr { + // this is temporary and assumes there exists only a single provider. + topology_ctrl_ref.read().await.topology.as_ref().unwrap() + .get_mix_provider_nodes() + .first() + .expect("Could not get a provider from the initial network topology, are you using the right directory server?") + .client_listener } pub fn start(self) -> Result<(), Box> { @@ -144,20 +106,14 @@ impl NymClient { let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); - // get initial topology; already filtered by health and version - let initial_topology = match rt.block_on(self.get_compatible_topology()) { - Ok(topology) => topology, - Err(err) => { - panic!("Failed to obtain initial network topology: {:?}", err); - } - }; + // TODO: when we switch to our graph topology, we need to remember to change 'Topology' type + let topology_controller = rt.block_on(topology_control::TopologyControl::::new( + self.directory.clone(), + TOPOLOGY_REFRESH_RATE, + )); - // this is temporary and assumes there exists only a single provider. - let provider_client_listener_address: SocketAddr = initial_topology - .get_mix_provider_nodes() - .first() - .expect("Could not get a provider from the supplied network topology, are you using the right directory server?") - .client_listener; + let provider_client_listener_address = + rt.block_on(self.get_provider_socket_address(topology_controller.get_inner_ref())); let mut provider_poller = provider_poller::ProviderPoller::new( poller_input_tx, @@ -189,13 +145,13 @@ impl NymClient { rt.spawn(cover_traffic_stream::start_loop_cover_traffic_stream( mix_tx.clone(), Destination::new(self.address, Default::default()), - initial_topology.clone(), + topology_controller.get_inner_ref(), )); // cloning arguments required by OutQueueControl; required due to move - let topology_clone = initial_topology.clone(); let self_address = self.address; let input_rx = self.input_rx; + let topology_ref = topology_controller.get_inner_ref(); // future constantly pumping traffic at some specified average rate // if a real message is available on 'input_rx' that might have been received from say @@ -206,7 +162,7 @@ impl NymClient { mix_tx, input_rx, Destination::new(self_address, Default::default()), - topology_clone, + topology_ref, ) .run_out_queue_control() .await @@ -225,7 +181,7 @@ impl NymClient { self.input_tx, received_messages_buffer_output_tx, self.address, - initial_topology, + topology_controller.get_inner_ref(), )); } SocketType::TCP => { @@ -234,12 +190,16 @@ impl NymClient { self.input_tx, received_messages_buffer_output_tx, self.address, - initial_topology, + topology_controller.get_inner_ref(), )); } SocketType::None => (), } + // future responsible for periodically polling directory server and updating + // the current global view of topology + let topology_refresher_future = rt.spawn(topology_controller.run_refresher()); + rt.block_on(async { let future_results = join!( received_messages_buffer_controllers_future, @@ -247,6 +207,7 @@ impl NymClient { loop_cover_traffic_future, out_queue_control_future, provider_polling_future, + topology_refresher_future, ); assert!( @@ -255,6 +216,7 @@ impl NymClient { && future_results.2.is_ok() && future_results.3.is_ok() && future_results.4.is_ok() + && future_results.5.is_ok() ); }); From c61b49c353a6c94831ac274f5f4503a51112f9b3 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 17:08:26 +0000 Subject: [PATCH 050/105] Corrected warnin message for out queue control --- nym-client/src/client/real_traffic_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index 66a80a2ebe..9d1ee055ab 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -88,7 +88,7 @@ impl OutQueueControl { let topology = read_lock.topology.as_ref(); if topology.is_none() { - warn!("No valid topology detected - won't send any loop cover message this time"); + warn!("No valid topology detected - won't send any loop cover or real message this time"); continue; } From d9ac85abb2449a89f8b9ff1ee3ac507ca40bb704 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 17:17:28 +0000 Subject: [PATCH 051/105] Removed hardcoded version numbers --- nym-client/src/client/topology_control.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nym-client/src/client/topology_control.rs b/nym-client/src/client/topology_control.rs index 8adf953dfb..6f749682dd 100644 --- a/nym-client/src/client/topology_control.rs +++ b/nym-client/src/client/topology_control.rs @@ -67,10 +67,11 @@ impl TopologyControl { let healthy_topology = healthcheck_scores.filter_topology_by_score(&full_topology, NODE_HEALTH_THRESHOLD); - // for time being assume same versioning, i.e. if client is running X.Y.Z, - // we're expecting mixes, providers and coconodes to also be running X.Y.Z - let versioned_healthy_topology = - healthy_topology.filter_node_versions("0.3.2", "0.3.2", built_info::PKG_VERSION); + let versioned_healthy_topology = healthy_topology.filter_node_versions( + built_info::PKG_VERSION, + built_info::PKG_VERSION, + built_info::PKG_VERSION, + ); // make sure you can still send a packet through the network: if !versioned_healthy_topology.can_construct_path_through() { From 4a2eab76e70746d60af40ef82cbf6e5ecd2a92b1 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 17:19:08 +0000 Subject: [PATCH 052/105] Decreased log severity for failing to send test packet during healthcheck --- common/healthcheck/src/path_check.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/healthcheck/src/path_check.rs b/common/healthcheck/src/path_check.rs index 62b94a86b1..42705b181d 100644 --- a/common/healthcheck/src/path_check.rs +++ b/common/healthcheck/src/path_check.rs @@ -223,7 +223,7 @@ impl PathChecker { debug!("sending test packet to {}", first_node_address); match first_node_client.send(packet, first_node_address).await { Err(err) => { - warn!("failed to send packet to {} - {}", first_node_address, err); + info!("failed to send packet to {} - {}", first_node_address, err); if self .paths_status .insert(path_identifier, PathStatus::Unhealthy) From 143d45904c872f655ff42435442c4499077cb340 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Thu, 23 Jan 2020 17:19:40 +0000 Subject: [PATCH 053/105] missing import --- common/healthcheck/src/path_check.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/healthcheck/src/path_check.rs b/common/healthcheck/src/path_check.rs index 42705b181d..89fe995880 100644 --- a/common/healthcheck/src/path_check.rs +++ b/common/healthcheck/src/path_check.rs @@ -1,6 +1,6 @@ use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; use itertools::Itertools; -use log::{debug, error, trace, warn}; +use log::{debug, error, info, trace, warn}; use mix_client::MixClient; use provider_client::ProviderClient; use sphinx::header::delays::Delay; From 4cb1719ee42f546bfe59c09ae599efb3327cf13a Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:03:02 +0000 Subject: [PATCH 054/105] Back to compiling after some self-experimentation and learning... --- Cargo.lock | 37 +++- .../clients/directory-client/src/presence.rs | 61 +++--- common/clients/mix-client/Cargo.toml | 3 +- common/clients/mix-client/src/packet.rs | 2 +- common/healthcheck/Cargo.toml | 3 +- common/healthcheck/src/path_check.rs | 4 +- common/healthcheck/src/result.rs | 23 +-- common/healthcheck/src/score.rs | 7 +- common/topology/Cargo.toml | 3 +- common/topology/src/coco.rs | 13 ++ common/topology/src/lib.rs | 193 +++++++----------- common/topology/src/mix.rs | 37 ++++ common/topology/src/provider.rs | 43 ++++ nym-client/Cargo.toml | 3 +- nym-client/src/client/mod.rs | 2 +- 15 files changed, 257 insertions(+), 177 deletions(-) create mode 100644 common/topology/src/coco.rs create mode 100644 common/topology/src/mix.rs create mode 100644 common/topology/src/provider.rs diff --git a/Cargo.lock b/Cargo.lock index 252451a01d..ac86ab4e59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,7 +856,7 @@ dependencies = [ "serde", "serde_derive", "sfw-provider-requests", - "sphinx", + "sphinx 0.1.0", "tokio 0.2.9", "topology", ] @@ -1257,7 +1257,7 @@ dependencies = [ "pretty_env_logger", "rand 0.7.2", "rand_distr", - "sphinx", + "sphinx 0.1.0", "tokio 0.2.9", "topology", ] @@ -1364,7 +1364,7 @@ dependencies = [ "serde", "serde_json", "sfw-provider-requests", - "sphinx", + "sphinx 0.1.0", "tokio 0.2.9", "tokio-tungstenite", "topology", @@ -1385,7 +1385,7 @@ dependencies = [ "futures 0.3.1", "log", "pretty_env_logger", - "sphinx", + "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", "tokio 0.2.9", ] @@ -1410,7 +1410,7 @@ dependencies = [ "serde_json", "sfw-provider-requests", "sha2", - "sphinx", + "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", "tokio 0.2.9", ] @@ -1618,7 +1618,7 @@ dependencies = [ "log", "pretty_env_logger", "sfw-provider-requests", - "sphinx", + "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", "tokio 0.2.9", ] @@ -2025,7 +2025,7 @@ dependencies = [ name = "sfw-provider-requests" version = "0.1.0" dependencies = [ - "sphinx", + "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", ] [[package]] @@ -2116,6 +2116,27 @@ dependencies = [ "sha2", ] +[[package]] +name = "sphinx" +version = "0.1.0" +dependencies = [ + "aes-ctr", + "arrayref", + "base64 0.11.0", + "blake2", + "byteorder", + "chacha", + "curve25519-dalek", + "hkdf", + "hmac", + "lioness", + "log", + "rand 0.7.2", + "rand_distr", + "rand_os", + "sha2", +] + [[package]] name = "stream-cipher" version = "0.3.2" @@ -2427,7 +2448,7 @@ dependencies = [ "pretty_env_logger", "rand 0.7.2", "serde", - "sphinx", + "sphinx 0.1.0", "version-checker", ] diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index 8be3c73b9e..469a1dd026 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -5,7 +5,10 @@ use serde::{Deserialize, Serialize}; use std::convert::TryInto; use std::io; use std::net::ToSocketAddrs; -use topology::{CocoNode, MixNode, MixProviderNode, NymTopology}; +use topology::coco; +use topology::mix; +use topology::provider; +use topology::NymTopology; #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -16,9 +19,9 @@ pub struct CocoPresence { pub version: String, } -impl Into for CocoPresence { - fn into(self) -> topology::CocoNode { - topology::CocoNode { +impl Into for CocoPresence { + fn into(self) -> topology::coco::Node { + topology::coco::Node { host: self.host, pub_key: self.pub_key, last_seen: self.last_seen, @@ -27,8 +30,8 @@ impl Into for CocoPresence { } } -impl From for CocoPresence { - fn from(cn: CocoNode) -> Self { +impl From for CocoPresence { + fn from(cn: coco::Node) -> Self { CocoPresence { host: cn.host, pub_key: cn.pub_key, @@ -48,10 +51,10 @@ pub struct MixNodePresence { pub version: String, } -impl TryInto for MixNodePresence { +impl TryInto for MixNodePresence { type Error = io::Error; - fn try_into(self) -> Result { + fn try_into(self) -> Result { let resolved_hostname = self.host.to_socket_addrs()?.next(); if resolved_hostname.is_none() { return Err(io::Error::new( @@ -60,7 +63,7 @@ impl TryInto for MixNodePresence { )); } - Ok(topology::MixNode { + Ok(topology::mix::Node { host: resolved_hostname.unwrap(), pub_key: self.pub_key, layer: self.layer, @@ -70,8 +73,8 @@ impl TryInto for MixNodePresence { } } -impl From for MixNodePresence { - fn from(mn: MixNode) -> Self { +impl From for MixNodePresence { + fn from(mn: mix::Node) -> Self { MixNodePresence { host: mn.host.to_string(), pub_key: mn.pub_key, @@ -93,9 +96,9 @@ pub struct MixProviderPresence { pub version: String, } -impl Into for MixProviderPresence { - fn into(self) -> topology::MixProviderNode { - topology::MixProviderNode { +impl Into for MixProviderPresence { + fn into(self) -> provider::Node { + provider::Node { client_listener: self.client_listener.parse().unwrap(), mixnet_listener: self.mixnet_listener.parse().unwrap(), pub_key: self.pub_key, @@ -110,8 +113,8 @@ impl Into for MixProviderPresence { } } -impl From for MixProviderPresence { - fn from(mpn: MixProviderNode) -> Self { +impl From for MixProviderPresence { + fn from(mpn: provider::Node) -> Self { MixProviderPresence { client_listener: mpn.client_listener.to_string(), mixnet_listener: mpn.mixnet_listener.to_string(), @@ -133,16 +136,16 @@ pub struct MixProviderClient { pub pub_key: String, } -impl Into for MixProviderClient { - fn into(self) -> topology::MixProviderClient { - topology::MixProviderClient { +impl Into for MixProviderClient { + fn into(self) -> topology::provider::Client { + topology::provider::Client { pub_key: self.pub_key, } } } -impl From for MixProviderClient { - fn from(mpc: topology::MixProviderClient) -> Self { +impl From for MixProviderClient { + fn from(mpc: topology::provider::Client) -> Self { MixProviderClient { pub_key: mpc.pub_key, } @@ -174,9 +177,9 @@ impl NymTopology for Topology { } fn new_from_nodes( - mix_nodes: Vec, - mix_provider_nodes: Vec, - coco_nodes: Vec, + mix_nodes: Vec, + mix_provider_nodes: Vec, + coco_nodes: Vec, ) -> Self { Topology { coco_nodes: coco_nodes.into_iter().map(|node| node.into()).collect(), @@ -188,21 +191,21 @@ impl NymTopology for Topology { } } - fn get_mix_nodes(&self) -> Vec { + fn mix_nodes(&self) -> Vec { self.mix_nodes .iter() .filter_map(|x| x.clone().try_into().ok()) .collect() } - fn get_mix_provider_nodes(&self) -> Vec { + fn providers(&self) -> Vec { self.mix_provider_nodes .iter() .map(|x| x.clone().into()) .collect() } - fn get_coco_nodes(&self) -> Vec { + fn coco_nodes(&self) -> Vec { self.coco_nodes.iter().map(|x| x.clone().into()).collect() } } @@ -223,7 +226,7 @@ mod converting_mixnode_presence_into_topology_mixnode { version: "".to_string(), }; - let result: Result = mix_presence.try_into(); + let result: Result = mix_presence.try_into(); assert!(result.is_err()) } @@ -239,7 +242,7 @@ mod converting_mixnode_presence_into_topology_mixnode { version: "".to_string(), }; - let result: Result = mix_presence.try_into(); + let result: Result = mix_presence.try_into(); assert!(result.is_ok()) } } diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index a7ef6e35c8..37722aa238 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -18,5 +18,6 @@ addressing = {path = "../../addressing"} topology = {path = "../../topology"} ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { path = "../../../../sphinx"} diff --git a/common/clients/mix-client/src/packet.rs b/common/clients/mix-client/src/packet.rs index cc1a027cb6..7999d1e620 100644 --- a/common/clients/mix-client/src/packet.rs +++ b/common/clients/mix-client/src/packet.rs @@ -28,7 +28,7 @@ pub fn encapsulate_message( topology: &T, average_delay: f64, ) -> (SocketAddr, SphinxPacket) { - let mut providers = topology.get_mix_provider_nodes(); + let mut providers = topology.providers(); let provider = providers.pop().unwrap().into(); let route = topology.route_to(provider).unwrap(); diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index 918efa0df9..22f55f572a 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -25,6 +25,7 @@ sfw-provider-requests = { path = "../../sfw-provider/sfw-provider-requests" } topology = {path = "../topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { path = "../../../sphinx"} [dev-dependencies] diff --git a/common/healthcheck/src/path_check.rs b/common/healthcheck/src/path_check.rs index 62b94a86b1..e5d90eb70a 100644 --- a/common/healthcheck/src/path_check.rs +++ b/common/healthcheck/src/path_check.rs @@ -6,7 +6,7 @@ use provider_client::ProviderClient; use sphinx::header::delays::Delay; use sphinx::route::{Destination, Node as SphinxNode}; use std::collections::HashMap; -use topology::MixProviderNode; +use topology::provider; #[derive(Debug, PartialEq, Clone)] pub enum PathStatus { @@ -27,7 +27,7 @@ pub(crate) struct PathChecker { impl PathChecker { pub(crate) async fn new( - providers: Vec, + providers: Vec, ephemeral_keys: DummyMixIdentityKeyPair, ) -> Self { let mut provider_clients = HashMap::new(); diff --git a/common/healthcheck/src/result.rs b/common/healthcheck/src/result.rs index 79ec6a291f..b7aae8d33c 100644 --- a/common/healthcheck/src/result.rs +++ b/common/healthcheck/src/result.rs @@ -28,8 +28,8 @@ impl HealthCheckResult { fn zero_score(topology: T) -> Self { warn!("The network is unhealthy, could not send any packets - returning zero score!"); - let mixes = topology.get_mix_nodes(); - let providers = topology.get_mix_provider_nodes(); + let mixes = topology.mix_nodes(); + let providers = topology.providers(); let health = mixes .into_iter() @@ -58,7 +58,7 @@ impl HealthCheckResult { score_threshold: f64, ) -> T { let filtered_mix_nodes = topology - .get_mix_nodes() + .mix_nodes() .into_iter() .filter(|node| { match self.node_score(NodeAddressBytes::from_b64_string(node.pub_key.clone())) { @@ -72,7 +72,7 @@ impl HealthCheckResult { .collect(); let filtered_provider_nodes = topology - .get_mix_provider_nodes() + .providers() .into_iter() .filter(|node| { match self.node_score(NodeAddressBytes::from_b64_string(node.pub_key.clone())) { @@ -85,7 +85,7 @@ impl HealthCheckResult { }) .collect(); // coco nodes remain unchanged as no healthcheck is being run on them or time being - let filtered_coco_nodes = topology.get_coco_nodes(); + let filtered_coco_nodes = topology.coco_nodes(); T::new_from_nodes( filtered_mix_nodes, @@ -110,19 +110,16 @@ impl HealthCheckResult { // create entries for all nodes let mut score_map = HashMap::new(); - topology.get_mix_nodes().into_iter().for_each(|node| { + topology.mix_nodes().into_iter().for_each(|node| { score_map.insert(node.get_pub_key_bytes(), NodeScore::from_mixnode(node)); }); - topology - .get_mix_provider_nodes() - .into_iter() - .for_each(|node| { - score_map.insert(node.get_pub_key_bytes(), NodeScore::from_provider(node)); - }); + topology.providers().into_iter().for_each(|node| { + score_map.insert(node.get_pub_key_bytes(), NodeScore::from_provider(node)); + }); let ephemeral_keys = DummyMixIdentityKeyPair::new(); - let providers = topology.get_mix_provider_nodes(); + let providers = topology.providers(); let mut path_checker = PathChecker::new(providers, ephemeral_keys).await; for i in 0..iterations { diff --git a/common/healthcheck/src/score.rs b/common/healthcheck/src/score.rs index 8f97cf96a8..4d29e8f5b6 100644 --- a/common/healthcheck/src/score.rs +++ b/common/healthcheck/src/score.rs @@ -4,7 +4,8 @@ use std::cmp::Ordering; use std::fmt::Error; use std::fmt::Formatter; use std::net::SocketAddr; -use topology::{MixNode, MixProviderNode}; +use topology::mix; +use topology::provider; // TODO: should 'nodetype' really be part of healthcheck::score @@ -91,7 +92,7 @@ impl PartialEq for NodeScore { } impl NodeScore { - pub(crate) fn from_mixnode(node: MixNode) -> Self { + pub(crate) fn from_mixnode(node: mix::Node) -> Self { NodeScore { typ: NodeType::Mix, pub_key: NodeAddressBytes::from_b64_string(node.pub_key), @@ -103,7 +104,7 @@ impl NodeScore { } } - pub(crate) fn from_provider(node: MixProviderNode) -> Self { + pub(crate) fn from_provider(node: provider::Node) -> Self { NodeScore { typ: NodeType::MixProvider, pub_key: NodeAddressBytes::from_b64_string(node.pub_key), diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index e9e8022c91..bc6d39ef7a 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -20,4 +20,5 @@ addressing = {path = "../addressing"} version-checker = {path = "../version-checker" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { path = "../../../sphinx"} diff --git a/common/topology/src/coco.rs b/common/topology/src/coco.rs new file mode 100644 index 0000000000..320064d08e --- /dev/null +++ b/common/topology/src/coco.rs @@ -0,0 +1,13 @@ +#[derive(Debug, Clone)] +pub struct Node { + pub host: String, + pub pub_key: String, + pub last_seen: u64, + pub version: String, +} + +impl super::Versioned for Node { + fn get_version(&self) -> String { + self.version.clone() + } +} diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 8fadc5c2ab..486a4340bc 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -1,103 +1,29 @@ -use addressing; -use curve25519_dalek::montgomery::MontgomeryPoint; use itertools::Itertools; use rand::seq::IteratorRandom; -use sphinx::route::{Node as SphinxNode, NodeAddressBytes}; +use sphinx::route::Node as SphinxNode; use std::cmp::max; use std::collections::HashMap; -use std::net::SocketAddr; +use std::marker::PhantomData; use version_checker; -#[derive(Debug, Clone)] -pub struct MixNode { - pub host: SocketAddr, - pub pub_key: String, - pub layer: u64, - pub last_seen: u64, - pub version: String, -} - -impl Into for MixNode { - fn into(self) -> SphinxNode { - let address_bytes = addressing::encoded_bytes_from_socket_address(self.host); - let key_bytes = self.get_pub_key_bytes(); - let key = MontgomeryPoint(key_bytes); - - SphinxNode::new(NodeAddressBytes::from_bytes(address_bytes), key) - } -} - -impl MixNode { - pub fn get_pub_key_bytes(&self) -> [u8; 32] { - let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); - let mut key_bytes = [0; 32]; - key_bytes.copy_from_slice(&decoded_key_bytes[..]); - key_bytes - } -} - -#[derive(Debug, Clone)] -pub struct MixProviderClient { - pub pub_key: String, -} - -#[derive(Debug, Clone)] -pub struct MixProviderNode { - pub client_listener: SocketAddr, - pub mixnet_listener: SocketAddr, - pub pub_key: String, - pub registered_clients: Vec, - pub last_seen: u64, - pub version: String, -} - -impl Into for MixProviderNode { - fn into(self) -> SphinxNode { - let address_bytes = addressing::encoded_bytes_from_socket_address(self.mixnet_listener); - let key_bytes = self.get_pub_key_bytes(); - let key = MontgomeryPoint(key_bytes); - - SphinxNode::new(NodeAddressBytes::from_bytes(address_bytes), key) - } -} - -impl MixProviderNode { - pub fn get_pub_key_bytes(&self) -> [u8; 32] { - let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); - let mut key_bytes = [0; 32]; - key_bytes.copy_from_slice(&decoded_key_bytes[..]); - key_bytes - } -} - -#[derive(Debug, Clone)] -pub struct CocoNode { - pub host: String, - pub pub_key: String, - pub last_seen: u64, - pub version: String, -} - -#[derive(Debug)] -pub enum NymTopologyError { - InvalidMixLayerError, - MissingLayerError(Vec), -} +pub mod coco; +pub mod mix; +pub mod provider; pub trait NymTopology: Sized { fn new(directory_server: String) -> Self; fn new_from_nodes( - mix_nodes: Vec, - mix_provider_nodes: Vec, - coco_nodes: Vec, + mix_nodes: Vec, + mix_provider_nodes: Vec, + coco_nodes: Vec, ) -> Self; - fn get_mix_nodes(&self) -> Vec; - fn get_mix_provider_nodes(&self) -> Vec; - fn get_coco_nodes(&self) -> Vec; - fn make_layered_topology(&self) -> Result>, NymTopologyError> { - let mut layered_topology: HashMap> = HashMap::new(); + fn mix_nodes(&self) -> Vec; + fn providers(&self) -> Vec; + fn coco_nodes(&self) -> Vec; + fn make_layered_topology(&self) -> Result>, NymTopologyError> { + let mut layered_topology: HashMap> = HashMap::new(); let mut highest_layer = 0; - for mix in self.get_mix_nodes() { + for mix in self.mix_nodes() { // we need to have extra space for provider if mix.layer > sphinx::constants::MAX_PATH_LENGTH as u64 { return Err(NymTopologyError::InvalidMixLayerError); @@ -125,6 +51,8 @@ pub trait NymTopology: Sized { Ok(layered_topology) } + + // Tries to get a route through the mix network fn mix_route(&self) -> Result, NymTopologyError> { let mut layered_topology = self.make_layered_topology()?; let num_layers = layered_topology.len(); @@ -137,7 +65,7 @@ pub trait NymTopology: Sized { Ok(route) } - // sets a route to specific provider + // Sets up a route to a specific provider fn route_to(&self, provider_node: SphinxNode) -> Result, NymTopologyError> { Ok(self .mix_route()? @@ -148,7 +76,7 @@ pub trait NymTopology: Sized { fn all_paths(&self) -> Result>, NymTopologyError> { let mut layered_topology = self.make_layered_topology()?; - let providers = self.get_mix_provider_nodes(); + let providers = self.providers(); let sorted_layers: Vec> = (1..=layered_topology.len() as u64) .map(|layer| layered_topology.remove(&layer).unwrap()) // get all nodes per layer @@ -172,32 +100,23 @@ pub trait NymTopology: Sized { expected_provider_version: &str, expected_coco_version: &str, ) -> Self { - let filtered_mixes = self - .get_mix_nodes() - .iter() - .filter(|mix_node| { - version_checker::is_compatible(&mix_node.version, expected_mix_version) - }) - .cloned() - .collect(); - let filtered_providers = self - .get_mix_provider_nodes() - .iter() - .filter(|provider_node| { - version_checker::is_compatible(&provider_node.version, expected_provider_version) - }) - .cloned() - .collect(); - let filtered_coco_nodes = self - .get_coco_nodes() - .iter() - .filter(|coco_node| { - version_checker::is_compatible(&coco_node.version, expected_coco_version) - }) - .cloned() - .collect(); + let unversioned_mixnodes = self.mix_nodes(); + // let f = Filter::new(unversioned_mixnodes) + // unversioned_mixnodes - Self::new_from_nodes(filtered_mixes, filtered_providers, filtered_coco_nodes) + let mixes = self.filter::(self.mix_nodes(), expected_mix_version); + let providers = self.filter::(self.providers(), expected_provider_version); + let cocos = self.filter::(self.coco_nodes(), expected_coco_version); + + Self::new_from_nodes(mixes, providers, cocos) + } + + fn filter(&self, nodes: Vec, expected_version: &str) -> Vec { + nodes + .iter() + .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) + .cloned() + .collect() } fn can_construct_path_through(&self) -> bool { @@ -208,4 +127,46 @@ pub trait NymTopology: Sized { } } -// TODO: tests... +pub trait Versioned: Clone + Sized { + fn get_version(&self) -> String; +} + +pub struct Filter { + _phantom_data: PhantomData, +} + +impl Filter { + fn new() -> Self { + Self { + _phantom_data: PhantomData, + } + } + + fn run(&self, nodes: Vec, expected_version: &str) -> Vec { + nodes + .iter() + .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) + .cloned() + .collect() + } +} + +// impl VersionFilter { +// fn new() -> Self { +// Self {} +// } + +// fn run(&self, nodes: Vec, expected_version: &str) -> Vec { +// nodes +// .iter() +// .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) +// .cloned() +// .collect() +// } +// } + +#[derive(Debug)] +pub enum NymTopologyError { + InvalidMixLayerError, + MissingLayerError(Vec), +} diff --git a/common/topology/src/mix.rs b/common/topology/src/mix.rs new file mode 100644 index 0000000000..0501a9f314 --- /dev/null +++ b/common/topology/src/mix.rs @@ -0,0 +1,37 @@ +use sphinx::route::Node as SphinxNode; +use sphinx::route::NodeAddressBytes; +use std::net::SocketAddr; + +#[derive(Debug, Clone)] +pub struct Node { + pub host: SocketAddr, + pub pub_key: String, + pub layer: u64, + pub last_seen: u64, + pub version: String, +} + +impl Node { + pub fn get_pub_key_bytes(&self) -> [u8; 32] { + let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); + let mut key_bytes = [0; 32]; + key_bytes.copy_from_slice(&decoded_key_bytes[..]); + key_bytes + } +} + +impl super::Versioned for Node { + fn get_version(&self) -> String { + self.version.clone() + } +} + +impl Into for Node { + fn into(self) -> SphinxNode { + let address_bytes = addressing::encoded_bytes_from_socket_address(self.host); + let key_bytes = self.get_pub_key_bytes(); + let key = sphinx::key::new(key_bytes); + + SphinxNode::new(NodeAddressBytes::from_bytes(address_bytes), key) + } +} diff --git a/common/topology/src/provider.rs b/common/topology/src/provider.rs new file mode 100644 index 0000000000..f887a6989a --- /dev/null +++ b/common/topology/src/provider.rs @@ -0,0 +1,43 @@ +use sphinx::route::Node as SphinxNode; +use sphinx::route::NodeAddressBytes; +use std::net::SocketAddr; + +#[derive(Debug, Clone)] +pub struct Client { + pub pub_key: String, +} + +#[derive(Debug, Clone)] +pub struct Node { + pub client_listener: SocketAddr, + pub mixnet_listener: SocketAddr, + pub pub_key: String, + pub registered_clients: Vec, + pub last_seen: u64, + pub version: String, +} + +impl Node { + pub fn get_pub_key_bytes(&self) -> [u8; 32] { + let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); + let mut key_bytes = [0; 32]; + key_bytes.copy_from_slice(&decoded_key_bytes[..]); + key_bytes + } +} + +impl super::Versioned for Node { + fn get_version(&self) -> String { + self.version.clone() + } +} + +impl Into for Node { + fn into(self) -> SphinxNode { + let address_bytes = addressing::encoded_bytes_from_socket_address(self.mixnet_listener); + let key_bytes = self.get_pub_key_bytes(); + let key = sphinx::key::new(key_bytes); + + SphinxNode::new(NodeAddressBytes::from_bytes(address_bytes), key) + } +} diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 51730b0dd1..3133e21703 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -40,7 +40,8 @@ sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" } topology = {path = "../common/topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { path = "../../sphinx"} # putting this explicitly below everything and most likely, the next time we look into it, it will already have a proper release tokio-tungstenite = { git = "https://github.com/snapview/tokio-tungstenite", rev="308d9680c0e59dd1e8651659a775c05df937934e" } diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index 8359716f71..98e4840be0 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -154,7 +154,7 @@ impl NymClient { // this is temporary and assumes there exists only a single provider. let provider_client_listener_address: SocketAddr = initial_topology - .get_mix_provider_nodes() + .providers() .first() .expect("Could not get a provider from the supplied network topology, are you using the right directory server?") .client_listener; From af1cdccda2fa2f720b7f55d7ce6892bc35f88a93 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:09:36 +0000 Subject: [PATCH 055/105] Using the new topology filter --- common/topology/src/lib.rs | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 486a4340bc..f960f559e3 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -100,13 +100,9 @@ pub trait NymTopology: Sized { expected_provider_version: &str, expected_coco_version: &str, ) -> Self { - let unversioned_mixnodes = self.mix_nodes(); - // let f = Filter::new(unversioned_mixnodes) - // unversioned_mixnodes - - let mixes = self.filter::(self.mix_nodes(), expected_mix_version); - let providers = self.filter::(self.providers(), expected_provider_version); - let cocos = self.filter::(self.coco_nodes(), expected_coco_version); + let mixes = Filter::new(self.mix_nodes()).run(expected_mix_version); + let providers = Filter::new(self.providers()).run(expected_provider_version); + let cocos = Filter::new(self.coco_nodes()).run(expected_coco_version); Self::new_from_nodes(mixes, providers, cocos) } @@ -132,18 +128,16 @@ pub trait Versioned: Clone + Sized { } pub struct Filter { - _phantom_data: PhantomData, + nodes: Vec, } impl Filter { - fn new() -> Self { - Self { - _phantom_data: PhantomData, - } + fn new(nodes: Vec) -> Self { + Self { nodes } } - fn run(&self, nodes: Vec, expected_version: &str) -> Vec { - nodes + fn run(&self, expected_version: &str) -> Vec { + self.nodes .iter() .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) .cloned() @@ -151,20 +145,6 @@ impl Filter { } } -// impl VersionFilter { -// fn new() -> Self { -// Self {} -// } - -// fn run(&self, nodes: Vec, expected_version: &str) -> Vec { -// nodes -// .iter() -// .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) -// .cloned() -// .collect() -// } -// } - #[derive(Debug)] pub enum NymTopologyError { InvalidMixLayerError, From 4c44df46f87409dcc67203bd39e0369746f0d73b Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:19:09 +0000 Subject: [PATCH 056/105] topology: removing unused PhantomData import --- common/topology/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index f960f559e3..36dd8fa494 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -3,7 +3,6 @@ use rand::seq::IteratorRandom; use sphinx::route::Node as SphinxNode; use std::cmp::max; use std::collections::HashMap; -use std::marker::PhantomData; use version_checker; pub mod coco; From 6e03375ec79bd4eaeb0be0a2bc0060e7038eac12 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:47:41 +0000 Subject: [PATCH 057/105] topology: moved filtering into its own home --- common/topology/src/coco.rs | 4 +++- common/topology/src/filter.rs | 16 +++++++++++++ common/topology/src/lib.rs | 41 ++++++--------------------------- common/topology/src/mix.rs | 3 ++- common/topology/src/provider.rs | 3 ++- 5 files changed, 30 insertions(+), 37 deletions(-) create mode 100644 common/topology/src/filter.rs diff --git a/common/topology/src/coco.rs b/common/topology/src/coco.rs index 320064d08e..be3c20f367 100644 --- a/common/topology/src/coco.rs +++ b/common/topology/src/coco.rs @@ -1,3 +1,5 @@ +use crate::filter; + #[derive(Debug, Clone)] pub struct Node { pub host: String, @@ -6,7 +8,7 @@ pub struct Node { pub version: String, } -impl super::Versioned for Node { +impl filter::Versioned for Node { fn get_version(&self) -> String { self.version.clone() } diff --git a/common/topology/src/filter.rs b/common/topology/src/filter.rs new file mode 100644 index 0000000000..846db99134 --- /dev/null +++ b/common/topology/src/filter.rs @@ -0,0 +1,16 @@ +pub trait Versioned: Clone { + fn get_version(&self) -> String; +} + +pub trait VersionFiltererable { + fn filter_by_version(&self, expected_version: &str) -> Self; +} + +impl VersionFiltererable for Vec { + fn filter_by_version(&self, expected_version: &str) -> Self { + self.iter() + .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) + .cloned() + .collect() + } +} diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 36dd8fa494..d4416d1b8d 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -1,11 +1,12 @@ +use crate::filter::VersionFiltererable; use itertools::Itertools; use rand::seq::IteratorRandom; use sphinx::route::Node as SphinxNode; use std::cmp::max; use std::collections::HashMap; -use version_checker; pub mod coco; +mod filter; pub mod mix; pub mod provider; @@ -99,21 +100,15 @@ pub trait NymTopology: Sized { expected_provider_version: &str, expected_coco_version: &str, ) -> Self { - let mixes = Filter::new(self.mix_nodes()).run(expected_mix_version); - let providers = Filter::new(self.providers()).run(expected_provider_version); - let cocos = Filter::new(self.coco_nodes()).run(expected_coco_version); + let mixes = self.mix_nodes().filter_by_version(expected_mix_version); + let providers = self + .providers() + .filter_by_version(expected_provider_version); + let cocos = self.coco_nodes().filter_by_version(expected_coco_version); Self::new_from_nodes(mixes, providers, cocos) } - fn filter(&self, nodes: Vec, expected_version: &str) -> Vec { - nodes - .iter() - .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) - .cloned() - .collect() - } - fn can_construct_path_through(&self) -> bool { match self.make_layered_topology() { Ok(_) => true, @@ -122,28 +117,6 @@ pub trait NymTopology: Sized { } } -pub trait Versioned: Clone + Sized { - fn get_version(&self) -> String; -} - -pub struct Filter { - nodes: Vec, -} - -impl Filter { - fn new(nodes: Vec) -> Self { - Self { nodes } - } - - fn run(&self, expected_version: &str) -> Vec { - self.nodes - .iter() - .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) - .cloned() - .collect() - } -} - #[derive(Debug)] pub enum NymTopologyError { InvalidMixLayerError, diff --git a/common/topology/src/mix.rs b/common/topology/src/mix.rs index 0501a9f314..8ff318920c 100644 --- a/common/topology/src/mix.rs +++ b/common/topology/src/mix.rs @@ -1,3 +1,4 @@ +use crate::filter; use sphinx::route::Node as SphinxNode; use sphinx::route::NodeAddressBytes; use std::net::SocketAddr; @@ -20,7 +21,7 @@ impl Node { } } -impl super::Versioned for Node { +impl filter::Versioned for Node { fn get_version(&self) -> String { self.version.clone() } diff --git a/common/topology/src/provider.rs b/common/topology/src/provider.rs index f887a6989a..9bbcfa6d86 100644 --- a/common/topology/src/provider.rs +++ b/common/topology/src/provider.rs @@ -1,3 +1,4 @@ +use crate::filter; use sphinx::route::Node as SphinxNode; use sphinx::route::NodeAddressBytes; use std::net::SocketAddr; @@ -26,7 +27,7 @@ impl Node { } } -impl super::Versioned for Node { +impl filter::Versioned for Node { fn get_version(&self) -> String { self.version.clone() } From 5e22d3e9a9e5d8387227035cfe7fbfef251f7edc Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:48:03 +0000 Subject: [PATCH 058/105] topology: get_version() is now version() --- common/topology/src/coco.rs | 2 +- common/topology/src/filter.rs | 4 ++-- common/topology/src/mix.rs | 2 +- common/topology/src/provider.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/topology/src/coco.rs b/common/topology/src/coco.rs index be3c20f367..bfc20a093e 100644 --- a/common/topology/src/coco.rs +++ b/common/topology/src/coco.rs @@ -9,7 +9,7 @@ pub struct Node { } impl filter::Versioned for Node { - fn get_version(&self) -> String { + fn version(&self) -> String { self.version.clone() } } diff --git a/common/topology/src/filter.rs b/common/topology/src/filter.rs index 846db99134..3007dce1a2 100644 --- a/common/topology/src/filter.rs +++ b/common/topology/src/filter.rs @@ -1,5 +1,5 @@ pub trait Versioned: Clone { - fn get_version(&self) -> String; + fn version(&self) -> String; } pub trait VersionFiltererable { @@ -9,7 +9,7 @@ pub trait VersionFiltererable { impl VersionFiltererable for Vec { fn filter_by_version(&self, expected_version: &str) -> Self { self.iter() - .filter(|node| version_checker::is_compatible(&node.get_version(), expected_version)) + .filter(|node| version_checker::is_compatible(&node.version(), expected_version)) .cloned() .collect() } diff --git a/common/topology/src/mix.rs b/common/topology/src/mix.rs index 8ff318920c..5236e191fa 100644 --- a/common/topology/src/mix.rs +++ b/common/topology/src/mix.rs @@ -22,7 +22,7 @@ impl Node { } impl filter::Versioned for Node { - fn get_version(&self) -> String { + fn version(&self) -> String { self.version.clone() } } diff --git a/common/topology/src/provider.rs b/common/topology/src/provider.rs index 9bbcfa6d86..e9d4869e0e 100644 --- a/common/topology/src/provider.rs +++ b/common/topology/src/provider.rs @@ -28,7 +28,7 @@ impl Node { } impl filter::Versioned for Node { - fn get_version(&self) -> String { + fn version(&self) -> String { self.version.clone() } } From 1579aa58d5ea92c3905f00750d8526679bfc7563 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 11:53:27 +0000 Subject: [PATCH 059/105] version checker: renamed is_compatible() to is_minor_version_compatible() --- common/topology/src/filter.rs | 2 +- common/version-checker/src/lib.rs | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/common/topology/src/filter.rs b/common/topology/src/filter.rs index 3007dce1a2..95cbcaf1ab 100644 --- a/common/topology/src/filter.rs +++ b/common/topology/src/filter.rs @@ -9,7 +9,7 @@ pub trait VersionFiltererable { impl VersionFiltererable for Vec { fn filter_by_version(&self, expected_version: &str) -> Self { self.iter() - .filter(|node| version_checker::is_compatible(&node.version(), expected_version)) + .filter(|node| version_checker::is_minor_version_compatible(&node.version(), expected_version)) .cloned() .collect() } diff --git a/common/version-checker/src/lib.rs b/common/version-checker/src/lib.rs index 48125479af..7ab389e1ed 100644 --- a/common/version-checker/src/lib.rs +++ b/common/version-checker/src/lib.rs @@ -6,7 +6,7 @@ use semver::VersionReq; /// concrete version number, because that's what we'll have in our Cargo.toml files (e.g. 0.3.2). /// The patch number in the requirement gets dropped and replaced with a wildcard (0.3.*) as all /// minor versions should be compatible with each other. -pub fn is_compatible(version: &str, req: &str) -> bool { +pub fn is_minor_version_compatible(version: &str, req: &str) -> bool { let version = Version::parse(version).unwrap(); let tmp = Version::parse(req).unwrap(); let wildcard = format!("{}.{}.*", tmp.major, tmp.minor).to_string(); @@ -21,31 +21,37 @@ mod tests { #[test] fn version_0_3_0_is_compatible_with_requirement_0_3_x() { - assert!(is_compatible("0.3.0", "0.3.2")); + assert!(is_minor_version_compatible("0.3.0", "0.3.2")); } #[test] fn version_0_3_1_is_compatible_with_minimum_requirement_0_3_x() { - assert!(is_compatible("0.3.1", "0.3.2")); + assert!(is_minor_version_compatible("0.3.1", "0.3.2")); } #[test] fn version_0_3_2_is_compatible_with_minimum_requirement_0_3_x() { - assert!(is_compatible("0.3.2", "0.3.0")); + assert!(is_minor_version_compatible("0.3.2", "0.3.0")); } #[test] fn version_0_2_0_is_not_compatible_with_requirement_0_3_x() { - assert!(!is_compatible("0.2.0", "0.3.2")); + assert!(!is_minor_version_compatible("0.2.0", "0.3.2")); } #[test] fn version_0_4_0_is_not_compatible_with_requirement_0_3_x() { - assert!(!is_compatible("0.4.0", "0.3.2")); + assert!(!is_minor_version_compatible("0.4.0", "0.3.2")); } #[test] fn version_1_3_2_is_not_compatible_with_requirement_0_3_x() { - assert!(!is_compatible("1.3.2", "0.3.2")); + assert!(!is_minor_version_compatible("1.3.2", "0.3.2")); + } + + #[test] + #[should_panic] + fn panics_on_foo_version() { + assert!(!is_minor_version_compatible("foo", "bar")); } } From 772d4281589f5745ad17bc1c264dff31dcfe5c6e Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 12:02:01 +0000 Subject: [PATCH 060/105] version-checker: toughened up on version parsing panics --- common/version-checker/src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/common/version-checker/src/lib.rs b/common/version-checker/src/lib.rs index 7ab389e1ed..55232a9885 100644 --- a/common/version-checker/src/lib.rs +++ b/common/version-checker/src/lib.rs @@ -7,10 +7,16 @@ use semver::VersionReq; /// The patch number in the requirement gets dropped and replaced with a wildcard (0.3.*) as all /// minor versions should be compatible with each other. pub fn is_minor_version_compatible(version: &str, req: &str) -> bool { - let version = Version::parse(version).unwrap(); - let tmp = Version::parse(req).unwrap(); + let version = match Version::parse(version) { + Ok(v) => v, + Err(_) => return false, + }; + let tmp = match Version::parse(req) { + Ok(v) => v, + Err(_) => return false, + }; let wildcard = format!("{}.{}.*", tmp.major, tmp.minor).to_string(); - let semver_requirement = VersionReq::parse(&wildcard).unwrap(); + let semver_requirement = VersionReq::parse(&wildcard).expect("panicked on semver requirement parsing. This should never happen as inputs should already have been sanitized."); semver_requirement.matches(&version) } @@ -50,8 +56,12 @@ mod tests { } #[test] - #[should_panic] - fn panics_on_foo_version() { - assert!(!is_minor_version_compatible("foo", "bar")); + fn returns_false_on_foo_version() { + assert!(!is_minor_version_compatible("foo", "0.3.2")); + } + + #[test] + fn returns_false_on_bar_version() { + assert!(!is_minor_version_compatible("0.3.2", "bar")); } } From 94314e332d34f42849faa2eddefc087da2e0f6d8 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 12:36:57 +0000 Subject: [PATCH 061/105] sphinx: restoring commit version --- common/clients/mix-client/Cargo.toml | 4 ++-- common/healthcheck/Cargo.toml | 4 ++-- common/topology/Cargo.toml | 4 ++-- nym-client/Cargo.toml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index 37722aa238..ccfba9d9e3 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -18,6 +18,6 @@ addressing = {path = "../../addressing"} topology = {path = "../../topology"} ## will be moved to proper dependencies once released -# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } -sphinx = { path = "../../../../sphinx"} +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +# sphinx = { path = "../../../../sphinx"} diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index 22f55f572a..834a72c65e 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -25,7 +25,7 @@ sfw-provider-requests = { path = "../../sfw-provider/sfw-provider-requests" } topology = {path = "../topology" } ## will be moved to proper dependencies once released -# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } -sphinx = { path = "../../../sphinx"} +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +# sphinx = { path = "../../../sphinx"} [dev-dependencies] diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index bc6d39ef7a..6e0e65bf50 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -20,5 +20,5 @@ addressing = {path = "../addressing"} version-checker = {path = "../version-checker" } ## will be moved to proper dependencies once released -# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } -sphinx = { path = "../../../sphinx"} +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +# sphinx = { path = "../../../sphinx"} diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 3133e21703..24f30726df 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -40,8 +40,8 @@ sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" } topology = {path = "../common/topology" } ## will be moved to proper dependencies once released -# sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } -sphinx = { path = "../../sphinx"} +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +# sphinx = { path = "../../sphinx"} # putting this explicitly below everything and most likely, the next time we look into it, it will already have a proper release tokio-tungstenite = { git = "https://github.com/snapview/tokio-tungstenite", rev="308d9680c0e59dd1e8651659a775c05df937934e" } From 6e676fc1e41b95e896c9f446af988d44041805fd Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 12:38:55 +0000 Subject: [PATCH 062/105] ibid --- Cargo.lock | 38 +++++-------------- common/clients/provider-client/Cargo.toml | 2 +- mixnode/Cargo.toml | 2 +- sfw-provider/Cargo.toml | 2 +- sfw-provider/sfw-provider-requests/Cargo.toml | 2 +- 5 files changed, 13 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac86ab4e59..090c0ae3f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,7 +856,7 @@ dependencies = [ "serde", "serde_derive", "sfw-provider-requests", - "sphinx 0.1.0", + "sphinx", "tokio 0.2.9", "topology", ] @@ -1257,7 +1257,7 @@ dependencies = [ "pretty_env_logger", "rand 0.7.2", "rand_distr", - "sphinx 0.1.0", + "sphinx", "tokio 0.2.9", "topology", ] @@ -1364,7 +1364,7 @@ dependencies = [ "serde", "serde_json", "sfw-provider-requests", - "sphinx 0.1.0", + "sphinx", "tokio 0.2.9", "tokio-tungstenite", "topology", @@ -1385,7 +1385,7 @@ dependencies = [ "futures 0.3.1", "log", "pretty_env_logger", - "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", + "sphinx", "tokio 0.2.9", ] @@ -1410,7 +1410,7 @@ dependencies = [ "serde_json", "sfw-provider-requests", "sha2", - "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", + "sphinx", "tokio 0.2.9", ] @@ -1618,7 +1618,7 @@ dependencies = [ "log", "pretty_env_logger", "sfw-provider-requests", - "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", + "sphinx", "tokio 0.2.9", ] @@ -2025,7 +2025,7 @@ dependencies = [ name = "sfw-provider-requests" version = "0.1.0" dependencies = [ - "sphinx 0.1.0 (git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f)", + "sphinx", ] [[package]] @@ -2098,27 +2098,7 @@ dependencies = [ [[package]] name = "sphinx" version = "0.1.0" -source = "git+https://github.com/nymtech/sphinx?rev=1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f#1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" -dependencies = [ - "aes-ctr", - "arrayref", - "base64 0.11.0", - "blake2", - "byteorder", - "chacha", - "curve25519-dalek", - "hkdf", - "hmac", - "lioness", - "rand 0.7.2", - "rand_distr", - "rand_os", - "sha2", -] - -[[package]] -name = "sphinx" -version = "0.1.0" +source = "git+https://github.com/nymtech/sphinx?rev=47c4fbba8272c4aee03cc8165220f33572a96602#47c4fbba8272c4aee03cc8165220f33572a96602" dependencies = [ "aes-ctr", "arrayref", @@ -2448,7 +2428,7 @@ dependencies = [ "pretty_env_logger", "rand 0.7.2", "serde", - "sphinx 0.1.0", + "sphinx", "version-checker", ] diff --git a/common/clients/provider-client/Cargo.toml b/common/clients/provider-client/Cargo.toml index 450a76cb2b..07c368f9d6 100644 --- a/common/clients/provider-client/Cargo.toml +++ b/common/clients/provider-client/Cargo.toml @@ -16,4 +16,4 @@ tokio = { version = "0.2", features = ["full"] } sfw-provider-requests = { path = "../../../sfw-provider/sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index e538c329d9..d02f8bcc22 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -22,7 +22,7 @@ addressing = {path = "../common/addressing" } directory-client = { path = "../common/clients/directory-client" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } [build-dependencies] built = "0.3.2" diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index 4c0fc7c132..bce8917ad1 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -29,7 +29,7 @@ directory-client = { path = "../common/clients/directory-client" } sfw-provider-requests = { path = "./sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } [build-dependencies] built = "0.3.2" diff --git a/sfw-provider/sfw-provider-requests/Cargo.toml b/sfw-provider/sfw-provider-requests/Cargo.toml index 97763acb6c..0f31948e50 100644 --- a/sfw-provider/sfw-provider-requests/Cargo.toml +++ b/sfw-provider/sfw-provider-requests/Cargo.toml @@ -7,4 +7,4 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -sphinx = { git = "https://github.com/nymtech/sphinx", rev="1d8cefcb6a0cb8e87d00d89eb1ccf2839e92aa1f" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } From 9dbe74bdb1265eda531fdab69fdfca130080f515 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 12:48:46 +0000 Subject: [PATCH 063/105] topology: renamed filter trait --- common/topology/src/filter.rs | 8 +++++--- common/topology/src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/topology/src/filter.rs b/common/topology/src/filter.rs index 95cbcaf1ab..68242b8b92 100644 --- a/common/topology/src/filter.rs +++ b/common/topology/src/filter.rs @@ -2,14 +2,16 @@ pub trait Versioned: Clone { fn version(&self) -> String; } -pub trait VersionFiltererable { +pub trait VersionFilterable { fn filter_by_version(&self, expected_version: &str) -> Self; } -impl VersionFiltererable for Vec { +impl VersionFilterable for Vec { fn filter_by_version(&self, expected_version: &str) -> Self { self.iter() - .filter(|node| version_checker::is_minor_version_compatible(&node.version(), expected_version)) + .filter(|node| { + version_checker::is_minor_version_compatible(&node.version(), expected_version) + }) .cloned() .collect() } diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index bfb5568872..94ac5a74b4 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -1,4 +1,4 @@ -use crate::filter::VersionFiltererable; +use crate::filter::VersionFilterable; use itertools::Itertools; use rand::seq::IteratorRandom; use sphinx::route::Node as SphinxNode; From 5c1559dd78f9a8ccfbc82a5ca80f31fe8161deae Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 12:52:16 +0000 Subject: [PATCH 064/105] nym-client: using the new Sphinx max packet length constant --- nym-client/src/sockets/tcp.rs | 10 ++++++++++ nym-client/src/sockets/ws.rs | 12 ++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index df02c5074a..e37c901c9a 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -95,6 +95,16 @@ impl ClientRequest { mut input_tx: mpsc::UnboundedSender, ) -> ServerResponse { trace!("sending to: {:?}, msg: {:?}", recipient_address, msg); + if msg.len() > sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH { + return ServerResponse::Error { + message: format!( + "too long message. Sent {} bytes while the maximum is {}", + msg.len(), + sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH + ) + .to_string(), + }; + } let dummy_surb = [0; 16]; let input_msg = InputMessage(Destination::new(recipient_address, dummy_surb), msg); input_tx.send(input_msg).await.unwrap(); diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 0a2807038e..fa44b925a7 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -188,20 +188,12 @@ impl ClientRequest { mut input_tx: mpsc::UnboundedSender, ) -> ServerResponse { let message_bytes = msg.into_bytes(); - // TODO: wait until 0.4.0 release to replace those constants with newly exposed - // sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH - // we can't do it now for compatibility reasons as most recent sphinx revision - // has breaking changes due to packet format changes - let maximum_plaintext_length = sphinx::constants::PAYLOAD_SIZE - - sphinx::constants::SECURITY_PARAMETER - - sphinx::constants::DESTINATION_ADDRESS_LENGTH - - 1; - if message_bytes.len() > maximum_plaintext_length { + if message_bytes.len() > sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH { return ServerResponse::Error { message: format!( "too long message. Sent {} bytes while the maximum is {}", message_bytes.len(), - maximum_plaintext_length + sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH ) .to_string(), }; From f5bbfa89752e16877d10987b963da28ca3f321a3 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 13:17:15 +0000 Subject: [PATCH 065/105] cargo: updating all crates to use newest Sphinx --- Cargo.lock | 2 +- common/clients/mix-client/Cargo.toml | 2 +- common/clients/provider-client/Cargo.toml | 2 +- common/healthcheck/Cargo.toml | 2 +- common/topology/Cargo.toml | 2 +- mixnode/Cargo.toml | 2 +- nym-client/Cargo.toml | 2 +- sfw-provider/Cargo.toml | 2 +- sfw-provider/sfw-provider-requests/Cargo.toml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 090c0ae3f7..27f61ec5a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2098,7 +2098,7 @@ dependencies = [ [[package]] name = "sphinx" version = "0.1.0" -source = "git+https://github.com/nymtech/sphinx?rev=47c4fbba8272c4aee03cc8165220f33572a96602#47c4fbba8272c4aee03cc8165220f33572a96602" +source = "git+https://github.com/nymtech/sphinx?rev=c3ba1447d61d6eb900fcdea0c20b3070a22bf11a#c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" dependencies = [ "aes-ctr", "arrayref", diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index ccfba9d9e3..dcd16d33d5 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -18,6 +18,6 @@ addressing = {path = "../../addressing"} topology = {path = "../../topology"} ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } # sphinx = { path = "../../../../sphinx"} diff --git a/common/clients/provider-client/Cargo.toml b/common/clients/provider-client/Cargo.toml index 07c368f9d6..1c1d87b7b1 100644 --- a/common/clients/provider-client/Cargo.toml +++ b/common/clients/provider-client/Cargo.toml @@ -16,4 +16,4 @@ tokio = { version = "0.2", features = ["full"] } sfw-provider-requests = { path = "../../../sfw-provider/sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index 834a72c65e..aaacb88d66 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -25,7 +25,7 @@ sfw-provider-requests = { path = "../../sfw-provider/sfw-provider-requests" } topology = {path = "../topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } # sphinx = { path = "../../../sphinx"} [dev-dependencies] diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index 6e0e65bf50..905df9f2da 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -20,5 +20,5 @@ addressing = {path = "../addressing"} version-checker = {path = "../version-checker" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } # sphinx = { path = "../../../sphinx"} diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index d02f8bcc22..52db3fa9b6 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -22,7 +22,7 @@ addressing = {path = "../common/addressing" } directory-client = { path = "../common/clients/directory-client" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } [build-dependencies] built = "0.3.2" diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 24f30726df..eade92dd4b 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -40,7 +40,7 @@ sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" } topology = {path = "../common/topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } # sphinx = { path = "../../sphinx"} # putting this explicitly below everything and most likely, the next time we look into it, it will already have a proper release diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index bce8917ad1..7bea6173d0 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -29,7 +29,7 @@ directory-client = { path = "../common/clients/directory-client" } sfw-provider-requests = { path = "./sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } [build-dependencies] built = "0.3.2" diff --git a/sfw-provider/sfw-provider-requests/Cargo.toml b/sfw-provider/sfw-provider-requests/Cargo.toml index 0f31948e50..9b207791e3 100644 --- a/sfw-provider/sfw-provider-requests/Cargo.toml +++ b/sfw-provider/sfw-provider-requests/Cargo.toml @@ -7,4 +7,4 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -sphinx = { git = "https://github.com/nymtech/sphinx", rev="47c4fbba8272c4aee03cc8165220f33572a96602" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } From a187399c7de827bbbc86268c745090ffe7d6a6d5 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 13:17:37 +0000 Subject: [PATCH 066/105] sphinx: gracefully handling packet unwrap failures --- mixnode/src/node/mod.rs | 9 +++++++-- sfw-provider/src/provider/mix_handling/mod.rs | 16 +++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index 1ae4b6e65d..78f49f33f1 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -39,6 +39,7 @@ impl Config { pub enum MixProcessingError { SphinxRecoveryError, ReceivedFinalHopError, + SphinxProcessingError, } impl From for MixProcessingError { @@ -115,10 +116,14 @@ impl PacketProcessor { let packet = SphinxPacket::from_bytes(packet_data.to_vec())?; let (next_packet, next_hop_address, delay) = match packet.process(processing_data.secret_key) { - ProcessedPacket::ProcessedPacketForwardHop(packet, address, delay) => { + Ok(ProcessedPacket::ProcessedPacketForwardHop(packet, address, delay)) => { (packet, address, delay) } - _ => return Err(MixProcessingError::ReceivedFinalHopError), + Ok(_) => return Err(MixProcessingError::ReceivedFinalHopError), + Err(e) => { + warn!("Failed to unwrap Sphinx pakcet: {:?}", e); + return Err(MixProcessingError::SphinxProcessingError); + } }; let next_mix = MixPeer::new(next_hop_address); diff --git a/sfw-provider/src/provider/mix_handling/mod.rs b/sfw-provider/src/provider/mix_handling/mod.rs index 1670311dd1..c7a4e8cc02 100644 --- a/sfw-provider/src/provider/mix_handling/mod.rs +++ b/sfw-provider/src/provider/mix_handling/mod.rs @@ -1,5 +1,6 @@ use crate::provider::storage::StoreData; use crypto::identity::DummyMixIdentityPrivateKey; +use log::warn; use sphinx::{ProcessedPacket, SphinxPacket}; use std::path::PathBuf; use std::sync::{Arc, RwLock}; @@ -8,11 +9,12 @@ use std::sync::{Arc, RwLock}; // DUPLICATE WITH MIXNODE CODE!!! #[derive(Debug)] pub enum MixProcessingError { - SphinxRecoveryError, - ReceivedForwardHopError, + FileIOFailure, InvalidPayload, NonMatchingRecipient, - FileIOFailure, + ReceivedForwardHopError, + SphinxRecoveryError, + SphinxProcessingError, } impl From for MixProcessingError { @@ -63,10 +65,14 @@ impl MixPacketProcessor { let read_processing_data = processing_data.read().unwrap(); let (client_address, client_surb_id, payload) = match packet.process(read_processing_data.secret_key.as_scalar()) { - ProcessedPacket::ProcessedPacketFinalHop(client_address, surb_id, payload) => { + Ok(ProcessedPacket::ProcessedPacketFinalHop(client_address, surb_id, payload)) => { (client_address, surb_id, payload) } - _ => return Err(MixProcessingError::ReceivedForwardHopError), + Ok(_) => return Err(MixProcessingError::ReceivedForwardHopError), + Err(e) => { + warn!("Error unwrapping Sphinx packet: {:?}", e); + return Err(MixProcessingError::SphinxProcessingError); + } }; // TODO: should provider try to be recovering plaintext? this would potentially make client retrieve messages of non-constant length, From 44e146e2a24657b3a3c6609b9b88f0a8c66dcf39 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:03:43 +0000 Subject: [PATCH 067/105] Removed calls to unwrap at mixnode --- mixnode/src/node/mod.rs | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index 78f49f33f1..df389d4bc8 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -109,9 +109,14 @@ impl PacketProcessor { ) -> Result { // we received something resembling a sphinx packet, report it! let processing_data = processing_data.lock().await; - let mut received_sender = processing_data.received_metrics_tx.clone(); + let mut received_metrics_tx = processing_data.received_metrics_tx.clone(); - received_sender.send(()).await.unwrap(); + // if unwrap failed it means our metrics reporter died, so we should exit application and + // force restart + if received_metrics_tx.send(()).await.is_err() { + error!("failed to send metrics data to the controller - the underlying thread probably died!"); + std::process::exit(1); + } let packet = SphinxPacket::from_bytes(packet_data.to_vec())?; let (next_packet, next_hop_address, delay) = @@ -140,11 +145,16 @@ impl PacketProcessor { async fn wait_and_forward(mut forwarding_data: ForwardingData) { let delay_duration = Duration::from_nanos(forwarding_data.delay.get_value()); tokio::time::delay_for(delay_duration).await; - forwarding_data + + if forwarding_data .sent_metrics_tx .send(forwarding_data.recipient.to_string()) .await - .unwrap(); + .is_err() + { + error!("failed to send metrics data to the controller - the underlying thread probably died!"); + std::process::exit(1); + } trace!("RECIPIENT: {:?}", forwarding_data.recipient); match forwarding_data @@ -188,7 +198,6 @@ impl MixNode { mut socket: tokio::net::TcpStream, processing_data: Arc>, ) { - // NOTE: processing_data is copied here!! let mut buf = [0u8; sphinx::PACKET_SIZE]; // In a loop, read data from the socket and write the data back. @@ -200,12 +209,18 @@ impl MixNode { return; } Ok(_) => { - let fwd_data = PacketProcessor::process_sphinx_data_packet( + let fwd_data = match PacketProcessor::process_sphinx_data_packet( buf.as_ref(), processing_data.clone(), ) .await - .unwrap(); + { + Ok(fwd_data) => fwd_data, + Err(e) => { + warn!("failed to process sphinx packet: {:?}", e); + return; + } + }; PacketProcessor::wait_and_forward(fwd_data).await; } Err(e) => { From ce56b525bbd6e7d26e14d957c681035a9579db4b Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:12:07 +0000 Subject: [PATCH 068/105] socket_address_from_encoded_bytes returning Result --- common/addressing/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/addressing/src/lib.rs b/common/addressing/src/lib.rs index a2660471f4..529d93924d 100644 --- a/common/addressing/src/lib.rs +++ b/common/addressing/src/lib.rs @@ -62,8 +62,8 @@ pub fn encoded_bytes_from_socket_address(address: SocketAddr) -> [u8; 32] { address_bytes } -pub fn socket_address_from_encoded_bytes(b: [u8; 32]) -> SocketAddr { - let address_type: AddressType = b[0].try_into().unwrap(); +pub fn socket_address_from_encoded_bytes(b: [u8; 32]) -> Result { + let address_type: AddressType = b[0].try_into()?; let port: u16 = u16::from_be_bytes([b[1], b[2]]); @@ -76,5 +76,5 @@ pub fn socket_address_from_encoded_bytes(b: [u8; 32]) -> SocketAddr { } }; - SocketAddr::new(ip, port) + Ok(SocketAddr::new(ip, port)) } From 6f83785b3a56e03911041a7221dbf26da0e6cf4b Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:12:28 +0000 Subject: [PATCH 069/105] Creating mix peer checks for correct address --- mixnode/src/mix_peer.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/mixnode/src/mix_peer.rs b/mixnode/src/mix_peer.rs index 1ddb0a3481..8a8579a780 100644 --- a/mixnode/src/mix_peer.rs +++ b/mixnode/src/mix_peer.rs @@ -1,4 +1,5 @@ use addressing; +use addressing::AddressTypeError; use sphinx::route::NodeAddressBytes; use std::error::Error; use std::net::SocketAddr; @@ -9,14 +10,27 @@ pub struct MixPeer { connection: SocketAddr, } +#[derive(Debug)] +pub enum MixPeerError { + InvalidAddressError, +} + +impl From for MixPeerError { + fn from(_: AddressTypeError) -> Self { + use MixPeerError::*; + + InvalidAddressError + } +} + impl MixPeer { // note that very soon `next_hop_address` will be changed to `next_hop_metadata` - pub fn new(next_hop_address: NodeAddressBytes) -> MixPeer { + pub fn new(next_hop_address: NodeAddressBytes) -> Result { let next_hop_socket_address = - addressing::socket_address_from_encoded_bytes(next_hop_address.to_bytes()); - MixPeer { + addressing::socket_address_from_encoded_bytes(next_hop_address.to_bytes())?; + Ok(MixPeer { connection: next_hop_socket_address, - } + }) } pub async fn send(&self, bytes: Vec) -> Result<(), Box> { From 9913f6f041c7fd51addba94a164c7ad29a1e6329 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:12:41 +0000 Subject: [PATCH 070/105] Additional mixnode error checks --- mixnode/src/node/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index df389d4bc8..812f4e68a6 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -40,6 +40,7 @@ pub enum MixProcessingError { SphinxRecoveryError, ReceivedFinalHopError, SphinxProcessingError, + InvalidHopAddress, } impl From for MixProcessingError { @@ -131,7 +132,10 @@ impl PacketProcessor { } }; - let next_mix = MixPeer::new(next_hop_address); + let next_mix = match MixPeer::new(next_hop_address) { + Ok(next_mix) => next_mix, + Err(_) => return Err(MixProcessingError::InvalidHopAddress), + }; let fwd_data = ForwardingData::new( next_packet, From 3db34e77179c41f177fa82e84de0491914a23ca4 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:14:49 +0000 Subject: [PATCH 071/105] typo --- mixnode/src/node/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index 812f4e68a6..4ac744ce76 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -127,7 +127,7 @@ impl PacketProcessor { } Ok(_) => return Err(MixProcessingError::ReceivedFinalHopError), Err(e) => { - warn!("Failed to unwrap Sphinx pakcet: {:?}", e); + warn!("Failed to unwrap Sphinx packet: {:?}", e); return Err(MixProcessingError::SphinxProcessingError); } }; From 0093951507e8ebf548a39f97ab4ab9d0e660e51c Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:37:36 +0000 Subject: [PATCH 072/105] Improved usage of unwraps and expects for provider --- .../sfw-provider-requests/src/responses.rs | 4 +- .../src/provider/client_handling/mod.rs | 5 ++- sfw-provider/src/provider/mix_handling/mod.rs | 10 ++++- sfw-provider/src/provider/mod.rs | 9 ++++- sfw-provider/src/provider/storage/mod.rs | 38 +++++++++++++------ 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/sfw-provider/sfw-provider-requests/src/responses.rs b/sfw-provider/sfw-provider-requests/src/responses.rs index 870f527334..c15017c76d 100644 --- a/sfw-provider/sfw-provider-requests/src/responses.rs +++ b/sfw-provider/sfw-provider-requests/src/responses.rs @@ -45,8 +45,8 @@ fn read_be_u16(input: &mut &[u8]) -> u16 { u16::from_be_bytes(int_bytes.try_into().unwrap()) } -// TODO: currently this allows for maximum 64kB payload - if we go over that in sphinx, -// we need to update this code. +// TODO: currently this allows for maximum 64kB payload - +// if we go over that in sphinx we need to update this code. impl ProviderResponse for PullResponse { // num_msgs || len1 || len2 || ... || msg1 || msg2 || ... fn to_bytes(&self) -> Vec { diff --git a/sfw-provider/src/provider/client_handling/mod.rs b/sfw-provider/src/provider/client_handling/mod.rs index b79c1962f7..034e946152 100644 --- a/sfw-provider/src/provider/client_handling/mod.rs +++ b/sfw-provider/src/provider/client_handling/mod.rs @@ -154,8 +154,9 @@ impl ClientRequestProcessor { } fn generate_new_auth_token(data: Vec, key: DummyMixIdentityPrivateKey) -> AuthToken { - let mut auth_token_raw = - HmacSha256::new_varkey(&key.to_bytes()).expect("HMAC can take key of any size"); + // also note that `new_varkey` doesn't even have an execution branch returning an error + let mut auth_token_raw = HmacSha256::new_varkey(&key.to_bytes()) + .expect("HMAC should be able take key of any size"); auth_token_raw.input(&data); let mut auth_token = [0u8; 32]; auth_token.copy_from_slice(&auth_token_raw.result().code().to_vec()); diff --git a/sfw-provider/src/provider/mix_handling/mod.rs b/sfw-provider/src/provider/mix_handling/mod.rs index c7a4e8cc02..338943efd1 100644 --- a/sfw-provider/src/provider/mix_handling/mod.rs +++ b/sfw-provider/src/provider/mix_handling/mod.rs @@ -1,6 +1,6 @@ use crate::provider::storage::StoreData; use crypto::identity::DummyMixIdentityPrivateKey; -use log::warn; +use log::{error, warn}; use sphinx::{ProcessedPacket, SphinxPacket}; use std::path::PathBuf; use std::sync::{Arc, RwLock}; @@ -62,7 +62,13 @@ impl MixPacketProcessor { processing_data: &RwLock, ) -> Result { let packet = SphinxPacket::from_bytes(packet_data.to_vec())?; - let read_processing_data = processing_data.read().unwrap(); + let read_processing_data = match processing_data.read() { + Ok(guard) => guard, + Err(e) => { + error!("processing data lock was poisoned! - {:?}", e); + std::process::exit(1) + } + }; let (client_address, client_surb_id, payload) = match packet.process(read_processing_data.secret_key.as_scalar()) { Ok(ProcessedPacket::ProcessedPacketFinalHop(client_address, surb_id, payload)) => { diff --git a/sfw-provider/src/provider/mod.rs b/sfw-provider/src/provider/mod.rs index bc32073da8..9e1f766f48 100644 --- a/sfw-provider/src/provider/mod.rs +++ b/sfw-provider/src/provider/mod.rs @@ -147,9 +147,16 @@ impl ServiceProvider { return; } }; + let processing_data_lock = match processing_data.read() { + Ok(guard) => guard, + Err(e) => { + error!("processing data lock was poisoned! - {:?}", e); + std::process::exit(1) + } + }; ClientStorage::store_processed_data( store_data, - processing_data.read().unwrap().store_dir.as_path(), + processing_data_lock.store_dir.as_path(), ) .unwrap_or_else(|e| { error!("failed to store processed sphinx message; err = {:?}", e); diff --git a/sfw-provider/src/provider/storage/mod.rs b/sfw-provider/src/provider/storage/mod.rs index fb7607a506..d19d427b55 100644 --- a/sfw-provider/src/provider/storage/mod.rs +++ b/sfw-provider/src/provider/storage/mod.rs @@ -90,18 +90,10 @@ impl ClientStorage { } let msgs: Vec<_> = std::fs::read_dir(full_store_dir)? - .map(|entry| entry.unwrap()) - .filter(|entry| { - let is_file = entry.metadata().unwrap().is_file(); - if !is_file { - error!( - "potentially corrupted client inbox! - found a non-file - {:?}", - entry.path() - ); - } - is_file - }) + .filter_map(|entry| entry.ok()) + .filter(|entry| ClientStorage::is_valid_file(entry)) .map(|entry| { + // Not yet sure how to exactly get rid of those unwraps let content = std::fs::read(entry.path()).unwrap(); ClientStorage::delete_file(entry.path()).unwrap(); content @@ -113,6 +105,30 @@ impl ClientStorage { Ok(msgs) } + fn is_valid_file(entry: &std::fs::DirEntry) -> bool { + let metadata = match entry.metadata() { + Ok(meta) => meta, + Err(e) => { + error!( + "potentially corrupted client inbox! ({:?} - failed to read its metadata - {:?}", + entry.path(), + e, + ); + return false; + } + }; + + let is_file = metadata.is_file(); + if !is_file { + error!( + "potentially corrupted client inbox! - found a non-file - {:?}", + entry.path() + ); + } + + is_file + } + // TODO: THIS NEEDS A LOCKING MECHANISM!!! (or a db layer on top - basically 'ClientStorage' on steroids) // TODO 2: This should only be called AFTER we sent the reply. Because if client's connection failed after sending request // the messages would be deleted but he wouldn't have received them From d41023d26f55b3b0073fae7847768d67dc4e8294 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 14:44:40 +0000 Subject: [PATCH 073/105] Better unwrap usage for nym-client --- nym-client/src/client/cover_traffic_stream.rs | 14 +++++++------- nym-client/src/client/real_traffic_stream.rs | 17 +++++++++-------- nym-client/src/sockets/tcp.rs | 17 +++++++++-------- nym-client/src/sockets/ws.rs | 17 +++++++++-------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/nym-client/src/client/cover_traffic_stream.rs b/nym-client/src/client/cover_traffic_stream.rs index f1caedc740..e03fc0abeb 100644 --- a/nym-client/src/client/cover_traffic_stream.rs +++ b/nym-client/src/client/cover_traffic_stream.rs @@ -20,13 +20,13 @@ pub(crate) async fn start_loop_cover_traffic_stream( tokio::time::delay_for(delay_duration).await; let read_lock = topology_ctrl_ref.read().await; - let topology = read_lock.topology.as_ref(); - if topology.is_none() { - warn!("No valid topology detected - won't send any loop cover message this time"); - continue; - } - - let topology = topology.unwrap(); + let topology = match read_lock.topology.as_ref() { + None => { + warn!("No valid topology detected - won't send any loop cover message this time"); + continue; + } + Some(topology) => topology, + }; let cover_message = mix_client::packet::loop_cover_message(our_info.address, our_info.identifier, topology); diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index 9d1ee055ab..68fb7f0098 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -85,14 +85,15 @@ impl OutQueueControl { while let Some(next_message) = self.next().await { trace!("created new message"); let read_lock = self.topology_ctrl_ref.read().await; - let topology = read_lock.topology.as_ref(); - - if topology.is_none() { - warn!("No valid topology detected - won't send any loop cover or real message this time"); - continue; - } - - let topology = topology.unwrap(); + let topology = match read_lock.topology.as_ref() { + None => { + warn!( + "No valid topology detected - won't send any loop cover message this time" + ); + continue; + } + Some(topology) => topology, + }; let next_packet = match next_message { StreamMessage::Cover => mix_client::packet::loop_cover_message( diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index 8f49707c60..69440809d3 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -120,15 +120,16 @@ impl ClientRequest { return ServerResponse::Error { message: e }; } - let messages = res_rx.map(|msg| msg).await; + let messages = match res_rx.map(|msg| msg).await { + Ok(messages) => messages, + Err(e) => { + warn!("Failed to fetch client messages - {:?}", e); + return ServerResponse::Error { + message: format!("Server failed to receive messages").to_string(), + }; + } + }; - if messages.is_err() { - return ServerResponse::Error { - message: "Server failed to receive messages".to_string(), - }; - } - - let messages = messages.unwrap(); trace!("fetched {} messages", messages.len()); ServerResponse::Fetch { messages } } diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 598a146b7c..c50cbb6037 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -234,16 +234,17 @@ impl ClientRequest { }; } - let messages = res_rx.map(|msg| msg).await; - if messages.is_err() { - warn!("Failed to handle_fetch. messages is an error"); - return ServerResponse::Error { - message: "Server failed to receive messages".to_string(), - }; - } + let messages = match res_rx.map(|msg| msg).await { + Ok(messages) => messages, + Err(e) => { + warn!("Failed to fetch client messages - {:?}", e); + return ServerResponse::Error { + message: format!("Server failed to receive messages").to_string(), + }; + } + }; let messages = messages - .unwrap() .into_iter() .map(|message| { match std::str::from_utf8(&message) { From 6fb991a1b028fe51a79af2f55e6ff9f0a39e1137 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 14:54:27 +0000 Subject: [PATCH 074/105] mixnode: typo fix in comments --- mixnode/src/node/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index 78f49f33f1..c8ce9ae783 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -121,7 +121,7 @@ impl PacketProcessor { } Ok(_) => return Err(MixProcessingError::ReceivedFinalHopError), Err(e) => { - warn!("Failed to unwrap Sphinx pakcet: {:?}", e); + warn!("Failed to unwrap Sphinx packet: {:?}", e); return Err(MixProcessingError::SphinxProcessingError); } }; From 28d8fbedc9be75bf35d09d983536a7ed4b8db911 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 14:57:39 +0000 Subject: [PATCH 075/105] nym-client: revamped error message in websocket --- nym-client/src/sockets/ws.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 598a146b7c..28d8b0e196 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -191,7 +191,7 @@ impl ClientRequest { if message_bytes.len() > sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH { return ServerResponse::Error { message: format!( - "too long message. Sent {} bytes while the maximum is {}", + "message too long. Sent {} bytes, but the maximum is {}", message_bytes.len(), sphinx::constants::MAXIMUM_PLAINTEXT_LENGTH ) From d744fa587f5f72dcf5a5ca80f6aef57596e379f8 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 15:11:03 +0000 Subject: [PATCH 076/105] topology: removing curve25519_dalek crate, it's no longer needed --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ba55ec418a..2a639ed825 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2428,7 +2428,6 @@ version = "0.1.0" dependencies = [ "addressing", "base64 0.11.0", - "curve25519-dalek", "itertools", "log", "pretty_env_logger", From a4ead85a0145a4f4c256a0e2575e89f920e4be32 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 15:11:16 +0000 Subject: [PATCH 077/105] topology: added bs58 crate --- Cargo.lock | 1 + common/topology/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 2a639ed825..c7f39eb1ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2428,6 +2428,7 @@ version = "0.1.0" dependencies = [ "addressing", "base64 0.11.0", + "bs58", "itertools", "log", "pretty_env_logger", diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index 905df9f2da..dcebcd6796 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] base64 = "0.11.0" -curve25519-dalek = "1.2.3" +bs58 = "0.3.0" itertools = "0.8.2" log = "0.4" pretty_env_logger = "0.3" From 1b787c94bfe45190008ebc7fa9ab59fada8bbe53 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Fri, 24 Jan 2020 15:31:40 +0000 Subject: [PATCH 078/105] Removed unwrap/expect calls wherever feasible and introduced Result types where appropriate --- .../clients/directory-client/src/presence.rs | 35 ++++++------ common/clients/mix-client/src/packet.rs | 51 +++++++++++++++--- common/clients/mix-client/src/poisson.rs | 3 ++ common/clients/provider-client/src/lib.rs | 13 +++-- common/crypto/src/encryption/x25519.rs | 1 + common/healthcheck/src/path_check.rs | 34 +++++++++--- common/healthcheck/src/result.rs | 6 +-- common/pemstore/src/pemstore.rs | 53 +++++++++++-------- common/topology/src/lib.rs | 2 + nym-client/src/client/cover_traffic_stream.rs | 18 +++++-- nym-client/src/client/real_traffic_stream.rs | 13 ++++- nym-client/src/commands/init.rs | 2 +- nym-client/src/commands/tcpsocket.rs | 4 +- nym-client/src/commands/websocket.rs | 4 +- 14 files changed, 174 insertions(+), 65 deletions(-) diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index f56305f172..7d1b30c24b 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -9,6 +9,7 @@ use std::net::ToSocketAddrs; use topology::coco; use topology::mix; use topology::provider; +use topology::provider::Node; use topology::NymTopology; // special version of 'PartialEq' that does not care about last_seen field (where applicable) @@ -62,16 +63,18 @@ impl TryInto for MixNodePresence { type Error = io::Error; fn try_into(self) -> Result { - let resolved_hostname = self.host.to_socket_addrs()?.next(); - if resolved_hostname.is_none() { - return Err(io::Error::new( - io::ErrorKind::Other, - "no valid socket address", - )); - } + let resolved_hostname = match self.host.to_socket_addrs()?.next() { + None => { + return Err(io::Error::new( + io::ErrorKind::Other, + "no valid socket address", + )); + } + Some(host) => host, + }; Ok(topology::mix::Node { - host: resolved_hostname.unwrap(), + host: resolved_hostname, pub_key: self.pub_key, layer: self.layer, last_seen: self.last_seen, @@ -129,11 +132,13 @@ impl PresenceEq for MixProviderPresence { } } -impl Into for MixProviderPresence { - fn into(self) -> topology::provider::Node { - topology::provider::Node { - client_listener: self.client_listener.parse().unwrap(), - mixnet_listener: self.mixnet_listener.parse().unwrap(), +impl TryInto for MixProviderPresence { + type Error = std::net::AddrParseError; + + fn try_into(self) -> Result { + Ok(topology::provider::Node { + client_listener: self.client_listener.parse()?, + mixnet_listener: self.mixnet_listener.parse()?, pub_key: self.pub_key, registered_clients: self .registered_clients @@ -142,7 +147,7 @@ impl Into for MixProviderPresence { .collect(), last_seen: self.last_seen, version: self.version, - } + }) } } @@ -345,7 +350,7 @@ impl NymTopology for Topology { fn providers(&self) -> Vec { self.mix_provider_nodes .iter() - .map(|x| x.clone().into()) + .filter_map(|x| x.clone().try_into().ok()) .collect() } diff --git a/common/clients/mix-client/src/packet.rs b/common/clients/mix-client/src/packet.rs index 7999d1e620..f1ee832651 100644 --- a/common/clients/mix-client/src/packet.rs +++ b/common/clients/mix-client/src/packet.rs @@ -1,17 +1,49 @@ use addressing; +use addressing::AddressTypeError; use sphinx::route::{Destination, DestinationAddressBytes, SURBIdentifier}; use sphinx::SphinxPacket; use std::net::SocketAddr; -use topology::NymTopology; +use topology::{NymTopology, NymTopologyError}; pub const LOOP_COVER_MESSAGE_PAYLOAD: &[u8] = b"The cake is a lie!"; pub const LOOP_COVER_MESSAGE_AVERAGE_DELAY: f64 = 2.0; +#[derive(Debug)] +pub enum SphinxPacketEncapsulationError { + NoValidProvidersError, + InvalidTopologyError, + SphinxEncapsulationError(sphinx::header::SphinxUnwrapError), + InvalidFirstMixAddress, +} + +impl From for SphinxPacketEncapsulationError { + fn from(_: NymTopologyError) -> Self { + use SphinxPacketEncapsulationError::*; + InvalidTopologyError + } +} + +// it is correct error we're converting from, it just has an unfortunate name +// related issue: https://github.com/nymtech/sphinx/issues/40 +impl From for SphinxPacketEncapsulationError { + fn from(err: sphinx::header::SphinxUnwrapError) -> Self { + use SphinxPacketEncapsulationError::*; + SphinxEncapsulationError(err) + } +} + +impl From for SphinxPacketEncapsulationError { + fn from(_: AddressTypeError) -> Self { + use SphinxPacketEncapsulationError::*; + InvalidFirstMixAddress + } +} + pub fn loop_cover_message( our_address: DestinationAddressBytes, surb_id: SURBIdentifier, topology: &T, -) -> (SocketAddr, SphinxPacket) { +) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> { let destination = Destination::new(our_address, surb_id); encapsulate_message( @@ -27,19 +59,24 @@ pub fn encapsulate_message( message: Vec, topology: &T, average_delay: f64, -) -> (SocketAddr, SphinxPacket) { +) -> Result<(SocketAddr, SphinxPacket), SphinxPacketEncapsulationError> { let mut providers = topology.providers(); + if providers.len() == 0 { + return Err(SphinxPacketEncapsulationError::NoValidProvidersError); + } + // unwrap is fine here as we asserted there is at least single provider let provider = providers.pop().unwrap().into(); - let route = topology.route_to(provider).unwrap(); + let route = topology.route_to(provider)?; let delays = sphinx::header::delays::generate(route.len(), average_delay); // build the packet - let packet = sphinx::SphinxPacket::new(message, &route[..], &recipient, &delays).unwrap(); + let packet = sphinx::SphinxPacket::new(message, &route[..], &recipient, &delays)?; + // we know the mix route must be valid otherwise we would have already returned an error let first_node_address = - addressing::socket_address_from_encoded_bytes(route.first().unwrap().address.to_bytes()); + addressing::socket_address_from_encoded_bytes(route.first().unwrap().address.to_bytes())?; - (first_node_address, packet) + Ok((first_node_address, packet)) } diff --git a/common/clients/mix-client/src/poisson.rs b/common/clients/mix-client/src/poisson.rs index d4991f0dd8..6474d3c65b 100644 --- a/common/clients/mix-client/src/poisson.rs +++ b/common/clients/mix-client/src/poisson.rs @@ -1,6 +1,9 @@ use rand_distr::{Distribution, Exp}; pub fn sample(average_delay: f64) -> f64 { + // this is our internal code used by our traffic streams + // the error is only thrown if average delay is less than 0, which will never happen + // so call to unwrap is perfectly safe here let exp = Exp::new(1.0 / average_delay).unwrap(); exp.sample(&mut rand::thread_rng()) } diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index befac332c9..07e924f5af 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -66,7 +66,7 @@ impl ProviderClient { pub async fn send_request(&self, bytes: Vec) -> Result, ProviderClientError> { let mut socket = tokio::net::TcpStream::connect(self.provider_network_address).await?; - socket.set_keepalive(Some(Duration::from_secs(2))).unwrap(); + socket.set_keepalive(Some(Duration::from_secs(2)))?; socket.write_all(&bytes[..]).await?; if let Err(e) = socket.shutdown(Shutdown::Write) { warn!("failed to close write part of the socket; err = {:?}", e) @@ -82,11 +82,14 @@ impl ProviderClient { } pub async fn retrieve_messages(&self) -> Result>, ProviderClientError> { - if self.auth_token.is_none() { - return Err(ProviderClientError::EmptyAuthTokenError); - } + let auth_token = match self.auth_token { + Some(token) => token, + None => { + return Err(ProviderClientError::EmptyAuthTokenError); + } + }; - let pull_request = PullRequest::new(self.our_address, self.auth_token.unwrap()); + let pull_request = PullRequest::new(self.our_address, auth_token); let bytes = pull_request.to_bytes(); let response = self.send_request(bytes).await?; diff --git a/common/crypto/src/encryption/x25519.rs b/common/crypto/src/encryption/x25519.rs index d7d5be9f62..44d86783da 100644 --- a/common/crypto/src/encryption/x25519.rs +++ b/common/crypto/src/encryption/x25519.rs @@ -56,6 +56,7 @@ impl MixnetEncryptionPrivateKey for PrivateKey { fn from_bytes(b: &[u8]) -> Self { let mut bytes = [0; 32]; bytes.copy_from_slice(&b[..]); + // due to trait restriction we have no choice but to panic if this fails let key = Scalar::from_canonical_bytes(bytes).unwrap(); Self(key) } diff --git a/common/healthcheck/src/path_check.rs b/common/healthcheck/src/path_check.rs index f2d8b4879b..e98666485e 100644 --- a/common/healthcheck/src/path_check.rs +++ b/common/healthcheck/src/path_check.rs @@ -153,16 +153,23 @@ impl PathChecker { let mut provider_messages = Vec::new(); for provider_client in self.provider_clients.values() { // if it was none all associated paths were already marked as unhealthy - if provider_client.is_some() { - let pc = provider_client.as_ref().unwrap(); - provider_messages.extend(self.resolve_pending_provider_checks(pc).await); - } + let pc = match provider_client { + Some(pc) => pc, + None => continue, + }; + + provider_messages.extend(self.resolve_pending_provider_checks(pc).await); } self.update_path_statuses(provider_messages); } pub(crate) async fn send_test_packet(&mut self, path: &Vec, iteration: u8) { + if path.len() == 0 { + warn!("trying to send test packet through an empty path!"); + return; + } + debug!("Checking path: {:?} ({})", path, iteration); let path_identifier = PathChecker::unique_path_key(path, iteration); @@ -171,7 +178,13 @@ impl PathChecker { // does provider exist? let provider_client = self .provider_clients - .get(&path.last().unwrap().pub_key.to_bytes()) + .get( + &path + .last() + .expect("We checked the path to contain at least one entry") + .pub_key + .to_bytes(), + ) .unwrap(); if provider_client.is_none() { @@ -186,10 +199,15 @@ impl PathChecker { return; } - let layer_one_mix = path.first().unwrap(); + let layer_one_mix = path + .first() + .expect("We checked the path to contain at least one entry"); let first_node_key = layer_one_mix.pub_key.to_bytes(); + + // we generated the bytes data so unwrap is fine let first_node_address = - addressing::socket_address_from_encoded_bytes(layer_one_mix.address.to_bytes()); + addressing::socket_address_from_encoded_bytes(layer_one_mix.address.to_bytes()) + .unwrap(); let first_node_client = self .layer_one_clients @@ -208,10 +226,12 @@ impl PathChecker { return; } + // we already checked for 'None' case let first_node_client = first_node_client.as_ref().unwrap(); let delays: Vec<_> = path.iter().map(|_| Delay::new(0)).collect(); + // all of the data used to create the packet was created by us let packet = sphinx::SphinxPacket::new( path_identifier.clone(), &path[..], diff --git a/common/healthcheck/src/result.rs b/common/healthcheck/src/result.rs index b7aae8d33c..de70873e38 100644 --- a/common/healthcheck/src/result.rs +++ b/common/healthcheck/src/result.rs @@ -14,9 +14,9 @@ pub struct HealthCheckResult(Vec); impl std::fmt::Display for HealthCheckResult { fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { write!(f, "NETWORK HEALTH\n==============\n")?; - self.0 - .iter() - .for_each(|score| write!(f, "{}\n", score).unwrap()); + for score in self.0.iter() { + write!(f, "{}\n", score)? + } Ok(()) } } diff --git a/common/pemstore/src/pemstore.rs b/common/pemstore/src/pemstore.rs index d16f8574fa..9c7a914d02 100644 --- a/common/pemstore/src/pemstore.rs +++ b/common/pemstore/src/pemstore.rs @@ -1,6 +1,7 @@ use crate::pathfinder::PathFinder; use pem::{encode, parse, Pem}; use std::fs::File; +use std::io; use std::io::prelude::*; use std::path::PathBuf; @@ -24,43 +25,50 @@ impl PemStore { } } - pub fn read_identity(&self) -> IDPair + pub fn read_identity(&self) -> io::Result where IDPair: crypto::identity::MixnetIdentityKeyPair, Priv: crypto::identity::MixnetIdentityPrivateKey, Pub: crypto::identity::MixnetIdentityPublicKey, { - let private_pem = self.read_pem_file(self.private_mix_key.clone()); - let public_pem = self.read_pem_file(self.public_mix_key.clone()); + let private_pem = self.read_pem_file(self.private_mix_key.clone())?; + let public_pem = self.read_pem_file(self.public_mix_key.clone())?; let key_pair = IDPair::from_bytes(&private_pem.contents, &public_pem.contents); - assert_eq!(key_pair.private_key().pem_type(), private_pem.tag); - assert_eq!(key_pair.public_key().pem_type(), public_pem.tag); + if key_pair.private_key().pem_type() != private_pem.tag { + return Err(io::Error::new( + io::ErrorKind::Other, + "unexpected private key pem tag", + )); + } - key_pair + if key_pair.public_key().pem_type() != public_pem.tag { + return Err(io::Error::new( + io::ErrorKind::Other, + "unexpected public key pem tag", + )); + } + + Ok(key_pair) } - fn read_pem_file(&self, filepath: PathBuf) -> Pem { - let mut pem_bytes = File::open(filepath).expect("Could not open stored keys from disk."); + fn read_pem_file(&self, filepath: PathBuf) -> io::Result { + let mut pem_bytes = File::open(filepath)?; let mut buf = Vec::new(); - pem_bytes - .read_to_end(&mut buf) - .expect("PEM bytes reading failed."); - let pem = parse(&buf).expect("PEM parsing failed while reading keys"); - - pem + pem_bytes.read_to_end(&mut buf)?; + parse(&buf).map_err(|e| io::Error::new(io::ErrorKind::Other, e)) } // This should be refactored and made more generic for when we have other kinds of // KeyPairs that we want to persist (e.g. validator keypairs, or keys for // signing vs encryption). However, for the moment, it does the job. - pub fn write_identity(&self, key_pair: IDPair) + pub fn write_identity(&self, key_pair: IDPair) -> io::Result<()> where IDPair: crypto::identity::MixnetIdentityKeyPair, Priv: crypto::identity::MixnetIdentityPrivateKey, Pub: crypto::identity::MixnetIdentityPublicKey, { - std::fs::create_dir_all(self.config_dir.clone()).unwrap(); + std::fs::create_dir_all(self.config_dir.clone())?; let private_key = key_pair.private_key(); let public_key = key_pair.public_key(); @@ -68,22 +76,25 @@ impl PemStore { self.private_mix_key.clone(), private_key.to_bytes(), private_key.pem_type(), - ); + )?; self.write_pem_file( self.public_mix_key.clone(), public_key.to_bytes(), public_key.pem_type(), - ); + )?; + Ok(()) } - fn write_pem_file(&self, filepath: PathBuf, data: Vec, tag: String) { + fn write_pem_file(&self, filepath: PathBuf, data: Vec, tag: String) -> io::Result<()> { let pem = Pem { tag, contents: data, }; let key = encode(&pem); - let mut file = File::create(filepath).unwrap(); - file.write_all(key.as_bytes()).unwrap(); + let mut file = File::create(filepath)?; + file.write_all(key.as_bytes())?; + + Ok(()) } } diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 94ac5a74b4..1004ef7ca3 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -57,6 +57,8 @@ pub trait NymTopology: Sized + PartialEq + std::fmt::Debug + Send + Sync { let mut layered_topology = self.make_layered_topology()?; let num_layers = layered_topology.len(); let route = (1..=num_layers as u64) + // unwrap is safe for 'remove' as it it failed, it implied the entry never existed + // in the map in the first place which would contradict what we've just done .map(|layer| layered_topology.remove(&layer).unwrap()) // for each layer .map(|nodes| nodes.into_iter().choose(&mut rand::thread_rng()).unwrap()) // choose random node .map(|random_node| random_node.into()) // and convert it into sphinx specific node format diff --git a/nym-client/src/client/cover_traffic_stream.rs b/nym-client/src/client/cover_traffic_stream.rs index e03fc0abeb..c48c0c4801 100644 --- a/nym-client/src/client/cover_traffic_stream.rs +++ b/nym-client/src/client/cover_traffic_stream.rs @@ -2,7 +2,7 @@ use crate::client::mix_traffic::MixMessage; use crate::client::topology_control::TopologyInnerRef; use crate::client::LOOP_COVER_AVERAGE_DELAY; use futures::channel::mpsc; -use log::{info, trace, warn}; +use log::{error, info, trace, warn}; use sphinx::route::Destination; use std::time::Duration; use topology::NymTopology; @@ -28,8 +28,20 @@ pub(crate) async fn start_loop_cover_traffic_stream( Some(topology) => topology, }; - let cover_message = - mix_client::packet::loop_cover_message(our_info.address, our_info.identifier, topology); + let cover_message = match mix_client::packet::loop_cover_message( + our_info.address, + our_info.identifier, + topology, + ) { + Ok(message) => message, + Err(err) => { + error!( + "Somehow we managed to create an invalid cover message - {:?}", + err + ); + continue; + } + }; // if this one fails, there's no retrying because it means that either: // - we run out of memory diff --git a/nym-client/src/client/real_traffic_stream.rs b/nym-client/src/client/real_traffic_stream.rs index 68fb7f0098..fc8069c12b 100644 --- a/nym-client/src/client/real_traffic_stream.rs +++ b/nym-client/src/client/real_traffic_stream.rs @@ -4,7 +4,7 @@ use crate::client::{InputMessage, MESSAGE_SENDING_AVERAGE_DELAY}; use futures::channel::mpsc; use futures::task::{Context, Poll}; use futures::{Future, Stream, StreamExt}; -use log::{info, trace, warn}; +use log::{error, info, trace, warn}; use sphinx::route::Destination; use std::pin::Pin; use std::time::Duration; @@ -109,6 +109,17 @@ impl OutQueueControl { ), }; + let next_packet = match next_packet { + Ok(message) => message, + Err(err) => { + error!( + "Somehow we managed to create an invalid traffic message - {:?}", + err + ); + continue; + } + }; + // if this one fails, there's no retrying because it means that either: // - we run out of memory // - the receiver channel is closed diff --git a/nym-client/src/commands/init.rs b/nym-client/src/commands/init.rs index 4ac9c47db4..bc4403e6b9 100644 --- a/nym-client/src/commands/init.rs +++ b/nym-client/src/commands/init.rs @@ -12,7 +12,7 @@ pub fn execute(matches: &ArgMatches) { println!("Writing keypairs to {:?}...", pathfinder.config_dir); let mix_keys = crypto::identity::DummyMixIdentityKeyPair::new(); let pem_store = PemStore::new(pathfinder); - pem_store.write_identity(mix_keys); + pem_store.write_identity(mix_keys).unwrap(); println!("Client configuration completed.\n\n\n") } diff --git a/nym-client/src/commands/tcpsocket.rs b/nym-client/src/commands/tcpsocket.rs index 81e0ef72d5..87901f5543 100644 --- a/nym-client/src/commands/tcpsocket.rs +++ b/nym-client/src/commands/tcpsocket.rs @@ -27,7 +27,9 @@ pub fn execute(matches: &ArgMatches) { .expect("Failed to extract the socket address from the iterator"); // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type? - let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)).read_identity(); + let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)) + .read_identity() + .unwrap(); // TODO: reading auth_token from disk (if exists); println!("Public key: {}", keypair.public_key.to_b64_string()); diff --git a/nym-client/src/commands/websocket.rs b/nym-client/src/commands/websocket.rs index 644e023306..5359c10871 100644 --- a/nym-client/src/commands/websocket.rs +++ b/nym-client/src/commands/websocket.rs @@ -27,7 +27,9 @@ pub fn execute(matches: &ArgMatches) { .expect("Failed to extract the socket address from the iterator"); // TODO: currently we know we are reading the 'DummyMixIdentityKeyPair', but how to properly assert the type? - let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)).read_identity(); + let keypair: DummyMixIdentityKeyPair = PemStore::new(ClientPathfinder::new(id)) + .read_identity() + .unwrap(); // TODO: reading auth_token from disk (if exists); From 0e85f6de0fd94dc8b39724c9492d950deff00ece Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 15:39:27 +0000 Subject: [PATCH 079/105] topology: using base58 for decoding keys --- common/topology/src/mix.rs | 3 +-- common/topology/src/provider.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/common/topology/src/mix.rs b/common/topology/src/mix.rs index 5236e191fa..4f5a9c2b2e 100644 --- a/common/topology/src/mix.rs +++ b/common/topology/src/mix.rs @@ -14,9 +14,8 @@ pub struct Node { impl Node { pub fn get_pub_key_bytes(&self) -> [u8; 32] { - let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); let mut key_bytes = [0; 32]; - key_bytes.copy_from_slice(&decoded_key_bytes[..]); + bs58::decode(&self.pub_key).into(&mut key_bytes).unwrap(); key_bytes } } diff --git a/common/topology/src/provider.rs b/common/topology/src/provider.rs index e9d4869e0e..a4d268822b 100644 --- a/common/topology/src/provider.rs +++ b/common/topology/src/provider.rs @@ -20,9 +20,8 @@ pub struct Node { impl Node { pub fn get_pub_key_bytes(&self) -> [u8; 32] { - let decoded_key_bytes = base64::decode_config(&self.pub_key, base64::URL_SAFE).unwrap(); let mut key_bytes = [0; 32]; - key_bytes.copy_from_slice(&decoded_key_bytes[..]); + bs58::decode(&self.pub_key).into(&mut key_bytes).unwrap(); key_bytes } } From d6ce495e08901ad7ddab632056c45fe29b6f4f8b Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 15:42:27 +0000 Subject: [PATCH 080/105] git: ignoring local dictionary --- .gitignore | 1 + common/clients/directory-client/src/presence.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6ddcea9a0c..74a82c4e13 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .idea target .env +/.vscode/settings.json diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs index f56305f172..664274007a 100644 --- a/common/clients/directory-client/src/presence.rs +++ b/common/clients/directory-client/src/presence.rs @@ -11,8 +11,8 @@ use topology::mix; use topology::provider; use topology::NymTopology; -// special version of 'PartialEq' that does not care about last_seen field (where applicable) -// or order of elements in vectors +// special version of 'PartialEq' that does not care about last_seen field +//(where applicable) or order of elements in vectors trait PresenceEq { fn presence_eq(&self, other: &Self) -> bool; } From a1af0b9345d2b0ac132aebc6baf054ef110c3dcc Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:18:22 +0000 Subject: [PATCH 081/105] directory-client: ripped presence apart a bit --- .../clients/directory-client/src/presence.rs | 392 ------------------ .../src/presence/coconodes.rs | 56 +++ .../directory-client/src/presence/mixnodes.rs | 72 ++++ .../directory-client/src/presence/mod.rs | 134 ++++++ .../src/presence/providers.rs | 149 +++++++ .../src/requests/presence_coconodes_post.rs | 4 +- .../src/requests/presence_mixnodes_post.rs | 4 +- .../src/requests/presence_providers_post.rs | 4 +- mixnode/src/node/presence.rs | 2 +- sfw-provider/src/provider/mod.rs | 2 +- sfw-provider/src/provider/presence.rs | 2 +- 11 files changed, 420 insertions(+), 401 deletions(-) delete mode 100644 common/clients/directory-client/src/presence.rs create mode 100644 common/clients/directory-client/src/presence/coconodes.rs create mode 100644 common/clients/directory-client/src/presence/mixnodes.rs create mode 100644 common/clients/directory-client/src/presence/mod.rs create mode 100644 common/clients/directory-client/src/presence/providers.rs diff --git a/common/clients/directory-client/src/presence.rs b/common/clients/directory-client/src/presence.rs deleted file mode 100644 index 664274007a..0000000000 --- a/common/clients/directory-client/src/presence.rs +++ /dev/null @@ -1,392 +0,0 @@ -use crate::requests::presence_topology_get::PresenceTopologyGetRequester; -use crate::{Client, Config, DirectoryClient}; -use log::*; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use std::convert::TryInto; -use std::io; -use std::net::ToSocketAddrs; -use topology::coco; -use topology::mix; -use topology::provider; -use topology::NymTopology; - -// special version of 'PartialEq' that does not care about last_seen field -//(where applicable) or order of elements in vectors -trait PresenceEq { - fn presence_eq(&self, other: &Self) -> bool; -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CocoPresence { - pub host: String, - pub pub_key: String, - pub last_seen: u64, - pub version: String, -} - -impl Into for CocoPresence { - fn into(self) -> topology::coco::Node { - topology::coco::Node { - host: self.host, - pub_key: self.pub_key, - last_seen: self.last_seen, - version: self.version, - } - } -} - -impl From for CocoPresence { - fn from(cn: coco::Node) -> Self { - CocoPresence { - host: cn.host, - pub_key: cn.pub_key, - last_seen: cn.last_seen, - version: cn.version, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct MixNodePresence { - pub host: String, - pub pub_key: String, - pub layer: u64, - pub last_seen: u64, - pub version: String, -} - -impl TryInto for MixNodePresence { - type Error = io::Error; - - fn try_into(self) -> Result { - let resolved_hostname = self.host.to_socket_addrs()?.next(); - if resolved_hostname.is_none() { - return Err(io::Error::new( - io::ErrorKind::Other, - "no valid socket address", - )); - } - - Ok(topology::mix::Node { - host: resolved_hostname.unwrap(), - pub_key: self.pub_key, - layer: self.layer, - last_seen: self.last_seen, - version: self.version, - }) - } -} - -impl From for MixNodePresence { - fn from(mn: mix::Node) -> Self { - MixNodePresence { - host: mn.host.to_string(), - pub_key: mn.pub_key, - layer: mn.layer, - last_seen: mn.last_seen, - version: mn.version, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct MixProviderPresence { - pub client_listener: String, - pub mixnet_listener: String, - pub pub_key: String, - pub registered_clients: Vec, - pub last_seen: u64, - pub version: String, -} - -impl PresenceEq for MixProviderPresence { - fn presence_eq(&self, other: &Self) -> bool { - if self.registered_clients.len() != other.registered_clients.len() { - return false; - } - - if self.client_listener != other.client_listener - || self.mixnet_listener != other.mixnet_listener - || self.pub_key != other.pub_key - || self.version != other.version - { - return false; - } - - let clients_self_set: HashSet<_> = - self.registered_clients.iter().map(|c| &c.pub_key).collect(); - let clients_other_set: HashSet<_> = other - .registered_clients - .iter() - .map(|c| &c.pub_key) - .collect(); - - clients_self_set == clients_other_set - } -} - -impl Into for MixProviderPresence { - fn into(self) -> topology::provider::Node { - topology::provider::Node { - client_listener: self.client_listener.parse().unwrap(), - mixnet_listener: self.mixnet_listener.parse().unwrap(), - pub_key: self.pub_key, - registered_clients: self - .registered_clients - .into_iter() - .map(|c| c.into()) - .collect(), - last_seen: self.last_seen, - version: self.version, - } - } -} - -impl From for MixProviderPresence { - fn from(mpn: provider::Node) -> Self { - MixProviderPresence { - client_listener: mpn.client_listener.to_string(), - mixnet_listener: mpn.mixnet_listener.to_string(), - pub_key: mpn.pub_key, - registered_clients: mpn - .registered_clients - .into_iter() - .map(|c| c.into()) - .collect(), - last_seen: mpn.last_seen, - version: mpn.version, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct MixProviderClient { - pub pub_key: String, -} - -impl Into for MixProviderClient { - fn into(self) -> topology::provider::Client { - topology::provider::Client { - pub_key: self.pub_key, - } - } -} - -impl From for MixProviderClient { - fn from(mpc: topology::provider::Client) -> Self { - MixProviderClient { - pub_key: mpc.pub_key, - } - } -} - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration - let self_set: HashSet<_> = self - .iter() - .map(|c| (&c.host, &c.pub_key, &c.version)) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|c| (&c.host, &c.pub_key, &c.version)) - .collect(); - - self_set == other_set - } -} - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration - let self_set: HashSet<_> = self - .iter() - .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) - .collect(); - - self_set == other_set - } -} - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration. - // We also don't care about order of registered_clients - - // since we're going to be getting rid of this very soon anyway, just clone registered - // clients vector and sort it - - let self_set: HashSet<_> = self - .iter() - .map(|p| { - ( - &p.client_listener, - &p.mixnet_listener, - &p.pub_key, - &p.version, - p.registered_clients - .iter() - .cloned() - .map(|c| c.pub_key) - .collect::>() - .sort(), - ) - }) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|p| { - ( - &p.client_listener, - &p.mixnet_listener, - &p.pub_key, - &p.version, - p.registered_clients - .iter() - .cloned() - .map(|c| c.pub_key) - .collect::>() - .sort(), - ) - }) - .collect(); - - self_set == other_set - } -} - -// Topology shows us the current state of the overall Nym network -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Topology { - pub coco_nodes: Vec, - pub mix_nodes: Vec, - pub mix_provider_nodes: Vec, -} - -impl PartialEq for Topology { - // we need a custom implementation as the order of nodes does not matter - // also we do not care about 'last_seen' when comparing topologies - fn eq(&self, other: &Self) -> bool { - if !self.coco_nodes.presence_eq(&other.coco_nodes) - || !self.mix_nodes.presence_eq(&other.mix_nodes) - || !self - .mix_provider_nodes - .presence_eq(&other.mix_provider_nodes) - { - return false; - } - - true - } -} - -impl NymTopology for Topology { - fn new(directory_server: String) -> Self { - debug!("Using directory server: {:?}", directory_server); - let directory_config = Config { - base_url: directory_server, - }; - let directory = Client::new(directory_config); - - let topology = directory - .presence_topology - .get() - .expect("Failed to retrieve network topology."); - topology - } - - fn new_from_nodes( - mix_nodes: Vec, - mix_provider_nodes: Vec, - coco_nodes: Vec, - ) -> Self { - Topology { - coco_nodes: coco_nodes.into_iter().map(|node| node.into()).collect(), - mix_nodes: mix_nodes.into_iter().map(|node| node.into()).collect(), - mix_provider_nodes: mix_provider_nodes - .into_iter() - .map(|node| node.into()) - .collect(), - } - } - - fn mix_nodes(&self) -> Vec { - self.mix_nodes - .iter() - .filter_map(|x| x.clone().try_into().ok()) - .collect() - } - - fn providers(&self) -> Vec { - self.mix_provider_nodes - .iter() - .map(|x| x.clone().into()) - .collect() - } - - fn coco_nodes(&self) -> Vec { - self.coco_nodes.iter().map(|x| x.clone().into()).collect() - } -} - -#[cfg(test)] -mod converting_mixnode_presence_into_topology_mixnode { - use super::*; - - #[test] - fn it_returns_error_on_unresolvable_hostname() { - let unresolvable_hostname = "foomp.foomp.foomp:1234"; - - let mix_presence = MixNodePresence { - host: unresolvable_hostname.to_string(), - pub_key: "".to_string(), - layer: 0, - last_seen: 0, - version: "".to_string(), - }; - - let result: Result = mix_presence.try_into(); - assert!(result.is_err()) - } - - #[test] - fn it_returns_resolved_ip_on_resolvable_hostname() { - let resolvable_hostname = "nymtech.net:1234"; - - let mix_presence = MixNodePresence { - host: resolvable_hostname.to_string(), - pub_key: "".to_string(), - layer: 0, - last_seen: 0, - version: "".to_string(), - }; - - let result: Result = mix_presence.try_into(); - assert!(result.is_ok()) - } -} diff --git a/common/clients/directory-client/src/presence/coconodes.rs b/common/clients/directory-client/src/presence/coconodes.rs new file mode 100644 index 0000000000..ff1d6c80f0 --- /dev/null +++ b/common/clients/directory-client/src/presence/coconodes.rs @@ -0,0 +1,56 @@ +use super::PresenceEq; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use topology::coco; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CocoPresence { + pub host: String, + pub pub_key: String, + pub last_seen: u64, + pub version: String, +} + +impl Into for CocoPresence { + fn into(self) -> topology::coco::Node { + topology::coco::Node { + host: self.host, + pub_key: self.pub_key, + last_seen: self.last_seen, + version: self.version, + } + } +} + +impl From for CocoPresence { + fn from(cn: coco::Node) -> Self { + CocoPresence { + host: cn.host, + pub_key: cn.pub_key, + last_seen: cn.last_seen, + version: cn.version, + } + } +} + +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration + let self_set: HashSet<_> = self + .iter() + .map(|c| (&c.host, &c.pub_key, &c.version)) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|c| (&c.host, &c.pub_key, &c.version)) + .collect(); + + self_set == other_set + } +} diff --git a/common/clients/directory-client/src/presence/mixnodes.rs b/common/clients/directory-client/src/presence/mixnodes.rs new file mode 100644 index 0000000000..aba2d73cf9 --- /dev/null +++ b/common/clients/directory-client/src/presence/mixnodes.rs @@ -0,0 +1,72 @@ +use super::PresenceEq; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::convert::TryInto; +use std::io; +use std::net::ToSocketAddrs; +use topology::mix; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MixNodePresence { + pub host: String, + pub pub_key: String, + pub layer: u64, + pub last_seen: u64, + pub version: String, +} + +impl TryInto for MixNodePresence { + type Error = io::Error; + + fn try_into(self) -> Result { + let resolved_hostname = self.host.to_socket_addrs()?.next(); + if resolved_hostname.is_none() { + return Err(io::Error::new( + io::ErrorKind::Other, + "no valid socket address", + )); + } + + Ok(topology::mix::Node { + host: resolved_hostname.unwrap(), + pub_key: self.pub_key, + layer: self.layer, + last_seen: self.last_seen, + version: self.version, + }) + } +} + +impl From for MixNodePresence { + fn from(mn: mix::Node) -> Self { + MixNodePresence { + host: mn.host.to_string(), + pub_key: mn.pub_key, + layer: mn.layer, + last_seen: mn.last_seen, + version: mn.version, + } + } +} + +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration + let self_set: HashSet<_> = self + .iter() + .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) + .collect(); + + self_set == other_set + } +} diff --git a/common/clients/directory-client/src/presence/mod.rs b/common/clients/directory-client/src/presence/mod.rs new file mode 100644 index 0000000000..10d8b1a5e4 --- /dev/null +++ b/common/clients/directory-client/src/presence/mod.rs @@ -0,0 +1,134 @@ +use crate::requests::presence_topology_get::PresenceTopologyGetRequester; +use crate::{Client, Config, DirectoryClient}; +use log::*; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::convert::TryInto; +use std::io; +use std::net::ToSocketAddrs; +use topology::coco; +use topology::mix; +use topology::provider; +use topology::NymTopology; + +pub mod coconodes; +pub mod mixnodes; +pub mod providers; + +// special version of 'PartialEq' that does not care about last_seen field +//(where applicable) or order of elements in vectors +trait PresenceEq { + fn presence_eq(&self, other: &Self) -> bool; +} + +// Topology shows us the current state of the overall Nym network +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Topology { + pub coco_nodes: Vec, + pub mix_nodes: Vec, + pub mix_provider_nodes: Vec, +} + +impl PartialEq for Topology { + // we need a custom implementation as the order of nodes does not matter + // also we do not care about 'last_seen' when comparing topologies + fn eq(&self, other: &Self) -> bool { + if !self.coco_nodes.presence_eq(&other.coco_nodes) + || !self.mix_nodes.presence_eq(&other.mix_nodes) + || !self + .mix_provider_nodes + .presence_eq(&other.mix_provider_nodes) + { + return false; + } + + true + } +} + +impl NymTopology for Topology { + fn new(directory_server: String) -> Self { + debug!("Using directory server: {:?}", directory_server); + let directory_config = Config { + base_url: directory_server, + }; + let directory = Client::new(directory_config); + + let topology = directory + .presence_topology + .get() + .expect("Failed to retrieve network topology."); + topology + } + + fn new_from_nodes( + mix_nodes: Vec, + mix_provider_nodes: Vec, + coco_nodes: Vec, + ) -> Self { + Topology { + coco_nodes: coco_nodes.into_iter().map(|node| node.into()).collect(), + mix_nodes: mix_nodes.into_iter().map(|node| node.into()).collect(), + mix_provider_nodes: mix_provider_nodes + .into_iter() + .map(|node| node.into()) + .collect(), + } + } + + fn mix_nodes(&self) -> Vec { + self.mix_nodes + .iter() + .filter_map(|x| x.clone().try_into().ok()) + .collect() + } + + fn providers(&self) -> Vec { + self.mix_provider_nodes + .iter() + .map(|x| x.clone().into()) + .collect() + } + + fn coco_nodes(&self) -> Vec { + self.coco_nodes.iter().map(|x| x.clone().into()).collect() + } +} + +#[cfg(test)] +mod converting_mixnode_presence_into_topology_mixnode { + use super::*; + + #[test] + fn it_returns_error_on_unresolvable_hostname() { + let unresolvable_hostname = "foomp.foomp.foomp:1234"; + + let mix_presence = mixnodes::MixNodePresence { + host: unresolvable_hostname.to_string(), + pub_key: "".to_string(), + layer: 0, + last_seen: 0, + version: "".to_string(), + }; + + let result: Result = mix_presence.try_into(); + assert!(result.is_err()) + } + + #[test] + fn it_returns_resolved_ip_on_resolvable_hostname() { + let resolvable_hostname = "nymtech.net:1234"; + + let mix_presence = mixnodes::MixNodePresence { + host: resolvable_hostname.to_string(), + pub_key: "".to_string(), + layer: 0, + last_seen: 0, + version: "".to_string(), + }; + + let result: Result = mix_presence.try_into(); + assert!(result.is_ok()) + } +} diff --git a/common/clients/directory-client/src/presence/providers.rs b/common/clients/directory-client/src/presence/providers.rs new file mode 100644 index 0000000000..b96bcf6a9e --- /dev/null +++ b/common/clients/directory-client/src/presence/providers.rs @@ -0,0 +1,149 @@ +use super::PresenceEq; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use topology::provider; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MixProviderPresence { + pub client_listener: String, + pub mixnet_listener: String, + pub pub_key: String, + pub registered_clients: Vec, + pub last_seen: u64, + pub version: String, +} + +impl PresenceEq for MixProviderPresence { + fn presence_eq(&self, other: &Self) -> bool { + if self.registered_clients.len() != other.registered_clients.len() { + return false; + } + + if self.client_listener != other.client_listener + || self.mixnet_listener != other.mixnet_listener + || self.pub_key != other.pub_key + || self.version != other.version + { + return false; + } + + let clients_self_set: HashSet<_> = + self.registered_clients.iter().map(|c| &c.pub_key).collect(); + let clients_other_set: HashSet<_> = other + .registered_clients + .iter() + .map(|c| &c.pub_key) + .collect(); + + clients_self_set == clients_other_set + } +} + +impl Into for MixProviderPresence { + fn into(self) -> topology::provider::Node { + topology::provider::Node { + client_listener: self.client_listener.parse().unwrap(), + mixnet_listener: self.mixnet_listener.parse().unwrap(), + pub_key: self.pub_key, + registered_clients: self + .registered_clients + .into_iter() + .map(|c| c.into()) + .collect(), + last_seen: self.last_seen, + version: self.version, + } + } +} + +impl From for MixProviderPresence { + fn from(mpn: provider::Node) -> Self { + MixProviderPresence { + client_listener: mpn.client_listener.to_string(), + mixnet_listener: mpn.mixnet_listener.to_string(), + pub_key: mpn.pub_key, + registered_clients: mpn + .registered_clients + .into_iter() + .map(|c| c.into()) + .collect(), + last_seen: mpn.last_seen, + version: mpn.version, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MixProviderClient { + pub pub_key: String, +} + +impl Into for MixProviderClient { + fn into(self) -> topology::provider::Client { + topology::provider::Client { + pub_key: self.pub_key, + } + } +} + +impl From for MixProviderClient { + fn from(mpc: topology::provider::Client) -> Self { + MixProviderClient { + pub_key: mpc.pub_key, + } + } +} + +impl PresenceEq for Vec { + fn presence_eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + // we can't take the whole thing into set as it does not implement 'Eq' and we can't + // derive it as we don't want to take 'last_seen' into consideration. + // We also don't care about order of registered_clients + + // since we're going to be getting rid of this very soon anyway, just clone registered + // clients vector and sort it + + let self_set: HashSet<_> = self + .iter() + .map(|p| { + ( + &p.client_listener, + &p.mixnet_listener, + &p.pub_key, + &p.version, + p.registered_clients + .iter() + .cloned() + .map(|c| c.pub_key) + .collect::>() + .sort(), + ) + }) + .collect(); + let other_set: HashSet<_> = other + .iter() + .map(|p| { + ( + &p.client_listener, + &p.mixnet_listener, + &p.pub_key, + &p.version, + p.registered_clients + .iter() + .cloned() + .map(|c| c.pub_key) + .collect::>() + .sort(), + ) + }) + .collect(); + + self_set == other_set + } +} diff --git a/common/clients/directory-client/src/requests/presence_coconodes_post.rs b/common/clients/directory-client/src/requests/presence_coconodes_post.rs index 40fe4bd2e2..cc34c13272 100644 --- a/common/clients/directory-client/src/requests/presence_coconodes_post.rs +++ b/common/clients/directory-client/src/requests/presence_coconodes_post.rs @@ -1,4 +1,4 @@ -use crate::presence::CocoPresence; +use crate::presence::coconodes::CocoPresence; use reqwest::Response; pub struct Request { @@ -73,7 +73,7 @@ mod metrics_get_request { #[cfg(test)] mod fixtures { - use crate::presence::CocoPresence; + use crate::presence::coconodes::CocoPresence; pub fn new_presence() -> CocoPresence { CocoPresence { diff --git a/common/clients/directory-client/src/requests/presence_mixnodes_post.rs b/common/clients/directory-client/src/requests/presence_mixnodes_post.rs index 3b5d942332..df110c2fea 100644 --- a/common/clients/directory-client/src/requests/presence_mixnodes_post.rs +++ b/common/clients/directory-client/src/requests/presence_mixnodes_post.rs @@ -1,4 +1,4 @@ -use crate::presence::MixNodePresence; +use crate::presence::mixnodes::MixNodePresence; use reqwest::Response; pub struct Request { @@ -73,7 +73,7 @@ mod metrics_get_request { #[cfg(test)] mod fixtures { - use crate::presence::MixNodePresence; + use crate::presence::mixnodes::MixNodePresence; pub fn new_presence() -> MixNodePresence { MixNodePresence { diff --git a/common/clients/directory-client/src/requests/presence_providers_post.rs b/common/clients/directory-client/src/requests/presence_providers_post.rs index f5f522fa93..d926700334 100644 --- a/common/clients/directory-client/src/requests/presence_providers_post.rs +++ b/common/clients/directory-client/src/requests/presence_providers_post.rs @@ -1,4 +1,4 @@ -use crate::presence::MixProviderPresence; +use crate::presence::providers::MixProviderPresence; use reqwest::Response; pub struct Request { @@ -72,7 +72,7 @@ mod metrics_get_request { } #[cfg(test)] mod fixtures { - use crate::presence::MixProviderPresence; + use crate::presence::providers::MixProviderPresence; pub fn new_presence() -> MixProviderPresence { MixProviderPresence { diff --git a/mixnode/src/node/presence.rs b/mixnode/src/node/presence.rs index 283a6a3afe..b017bacdea 100644 --- a/mixnode/src/node/presence.rs +++ b/mixnode/src/node/presence.rs @@ -1,5 +1,5 @@ use crate::node; -use directory_client::presence::MixNodePresence; +use directory_client::presence::mixnodes::MixNodePresence; use directory_client::requests::presence_mixnodes_post::PresenceMixNodesPoster; use directory_client::DirectoryClient; use log::{debug, error}; diff --git a/sfw-provider/src/provider/mod.rs b/sfw-provider/src/provider/mod.rs index bc32073da8..539ecbe463 100644 --- a/sfw-provider/src/provider/mod.rs +++ b/sfw-provider/src/provider/mod.rs @@ -2,7 +2,7 @@ use crate::provider::client_handling::{ClientProcessingData, ClientRequestProces use crate::provider::mix_handling::{MixPacketProcessor, MixProcessingData}; use crate::provider::storage::ClientStorage; use crypto::identity::{DummyMixIdentityPrivateKey, DummyMixIdentityPublicKey}; -use directory_client::presence::MixProviderClient; +use directory_client::presence::providers::MixProviderClient; use futures::io::Error; use futures::lock::Mutex as FMutex; use log::*; diff --git a/sfw-provider/src/provider/presence.rs b/sfw-provider/src/provider/presence.rs index 567b0a56dc..d2ec3c1970 100644 --- a/sfw-provider/src/provider/presence.rs +++ b/sfw-provider/src/provider/presence.rs @@ -1,6 +1,6 @@ use crate::provider::ClientLedger; use crypto::identity::DummyMixIdentityPublicKey; -use directory_client::presence::MixProviderPresence; +use directory_client::presence::providers::MixProviderPresence; use directory_client::requests::presence_providers_post::PresenceMixProviderPoster; use directory_client::DirectoryClient; use futures::lock::Mutex as FMutex; From 8cf37e10cec6b3ff6d7bdb46bc40ffd516cbc355 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:21:23 +0000 Subject: [PATCH 082/105] directory-client: removing unused imports --- common/clients/directory-client/src/presence/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/clients/directory-client/src/presence/mod.rs b/common/clients/directory-client/src/presence/mod.rs index 10d8b1a5e4..1c45b39445 100644 --- a/common/clients/directory-client/src/presence/mod.rs +++ b/common/clients/directory-client/src/presence/mod.rs @@ -2,10 +2,8 @@ use crate::requests::presence_topology_get::PresenceTopologyGetRequester; use crate::{Client, Config, DirectoryClient}; use log::*; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use std::convert::TryInto; use std::io; -use std::net::ToSocketAddrs; use topology::coco; use topology::mix; use topology::provider; From 80537045b47c5937c60e80ed60860daecb528460 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:22:44 +0000 Subject: [PATCH 083/105] directory-client: fullpath on std::io::Error needed for some reason --- common/clients/directory-client/src/presence/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/clients/directory-client/src/presence/mod.rs b/common/clients/directory-client/src/presence/mod.rs index 1c45b39445..b759f576e7 100644 --- a/common/clients/directory-client/src/presence/mod.rs +++ b/common/clients/directory-client/src/presence/mod.rs @@ -3,7 +3,6 @@ use crate::{Client, Config, DirectoryClient}; use log::*; use serde::{Deserialize, Serialize}; use std::convert::TryInto; -use std::io; use topology::coco; use topology::mix; use topology::provider; @@ -110,7 +109,7 @@ mod converting_mixnode_presence_into_topology_mixnode { version: "".to_string(), }; - let result: Result = mix_presence.try_into(); + let result: Result = mix_presence.try_into(); assert!(result.is_err()) } @@ -126,7 +125,7 @@ mod converting_mixnode_presence_into_topology_mixnode { version: "".to_string(), }; - let result: Result = mix_presence.try_into(); + let result: Result = mix_presence.try_into(); assert!(result.is_ok()) } } From aa7ff75be71b89636bb233cfa1818112cb2384f6 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:29:02 +0000 Subject: [PATCH 084/105] mixnode: replaced base64 with base58 --- Cargo.lock | 2 +- mixnode/Cargo.toml | 2 +- mixnode/src/node/mod.rs | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7f39eb1ef..5f15a91993 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1382,7 +1382,7 @@ name = "nym-mixnode" version = "0.3.3" dependencies = [ "addressing", - "base64 0.11.0", + "bs58", "built", "clap", "curve25519-dalek", diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index 52db3fa9b6..e7e7282f92 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -base64 = "0.11.0" +bs58 = "0.3.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dotenv = "0.15.0" diff --git a/mixnode/src/node/mod.rs b/mixnode/src/node/mod.rs index c8ce9ae783..70f69ce80e 100644 --- a/mixnode/src/node/mod.rs +++ b/mixnode/src/node/mod.rs @@ -31,7 +31,7 @@ pub struct Config { impl Config { pub fn public_key_string(&self) -> String { let key_bytes = self.public_key.to_bytes().to_vec(); - base64::encode_config(&key_bytes, base64::URL_SAFE) + bs58::encode(&key_bytes).into_string() } } @@ -232,8 +232,7 @@ impl MixNode { let directory_cfg = directory_client::Config { base_url: self.directory_server.clone(), }; - let pub_key_str = - base64::encode_config(&self.public_key.to_bytes().to_vec(), base64::URL_SAFE); + let pub_key_str = bs58::encode(&self.public_key.to_bytes().to_vec()).into_string(); rt.spawn({ let presence_notifier = presence::Notifier::new(&config); From d71d2effe65dc73879b7a0a2c7d32ee51a9c27c3 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:29:41 +0000 Subject: [PATCH 085/105] topology: removing base64 crate --- Cargo.lock | 1 - common/topology/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f15a91993..f3fa287c2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2427,7 +2427,6 @@ name = "topology" version = "0.1.0" dependencies = [ "addressing", - "base64 0.11.0", "bs58", "itertools", "log", diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index dcebcd6796..d7d1b45501 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -7,7 +7,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -base64 = "0.11.0" bs58 = "0.3.0" itertools = "0.8.2" log = "0.4" From da6aac176b4842bcc473878cf171b64f9d88d03c Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:34:55 +0000 Subject: [PATCH 086/105] nym-client: adding bs58 crate --- Cargo.lock | 1 + nym-client/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f3fa287c2d..cca62e2495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,6 +1350,7 @@ version = "0.3.3" dependencies = [ "addressing", "base64 0.11.0", + "bs58", "built", "clap", "crypto", diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index eade92dd4b..a33d6ede32 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -13,6 +13,7 @@ path = "src/lib.rs" [dependencies] base64 = "0.11.0" +bs58 = "0.3.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dirs = "2.0.2" From 559cffdb560c0691a2191e4b7077a9542ee23266 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:37:35 +0000 Subject: [PATCH 087/105] sfw-provider: swapping out base64 for base58 --- Cargo.lock | 2 +- sfw-provider/Cargo.toml | 2 +- sfw-provider/src/provider/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cca62e2495..3fe3eea9b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1400,7 +1400,7 @@ dependencies = [ name = "nym-sfw-provider" version = "0.3.3" dependencies = [ - "base64 0.11.0", + "bs58", "built", "clap", "crypto", diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index 7bea6173d0..8fccc11232 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -base64 = "0.11.0" +bs58 = "0.3.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dotenv = "0.15.0" diff --git a/sfw-provider/src/provider/mod.rs b/sfw-provider/src/provider/mod.rs index 539ecbe463..368b0dd64b 100644 --- a/sfw-provider/src/provider/mod.rs +++ b/sfw-provider/src/provider/mod.rs @@ -87,7 +87,7 @@ impl ClientLedger { fn current_clients(&self) -> Vec { self.0 .iter() - .map(|(_, v)| base64::encode_config(v, base64::URL_SAFE)) + .map(|(_, v)| bs58::encode(v).into_string()) .map(|pub_key| MixProviderClient { pub_key }) .collect() } From e1b3a22bc57438b5a2edeee63e5e8c91d93a9f50 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:45:40 +0000 Subject: [PATCH 088/105] nym-client: removed the hex crate --- Cargo.lock | 1 - nym-client/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fe3eea9b4..2c59310c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1360,7 +1360,6 @@ dependencies = [ "dotenv", "futures 0.3.1", "healthcheck", - "hex", "log", "mix-client", "pem", diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index a33d6ede32..7f31d21111 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -19,7 +19,6 @@ curve25519-dalek = "1.2.3" dirs = "2.0.2" dotenv = "0.15.0" futures = "0.3.1" -hex = "0.4.0" log = "0.4" pem = "0.7.0" pretty_env_logger = "0.3" From a1982680092d37a68779fba0adb1a51371667cd4 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:46:12 +0000 Subject: [PATCH 089/105] nym-client: renamed suspicious hex variable (which contains bytes) --- nym-client/src/sockets/ws.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index 8840e6f95e..e2c2267414 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -205,7 +205,7 @@ impl ClientRequest { message: e.to_string(), }; } - Ok(hex) => hex, + Ok(bytes) => bytes, }; if address_vec.len() != 32 { From 83aa0a37b16bb5036e630cfc5ef5066a77af21e3 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Fri, 24 Jan 2020 16:50:06 +0000 Subject: [PATCH 090/105] sfw-provider: changed on-disk storage location from hex to base58 --- Cargo.lock | 7 ------- sfw-provider/Cargo.toml | 1 - sfw-provider/src/provider/client_handling/mod.rs | 2 +- sfw-provider/src/provider/storage/mod.rs | 4 ++-- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c59310c35..ef7e4a13a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -876,12 +876,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" - [[package]] name = "hkdf" version = "0.8.0" @@ -1407,7 +1401,6 @@ dependencies = [ "directory-client", "dotenv", "futures 0.3.1", - "hex", "hmac", "log", "pretty_env_logger", diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index 8fccc11232..f590ece187 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -12,7 +12,6 @@ bs58 = "0.3.0" clap = "2.33.0" curve25519-dalek = "1.2.3" dotenv = "0.15.0" -hex = "0.4.0" futures = "0.3.1" log = "0.4" pretty_env_logger = "0.3" diff --git a/sfw-provider/src/provider/client_handling/mod.rs b/sfw-provider/src/provider/client_handling/mod.rs index 034e946152..e3b9dcf86e 100644 --- a/sfw-provider/src/provider/client_handling/mod.rs +++ b/sfw-provider/src/provider/client_handling/mod.rs @@ -148,7 +148,7 @@ impl ClientRequestProcessor { client_address: sphinx::route::DestinationAddressBytes, store_dir: &Path, ) -> io::Result<()> { - let client_dir_name = hex::encode(client_address); + let client_dir_name = bs58::encode(client_address).into_string(); let full_store_dir = store_dir.join(client_dir_name); std::fs::create_dir_all(full_store_dir) } diff --git a/sfw-provider/src/provider/storage/mod.rs b/sfw-provider/src/provider/storage/mod.rs index d19d427b55..73c32a84a3 100644 --- a/sfw-provider/src/provider/storage/mod.rs +++ b/sfw-provider/src/provider/storage/mod.rs @@ -60,7 +60,7 @@ impl ClientStorage { } pub fn store_processed_data(store_data: StoreData, store_dir: &Path) -> io::Result<()> { - let client_dir_name = hex::encode(store_data.client_address); + let client_dir_name = bs58::encode(store_data.client_address).into_string(); let full_store_dir = store_dir.join(client_dir_name); let full_store_path = full_store_dir.join(ClientStorage::generate_random_file_name()); debug!( @@ -81,7 +81,7 @@ impl ClientStorage { client_address: DestinationAddressBytes, store_dir: &Path, ) -> Result>, StoreError> { - let client_dir_name = hex::encode(client_address); + let client_dir_name = bs58::encode(client_address).into_string(); let full_store_dir = store_dir.join(client_dir_name); trace!("going to lookup: {:?}!", full_store_dir); From c94691d68e4eea12f2313d525dda50de3beedb42 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 11:35:11 +0000 Subject: [PATCH 091/105] nym-client: swapping bs58 for base64 --- nym-client/src/sockets/tcp.rs | 4 +--- nym-client/src/sockets/ws.rs | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nym-client/src/sockets/tcp.rs b/nym-client/src/sockets/tcp.rs index 69440809d3..2b2b309070 100644 --- a/nym-client/src/sockets/tcp.rs +++ b/nym-client/src/sockets/tcp.rs @@ -142,9 +142,7 @@ impl ClientRequest { .providers() .iter() .flat_map(|provider| provider.registered_clients.iter()) - .filter_map(|client| { - base64::decode_config(&client.pub_key, base64::URL_SAFE).ok() - }) + .filter_map(|client| bs58::decode(&client.pub_key).into_vec().ok()) .collect(); ServerResponse::GetClients { clients } } diff --git a/nym-client/src/sockets/ws.rs b/nym-client/src/sockets/ws.rs index e2c2267414..15dedb37c9 100644 --- a/nym-client/src/sockets/ws.rs +++ b/nym-client/src/sockets/ws.rs @@ -199,7 +199,7 @@ impl ClientRequest { }; } - let address_vec = match base64::decode_config(&recipient_address, base64::URL_SAFE) { + let address_vec = match bs58::decode(&recipient_address).into_vec() { Err(e) => { return ServerResponse::Error { message: e.to_string(), @@ -280,7 +280,7 @@ impl ClientRequest { } async fn handle_own_details(self_address_bytes: DestinationAddressBytes) -> ServerResponse { - let self_address = base64::encode_config(&self_address_bytes, base64::URL_SAFE); + let self_address = bs58::encode(&self_address_bytes).into_string(); ServerResponse::OwnDetails { address: self_address, } From bdda6e2cf59bf4f51b7bba2bd48c2b025d2f2127 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 11:52:44 +0000 Subject: [PATCH 092/105] Removed base64 in favour of base58. --- Cargo.lock | 159 ++++++++++++++++++++---------------------- nym-client/Cargo.toml | 1 - 2 files changed, 74 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef7e4a13a5..d2857ab45f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,9 +74,9 @@ checksum = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" [[package]] name = "arrayref" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" [[package]] name = "arrayvec" @@ -119,9 +119,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.40" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" +checksum = "b4b1549d804b6c73f4817df2ba073709e96e426f12987127c48e6745568c350b" dependencies = [ "backtrace-sys", "cfg-if", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "bytes" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10004c15deb332055f7a4a208190aed362cf9a7c2f6ab70a305fba50e1105f38" +checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" [[package]] name = "c2-chacha" @@ -340,9 +340,9 @@ dependencies = [ [[package]] name = "colored" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f930f8b286023ed451756fe2527d73484d667adf9e905e9932e81d52996a343a" +checksum = "8815e2ab78f3a59928fc32e141fbeece88320a240e43f47b2fd64ea3a88a5b3d" dependencies = [ "atty", "lazy_static", @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "cookie" @@ -470,7 +470,7 @@ dependencies = [ "curve25519-dalek", "log", "pretty_env_logger", - "rand 0.7.2", + "rand 0.7.3", "rand_os", ] @@ -863,7 +863,7 @@ dependencies = [ "serde_derive", "sfw-provider-requests", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", "topology", ] @@ -1001,11 +1001,11 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2" +checksum = "0b54058f0a6ff80b6803da8faf8997cde53872b38f4023728f6830b06cd3c0dc" dependencies = [ - "autocfg 0.1.7", + "autocfg 1.0.0", ] [[package]] @@ -1037,15 +1037,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" [[package]] name = "jobserver" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230ae9adf468173aecd4176c7233bddc84a15871a586c5971ace9a55f881c075" +checksum = "67b06c1b455f1cf4269a8cfc320ab930a810e2375a42af5075eb8a8b36405ce0" dependencies = [ "libc", ] @@ -1146,9 +1146,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" +checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" [[package]] name = "memoffset" @@ -1255,10 +1255,10 @@ dependencies = [ "addressing", "log", "pretty_env_logger", - "rand 0.7.2", + "rand 0.7.3", "rand_distr", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", "topology", ] @@ -1275,7 +1275,7 @@ dependencies = [ "lazy_static", "log", "percent-encoding 1.0.1", - "rand 0.7.2", + "rand 0.7.3", "regex", "serde_json", ] @@ -1330,9 +1330,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72" +checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" dependencies = [ "hermit-abi", "libc", @@ -1343,7 +1343,6 @@ name = "nym-client" version = "0.3.3" dependencies = [ "addressing", - "base64 0.11.0", "bs58", "built", "clap", @@ -1365,7 +1364,7 @@ dependencies = [ "serde_json", "sfw-provider-requests", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", "tokio-tungstenite", "topology", "tungstenite", @@ -1386,7 +1385,7 @@ dependencies = [ "log", "pretty_env_logger", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", ] [[package]] @@ -1404,13 +1403,13 @@ dependencies = [ "hmac", "log", "pretty_env_logger", - "rand 0.7.2", + "rand 0.7.3", "serde", "serde_json", "sfw-provider-requests", "sha2", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", ] [[package]] @@ -1426,7 +1425,7 @@ dependencies = [ "pretty_env_logger", "serde", "serde_derive", - "tokio 0.2.9", + "tokio 0.2.10", "toml", ] @@ -1530,18 +1529,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b90146c7216e4cb534069fb91366de4ea0ea353105ee45ed297e2d1619e469" +checksum = "75fca1c4ff21f60ca2d37b80d72b63dab823a9d19d3cda3a81d18bc03f0ba8c5" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44ca92f893f0656d3cba8158dd0f2b99b94de256a4a54e870bd6922fcc6c8355" +checksum = "6544cd4e4ecace61075a6ec78074beeef98d58aa9a3d07d053d993b2946a90d6" dependencies = [ "proc-macro2", "quote", @@ -1550,9 +1549,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8822eb8bb72452f038ebf6048efa02c3fe22bf83f76519c9583e47fc194a422" +checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" [[package]] name = "pin-utils" @@ -1602,9 +1601,9 @@ checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" [[package]] name = "proc-macro2" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0319972dcae462681daf4da1adeeaa066e3ebd29c69be96c6abb1259d2ee2bcc" +checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" dependencies = [ "unicode-xid", ] @@ -1618,7 +1617,7 @@ dependencies = [ "pretty_env_logger", "sfw-provider-requests", "sphinx", - "tokio 0.2.9", + "tokio 0.2.10", ] [[package]] @@ -1670,9 +1669,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom", "libc", @@ -1731,7 +1730,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96977acbdd3a6576fb1d27391900035bf3863d4a16422973a409b488cf29ffb2" dependencies = [ - "rand 0.7.2", + "rand 0.7.3", ] [[package]] @@ -1822,12 +1821,11 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "redox_users" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ecedbca3bf205f8d8f5c2b44d83cd0690e39ee84b951ed649e9f1841132b66d" +checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" dependencies = [ - "failure", - "rand_os", + "getrandom", "redox_syscall", "rust-argon2", ] @@ -1895,13 +1893,14 @@ dependencies = [ [[package]] name = "rust-argon2" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca4eaef519b494d1f2848fc602d18816fed808a981aedf4f1f00ceb7c9d32cf" +checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" dependencies = [ - "base64 0.10.1", + "base64 0.11.0", "blake2b_simd", - "crossbeam-utils 0.6.6", + "constant_time_eq", + "crossbeam-utils 0.7.0", ] [[package]] @@ -1999,9 +1998,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c575e0cc52bdd09b47f330f646cf59afc586e9c4e3ccd6fc1f625b8ea1dad7" +checksum = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b" dependencies = [ "itoa", "ryu", @@ -2110,7 +2109,7 @@ dependencies = [ "hmac", "lioness", "log", - "rand 0.7.2", + "rand 0.7.3", "rand_distr", "rand_os", "sha2", @@ -2154,9 +2153,9 @@ checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4ff033220a41d1a57d8125eab57bf5263783dfdcc18688b1dacc6ce9651ef8" +checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" dependencies = [ "proc-macro2", "quote", @@ -2183,7 +2182,7 @@ checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ "cfg-if", "libc", - "rand 0.7.2", + "rand 0.7.3", "redox_syscall", "remove_dir_all", "winapi 0.3.8", @@ -2191,11 +2190,11 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" dependencies = [ - "wincolor", + "winapi-util", ] [[package]] @@ -2209,9 +2208,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ddf1ad580c7e3d1efff877d972bcc93f995556b9087a5a259630985c88ceab" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" dependencies = [ "lazy_static", ] @@ -2248,11 +2247,11 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa2fdcfa937b20cb3c822a635ceecd5fc1a27a6a474527e5516aa24b8c8820a" +checksum = "c1fc73332507b971a5010664991a441b5ee0de92017f5a0e8b00fd684573045b" dependencies = [ - "bytes 0.5.3", + "bytes 0.5.4", "fnv", "futures-core", "iovec", @@ -2402,15 +2401,15 @@ dependencies = [ "futures 0.3.1", "log", "pin-project", - "tokio 0.2.9", + "tokio 0.2.10", "tungstenite", ] [[package]] name = "toml" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" +checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" dependencies = [ "serde", ] @@ -2424,7 +2423,7 @@ dependencies = [ "itertools", "log", "pretty_env_logger", - "rand 0.7.2", + "rand 0.7.3", "serde", "sphinx", "version-checker", @@ -2459,7 +2458,7 @@ dependencies = [ "input_buffer", "log", "native-tls", - "rand 0.7.2", + "rand 0.7.3", "sha-1", "url 2.1.1", "utf-8", @@ -2491,9 +2490,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ "smallvec 1.1.0", ] @@ -2633,9 +2632,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" +checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" dependencies = [ "winapi 0.3.8", ] @@ -2646,16 +2645,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "wincolor" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f5016b18804d24db43cebf3c77269e7569b8954a8464501c216cc5e070eaa9" -dependencies = [ - "winapi 0.3.8", - "winapi-util", -] - [[package]] name = "winreg" version = "0.6.2" diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 7f31d21111..6b61400659 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -12,7 +12,6 @@ name = "nym_client" path = "src/lib.rs" [dependencies] -base64 = "0.11.0" bs58 = "0.3.0" clap = "2.33.0" curve25519-dalek = "1.2.3" From 416e1f19e0a5d3874d9bd1d04bcaea9816858820 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 12:09:39 +0000 Subject: [PATCH 093/105] cargo: updating sphinx to latest pinned version --- Cargo.lock | 4 ++-- common/clients/mix-client/Cargo.toml | 2 +- common/clients/provider-client/Cargo.toml | 2 +- common/topology/Cargo.toml | 2 +- mixnode/Cargo.toml | 2 +- nym-client/Cargo.toml | 2 +- sfw-provider/sfw-provider-requests/Cargo.toml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2857ab45f..c7566bb6e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2096,12 +2096,12 @@ dependencies = [ [[package]] name = "sphinx" version = "0.1.0" -source = "git+https://github.com/nymtech/sphinx?rev=c3ba1447d61d6eb900fcdea0c20b3070a22bf11a#c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" +source = "git+https://github.com/nymtech/sphinx?rev=8424f4b0933b4a7f64ae828b2edefc5ba43a9e79#8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" dependencies = [ "aes-ctr", "arrayref", - "base64 0.11.0", "blake2", + "bs58", "byteorder", "chacha", "curve25519-dalek", diff --git a/common/clients/mix-client/Cargo.toml b/common/clients/mix-client/Cargo.toml index dcd16d33d5..ae7091c62c 100644 --- a/common/clients/mix-client/Cargo.toml +++ b/common/clients/mix-client/Cargo.toml @@ -18,6 +18,6 @@ addressing = {path = "../../addressing"} topology = {path = "../../topology"} ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } # sphinx = { path = "../../../../sphinx"} diff --git a/common/clients/provider-client/Cargo.toml b/common/clients/provider-client/Cargo.toml index 1c1d87b7b1..786b3c2a29 100644 --- a/common/clients/provider-client/Cargo.toml +++ b/common/clients/provider-client/Cargo.toml @@ -16,4 +16,4 @@ tokio = { version = "0.2", features = ["full"] } sfw-provider-requests = { path = "../../../sfw-provider/sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } diff --git a/common/topology/Cargo.toml b/common/topology/Cargo.toml index d7d1b45501..c3c23e733d 100644 --- a/common/topology/Cargo.toml +++ b/common/topology/Cargo.toml @@ -19,5 +19,5 @@ addressing = {path = "../addressing"} version-checker = {path = "../version-checker" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } # sphinx = { path = "../../../sphinx"} diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index e7e7282f92..23c9e7324f 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -22,7 +22,7 @@ addressing = {path = "../common/addressing" } directory-client = { path = "../common/clients/directory-client" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } [build-dependencies] built = "0.3.2" diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 6b61400659..52f5cb89da 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -39,7 +39,7 @@ sfw-provider-requests = { path = "../sfw-provider/sfw-provider-requests" } topology = {path = "../common/topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } # sphinx = { path = "../../sphinx"} # putting this explicitly below everything and most likely, the next time we look into it, it will already have a proper release diff --git a/sfw-provider/sfw-provider-requests/Cargo.toml b/sfw-provider/sfw-provider-requests/Cargo.toml index 9b207791e3..12a8343bba 100644 --- a/sfw-provider/sfw-provider-requests/Cargo.toml +++ b/sfw-provider/sfw-provider-requests/Cargo.toml @@ -7,4 +7,4 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } From b009eb0a4d1fc064dfffa7a129c43473424e5429 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 12:09:50 +0000 Subject: [PATCH 094/105] ibid --- common/healthcheck/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index aaacb88d66..6e61e17707 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -25,7 +25,7 @@ sfw-provider-requests = { path = "../../sfw-provider/sfw-provider-requests" } topology = {path = "../topology" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } # sphinx = { path = "../../../sphinx"} [dev-dependencies] From 1ab4e3efeb72ea23175658ccd3f3492e079f7e3f Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 12:09:57 +0000 Subject: [PATCH 095/105] ibid --- sfw-provider/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index f590ece187..3d7e39f7b5 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -28,7 +28,7 @@ directory-client = { path = "../common/clients/directory-client" } sfw-provider-requests = { path = "./sfw-provider-requests" } ## will be moved to proper dependencies once released -sphinx = { git = "https://github.com/nymtech/sphinx", rev="c3ba1447d61d6eb900fcdea0c20b3070a22bf11a" } +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } [build-dependencies] built = "0.3.2" From a55b87809e4be139c166087150f063e4babb56d5 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 12:10:18 +0000 Subject: [PATCH 096/105] encoding: updating all remaining usages of base64 to base58 --- common/crypto/src/identity/mod.rs | 4 ++-- common/healthcheck/src/result.rs | 4 ++-- common/healthcheck/src/score.rs | 6 +++--- nym-client/src/commands/tcpsocket.rs | 2 +- nym-client/src/commands/websocket.rs | 2 +- sfw-provider/src/provider/presence.rs | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/common/crypto/src/identity/mod.rs b/common/crypto/src/identity/mod.rs index 2a580547bf..3a2253813b 100644 --- a/common/crypto/src/identity/mod.rs +++ b/common/crypto/src/identity/mod.rs @@ -100,12 +100,12 @@ impl PemStorable for DummyMixIdentityPublicKey { } impl DummyMixIdentityPublicKey { - pub fn to_b64_string(&self) -> String { + pub fn to_base58_string(&self) -> String { bs58::encode(&self.to_bytes()).into_string() } #[allow(dead_code)] - fn from_b64_string(val: String) -> Self { + fn from_base58_string(val: String) -> Self { Self::from_bytes(&bs58::decode(&val).into_vec().unwrap()) } } diff --git a/common/healthcheck/src/result.rs b/common/healthcheck/src/result.rs index de70873e38..1b0121f5fb 100644 --- a/common/healthcheck/src/result.rs +++ b/common/healthcheck/src/result.rs @@ -61,7 +61,7 @@ impl HealthCheckResult { .mix_nodes() .into_iter() .filter(|node| { - match self.node_score(NodeAddressBytes::from_b64_string(node.pub_key.clone())) { + match self.node_score(NodeAddressBytes::from_base58_string(node.pub_key.clone())) { None => { error!("Unknown node in topology - {:?}", node); false @@ -75,7 +75,7 @@ impl HealthCheckResult { .providers() .into_iter() .filter(|node| { - match self.node_score(NodeAddressBytes::from_b64_string(node.pub_key.clone())) { + match self.node_score(NodeAddressBytes::from_base58_string(node.pub_key.clone())) { None => { error!("Unknown node in topology - {:?}", node); false diff --git a/common/healthcheck/src/score.rs b/common/healthcheck/src/score.rs index 4d29e8f5b6..7262d4ee0d 100644 --- a/common/healthcheck/src/score.rs +++ b/common/healthcheck/src/score.rs @@ -95,7 +95,7 @@ impl NodeScore { pub(crate) fn from_mixnode(node: mix::Node) -> Self { NodeScore { typ: NodeType::Mix, - pub_key: NodeAddressBytes::from_b64_string(node.pub_key), + pub_key: NodeAddressBytes::from_base58_string(node.pub_key), addresses: vec![node.host], version: node.version, layer: format!("layer {}", node.layer), @@ -107,7 +107,7 @@ impl NodeScore { pub(crate) fn from_provider(node: provider::Node) -> Self { NodeScore { typ: NodeType::MixProvider, - pub_key: NodeAddressBytes::from_b64_string(node.pub_key), + pub_key: NodeAddressBytes::from_base58_string(node.pub_key), addresses: vec![node.mixnet_listener, node.client_listener], version: node.version, layer: format!("provider"), @@ -149,7 +149,7 @@ impl std::fmt::Display for NodeScore { return Err(std::fmt::Error); } }; - let stringified_key = self.pub_key.to_b64_string(); + let stringified_key = self.pub_key.to_base58_string(); write!( f, "({})\t{}/{}\t({}%)\t|| {}\tv{} <{}> - {}", diff --git a/nym-client/src/commands/tcpsocket.rs b/nym-client/src/commands/tcpsocket.rs index 87901f5543..a050e78832 100644 --- a/nym-client/src/commands/tcpsocket.rs +++ b/nym-client/src/commands/tcpsocket.rs @@ -32,7 +32,7 @@ pub fn execute(matches: &ArgMatches) { .unwrap(); // TODO: reading auth_token from disk (if exists); - println!("Public key: {}", keypair.public_key.to_b64_string()); + println!("Public key: {}", keypair.public_key.to_base58_string()); let mut temporary_address = [0u8; 32]; let public_key_bytes = keypair.public_key().to_bytes(); diff --git a/nym-client/src/commands/websocket.rs b/nym-client/src/commands/websocket.rs index 5359c10871..82437514a2 100644 --- a/nym-client/src/commands/websocket.rs +++ b/nym-client/src/commands/websocket.rs @@ -33,7 +33,7 @@ pub fn execute(matches: &ArgMatches) { // TODO: reading auth_token from disk (if exists); - println!("Public key: {}", keypair.public_key.to_b64_string()); + println!("Public key: {}", keypair.public_key.to_base58_string()); let mut temporary_address = [0u8; 32]; let public_key_bytes = keypair.public_key().to_bytes(); diff --git a/sfw-provider/src/provider/presence.rs b/sfw-provider/src/provider/presence.rs index d2ec3c1970..adfcebaddc 100644 --- a/sfw-provider/src/provider/presence.rs +++ b/sfw-provider/src/provider/presence.rs @@ -34,7 +34,7 @@ impl Notifier { net_client, client_listener: client_listener.to_string(), mixnet_listener: mixnet_listener.to_string(), - pub_key: pub_key.to_b64_string(), + pub_key: pub_key.to_base58_string(), client_ledger, } } From e5580df77b3f058a0bc3f04a5e8b5790fdff598e Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Mon, 27 Jan 2020 14:59:56 +0000 Subject: [PATCH 097/105] directory-client: removed premature equality check which caused a lock update problem --- .../src/presence/coconodes.rs | 23 ------ .../directory-client/src/presence/mixnodes.rs | 23 ------ .../directory-client/src/presence/mod.rs | 23 ------ .../src/presence/providers.rs | 80 ------------------- common/topology/src/lib.rs | 2 +- nym-client/src/client/topology_control.rs | 21 +---- 6 files changed, 3 insertions(+), 169 deletions(-) diff --git a/common/clients/directory-client/src/presence/coconodes.rs b/common/clients/directory-client/src/presence/coconodes.rs index ff1d6c80f0..e2b61f0771 100644 --- a/common/clients/directory-client/src/presence/coconodes.rs +++ b/common/clients/directory-client/src/presence/coconodes.rs @@ -1,6 +1,4 @@ -use super::PresenceEq; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use topology::coco; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -33,24 +31,3 @@ impl From for CocoPresence { } } } - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration - let self_set: HashSet<_> = self - .iter() - .map(|c| (&c.host, &c.pub_key, &c.version)) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|c| (&c.host, &c.pub_key, &c.version)) - .collect(); - - self_set == other_set - } -} diff --git a/common/clients/directory-client/src/presence/mixnodes.rs b/common/clients/directory-client/src/presence/mixnodes.rs index aba2d73cf9..755c9499a4 100644 --- a/common/clients/directory-client/src/presence/mixnodes.rs +++ b/common/clients/directory-client/src/presence/mixnodes.rs @@ -1,6 +1,4 @@ -use super::PresenceEq; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use std::convert::TryInto; use std::io; use std::net::ToSocketAddrs; @@ -49,24 +47,3 @@ impl From for MixNodePresence { } } } - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration - let self_set: HashSet<_> = self - .iter() - .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|m| (&m.host, &m.pub_key, &m.version, &m.layer)) - .collect(); - - self_set == other_set - } -} diff --git a/common/clients/directory-client/src/presence/mod.rs b/common/clients/directory-client/src/presence/mod.rs index b759f576e7..b595953758 100644 --- a/common/clients/directory-client/src/presence/mod.rs +++ b/common/clients/directory-client/src/presence/mod.rs @@ -12,12 +12,6 @@ pub mod coconodes; pub mod mixnodes; pub mod providers; -// special version of 'PartialEq' that does not care about last_seen field -//(where applicable) or order of elements in vectors -trait PresenceEq { - fn presence_eq(&self, other: &Self) -> bool; -} - // Topology shows us the current state of the overall Nym network #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -27,23 +21,6 @@ pub struct Topology { pub mix_provider_nodes: Vec, } -impl PartialEq for Topology { - // we need a custom implementation as the order of nodes does not matter - // also we do not care about 'last_seen' when comparing topologies - fn eq(&self, other: &Self) -> bool { - if !self.coco_nodes.presence_eq(&other.coco_nodes) - || !self.mix_nodes.presence_eq(&other.mix_nodes) - || !self - .mix_provider_nodes - .presence_eq(&other.mix_provider_nodes) - { - return false; - } - - true - } -} - impl NymTopology for Topology { fn new(directory_server: String) -> Self { debug!("Using directory server: {:?}", directory_server); diff --git a/common/clients/directory-client/src/presence/providers.rs b/common/clients/directory-client/src/presence/providers.rs index b96bcf6a9e..7528accd93 100644 --- a/common/clients/directory-client/src/presence/providers.rs +++ b/common/clients/directory-client/src/presence/providers.rs @@ -1,6 +1,4 @@ -use super::PresenceEq; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use topology::provider; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -14,32 +12,6 @@ pub struct MixProviderPresence { pub version: String, } -impl PresenceEq for MixProviderPresence { - fn presence_eq(&self, other: &Self) -> bool { - if self.registered_clients.len() != other.registered_clients.len() { - return false; - } - - if self.client_listener != other.client_listener - || self.mixnet_listener != other.mixnet_listener - || self.pub_key != other.pub_key - || self.version != other.version - { - return false; - } - - let clients_self_set: HashSet<_> = - self.registered_clients.iter().map(|c| &c.pub_key).collect(); - let clients_other_set: HashSet<_> = other - .registered_clients - .iter() - .map(|c| &c.pub_key) - .collect(); - - clients_self_set == clients_other_set - } -} - impl Into for MixProviderPresence { fn into(self) -> topology::provider::Node { topology::provider::Node { @@ -95,55 +67,3 @@ impl From for MixProviderClient { } } } - -impl PresenceEq for Vec { - fn presence_eq(&self, other: &Self) -> bool { - if self.len() != other.len() { - return false; - } - - // we can't take the whole thing into set as it does not implement 'Eq' and we can't - // derive it as we don't want to take 'last_seen' into consideration. - // We also don't care about order of registered_clients - - // since we're going to be getting rid of this very soon anyway, just clone registered - // clients vector and sort it - - let self_set: HashSet<_> = self - .iter() - .map(|p| { - ( - &p.client_listener, - &p.mixnet_listener, - &p.pub_key, - &p.version, - p.registered_clients - .iter() - .cloned() - .map(|c| c.pub_key) - .collect::>() - .sort(), - ) - }) - .collect(); - let other_set: HashSet<_> = other - .iter() - .map(|p| { - ( - &p.client_listener, - &p.mixnet_listener, - &p.pub_key, - &p.version, - p.registered_clients - .iter() - .cloned() - .map(|c| c.pub_key) - .collect::>() - .sort(), - ) - }) - .collect(); - - self_set == other_set - } -} diff --git a/common/topology/src/lib.rs b/common/topology/src/lib.rs index 1004ef7ca3..3067b30b47 100644 --- a/common/topology/src/lib.rs +++ b/common/topology/src/lib.rs @@ -10,7 +10,7 @@ mod filter; pub mod mix; pub mod provider; -pub trait NymTopology: Sized + PartialEq + std::fmt::Debug + Send + Sync { +pub trait NymTopology: Sized + std::fmt::Debug + Send + Sync { fn new(directory_server: String) -> Self; fn new_from_nodes( mix_nodes: Vec, diff --git a/nym-client/src/client/topology_control.rs b/nym-client/src/client/topology_control.rs index 6f749682dd..426c6be821 100644 --- a/nym-client/src/client/topology_control.rs +++ b/nym-client/src/client/topology_control.rs @@ -58,7 +58,7 @@ impl TopologyControl { let healthcheck_scores = match healthcheck_result { Err(err) => { - error!("Error while performing the healtcheck: {:?}", err); + error!("Error while performing the healthcheck: {:?}", err); return Err(TopologyError::HealthCheckError); } Ok(scores) => scores, @@ -91,18 +91,6 @@ impl TopologyControl { write_lock.topology = new_topology; } - async fn should_update_topology(&mut self, new_topology: &Option) -> bool { - let read_lock = self.inner.read().await; - match new_topology { - // if new topology is invalid, we MUST update to it as it is impossible to send packets through - None => true, - Some(new_topology) => match &read_lock.topology { - None => true, - Some(old_topology) => new_topology != old_topology, - }, - } - } - pub(crate) async fn run_refresher(mut self) { info!("Starting topology refresher"); let delay_duration = Duration::from_secs_f64(self.refresh_rate); @@ -119,12 +107,7 @@ impl TopologyControl { } }; - if self.should_update_topology(&new_topology).await { - info!("Detected changes in topology - updating global view!"); - - self.update_global_topology(new_topology).await; - } - + self.update_global_topology(new_topology).await; tokio::time::delay_for(delay_duration).await; } } From 6168cb727620a6288582bb6d58872f21348e4c05 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:08:12 +0000 Subject: [PATCH 098/105] Decreased logging severity for failing to close read tcp socket --- common/clients/provider-client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/clients/provider-client/src/lib.rs b/common/clients/provider-client/src/lib.rs index 07e924f5af..e24f893ac9 100644 --- a/common/clients/provider-client/src/lib.rs +++ b/common/clients/provider-client/src/lib.rs @@ -75,7 +75,7 @@ impl ProviderClient { let mut response = Vec::new(); socket.read_to_end(&mut response).await?; if let Err(e) = socket.shutdown(Shutdown::Read) { - warn!("failed to close read part of the socket; err = {:?}", e) + debug!("failed to close read part of the socket; err = {:?}. It was probably already closed by the provider", e) } Ok(response) From 4ace90ad8be254f54b4a45c14ddeb3cba61a49f9 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:08:50 +0000 Subject: [PATCH 099/105] Clone requirement on identity keys + 'derive address' method on public key --- common/crypto/src/identity/mod.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/common/crypto/src/identity/mod.rs b/common/crypto/src/identity/mod.rs index 3a2253813b..cf39000e5d 100644 --- a/common/crypto/src/identity/mod.rs +++ b/common/crypto/src/identity/mod.rs @@ -4,8 +4,9 @@ use crate::encryption::{ use crate::{encryption, PemStorable}; use bs58; use curve25519_dalek::scalar::Scalar; +use sphinx::route::DestinationAddressBytes; -pub trait MixnetIdentityKeyPair +pub trait MixnetIdentityKeyPair: Clone where Priv: MixnetIdentityPrivateKey, Pub: MixnetIdentityPublicKey, @@ -19,16 +20,20 @@ where } pub trait MixnetIdentityPublicKey: - Sized + PemStorable + for<'a> From<&'a ::PrivateKeyMaterial> + Sized + + PemStorable + + Clone + + for<'a> From<&'a ::PrivateKeyMaterial> { // we need to couple public and private keys together type PrivateKeyMaterial: MixnetIdentityPrivateKey; + fn derive_address(&self) -> DestinationAddressBytes; fn to_bytes(&self) -> Vec; fn from_bytes(b: &[u8]) -> Self; } -pub trait MixnetIdentityPrivateKey: Sized + PemStorable { +pub trait MixnetIdentityPrivateKey: Sized + PemStorable + Clone { // we need to couple public and private keys together type PublicKeyMaterial: MixnetIdentityPublicKey; @@ -45,7 +50,7 @@ pub trait MixnetIdentityPrivateKey: Sized + PemStorable { // for time being define a dummy identity using x25519 encryption keys (as we've done so far) // and replace it with proper keys, like ed25519 later on - +#[derive(Clone)] pub struct DummyMixIdentityKeyPair { pub private_key: DummyMixIdentityPrivateKey, pub public_key: DummyMixIdentityPublicKey, @@ -84,6 +89,14 @@ pub struct DummyMixIdentityPublicKey(encryption::x25519::PublicKey); impl MixnetIdentityPublicKey for DummyMixIdentityPublicKey { type PrivateKeyMaterial = DummyMixIdentityPrivateKey; + fn derive_address(&self) -> DestinationAddressBytes { + let mut temporary_address = [0u8; 32]; + let public_key_bytes = self.to_bytes(); + temporary_address.copy_from_slice(&public_key_bytes[..]); + + temporary_address + } + fn to_bytes(&self) -> Vec { self.0.to_bytes() } From c662dad57f70d6a1be942393d6d0677b12c69247 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:10:29 +0000 Subject: [PATCH 100/105] Healthchecker getting keypair from outside rather than generating new one every run --- common/healthcheck/src/lib.rs | 28 +++++++++++++++-- common/healthcheck/src/path_check.rs | 46 ++++++++++++++++++---------- common/healthcheck/src/result.rs | 26 +++++++++++++--- 3 files changed, 76 insertions(+), 24 deletions(-) diff --git a/common/healthcheck/src/lib.rs b/common/healthcheck/src/lib.rs index d2dccfb77e..b95eefd454 100644 --- a/common/healthcheck/src/lib.rs +++ b/common/healthcheck/src/lib.rs @@ -1,8 +1,10 @@ use crate::result::HealthCheckResult; +use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey}; use directory_client::requests::presence_topology_get::PresenceTopologyGetRequester; use directory_client::DirectoryClient; use log::{debug, error, info, trace}; use std::fmt::{Error, Formatter}; +use std::marker::PhantomData; use std::time::Duration; use topology::NymTopologyError; @@ -34,15 +36,29 @@ impl From for HealthCheckerError { } } -pub struct HealthChecker { +pub struct HealthChecker +where + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, +{ directory_client: directory_client::Client, interval: Duration, num_test_packets: usize, resolution_timeout: Duration, + identity_keypair: IDPair, + + _phantom_private: PhantomData, + _phantom_public: PhantomData, } -impl HealthChecker { - pub fn new(config: config::HealthCheck) -> Self { +impl HealthChecker +where + IDPair: crypto::identity::MixnetIdentityKeyPair, + Priv: crypto::identity::MixnetIdentityPrivateKey, + Pub: crypto::identity::MixnetIdentityPublicKey, +{ + pub fn new(config: config::HealthCheck, identity_keypair: IDPair) -> Self { debug!( "healthcheck will be using the following directory server: {:?}", config.directory_server @@ -53,11 +69,16 @@ impl HealthChecker { interval: Duration::from_secs_f64(config.interval), resolution_timeout: Duration::from_secs_f64(config.resolution_timeout), num_test_packets: config.num_test_packets, + identity_keypair, + + _phantom_private: PhantomData, + _phantom_public: PhantomData, } } pub async fn do_check(&self) -> Result { trace!("going to perform a healthcheck!"); + let current_topology = match self.directory_client.presence_topology.get() { Ok(topology) => topology, Err(err) => { @@ -71,6 +92,7 @@ impl HealthChecker { current_topology, self.num_test_packets, self.resolution_timeout, + &self.identity_keypair, ) .await; healthcheck_result.sort_scores(); diff --git a/common/healthcheck/src/path_check.rs b/common/healthcheck/src/path_check.rs index e98666485e..b1d2617ad8 100644 --- a/common/healthcheck/src/path_check.rs +++ b/common/healthcheck/src/path_check.rs @@ -1,8 +1,8 @@ -use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey}; use itertools::Itertools; use log::{debug, error, info, trace, warn}; use mix_client::MixClient; -use provider_client::ProviderClient; +use provider_client::{ProviderClient, ProviderClientError}; use sphinx::header::delays::Delay; use sphinx::route::{Destination, Node as SphinxNode}; use std::collections::HashMap; @@ -23,28 +23,36 @@ pub(crate) struct PathChecker { layer_one_clients: HashMap<[u8; 32], Option>, paths_status: HashMap, PathStatus>, our_destination: Destination, + check_id: [u8; 16], } impl PathChecker { - pub(crate) async fn new( + pub(crate) async fn new( providers: Vec, - ephemeral_keys: DummyMixIdentityKeyPair, - ) -> Self { + identity_keys: &IDPair, + check_id: [u8; 16], + ) -> Self + where + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, + { let mut provider_clients = HashMap::new(); - let mut temporary_address = [0u8; 32]; - let public_key_bytes = ephemeral_keys.public_key().to_bytes(); - temporary_address.copy_from_slice(&public_key_bytes[..]); + let address = identity_keys.public_key().derive_address(); for provider in providers { - let mut provider_client = - ProviderClient::new(provider.client_listener, temporary_address, None); + let mut provider_client = ProviderClient::new(provider.client_listener, address, None); let insertion_result = match provider_client.register().await { Ok(token) => { debug!("registered at provider {}", provider.pub_key); provider_client.update_token(token); provider_clients.insert(provider.get_pub_key_bytes(), Some(provider_client)) } + Err(ProviderClientError::ClientAlreadyRegisteredError) => { + info!("We were already registered"); + provider_clients.insert(provider.get_pub_key_bytes(), Some(provider_client)) + } Err(err) => { warn!( "failed to register at provider {} - {:?}", @@ -62,15 +70,19 @@ impl PathChecker { PathChecker { provider_clients, layer_one_clients: HashMap::new(), - our_destination: Destination::new(temporary_address, Default::default()), + our_destination: Destination::new(address, Default::default()), paths_status: HashMap::new(), + check_id, } } // iteration is used to distinguish packets sent through the same path (as the healthcheck // may try to send say 10 packets through given path) - fn unique_path_key(path: &Vec, iteration: u8) -> Vec { - std::iter::once(iteration) + fn unique_path_key(path: &Vec, check_id: [u8; 16], iteration: u8) -> Vec { + check_id + .iter() + .cloned() + .chain(std::iter::once(iteration)) .chain( path.iter() .map(|node| node.pub_key.to_bytes().to_vec()) @@ -80,10 +92,10 @@ impl PathChecker { } pub(crate) fn path_key_to_node_keys(path_key: Vec) -> Vec<[u8; 32]> { - assert_eq!(path_key.len() % 32, 1); + assert_eq!(path_key.len() % 32, 17); path_key .into_iter() - .skip(1) // remove first byte as it represents the iteration number which we do not care about now + .skip(16 + 1) // remove 16 + 1 bytes as it represents check_id and the iteration number which we do not care about now .chunks(32) .into_iter() .map(|key_chunk| { @@ -135,6 +147,8 @@ impl PathChecker { if msg == sfw_provider_requests::DUMMY_MESSAGE_CONTENT { // finish iterating the loop as the messages might not be ordered should_stop = true; + } else if msg[..16] != self.check_id { + warn!("received response from previous healthcheck") } else { provider_messages.push(msg); } @@ -171,7 +185,7 @@ impl PathChecker { } debug!("Checking path: {:?} ({})", path, iteration); - let path_identifier = PathChecker::unique_path_key(path, iteration); + let path_identifier = PathChecker::unique_path_key(path, self.check_id, iteration); // check if there is even any point in sending the packet diff --git a/common/healthcheck/src/result.rs b/common/healthcheck/src/result.rs index 1b0121f5fb..83d1f8778c 100644 --- a/common/healthcheck/src/result.rs +++ b/common/healthcheck/src/result.rs @@ -1,7 +1,8 @@ use crate::path_check::{PathChecker, PathStatus}; use crate::score::NodeScore; -use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair}; +use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey}; use log::{debug, error, info, warn}; +use rand_os::rand_core::RngCore; use sphinx::route::NodeAddressBytes; use std::collections::HashMap; use std::fmt::{Error, Formatter}; @@ -94,15 +95,31 @@ impl HealthCheckResult { ) } - pub async fn calculate( + fn generate_check_id() -> [u8; 16] { + let mut id = [0u8; 16]; + let mut rng = rand_os::OsRng::new().unwrap(); + rng.fill_bytes(&mut id); + id + } + + pub async fn calculate( topology: T, iterations: usize, resolution_timeout: Duration, - ) -> Self { + identity_keys: &IDPair, + ) -> Self + where + T: NymTopology, + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, + { // currently healthchecker supports only up to 255 iterations - if we somehow // find we need more, it's relatively easy change assert!(iterations <= 255); + let check_id = Self::generate_check_id(); + let all_paths = match topology.all_paths() { Ok(paths) => paths, Err(_) => return Self::zero_score(topology), @@ -118,10 +135,9 @@ impl HealthCheckResult { score_map.insert(node.get_pub_key_bytes(), NodeScore::from_provider(node)); }); - let ephemeral_keys = DummyMixIdentityKeyPair::new(); let providers = topology.providers(); - let mut path_checker = PathChecker::new(providers, ephemeral_keys).await; + let mut path_checker = PathChecker::new(providers, identity_keys, check_id).await; for i in 0..iterations { debug!("running healthcheck iteration {} / {}", i + 1, iterations); for path in &all_paths { From 52aede3d9986b83ec452b8ee925f4d561ef7e27b Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:10:53 +0000 Subject: [PATCH 101/105] More generic nym-client with regards to identity keypair --- nym-client/src/client/mod.rs | 54 ++++++++++----- nym-client/src/client/topology_control.rs | 80 +++++++++++++++-------- nym-client/src/commands/tcpsocket.rs | 7 +- nym-client/src/commands/websocket.rs | 7 +- 4 files changed, 94 insertions(+), 54 deletions(-) diff --git a/nym-client/src/client/mod.rs b/nym-client/src/client/mod.rs index c2435acf0d..4d4e0da408 100644 --- a/nym-client/src/client/mod.rs +++ b/nym-client/src/client/mod.rs @@ -3,12 +3,14 @@ use crate::client::received_buffer::ReceivedMessagesBuffer; use crate::client::topology_control::TopologyInnerRef; use crate::sockets::tcp; use crate::sockets::ws; +use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey}; use directory_client::presence::Topology; use futures::channel::mpsc; use futures::join; use log::*; use sfw_provider_requests::AuthToken; -use sphinx::route::{Destination, DestinationAddressBytes}; +use sphinx::route::Destination; +use std::marker::PhantomData; use std::net::SocketAddr; use tokio::runtime::Runtime; use topology::NymTopology; @@ -35,9 +37,13 @@ pub enum SocketType { None, } -pub struct NymClient { - // to be replaced by something else I guess - address: DestinationAddressBytes, +pub struct NymClient +where + IDPair: MixnetIdentityKeyPair + Send + Sync, + Priv: MixnetIdentityPrivateKey + Send + Sync, + Pub: MixnetIdentityPublicKey + Send + Sync, +{ + keypair: IDPair, // to be used by "send" function or socket, etc pub input_tx: mpsc::UnboundedSender, @@ -47,14 +53,22 @@ pub struct NymClient { directory: String, auth_token: Option, socket_type: SocketType, + + _phantom_private: PhantomData, + _phantom_public: PhantomData, } #[derive(Debug)] pub struct InputMessage(pub Destination, pub Vec); -impl NymClient { +impl NymClient +where + IDPair: MixnetIdentityKeyPair + Send + Sync, + Priv: MixnetIdentityPrivateKey + Send + Sync, + Pub: MixnetIdentityPublicKey + Send + Sync, +{ pub fn new( - address: DestinationAddressBytes, + keypair: IDPair, socket_listening_address: SocketAddr, directory: String, auth_token: Option, @@ -63,13 +77,15 @@ impl NymClient { let (input_tx, input_rx) = mpsc::unbounded::(); NymClient { - address, + keypair, input_tx, input_rx, socket_listening_address, directory, auth_token, socket_type, + _phantom_private: PhantomData, + _phantom_public: PhantomData, } } @@ -106,11 +122,18 @@ impl NymClient { let (received_messages_buffer_output_tx, received_messages_buffer_output_rx) = mpsc::unbounded(); + let self_address = self.keypair.public_key().derive_address(); + + // generate same type of keys we have as our identity + let healthcheck_keys = IDPair::new(); + // TODO: when we switch to our graph topology, we need to remember to change 'Topology' type - let topology_controller = rt.block_on(topology_control::TopologyControl::::new( - self.directory.clone(), - TOPOLOGY_REFRESH_RATE, - )); + let topology_controller = + rt.block_on(topology_control::TopologyControl::::new( + self.directory.clone(), + TOPOLOGY_REFRESH_RATE, + healthcheck_keys, + )); let provider_client_listener_address = rt.block_on(self.get_provider_socket_address(topology_controller.get_inner_ref())); @@ -118,7 +141,7 @@ impl NymClient { let mut provider_poller = provider_poller::ProviderPoller::new( poller_input_tx, provider_client_listener_address, - self.address, + self_address, self.auth_token, ); @@ -144,12 +167,11 @@ impl NymClient { let loop_cover_traffic_future = rt.spawn(cover_traffic_stream::start_loop_cover_traffic_stream( mix_tx.clone(), - Destination::new(self.address, Default::default()), + Destination::new(self_address, Default::default()), topology_controller.get_inner_ref(), )); // cloning arguments required by OutQueueControl; required due to move - let self_address = self.address; let input_rx = self.input_rx; let topology_ref = topology_controller.get_inner_ref(); @@ -180,7 +202,7 @@ impl NymClient { self.socket_listening_address, self.input_tx, received_messages_buffer_output_tx, - self.address, + self_address, topology_controller.get_inner_ref(), )); } @@ -189,7 +211,7 @@ impl NymClient { self.socket_listening_address, self.input_tx, received_messages_buffer_output_tx, - self.address, + self_address, topology_controller.get_inner_ref(), )); } diff --git a/nym-client/src/client/topology_control.rs b/nym-client/src/client/topology_control.rs index 426c6be821..9daafc0572 100644 --- a/nym-client/src/client/topology_control.rs +++ b/nym-client/src/client/topology_control.rs @@ -1,4 +1,6 @@ use crate::built_info; +use crypto::identity::{MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey}; +use healthcheck::HealthChecker; use log::{error, info, trace, warn}; use std::sync::Arc; use std::time::Duration; @@ -10,10 +12,17 @@ const NODE_HEALTH_THRESHOLD: f64 = 0.0; // auxiliary type for ease of use pub type TopologyInnerRef = Arc>>; -pub(crate) struct TopologyControl { +pub(crate) struct TopologyControl +where + T: NymTopology, + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, +{ directory_server: String, - refresh_rate: f64, inner: Arc>>, + health_checker: HealthChecker, + refresh_rate: f64, } #[derive(Debug)] @@ -22,40 +31,56 @@ enum TopologyError { NoValidPathsError, } -impl TopologyControl { - pub(crate) async fn new(directory_server: String, refresh_rate: f64) -> Self { - let initial_topology = match Self::get_current_compatible_topology(directory_server.clone()) - .await - { +impl TopologyControl +where + T: NymTopology, + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, +{ + pub(crate) async fn new( + directory_server: String, + refresh_rate: f64, + identity_keypair: IDPair, + ) -> Self { + // topology control run a healthcheck to determine healthy-ish nodes: + // this is a temporary solution as the healthcheck will eventually be moved to validators + + let healthcheck_config = healthcheck::config::HealthCheck { + directory_server: directory_server.clone(), + // those are literally irrelevant when running single check + interval: 100000.0, + resolution_timeout: 5.0, + num_test_packets: 2, + }; + let health_checker = healthcheck::HealthChecker::new(healthcheck_config, identity_keypair); + + let mut topology_control = TopologyControl { + directory_server, + refresh_rate, + inner: Arc::new(FRwLock::new(Inner::new(None))), + health_checker, + }; + + // best effort approach to try to get a valid topology after call to 'new' + let initial_topology = match topology_control.get_current_compatible_topology().await { Ok(topology) => Some(topology), Err(err) => { error!("Initial topology is invalid - {:?}. Right now it will be impossible to send any packets through the mixnet!", err); None } }; + topology_control + .update_global_topology(initial_topology) + .await; - TopologyControl { - directory_server, - refresh_rate, - inner: Arc::new(FRwLock::new(Inner::new(initial_topology))), - } + topology_control } - async fn get_current_compatible_topology(directory_server: String) -> Result { - let full_topology = T::new(directory_server.clone()); - - // run a healthcheck to determine healthy-ish nodes: - // this is a temporary solution as the healthcheck will eventually be moved to validators - let healthcheck_config = healthcheck::config::HealthCheck { - directory_server, - // those are literally irrelevant when running single check - interval: 100000.0, - resolution_timeout: 5.0, - num_test_packets: 2, - }; - let healthcheck = healthcheck::HealthChecker::new(healthcheck_config); - let healthcheck_result = healthcheck.do_check().await; + async fn get_current_compatible_topology(&self) -> Result { + let full_topology = T::new(self.directory_server.clone()); + let healthcheck_result = self.health_checker.do_check().await; let healthcheck_scores = match healthcheck_result { Err(err) => { error!("Error while performing the healthcheck: {:?}", err); @@ -96,8 +121,7 @@ impl TopologyControl { let delay_duration = Duration::from_secs_f64(self.refresh_rate); loop { trace!("Refreshing the topology"); - let new_topology_res = - Self::get_current_compatible_topology(self.directory_server.clone()).await; + let new_topology_res = self.get_current_compatible_topology().await; let new_topology = match new_topology_res { Ok(topology) => Some(topology), diff --git a/nym-client/src/commands/tcpsocket.rs b/nym-client/src/commands/tcpsocket.rs index a050e78832..7a98a91681 100644 --- a/nym-client/src/commands/tcpsocket.rs +++ b/nym-client/src/commands/tcpsocket.rs @@ -1,7 +1,7 @@ use crate::client::{NymClient, SocketType}; use crate::config::persistance::pathfinder::ClientPathfinder; use clap::ArgMatches; -use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use crypto::identity::DummyMixIdentityKeyPair; use pemstore::pemstore::PemStore; use std::net::ToSocketAddrs; @@ -34,12 +34,9 @@ pub fn execute(matches: &ArgMatches) { println!("Public key: {}", keypair.public_key.to_base58_string()); - let mut temporary_address = [0u8; 32]; - let public_key_bytes = keypair.public_key().to_bytes(); - temporary_address.copy_from_slice(&public_key_bytes[..]); let auth_token = None; let client = NymClient::new( - temporary_address, + keypair, socket_address.clone(), directory_server, auth_token, diff --git a/nym-client/src/commands/websocket.rs b/nym-client/src/commands/websocket.rs index 82437514a2..3193c1ffde 100644 --- a/nym-client/src/commands/websocket.rs +++ b/nym-client/src/commands/websocket.rs @@ -1,7 +1,7 @@ use crate::client::{NymClient, SocketType}; use crate::config::persistance::pathfinder::ClientPathfinder; use clap::ArgMatches; -use crypto::identity::{DummyMixIdentityKeyPair, MixnetIdentityKeyPair, MixnetIdentityPublicKey}; +use crypto::identity::DummyMixIdentityKeyPair; use pemstore::pemstore::PemStore; use std::net::ToSocketAddrs; @@ -35,12 +35,9 @@ pub fn execute(matches: &ArgMatches) { println!("Public key: {}", keypair.public_key.to_base58_string()); - let mut temporary_address = [0u8; 32]; - let public_key_bytes = keypair.public_key().to_bytes(); - temporary_address.copy_from_slice(&public_key_bytes[..]); let auth_token = None; let client = NymClient::new( - temporary_address, + keypair, socket_address, directory_server, auth_token, From a9ad9904528b6556b43b49021ff41ea8b172dc84 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:11:09 +0000 Subject: [PATCH 102/105] Missing cargo.toml changes --- common/crypto/Cargo.toml | 3 +++ common/healthcheck/Cargo.toml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/common/crypto/Cargo.toml b/common/crypto/Cargo.toml index 5e4de60816..9cda304ed6 100644 --- a/common/crypto/Cargo.toml +++ b/common/crypto/Cargo.toml @@ -13,3 +13,6 @@ log = "0.4" pretty_env_logger = "0.3" rand = "0.7.2" rand_os = "0.1" + +## will be moved to proper dependencies once released +sphinx = { git = "https://github.com/nymtech/sphinx", rev="8424f4b0933b4a7f64ae828b2edefc5ba43a9e79" } \ No newline at end of file diff --git a/common/healthcheck/Cargo.toml b/common/healthcheck/Cargo.toml index 6e61e17707..f972b56af1 100644 --- a/common/healthcheck/Cargo.toml +++ b/common/healthcheck/Cargo.toml @@ -11,6 +11,8 @@ futures = "0.3.1" itertools = "0.8.2" log = "0.4.8" pretty_env_logger = "0.3" +rand_os = "0.1" +rand = "0.7.2" serde = "1.0.104" serde_derive = "1.0.104" tokio = { version = "0.2", features = ["full"] } From b0e5f7c808101c5c3c72d2d5b35919314b3fbc43 Mon Sep 17 00:00:00 2001 From: Jedrzej Stuczynski Date: Mon, 27 Jan 2020 17:11:28 +0000 Subject: [PATCH 103/105] Validator using new healthcheck + added identity key --- Cargo.lock | 4 ++++ validator/Cargo.toml | 1 + validator/src/validator/mod.rs | 24 ++++++++++++++++++++---- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7566bb6e6..3af9d02bc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,6 +472,7 @@ dependencies = [ "pretty_env_logger", "rand 0.7.3", "rand_os", + "sphinx", ] [[package]] @@ -859,6 +860,8 @@ dependencies = [ "mix-client", "pretty_env_logger", "provider-client", + "rand 0.7.3", + "rand_os", "serde", "serde_derive", "sfw-provider-requests", @@ -1418,6 +1421,7 @@ version = "0.1.0" dependencies = [ "built", "clap", + "crypto", "dotenv", "futures 0.3.1", "healthcheck", diff --git a/validator/Cargo.toml b/validator/Cargo.toml index ead9eade21..6a6e83066c 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -20,6 +20,7 @@ tokio = { version = "0.2", features = ["full"] } toml = "0.5.5" ## internal +crypto = {path = "../common/crypto"} healthcheck = {path = "../common/healthcheck" } [build-dependencies] diff --git a/validator/src/validator/mod.rs b/validator/src/validator/mod.rs index 608b7dd852..6ed56fa42b 100644 --- a/validator/src/validator/mod.rs +++ b/validator/src/validator/mod.rs @@ -1,20 +1,36 @@ use crate::validator::config::Config; +use crypto::identity::{ + DummyMixIdentityKeyPair, DummyMixIdentityPrivateKey, DummyMixIdentityPublicKey, + MixnetIdentityKeyPair, MixnetIdentityPrivateKey, MixnetIdentityPublicKey, +}; use healthcheck::HealthChecker; use log::debug; use tokio::runtime::Runtime; pub mod config; -pub struct Validator { - heath_check: HealthChecker, +// allow for a generic validator +pub struct Validator +where + IDPair: MixnetIdentityKeyPair, + Priv: MixnetIdentityPrivateKey, + Pub: MixnetIdentityPublicKey, +{ + #[allow(dead_code)] + identity_keypair: IDPair, + heath_check: HealthChecker, } -impl Validator { +// but for time being, since it's a dummy one, have it use dummy keys +impl Validator { pub fn new(config: Config) -> Self { debug!("validator new"); + let dummy_keypair = DummyMixIdentityKeyPair::new(); + Validator { - heath_check: HealthChecker::new(config.health_check), + identity_keypair: dummy_keypair.clone(), + heath_check: HealthChecker::new(config.health_check, dummy_keypair), } } From b8fed18c774fcddc24fd4729d462f40fd549c066 Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Tue, 28 Jan 2020 11:50:11 +0000 Subject: [PATCH 104/105] cargo: bumping all binary crate version numbers to 0.4.0-rc.1 --- Cargo.lock | 8 ++++---- mixnode/Cargo.toml | 2 +- nym-client/Cargo.toml | 2 +- sfw-provider/Cargo.toml | 2 +- validator/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3af9d02bc2..14b3f014ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1343,7 +1343,7 @@ dependencies = [ [[package]] name = "nym-client" -version = "0.3.3" +version = "0.4.0-rc.1" dependencies = [ "addressing", "bs58", @@ -1375,7 +1375,7 @@ dependencies = [ [[package]] name = "nym-mixnode" -version = "0.3.3" +version = "0.4.0-rc.1" dependencies = [ "addressing", "bs58", @@ -1393,7 +1393,7 @@ dependencies = [ [[package]] name = "nym-sfw-provider" -version = "0.3.3" +version = "0.4.0-rc.1" dependencies = [ "bs58", "built", @@ -1417,7 +1417,7 @@ dependencies = [ [[package]] name = "nym-validator" -version = "0.1.0" +version = "0.4.0-rc.1" dependencies = [ "built", "clap", diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index 23c9e7324f..a87431cd2e 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -1,7 +1,7 @@ [package] build = "build.rs" name = "nym-mixnode" -version = "0.3.3" +version = "0.4.0-rc.1" authors = ["Dave Hrycyszyn ", "Jędrzej Stuczyński "] edition = "2018" diff --git a/nym-client/Cargo.toml b/nym-client/Cargo.toml index 52f5cb89da..ec184005c2 100644 --- a/nym-client/Cargo.toml +++ b/nym-client/Cargo.toml @@ -1,7 +1,7 @@ [package] build = "build.rs" name = "nym-client" -version = "0.3.3" +version = "0.4.0-rc.1" authors = ["Dave Hrycyszyn ", "Jędrzej Stuczyński "] edition = "2018" diff --git a/sfw-provider/Cargo.toml b/sfw-provider/Cargo.toml index 3d7e39f7b5..5bb8306412 100644 --- a/sfw-provider/Cargo.toml +++ b/sfw-provider/Cargo.toml @@ -1,7 +1,7 @@ [package] build = "build.rs" name = "nym-sfw-provider" -version = "0.3.3" +version = "0.4.0-rc.1" authors = ["Dave Hrycyszyn ", "Jędrzej Stuczyński "] edition = "2018" diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 6a6e83066c..5bf0cf583a 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -1,7 +1,7 @@ [package] build = "build.rs" name = "nym-validator" -version = "0.1.0" +version = "0.4.0-rc.1" authors = ["Jedrzej Stuczynski "] edition = "2018" From 1bc68ddfa1eaa5c6d4b04a50acf6228f2e2d686a Mon Sep 17 00:00:00 2001 From: Dave Hrycyszyn Date: Tue, 28 Jan 2020 12:05:39 +0000 Subject: [PATCH 105/105] changelog: moving to a unified, autogenerated changelog --- CHANGELOG.md | 121 ++++++++++++++++++++++++++++++++++++++ mixnode/CHANGELOG.md | 39 ------------ nym-client/CHANGELOG.md | 41 ------------- sfw-provider/CHANGELOG.md | 38 ------------ validator/CHANGELOG.md | 5 -- 5 files changed, 121 insertions(+), 123 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 mixnode/CHANGELOG.md delete mode 100644 nym-client/CHANGELOG.md delete mode 100644 sfw-provider/CHANGELOG.md delete mode 100644 validator/CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..5dfcddc34a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,121 @@ +# Changelog + +## [Unreleased](https://github.com/nymtech/nym/tree/HEAD) + +[Full Changelog](https://github.com/nymtech/nym/compare/v0.3.3...HEAD) + +**Closed issues:** + +- Check Sphinx packet length in client [\#98](https://github.com/nymtech/nym/issues/98) +- workflow test [\#97](https://github.com/nymtech/nym/issues/97) +- Client SemVer [\#85](https://github.com/nymtech/nym/issues/85) +- Move PemStore [\#74](https://github.com/nymtech/nym/issues/74) +- Periodic client refresh [\#70](https://github.com/nymtech/nym/issues/70) +- Logging [\#68](https://github.com/nymtech/nym/issues/68) +- Nym-client refactor [\#67](https://github.com/nymtech/nym/issues/67) +- Stop panicking! [\#66](https://github.com/nymtech/nym/issues/66) +- Fix Mixnode Panic on Sphinx packet replay [\#65](https://github.com/nymtech/nym/issues/65) +- Convert older code to start using logging [\#35](https://github.com/nymtech/nym/issues/35) +- Convert to non-url-safe base64 \(everywhere\) [\#28](https://github.com/nymtech/nym/issues/28) +- If a thread blows at startup, panic the entire application [\#15](https://github.com/nymtech/nym/issues/15) +- Split provider/mod.rs [\#10](https://github.com/nymtech/nym/issues/10) + +**Merged pull requests:** + +- Feature/health checker with existing keys [\#105](https://github.com/nymtech/nym/pull/105) ([jstuczyn](https://github.com/jstuczyn)) +- Feature/remove topology equality check [\#104](https://github.com/nymtech/nym/pull/104) ([futurechimp](https://github.com/futurechimp)) +- Feature/base58 [\#102](https://github.com/nymtech/nym/pull/102) ([futurechimp](https://github.com/futurechimp)) +- Feature/panic improvements [\#101](https://github.com/nymtech/nym/pull/101) ([jstuczyn](https://github.com/jstuczyn)) +- Feature/fix sphinx unwraps [\#100](https://github.com/nymtech/nym/pull/100) ([futurechimp](https://github.com/futurechimp)) +- Feature/check packet length [\#99](https://github.com/nymtech/nym/pull/99) ([futurechimp](https://github.com/futurechimp)) +- Feature/version filtering improvements [\#96](https://github.com/nymtech/nym/pull/96) ([futurechimp](https://github.com/futurechimp)) +- Feature/refreshing topology [\#94](https://github.com/nymtech/nym/pull/94) ([jstuczyn](https://github.com/jstuczyn)) +- Feature/consistent logging [\#93](https://github.com/nymtech/nym/pull/93) ([futurechimp](https://github.com/futurechimp)) +- Feature/semver client [\#92](https://github.com/nymtech/nym/pull/92) ([futurechimp](https://github.com/futurechimp)) +- Feature/client refactor [\#91](https://github.com/nymtech/nym/pull/91) ([jstuczyn](https://github.com/jstuczyn)) +- Release builds should no longer silently fail - everything will be im… [\#89](https://github.com/nymtech/nym/pull/89) ([jstuczyn](https://github.com/jstuczyn)) + +## [v0.3.3](https://github.com/nymtech/nym/tree/v0.3.3) (2020-01-20) + +[Full Changelog](https://github.com/nymtech/nym/compare/v0.3.2...v0.3.3) + +**Fixed bugs:** + +- Nym client crashing and disconnecting local websocket with complaint about binary data [\#56](https://github.com/nymtech/nym/issues/56) + +**Closed issues:** + +- Websocket text fix [\#64](https://github.com/nymtech/nym/issues/64) +- Restore nym-client lib in crate [\#63](https://github.com/nymtech/nym/issues/63) +- Make websocket not crash on ping or pong messages [\#62](https://github.com/nymtech/nym/issues/62) +- Messages returned by fetch are base64 encoded [\#55](https://github.com/nymtech/nym/issues/55) +- Check layer 1 connectivity at client start [\#38](https://github.com/nymtech/nym/issues/38) +- Check required sfw-provider args [\#27](https://github.com/nymtech/nym/issues/27) +- Make Electron app work with new Rust mixnet client [\#16](https://github.com/nymtech/nym/issues/16) +- Take version numbers into account when picking routes [\#14](https://github.com/nymtech/nym/issues/14) + +**Merged pull requests:** + +- Feature/websocket improvements [\#88](https://github.com/nymtech/nym/pull/88) ([jstuczyn](https://github.com/jstuczyn)) +- Using println rather than log for startup banner, it's not an error [\#87](https://github.com/nymtech/nym/pull/87) ([futurechimp](https://github.com/futurechimp)) +- Feature/nym client lib [\#61](https://github.com/nymtech/nym/pull/61) ([jstuczyn](https://github.com/jstuczyn)) + +## [v0.3.2](https://github.com/nymtech/nym/tree/v0.3.2) (2020-01-17) + +[Full Changelog](https://github.com/nymtech/nym/compare/v0.3.1...v0.3.2) + +**Merged pull requests:** + +- Feature/separate presence address [\#59](https://github.com/nymtech/nym/pull/59) ([jstuczyn](https://github.com/jstuczyn)) + +## [v0.3.1](https://github.com/nymtech/nym/tree/v0.3.1) (2020-01-16) + +[Full Changelog](https://github.com/nymtech/nym/compare/v0.3.0...v0.3.1) + +**Merged pull requests:** + +- Bugfix/presence client crash [\#58](https://github.com/nymtech/nym/pull/58) ([jstuczyn](https://github.com/jstuczyn)) + +## [v0.3.0](https://github.com/nymtech/nym/tree/v0.3.0) (2020-01-14) + +[Full Changelog](https://github.com/nymtech/nym/compare/v0.2.0...v0.3.0) + +**Closed issues:** + +- Error referring to mismatched types caused by topology [\#46](https://github.com/nymtech/nym/issues/46) +- Provider needs to announce two of its addresses \(+ issue of hardcoded port\) [\#32](https://github.com/nymtech/nym/issues/32) +- Port run\_network.sh from old Go mixnet [\#30](https://github.com/nymtech/nym/issues/30) +- Health Checker inside Validator [\#29](https://github.com/nymtech/nym/issues/29) +- Combine Rust repositories into a single master repo containing multiple crates [\#24](https://github.com/nymtech/nym/issues/24) +- Fix the version numbers on provider and mixnode [\#23](https://github.com/nymtech/nym/issues/23) +- Travis releases for Nym [\#22](https://github.com/nymtech/nym/issues/22) +- Add version number to presence [\#4](https://github.com/nymtech/nym/issues/4) +- Add version number to presence [\#1](https://github.com/nymtech/nym/issues/1) + +**Merged pull requests:** + +- Feature/client topology filtering [\#54](https://github.com/nymtech/nym/pull/54) ([jstuczyn](https://github.com/jstuczyn)) +- print public key for nym client tools [\#53](https://github.com/nymtech/nym/pull/53) ([mileschet](https://github.com/mileschet)) +- Showing binding warning on binding to localhost, 0.0.0.0 or 127.0.0.1 [\#52](https://github.com/nymtech/nym/pull/52) ([jstuczyn](https://github.com/jstuczyn)) +- validator: moving sample config files into sample configs directory [\#51](https://github.com/nymtech/nym/pull/51) ([futurechimp](https://github.com/futurechimp)) +- Feature/validator health checker [\#50](https://github.com/nymtech/nym/pull/50) ([jstuczyn](https://github.com/jstuczyn)) +- Improving websocket connection errors [\#49](https://github.com/nymtech/nym/pull/49) ([futurechimp](https://github.com/futurechimp)) +- Feature/crypto crate [\#48](https://github.com/nymtech/nym/pull/48) ([jstuczyn](https://github.com/jstuczyn)) +- Feature/chill log messages [\#43](https://github.com/nymtech/nym/pull/43) ([futurechimp](https://github.com/futurechimp)) +- persistence: improving PEM file reading and parsing failure messages [\#42](https://github.com/nymtech/nym/pull/42) ([futurechimp](https://github.com/futurechimp)) +- Providing a nicer error than "failed on unwrap\(\)" when topology retri… [\#41](https://github.com/nymtech/nym/pull/41) ([futurechimp](https://github.com/futurechimp)) +- Prettying up sfw-provider start sequence a bit. [\#40](https://github.com/nymtech/nym/pull/40) ([futurechimp](https://github.com/futurechimp)) +- Removing the run command from code and documentation [\#39](https://github.com/nymtech/nym/pull/39) ([futurechimp](https://github.com/futurechimp)) +- Feature/script/localnet [\#37](https://github.com/nymtech/nym/pull/37) ([futurechimp](https://github.com/futurechimp)) +- Feature/arguments fix [\#36](https://github.com/nymtech/nym/pull/36) ([jstuczyn](https://github.com/jstuczyn)) +- Feature/crates cleanup [\#34](https://github.com/nymtech/nym/pull/34) ([jstuczyn](https://github.com/jstuczyn)) +- Features/version number to presence [\#25](https://github.com/nymtech/nym/pull/25) ([aniampio](https://github.com/aniampio)) +- Feature/minor client changes [\#7](https://github.com/nymtech/nym/pull/7) ([jstuczyn](https://github.com/jstuczyn)) + +## [v0.2.0](https://github.com/nymtech/nym/tree/v0.2.0) (2020-01-07) + +[Full Changelog](https://github.com/nymtech/nym/compare/3c64a2facd753f4f2f431e7f888e54842e2bc64e...v0.2.0) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/mixnode/CHANGELOG.md b/mixnode/CHANGELOG.md deleted file mode 100644 index 09fcd6f114..0000000000 --- a/mixnode/CHANGELOG.md +++ /dev/null @@ -1,39 +0,0 @@ -# nym-mixnode Changelog - -## 0.3.3 - -* Version increase for consistency with `nym-client` - -## 0.3.2 - -* added separate announce address -* allows announcing dns hostname instead of an ip address - -## 0.3.1 - -* Fixed crash when directory server goes down - -## 0.3.0 - -* cleaned up a lot of internal dependencies -* reporting version to the directory server -* printing warning on trying to bind to "localhost", "127.0.0.1" or "0.0.0.0" -* more informative error messages -* generalised identity keys -* generalised Topology handling -* started slow transition to `log` crate by `nym-client` -* start of 'MixMining' -* start of validator node - -## 0.2.0 - -* removed the `--local` flag -* introduced `--directory` argument to support arbitrary directory servers. Leaving it out will point the node at the "https://directory.nymtech.net" alpha testnet server -* the `host` argument is now required -* IPv6 support -* mixnode version number is now shown at node start -* directory server location is now shown at node start - -## 0.1.0 - Initial Release - -* The bare minimum set of features required by a Nym Mixnode diff --git a/nym-client/CHANGELOG.md b/nym-client/CHANGELOG.md deleted file mode 100644 index 10796961c0..0000000000 --- a/nym-client/CHANGELOG.md +++ /dev/null @@ -1,41 +0,0 @@ -# nym-client Changelog - -## 0.3.3 - -* websocket handling of 'ping', 'pong' and 'close' messages -* websocket not crashing on binary messages -* websocket returning text rather than base64 -* restored `nym-client` lib functionality - -## 0.3.2 - -* allows receiving topology with dns hostname instead of an ip address - -## 0.3.1 - -* Version increase for consistency with `nym-mixnode` and `nym-sfw-provider` - -## 0.3.0 - -* cleaned up a lot of internal dependencies -* reporting version to the directory server -* printing warning on trying to bind to "localhost", "127.0.0.1" or "0.0.0.0" -* more informative error messages -* generalised identity keys -* generalised Topology handling -* started slow transition to `log` crate by `nym-client` -* start of 'MixMining' -* start of validator node - -## 0.2.0 - -* removed the `--local` flag -* introduced `--directory` argument to support arbitrary directory servers. Leaving it out will point the node at the "https://directory.nymtech.net" alpha testnet server -* IPv6 support -* client version number is now shown at node start -* directory server location is now shown at node start -* decrease default delays - -## 0.1.0 - Initial Release - -* The bare minimum set of features required by a Nym Client diff --git a/sfw-provider/CHANGELOG.md b/sfw-provider/CHANGELOG.md deleted file mode 100644 index 4c3c0773be..0000000000 --- a/sfw-provider/CHANGELOG.md +++ /dev/null @@ -1,38 +0,0 @@ -# nym-sfw-provider Changelog - -## 0.3.3 - -* Version increase for consistency with `nym-client` - -## 0.3.2 - -* Version increase for consistency with `nym-mixnode` and `nym-client` - -## 0.3.1 - -* Fixed crash when directory server goes down - -## 0.3.0 - -* cleaned up a lot of internal dependencies -* reporting version to the directory server -* printing warning on trying to bind to "localhost", "127.0.0.1" or "0.0.0.0" -* more informative error messages -* generalised identity keys -* generalised Topology handling -* started slow transition to `log` crate by `nym-client` -* start of 'MixMining' -* start of validator node - - -## 0.2.0 - -* removed the `--local` flag -* introduced `--directory` argument to support arbitrary directory servers. Leaving it out will point the node at the "https://directory.nymtech.net" alpha testnet server -* IPv6 support -* provider version number is now shown at node start -* directory server location is now shown at node start - -## 0.1.0 - Initial Release - -* The bare minimum set of features required by a Nym Store and Forward Provider diff --git a/validator/CHANGELOG.md b/validator/CHANGELOG.md deleted file mode 100644 index 35f6449c19..0000000000 --- a/validator/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# nym-validator Changelog - -## 0.1.0 - Initial Release - -* Initial take on the health checker that is going to be run by the Nym Validator