Add two more sphinx extended packet sizes (8kb and 16kb) (#1694)

* Rename to ExtendedPacketSize32

* Add two more extended packet sizes

* Update config handling for new packet sizes

* Update wasm-client

* Changelog: update

* wasm-client: fix ref

* Switch use enum instead of string for config

CHANGELOG.md

@@ -16,6 +16,7 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https://
 - wasm-client: uses updated wasm-compatible `client-core` so that it's now capable of packet retransmission, cover traffic and poisson delay (among other things!) ([#1673])
 - validator-api: add `interval_operating_cost` and `profit_margin_percent` to cmpute reward estimation endpoint
 - vesting-contract: optional locked token pledge cap per account ([#1687]), defaults to 100_000 NYM
+- clients: add testing-only support for two more extended packet sizes (8kb and 16kb).
 
 ### Fixed
 

clients/client-core/src/config/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use config::NymConfig;
+use nymsphinx::params::PacketSize;
 use serde::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -247,8 +248,8 @@ impl<T: NymConfig> Config<T> {
         self.debug.disable_main_poisson_packet_distribution
     }
 
-    pub fn get_use_extended_packet_size(&self) -> bool {
-        self.debug.use_extended_packet_size
+    pub fn get_use_extended_packet_size(&self) -> Option<ExtendedPacketSize> {
+        self.debug.use_extended_packet_size.clone()
     }
 
     pub fn get_version(&self) -> &str {
@@ -470,8 +471,16 @@ pub struct Debug {
     /// poisson distribution.
     pub disable_main_poisson_packet_distribution: bool,
 
-    /// Controls whether the sent sphinx packet use the NON-DEFAULT bigger size.
-    pub use_extended_packet_size: bool,
+    /// Controls whether the sent sphinx packet use a NON-DEFAULT bigger size.
+    pub use_extended_packet_size: Option<ExtendedPacketSize>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ExtendedPacketSize {
+    Extended8,
+    Extended16,
+    Extended32,
 }
 
 impl Default for Debug {
@@ -488,7 +497,17 @@ impl Default for Debug {
             topology_resolution_timeout: DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT,
             disable_loop_cover_traffic_stream: false,
             disable_main_poisson_packet_distribution: false,
-            use_extended_packet_size: false,
+            use_extended_packet_size: None,
+        }
+    }
+}
+
+impl From<ExtendedPacketSize> for PacketSize {
+    fn from(size: ExtendedPacketSize) -> PacketSize {
+        match size {
+            ExtendedPacketSize::Extended8 => PacketSize::ExtendedPacket8,
+            ExtendedPacketSize::Extended16 => PacketSize::ExtendedPacket16,
+            ExtendedPacketSize::Extended32 => PacketSize::ExtendedPacket32,
         }
     }
 }

clients/native/src/client/mod.rs

@@ -31,7 +31,6 @@ use log::*;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::addressing::nodes::NodeIdentity;
 use nymsphinx::anonymous_replies::ReplySurb;
-use nymsphinx::params::PacketSize;
 use nymsphinx::receiver::ReconstructedMessage;
 use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
 
@@ -103,8 +102,9 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
         }
 
         stream.start_with_shutdown(shutdown);
@@ -132,8 +132,9 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
         }
 
         info!("Starting real traffic stream...");

clients/socks5/src/client/mod.rs

@@ -36,7 +36,6 @@ use gateway_client::{
 use log::*;
 use nymsphinx::addressing::clients::Recipient;
 use nymsphinx::addressing::nodes::NodeIdentity;
-use nymsphinx::params::PacketSize;
 use task::{wait_for_signal, ShutdownListener, ShutdownNotifier};
 
 pub mod config;
@@ -103,8 +102,9 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            stream.set_custom_packet_size(size.into());
         }
 
         stream.start_with_shutdown(shutdown);
@@ -132,8 +132,9 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.get_base().get_use_extended_packet_size() {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = self.config.get_base().get_use_extended_packet_size() {
+            log::debug!("Setting extended packet size: {:?}", size);
+            controller_config.set_custom_packet_size(size.into());
         }
 
         info!("Starting real traffic stream...");

clients/webassembly/src/client/config.rs

@@ -4,7 +4,7 @@
 // due to expansion of #[wasm_bindgen] macro on `Debug` Config struct
 #![allow(clippy::drop_non_drop)]
 
-use client_core::config::{Debug as ConfigDebug, GatewayEndpoint};
+use client_core::config::{Debug as ConfigDebug, ExtendedPacketSize, GatewayEndpoint};
 use std::time::Duration;
 use url::Url;
 use wasm_bindgen::prelude::*;
@@ -107,6 +107,11 @@ pub struct Debug {
 
 impl From<Debug> for ConfigDebug {
     fn from(debug: Debug) -> Self {
+        // For now we just always use the (older) 32kb extended size
+        let use_extended_packet_size = debug
+            .use_extended_packet_size
+            .then(|| ExtendedPacketSize::Extended32);
+
         ConfigDebug {
             average_packet_delay: Duration::from_millis(debug.average_packet_delay_ms),
             average_ack_delay: Duration::from_millis(debug.average_ack_delay_ms),
@@ -126,7 +131,7 @@ impl From<Debug> for ConfigDebug {
             disable_loop_cover_traffic_stream: debug.disable_loop_cover_traffic_stream,
             disable_main_poisson_packet_distribution: debug
                 .disable_main_poisson_packet_distribution,
-            use_extended_packet_size: debug.use_extended_packet_size,
+            use_extended_packet_size,
         }
     }
 }
@@ -148,7 +153,7 @@ impl From<ConfigDebug> for Debug {
             disable_loop_cover_traffic_stream: debug.disable_loop_cover_traffic_stream,
             disable_main_poisson_packet_distribution: debug
                 .disable_main_poisson_packet_distribution,
-            use_extended_packet_size: debug.use_extended_packet_size,
+            use_extended_packet_size: debug.use_extended_packet_size.is_some(),
         }
     }
 }

clients/webassembly/src/client/mod.rs

@@ -22,7 +22,6 @@ use gateway_client::{
     MixnetMessageSender,
 };
 use nymsphinx::addressing::clients::Recipient;
-use nymsphinx::params::PacketSize;
 use rand::rngs::OsRng;
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::spawn_local;
@@ -110,8 +109,8 @@ impl NymClient {
             topology_accessor,
         );
 
-        if self.config.debug.use_extended_packet_size {
-            stream.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = &self.config.debug.use_extended_packet_size {
+            stream.set_custom_packet_size(size.clone().into());
         }
 
         stream.start();
@@ -135,8 +134,8 @@ impl NymClient {
             self.as_mix_recipient(),
         );
 
-        if self.config.debug.use_extended_packet_size {
-            controller_config.set_custom_packet_size(PacketSize::ExtendedPacket)
+        if let Some(size) = &self.config.debug.use_extended_packet_size {
+            controller_config.set_custom_packet_size(size.clone().into());
         }
 
         console_log!("Starting real traffic stream...");

common/client-libs/gateway-client/src/packet_router.rs

@@ -62,9 +62,19 @@ impl PacketRouter {
                 trace!("routing regular packet");
                 received_messages.push(received_packet);
             } else if received_packet.len()
-                == PacketSize::ExtendedPacket.plaintext_size() - ack_overhead
+                == PacketSize::ExtendedPacket8.plaintext_size() - ack_overhead
             {
-                trace!("routing extended packet");
+                trace!("routing extended8 packet");
+                received_messages.push(received_packet);
+            } else if received_packet.len()
+                == PacketSize::ExtendedPacket16.plaintext_size() - ack_overhead
+            {
+                trace!("routing extended16 packet");
+                received_messages.push(received_packet);
+            } else if received_packet.len()
+                == PacketSize::ExtendedPacket32.plaintext_size() - ack_overhead
+            {
+                trace!("routing extended32 packet");
                 received_messages.push(received_packet);
             } else {
                 // this can happen if other clients are not padding their messages

common/mixnode-common/src/packet_processor/processor.rs

@@ -116,7 +116,10 @@ impl SphinxPacketProcessor {
                 trace!("received an ack packet!");
                 Ok((None, data))
             }
-            PacketSize::RegularPacket | PacketSize::ExtendedPacket => {
+            PacketSize::RegularPacket
+            | PacketSize::ExtendedPacket8
+            | PacketSize::ExtendedPacket16
+            | PacketSize::ExtendedPacket32 => {
                 trace!("received a normal packet!");
                 let (ack_data, message) = self.split_hop_data_into_ack_and_message(data)?;
                 let (ack_first_hop, ack_packet) = SurbAck::try_recover_first_hop_packet(&ack_data)?;

common/nymsphinx/forwarding/src/packet.rs

@@ -25,8 +25,10 @@ impl Display for MixPacketFormattingError {
             InvalidPacketSize(actual) =>
                 write!(
                     f,
-                    "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {} (EXTENDED))",
-                    actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(), PacketSize::ExtendedPacket.size()
+                    "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {}, {}, {} (EXTENDED))",
+                    actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(),
+                    PacketSize::ExtendedPacket8.size(), PacketSize::ExtendedPacket16.size(),
+                    PacketSize::ExtendedPacket32.size()
                 ),
             MalformedSphinxPacket => write!(f, "received sphinx packet was malformed"),
             InvalidPacketMode => write!(f, "provided packet mode is invalid")

common/nymsphinx/framing/src/codec.rs

@@ -217,7 +217,9 @@ mod packet_encoding {
             let packet_sizes = vec![
                 PacketSize::AckPacket,
                 PacketSize::RegularPacket,
-                PacketSize::ExtendedPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
             ];
             for packet_size in packet_sizes {
                 let header = Header {
@@ -255,7 +257,9 @@ mod packet_encoding {
             let packet_sizes = vec![
                 PacketSize::AckPacket,
                 PacketSize::RegularPacket,
-                PacketSize::ExtendedPacket,
+                PacketSize::ExtendedPacket8,
+                PacketSize::ExtendedPacket16,
+                PacketSize::ExtendedPacket32,
             ];
 
             for packet_size in packet_sizes {

common/nymsphinx/framing/src/packet.rs

@@ -140,7 +140,9 @@ mod header_encoding {
         let packet_sizes = vec![
             PacketSize::AckPacket,
             PacketSize::RegularPacket,
-            PacketSize::ExtendedPacket,
+            PacketSize::ExtendedPacket8,
+            PacketSize::ExtendedPacket16,
+            PacketSize::ExtendedPacket32,
         ];
         for packet_size in packet_sizes {
             let header = Header {

common/nymsphinx/params/src/packet_sizes.rs

@@ -5,6 +5,7 @@ use crate::FRAG_ID_LEN;
 use nymsphinx_types::header::HEADER_SIZE;
 use nymsphinx_types::PAYLOAD_OVERHEAD_SIZE;
 use std::convert::TryFrom;
+use std::str::FromStr;
 
 // it's up to the smart people to figure those values out : )
 const REGULAR_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 2 * 1024;
@@ -15,11 +16,16 @@ const REGULAR_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 2 * 102
 const ACK_IV_SIZE: usize = 16;
 
 const ACK_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + ACK_IV_SIZE + FRAG_ID_LEN;
-const EXTENDED_PACKET_SIZE: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 32 * 1024;
+const EXTENDED_PACKET_SIZE_8: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 8 * 1024;
+const EXTENDED_PACKET_SIZE_16: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 16 * 1024;
+const EXTENDED_PACKET_SIZE_32: usize = HEADER_SIZE + PAYLOAD_OVERHEAD_SIZE + 32 * 1024;
 
 #[derive(Debug)]
 pub struct InvalidPacketSize;
 
+#[derive(Debug)]
+pub struct InvalidExtendedPacketSize;
+
 #[repr(u8)]
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum PacketSize {
@@ -30,7 +36,28 @@ pub enum PacketSize {
     AckPacket = 2,
 
     // for example for streaming fast and furious in uncompressed 10bit 4K HDR quality
-    ExtendedPacket = 3,
+    ExtendedPacket32 = 3,
+
+    // for example for streaming fast and furious in heavily compressed lossy RealPlayer quality
+    ExtendedPacket8 = 4,
+
+    // for example for streaming fast and furious in compressed XviD quality
+    ExtendedPacket16 = 5,
+}
+
+impl FromStr for PacketSize {
+    type Err = InvalidPacketSize;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "regular" => Ok(Self::RegularPacket),
+            "ack" => Ok(Self::AckPacket),
+            "extended8" => Ok(Self::ExtendedPacket8),
+            "extended16" => Ok(Self::ExtendedPacket16),
+            "extended32" => Ok(Self::ExtendedPacket32),
+            _ => Err(InvalidPacketSize),
+        }
+    }
 }
 
 impl TryFrom<u8> for PacketSize {
@@ -40,7 +67,9 @@ impl TryFrom<u8> for PacketSize {
         match value {
             _ if value == (PacketSize::RegularPacket as u8) => Ok(Self::RegularPacket),
             _ if value == (PacketSize::AckPacket as u8) => Ok(Self::AckPacket),
-            _ if value == (PacketSize::ExtendedPacket as u8) => Ok(Self::ExtendedPacket),
+            _ if value == (PacketSize::ExtendedPacket8 as u8) => Ok(Self::ExtendedPacket8),
+            _ if value == (PacketSize::ExtendedPacket16 as u8) => Ok(Self::ExtendedPacket16),
+            _ if value == (PacketSize::ExtendedPacket32 as u8) => Ok(Self::ExtendedPacket32),
             _ => Err(InvalidPacketSize),
         }
     }
@@ -51,7 +80,9 @@ impl PacketSize {
         match self {
             PacketSize::RegularPacket => REGULAR_PACKET_SIZE,
             PacketSize::AckPacket => ACK_PACKET_SIZE,
-            PacketSize::ExtendedPacket => EXTENDED_PACKET_SIZE,
+            PacketSize::ExtendedPacket8 => EXTENDED_PACKET_SIZE_8,
+            PacketSize::ExtendedPacket16 => EXTENDED_PACKET_SIZE_16,
+            PacketSize::ExtendedPacket32 => EXTENDED_PACKET_SIZE_32,
         }
     }
 
@@ -68,12 +99,33 @@ impl PacketSize {
             Ok(PacketSize::RegularPacket)
         } else if PacketSize::AckPacket.size() == size {
             Ok(PacketSize::AckPacket)
-        } else if PacketSize::ExtendedPacket.size() == size {
-            Ok(PacketSize::ExtendedPacket)
+        } else if PacketSize::ExtendedPacket8.size() == size {
+            Ok(PacketSize::ExtendedPacket8)
+        } else if PacketSize::ExtendedPacket16.size() == size {
+            Ok(PacketSize::ExtendedPacket16)
+        } else if PacketSize::ExtendedPacket32.size() == size {
+            Ok(PacketSize::ExtendedPacket32)
         } else {
             Err(InvalidPacketSize)
         }
     }
+
+    pub fn is_extended_size(&self) -> bool {
+        match self {
+            PacketSize::RegularPacket | PacketSize::AckPacket => false,
+            PacketSize::ExtendedPacket8
+            | PacketSize::ExtendedPacket16
+            | PacketSize::ExtendedPacket32 => true,
+        }
+    }
+
+    pub fn as_extended_size(self) -> Option<Self> {
+        if self.is_extended_size() {
+            Some(self)
+        } else {
+            None
+        }
+    }
 }
 
 impl Default for PacketSize {

gateway/gateway-requests/src/types.rs

@@ -82,8 +82,10 @@ impl fmt::Display for GatewayRequestsError {
             RequestOfInvalidSize(actual) =>
                 write!(
                 f,
-                "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {} (EXTENDED))",
-                actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(), PacketSize::ExtendedPacket.size()
+                "received request had invalid size. (actual: {}, but expected one of: {} (ACK), {} (REGULAR), {}, {}, {} (EXTENDED))",
+                actual, PacketSize::AckPacket.size(), PacketSize::RegularPacket.size(),
+                PacketSize::ExtendedPacket8.size(), PacketSize::ExtendedPacket16.size(),
+                PacketSize::ExtendedPacket32.size()
             ),
             MalformedSphinxPacket => write!(f, "received sphinx packet was malformed"),
             MalformedEncryption => write!(f, "the received encrypted data was malformed"),