From 14961d231ecedeb8fd036db703ff9e7389b96112 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Fri, 3 Nov 2023 12:46:55 +0100 Subject: [PATCH 1/9] - add checks around message lengths --- wasm/client/src/encoded_payload_helper.rs | 66 ++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/wasm/client/src/encoded_payload_helper.rs b/wasm/client/src/encoded_payload_helper.rs index 17ae92fbf8..0f980288d5 100644 --- a/wasm/client/src/encoded_payload_helper.rs +++ b/wasm/client/src/encoded_payload_helper.rs @@ -86,15 +86,25 @@ pub fn decode_payload(message: Vec) -> Result { pub(crate) fn parse_payload(message: &[u8]) -> anyhow::Result<(EncodedPayloadMetadata, &[u8])> { // 1st 8 bytes are the size (as u64 big endian) let mut size = [0u8; 8]; + if message.len() < 8 { + return Err(anyhow::anyhow!( + "Message is too short to contain size information" + )); + } size.clone_from_slice(&message[0..8]); - let size = u64::from_be_bytes(size) as usize; + let metadata_size = u64::from_be_bytes(size) as usize; - // then the metadata - let metadata = String::from_utf8_lossy(&message[8..8 + size]).into_owned(); - let metadata: EncodedPayloadMetadata = serde_json::from_str(metadata.as_str())?; + if metadata_size + 8 > message.len() { + return Err(anyhow::anyhow!( + "Metadata size with prefix exceeds message length" + )); + } - // finally the payload - let payload = &message[8 + size..]; + //then the metadata + let metadata = String::from_utf8_lossy(&message[8..8 + metadata_size]).into_owned(); + let metadata: EncodedPayloadMetadata = serde_json::from_str(&metadata)?; + + let payload = &message[8 + metadata_size..]; Ok((metadata, payload)) } @@ -235,4 +245,48 @@ mod tests { assert_eq!(res.0.headers, None); assert_eq!(res.1, empty); } + + #[wasm_bindgen_test] + async fn test_parse_payload_too_short() { + let message = vec![0u8; 7]; + let result = parse_payload(&message); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(error.to_string(), "message is too short to contain size information"); + } + + #[wasm_bindgen_test] + async fn test_parse_payload_size_exceeds_length() { + let mut message = vec![0u8; 8]; + message.extend(vec![1u8; 10]); + message[0..8].copy_from_slice(&(20u64.to_be_bytes())); + + let result = parse_payload(&message); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(error.to_string(), "metadata size with prefix exceeds message length"); + } + + #[wasm_bindgen_test] + async fn test_parse_payload_valid() { + let metadata = EncodedPayloadMetadata { + mime_type: "text/plain".to_string(), + headers: Some("test headers".to_string()), + }; + let payload_data = vec![2u8, 3u8, 5u8]; + + let serialized_metadata = serde_json::to_string(&metadata).unwrap(); + let metadata_length = serialized_metadata.len() as u64; + + let mut message = metadata_length.to_be_bytes().to_vec(); + message.extend_from_slice(serialized_metadata.as_bytes()); + message.extend_from_slice(&payload_data); + + let (parsed_metadata, parsed_payload) = parse_payload(&message).unwrap(); + + assert_eq!(parsed_metadata.mime_type, metadata.mime_type); + assert!(parsed_metadata.headers.is_some()); + assert_eq!(parsed_metadata.headers.unwrap(), "test headers"); + assert_eq!(parsed_payload, payload_data.as_slice()); + } } From 41bbbed70493d724ebac689acb8baac6fe1fa2c2 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Fri, 3 Nov 2023 12:49:35 +0100 Subject: [PATCH 2/9] keep notes --- wasm/client/src/encoded_payload_helper.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/wasm/client/src/encoded_payload_helper.rs b/wasm/client/src/encoded_payload_helper.rs index 0f980288d5..9f98636ece 100644 --- a/wasm/client/src/encoded_payload_helper.rs +++ b/wasm/client/src/encoded_payload_helper.rs @@ -104,6 +104,7 @@ pub(crate) fn parse_payload(message: &[u8]) -> anyhow::Result<(EncodedPayloadMet let metadata = String::from_utf8_lossy(&message[8..8 + metadata_size]).into_owned(); let metadata: EncodedPayloadMetadata = serde_json::from_str(&metadata)?; + //finally the payload let payload = &message[8 + metadata_size..]; Ok((metadata, payload)) From 91f383d5aced819feaeed236d8ec670822880223 Mon Sep 17 00:00:00 2001 From: benedettadavico Date: Tue, 7 Nov 2023 07:58:27 +0100 Subject: [PATCH 3/9] Bump mixnode version and update changelog --- CHANGELOG.md | 14 ++++++++++++++ Cargo.lock | 2 +- mixnode/Cargo.toml | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04209d93f7..554b9bcb3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https:// ## [Unreleased] +## [2023.4-galaxy] (2023-11-07) + +- DRY up client cli ([#4077]) +- [mixnode] replace rocket with axum ([#4071]) +- incorporate the nym node HTTP api into the mixnode ([#4070]) +- replaced '--disable-sign-ext' with '--signext-lowering' when running wasm-opt ([#3896]) + +[#4077]: https://github.com/nymtech/nym/pull/4077 +[#4071]: https://github.com/nymtech/nym/pull/4071 +[#4070]: https://github.com/nymtech/nym/issues/4070 +[#3896]: https://github.com/nymtech/nym/pull/3896 + ## [2023.3-kinder] (2023-10-31) - suppress error output ([#4056]) @@ -15,6 +27,7 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https:// - use saturating sub in case outfox is not enabled ([#3986]) - Fix sorting for mixnodes and gateways ([#3985]) - Gateway client registry and api routes ([#3955]) +- Feature/configurable socks5 bind address ([#3992]) [#4056]: https://github.com/nymtech/nym/pull/4056 [#4042]: https://github.com/nymtech/nym/pull/4042 @@ -25,6 +38,7 @@ Post 1.0.0 release, the changelog format is based on [Keep a Changelog](https:// [#3986]: https://github.com/nymtech/nym/pull/3986 [#3985]: https://github.com/nymtech/nym/pull/3985 [#3955]: https://github.com/nymtech/nym/pull/3955 +[#3992]: https://github.com/nymtech/nym/pull/3992 ## [2023.1-milka] (2023-09-24) diff --git a/Cargo.lock b/Cargo.lock index 4190432be9..943d8d6ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6703,7 +6703,7 @@ dependencies = [ [[package]] name = "nym-mixnode" -version = "1.1.32" +version = "1.1.33" dependencies = [ "anyhow", "axum", diff --git a/mixnode/Cargo.toml b/mixnode/Cargo.toml index 2c8636f94b..872ca55b92 100644 --- a/mixnode/Cargo.toml +++ b/mixnode/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-mixnode" -version = "1.1.32" +version = "1.1.33" authors = [ "Dave Hrycyszyn ", "Jędrzej Stuczyński ", From 40b2729a018e2cd4aa42ca71e03e16c7df41df25 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Tue, 7 Nov 2023 11:19:57 +0100 Subject: [PATCH 4/9] attempt fixing parseBody - why? when using the mixfetch SDK, i was encountering issues, when posting requests to specific endpoints. It was not parsing the response correctly with: Error: panic:syscall/js: Value.Call: property getReader is not a function By updating the above, i've tested this works on all variations of post and get request using mixfetch. Locally I had to upgrade my version of go to 1.20 --- .../internal/jstypes/conv/request.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go b/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go index 27621e83ae..81d46fee49 100644 --- a/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go +++ b/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go @@ -223,13 +223,20 @@ func parseHeaders(headers js.Value, reqOpts types.RequestOptions, method string) func parseBody(request *js.Value) (io.Reader, error) { jsBody := request.Get(fieldRequestBody) var bodyReader io.Reader + if !jsBody.IsUndefined() && !jsBody.IsNull() { - log.Debug("stream body - getReader") - bodyReader = external.NewStreamReader(jsBody.Call("getReader")) + // Check to see if getReader is a function + if jsBody.InstanceOf(js.Global().Get("ReadableStream")) && jsBody.Get("getReader").Type() == js.TypeFunction { + log.Debug("stream body - getReader") + bodyReader = external.NewStreamReader(jsBody.Call("getReader")) + } else { + // Fall back to using ArrayBuffer + // https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer + log.Debug("getReader not available - fallback to ArrayBuffer") + bodyReader = external.NewArrayReader(request.Call("arrayBuffer")) + } } else { - log.Debug("unstremable body - fallback to ArrayBuffer") - // Fall back to using ArrayBuffer - // https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer + log.Debug("unstreamable body - fallback to ArrayBuffer") bodyReader = external.NewArrayReader(request.Call("arrayBuffer")) } @@ -237,9 +244,11 @@ func parseBody(request *js.Value) (io.Reader, error) { if err != nil { return nil, err } + // Leaving historical notes as reference points // TODO: that seems super awkward. we're constructing a reader only to fully consume it // and create it all over again so that we the recipient wouldn't complain about content-length // surely there must be a better way? + return bytes.NewReader(bodyBytes), nil } From 1f83b6f4e88249892f043c1d04bce595a99849d2 Mon Sep 17 00:00:00 2001 From: Tommy Verrall <60836166+tommyv1987@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:38:06 +0100 Subject: [PATCH 5/9] Update request.go with PR comments --- .../go-mix-conn/internal/jstypes/conv/request.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go b/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go index 81d46fee49..883ce82724 100644 --- a/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go +++ b/wasm/mix-fetch/go-mix-conn/internal/jstypes/conv/request.go @@ -224,22 +224,14 @@ func parseBody(request *js.Value) (io.Reader, error) { jsBody := request.Get(fieldRequestBody) var bodyReader io.Reader - if !jsBody.IsUndefined() && !jsBody.IsNull() { + if jsBody.InstanceOf(js.Global().Get("ReadableStream")) && jsBody.Get("getReader").Type() == js.TypeFunction { // Check to see if getReader is a function - if jsBody.InstanceOf(js.Global().Get("ReadableStream")) && jsBody.Get("getReader").Type() == js.TypeFunction { - log.Debug("stream body - getReader") - bodyReader = external.NewStreamReader(jsBody.Call("getReader")) - } else { - // Fall back to using ArrayBuffer - // https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer - log.Debug("getReader not available - fallback to ArrayBuffer") - bodyReader = external.NewArrayReader(request.Call("arrayBuffer")) - } + log.Debug("stream body - getReader") + bodyReader = external.NewStreamReader(jsBody.Call("getReader")) } else { log.Debug("unstreamable body - fallback to ArrayBuffer") bodyReader = external.NewArrayReader(request.Call("arrayBuffer")) } - bodyBytes, err := io.ReadAll(bodyReader) if err != nil { return nil, err From b6ccab79d2dd1bec01954f9f0c47ebffe3b638f6 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Tue, 7 Nov 2023 13:18:25 +0100 Subject: [PATCH 6/9] pr comments - update based on comments --- wasm/client/src/encoded_payload_helper.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/wasm/client/src/encoded_payload_helper.rs b/wasm/client/src/encoded_payload_helper.rs index 9f98636ece..22267f5506 100644 --- a/wasm/client/src/encoded_payload_helper.rs +++ b/wasm/client/src/encoded_payload_helper.rs @@ -87,22 +87,19 @@ pub(crate) fn parse_payload(message: &[u8]) -> anyhow::Result<(EncodedPayloadMet // 1st 8 bytes are the size (as u64 big endian) let mut size = [0u8; 8]; if message.len() < 8 { - return Err(anyhow::anyhow!( - "Message is too short to contain size information" - )); + bail!("Message is too short to contain size information") } size.clone_from_slice(&message[0..8]); let metadata_size = u64::from_be_bytes(size) as usize; if metadata_size + 8 > message.len() { return Err(anyhow::anyhow!( - "Metadata size with prefix exceeds message length" + format!("Metadata size: {}, exceeds message with length of: {}", metadata_size, message.len()), )); } //then the metadata - let metadata = String::from_utf8_lossy(&message[8..8 + metadata_size]).into_owned(); - let metadata: EncodedPayloadMetadata = serde_json::from_str(&metadata)?; + let metadata: EncodedPayloadMetadata = serde_json::from_slice(&message[8..8 + metadata_size])?; //finally the payload let payload = &message[8 + metadata_size..]; @@ -253,7 +250,7 @@ mod tests { let result = parse_payload(&message); assert!(result.is_err()); let error = result.unwrap_err(); - assert_eq!(error.to_string(), "message is too short to contain size information"); + assert_eq!(error.to_string(), "Message is too short to contain size information"); } #[wasm_bindgen_test] @@ -265,7 +262,7 @@ mod tests { let result = parse_payload(&message); assert!(result.is_err()); let error = result.unwrap_err(); - assert_eq!(error.to_string(), "metadata size with prefix exceeds message length"); + assert_eq!(error.to_string(), "Metadata size: 20, exceeds message with length of: 18"); } #[wasm_bindgen_test] From 189fd0ece4d28a5d4264d57becfd30990173e18b Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Tue, 7 Nov 2023 13:20:25 +0100 Subject: [PATCH 7/9] insert import --- wasm/client/src/encoded_payload_helper.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/wasm/client/src/encoded_payload_helper.rs b/wasm/client/src/encoded_payload_helper.rs index 22267f5506..300dbbc8f4 100644 --- a/wasm/client/src/encoded_payload_helper.rs +++ b/wasm/client/src/encoded_payload_helper.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; +use anyhow::bail; #[wasm_bindgen(typescript_custom_section)] const TS_DEFS: &'static str = r#" From a0c667927c0504a31a9edf6396a9412a5ce35446 Mon Sep 17 00:00:00 2001 From: Tommy Verrall Date: Tue, 7 Nov 2023 16:21:59 +0100 Subject: [PATCH 8/9] one last change --- wasm/client/src/encoded_payload_helper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/client/src/encoded_payload_helper.rs b/wasm/client/src/encoded_payload_helper.rs index 300dbbc8f4..d0df6edf87 100644 --- a/wasm/client/src/encoded_payload_helper.rs +++ b/wasm/client/src/encoded_payload_helper.rs @@ -93,7 +93,7 @@ pub(crate) fn parse_payload(message: &[u8]) -> anyhow::Result<(EncodedPayloadMet size.clone_from_slice(&message[0..8]); let metadata_size = u64::from_be_bytes(size) as usize; - if metadata_size + 8 > message.len() { + if metadata_size + 8 != message.len() { return Err(anyhow::anyhow!( format!("Metadata size: {}, exceeds message with length of: {}", metadata_size, message.len()), )); From 8f53f095fb60b52359b64e527da5602f62948431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Tue, 7 Nov 2023 15:35:02 +0000 Subject: [PATCH 9/9] Change default http API timeout from 3s to 10s (#4117) --- common/http-api-client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/http-api-client/src/lib.rs b/common/http-api-client/src/lib.rs index c3c4f98f09..ebf186e126 100644 --- a/common/http-api-client/src/lib.rs +++ b/common/http-api-client/src/lib.rs @@ -11,7 +11,7 @@ use thiserror::Error; use tracing::warn; use url::Url; -pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(3); +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); pub type PathSegments<'a> = &'a [&'a str]; pub type Params<'a, K, V> = &'a [(K, V)];