feat: pass custom HTTP client through SDK stack for domain fronting

- Add with_nym_api_client() to BaseClientBuilder, MixnetClientBuilder, and RegistrationClientBuilderConfig

- Modify nym_api_provider to fetch all nodes then filter by supported_roles.entry (fixes metadata inconsistency)

- Update helpers.rs to build HTTP client with all nym_apis URLs and retries for fallback support

- Fix SDK to use entry_capable_nodes() instead of entry_gateways() for broader gateway selection

This enables domain fronting and URL rotation throughout the entire SDK stack, improving censorship resistance and connection reliability. All changes are backward compatible - custom client is optional.
This commit is contained in:
Tommy Verrall
2025-10-17 08:36:23 +02:00
committed by Jędrzej Stuczyński
parent a2856552d8
commit 29cf5058a6
6 changed files with 426 additions and 34 deletions
@@ -199,6 +199,7 @@ pub struct BaseClientBuilder<C, S: MixnetClientStorage> {
custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send>>,
shutdown: Option<ShutdownTracker>,
user_agent: Option<UserAgent>,
custom_nym_api_client: Option<nym_http_api_client::Client>,
setup_method: GatewaySetup,
@@ -227,6 +228,7 @@ where
custom_gateway_transceiver: None,
shutdown: None,
user_agent: None,
custom_nym_api_client: None,
setup_method: GatewaySetup::MustLoad { gateway_id: None },
#[cfg(unix)]
connection_fd_callback: None,
@@ -234,6 +236,12 @@ where
}
}
#[must_use]
pub fn with_nym_api_client(mut self, client: nym_http_api_client::Client) -> Self {
self.custom_nym_api_client = Some(client);
self
}
#[must_use]
pub fn with_derivation_material(
mut self,
@@ -820,7 +828,17 @@ where
fn construct_nym_api_client(
config: &Config,
user_agent: Option<UserAgent>,
custom_client: Option<nym_http_api_client::Client>,
) -> Result<nym_http_api_client::Client, ClientCoreError> {
// If a custom client was provided (e.g., with domain fronting support), use it
if let Some(client) = custom_client {
tracing::info!("Using CUSTOM nym-api HTTP client (with domain fronting support)");
return Ok(client);
}
tracing::warn!("No custom HTTP client provided - creating DEFAULT client from config");
// Otherwise, create a basic client
let mut nym_api_urls = config.get_nym_api_endpoints();
nym_api_urls.shuffle(&mut thread_rng());
@@ -911,7 +929,11 @@ where
.dkg_query_client
.map(|client| BandwidthController::new(credential_store, client));
let nym_api_client = Self::construct_nym_api_client(&self.config, self.user_agent.clone())?;
let nym_api_client = Self::construct_nym_api_client(
&self.config,
self.user_agent.clone(),
self.custom_nym_api_client,
)?;
let key_rotation_config = Self::determine_key_rotation_state(&nym_api_client).await?;
let topology_provider = Self::setup_topology_provider(
@@ -6,6 +6,7 @@ use nym_mixnet_contract_common::EpochRewardedSet;
use nym_topology::provider_trait::{ToTopologyMetadata, TopologyProvider};
use nym_topology::NymTopology;
use nym_validator_client::nym_api::NymApiClientExt;
use nym_validator_client::nym_nodes::NodeRole;
use rand::prelude::SliceRandom;
use rand::thread_rng;
use std::cmp::min;
@@ -135,31 +136,31 @@ impl NymApiTopologyProvider {
} else {
// if we're not using extended topology, we're only getting active set mixnodes and gateways
let mixnodes_fut = self
.validator_client
.get_all_basic_active_mixing_assigned_nodes_with_metadata();
// Get ALL bonded nodes and filter for both mixnodes and gateways
// This ensures consistent metadata between both calls
let all_nodes_fut = self.validator_client.get_all_basic_nodes_with_metadata();
// TODO: we really should be getting ACTIVE gateways only
let gateways_fut = self
.validator_client
.get_all_basic_entry_assigned_nodes_with_metadata();
let (rewarded_set, all_nodes_res) = futures::try_join!(rewarded_set_fut, all_nodes_fut)
.inspect_err(|err| {
error!("failed to get network nodes: {err}");
})
.ok()?;
let (rewarded_set, mixnodes_res, gateways_res) =
futures::try_join!(rewarded_set_fut, mixnodes_fut, gateways_fut)
.inspect_err(|err| {
error!("failed to get network nodes: {err}");
})
.ok()?;
let metadata = all_nodes_res.metadata;
let all_nodes = all_nodes_res.nodes;
let metadata = mixnodes_res.metadata;
let mixnodes = mixnodes_res.nodes;
// Filter for active mixing nodes (role=Mixnode)
let mixnodes: Vec<_> = all_nodes
.iter()
.filter(|n| n.supported_roles.mixnode && matches!(n.role, NodeRole::Mixnode { .. }))
.cloned()
.collect();
if !gateways_res.metadata.consistency_check(&metadata) {
warn!("inconsistent nodes metadata between mixnodes and gateways calls! {metadata:?} and {:?}", gateways_res.metadata);
return None;
}
let gateways = gateways_res.nodes;
// Filter for entry-capable nodes (supported_roles.entry == true)
let gateways: Vec<_> = all_nodes
.into_iter()
.filter(|n| n.supported_roles.entry)
.collect();
debug!(
"there are {} mixnodes and {} gateways in total (before performance filtering)",
@@ -221,3 +222,27 @@ impl TopologyProvider for NymApiTopologyProvider {
Some(topology)
}
}
#[cfg(test)]
mod tests {
#[test]
fn test_filtering_logic() {
// This test verifies the filtering logic used in get_current_compatible_topology
// We test that the role matching pattern works correctly
use nym_validator_client::nym_nodes::NodeRole;
// Test that the pattern matching for mixnodes works
let mixnode_role = NodeRole::Mixnode { layer: 1 };
assert!(
matches!(mixnode_role, NodeRole::Mixnode { .. }),
"Should match mixnode role pattern"
);
// Test that other roles don't match
let entry_role = NodeRole::EntryGateway;
assert!(
!matches!(entry_role, NodeRole::Mixnode { .. }),
"Entry gateway should not match mixnode pattern"
);
}
}
+178 -10
View File
@@ -89,16 +89,22 @@ async fn get_all_basic_entry_nodes_with_metadata(
client: &nym_http_api_client::Client,
use_bincode: bool,
) -> Result<SkimmedNodesWithMetadata, ClientCoreError> {
// Get first page to obtain metadata
// Get ALL nodes (not just entry-assigned) because in some environments (like sandbox),
// nodes may be capable of entry gateway role but not currently assigned to it
let mut page = 0;
let res = client
.get_basic_entry_assigned_nodes_v2(false, Some(page), None, use_bincode)
.get_basic_nodes_v2(false, Some(page), None, use_bincode)
.await?;
let mut nodes = res.nodes.data;
let metadata = res.metadata;
if res.nodes.pagination.total == nodes.len() {
return Ok(SkimmedNodesWithMetadata::new(nodes, metadata));
// Filter for entry-capable nodes (nodes with supported_roles.entry == true)
let entry_nodes: Vec<_> = nodes
.into_iter()
.filter(|n| n.supported_roles.entry)
.collect();
return Ok(SkimmedNodesWithMetadata::new(entry_nodes, metadata));
}
page += 1;
@@ -106,7 +112,7 @@ async fn get_all_basic_entry_nodes_with_metadata(
// Collect remaining pages
loop {
let mut res = client
.get_basic_entry_assigned_nodes_v2(false, Some(page), None, use_bincode)
.get_basic_nodes_v2(false, Some(page), None, use_bincode)
.await?;
if !metadata.consistency_check(&res.metadata) {
@@ -123,7 +129,13 @@ async fn get_all_basic_entry_nodes_with_metadata(
}
}
Ok(SkimmedNodesWithMetadata::new(nodes, metadata))
// Filter for entry-capable nodes (nodes with supported_roles.entry == true)
let entry_nodes: Vec<_> = nodes
.into_iter()
.filter(|n| n.supported_roles.entry)
.collect();
Ok(SkimmedNodesWithMetadata::new(entry_nodes, metadata))
}
impl<'a, G: ConnectableGateway> GatewayWithLatency<'a, G> {
@@ -182,13 +194,41 @@ pub async fn gateways_for_init<R: Rng>(
// filter out gateways below minimum performance and ones that could operate as a mixnode
// (we don't want instability)
let valid_gateways = gateways
let total_fetched = gateways.len();
let after_role_filter_count = gateways
.iter()
.filter(|g| ignore_epoch_roles || !g.supported_roles.mixnode)
.count();
tracing::debug!(
"After role filter: {} (removed {} mixnode-capable)",
after_role_filter_count,
total_fetched - after_role_filter_count
);
let after_performance_filter_count = gateways
.iter()
.filter(|g| ignore_epoch_roles || !g.supported_roles.mixnode)
.filter(|g| g.performance.round_to_integer() >= minimum_performance)
.count();
tracing::debug!(
"After performance filter (>= {}%): {} (removed {} low-performance)",
minimum_performance,
after_performance_filter_count,
after_role_filter_count - after_performance_filter_count
);
let valid_gateways: Vec<RoutingNode> = gateways
.iter()
.filter(|g| ignore_epoch_roles || !g.supported_roles.mixnode)
.filter(|g| g.performance.round_to_integer() >= minimum_performance)
.filter_map(|gateway| gateway.try_into().ok())
.collect::<Vec<_>>();
tracing::debug!("After checking validity: {}", valid_gateways.len());
.collect();
tracing::debug!(
"After conversion to RoutingNode: {} (removed {} invalid)",
valid_gateways.len(),
after_performance_filter_count - valid_gateways.len()
);
tracing::trace!("Valid gateways: {valid_gateways:#?}");
tracing::info!(
@@ -196,6 +236,14 @@ pub async fn gateways_for_init<R: Rng>(
valid_gateways.len()
);
for gw in &valid_gateways {
tracing::debug!(
"Available gateway: {} (node_id: {})",
gw.identity_key.to_base58_string(),
gw.node_id
);
}
Ok(valid_gateways)
}
@@ -355,13 +403,25 @@ pub(super) fn get_specified_gateway(
must_use_tls: bool,
) -> Result<RoutingNode, ClientCoreError> {
tracing::debug!("Requesting specified gateway: {gateway_identity}");
tracing::debug!("Searching in {} available gateways", gateways.len());
for gw in gateways {
tracing::debug!(
"Gateway in list: {} (node_id: {})",
gw.identity_key.to_base58_string(),
gw.node_id
);
}
let user_gateway = ed25519::PublicKey::from_base58_string(gateway_identity)
.map_err(ClientCoreError::UnableToCreatePublicKeyFromGatewayId)?;
let gateway = gateways
.iter()
.find(|gateway| gateway.identity_key == user_gateway)
.ok_or_else(|| ClientCoreError::NoGatewayWithId(gateway_identity.to_string()))?;
.ok_or_else(|| {
tracing::error!("Gateway {gateway_identity} NOT FOUND in available gateways list!");
ClientCoreError::NoGatewayWithId(gateway_identity.to_string())
})?;
let Some(entry_details) = gateway.entry.as_ref() else {
return Err(ClientCoreError::UnsupportedEntry {
@@ -427,7 +487,6 @@ pub(super) async fn register_with_gateway(
#[cfg(test)]
mod tests {
use super::*;
use url::Url;
#[test]
@@ -473,4 +532,113 @@ mod tests {
assert!(nym_api_urls.is_empty(), "Empty list should remain empty");
}
#[test]
fn test_gateway_filtering_logic() {
// NOTE: This test validates the filtering logic in isolation.
// It does NOT test the actual implementation in get_all_basic_entry_nodes_with_metadata
// or gateways_for_init (which would require mocking the HTTP client).
// The real proof is building and running the daemon.
//
// Test the core filtering logic used in gateways_for_init:
// 1. Filter by supported_roles.entry (not by epoch role assignment)
// 2. Filter by performance
//
// This test verifies the fix where nodes have role=Inactive
// but supported_roles.entry=true
#[derive(Debug)]
struct TestNode {
id: u32,
entry_capable: bool, // supported_roles.entry
mixnode_capable: bool, // supported_roles.mixnode
performance: u8, // 0-100
}
let nodes = vec![
// Node 53: entry-capable, good performance
TestNode {
id: 53,
entry_capable: true,
mixnode_capable: false,
performance: 100,
},
// Node 97: entry-capable (but role=Inactive in sandbox), good performance
TestNode {
id: 97,
entry_capable: true,
mixnode_capable: false,
performance: 100,
},
// Node 75: NOT entry-capable (mixnode only)
TestNode {
id: 75,
entry_capable: false,
mixnode_capable: true,
performance: 100,
},
// Node 99: entry-capable but low performance
TestNode {
id: 99,
entry_capable: true,
mixnode_capable: false,
performance: 0,
},
];
let minimum_performance = 50;
let ignore_epoch_roles = true;
// Step 1: Filter by supported_roles.entry (this is what the fix enables)
let entry_capable: Vec<_> = nodes.iter().filter(|n| n.entry_capable).collect();
assert_eq!(
entry_capable.len(),
3,
"Should have 3 entry-capable nodes (53, 97, 99) - this includes Inactive nodes!"
);
// Step 2: Filter by role (exclude mixnode-capable if not ignoring epoch roles)
let after_role_filter: Vec<_> = entry_capable
.iter()
.filter(|g| ignore_epoch_roles || !g.mixnode_capable)
.collect();
assert_eq!(
after_role_filter.len(),
3,
"All entry-capable nodes pass role filter with ignore_epoch_roles=true"
);
// Step 3: Filter by performance
let after_performance_filter: Vec<_> = after_role_filter
.iter()
.filter(|g| g.performance >= minimum_performance)
.collect();
assert_eq!(
after_performance_filter.len(),
2,
"Should have 2 nodes after performance filter (53 and 97, excluding 99 with 0%)"
);
// Verify the correct nodes made it through
let node_ids: Vec<u32> = after_performance_filter.iter().map(|n| n.id).collect();
assert!(
node_ids.contains(&53),
"Node 53 (actively assigned entry gateway) should be included"
);
assert!(
node_ids.contains(&97),
"Node 97 (Inactive but entry-capable) should be included - THIS IS THE FIX!"
);
assert!(
!node_ids.contains(&75),
"Node 75 (mixnode-only, not entry-capable) should be excluded"
);
assert!(
!node_ids.contains(&99),
"Node 99 (low performance) should be excluded"
);
}
}
+1
View File
@@ -22,6 +22,7 @@ nym-authenticator-client = { path = "../nym-authenticator-client" }
nym-bandwidth-controller = { path = "../common/bandwidth-controller" }
nym-credential-storage = { path = "../common/credential-storage" }
nym-credentials-interface = { path = "../common/credentials-interface" }
nym-http-api-client = { path = "../common/http-api-client" }
nym-ip-packet-client = { path = "../nym-ip-packet-client" }
nym-registration-common = { path = "../common/registration" }
nym-sdk = { path = "../sdk/rust/nym-sdk" }
+80 -1
View File
@@ -34,6 +34,7 @@ pub struct BuilderConfig {
pub two_hops: bool,
pub user_agent: UserAgent,
pub custom_topology_provider: Box<dyn TopologyProvider + Send + Sync>,
pub custom_nym_api_client: Option<nym_http_api_client::Client>,
pub network_env: NymNetworkDetails,
pub cancel_token: CancellationToken,
#[cfg(unix)]
@@ -56,6 +57,44 @@ pub struct MixnetClientConfig {
}
impl BuilderConfig {
/// Create a new BuilderConfig without domain fronting support
///
/// For domain fronting support, set `custom_nym_api_client` to Some(client) after creation
#[allow(clippy::too_many_arguments)]
pub fn new(
entry_node: NymNodeWithKeys,
exit_node: NymNodeWithKeys,
data_path: Option<PathBuf>,
mixnet_client_config: MixnetClientConfig,
two_hops: bool,
user_agent: UserAgent,
custom_topology_provider: Box<dyn TopologyProvider + Send + Sync>,
network_env: NymNetworkDetails,
cancel_token: CancellationToken,
#[cfg(unix)] connection_fd_callback: Arc<dyn Fn(RawFd) + Send + Sync>,
) -> Self {
Self {
entry_node,
exit_node,
data_path,
mixnet_client_config,
two_hops,
user_agent,
custom_topology_provider,
custom_nym_api_client: None,
network_env,
cancel_token,
#[cfg(unix)]
connection_fd_callback,
}
}
/// Set a custom nym-api HTTP client (for domain fronting support)
pub fn with_nym_api_client(mut self, client: nym_http_api_client::Client) -> Self {
self.custom_nym_api_client = Some(client);
self
}
pub fn mixnet_client_debug_config(&self) -> DebugConfig {
if self.two_hops {
two_hop_debug_config(&self.mixnet_client_config)
@@ -108,7 +147,7 @@ impl BuilderConfig {
RememberMe::new_mixnet()
};
let builder = builder
let mut builder = builder
.with_user_agent(self.user_agent)
.request_gateway(self.entry_node.node.identity.to_string())
.network_details(self.network_env)
@@ -117,6 +156,13 @@ impl BuilderConfig {
.with_remember_me(remember_me)
.custom_topology_provider(self.custom_topology_provider);
if let Some(nym_api_client) = self.custom_nym_api_client {
tracing::info!("Registration client: Passing custom nym-api HTTP client to SDK");
builder = builder.with_nym_api_client(nym_api_client);
} else {
tracing::warn!("Registration client: No custom nym-api HTTP client provided");
}
#[cfg(unix)]
let builder = builder.with_connection_fd_callback(self.connection_fd_callback);
@@ -206,3 +252,36 @@ fn log_mixnet_client_config(debug_config: &DebugConfig) {
fn true_to_disabled(val: bool) -> &'static str {
if val { "disabled" } else { "enabled" }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_mixnet_client_config_default_values() {
let config = MixnetClientConfig::default();
assert!(!config.disable_poisson_rate);
assert!(!config.disable_background_cover_traffic);
assert_eq!(config.min_mixnode_performance, None);
assert_eq!(config.min_gateway_performance, None);
}
#[test]
fn test_builder_config_has_custom_client_field() {
// Verify that BuilderConfig has the custom_nym_api_client field
// by creating a simple HTTP client
let http_client = nym_http_api_client::Client::builder(
nym_http_api_client::Url::parse("https://validator.nymtech.net/api").unwrap(),
)
.expect("Failed to create client builder")
.build()
.expect("Failed to build client");
// Verify the client works
let urls = http_client.base_urls();
assert!(
!urls.is_empty() && urls[0].as_str().contains("validator.nymtech.net"),
"HTTP client should be configured with correct URL"
);
}
}
+98 -1
View File
@@ -54,6 +54,7 @@ pub struct MixnetClientBuilder<S: MixnetClientStorage = Ephemeral> {
custom_topology_provider: Option<Box<dyn TopologyProvider + Send + Sync>>,
custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send + Sync>>,
custom_shutdown: Option<ShutdownTracker>,
custom_nym_api_client: Option<nym_http_api_client::Client>,
force_tls: bool,
user_agent: Option<UserAgent>,
#[cfg(unix)]
@@ -92,6 +93,7 @@ impl MixnetClientBuilder<OnDiskPersistent> {
socks5_config: None,
wait_for_gateway: false,
custom_topology_provider: None,
custom_nym_api_client: None,
storage: storage_paths
.initialise_default_persistent_storage()
.await?,
@@ -129,6 +131,7 @@ where
wait_for_gateway: false,
custom_topology_provider: None,
custom_gateway_transceiver: None,
custom_nym_api_client: None,
custom_shutdown: None,
force_tls: false,
user_agent: None,
@@ -152,6 +155,7 @@ where
wait_for_gateway: self.wait_for_gateway,
custom_topology_provider: self.custom_topology_provider,
custom_gateway_transceiver: self.custom_gateway_transceiver,
custom_nym_api_client: self.custom_nym_api_client,
custom_shutdown: self.custom_shutdown,
force_tls: self.force_tls,
user_agent: self.user_agent,
@@ -283,6 +287,12 @@ where
self
}
#[must_use]
pub fn with_nym_api_client(mut self, client: nym_http_api_client::Client) -> Self {
self.custom_nym_api_client = Some(client);
self
}
#[must_use]
pub fn with_statistics_reporting(mut self, config: StatsReporting) -> Self {
self.config.debug_config.stats_reporting = config;
@@ -323,6 +333,7 @@ where
client.custom_gateway_transceiver = self.custom_gateway_transceiver;
client.custom_topology_provider = self.custom_topology_provider;
client.custom_nym_api_client = self.custom_nym_api_client;
client.custom_shutdown = self.custom_shutdown;
client.wait_for_gateway = self.wait_for_gateway;
client.force_tls = self.force_tls;
@@ -372,6 +383,9 @@ where
/// advanced usage of custom gateways
custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send + Sync>>,
/// Custom nym-api HTTP client (for domain fronting support)
custom_nym_api_client: Option<nym_http_api_client::Client>,
/// Attempt to wait for the selected gateway (if applicable) to come online if its currently not bonded.
wait_for_gateway: bool,
@@ -440,6 +454,7 @@ where
storage,
custom_topology_provider: None,
custom_gateway_transceiver: None,
custom_nym_api_client: None,
wait_for_gateway: false,
force_tls: false,
custom_shutdown: None,
@@ -535,7 +550,9 @@ where
async fn available_gateways(&mut self) -> Result<Vec<RoutingNode>, ClientCoreError> {
if let Some(ref mut custom_provider) = self.custom_topology_provider {
if let Some(topology) = custom_provider.get_new_topology().await {
return Ok(topology.entry_gateways().cloned().collect());
// Use entry_capable_nodes() instead of entry_gateways() to include
// all entry-capable nodes, not just actively assigned ones
return Ok(topology.entry_capable_nodes().cloned().collect());
}
}
@@ -696,6 +713,13 @@ where
base_builder = base_builder.with_user_agent(user_agent);
}
if let Some(nym_api_client) = self.custom_nym_api_client {
tracing::info!("SDK: Passing custom nym-api HTTP client to BaseClientBuilder");
base_builder = base_builder.with_nym_api_client(nym_api_client);
} else {
tracing::warn!("SDK: No custom nym-api HTTP client configured");
}
if let Some(topology_provider) = self.custom_topology_provider {
base_builder = base_builder.with_topology_provider(topology_provider);
}
@@ -858,3 +882,76 @@ impl IncludedSurbs {
Self::ExposeSelfAddress
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_mixnet_builder_default_no_custom_client() {
let builder = MixnetClientBuilder::new_ephemeral();
assert!(
builder.build().is_ok(),
"Builder should succeed without custom client"
);
}
#[test]
fn test_mixnet_builder_with_custom_client() {
let http_client = nym_http_api_client::Client::builder(
nym_http_api_client::Url::parse("https://validator.nymtech.net/api").unwrap(),
)
.expect("Failed to create client builder")
.build()
.expect("Failed to build client");
let builder = MixnetClientBuilder::new_ephemeral().with_nym_api_client(http_client);
assert!(
builder.build().is_ok(),
"Builder should succeed with custom client"
);
}
// Note: Tests for entry_capable_nodes() vs entry_gateways() are in nym-topology crate
// These tests verify the builder functionality only
#[test]
fn test_custom_client_transfer_through_build() {
let http_client = nym_http_api_client::Client::builder(
nym_http_api_client::Url::parse("https://validator.nymtech.net/api").unwrap(),
)
.expect("Failed to create client builder")
.build()
.expect("Failed to build client");
let builder = MixnetClientBuilder::new_ephemeral().with_nym_api_client(http_client);
let disconnected_client = builder.build();
assert!(
disconnected_client.is_ok(),
"Build should transfer custom_nym_api_client successfully"
);
}
#[test]
fn test_builder_storage_transfer_includes_custom_client() {
use nym_client_core::client::base_client::storage::Ephemeral;
let http_client = nym_http_api_client::Client::builder(
nym_http_api_client::Url::parse("https://validator.nymtech.net/api").unwrap(),
)
.expect("Failed to create client builder")
.build()
.expect("Failed to build client");
let builder = MixnetClientBuilder::new_ephemeral().with_nym_api_client(http_client);
let new_storage = Ephemeral::default();
let builder_with_new_storage = builder.set_storage(new_storage);
assert!(
builder_with_new_storage.build().is_ok(),
"set_storage should preserve custom_nym_api_client"
);
}
}