GRIN/nym
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HTTP Client Retries, Fallbacks, and Redirects (#5789)

Browse Source 
updates to nym HTTP api client with multiple features relating to request domains
This commit is contained in:
Jack Wampler
2025-05-29 10:37:07 -06:00
committed by GitHub
parent 56ed915626
commit 814ee45b4d
13 changed files with 1080 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -5848,6 +5848,7 @@ dependencies = [
"encoding_rs",
"hickory-resolver",
"http 1.3.1",
"itertools 0.14.0",
"mime",
"nym-bin-common",
"nym-http-api-common",
common/client-libs/validator-client/src/client.rs
+4 -4
View File
@@ -200,11 +200,11 @@ impl<C, S> Client<C, S> {
#[allow(deprecated)]
impl<C, S> Client<C, S> {
pub fn api_url(&self) -> &Url {
self.nym_api.current_url()
self.nym_api.current_url().as_ref()
}
pub fn change_nym_api(&mut self, new_endpoint: Url) {
self.nym_api.change_base_url(new_endpoint)
self.nym_api.change_base_urls(vec![new_endpoint.into()])
}
#[deprecated]
@@ -402,11 +402,11 @@ impl NymApiClient {
}
pub fn api_url(&self) -> &Url {
self.nym_api.current_url()
self.nym_api.current_url().as_ref()
}
pub fn change_nym_api(&mut self, new_endpoint: Url) {
self.nym_api.change_base_url(new_endpoint);
self.nym_api.change_base_urls(vec![new_endpoint.into()]);
}
#[deprecated(note = "use get_all_basic_active_mixing_assigned_nodes instead")]
common/client-libs/validator-client/src/connection_tester.rs
+2 -2
View File
@@ -5,12 +5,12 @@ use crate::{NymApiClient, QueryHttpRpcNyxdClient, ValidatorClientError};
use colored::Colorize;
use core::fmt;
use itertools::Itertools;
use nym_http_api_client::Url;
use nym_network_defaults::NymNetworkDetails;
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::time::Duration;
use tokio::time::timeout;
use url::Url;
const MAX_URLS_TESTED: usize = 200;
const CONNECTION_TEST_TIMEOUT_SEC: u64 = 2;
@@ -88,7 +88,7 @@ fn setup_connection_tests<H: BuildHasher + 'static>(
});
let api_connection_test_clients = api_urls.map(|(network, url)| {
ClientForConnectionTest::Api(network, url.clone(), NymApiClient::new(url))
ClientForConnectionTest::Api(network, url.clone(), NymApiClient::new(url.into()))
});
nyxd_connection_test_clients.chain(api_connection_test_clients)
common/http-api-client/Cargo.toml
+6 -1
View File
@@ -10,10 +10,14 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
default=["tunneling"]
tunneling=[]
[dependencies]
async-trait = { workspace = true }
bincode = { workspace = true }
reqwest = { workspace = true, features = ["json", "gzip", "deflate", "brotli", "zstd"] }
reqwest = { workspace = true, features = ["json", "gzip", "deflate", "brotli", "zstd", "rustls-tls"] }
http.workspace = true
url = { workspace = true }
once_cell = { workspace = true }
@@ -21,6 +25,7 @@ serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
itertools = { workspace = true }
# used for decoding text responses (they were already implicitly included)
bytes = { workspace = true }
common/http-api-client/src/fronted.rs
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: Apache-2.0
//! Utilities for and implementation of request tunneling
use std::sync::atomic::{AtomicBool, Ordering};
use tracing::warn;
use crate::ClientBuilder;
// #[cfg(feature = "tunneling")]
#[derive(Debug)]
pub(crate) struct Front {
pub(crate) policy: FrontPolicy,
enabled: AtomicBool,
}
impl Clone for Front {
fn clone(&self) -> Self {
Self {
policy: self.policy.clone(),
enabled: AtomicBool::new(self.enabled.load(Ordering::Relaxed)),
}
}
}
impl Front {
pub(crate) fn new(policy: FrontPolicy) -> Self {
Self {
enabled: AtomicBool::new(policy == FrontPolicy::Always),
policy,
}
}
pub(crate) fn is_enabled(&self) -> bool {
match self.policy {
FrontPolicy::Off => false,
FrontPolicy::OnRetry => self.enabled.load(Ordering::Relaxed),
FrontPolicy::Always => true,
}
}
// Used to indicate that the client hit an error that should trigger the retry policy
// to enable fronting.
pub(crate) fn retry_enable(&self) {
if self.is_enabled() {
return;
}
if matches!(self.policy, FrontPolicy::OnRetry) {
self.enabled.store(true, Ordering::Relaxed);
}
}
}
#[derive(Debug, Default, PartialEq, Clone)]
#[cfg(feature = "tunneling")]
pub enum FrontPolicy {
Always,
OnRetry,
#[default]
Off,
}
impl ClientBuilder {
/// Enable and configure request tunneling for API requests.
#[cfg(feature = "tunneling")]
pub fn with_fronting(mut self, policy: FrontPolicy) -> Self {
let front = Front::new(policy);
// Check if any of the supplied urls even support fronting
if !self.urls.iter().any(|url| url.has_front()) {
warn!("fronting is enabled, but none of the supplied urls have configured fronting domains");
}
self.front = Some(front);
self
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{ApiClientCore, Url, NO_PARAMS};
#[tokio::test]
async fn nym_api_works() {
let url1 = Url::new(
"https://validator.global.ssl.fastly.net",
Some(vec!["https://yelp.global.ssl.fastly.net"]),
)
.unwrap(); // fastly
// let url2 = Url::new(
// "https://validator.nymtech.net",
// Some(vec!["https://cdn77.com"]),
// ).unwrap(); // cdn77
let client = ClientBuilder::new::<_, &str>(url1)
.expect("bad url")
.with_fronting(FrontPolicy::Always)
.build::<&str>()
.expect("failed to build client");
let response = client
.send_request::<_, (), &str, &str, &str>(
reqwest::Method::GET,
&["api", "v1", "network", "details"],
NO_PARAMS,
None,
)
.await
.expect("failed get request");
// println!("{response:?}");
assert_eq!(response.status(), 200);
}
}
common/http-api-client/src/lib.rs
+412 -304
View File
@@ -136,29 +136,33 @@
//! ```
#![warn(missing_docs)]
pub use reqwest::{IntoUrl, StatusCode};
pub use reqwest::StatusCode;
use crate::path::RequestPath;
use async_trait::async_trait;
use bytes::Bytes;
use http::header::CONTENT_TYPE;
use http::HeaderMap;
use itertools::Itertools;
use mime::Mime;
use reqwest::header::HeaderValue;
use reqwest::{RequestBuilder, Response};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::fmt::Display;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use thiserror::Error;
use tracing::{debug, instrument, warn};
use url::Url;
#[cfg(not(target_arch = "wasm32"))]
use std::net::SocketAddr;
#[cfg(not(target_arch = "wasm32"))]
use std::sync::Arc;
#[cfg(feature = "tunneling")]
mod fronted;
mod url;
pub use url::{IntoUrl, Url};
mod user_agent;
pub use user_agent::UserAgent;
@@ -247,207 +251,6 @@ impl HttpClientError {
}
}
/// A `ClientBuilder` can be used to create a [`Client`] with custom configuration applied consistently
/// and state tracked across subsequent requests.
pub struct ClientBuilder {
url: Url,
timeout: Option<Duration>,
custom_user_agent: bool,
reqwest_client_builder: reqwest::ClientBuilder,
#[allow(dead_code)] // not dead code, just unused in wasm
use_secure_dns: bool,
}
impl ClientBuilder {
/// Constructs a new `ClientBuilder`.
///
/// This is the same as `Client::builder()`.
pub fn new<U, E>(url: U) -> Result<Self, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
let str_url = url.as_str();
// a naive check: if the provided URL does not start with http(s), add that scheme
if !str_url.starts_with("http") {
let alt = format!("http://{str_url}");
warn!("the provided url ('{str_url}') does not contain scheme information. Changing it to '{alt}' ...");
// TODO: or should we maybe default to https?
Self::new(alt)
} else {
Ok(Self::new_with_url(url.into_url()?))
}
}
/// Constructs a new http `ClientBuilder` from a valid url.
pub fn new_with_url(url: Url) -> Self {
if !url.scheme().starts_with("http") {
warn!("the provided url ('{url}') does not use HTTP / HTTPS scheme");
}
#[cfg(target_arch = "wasm32")]
let reqwest_client_builder = reqwest::ClientBuilder::new();
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client_builder = {
// Note: I believe the manual enable calls for the compression methods are extra
// as the various compression features for `reqwest` crate should be enabled
// just by including the feature which:
// `"Enable[s] auto decompression by checking the Content-Encoding response header."`
//
// I am going to leave these here anyways so that removing a decompression method
// from the features list will throw an error if it is not also removed here.
reqwest::ClientBuilder::new()
.gzip(true)
.deflate(true)
.brotli(true)
.zstd(true)
};
ClientBuilder {
url,
timeout: None,
custom_user_agent: false,
reqwest_client_builder,
use_secure_dns: true,
}
}
/// Enables a total request timeout other than the default.
///
/// The timeout is applied from when the request starts connecting until the response body has finished. Also considered a total deadline.
///
/// Default is [`DEFAULT_TIMEOUT`].
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
/// Provide a pre-configured [`reqwest::ClientBuilder`]
pub fn with_reqwest_builder(mut self, reqwest_builder: reqwest::ClientBuilder) -> Self {
self.reqwest_client_builder = reqwest_builder;
self
}
/// Sets the `User-Agent` header to be used by this client.
pub fn with_user_agent<V>(mut self, value: V) -> Self
where
V: TryInto<HeaderValue>,
V::Error: Into<http::Error>,
{
self.custom_user_agent = true;
self.reqwest_client_builder = self.reqwest_client_builder.user_agent(value);
self
}
/// Override DNS resolution for specific domains to particular IP addresses.
///
/// Set the port to `0` to use the conventional port for the given scheme (e.g. 80 for http).
/// Ports in the URL itself will always be used instead of the port in the overridden addr.
#[cfg(not(target_arch = "wasm32"))]
pub fn resolve_to_addrs(mut self, domain: &str, addrs: &[SocketAddr]) -> ClientBuilder {
self.reqwest_client_builder = self.reqwest_client_builder.resolve_to_addrs(domain, addrs);
self
}
/// Returns a Client that uses this ClientBuilder configuration.
pub fn build<E>(self) -> Result<Client, HttpClientError<E>>
where
E: Display,
{
#[cfg(target_arch = "wasm32")]
let reqwest_client = self.reqwest_client_builder.build()?;
// TODO: we should probably be propagating the error rather than panicking,
// but that'd break bunch of things due to type changes
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client = {
let mut builder = self
.reqwest_client_builder
.timeout(self.timeout.unwrap_or(DEFAULT_TIMEOUT));
// if no custom user agent was set, use a default
if !self.custom_user_agent {
builder =
builder.user_agent(format!("nym-http-api-client/{}", env!("CARGO_PKG_VERSION")))
}
// unless explicitly disabled use the DoT/DoH enabled resolver
if self.use_secure_dns {
builder = builder.dns_resolver(Arc::new(HickoryDnsResolver::default()));
}
builder.build()?
};
Ok(Client {
base_url: self.url,
reqwest_client,
#[cfg(target_arch = "wasm32")]
request_timeout: self.timeout.unwrap_or(DEFAULT_TIMEOUT),
})
}
}
/// A simple extendable client wrapper for http request with extra url sanitization.
#[derive(Debug, Clone)]
pub struct Client {
base_url: Url,
reqwest_client: reqwest::Client,
#[cfg(target_arch = "wasm32")]
request_timeout: Duration,
}
impl Client {
/// Create a new http `Client`
// no timeout until https://github.com/seanmonstar/reqwest/issues/1135 is fixed
//
// In order to prevent interference in API requests at the DNS phase we default to a resolver
// that uses DoT and DoH.
pub fn new(base_url: Url, timeout: Option<Duration>) -> Self {
Self::new_url::<_, String>(base_url, timeout).expect(
"we provided valid url and we were unwrapping previous construction errors anyway",
)
}
/// Attempt to create a new http client from a something that can be converted to a URL
pub fn new_url<U, E>(url: U, timeout: Option<Duration>) -> Result<Self, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
let builder = Self::builder(url)?;
match timeout {
Some(timeout) => builder.with_timeout(timeout).build(),
None => builder.build(),
}
}
/// Creates a [`ClientBuilder`] to configure a [`Client`].
///
/// This is the same as [`ClientBuilder::new()`].
pub fn builder<U, E>(url: U) -> Result<ClientBuilder, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
ClientBuilder::new(url)
}
/// Update the host that this client uses when sending API requests.
pub fn change_base_url(&mut self, new_url: Url) {
self.base_url = new_url
}
/// Get the currently configured host that this client uses when sending API requests.
pub fn current_url(&self) -> &Url {
&self.base_url
}
}
/// Core functionality required for types acting as API clients.
///
/// This trait defines the "skinny waist" of behaviors that are required by an API client. More
@@ -548,6 +351,351 @@ pub trait ApiClientCore {
}
}
/// A `ClientBuilder` can be used to create a [`Client`] with custom configuration applied consistently
/// and state tracked across subsequent requests.
pub struct ClientBuilder {
urls: Vec<Url>,
timeout: Option<Duration>,
custom_user_agent: bool,
reqwest_client_builder: reqwest::ClientBuilder,
#[allow(dead_code)] // not dead code, just unused in wasm
use_secure_dns: bool,
#[cfg(feature = "tunneling")]
front: Option<fronted::Front>,
retry_limit: usize,
}
impl ClientBuilder {
/// Constructs a new `ClientBuilder`.
///
/// This is the same as `Client::builder()`.
pub fn new<U, E>(url: U) -> Result<Self, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
let str_url = url.as_str();
// a naive check: if the provided URL does not start with http(s), add that scheme
if !str_url.starts_with("http") {
let alt = format!("http://{str_url}");
warn!("the provided url ('{str_url}') does not contain scheme information. Changing it to '{alt}' ...");
// TODO: or should we maybe default to https?
Self::new(alt)
} else {
let url = url.to_url()?;
Ok(Self::new_with_urls(vec![url]))
}
}
/// Constructs a new http `ClientBuilder` from a valid url.
pub fn new_with_urls(urls: Vec<Url>) -> Self {
let urls = Self::check_urls(urls);
#[cfg(target_arch = "wasm32")]
let reqwest_client_builder = reqwest::ClientBuilder::new();
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client_builder = {
// Note: I believe the manual enable calls for the compression methods are extra
// as the various compression features for `reqwest` crate should be enabled
// just by including the feature which:
// `"Enable[s] auto decompression by checking the Content-Encoding response header."`
//
// I am going to leave these here anyways so that removing a decompression method
// from the features list will throw an error if it is not also removed here.
reqwest::ClientBuilder::new()
.gzip(true)
.deflate(true)
.brotli(true)
.zstd(true)
};
ClientBuilder {
urls,
timeout: None,
custom_user_agent: false,
reqwest_client_builder,
use_secure_dns: true,
#[cfg(feature = "tunneling")]
front: None,
retry_limit: 0,
}
}
/// Add an additional URL to the set usable by this constructed `Client`
pub fn add_url(mut self, url: Url) -> Self {
self.urls.push(url);
self
}
fn check_urls(mut urls: Vec<Url>) -> Vec<Url> {
// remove any duplicate URLs
urls = urls.into_iter().unique().collect();
// warn about any invalid URLs
urls.iter()
.filter(|url| url.scheme().starts_with("http"))
.for_each(|url| {
warn!("the provided url ('{url}') does not use HTTP / HTTPS scheme");
});
urls
}
/// Enables a total request timeout other than the default.
///
/// The timeout is applied from when the request starts connecting until the response body has finished. Also considered a total deadline.
///
/// Default is [`DEFAULT_TIMEOUT`].
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
/// Sets the maximum number of retries for a request. This defaults to 0, indicating no retries.
///
/// Note that setting a retry limit of 3 (for example) will result in 4 attempts to send the
/// request in the case that all are unsuccessful.
///
/// If multiple urls (or fronting configurations if enabled) are available, retried requests
/// will be sent to the next URL in the list.
pub fn with_retries(mut self, retry_limit: usize) -> Self {
self.retry_limit = retry_limit;
self
}
/// Provide a pre-configured [`reqwest::ClientBuilder`]
pub fn with_reqwest_builder(mut self, reqwest_builder: reqwest::ClientBuilder) -> Self {
self.reqwest_client_builder = reqwest_builder;
self
}
/// Sets the `User-Agent` header to be used by this client.
pub fn with_user_agent<V>(mut self, value: V) -> Self
where
V: TryInto<HeaderValue>,
V::Error: Into<http::Error>,
{
self.custom_user_agent = true;
self.reqwest_client_builder = self.reqwest_client_builder.user_agent(value);
self
}
/// Override DNS resolution for specific domains to particular IP addresses.
///
/// Set the port to `0` to use the conventional port for the given scheme (e.g. 80 for http).
/// Ports in the URL itself will always be used instead of the port in the overridden addr.
#[cfg(not(target_arch = "wasm32"))]
pub fn resolve_to_addrs(mut self, domain: &str, addrs: &[SocketAddr]) -> ClientBuilder {
self.reqwest_client_builder = self.reqwest_client_builder.resolve_to_addrs(domain, addrs);
self
}
/// Returns a Client that uses this ClientBuilder configuration.
pub fn build<E>(self) -> Result<Client, HttpClientError<E>>
where
E: Display,
{
#[cfg(target_arch = "wasm32")]
let reqwest_client = self.reqwest_client_builder.build()?;
// TODO: we should probably be propagating the error rather than panicking,
// but that'd break bunch of things due to type changes
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client = {
let mut builder = self
.reqwest_client_builder
.timeout(self.timeout.unwrap_or(DEFAULT_TIMEOUT));
// if no custom user agent was set, use a default
if !self.custom_user_agent {
builder =
builder.user_agent(format!("nym-http-api-client/{}", env!("CARGO_PKG_VERSION")))
}
// unless explicitly disabled use the DoT/DoH enabled resolver
if self.use_secure_dns {
builder = builder.dns_resolver(Arc::new(HickoryDnsResolver::default()));
}
builder.build()?
};
let client = Client {
base_urls: self.urls,
current_idx: Arc::new(AtomicUsize::new(0)),
reqwest_client,
#[cfg(feature = "tunneling")]
front: self.front,
#[cfg(target_arch = "wasm32")]
request_timeout: self.timeout.unwrap_or(DEFAULT_TIMEOUT),
retry_limit: self.retry_limit,
};
Ok(client)
}
}
/// A simple extendable client wrapper for http request with extra url sanitization.
#[derive(Debug, Clone)]
pub struct Client {
base_urls: Vec<Url>,
current_idx: Arc<AtomicUsize>,
reqwest_client: reqwest::Client,
#[cfg(feature = "tunneling")]
front: Option<fronted::Front>,
#[cfg(target_arch = "wasm32")]
request_timeout: Duration,
retry_limit: usize,
}
impl Client {
/// Create a new http `Client`
// no timeout until https://github.com/seanmonstar/reqwest/issues/1135 is fixed
//
// In order to prevent interference in API requests at the DNS phase we default to a resolver
// that uses DoT and DoH.
pub fn new(base_url: ::url::Url, timeout: Option<Duration>) -> Self {
Self::new_url::<_, String>(base_url, timeout).expect(
"we provided valid url and we were unwrapping previous construction errors anyway",
)
}
/// Attempt to create a new http client from a something that can be converted to a URL
pub fn new_url<U, E>(url: U, timeout: Option<Duration>) -> Result<Self, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
let builder = Self::builder(url)?;
match timeout {
Some(timeout) => builder.with_timeout(timeout).build(),
None => builder.build(),
}
}
/// Creates a [`ClientBuilder`] to configure a [`Client`].
///
/// This is the same as [`ClientBuilder::new()`].
pub fn builder<U, E>(url: U) -> Result<ClientBuilder, HttpClientError<E>>
where
U: IntoUrl,
E: Display,
{
ClientBuilder::new(url)
}
/// Update the set of hosts that this client uses when sending API requests.
pub fn change_base_urls(&mut self, new_urls: Vec<Url>) {
self.current_idx.store(0, Ordering::Relaxed);
self.base_urls = new_urls
}
/// Get the currently configured host that this client uses when sending API requests.
pub fn current_url(&self) -> &Url {
&self.base_urls[self.current_idx.load(std::sync::atomic::Ordering::Relaxed)]
}
/// Get the currently configured host that this client uses when sending API requests.
pub fn base_urls(&self) -> &[Url] {
&self.base_urls
}
/// Get a mutable reference to the hosts that this client uses when sending API requests.
pub fn base_urls_mut(&mut self) -> &mut [Url] {
&mut self.base_urls
}
/// Change the currently configured limit on the number of retries for a request.
pub fn change_retry_limit(&mut self, limit: usize) {
self.retry_limit = limit;
}
/// If multiple base urls are available rotate to next (e.g. when the current one resulted in an error)
fn update_host(&self) {
#[cfg(feature = "tunneling")]
if let Some(ref front) = self.front {
if front.is_enabled() {
// if we are using fronting, try updating to the next front
let url = self.current_url();
// try to update the current host to use a next front, if one is available, otherwise
// we move on and try the next base url (if one is available)
if url.has_front() && !url.update() {
// we swapped to the next front for the current host
return;
}
}
}
if self.base_urls.len() > 1 {
let orig = self.current_idx.load(Ordering::Relaxed);
let mut next = (orig + 1) % self.base_urls.len();
// if fronting is enabled we want to update to a host that has fronts configured
#[cfg(feature = "tunneling")]
if let Some(ref front) = self.front {
if front.is_enabled() {
while next != orig {
if self.base_urls[next].has_front() {
// we have a front for the next host, so we can use it
break;
}
next = (next + 1) % self.base_urls.len();
}
}
}
self.current_idx.store(next, Ordering::Relaxed);
}
}
/// Make modifications to the request to apply the current state of this client i.e. the
/// currently configured host. This is required as a caller may use this client to create a
/// request, but then have the state of the client change before the caller uses the client to
/// send their request.
///
/// This enures that the outgoing requests benefit from the configured fallback mechanisms, even
/// for requests that were created before the state of the client changed.
///
/// This method assumes that any updates to the state of the client are made before the call to
/// this method. For example, if the client is configured to rotate hosts after each error, this
/// method should be called after the host has been updated -- i.e. as part of the subsequent
/// send.
fn apply_hosts_to_req(&self, r: &mut reqwest::Request) {
let url = self.current_url();
r.url_mut().set_host(url.host_str()).unwrap();
#[cfg(feature = "tunneling")]
if let Some(ref front) = self.front {
if front.is_enabled() {
// this should never fail as we are transplanting the host from one url to another
r.url_mut().set_host(url.front_str()).unwrap();
let actual_host: HeaderValue = url
.host_str()
.unwrap_or("")
.parse()
.unwrap_or(HeaderValue::from_static(""));
// If the map did have this key present, the new value is associated with the key
// and all previous values are removed. (reqwest HeaderMap docs)
_ = r.headers_mut().insert(reqwest::header::HOST, actual_host);
}
}
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl ApiClientCore for Client {
@@ -565,33 +713,77 @@ impl ApiClientCore for Client {
K: AsRef<str>,
V: AsRef<str>,
{
let url = sanitize_url(&self.base_url, path, params);
let url = self.current_url();
let url = sanitize_url(url, path, params);
let mut request = self.reqwest_client.request(method.clone(), url);
let mut req = reqwest::Request::new(method, url.into());
self.apply_hosts_to_req(&mut req);
let mut rb = RequestBuilder::from_parts(self.reqwest_client.clone(), req);
if let Some(body) = json_body {
request = request.json(body);
rb = rb.json(body);
}
request
rb
}
async fn send<E>(&self, request: RequestBuilder) -> Result<Response, HttpClientError<E>>
where
E: Display,
{
#[cfg(target_arch = "wasm32")]
{
Ok(
wasmtimer::tokio::timeout(self.request_timeout, request.send())
.await
.map_err(|_timeout| HttpClientError::RequestTimeout)??,
)
}
let mut attempts = 0;
loop {
// try_clone may fail if the body is a stream in which case using retries is not advised.
let r = request
.try_clone()
.ok_or(HttpClientError::GenericRequestFailure(
"failed to send request".to_string(),
))?;
#[cfg(not(target_arch = "wasm32"))]
{
Ok(request.send().await?)
// apply any changes based on the current state of the client wrt. hosts,
// fronting domains, etc.
let mut req = r.build()?;
self.apply_hosts_to_req(&mut req);
#[cfg(target_arch = "wasm32")]
let response: Result<Response, HttpClientError<E>> = {
Ok(wasmtimer::tokio::timeout(
self.request_timeout,
self.reqwest_client.execute(req),
)
.await
.map_err(|_timeout| HttpClientError::RequestTimeout)??)
};
#[cfg(not(target_arch = "wasm32"))]
let response = self.reqwest_client.execute(req).await;
match response {
Ok(resp) => return Ok(resp),
Err(e) => {
// if we have multiple urls, update to the next
self.update_host();
#[cfg(feature = "tunneling")]
if let Some(ref front) = self.front {
// If fronting is set to be enabled on error, enable domain fronting as we
// have encountered an error.
front.retry_enable();
}
if attempts < self.retry_limit {
warn!("Retrying request due to http error: {}", e);
attempts += 1;
continue;
}
// if we have exhausted our attempts, return the error
#[allow(clippy::useless_conversion)] // conversion considered useless in wasm
return Err(e.into());
}
}
}
}
}
@@ -1049,88 +1241,4 @@ fn try_get_mime_type(headers: &HeaderMap) -> Option<Mime> {
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitizing_urls() {
let base_url: Url = "http://foomp.com".parse().unwrap();
// works with a full string
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, "/foo//bar/", NO_PARAMS).as_str()
);
// (and leading slash doesn't matter)
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, "foo//bar/", NO_PARAMS).as_str()
);
// works with 1 segment
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["foo"], NO_PARAMS).as_str()
);
// works with 2 segments
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "bar"], NO_PARAMS).as_str()
);
// works with leading slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["/foo"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["/foo", "bar"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "/bar"], NO_PARAMS).as_str()
);
// works with trailing slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["foo/"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo/", "bar"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "bar/"], NO_PARAMS).as_str()
);
// works with both leading and trailing slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["/foo/"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["/foo/", "/bar/"], NO_PARAMS).as_str()
);
// adds params
assert_eq!(
"http://foomp.com/foo/bar?foomp=baz",
sanitize_url(&base_url, &["foo", "bar"], &[("foomp", "baz")]).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar?arg1=val1&arg2=val2",
sanitize_url(
&base_url,
&["/foo/", "/bar/"],
&[("arg1", "val1"), ("arg2", "val2")]
)
.as_str()
);
}
}
mod tests;
common/http-api-client/src/tests.rs
+224
View File
@@ -0,0 +1,224 @@
use super::*;
#[test]
fn sanitizing_urls() {
let base_url: Url = "http://foomp.com".parse().unwrap();
// works with a full string
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, "/foo//bar/", NO_PARAMS).as_str()
);
// (and leading slash doesn't matter)
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, "foo//bar/", NO_PARAMS).as_str()
);
// works with 1 segment
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["foo"], NO_PARAMS).as_str()
);
// works with 2 segments
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "bar"], NO_PARAMS).as_str()
);
// works with leading slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["/foo"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["/foo", "bar"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "/bar"], NO_PARAMS).as_str()
);
// works with trailing slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["foo/"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo/", "bar"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["foo", "bar/"], NO_PARAMS).as_str()
);
// works with both leading and trailing slash
assert_eq!(
"http://foomp.com/foo",
sanitize_url(&base_url, &["/foo/"], NO_PARAMS).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar",
sanitize_url(&base_url, &["/foo/", "/bar/"], NO_PARAMS).as_str()
);
// adds params
assert_eq!(
"http://foomp.com/foo/bar?foomp=baz",
sanitize_url(&base_url, &["foo", "bar"], &[("foomp", "baz")]).as_str()
);
assert_eq!(
"http://foomp.com/foo/bar?arg1=val1&arg2=val2",
sanitize_url(
&base_url,
&["/foo/", "/bar/"],
&[("arg1", "val1"), ("arg2", "val2")]
)
.as_str()
);
}
// - Do the retries work
// - Do we use fallback urls on retry if multiple are provided
// - Do we use the next front on retry if multiple are provided
// - If we have more retries than urls, do we wrap back to the first one again
// - on error without retries is where we have multiple urls, is the url updated?
#[tokio::test]
async fn api_client_retry() -> Result<(), Box<dyn std::error::Error>> {
let client = ClientBuilder::new_with_urls(vec![
"http://broken.nym.badurl".parse()?,
"http://example.com/".parse()?,
])
.with_retries(3)
.build::<HttpClientError>()?;
let req = client.create_get_request(&["/"], NO_PARAMS);
let resp = client.send::<HttpClientError>(req).await?;
assert_eq!(resp.status(), 200);
// check that the url was updated
assert_eq!(client.current_url().as_str(), "http://example.com/");
Ok(())
}
#[test]
fn host_updating() {
let url = Url::new("http://example.com", None).unwrap();
let mut client = ClientBuilder::new::<_, &str>(url)
.unwrap()
.build::<&str>()
.unwrap();
// check that the url is set correctly
let current_url = client.current_url();
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), None);
// update the url
client.update_host();
// check that the url is still the same since there is one URL
assert_eq!(client.current_url().as_str(), "http://example.com/");
// =======================================
// we rotate through urls when available
let new_urls = vec![
Url::new("http://example.com", None).unwrap(),
Url::new("http://example.org", None).unwrap(),
];
client.change_base_urls(new_urls);
assert_eq!(client.current_url().as_str(), "http://example.com/");
client.update_host();
// check that the url got updated now that there are multiple URLs
assert_eq!(client.current_url().as_str(), "http://example.org/");
assert_eq!(client.current_url().front_str(), None);
client.update_host();
assert_eq!(client.current_url().as_str(), "http://example.com/");
// =======================================
// we rotate through urls when available if fronting is disabled
let new_urls = vec![
Url::new(
"http://example.com",
Some(vec!["http://front1.com", "http://front2.com"]),
)
.unwrap(),
Url::new("http://example.org", None).unwrap(),
];
client.change_base_urls(new_urls);
assert_eq!(client.current_url().as_str(), "http://example.com/");
client.update_host();
// check that the url got updated now that there are multiple URLs
assert_eq!(client.current_url().as_str(), "http://example.org/");
}
#[test]
#[cfg(feature = "tunneling")]
fn fronted_host_updating() {
let url = Url::new("http://example.com", Some(vec!["http://front1.com"])).unwrap();
let mut client = ClientBuilder::new::<_, &str>(url)
.unwrap()
.with_fronting(crate::fronted::FrontPolicy::Always)
.build::<&str>()
.unwrap();
// check that the url is set correctly
let current_url = client.current_url();
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), Some("front1.com"));
// update the url
client.update_host();
// check that the url is still the same since there is one URL and one front
let current_url = client.current_url();
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), Some("front1.com"));
// =======================================
// we rotate through front urls when available if fronting is enabled
let new_urls = vec![
Url::new(
"http://example.com",
Some(vec!["http://front1.com", "http://front2.com"]),
)
.unwrap(),
Url::new("http://example.org", None).unwrap(),
];
client.change_base_urls(new_urls);
let current_url = client.current_url();
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), Some("front1.com"));
// update the url - this should keep the same host but change the front
client.update_host();
let current_url = client.current_url();
// check that the url is still the same since there is one URL
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), Some("front2.com"));
// update the url - this should wrap around to the first front as the second url is not fronted
client.update_host();
let current_url = client.current_url();
assert_eq!(current_url.as_str(), "http://example.com/");
assert_eq!(current_url.front_str(), Some("front1.com"));
}
common/http-api-client/src/url.rs
+291
View File
@@ -0,0 +1,291 @@
//! Url handling for the HTTP API client.
//!
//! This module provides a `Url` struct that wraps around the `url::Url` type and adds
//! functionality for handling front domains, which are used for reverse proxying.
use std::fmt::Display;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use itertools::Itertools;
use url::form_urlencoded;
pub use url::ParseError;
/// A trait to try to convert some type into a `Url`.
pub trait IntoUrl {
/// Parse as a valid `Url`
fn to_url(self) -> Result<Url, ParseError>;
/// Returns the string representation of the URL.
fn as_str(&self) -> &str;
}
impl IntoUrl for &str {
fn to_url(self) -> Result<Url, ParseError> {
let url = url::Url::parse(self)?;
Ok(url.into())
}
fn as_str(&self) -> &str {
self
}
}
impl IntoUrl for String {
fn to_url(self) -> Result<Url, ParseError> {
let url = url::Url::parse(&self)?;
Ok(url.into())
}
fn as_str(&self) -> &str {
self
}
}
impl IntoUrl for reqwest::Url {
fn to_url(self) -> Result<Url, ParseError> {
Ok(self.into())
}
fn as_str(&self) -> &str {
self.as_str()
}
}
/// When configuring fronting, some configurations will require a specific backend host
/// to be used for the request to be properly reverse proxied.
#[derive(Debug, Clone)]
pub struct Url {
url: url::Url,
fronts: Option<Vec<url::Url>>,
current_front: Arc<AtomicUsize>,
}
impl IntoUrl for Url {
fn to_url(self) -> Result<Url, ParseError> {
Ok(self)
}
fn as_str(&self) -> &str {
self.url.as_str()
}
}
impl PartialEq for Url {
fn eq(&self, other: &Self) -> bool {
let current = self.current_front.load(Ordering::Relaxed);
let other_current = other.current_front.load(Ordering::Relaxed);
self.fronts == other.fronts && self.url == other.url && current == other_current
}
}
impl Eq for Url {}
impl std::hash::Hash for Url {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
let current = self.current_front.load(Ordering::Relaxed);
self.fronts.hash(state);
self.url.hash(state);
current.hash(state);
}
}
impl Display for Url {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.fronts {
Some(ref fronts) => {
let current = self.current_front.load(Ordering::Relaxed);
if let Some(front) = fronts.get(current) {
write!(f, "{front}=>{}", self.url)
} else {
write!(f, "{}", self.url)
}
}
None => write!(f, "{}", self.url),
}
}
}
impl From<Url> for url::Url {
fn from(val: Url) -> Self {
val.url
}
}
impl From<reqwest::Url> for Url {
fn from(url: url::Url) -> Self {
Self {
url,
fronts: None,
current_front: Arc::new(AtomicUsize::new(0)),
}
}
}
impl AsRef<url::Url> for Url {
fn as_ref(&self) -> &url::Url {
&self.url
}
}
impl AsMut<url::Url> for Url {
fn as_mut(&mut self) -> &mut url::Url {
&mut self.url
}
}
impl std::str::FromStr for Url {
type Err = url::ParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let url = url::Url::parse(s)?;
Ok(Self {
url,
fronts: None,
current_front: Arc::new(AtomicUsize::new(0)),
})
}
}
impl Url {
/// Create a new `Url` instance with the given something that can be parsed as a URL and
/// optional tunneling domains
pub fn new<U: reqwest::IntoUrl>(
url: U,
fronts: Option<Vec<U>>,
) -> Result<Self, reqwest::Error> {
let mut url = Self {
url: url.into_url()?,
fronts: None,
current_front: Arc::new(AtomicUsize::new(0)),
};
// ensure that the provided URLs are valid
if let Some(front_domains) = fronts {
let f: Vec<reqwest::Url> = front_domains
.into_iter()
.map(|front| front.into_url())
.try_collect()?;
url.fronts = Some(f);
}
Ok(url)
}
/// Parse an absolute URL from a string.
pub fn parse(s: &str) -> Result<Self, ParseError> {
let url = url::Url::parse(s)?;
Ok(Self {
url,
fronts: None,
current_front: Arc::new(AtomicUsize::new(0)),
})
}
/// Returns true if the URL has a front domain set
pub fn has_front(&self) -> bool {
if let Some(fronts) = &self.fronts {
return !fronts.is_empty();
}
false
}
/// Return the string representation of the current front host (domain or IP address) for this
/// URL, if any.
pub fn front_str(&self) -> Option<&str> {
let current = self.current_front.load(Ordering::Relaxed);
self.fronts
.as_ref()
.and_then(|fronts| fronts.get(current))
.and_then(|url| url.host_str())
}
/// Return the string representation of the host (domain or IP address) for this URL, if any.
pub fn host_str(&self) -> Option<&str> {
self.url.host_str()
}
/// Return the serialization of this URL.
///
/// This is fast since that serialization is already stored in the inner url::Url struct.
pub fn as_str(&self) -> &str {
self.url.as_str()
}
/// Returns true if updating the front wraps back to the first front, or if no fronts are set
pub fn update(&self) -> bool {
if let Some(fronts) = &self.fronts {
if fronts.len() > 1 {
let current = self.current_front.load(Ordering::Relaxed);
let next = (current + 1) % fronts.len();
self.current_front.store(next, Ordering::Relaxed);
return next == 0;
}
}
true
}
/// Return the scheme of this URL, lower-cased, as an ASCII string without the ‘:’ delimiter.
pub fn scheme(&self) -> &str {
self.url.scheme()
}
/// Parse the URL’s query string, if any, as application/x-www-form-urlencoded and return an
/// iterator of (key, value) pairs.
pub fn query_pairs(&self) -> form_urlencoded::Parse<'_> {
self.url.query_pairs()
}
/// Manipulate this URL’s query string, viewed as a sequence of name/value pairs in
/// application/x-www-form-urlencoded syntax.
pub fn query_pairs_mut(&mut self) -> form_urlencoded::Serializer<'_, ::url::UrlQuery<'_>> {
self.url.query_pairs_mut()
}
/// Change this URL’s query string. If `query` is `None`, this URL’s query string will be cleared.
pub fn set_query(&mut self, query: Option<&str>) {
self.url.set_query(query);
}
/// Change this URL’s path.
pub fn set_path(&mut self, path: &str) {
self.url.set_path(path);
}
/// Change this URL’s scheme.
pub fn set_scheme(&mut self, scheme: &str) {
self.url.set_scheme(scheme).unwrap();
}
/// Change this URL’s host.
///
/// Removing the host (calling this with None) will also remove any username, password, and port number.
pub fn set_host(&mut self, host: &str) {
self.url.set_host(Some(host)).unwrap();
}
/// Change this URL’s port number.
///
/// Note that default port numbers are not reflected in the serialization.
///
/// If this URL is cannot-be-a-base, does not have a host, or has the `file` scheme; do nothing and return `Err`.
pub fn set_port(&mut self, port: u16) {
self.url.set_port(Some(port)).unwrap();
}
/// Return an object with methods to manipulate this URL’s path segments.
///
/// Return Err(()) if this URL is cannot-be-a-base.
pub fn path_segments(&self) -> Option<std::str::Split<'_, char>> {
self.url.path_segments()
}
/// Return an object with methods to manipulate this URL’s path segments.
///
/// Return Err(()) if this URL is cannot-be-a-base.
#[allow(clippy::result_unit_err)]
pub fn path_segments_mut(&mut self) -> Result<::url::PathSegmentsMut<'_>, ()> {
self.url.path_segments_mut()
}
}
nym-credential-proxy/nym-credential-proxy-requests/src/client.rs
+2 -1
View File
@@ -26,8 +26,9 @@ pub fn new_client(
base_url: impl IntoUrl,
bearer_token: impl Into<String>,
) -> Result<VpnApiClient, VpnApiClientError> {
let url = base_url.into_url()?;
Ok(VpnApiClient {
inner: Client::builder(base_url)?
inner: Client::builder(url)?
.with_user_agent(format!(
"nym-credential-proxy-requests/{}",
env!("CARGO_PKG_VERSION")
nym-node-status-api/nym-node-status-api/src/monitor/mod.rs
+5 -4
View File
@@ -106,10 +106,11 @@ impl Monitor {
.clone()
.expect("rust sdk mainnet default missing api_url");
let nym_api = nym_http_api_client::ClientBuilder::new_with_url(default_api_url)
.no_hickory_dns()
.with_timeout(self.nym_api_client_timeout)
.build::<&str>()?;
let nym_api =
nym_http_api_client::ClientBuilder::new_with_urls(vec![default_api_url.into()])
.no_hickory_dns()
.with_timeout(self.nym_api_client_timeout)
.build::<&str>()?;
let api_client = NymApiClient::from(nym_api);
nym-node-status-api/nym-node-status-api/src/node_scraper/mod.rs
+1 -1
View File
@@ -57,7 +57,7 @@ async fn run(
.clone()
.expect("rust sdk mainnet default missing api_url");
let nym_api = nym_http_api_client::ClientBuilder::new_with_url(default_api_url)
let nym_api = nym_http_api_client::ClientBuilder::new_with_urls(vec![default_api_url.into()])
.no_hickory_dns()
.with_timeout(nym_api_client_timeout)
.build::<&str>()?;
nym-node/src/node/mod.rs
+13 -12
View File
@@ -829,18 +829,19 @@ impl NymNode {
for nym_api_url in &self.config.mixnet.nym_api_urls {
info!("trying {nym_api_url}...");
let nym_api =
match nym_http_api_client::ClientBuilder::new_with_url(nym_api_url.clone())
.no_hickory_dns()
.with_user_agent(self.user_agent())
.build::<&str>()
{
Ok(b) => b,
Err(e) => {
warn!("failed to build http client for \"{nym_api_url}\": {e}",);
continue;
}
};
let nym_api = match nym_http_api_client::ClientBuilder::new_with_urls(vec![nym_api_url
.clone()
.into()])
.no_hickory_dns()
.with_user_agent(self.user_agent())
.build::<&str>()
{
Ok(b) => b,
Err(e) => {
warn!("failed to build http client for \"{nym_api_url}\": {e}",);
continue;
}
};
let client = NymApiClient::from(nym_api);
wasm/zknym-lib/src/vpn_api_client/client.rs
+1 -2
View File
@@ -8,8 +8,7 @@ use crate::vpn_api_client::types::{
};
use async_trait::async_trait;
pub use nym_http_api_client::Client;
use nym_http_api_client::{parse_response, ApiClient, PathSegments, NO_PARAMS};
use reqwest::IntoUrl;
use nym_http_api_client::{parse_response, ApiClient, IntoUrl, PathSegments, NO_PARAMS};
use serde::de::DeserializeOwned;
#[allow(dead_code)]