Feature/force refresh node (#5101)

* introduced nym-api endpoint for force refreshing described node data

* client code + updated return types

* nym-node to update self-described data cache on startup + change request type

* send request to all available nym-apis

* fixed 'is_stale' check
This commit is contained in:
Jędrzej Stuczyński
2024-11-06 09:17:44 +00:00
committed by GitHub
parent fd8dc63c88
commit c001059af9
15 changed files with 327 additions and 33 deletions
Generated
+1
View File
@@ -5906,6 +5906,7 @@ dependencies = [
"nym-sphinx-addressing",
"nym-task",
"nym-types",
"nym-validator-client",
"nym-wireguard",
"nym-wireguard-types",
"rand",
@@ -25,12 +25,12 @@ use nym_api_requests::models::{LegacyDescribedGateway, MixNodeBondAnnotated};
use nym_api_requests::nym_nodes::SkimmedNode;
use nym_coconut_dkg_common::types::EpochId;
use nym_http_api_client::UserAgent;
use nym_mixnet_contract_common::NymNodeDetails;
use nym_network_defaults::NymNetworkDetails;
use time::Date;
use url::Url;
pub use crate::nym_api::NymApiClientExt;
use nym_mixnet_contract_common::NymNodeDetails;
pub use nym_mixnet_contract_common::{
mixnode::MixNodeDetails, GatewayBond, IdentityKey, IdentityKeyRef, NodeId,
};
@@ -330,10 +330,10 @@ impl NymApiClient {
NymApiClient { nym_api }
}
pub fn new_with_user_agent(api_url: Url, user_agent: UserAgent) -> Self {
pub fn new_with_user_agent(api_url: Url, user_agent: impl Into<UserAgent>) -> Self {
let nym_api = nym_api::Client::builder::<_, ValidatorClientError>(api_url)
.expect("invalid api url")
.with_user_agent(user_agent)
.with_user_agent(user_agent.into())
.build::<ValidatorClientError>()
.expect("failed to build nym api client");
@@ -12,7 +12,7 @@ use nym_api_requests::ecash::models::{
use nym_api_requests::ecash::VerificationKeyResponse;
use nym_api_requests::models::{
AnnotationResponse, ApiHealthResponse, LegacyDescribedMixNode, NodePerformanceResponse,
NymNodeDescription,
NodeRefreshBody, NymNodeDescription,
};
use nym_api_requests::nym_nodes::PaginatedCachedNodesResponse;
use nym_api_requests::pagination::PaginatedResponse;
@@ -934,6 +934,18 @@ pub trait NymApiClientExt: ApiClient {
.await
}
async fn force_refresh_describe_cache(
&self,
request: &NodeRefreshBody,
) -> Result<(), NymAPIError> {
self.post_json(
&[routes::API_VERSION, "nym-nodes", "refresh-described"],
NO_PARAMS,
request,
)
.await
}
#[instrument(level = "debug", skip(self))]
async fn epoch_credentials(
&self,
+62
View File
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
use crate::helpers::unix_epoch;
use crate::helpers::PlaceholderJsonSchemaImpl;
use crate::legacy::{
LegacyGatewayBondWithId, LegacyMixNodeBondWithLayer, LegacyMixNodeDetailsWithLayer,
};
@@ -1143,6 +1144,67 @@ pub struct NoiseDetails {
pub ip_addresses: Vec<IpAddr>,
}
#[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)]
pub struct NodeRefreshBody {
#[serde(with = "bs58_ed25519_pubkey")]
#[schemars(with = "String")]
pub node_identity: ed25519::PublicKey,
// a poor man's nonce
pub request_timestamp: i64,
#[schemars(with = "PlaceholderJsonSchemaImpl")]
pub signature: ed25519::Signature,
}
impl NodeRefreshBody {
pub fn plaintext(node_identity: ed25519::PublicKey, request_timestamp: i64) -> Vec<u8> {
node_identity
.to_bytes()
.into_iter()
.chain(request_timestamp.to_be_bytes())
.chain(b"describe-cache-refresh-request".iter().copied())
.collect()
}
pub fn new(private_key: &ed25519::PrivateKey) -> Self {
let node_identity = private_key.public_key();
let request_timestamp = OffsetDateTime::now_utc().unix_timestamp();
let signature = private_key.sign(Self::plaintext(node_identity, request_timestamp));
NodeRefreshBody {
node_identity,
request_timestamp,
signature,
}
}
pub fn verify_signature(&self) -> bool {
self.node_identity
.verify(
Self::plaintext(self.node_identity, self.request_timestamp),
&self.signature,
)
.is_ok()
}
pub fn is_stale(&self) -> bool {
let Ok(encoded) = OffsetDateTime::from_unix_timestamp(self.request_timestamp) else {
return true;
};
let now = OffsetDateTime::now_utc();
if encoded > now {
return true;
}
if (encoded + Duration::from_secs(30)) < now {
return true;
}
false
}
}
#[cfg(test)]
mod tests {
use super::*;
+1
View File
@@ -1261,6 +1261,7 @@ struct TestFixture {
impl TestFixture {
fn build_app_state(storage: NymApiStorage) -> AppState {
AppState {
forced_refresh: Default::default(),
nym_contract_cache: NymContractCache::new(),
node_status_cache: NodeStatusCache::new(),
circulating_supply_cache: CirculatingSupplyCache::new("unym".to_owned()),
+49 -22
View File
@@ -9,9 +9,10 @@ use crate::support::config;
use crate::support::config::DEFAULT_NODE_DESCRIBE_BATCH_SIZE;
use async_trait::async_trait;
use futures::{stream, StreamExt};
use nym_api_requests::legacy::{LegacyGatewayBondWithId, LegacyMixNodeDetailsWithLayer};
use nym_api_requests::models::{DescribedNodeType, NymNodeData, NymNodeDescription};
use nym_config::defaults::DEFAULT_NYM_NODE_HTTP_PORT;
use nym_mixnet_contract_common::{LegacyMixLayer, NodeId};
use nym_mixnet_contract_common::{LegacyMixLayer, NodeId, NymNodeDetails};
use nym_node_requests::api::client::{NymNodeApiClientError, NymNodeApiClientExt};
use nym_topology::gateway::GatewayConversionError;
use nym_topology::mix::MixnodeConversionError;
@@ -151,6 +152,10 @@ pub struct DescribedNodes {
}
impl DescribedNodes {
pub fn force_update(&mut self, node: NymNodeDescription) {
self.nodes.insert(node.node_id, node);
}
pub fn get_description(&self, node_id: &NodeId) -> Option<&NymNodeData> {
self.nodes.get(node_id).map(|n| &n.description)
}
@@ -292,7 +297,7 @@ async fn try_get_description(
}
#[derive(Debug)]
struct RefreshData {
pub(crate) struct RefreshData {
host: String,
node_id: NodeId,
node_type: DescribedNodeType,
@@ -300,6 +305,39 @@ struct RefreshData {
port: Option<u16>,
}
impl<'a> From<&'a LegacyMixNodeDetailsWithLayer> for RefreshData {
fn from(node: &'a LegacyMixNodeDetailsWithLayer) -> Self {
RefreshData::new(
&node.bond_information.mix_node.host,
DescribedNodeType::LegacyMixnode,
node.mix_id(),
Some(node.bond_information.mix_node.http_api_port),
)
}
}
impl<'a> From<&'a LegacyGatewayBondWithId> for RefreshData {
fn from(node: &'a LegacyGatewayBondWithId) -> Self {
RefreshData::new(
&node.bond.gateway.host,
DescribedNodeType::LegacyGateway,
node.node_id,
None,
)
}
}
impl<'a> From<&'a NymNodeDetails> for RefreshData {
fn from(node: &'a NymNodeDetails) -> Self {
RefreshData::new(
&node.bond_information.node.host,
DescribedNodeType::NymNode,
node.node_id(),
node.bond_information.node.custom_http_port,
)
}
}
impl RefreshData {
pub fn new(
host: impl Into<String>,
@@ -315,7 +353,11 @@ impl RefreshData {
}
}
async fn try_refresh(self) -> Option<NymNodeDescription> {
pub(crate) fn node_id(&self) -> NodeId {
self.node_id
}
pub(crate) async fn try_refresh(self) -> Option<NymNodeDescription> {
match try_get_description(self).await {
Ok(description) => Some(description),
Err(err) => {
@@ -341,18 +383,13 @@ impl CacheItemProvider for NodeDescriptionProvider {
// - legacy gateways (because they might already be running nym-nodes, but haven't updated contract info)
// - nym-nodes
let mut nodes_to_query = Vec::new();
let mut nodes_to_query: Vec<RefreshData> = Vec::new();
match self.contract_cache.all_cached_legacy_mixnodes().await {
None => error!("failed to obtain mixnodes information from the cache"),
Some(legacy_mixnodes) => {
for node in &**legacy_mixnodes {
nodes_to_query.push(RefreshData::new(
&node.bond_information.mix_node.host,
DescribedNodeType::LegacyMixnode,
node.mix_id(),
Some(node.bond_information.mix_node.http_api_port),
))
nodes_to_query.push(node.into())
}
}
}
@@ -361,12 +398,7 @@ impl CacheItemProvider for NodeDescriptionProvider {
None => error!("failed to obtain gateways information from the cache"),
Some(legacy_gateways) => {
for node in &**legacy_gateways {
nodes_to_query.push(RefreshData::new(
&node.bond.gateway.host,
DescribedNodeType::LegacyGateway,
node.node_id,
None,
))
nodes_to_query.push(node.into())
}
}
}
@@ -375,12 +407,7 @@ impl CacheItemProvider for NodeDescriptionProvider {
None => error!("failed to obtain nym-nodes information from the cache"),
Some(nym_nodes) => {
for node in &**nym_nodes {
nodes_to_query.push(RefreshData::new(
&node.bond_information.node.host,
DescribedNodeType::NymNode,
node.node_id(),
node.bond_information.node.custom_http_port,
))
nodes_to_query.push(node.into())
}
}
}
+14
View File
@@ -355,6 +355,13 @@ impl AxumErrorResponse {
}
}
pub(crate) fn unauthorised(msg: impl Display) -> Self {
Self {
message: RequestError::new(msg.to_string()),
status: StatusCode::UNAUTHORIZED,
}
}
pub(crate) fn unprocessable_entity(msg: impl Display) -> Self {
Self {
message: RequestError::new(msg.to_string()),
@@ -375,6 +382,13 @@ impl AxumErrorResponse {
status: StatusCode::BAD_REQUEST,
}
}
pub(crate) fn too_many(msg: impl Display) -> Self {
Self {
message: RequestError::new(msg.to_string()),
status: StatusCode::TOO_MANY_REQUESTS,
}
}
}
impl From<UninitialisedCache> for AxumErrorResponse {
+44
View File
@@ -1,6 +1,7 @@
// Copyright 2023 - Nym Technologies SA <contact@nymtech.net>
// SPDX-License-Identifier: GPL-3.0-only
use crate::node_describe_cache::RefreshData;
use crate::nym_contract_cache::cache::data::CachedContractsInfo;
use crate::support::caching::Cache;
use data::ValidatorCacheData;
@@ -8,6 +9,7 @@ use nym_api_requests::legacy::{
LegacyGatewayBondWithId, LegacyMixNodeBondWithLayer, LegacyMixNodeDetailsWithLayer,
};
use nym_api_requests::models::MixnodeStatus;
use nym_crypto::asymmetric::ed25519;
use nym_mixnet_contract_common::{Interval, NodeId, NymNodeDetails, RewardedSet, RewardingParams};
use std::{
collections::HashSet,
@@ -352,6 +354,48 @@ impl NymContractCache {
self.legacy_mixnode_details(mix_id).await.1
}
pub async fn get_node_refresh_data(
&self,
node_identity: ed25519::PublicKey,
) -> Option<RefreshData> {
if !self.initialised() {
return None;
}
let inner = self.inner.read().await;
let encoded_identity = node_identity.to_base58_string();
// 1. check nymnodes
if let Some(nym_node) = inner
.nym_nodes
.iter()
.find(|n| n.bond_information.identity() == encoded_identity)
{
return Some(nym_node.into());
}
// 2. check legacy mixnodes
if let Some(mixnode) = inner
.legacy_mixnodes
.iter()
.find(|n| n.bond_information.identity() == encoded_identity)
{
return Some(mixnode.into());
}
// 3. check legacy gateways
if let Some(gateway) = inner
.legacy_gateways
.iter()
.find(|n| n.identity() == &encoded_identity)
{
return Some(gateway.into());
}
None
}
pub fn initialised(&self) -> bool {
self.initialised.load(Ordering::Relaxed)
}
+63 -4
View File
@@ -5,11 +5,11 @@ use crate::node_status_api::models::{AxumErrorResponse, AxumResult};
use crate::support::http::helpers::{NodeIdParam, PaginationRequest};
use crate::support::http::state::AppState;
use axum::extract::{Path, Query, State};
use axum::routing::get;
use axum::routing::{get, post};
use axum::{Json, Router};
use nym_api_requests::models::{
AnnotationResponse, NodeDatePerformanceResponse, NodePerformanceResponse, NoiseDetails,
NymNodeDescription, PerformanceHistoryResponse, UptimeHistoryResponse,
AnnotationResponse, NodeDatePerformanceResponse, NodePerformanceResponse, NodeRefreshBody,
NoiseDetails, NymNodeDescription, PerformanceHistoryResponse, UptimeHistoryResponse,
};
use nym_api_requests::pagination::{PaginatedResponse, Pagination};
use nym_contracts_common::NaiveFloat;
@@ -17,7 +17,8 @@ use nym_mixnet_contract_common::reward_params::Performance;
use nym_mixnet_contract_common::NymNodeDetails;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use time::Date;
use std::time::Duration;
use time::{Date, OffsetDateTime};
use utoipa::{IntoParams, ToSchema};
pub(crate) mod legacy;
@@ -25,6 +26,7 @@ pub(crate) mod unstable;
pub(crate) fn nym_node_routes() -> Router<AppState> {
Router::new()
.route("/refresh-described", post(refresh_described))
.route("/noise", get(nodes_noise))
.route("/bonded", get(get_bonded_nodes))
.route("/described", get(get_described_nodes))
@@ -42,6 +44,63 @@ pub(crate) fn nym_node_routes() -> Router<AppState> {
.route("/uptime-history/:node_id", get(get_node_uptime_history))
}
#[utoipa::path(
tag = "Nym Nodes",
post,
request_body = NodeRefreshBody,
path = "/refresh-described",
context_path = "/v1/nym-nodes",
)]
async fn refresh_described(
State(state): State<AppState>,
Json(request_body): Json<NodeRefreshBody>,
) -> AxumResult<Json<()>> {
let Some(refresh_data) = state
.nym_contract_cache()
.get_node_refresh_data(request_body.node_identity)
.await
else {
return Err(AxumErrorResponse::not_found(format!(
"node with identity {} does not seem to exist",
request_body.node_identity
)));
};
if !request_body.verify_signature() {
return Err(AxumErrorResponse::unauthorised("invalid request signature"));
}
if request_body.is_stale() {
return Err(AxumErrorResponse::bad_request("the request is stale"));
}
let node_id = refresh_data.node_id();
if let Some(last) = state.forced_refresh.last_refreshed(node_id).await {
// max 1 refresh a minute
let minute_ago = OffsetDateTime::now_utc() - Duration::from_secs(60);
if last > minute_ago {
return Err(AxumErrorResponse::too_many(
"already refreshed node in the last minute",
));
}
}
// to make sure you can't ddos the endpoint while a request is in progress
state.forced_refresh.set_last_refreshed(node_id).await;
if let Some(updated_data) = refresh_data.try_refresh().await {
let Ok(mut describe_cache) = state.described_nodes_cache.write().await else {
return Err(AxumErrorResponse::service_unavailable());
};
describe_cache.get_mut().force_update(updated_data)
} else {
return Err(AxumErrorResponse::unprocessable_entity(
"failed to refresh node description",
));
}
Ok(Json(()))
}
#[utoipa::path(
tag = "Nym Nodes",
get,
+12 -1
View File
@@ -6,7 +6,7 @@ use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
use time::OffsetDateTime;
use tokio::sync::{RwLock, RwLockReadGuard};
use tokio::sync::{RwLock, RwLockMappedWriteGuard, RwLockReadGuard, RwLockWriteGuard};
#[derive(Debug, Error)]
#[error("the cache item has not been initialised")]
@@ -45,6 +45,13 @@ impl<T> SharedCache<T> {
RwLockReadGuard::try_map(guard, |a| a.inner.as_ref()).map_err(|_| UninitialisedCache)
}
pub(crate) async fn write(
&self,
) -> Result<RwLockMappedWriteGuard<'_, Cache<T>>, UninitialisedCache> {
let guard = self.0.write().await;
RwLockWriteGuard::try_map(guard, |a| a.inner.as_mut()).map_err(|_| UninitialisedCache)
}
// ignores expiration data
#[allow(dead_code)]
pub(crate) async fn unchecked_get_inner(
@@ -134,6 +141,10 @@ impl<T> Cache<T> {
self.as_at = OffsetDateTime::now_utc()
}
pub(crate) fn get_mut(&mut self) -> &mut T {
&mut self.value
}
#[allow(dead_code)]
pub fn has_expired(&self, ttl: Duration, now: Option<OffsetDateTime>) -> bool {
let now = now.unwrap_or(OffsetDateTime::now_utc());
+1
View File
@@ -188,6 +188,7 @@ async fn start_nym_api_tasks_axum(config: &Config) -> anyhow::Result<ShutdownHan
};
let router = router.with_state(AppState {
forced_refresh: Default::default(),
nym_contract_cache: nym_contract_cache_state.clone(),
node_status_cache: node_status_cache_state.clone(),
circulating_supply_cache: circulating_supply_cache.clone(),
+1
View File
@@ -20,6 +20,7 @@ use utoipauto::utoipauto;
models::CirculatingSupplyResponse,
models::CoinSchema,
nym_mixnet_contract_common::Interval,
nym_api_requests::models::NodeRefreshBody,
nym_api_requests::models::GatewayStatusReportResponse,
nym_api_requests::models::GatewayUptimeHistoryResponse,
nym_api_requests::models::GatewayCoreStatusResponse,
+22 -1
View File
@@ -15,7 +15,9 @@ use nym_api_requests::models::{GatewayBondAnnotated, MixNodeBondAnnotated, NodeA
use nym_mixnet_contract_common::NodeId;
use nym_task::TaskManager;
use std::collections::HashMap;
use tokio::sync::RwLockReadGuard;
use std::sync::Arc;
use time::OffsetDateTime;
use tokio::sync::{RwLock, RwLockReadGuard};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
@@ -70,6 +72,7 @@ type AxumJoinHandle = JoinHandle<std::io::Result<()>>;
#[derive(Clone)]
pub(crate) struct AppState {
pub(crate) forced_refresh: ForcedRefresh,
pub(crate) nym_contract_cache: NymContractCache,
pub(crate) node_status_cache: NodeStatusCache,
pub(crate) circulating_supply_cache: CirculatingSupplyCache,
@@ -79,6 +82,24 @@ pub(crate) struct AppState {
pub(crate) node_info_cache: unstable::NodeInfoCache,
}
#[derive(Clone, Default)]
pub(crate) struct ForcedRefresh {
pub(crate) refreshes: Arc<RwLock<HashMap<NodeId, OffsetDateTime>>>,
}
impl ForcedRefresh {
pub(crate) async fn last_refreshed(&self, node_id: NodeId) -> Option<OffsetDateTime> {
self.refreshes.read().await.get(&node_id).copied()
}
pub(crate) async fn set_last_refreshed(&self, node_id: NodeId) {
self.refreshes
.write()
.await
.insert(node_id, OffsetDateTime::now_utc());
}
}
impl AppState {
pub(crate) fn nym_contract_cache(&self) -> &NymContractCache {
&self.nym_contract_cache
+1
View File
@@ -52,6 +52,7 @@ nym-sphinx-acknowledgements = { path = "../common/nymsphinx/acknowledgements" }
nym-sphinx-addressing = { path = "../common/nymsphinx/addressing" }
nym-task = { path = "../common/task" }
nym-types = { path = "../common/types" }
nym-validator-client = { path = "../common/client-libs/validator-client" }
nym-wireguard = { path = "../common/wireguard" }
nym-wireguard-types = { path = "../common/wireguard-types", default-features = false }
+40 -1
View File
@@ -32,13 +32,18 @@ use nym_node_http_api::{NymNodeHTTPServer, NymNodeRouter};
use nym_sphinx_acknowledgements::AckKey;
use nym_sphinx_addressing::Recipient;
use nym_task::{TaskClient, TaskManager};
use nym_validator_client::client::NymApiClientExt;
use nym_validator_client::models::NodeRefreshBody;
use nym_validator_client::NymApiClient;
use nym_wireguard::{peer_controller::PeerControlRequest, WireguardGatewayData};
use rand::rngs::OsRng;
use rand::{CryptoRng, RngCore};
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tracing::{debug, error, info, trace};
use tokio::time::timeout;
use tracing::{debug, error, info, trace, warn};
use zeroize::Zeroizing;
use self::helpers::load_x25519_wireguard_keypair;
@@ -740,6 +745,38 @@ impl NymNode {
.await?)
}
async fn try_refresh_remote_nym_api_cache(&self) {
info!("attempting to request described cache request from nym-api...");
if self.config.mixnet.nym_api_urls.is_empty() {
warn!("no nym-api urls available");
return;
}
for nym_api in &self.config.mixnet.nym_api_urls {
info!("trying {nym_api}...");
let client = NymApiClient::new_with_user_agent(nym_api.clone(), bin_info_owned!());
// make new request every time in case previous one takes longer and invalidates the signature
let request = NodeRefreshBody::new(self.ed25519_identity_keys.private_key());
match timeout(
Duration::from_secs(10),
client.nym_api.force_refresh_describe_cache(&request),
)
.await
{
Ok(Ok(_)) => {
info!("managed to refresh own self-described data cache")
}
Ok(Err(request_failure)) => {
warn!("failed to resolve the refresh request: {request_failure}")
}
Err(_timeout) => {
warn!("timed out while attempting to resolve the request. the cache might be stale")
}
};
}
}
pub(crate) async fn run(self) -> Result<(), NymNodeError> {
let mut task_manager = TaskManager::default().named("NymNode");
let http_server = self
@@ -754,6 +791,8 @@ impl NymNode {
}
});
self.try_refresh_remote_nym_api_cache().await;
match self.config.mode {
NodeMode::Mixnode => {
self.start_mixnode(task_manager.subscribe_named("mixnode"))?;