From fc53eec241d3250b5807cc9266ff8cf06e5e555d Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 1 Aug 2025 17:17:14 +0200 Subject: [PATCH] add telemetry builder to node.run() --- Cargo.lock | 9 ++ gateway/Cargo.toml | 7 + .../websocket/connection_handler/fresh.rs | 1 + nym-node/Cargo.toml | 11 ++ nym-node/src/node/mod.rs | 141 ++++++++++++++++++ 5 files changed, 169 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8a7eac4190..8ee91dfbeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6016,6 +6016,8 @@ dependencies = [ "nym-wireguard", "nym-wireguard-types 0.1.0", "opentelemetry 0.30.0", + "opentelemetry-otlp", + "opentelemetry-stdout", "opentelemetry_sdk 0.30.0", "rand 0.8.5", "sha2 0.10.9", @@ -6684,6 +6686,11 @@ dependencies = [ "nym-verloc", "nym-wireguard", "nym-wireguard-types 0.1.0", + "opentelemetry 0.30.0", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions 0.30.0", + "opentelemetry-stdout", + "opentelemetry_sdk 0.30.0", "rand 0.8.5", "rand_chacha 0.3.1", "serde", @@ -6697,7 +6704,9 @@ dependencies = [ "toml 0.8.23", "tower-http 0.5.2", "tracing", + "tracing-core", "tracing-indicatif", + "tracing-opentelemetry", "tracing-subscriber", "url", "utoipa", diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index d4602cabda..a92963e72a 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -85,6 +85,13 @@ opentelemetry_sdk = { version = "0.30.0", default-features = false, features = [ "rt-tokio", "experimental_metrics_custom_reader", ] } +opentelemetry-otlp = { workspace = true, features = [ + "metrics", + "grpc-tonic", + "tls", + "tls-webpki-roots", +] } +opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"] } [build-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/gateway/src/node/client_handling/websocket/connection_handler/fresh.rs b/gateway/src/node/client_handling/websocket/connection_handler/fresh.rs index 3635f632ac..9ce888e927 100644 --- a/gateway/src/node/client_handling/websocket/connection_handler/fresh.rs +++ b/gateway/src/node/client_handling/websocket/connection_handler/fresh.rs @@ -902,6 +902,7 @@ impl FreshHandler { let trace_id = opentelemetry::trace::TraceId::from_hex(&trace_id) .expect("Invalid trace ID format"); + // We don't need to try and preserve the SpanID, just the TraceID let id_generator = RandomIdGenerator::default(); let span_id = id_generator.new_span_id(); diff --git a/nym-node/Cargo.toml b/nym-node/Cargo.toml index 96be330954..b13f6fd36c 100644 --- a/nym-node/Cargo.toml +++ b/nym-node/Cargo.toml @@ -114,6 +114,17 @@ blake2 = "=0.8.1" sha2 = { workspace = true } hkdf = { workspace = true } +# signoz +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +tracing-core = { workspace = true } +tracing-opentelemetry = { workspace = true } +opentelemetry-otlp = { version = "0.30.0" } +opentelemetry-semantic-conventions = { workspace = true, features = [ + "semconv_experimental", +] } +opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"] } + [[bench]] name = "benchmarks" harness = false diff --git a/nym-node/src/node/mod.rs b/nym-node/src/node/mod.rs index c240686b8a..5df5abce4d 100644 --- a/nym-node/src/node/mod.rs +++ b/nym-node/src/node/mod.rs @@ -69,6 +69,23 @@ use tokio::sync::mpsc; use tracing::{debug, info, trace}; use zeroize::Zeroizing; +// signoz +use opentelemetry::trace::TracerProvider; +use opentelemetry::{global, KeyValue}; +use opentelemetry_otlp::tonic_types::metadata::MetadataMap; +use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig; +use opentelemetry_otlp::{WithExportConfig, WithTonicConfig}; +use opentelemetry_sdk::metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider}; +use opentelemetry_sdk::trace::{RandomIdGenerator, SdkTracerProvider}; +use opentelemetry_sdk::{trace::Sampler, Resource}; +use opentelemetry_semantic_conventions::{ + attribute::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_VERSION}, + SCHEMA_URL, +}; +use tracing_core::Level; +use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + pub mod bonding_information; pub mod description; pub mod helpers; @@ -1196,6 +1213,15 @@ impl NymNode { } pub(crate) async fn run(mut self) -> Result<(), NymNodeError> { + let key = std::env::var("SIGNOZ_INGESTION_KEY").expect("SIGNOZ_INGESTION_KEY not set"); + println!("SIGNOZ_INGESTION_KEY = {}", key); + let mut metadata = MetadataMap::new(); + metadata.insert( + "signoz-ingestion-key", + key.parse().expect("Could not parse key"), + ); + let _guard = init_tracing_subscriber(metadata); + let mut shutdown_signals = self.shutdown_manager.detach_shutdown_signals(); // listen for shutdown signal in case we received it when attempting to spawn all the tasks @@ -1215,3 +1241,118 @@ impl NymNode { Ok(()) } } + +fn resource() -> Resource { + Resource::builder() + .with_service_name("sdk-example-surb-reply") + .with_schema_url( + [ + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, "develop"), + ], + SCHEMA_URL, + ) + .build() +} + +fn init_tracer_provider(metadata: MetadataMap) -> anyhow::Result { + let endpoint = std::env::var("SIGNOZ_ENDPOINT").expect("SIGNOZ_ENDPOINT not set"); + println!("SIGNOZ_ENDPOINT = {}", endpoint); + + // Configure OTLP exporter with metadata + let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_metadata(metadata) + .with_endpoint(&endpoint); + + // Try with TLS - seems to work + if endpoint.starts_with("https://") { + exporter_builder = + exporter_builder.with_tls_config(ClientTlsConfig::new().with_enabled_roots()); + } + + let exporter = exporter_builder.build()?; + + let tracer = SdkTracerProvider::builder() + // Customize sampling strategy + .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased( + 1.0, + )))) + // If export trace to AWS X-Ray, you can use XrayIdGenerator + .with_id_generator(RandomIdGenerator::default()) + .with_resource(resource()) + .with_batch_exporter(exporter) + .build(); + + Ok(tracer) +} + +fn init_meter_provider() -> SdkMeterProvider { + let exporter = opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .with_temporality(opentelemetry_sdk::metrics::Temporality::default()) + .build() + .unwrap(); + + let reader = PeriodicReader::builder(exporter) + .with_interval(std::time::Duration::from_secs(30)) + .build(); + + // For debugging in development + let stdout_reader = + PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default()).build(); + + let meter_provider = MeterProviderBuilder::default() + .with_resource(resource()) + .with_reader(reader) + .with_reader(stdout_reader) + .build(); + + global::set_meter_provider(meter_provider.clone()); + + meter_provider +} + +fn init_tracing_subscriber(metadata: MetadataMap) -> OtelGuard { + let tracer_provider = init_tracer_provider(metadata).expect("Initializing tracer failed"); + let meter_provider = init_meter_provider(); + + let tracer = tracer_provider.tracer("tracing-otel-subscriber"); + + tracing_subscriber::registry() + // The global level filter prevents the exporter network stack + // from reentering the globally installed OpenTelemetryLayer with + // its own spans while exporting, as the libraries should not use + // tracing levels below DEBUG. If the OpenTelemetry layer needs to + // trace spans and events with higher verbosity levels, consider using + // per-layer filtering to target the telemetry layer specifically, + // e.g. by target matching. + .with(tracing_subscriber::filter::LevelFilter::from_level( + Level::DEBUG, + )) + .with(tracing_subscriber::fmt::layer()) + .with(MetricsLayer::new(meter_provider.clone())) + .with(OpenTelemetryLayer::new(tracer)) + .init(); + + OtelGuard { + tracer_provider, + meter_provider, + } +} + +struct OtelGuard { + tracer_provider: SdkTracerProvider, + meter_provider: SdkMeterProvider, +} + +impl Drop for OtelGuard { + fn drop(&mut self) { + if let Err(err) = self.tracer_provider.shutdown() { + eprintln!("{err:?}"); + } + if let Err(err) = self.meter_provider.shutdown() { + eprintln!("{err:?}"); + } + } +}