add telemetry builder to node.run()

Cargo.lock

@@ -6016,6 +6016,8 @@ dependencies = [
  "nym-wireguard",
  "nym-wireguard-types 0.1.0",
  "opentelemetry 0.30.0",
+ "opentelemetry-otlp",
+ "opentelemetry-stdout",
  "opentelemetry_sdk 0.30.0",
  "rand 0.8.5",
  "sha2 0.10.9",
@@ -6684,6 +6686,11 @@ dependencies = [
  "nym-verloc",
  "nym-wireguard",
  "nym-wireguard-types 0.1.0",
+ "opentelemetry 0.30.0",
+ "opentelemetry-otlp",
+ "opentelemetry-semantic-conventions 0.30.0",
+ "opentelemetry-stdout",
+ "opentelemetry_sdk 0.30.0",
  "rand 0.8.5",
  "rand_chacha 0.3.1",
  "serde",
@@ -6697,7 +6704,9 @@ dependencies = [
  "toml 0.8.23",
  "tower-http 0.5.2",
  "tracing",
+ "tracing-core",
  "tracing-indicatif",
+ "tracing-opentelemetry",
  "tracing-subscriber",
  "url",
  "utoipa",

gateway/Cargo.toml

@@ -85,6 +85,13 @@ opentelemetry_sdk = { version = "0.30.0", default-features = false, features = [
     "rt-tokio",
     "experimental_metrics_custom_reader",
 ] }
+opentelemetry-otlp = { workspace = true, features = [
+    "metrics",
+    "grpc-tonic",
+    "tls",
+    "tls-webpki-roots",
+] }
+opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"] }
 
 [build-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

gateway/src/node/client_handling/websocket/connection_handler/fresh.rs

@@ -902,6 +902,7 @@ impl<R, S> FreshHandler<R, S> {
                     let trace_id = opentelemetry::trace::TraceId::from_hex(&trace_id)
                         .expect("Invalid trace ID format");
 
+                    // We don't need to try and preserve the SpanID, just the TraceID
                     let id_generator = RandomIdGenerator::default();
                     let span_id = id_generator.new_span_id();
 

nym-node/Cargo.toml

@@ -114,6 +114,17 @@ blake2 = "=0.8.1"
 sha2 = { workspace = true }
 hkdf = { workspace = true }
 
+# signoz
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true }
+tracing-core = { workspace = true }
+tracing-opentelemetry = { workspace = true }
+opentelemetry-otlp = { version = "0.30.0" }
+opentelemetry-semantic-conventions = { workspace = true, features = [
+    "semconv_experimental",
+] }
+opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"] }
+
 [[bench]]
 name = "benchmarks"
 harness = false

nym-node/src/node/mod.rs

@@ -69,6 +69,23 @@ use tokio::sync::mpsc;
 use tracing::{debug, info, trace};
 use zeroize::Zeroizing;
 
+// signoz
+use opentelemetry::trace::TracerProvider;
+use opentelemetry::{global, KeyValue};
+use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
+use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig;
+use opentelemetry_otlp::{WithExportConfig, WithTonicConfig};
+use opentelemetry_sdk::metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider};
+use opentelemetry_sdk::trace::{RandomIdGenerator, SdkTracerProvider};
+use opentelemetry_sdk::{trace::Sampler, Resource};
+use opentelemetry_semantic_conventions::{
+    attribute::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_VERSION},
+    SCHEMA_URL,
+};
+use tracing_core::Level;
+use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
 pub mod bonding_information;
 pub mod description;
 pub mod helpers;
@@ -1196,6 +1213,15 @@ impl NymNode {
     }
 
     pub(crate) async fn run(mut self) -> Result<(), NymNodeError> {
+        let key = std::env::var("SIGNOZ_INGESTION_KEY").expect("SIGNOZ_INGESTION_KEY not set");
+        println!("SIGNOZ_INGESTION_KEY = {}", key);
+        let mut metadata = MetadataMap::new();
+        metadata.insert(
+            "signoz-ingestion-key",
+            key.parse().expect("Could not parse key"),
+        );
+        let _guard = init_tracing_subscriber(metadata);
+
         let mut shutdown_signals = self.shutdown_manager.detach_shutdown_signals();
 
         // listen for shutdown signal in case we received it when attempting to spawn all the tasks
@@ -1215,3 +1241,118 @@ impl NymNode {
         Ok(())
     }
 }
+
+fn resource() -> Resource {
+    Resource::builder()
+        .with_service_name("sdk-example-surb-reply")
+        .with_schema_url(
+            [
+                KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
+                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, "develop"),
+            ],
+            SCHEMA_URL,
+        )
+        .build()
+}
+
+fn init_tracer_provider(metadata: MetadataMap) -> anyhow::Result<SdkTracerProvider> {
+    let endpoint = std::env::var("SIGNOZ_ENDPOINT").expect("SIGNOZ_ENDPOINT not set");
+    println!("SIGNOZ_ENDPOINT = {}", endpoint);
+
+    // Configure OTLP exporter with metadata
+    let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
+        .with_metadata(metadata)
+        .with_endpoint(&endpoint);
+
+    // Try with TLS - seems to work
+    if endpoint.starts_with("https://") {
+        exporter_builder =
+            exporter_builder.with_tls_config(ClientTlsConfig::new().with_enabled_roots());
+    }
+
+    let exporter = exporter_builder.build()?;
+
+    let tracer = SdkTracerProvider::builder()
+        // Customize sampling strategy
+        .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
+            1.0,
+        ))))
+        // If export trace to AWS X-Ray, you can use XrayIdGenerator
+        .with_id_generator(RandomIdGenerator::default())
+        .with_resource(resource())
+        .with_batch_exporter(exporter)
+        .build();
+
+    Ok(tracer)
+}
+
+fn init_meter_provider() -> SdkMeterProvider {
+    let exporter = opentelemetry_otlp::MetricExporter::builder()
+        .with_tonic()
+        .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+        .build()
+        .unwrap();
+
+    let reader = PeriodicReader::builder(exporter)
+        .with_interval(std::time::Duration::from_secs(30))
+        .build();
+
+    // For debugging in development
+    let stdout_reader =
+        PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default()).build();
+
+    let meter_provider = MeterProviderBuilder::default()
+        .with_resource(resource())
+        .with_reader(reader)
+        .with_reader(stdout_reader)
+        .build();
+
+    global::set_meter_provider(meter_provider.clone());
+
+    meter_provider
+}
+
+fn init_tracing_subscriber(metadata: MetadataMap) -> OtelGuard {
+    let tracer_provider = init_tracer_provider(metadata).expect("Initializing tracer failed");
+    let meter_provider = init_meter_provider();
+
+    let tracer = tracer_provider.tracer("tracing-otel-subscriber");
+
+    tracing_subscriber::registry()
+        // The global level filter prevents the exporter network stack
+        // from reentering the globally installed OpenTelemetryLayer with
+        // its own spans while exporting, as the libraries should not use
+        // tracing levels below DEBUG. If the OpenTelemetry layer needs to
+        // trace spans and events with higher verbosity levels, consider using
+        // per-layer filtering to target the telemetry layer specifically,
+        // e.g. by target matching.
+        .with(tracing_subscriber::filter::LevelFilter::from_level(
+            Level::DEBUG,
+        ))
+        .with(tracing_subscriber::fmt::layer())
+        .with(MetricsLayer::new(meter_provider.clone()))
+        .with(OpenTelemetryLayer::new(tracer))
+        .init();
+
+    OtelGuard {
+        tracer_provider,
+        meter_provider,
+    }
+}
+
+struct OtelGuard {
+    tracer_provider: SdkTracerProvider,
+    meter_provider: SdkMeterProvider,
+}
+
+impl Drop for OtelGuard {
+    fn drop(&mut self) {
+        if let Err(err) = self.tracer_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+        if let Err(err) = self.meter_provider.shutdown() {
+            eprintln!("{err:?}");
+        }
+    }
+}