add opentelemetry for debugging and testing behind otel feature flag

Cargo.toml

@@ -300,8 +300,11 @@ nix = "0.27.1"
 notify = "5.1.0"
 okapi = "0.7.0"
 once_cell = "1.21.3"
-opentelemetry = "0.19.0"
-opentelemetry-jaeger = "0.18.0"
+opentelemetry = "0.30.0"
+opentelemetry-otlp = "0.30.0"
+opentelemetry-semantic-conventions = "0.30.0"
+opentelemetry_sdk = "0.30.0"
+opentelemetry-stdout = "0.30.0"
 parking_lot = "0.12.3"
 pem = "0.8"
 petgraph = "0.6.5"
@@ -360,8 +363,10 @@ toml = "0.8.22"
 tower = "0.5.2"
 tower-http = "0.5.2"
 tracing = "0.1.41"
+tracing-core = "0.1.33"
 tracing-log = "0.2"
-tracing-opentelemetry = "0.19.0"
+tracing-opentelemetry = "0.31.0"
+tracing-serde = "0.2.0"
 tracing-subscriber = "0.3.19"
 tracing-tree = "0.2.2"
 tracing-indicatif = "0.3.9"

common/bin-common/Cargo.toml

@@ -8,40 +8,57 @@ license = { workspace = true }
 repository = { workspace = true }
 
 [dependencies]
+chrono = { workspace = true, optional = true }
+cfg-if = { workspace = true }
 clap = { workspace = true, features = ["derive"], optional = true }
 clap_complete = { workspace = true, optional = true }
 clap_complete_fig = { workspace = true, optional = true }
 const-str = { workspace = true }
-log = { workspace = true }
+opentelemetry = { workspace = true, optional = true }
+opentelemetry-otlp = { workspace = true,features=["metrics", "grpc-tonic", "tls",
+    "tls-webpki-roots"], optional = true }
+opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"], optional = true }
+opentelemetry-stdout = { workspace = true, features = ["trace", "metrics"], optional = true }
+opentelemetry_sdk = { workspace = true, optional = true }
 schemars = { workspace = true, features = ["preserve_order"], optional = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true, optional = true }
-
-## tracing
-tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
-tracing-tree = { workspace = true, optional = true }
+thiserror = { workspace = true }
 tracing = { workspace = true, optional = true }
-opentelemetry-jaeger = { workspace = true, features = ["rt-tokio", "collector_client", "isahc_collector_client"], optional = true }
+tracing-core = { workspace = true, optional = true }
 tracing-opentelemetry = { workspace = true, optional = true }
+tracing-serde = { workspace = true, optional = true }
+tracing-subscriber = { workspace = true, features = ["env-filter", "json"], optional = true }
+tracing-tree = { workspace = true, optional = true }
 utoipa = { workspace = true, optional = true }
-opentelemetry = { workspace = true, features = ["rt-tokio"], optional = true }
-
 
 [build-dependencies]
 vergen = { workspace = true, features = ["build", "git", "gitcl", "rustc", "cargo"] }
 
 [features]
-default = []
+default = ["otel"] # 🚨🚨🚨 TODO: remove otel as a default feature 🚨🚨🚨
 openapi = ["utoipa"]
 output_format = ["serde_json", "dep:clap"]
 bin_info_schema = ["schemars"]
 basic_tracing = ["dep:tracing", "tracing-subscriber"]
+tokio-console = ["dep:tracing", "tracing-subscriber"]
 tracing = [
     "basic_tracing",
-    "tracing-tree",
-    "opentelemetry-jaeger",
+    "chrono",
+    "serde_json",
+    "tracing-core",
     "tracing-opentelemetry",
+    "tracing-serde",
+    "tracing-subscriber",
+    "tracing-tree",
+]
+otel = [
+    "tracing",
     "opentelemetry",
+    "opentelemetry-otlp",
+    "opentelemetry-semantic-conventions",
+    "opentelemetry-stdout",
+    "opentelemetry_sdk",
 ]
 clap = ["dep:clap", "dep:clap_complete", "dep:clap_complete_fig"]
 models = []

common/bin-common/src/lib.rs

@@ -4,6 +4,8 @@
 pub mod build_information;
 pub mod logging;
 
+pub mod opentelemetry;
+
 #[cfg(feature = "clap")]
 pub mod completions;
 

common/bin-common/src/logging/mod.rs

@@ -4,17 +4,6 @@
 use serde::{Deserialize, Serialize};
 use std::io::IsTerminal;
 
-#[cfg(feature = "tracing")]
-pub use opentelemetry;
-#[cfg(feature = "tracing")]
-pub use opentelemetry_jaeger;
-#[cfg(feature = "tracing")]
-pub use tracing_opentelemetry;
-#[cfg(feature = "tracing")]
-pub use tracing_subscriber;
-#[cfg(feature = "tracing")]
-pub use tracing_tree;
-
 #[derive(Debug, Default, Copy, Clone, Deserialize, PartialEq, Eq, Serialize)]
 #[serde(deny_unknown_fields)]
 pub struct LoggingSettings {
@@ -74,34 +63,7 @@ pub fn setup_tracing_logger() {
 #[macro_export]
 macro_rules! setup_tracing {
     ($service_name: expr) => {
-        use nym_bin_common::logging::tracing_subscriber::layer::SubscriberExt;
-        use nym_bin_common::logging::tracing_subscriber::util::SubscriberInitExt;
-
-        let registry = nym_bin_common::logging::tracing_subscriber::Registry::default()
-            .with(nym_bin_common::logging::tracing_subscriber::EnvFilter::from_default_env())
-            .with(
-                nym_bin_common::logging::tracing_tree::HierarchicalLayer::new(4)
-                    .with_targets(true)
-                    .with_bracketed_fields(true),
-            );
-
-        let tracer = nym_bin_common::logging::opentelemetry_jaeger::new_collector_pipeline()
-            .with_endpoint("http://44.199.230.10:14268/api/traces")
-            .with_service_name($service_name)
-            .with_isahc()
-            .with_trace_config(
-                nym_bin_common::logging::opentelemetry::sdk::trace::config().with_sampler(
-                    nym_bin_common::logging::opentelemetry::sdk::trace::Sampler::TraceIdRatioBased(
-                        0.1,
-                    ),
-                ),
-            )
-            .install_batch(nym_bin_common::logging::opentelemetry::runtime::Tokio)
-            .expect("Could not init tracer");
-
-        let telemetry = nym_bin_common::logging::tracing_opentelemetry::layer().with_tracer(tracer);
-
-        registry.with(telemetry).init();
+        crate::opentelemetry::setup_no_otel_logger()
     };
 }
 

common/bin-common/src/opentelemetry/error.rs

@@ -0,0 +1,21 @@
+// Copyright 2025 - Nym Technologies SA <contact@nymtech.net>
+// SPDX-License-Identifier: Apache-2.0
+
+use opentelemetry_otlp::ExporterBuildError;
+
+#[derive(thiserror::Error, Debug)]
+pub enum TracingError {
+    #[error("tracing logger already initialised")]
+    TracingLoggerAlreadyInitialised,
+
+    // #[error("I/O error: {0}")]
+    // IoError(#[from] std::io::Error),
+    #[error("Logging error: {0}")]
+    TracingTryInitError(tracing_subscriber::util::TryInitError),
+
+    #[error("{0}")]
+    TracingExporterBuildError(#[from] ExporterBuildError),
+
+    #[error("{0}")]
+    TracingFilterParseError(#[from] tracing_subscriber::filter::ParseError),
+}

common/bin-common/src/opentelemetry/mod.rs

@@ -0,0 +1,180 @@
+pub mod error;
+mod trace_id_format;
+
+use tracing::{info, Level};
+use tracing_subscriber::filter::Directive;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{fmt, Layer};
+
+use crate::logging::{default_tracing_env_filter, default_tracing_fmt_layer};
+use crate::opentelemetry::error::TracingError;
+use opentelemetry::trace::TracerProvider;
+use opentelemetry::{global, KeyValue};
+use opentelemetry_otlp::tonic_types::metadata::MetadataMap;
+use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig;
+use opentelemetry_otlp::{WithExportConfig, WithTonicConfig};
+use opentelemetry_sdk::metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider};
+use opentelemetry_sdk::trace::{RandomIdGenerator, SdkTracerProvider};
+use opentelemetry_sdk::{trace::Sampler, Resource};
+use opentelemetry_semantic_conventions::resource::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_VERSION};
+use opentelemetry_semantic_conventions::SCHEMA_URL;
+use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
+use tracing_subscriber::fmt::format::FmtSpan;
+
+pub(crate) fn granual_filtered_env() -> Result<tracing_subscriber::filter::EnvFilter, TracingError>
+{
+    fn directive_checked(directive: impl Into<String>) -> Result<Directive, TracingError> {
+        directive.into().parse().map_err(From::from)
+    }
+
+    let mut filter = default_tracing_env_filter();
+
+    // these crates are more granularly filtered
+    let filter_crates = ["defguard_wireguard_rs"];
+    for crate_name in filter_crates {
+        filter = filter.add_directive(directive_checked(format!("{crate_name}=warn"))?);
+    }
+    Ok(filter)
+}
+
+pub(crate) fn build_tracing_logger() -> Result<impl SubscriberExt, TracingError> {
+    let key =
+        std::env::var("SIGNOZ_INGESTION_KEY".to_string()).expect("SIGNOZ_INGESTION_KEY not set");
+    let mut metadata = MetadataMap::new();
+    metadata.insert(
+        "signoz-ingestion-key",
+        key.parse().expect("Could not parse signoz ingestion key"),
+    );
+
+    let tracer_provider = init_tracer_provider(metadata)?;
+    let meter_provider = init_meter_provider()?;
+    let tracer = tracer_provider.tracer("tracing-otel-subscriber");
+
+    let fmt_layer = fmt::layer()
+        .json()
+        .with_writer(std::io::stderr)
+        .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
+        .with_span_list(false)
+        .with_current_span(true)
+        .event_format(trace_id_format::TraceIdFormat);
+
+    let registry = tracing_subscriber::registry()
+        .with(fmt_layer)
+        .with(granual_filtered_env()?)
+        .with(tracing_subscriber::filter::LevelFilter::from_level(
+            Level::DEBUG,
+        ))
+        .with(MetricsLayer::new(meter_provider))
+        .with(OpenTelemetryLayer::new(tracer));
+
+    Ok(registry)
+}
+
+pub fn setup_tracing_logger() -> Result<(), TracingError> {
+    let stderr_layer =
+        default_tracing_fmt_layer(std::io::stderr).with_filter(granual_filtered_env()?);
+
+    cfg_if::cfg_if! {if #[cfg(feature = "tokio-console")] {
+        // instrument tokio console subscriber needs RUSTFLAGS="--cfg tokio_unstable" at build time
+        let console_layer = console_subscriber::spawn();
+
+        tracing_subscriber::registry()
+            .with(console_layer)
+            .with(stderr_layer)
+            .init();
+    } else {
+        tracing_subscriber::registry()
+            .with(stderr_layer)
+            .init();
+    }}
+
+    build_tracing_logger()?.init();
+    Ok(())
+}
+
+pub fn setup_no_otel_logger() -> Result<(), error::TracingError> {
+    // Only set up if not already initialized
+    if tracing::dispatcher::has_been_set() {
+        // It shouldn't be - this is really checking that it is torn down between async command executions
+        return Err(error::TracingError::TracingLoggerAlreadyInitialised);
+    }
+
+    let registry = tracing_subscriber::registry()
+        .with(default_tracing_fmt_layer(std::io::stderr))
+        .with(granual_filtered_env()?)
+        .with(tracing_subscriber::filter::LevelFilter::from_level(
+            Level::INFO,
+        ));
+
+    registry
+        .try_init()
+        .map_err(|e| error::TracingError::TracingTryInitError(e))?;
+
+    Ok(())
+}
+
+fn resource() -> Resource {
+    Resource::builder()
+        .with_service_name("nym-node")
+        .with_schema_url(
+            [
+                KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")),
+                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, "develop"),
+            ],
+            SCHEMA_URL,
+        )
+        .build()
+}
+
+fn init_tracer_provider(metadata: MetadataMap) -> Result<SdkTracerProvider, TracingError> {
+    let endpoint = std::env::var("SIGNOZ_ENDPOINT".to_string()).expect("SIGNOZ_ENDPOINT not set");
+    info!("SIGNOZ_ENDPOINT = {}", endpoint);
+
+    let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
+        .with_metadata(metadata)
+        .with_endpoint(&endpoint);
+
+    if endpoint.starts_with("https://") {
+        exporter_builder =
+            exporter_builder.with_tls_config(ClientTlsConfig::new().with_enabled_roots());
+    }
+
+    let exporter = exporter_builder.build()?;
+
+    let tracer = SdkTracerProvider::builder()
+        .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
+            1.0,
+        ))))
+        .with_id_generator(RandomIdGenerator::default())
+        .with_resource(resource())
+        .with_batch_exporter(exporter)
+        .build();
+
+    Ok(tracer)
+}
+
+fn init_meter_provider() -> Result<SdkMeterProvider, TracingError> {
+    let exporter = opentelemetry_otlp::MetricExporter::builder()
+        .with_tonic()
+        .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+        .build()?;
+
+    let reader = PeriodicReader::builder(exporter)
+        .with_interval(std::time::Duration::from_secs(30))
+        .build();
+
+    let stdout_reader =
+        PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default()).build();
+
+    let meter_provider = MeterProviderBuilder::default()
+        .with_resource(resource())
+        .with_reader(reader)
+        .with_reader(stdout_reader)
+        .build();
+
+    global::set_meter_provider(meter_provider.clone());
+
+    Ok(meter_provider)
+}

common/bin-common/src/opentelemetry/trace_id_format.rs

@@ -0,0 +1,88 @@
+use chrono::Utc;
+use opentelemetry::trace::TraceContextExt;
+use opentelemetry::{SpanId, TraceId};
+use serde::ser::{SerializeMap, Serializer as _};
+use std::io;
+use tracing::{Event, Subscriber};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+use tracing_serde::fields::AsMap;
+use tracing_serde::AsSerde;
+use tracing_subscriber::fmt::format::Writer;
+use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
+use tracing_subscriber::registry::LookupSpan;
+
+pub struct WriteAdaptor<'a> {
+    fmt_write: &'a mut dyn std::fmt::Write,
+}
+
+impl<'a> WriteAdaptor<'a> {
+    pub fn new(fmt_write: &'a mut dyn std::fmt::Write) -> Self {
+        Self { fmt_write }
+    }
+}
+
+impl<'a> io::Write for WriteAdaptor<'a> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        let s =
+            std::str::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+        self.fmt_write
+            .write_str(s)
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
+
+        Ok(s.as_bytes().len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+pub struct TraceIdFormat;
+
+impl<S, N> FormatEvent<S, N> for TraceIdFormat
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+    N: for<'writer> FormatFields<'writer> + 'static,
+{
+    fn format_event(
+        &self,
+        _ctx: &FmtContext<'_, S, N>,
+        mut writer: Writer<'_>,
+        event: &Event<'_>,
+    ) -> std::fmt::Result
+    where
+        S: Subscriber + for<'a> LookupSpan<'a>,
+    {
+        let meta = event.metadata();
+
+        let mut visit = || {
+            let mut serializer = serde_json::Serializer::new(WriteAdaptor::new(&mut writer));
+            let mut serializer = serializer.serialize_map(None)?;
+            serializer.serialize_entry("timestamp", &Utc::now().to_rfc3339())?;
+            serializer.serialize_entry("level", &meta.level().as_serde())?;
+            serializer.serialize_entry("fields", &event.field_map())?;
+            serializer.serialize_entry("target", meta.target())?;
+
+            let current_span = tracing::Span::current();
+            let context = current_span.context();
+            let span_ref = context.span();
+            let span_context = span_ref.span_context();
+
+            let trace_id = span_context.trace_id();
+            if trace_id != TraceId::INVALID {
+                serializer.serialize_entry("trace_id", &trace_id.to_string())?;
+
+                let span_id = span_context.span_id();
+                if span_id != SpanId::INVALID {
+                    serializer.serialize_entry("span_id", &span_id.to_string())?;
+                }
+            }
+
+            serializer.end()
+        };
+
+        visit().map_err(|_| std::fmt::Error)?;
+        writeln!(writer)
+    }
+}