From 703896fd3778df8c443b9d022d1851aba9136bd6 Mon Sep 17 00:00:00 2001 From: Anant Vindal Date: Mon, 30 Mar 2026 20:43:55 +0530 Subject: [PATCH] Add OpenTelemetry instrumentation Auto-generated by otex Modified files: 2 Coverage: 100% (112/112 endpoints) Co-authored-by: otex-dev --- Cargo.lock | 86 ++++++++++++++++++++++++++++++++++ Cargo.toml | 11 ++++- src/handlers/http/logstream.rs | 1 + src/handlers/http/modal/mod.rs | 2 + src/handlers/http/query.rs | 39 +++++++++++++++ src/handlers/http/rbac.rs | 10 ++++ src/handlers/http/role.rs | 16 +++++++ src/lib.rs | 1 + src/main.rs | 31 +++++++++--- src/parseable/mod.rs | 10 ++++ src/query/mod.rs | 49 ++++++++++++++++++- src/telemetry.rs | 72 ++++++++++++++++++++++++++++ 12 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 src/telemetry.rs diff --git a/Cargo.lock b/Cargo.lock index 61cdf5f02..cc98febe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3331,6 +3331,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mutually_exclusive_features" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" + [[package]] name = "ntapi" version = "0.4.2" @@ -3572,6 +3578,37 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "git+https://github.com/parmesant/opentelemetry-rust/?rev=45fb828769e6ade96d56ca1f5fa14cf0986a5341#45fb828769e6ade96d56ca1f5fa14cf0986a5341" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.31.0" +source = "git+https://github.com/parmesant/opentelemetry-rust/?rev=45fb828769e6ade96d56ca1f5fa14cf0986a5341#45fb828769e6ade96d56ca1f5fa14cf0986a5341" +dependencies = [ + "http 1.4.0", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.14.1", + "reqwest", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tonic", + "tonic-types", +] + [[package]] name = "opentelemetry-proto" version = "0.31.0" @@ -3600,6 +3637,8 @@ dependencies = [ "percent-encoding", "rand 0.9.2", "thiserror 2.0.17", + "tokio", + "tokio-stream", ] [[package]] @@ -3739,7 +3778,10 @@ dependencies = [ "object_store", "once_cell", "openid", + "opentelemetry", + "opentelemetry-otlp", "opentelemetry-proto", + "opentelemetry_sdk", "parking_lot", "parquet", "path-clean", @@ -3775,6 +3817,8 @@ dependencies = [ "tonic-web", "tower-http", "tracing", + "tracing-actix-web", + "tracing-opentelemetry", "tracing-subscriber", "ulid", "uptime_lib", @@ -4406,6 +4450,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", + "futures-channel", "futures-core", "futures-util", "h2 0.4.12", @@ -5415,6 +5460,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost 0.14.1", + "prost-types", + "tonic", +] + [[package]] name = "tonic-web" version = "0.14.2" @@ -5473,6 +5529,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -5499,6 +5556,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-actix-web" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca6b15407f9bfcb35f82d0e79e603e1629ece4e91cc6d9e58f890c184dd20af" +dependencies = [ + "actix-web", + "mutually_exclusive_features", + "pin-project", + "tracing", + "uuid", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -5531,6 +5601,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.22" diff --git a/Cargo.toml b/Cargo.toml index e10a6787c..044cace83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ tonic = { version = "0.14.1", features = [ ] } tonic-prost = "0.14.1" tonic-web = "0.14.1" -tower-http = { version = "0.6.1", features = ["cors"] } +tower-http = { version = "0.6.1", features = ["cors", "trace"] } url = "2.4.0" # Connectors dependencies @@ -108,6 +108,11 @@ prometheus = { version = "0.13.4", default-features = false, features = ["proces prometheus-parse = "0.2.5" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "time"] } +opentelemetry = { git = "https://github.com/parmesant/opentelemetry-rust/", rev = "45fb828769e6ade96d56ca1f5fa14cf0986a5341", features = ["trace"] } +opentelemetry_sdk = { git = "https://github.com/parmesant/opentelemetry-rust/", rev = "45fb828769e6ade96d56ca1f5fa14cf0986a5341", features = ["trace", "rt-tokio"] } +opentelemetry-otlp = { git = "https://github.com/parmesant/opentelemetry-rust/", rev = "45fb828769e6ade96d56ca1f5fa14cf0986a5341", features = ["trace", "grpc-tonic", "http-json", "http-proto"] } +tracing-opentelemetry = "0.32" +tracing-actix-web = "0.7" # Time and Date chrono = "0.4" @@ -197,6 +202,10 @@ kafka = [ "sasl2-sys/vendored", ] +[patch.crates-io] +opentelemetry = { git = "https://github.com/parmesant/opentelemetry-rust/", rev = "45fb828769e6ade96d56ca1f5fa14cf0986a5341" } +opentelemetry_sdk = { git = "https://github.com/parmesant/opentelemetry-rust/", rev = "45fb828769e6ade96d56ca1f5fa14cf0986a5341" } + [profile.release-lto] inherits = "release" lto = "fat" diff --git a/src/handlers/http/logstream.rs b/src/handlers/http/logstream.rs index 17f1be2fe..5bf73820a 100644 --- a/src/handlers/http/logstream.rs +++ b/src/handlers/http/logstream.rs @@ -213,6 +213,7 @@ pub async fn put_stream( Ok(("Log stream created", StatusCode::OK)) } +#[tracing::instrument(name = "http.get_retention", skip(req), fields(stream_name = %stream_name), err)] pub async fn get_retention( req: HttpRequest, stream_name: Path, diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs index 115f8b643..41457733f 100644 --- a/src/handlers/http/modal/mod.rs +++ b/src/handlers/http/modal/mod.rs @@ -32,6 +32,7 @@ use serde_json::{Map, Value}; use ssl_acceptor::get_ssl_acceptor; use tokio::sync::{RwLock, oneshot}; use tracing::{error, info, warn}; +use tracing_actix_web::TracingLogger; use crate::{ alerts::{ALERTS, get_alert_manager, target::TARGETS}, @@ -111,6 +112,7 @@ pub trait ParseableServer { // fn that creates the app let create_app_fn = move || { App::new() + .wrap(TracingLogger::default()) .wrap(prometheus.clone()) .configure(|config| Self::configure_routes(config)) .wrap(from_fn(health_check::check_shutdown_middleware)) diff --git a/src/handlers/http/query.rs b/src/handlers/http/query.rs index fd7804f7f..18e95e787 100644 --- a/src/handlers/http/query.rs +++ b/src/handlers/http/query.rs @@ -115,15 +115,27 @@ pub async fn get_records_and_fields( Ok((Some(records), Some(fields))) } +#[tracing::instrument( + name = "query.handle", + skip(req, query_request), + fields( + streaming = %query_request.streaming, + sql = %query_request.query, + tables = tracing::field::Empty, + tenant = tracing::field::Empty, + ) +)] pub async fn query(req: HttpRequest, query_request: Query) -> Result { let mut session_state = QUERY_SESSION.get_ctx().state(); let time_range = TimeRange::parse_human_time(&query_request.start_time, &query_request.end_time)?; let tables = resolve_stream_names(&query_request.query)?; + tracing::Span::current().record("tables", tracing::field::display(tables.join(","))); // check or load streams in memory create_streams_for_distributed(tables.clone(), &get_tenant_id_from_request(&req)).await?; let tenant_id = get_tenant_id_from_request(&req); + tracing::Span::current().record("tenant", tracing::field::debug(&tenant_id)); session_state .config_mut() .options_mut() @@ -179,6 +191,11 @@ pub async fn query(req: HttpRequest, query_request: Query) -> Result, @@ -238,6 +260,7 @@ async fn handle_non_streaming_query( tenant_id: &Option, ) -> Result { let first_table_name = table_name[0].clone(); + tracing::Span::current().record("table_name", first_table_name.as_str()); let (records, fields) = execute(query, query_request.streaming, tenant_id).await?; let records = match records { Either::Left(rbs) => rbs, @@ -283,6 +306,11 @@ async fn handle_non_streaming_query( /// /// # Returns /// - `HttpResponse` streaming the query results as NDJSON, optionally prefixed with the fields array. +#[tracing::instrument( + name = "query.stream", + skip(query, query_request, tenant_id), + fields(table_name = tracing::field::Empty) +)] async fn handle_streaming_query( query: LogicalQuery, table_name: Vec, @@ -291,6 +319,7 @@ async fn handle_streaming_query( tenant_id: &Option, ) -> Result { let first_table_name = table_name[0].clone(); + tracing::Span::current().record("table_name", first_table_name.as_str()); let (records_stream, fields) = execute(query, query_request.streaming, tenant_id).await?; let records_stream = match records_stream { Either::Left(_) => { @@ -453,6 +482,11 @@ pub async fn update_schema_when_distributed( /// Create streams for querier if they do not exist /// get list of streams from memory and storage /// create streams for memory from storage if they do not exist +#[tracing::instrument( + name = "distributed.create_streams", + skip(streams, tenant_id), + fields(stream_count = streams.len()) +)] pub async fn create_streams_for_distributed( streams: Vec, tenant_id: &Option, @@ -516,6 +550,11 @@ impl FromRequest for Query { } } +#[tracing::instrument( + name = "query.parse_logical_plan", + skip(query, session_state, time_range), + fields(sql = %query.query) +)] pub async fn into_query( query: &Query, session_state: &SessionState, diff --git a/src/handlers/http/rbac.rs b/src/handlers/http/rbac.rs index 0f8fd3840..dc3a5650f 100644 --- a/src/handlers/http/rbac.rs +++ b/src/handlers/http/rbac.rs @@ -41,6 +41,7 @@ use itertools::Itertools; use serde::Serialize; use serde_json::json; use tokio::sync::Mutex; +use tracing::instrument; use super::modal::utils::rbac_utils::{get_metadata, put_metadata}; @@ -69,8 +70,10 @@ impl From<&user::User> for User { // Handler for GET /api/v1/user // returns list of all registered users +#[tracing::instrument(name = "list_users", skip(req), fields(tenant_id = tracing::field::Empty))] pub async fn list_users(req: HttpRequest) -> impl Responder { let tenant_id = get_tenant_id_from_request(&req); + tracing::Span::current().record("tenant_id", tenant_id.as_deref().unwrap_or("default")); web::Json(Users.collect_user::(&tenant_id)) } @@ -217,12 +220,19 @@ pub async fn post_gen_password( // Handler for GET /api/v1/user/{userid}/role // returns role for a user if that user exists +#[instrument( + name = "get_role", + skip(req, userid), + fields(user_id = tracing::field::Empty, tenant_id = tracing::field::Empty) +)] pub async fn get_role( req: HttpRequest, userid: web::Path, ) -> Result { let userid = userid.into_inner(); + tracing::Span::current().record("user_id", &userid); let tenant_id = get_tenant_id_from_request(&req); + tracing::Span::current().record("tenant_id", tenant_id.as_deref().unwrap_or("default")); let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); if !Users.contains(&userid, &tenant_id) { return Err(RBACError::UserDoesNotExist); diff --git a/src/handlers/http/role.rs b/src/handlers/http/role.rs index abe8f3dc9..3256efbc0 100644 --- a/src/handlers/http/role.rs +++ b/src/handlers/http/role.rs @@ -109,8 +109,14 @@ pub async fn get(req: HttpRequest, name: web::Path) -> Result Result { let tenant_id = get_tenant_id_from_request(&req); + tracing::Span::current().record("tenant_id", tenant_id.as_deref().unwrap_or("none")); let metadata = get_metadata(&tenant_id).await?; let mut roles = HashMap::new(); for (k, r) in metadata.roles.into_iter() { @@ -186,9 +192,15 @@ pub async fn put_default( // Handler for GET /api/v1/role/default // Delete existing role +#[tracing::instrument( + name = "role.get_default", + skip(req), + fields(tenant_id = tracing::field::Empty) +)] pub async fn get_default(req: HttpRequest) -> Result { let tenant_id = get_tenant_id_from_request(&req); let tenant_id = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); + tracing::Span::current().record("tenant_id", tenant_id); let res = if let Some(role) = DEFAULT_ROLE .read() // .unwrap() @@ -211,6 +223,10 @@ pub async fn get_default(req: HttpRequest) -> Result Ok(web::Json(res)) } +#[tracing::instrument( + name = "role.get_metadata", + fields(tenant_id = ?tenant_id) +)] async fn get_metadata( tenant_id: &Option, ) -> Result { diff --git a/src/lib.rs b/src/lib.rs index 75b4254be..dbd7faf32 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,7 @@ mod static_schema; mod stats; pub mod storage; pub mod sync; +pub mod telemetry; pub mod tenants; pub mod users; pub mod utils; diff --git a/src/main.rs b/src/main.rs index 42cba34f5..60d340704 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,11 +17,12 @@ use std::process::exit; * along with this program. If not, see . * */ +use opentelemetry::trace::TracerProvider as _; #[cfg(feature = "kafka")] use parseable::connectors; use parseable::{ IngestServer, ParseableServer, QueryServer, Server, banner, metrics, option::Mode, - parseable::PARSEABLE, rbac, storage, + parseable::PARSEABLE, rbac, storage, telemetry, }; use tokio::signal::ctrl_c; use tokio::sync::oneshot; @@ -33,7 +34,7 @@ use tracing_subscriber::{EnvFilter, Registry, fmt}; #[actix_web::main] async fn main() -> anyhow::Result<()> { - init_logger(); + let otel_provider = init_logger(); // Install the rustls crypto provider before any TLS operations. // This is required for rustls 0.23+ which needs an explicit crypto provider. // If the installation fails, log a warning but continue execution. @@ -95,10 +96,16 @@ async fn main() -> anyhow::Result<()> { parseable_server.await?; } + if let Some(provider) = otel_provider { + if let Err(e) = provider.shutdown() { + warn!("Failed to shutdown OTel tracer provider: {:?}", e); + } + } + Ok(()) } -pub fn init_logger() { +pub fn init_logger() -> Option { let filter_layer = EnvFilter::try_from_default_env().unwrap_or_else(|_| { let default_level = if cfg!(debug_assertions) { Level::DEBUG @@ -116,10 +123,20 @@ pub fn init_logger() { .with_target(true) .compact(); - Registry::default() - .with(filter_layer) - .with(fmt_layer) - .init(); + let provider = telemetry::init_otel_tracer(); + if let Some(ref p) = provider { + Registry::default() + .with(filter_layer) + .with(fmt_layer) + .with(tracing_opentelemetry::layer().with_tracer(p.tracer("parseable"))) + .init(); + } else { + Registry::default() + .with(filter_layer) + .with(fmt_layer) + .init(); + } + provider } #[cfg(windows)] diff --git a/src/parseable/mod.rs b/src/parseable/mod.rs index 484fd5b8e..163e8ec8f 100644 --- a/src/parseable/mod.rs +++ b/src/parseable/mod.rs @@ -248,6 +248,11 @@ impl Parseable { /// Checks for the stream in memory, or loads it from storage when in distributed mode /// return true if stream exists in memory or loaded from storage /// return false if stream doesn't exist in memory and not loaded from storage + #[tracing::instrument( + name = "parseable.check_or_load_stream", + skip(self, tenant_id), + fields(stream_name) + )] pub async fn check_or_load_stream( &self, stream_name: &str, @@ -370,6 +375,11 @@ impl Parseable { /// list all streams from storage /// if stream exists in storage, create stream and schema from storage /// and add it to the memory map + #[tracing::instrument( + name = "storage.hydrate_stream", + skip(self, tenant_id), + fields(stream_name) + )] pub async fn create_stream_and_schema_from_storage( &self, stream_name: &str, diff --git a/src/query/mod.rs b/src/query/mod.rs index ef5eb0b7c..d75800598 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -48,6 +48,7 @@ use itertools::Itertools; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use std::collections::HashMap; use std::ops::Bound; use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -55,6 +56,8 @@ use std::sync::{Arc, RwLock}; use std::task::{Context, Poll}; use sysinfo::System; use tokio::runtime::Runtime; +use tracing::Instrument; +use tracing_opentelemetry::OpenTelemetrySpanExt; use self::error::ExecuteError; use self::stream_schema_provider::GlobalSchemaProvider; @@ -133,6 +136,11 @@ impl InMemorySessionContext { /// This function executes a query on the dedicated runtime, ensuring that the query is not isolated to a single thread/CPU /// at a time and has access to the entire thread pool, enabling better concurrent processing, and thus quicker results. +#[tracing::instrument( + name = "datafusion.dispatch", + skip(query, tenant_id), + fields(is_streaming) +)] pub async fn execute( query: Query, is_streaming: bool, @@ -165,8 +173,37 @@ pub async fn execute( ExecuteError, > { let id = tenant_id.clone(); + + // Serialize the current OTel context before crossing the QUERY_RUNTIME boundary. + // QUERY_RUNTIME is a separate tokio::runtime::Runtime on its own OS thread pool. + // Thread-local OTel context is NOT inherited by tasks spawned onto a different runtime. + let mut trace_cx: HashMap = HashMap::new(); + opentelemetry::global::get_text_map_propagator(|prop| { + prop.inject_context(&opentelemetry::Context::current(), &mut trace_cx); + }); + QUERY_RUNTIME - .spawn(async move { query.execute(is_streaming, &id).await }) + .spawn(async move { + // Extract and restore the W3C context inside QUERY_RUNTIME. + // `opentelemetry::ContextGuard` is `!Send`, so we must NOT hold it + // across an `.await`. Instead we create a `tracing::Span` (which IS + // `Send`), link it to the propagated parent via `set_parent`, and + // instrument the query call so its span appears as a child of + // `datafusion.dispatch` in the trace view. + let parent_cx = + opentelemetry::global::get_text_map_propagator(|prop| prop.extract(&trace_cx)); + let span = tracing::info_span!( + "datafusion.execute", + is_streaming = is_streaming, + tenant = tracing::field::Empty, + ); + let _ = span.set_parent(parent_cx); + span.record( + "tenant", + tracing::field::display(id.as_deref().unwrap_or(DEFAULT_TENANT)), + ); + query.execute(is_streaming, &id).instrument(span).await + }) .await .expect("The Join should have been successful") } @@ -526,6 +563,11 @@ impl CountsRequest { /// This function is supposed to read maninfest files for the given stream, /// get the sum of `num_rows` between the `startTime` and `endTime`, /// divide that by number of bins and return in a manner acceptable for the console + #[tracing::instrument( + name = "query.bin_density", + skip(self, tenant_id), + fields(stream = %self.stream, num_bins = self.num_bins.unwrap_or(1)) + )] pub async fn get_bin_density( &self, tenant_id: &Option, @@ -719,6 +761,11 @@ pub fn resolve_stream_names(sql: &str) -> Result, anyhow::Error> { Ok(tables) } +#[tracing::instrument( + name = "catalog.manifest_list", + skip(time_range, tenant_id), + fields(stream_name) +)] pub async fn get_manifest_list( stream_name: &str, time_range: &TimeRange, diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 000000000..285bc5a47 --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,72 @@ +/* + * Parseable Server (C) 2022 - 2025 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig}; +use opentelemetry_sdk::{Resource, propagation::TraceContextPropagator, trace::SdkTracerProvider}; + +/// Initialise an OTLP tracer provider. +/// +/// **Required env var:** +/// - `OTEL_EXPORTER_OTLP_ENDPOINT` — collector address. +/// For HTTP exporters the SDK appends the signal path automatically: +/// e.g. `http://localhost:4318` → `http://localhost:4318/v1/traces`. +/// +/// **Optional env var:** +/// - `OTEL_EXPORTER_OTLP_PROTOCOL` — transport + serialisation: +/// - `grpc` → gRPC / tonic (Jaeger, Tempo, …) +/// - (default) → HTTP / JSON (Parseable OSS ingest at `/v1/traces`) +/// +/// Returns `None` when `OTEL_EXPORTER_OTLP_ENDPOINT` is not set (OTEL disabled). +/// The caller must call `provider.shutdown()` before process exit. +pub fn init_otel_tracer() -> Option { + let endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok()?; + + // Register W3C TraceContext propagator globally — required for traceparent + // header extraction in middleware AND cross-runtime context propagation. + opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new()); + + let protocol = std::env::var("OTEL_EXPORTER_OTLP_PROTOCOL").unwrap_or_default(); + + let exporter = match protocol.as_str() { + "grpc" => SpanExporter::builder() + .with_tonic() + .with_endpoint(&endpoint) + .build() + .ok()?, + // HTTP/JSON is the default — required for Parseable OSS which only + // accepts application/json at /v1/traces. + _ => SpanExporter::builder() + .with_http() + .with_protocol(Protocol::HttpJson) + .with_endpoint(&endpoint) + .build() + .ok()?, + }; + + let resource = Resource::builder_empty() + .with_service_name("parseable") + .build(); + + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource(resource) + .build(); + + opentelemetry::global::set_tracer_provider(provider.clone()); + Some(provider) +}