From 23192b706d068a0e4eda96ca9b4ec9cc2e00122a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doru=20Bl=C3=A2nzeanu?= Date: Wed, 27 Nov 2024 19:31:17 +0200 Subject: [PATCH] Fix otlp tracing (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix otlp_tracing compilation issue Signed-off-by: Doru Blânzeanu * Fix logging filter so that jaeger can get the traces Signed-off-by: Doru Blânzeanu * Fix fmt check error Signed-off-by: Doru Blânzeanu * Add otlp_tracing test to ensure the traces are accessible for collectors Signed-off-by: Doru Blânzeanu --------- Signed-off-by: Doru Blânzeanu --- Cargo.lock | 29 ++-- docs/hyperlight-metrics-logs-and-traces.md | 6 +- src/hyperlight_host/Cargo.toml | 11 +- .../examples/otlp_tracing/main.rs | 124 ++++++++++++++---- 4 files changed, 129 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b9da8f02..071cfbd73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1107,6 +1107,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-otlp", + "opentelemetry-semantic-conventions", "opentelemetry_sdk", "page_size", "paste", @@ -1709,9 +1710,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" dependencies = [ "futures-core", "futures-sink", @@ -1723,9 +1724,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", @@ -1737,13 +1738,14 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1751,11 +1753,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + [[package]] name = "opentelemetry_sdk" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" dependencies = [ "async-trait", "futures-channel", @@ -1770,6 +1778,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -2882,9 +2891,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" dependencies = [ "js-sys", "once_cell", diff --git a/docs/hyperlight-metrics-logs-and-traces.md b/docs/hyperlight-metrics-logs-and-traces.md index accb33704..3a22e5a82 100644 --- a/docs/hyperlight-metrics-logs-and-traces.md +++ b/docs/hyperlight-metrics-logs-and-traces.md @@ -82,13 +82,13 @@ In the [examples/otlp_tracing](../src/hyperlight_host/examples/otlp_tracing) dir #### Linux ```bash -RUST_LOG='none,hyperlight-host=info,tracing=info' cargo run --example otlp_tracing +RUST_LOG='none,hyperlight_host=info,tracing=info' cargo run --example otlp_tracing ``` #### Windows ```powershell -$env:RUST_LOG='none,hyperlight-host=info,tracing=info';cargo run --example otlp_tracing +$env:RUST_LOG='none,hyperlight_host=info,tracing=info';cargo run --example otlp_tracing ``` The sample will run and generate trace data until any key is pressed. @@ -96,7 +96,7 @@ The sample will run and generate trace data until any key is pressed. To view the trace data, leave the example running and use the jaegertracing/all-in-one container image with the following command: ```console - docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.51 + docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 4317:4317 -p 16686:16686 jaegertracing/all-in-one:1.60 ``` NOTE: when running this on windows that this is a linux container, so you will need to ensure that docker is configured to run linux containers using WSL2. Alternatively, you can download the Jaeger binaries from [here](https://www.jaegertracing.io/download/). Extract the archive and run the `jaeger-all-in-one` executable as follows: diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index f4c36f416..0966177df 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -89,11 +89,12 @@ hyperlight-testing = { workspace = true } env_logger = "0.11.5" tracing-forest = { version = "0.1.6", features = ["uuid", "chrono", "smallvec", "serde", "env-filter"] } tracing = "0.1.37" -tracing-subscriber = { version = "0.3.18", features = ["std", "env-filter"] } -tracing-opentelemetry = "0.27.0" -opentelemetry = "0.26.0" -opentelemetry-otlp = { version = "0.26.0", features = ["default"] } -opentelemetry_sdk = { version = "0.26.0", features = ["rt-tokio"] } +tracing-subscriber = {version = "0.3.18", features = ["std", "env-filter"]} +tracing-opentelemetry = "0.28.0" +opentelemetry = "0.27.0" +opentelemetry-otlp = { version = "0.27.0", features = ["default"] } +opentelemetry-semantic-conventions = "0.27.0" +opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] } tokio = { version = "1.34.0", features = ["full"] } criterion = "0.5.1" tracing-chrome = "0.7.2" diff --git a/src/hyperlight_host/examples/otlp_tracing/main.rs b/src/hyperlight_host/examples/otlp_tracing/main.rs index d80e0a806..5548c4840 100644 --- a/src/hyperlight_host/examples/otlp_tracing/main.rs +++ b/src/hyperlight_host/examples/otlp_tracing/main.rs @@ -17,6 +17,8 @@ limitations under the License. use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType}; use rand::Rng; use tracing::{span, Level}; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; extern crate hyperlight_host; use std::error::Error; @@ -29,16 +31,19 @@ use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; use hyperlight_host::sandbox_state::transition::Noop; use hyperlight_host::{GuestBinary, MultiUseSandbox, Result as HyperlightResult}; use hyperlight_testing::simple_guest_as_string; -use opentelemetry::global::shutdown_tracer_provider; +use opentelemetry::global::{self, shutdown_tracer_provider}; use opentelemetry::trace::TracerProvider; use opentelemetry::KeyValue; -use opentelemetry_otlp::{new_exporter, new_pipeline, WithExportConfig}; +use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::runtime::Tokio; use opentelemetry_sdk::{trace, Resource}; -use tracing_subscriber::layer::SubscriberExt; +use opentelemetry_semantic_conventions::attribute::{SERVICE_NAME, SERVICE_VERSION}; +use opentelemetry_semantic_conventions::SCHEMA_URL; use tracing_subscriber::EnvFilter; use uuid::Uuid; +const ENDPOINT_ADDR: &str = "http://localhost:4317"; + fn fn_writer(_msg: String) -> HyperlightResult { Ok(0) } @@ -47,30 +52,51 @@ fn fn_writer(_msg: String) -> HyperlightResult { #[tokio::main] async fn main() -> Result<(), Box> { - let tracer = new_pipeline() - .tracing() - .with_exporter( - new_exporter() - .tonic() - .with_endpoint("http://localhost:4317/v1/traces"), + init_tracing_subscriber(ENDPOINT_ADDR)?; + + Ok(run_example(true)?) +} + +fn init_tracing_subscriber(addr: &str) -> Result<(), Box> { + let exporter = SpanExporter::builder() + .with_tonic() + .with_endpoint(addr) + .build()?; + + let provider = trace::TracerProvider::builder() + .with_config( + trace::Config::default().with_resource(Resource::from_schema_url( + vec![ + KeyValue::new(SERVICE_NAME, "hyperlight_otel_example"), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + ], + SCHEMA_URL, + )), ) - .with_trace_config(trace::Config::default().with_resource(Resource::new(vec![ - KeyValue::new("service.name", "hyperlight_otel_example"), - ]))) - .install_batch(Tokio) - .unwrap() - .tracer("trace-demo"); + .with_batch_exporter(exporter, Tokio) + .build(); + + global::set_tracer_provider(provider.clone()); + let tracer = provider.tracer("trace-demo"); + + let otel_layer = OpenTelemetryLayer::new(tracer); - let otel_layer = tracing_opentelemetry::OpenTelemetryLayer::new(tracer); + // Try using the environment otherwise set default filters + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + EnvFilter::from_default_env() + .add_directive("hyperlight_host=info".parse().unwrap()) + .add_directive("tracing=info".parse().unwrap()) + }); - tracing_subscriber::Registry::default() - .with(EnvFilter::from_default_env()) + tracing_subscriber::registry() + .with(filter) .with(otel_layer) .try_init()?; - Ok(run_example()?) + Ok(()) } -fn run_example() -> HyperlightResult<()> { + +fn run_example(wait_input: bool) -> HyperlightResult<()> { // Get the path to a simple guest binary. let hyperlight_guest_path = simple_guest_as_string().expect("Cannot find the guest binary at the expected location."); @@ -168,9 +194,12 @@ fn run_example() -> HyperlightResult<()> { join_handles.push(handle); } - println!("Press enter to exit..."); - let mut input = String::new(); - stdin().read_line(&mut input)?; + if wait_input { + println!("Press enter to exit..."); + let mut input = String::new(); + stdin().read_line(&mut input)?; + } + *should_exit.try_lock().unwrap() = true; for join_handle in join_handles { let result = join_handle.join(); @@ -180,3 +209,52 @@ fn run_example() -> HyperlightResult<()> { Ok(()) } + +#[cfg(test)] +mod test { + use hyperlight_host::{HyperlightError, Result}; + use tokio::io::AsyncReadExt; + use tokio::net::{TcpListener, TcpStream}; + + use super::*; + + const TESTER_ADDR: &str = "127.0.0.1:4317"; + + async fn handle(mut stream: TcpStream) -> Result<()> { + let mut buf = Vec::with_capacity(128); + let size = stream.read_buf(&mut buf).await?; + + if size > 0 { + Ok(()) + } else { + Err(HyperlightError::Error("Cannot read req body".to_string())) + } + } + + async fn check_otl_connection(addr: &str) -> Result<()> { + let listener = TcpListener::bind(addr).await?; + + let (stream, _) = listener.accept().await?; + + handle(stream).await + } + + #[tokio::test] + async fn test_subscriber() { + // Create task that generates spans + let task = tokio::spawn(async move { + let _ = init_tracing_subscriber(ENDPOINT_ADDR); + + // No need to wait for input, just generate some spans and exit + let _ = run_example(false); + }); + + // Create server that listens and checks to see if traces are received + let result = check_otl_connection(TESTER_ADDR).await; + + // Abort task in case it doesn't finish + task.abort(); + + assert!(result.is_ok()); + } +}