From 921a2cd531d8d0073f22bb1b9fc8d2ac2e26c7a5 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 21 Sep 2026 12:31:21 -0300 Subject: [PATCH] feat: export OTLP traces to a local file Relay's OpenTelemetry plugin could only ship spans to a collector. ATIF and ATOF can both be written to a file, so OTLP was the one trajectory output that could not be kept on disk. An evaluation harness that scores the trace it just produced, an offline replay, and any environment with no collector all had to stand up an OTLP receiver purely to catch the export and write it out. `[[components.config.opentelemetry.file_sinks]]` writes the same `ExportTraceServiceRequest` an endpoint would receive. The default `json_lines` format implements the OpenTelemetry Protocol File Exporter specification, one OTLP/JSON record per line; `proto` writes the same records length-delimited for consumers that would rather not pay JSON's size and parse cost. The conversion comes from `opentelemetry-proto`, the crate `opentelemetry-otlp` already uses to build its wire payload, so a span means the same thing whichever destination it goes to. That crate and `prost` were already present as dev dependencies of `nemo-relay` and as transitive dependencies of the workspace; this promotes them to direct dependencies and adds the `trace` and `with-serde` features. Nothing new enters `Cargo.lock` except `tempfile`, a dev dependency of the Python crate's tests. The endpoint and the file sink are separate config types, matching the split the bindings already expose. Options they share live in `SharedTraceOptions`, so a file sink cannot be given an endpoint, a transport, headers, or a timeout: those are absent from the type rather than rejected at runtime. Existing configuration files are unaffected, since `file_sinks` is a new optional array that is skipped on serialization when empty. Each export is flushed before it is reported as delivered, so a run that exits between batches leaves a readable prefix. Output is created with owner-only permissions and confined to `output_directory`, matching the ATOF and ATIF sinks. Two file sinks writing one path are rejected at activation. The process-global OTLP header variables are rejected only for a network destination, since they cannot reach a file. `nemo-relay plugins edit` lists file sinks beside trace endpoints, and the Python, Node.js, Go, and C FFI surfaces each gain the destination. Relates to #1089 Signed-off-by: Sandy Chapman Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + crates/core/Cargo.toml | 3 +- crates/core/src/observability/mod.rs | 1 + crates/core/src/observability/otel.rs | 747 +++++++++++------- crates/core/src/observability/otel_file.rs | 226 ++++++ .../src/observability/plugin_component.rs | 427 +++++++++- .../unit/observability/otel_file_tests.rs | 626 +++++++++++++++ .../tests/unit/observability/otel_tests.rs | 50 +- .../observability/plugin_component_tests.rs | 422 ++++++++++ crates/ffi/nemo_relay.h | 24 + crates/ffi/src/api/observability.rs | 162 ++++ .../tests/unit/api/coverage_sweeps_tests.rs | 251 ++++++ crates/node/observability.d.ts | 29 + crates/node/observability.js | 38 + crates/node/src/api/mod.rs | 128 ++- .../node/tests/observability_plugin_tests.mjs | 54 ++ crates/python/Cargo.toml | 1 + crates/python/src/py_types/mod.rs | 1 + crates/python/src/py_types/observability.rs | 184 ++++- .../tests/coverage/py_types_coverage_tests.rs | 89 ++- .../observability/opentelemetry.mdx | 50 ++ ...emo-relay-events-2026-09-21-15.24.52.jsonl | 0 ...emo-relay-events-2026-09-21-15.25.33.jsonl | 0 go/nemo_relay/nemo_relay.go | 103 +++ go/nemo_relay/observability_plugin.go | 26 + go/nemo_relay/otel_test.go | 169 ++++ python/nemo_relay/__init__.py | 2 + python/nemo_relay/__init__.pyi | 3 + python/nemo_relay/_native.pyi | 34 + python/tests/test_types.py | 73 ++ 30 files changed, 3597 insertions(+), 327 deletions(-) create mode 100644 crates/core/src/observability/otel_file.rs create mode 100644 crates/core/tests/unit/observability/otel_file_tests.rs create mode 100644 go/nemo_relay/nemo-relay-events-2026-09-21-15.24.52.jsonl create mode 100644 go/nemo_relay/nemo-relay-events-2026-09-21-15.25.33.jsonl diff --git a/Cargo.lock b/Cargo.lock index 37e7e17d4..7cfa0954d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1971,6 +1971,7 @@ dependencies = [ "pythonize", "serde", "serde_json", + "tempfile", "tokio", "tokio-stream", "uuid", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 1091a3f53..0983acc58 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -79,6 +79,8 @@ base64 = "0.22" ring = ">=0.17.13, <0.18" jsonschema = { version = "0.46.6", default-features = false } percent-encoding = "2" +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "trace", "with-serde"] } +prost = "0.14" opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "logs", "metrics", "http-proto", "http-json", "reqwest-client", "reqwest-rustls", "grpc-tonic", "gzip-http", "gzip-tonic", "zstd-http", "zstd-tonic", "tls-ring", "tls-roots"] } opentelemetry-http = "0.32" async-trait = "0.1" @@ -104,7 +106,6 @@ tokio = { version = "1", features = ["rt", "macros", "sync", "test-util", "rt-mu futures = "0.3" opentelemetry_sdk = { workspace = true, features = ["trace", "logs", "metrics", "testing"] } opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "logs", "metrics"] } -prost = "0.14" tonic = { version = "0.14.1", features = ["transport"] } tokio-stream = { version = "0.1", features = ["net"] } serde_json = "1" diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index 154853001..0017ef1e9 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -45,6 +45,7 @@ pub(crate) mod header_file; pub(crate) mod manual; pub(crate) mod openinference; pub mod otel; +pub mod otel_file; mod otel_genai; pub mod otel_logs; pub mod otel_metrics; diff --git a/crates/core/src/observability/otel.rs b/crates/core/src/observability/otel.rs index 51b568970..6b281a97d 100644 --- a/crates/core/src/observability/otel.rs +++ b/crates/core/src/observability/otel.rs @@ -18,6 +18,7 @@ use std::borrow::Cow; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Mutex}; @@ -28,6 +29,7 @@ use super::header_file::{ HeaderFileHttpClient, HeaderFileInterceptor, HeaderFileResolver, HeaderFiles, validate_header_files, }; +use super::otel_file::{OtlpFileFormat, OtlpFileSpanExporter}; use super::otel_signal::{ MetricMarkClassification, SignalRuntimeDiagnostics, TELEMETRY_SDK_RESOURCE_ATTRIBUTE_KEYS, automatic_otlp_http_client, automatic_protocol_is_grpc, automatic_protocol_is_unset, @@ -265,14 +267,26 @@ pub(super) fn validate_trace_endpoint(endpoint: &str) -> Result<()> { Ok(()) } -/// Configuration for the OpenTelemetry subscriber. +/// A local file destination for exported spans. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OtlpFileSinkSettings { + /// Directory the output is confined to. + pub output_directory: PathBuf, + /// Full path of the output file, which must live under `output_directory`. + pub path: PathBuf, + /// On-disk encoding. + pub format: OtlpFileFormat, + /// Whether an existing file is appended to rather than truncated. + pub append: bool, +} + +/// Trace options that apply to any destination. +/// +/// Held by both [`OpenTelemetryConfig`] and [`OpenTelemetryFileSinkConfig`] so +/// the projection and batching settings stay identical between them. #[derive(Debug, Clone)] -pub struct OpenTelemetryConfig { +pub(crate) struct SharedTraceOptions { otel_type: OpenTelemetryType, - endpoint: String, - headers: HashMap, - header_env: HashMap, - header_file: HeaderFiles, resource_attributes: HashMap, service_name: Option, service_namespace: Option, @@ -283,23 +297,16 @@ pub struct OpenTelemetryConfig { attribute_mappings: Vec, promote_metadata_prefixes: Vec, promote_resource_metadata_prefixes: Vec, - timeout: Duration, - transport: OtlpTransport, max_queue_size: Option, max_export_batch_size: Option, scheduled_delay: Option, completed_span_context_ttl: Duration, - automatic: bool, } -impl OpenTelemetryConfig { - fn default_values() -> Self { +impl SharedTraceOptions { + fn new(otel_type: OpenTelemetryType) -> Self { Self { - otel_type: OpenTelemetryType::Full, - endpoint: String::new(), - headers: HashMap::new(), - header_env: HashMap::new(), - header_file: HashMap::new(), + otel_type, resource_attributes: HashMap::new(), service_name: None, service_namespace: None, @@ -310,22 +317,247 @@ impl OpenTelemetryConfig { attribute_mappings: Vec::new(), promote_metadata_prefixes: Vec::new(), promote_resource_metadata_prefixes: Vec::new(), - timeout: Duration::from_secs(3), - transport: OtlpTransport::HttpBinary, max_queue_size: None, max_export_batch_size: None, scheduled_delay: None, completed_span_context_ttl: DEFAULT_COMPLETED_SPAN_CONTEXT_TTL, - automatic: false, + } + } +} + +/// Configuration for an OpenTelemetry subscriber exporting to a collector. +#[derive(Debug, Clone)] +pub struct OpenTelemetryConfig { + shared: SharedTraceOptions, + /// Whether the exporter takes its endpoint and protocol from OTEL_* vars. + automatic: bool, + endpoint: String, + transport: OtlpTransport, + headers: HashMap, + header_env: HashMap, + header_file: HeaderFiles, + timeout: Duration, +} + +/// Configuration for an OpenTelemetry subscriber writing OTLP to a local file. +/// +/// Carries no endpoint, transport, headers, or timeout: a file destination has +/// no use for any of them, so they are absent rather than rejected. +#[derive(Debug, Clone)] +pub struct OpenTelemetryFileSinkConfig { + shared: SharedTraceOptions, + sink: OtlpFileSinkSettings, +} + +/// Generates the builders and accessors every trace config shares. +/// +/// Both destinations carry the same projection, resource, and batching +/// options; only the transport-specific ones differ, and those are written out +/// on the type that has them. +macro_rules! shared_trace_options { + ($config:ty) => { + impl $config { + /// Sets the `service.name` resource attribute. + pub fn with_service_name(mut self, service_name: impl Into) -> Self { + self.shared.service_name = Some(service_name.into()); + self + } + + /// Sets the optional `service.namespace` resource attribute. + pub fn with_service_namespace(mut self, namespace: impl Into) -> Self { + self.shared.service_namespace = Some(namespace.into()); + self + } + + /// Sets the optional `service.version` resource attribute. + pub fn with_service_version(mut self, version: impl Into) -> Self { + self.shared.service_version = Some(version.into()); + self + } + + /// Sets the instrumentation scope name. + pub fn with_instrumentation_scope(mut self, scope: impl Into) -> Self { + self.shared.instrumentation_scope = scope.into(); + self + } + + /// Adds a resource attribute as a string key/value pair. + pub fn with_resource_attribute( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + self.shared + .resource_attributes + .insert(key.into(), value.into()); + self + } + + /// Selects how point-in-time marks are represented. + pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { + self.shared.mark_projection = mark_projection; + self + } + + /// Replaces the mark names excluded from tool projection. + pub fn with_mark_exclude_names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.shared.mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } + + /// Copies a projected attribute to a second attribute name. + pub fn with_attribute_mapping( + mut self, + key: impl Into, + alias: impl Into, + ) -> Self { + self.shared + .attribute_mappings + .push(OtlpAttributeMapping::new(key, alias)); + self + } + + /// Replaces the projected attribute aliases. + pub fn with_attribute_mappings(mut self, mappings: I) -> Self + where + I: IntoIterator, + { + self.shared.attribute_mappings = mappings.into_iter().collect(); + self + } + + /// Replaces the Event metadata prefixes copied to span attributes. + pub fn with_promote_metadata_prefixes(mut self, prefixes: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.shared.promote_metadata_prefixes = + prefixes.into_iter().map(Into::into).collect(); + self + } + + /// Replaces the root-scope metadata prefixes copied to resource attributes. + pub fn with_promote_resource_metadata_prefixes(mut self, prefixes: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.shared.promote_resource_metadata_prefixes = + prefixes.into_iter().map(Into::into).collect(); + self + } + + /// Sets how long completed scopes retain trace context for late marks. + pub fn with_completed_span_context_ttl(mut self, ttl: Duration) -> Self { + self.shared.completed_span_context_ttl = ttl; + self + } + + /// Overrides the batch processor queue size. + pub(crate) fn with_max_queue_size(mut self, max_queue_size: usize) -> Self { + self.shared.max_queue_size = Some(max_queue_size); + self + } + + /// Overrides the maximum number of spans exported in one batch. + pub(crate) fn with_max_export_batch_size( + mut self, + max_export_batch_size: usize, + ) -> Self { + self.shared.max_export_batch_size = Some(max_export_batch_size); + self + } + + /// Overrides the delay before a non-full batch is exported. + pub(crate) fn with_scheduled_delay(mut self, scheduled_delay: Duration) -> Self { + self.shared.scheduled_delay = Some(scheduled_delay); + self + } + + #[allow(dead_code)] + pub(crate) fn batch_overrides( + &self, + ) -> (Option, Option, Option) { + ( + self.shared.max_queue_size, + self.shared.max_export_batch_size, + self.shared.scheduled_delay, + ) + } + + #[allow(dead_code)] + pub(crate) fn completed_span_context_ttl(&self) -> Duration { + self.shared.completed_span_context_ttl + } + + #[allow(dead_code)] + pub(crate) fn promote_resource_metadata_prefixes(&self) -> &[String] { + &self.shared.promote_resource_metadata_prefixes + } + } + }; +} + +/// The config a subscriber was built from. +/// +/// Private: callers pass one of the two public config types. The runtime keeps +/// it because resource-metadata promotion builds additional providers after +/// startup, and those need the settings the subscriber was created with. +#[derive(Debug, Clone)] +pub(crate) enum TraceConfig { + Endpoint(OpenTelemetryConfig), + File(OpenTelemetryFileSinkConfig), +} + +impl TraceConfig { + fn shared(&self) -> &SharedTraceOptions { + match self { + Self::Endpoint(config) => &config.shared, + Self::File(config) => &config.shared, } } + /// The name used in diagnostics and delivery errors. Never a URL's + /// credentials or query: see [`trace_endpoint_log_identity`]. + fn label(&self) -> String { + match self { + Self::Endpoint(config) => config.endpoint.clone(), + Self::File(config) => config.sink.path.display().to_string(), + } + } + + /// Bounds retrying `force_flush` and `shutdown` while the batch + /// processor's queue is full. The queue fills from the producer side, so + /// this is not a property of the destination; an endpoint reuses its + /// request timeout and a file sink has no equivalent knob to reuse. + fn channel_full_retry_budget(&self) -> Duration { + match self { + Self::Endpoint(config) => config.timeout, + Self::File(_) => Duration::from_secs(3), + } + } +} + +shared_trace_options!(OpenTelemetryConfig); +shared_trace_options!(OpenTelemetryFileSinkConfig); + +impl OpenTelemetryConfig { /// Creates a typed OpenTelemetry exporter for a required OTLP endpoint. pub fn new(otel_type: OpenTelemetryType, endpoint: impl Into) -> Self { Self { - otel_type, + shared: SharedTraceOptions::new(otel_type), + automatic: false, endpoint: endpoint.into(), - ..Self::default_values() + transport: OtlpTransport::HttpBinary, + headers: HashMap::new(), + header_env: HashMap::new(), + header_file: HashMap::new(), + timeout: Duration::from_secs(3), } } @@ -333,30 +565,26 @@ impl OpenTelemetryConfig { /// exporter environment configuration. pub(crate) fn from_automatic_configuration() -> Self { Self { - endpoint: AUTOMATIC_OTLP_ENDPOINT_MARKER.to_string(), automatic: true, - ..Self::default_values() + ..Self::new( + OpenTelemetryType::Full, + AUTOMATIC_OTLP_ENDPOINT_MARKER.to_string(), + ) } } /// Creates an HTTP OTLP config for the given service name. #[cfg(test)] pub(crate) fn http_binary(service_name: impl Into) -> Self { - Self { - service_name: Some(service_name.into()), - transport: OtlpTransport::HttpBinary, - ..Self::default_values() - } + Self::new(OpenTelemetryType::Full, String::new()).with_service_name(service_name) } /// Creates a gRPC OTLP config for the given service name. #[cfg(test)] pub(crate) fn grpc(service_name: impl Into) -> Self { - Self { - service_name: Some(service_name.into()), - transport: OtlpTransport::Grpc, - ..Self::default_values() - } + Self::new(OpenTelemetryType::Full, String::new()) + .with_transport(OtlpTransport::Grpc) + .with_service_name(service_name) } /// Overrides the OTLP endpoint. If unset, exporter defaults and OTEL_* env vars apply. @@ -371,12 +599,6 @@ impl OpenTelemetryConfig { self } - /// Sets the `service.name` resource attribute. - pub fn with_service_name(mut self, service_name: impl Into) -> Self { - self.service_name = Some(service_name.into()); - self - } - /// Adds a header/metadata entry for the exporter. pub fn with_header(mut self, key: impl Into, value: impl Into) -> Self { self.headers.insert(key.into(), value.into()); @@ -389,6 +611,7 @@ impl OpenTelemetryConfig { self } + /// Maps an exporter header name to the file supplying its value. pub(crate) fn with_header_file( mut self, key: impl Into, @@ -403,147 +626,32 @@ impl OpenTelemetryConfig { self.headers.get(key).map(String::as_str) } - /// Adds a resource attribute as a string key/value pair. - pub fn with_resource_attribute( - mut self, - key: impl Into, - value: impl Into, - ) -> Self { - self.resource_attributes.insert(key.into(), value.into()); - self - } - /// Sets the OTLP request timeout. pub fn with_timeout(mut self, timeout: Duration) -> Self { self.timeout = timeout; self } +} - /// Overrides the batch processor queue size for this endpoint. - pub(crate) fn with_max_queue_size(mut self, max_queue_size: usize) -> Self { - self.max_queue_size = Some(max_queue_size); - self - } - - /// Overrides the maximum export batch size for this endpoint. - pub(crate) fn with_max_export_batch_size(mut self, max_export_batch_size: usize) -> Self { - self.max_export_batch_size = Some(max_export_batch_size); - self - } - - /// Overrides the maximum delay before exporting a non-full batch. - pub(crate) fn with_scheduled_delay(mut self, scheduled_delay: Duration) -> Self { - self.scheduled_delay = Some(scheduled_delay); - self - } - - /// Sets how long completed scopes retain their trace context for late marks. - /// - /// The value must be greater than zero. Subscriber construction fails with - /// [`OpenTelemetryError::ExporterBuild`] when the TTL is zero. - pub fn with_completed_span_context_ttl(mut self, ttl: Duration) -> Self { - self.completed_span_context_ttl = ttl; - self - } - - #[cfg(test)] - pub(crate) fn batch_overrides(&self) -> (Option, Option, Option) { - ( - self.max_queue_size, - self.max_export_batch_size, - self.scheduled_delay, - ) - } - - #[cfg(test)] - pub(crate) fn completed_span_context_ttl(&self) -> Duration { - self.completed_span_context_ttl - } - - /// Sets the service namespace resource attribute. - pub fn with_service_namespace(mut self, namespace: impl Into) -> Self { - self.service_namespace = Some(namespace.into()); - self - } - - /// Sets the service version resource attribute. - pub fn with_service_version(mut self, version: impl Into) -> Self { - self.service_version = Some(version.into()); - self - } - - /// Sets the instrumentation scope name used for emitted spans. - pub fn with_instrumentation_scope(mut self, scope: impl Into) -> Self { - self.instrumentation_scope = scope.into(); - self - } - - /// Selects how point-in-time marks are represented in exported traces. - pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { - self.mark_projection = mark_projection; - self - } - - /// Excludes named marks from tool projection. - pub fn with_mark_exclude_names(mut self, names: I) -> Self - where - I: IntoIterator, - S: Into, - { - self.mark_exclude_names = names.into_iter().map(Into::into).collect(); - self - } - - /// Adds a projected OpenTelemetry attribute alias. - pub fn with_attribute_mapping( - mut self, - key: impl Into, - alias: impl Into, - ) -> Self { - self.attribute_mappings - .push(OtlpAttributeMapping::new(key, alias)); - self - } - - /// Replaces projected OpenTelemetry attribute aliases. - pub fn with_attribute_mappings(mut self, mappings: I) -> Self - where - I: IntoIterator, - { - self.attribute_mappings = mappings.into_iter().collect(); - self - } - - /// Selects literal Event metadata prefixes copied to OTLP attributes. - pub fn with_promote_metadata_prefixes(mut self, prefixes: I) -> Self - where - I: IntoIterator, - S: Into, - { - self.promote_metadata_prefixes = prefixes.into_iter().map(Into::into).collect(); - self - } - - /// Selects literal root-scope Event metadata prefixes copied to OTLP resource attributes. - pub fn with_promote_resource_metadata_prefixes(mut self, prefixes: I) -> Self - where - I: IntoIterator, - S: Into, - { - self.promote_resource_metadata_prefixes = prefixes.into_iter().map(Into::into).collect(); - self +impl OpenTelemetryFileSinkConfig { + /// Creates a config writing OTLP to the file `sink` describes. + pub fn new(otel_type: OpenTelemetryType, sink: OtlpFileSinkSettings) -> Self { + Self { + shared: SharedTraceOptions::new(otel_type), + sink, + } } - #[cfg(test)] - pub(crate) fn promote_resource_metadata_prefixes(&self) -> &[String] { - &self.promote_resource_metadata_prefixes + /// Returns the file this config writes to. + pub fn sink(&self) -> &OtlpFileSinkSettings { + &self.sink } } #[cfg(test)] impl Default for OpenTelemetryConfig { fn default() -> Self { - Self::default_values() + Self::new(OpenTelemetryType::Full, String::new()) } } @@ -627,7 +735,12 @@ impl Drop for ExporterRuntime { impl OpenTelemetrySubscriber { /// Builds a subscriber backed by a new OTLP tracer provider. pub fn new(config: OpenTelemetryConfig) -> Result { - Self::new_with_runtime_diagnostics(config, None) + Self::new_with_runtime_diagnostics(TraceConfig::Endpoint(config), None) + } + + /// Creates a subscriber writing OTLP to a local file. + pub fn new_file_sink(config: OpenTelemetryFileSinkConfig) -> Result { + Self::new_with_runtime_diagnostics(TraceConfig::File(config), None) } pub(crate) fn new_for_plugin( @@ -635,73 +748,99 @@ impl OpenTelemetrySubscriber { endpoint_index: usize, ) -> Result { Self::new_with_runtime_diagnostics( - config, + TraceConfig::Endpoint(config), Some(format!("opentelemetry.traces[{endpoint_index}].endpoint")), ) } + /// Creates a plugin-managed subscriber for a configured file sink. + pub(crate) fn new_for_plugin_file_sink( + config: OpenTelemetryFileSinkConfig, + file_sink_index: usize, + ) -> Result { + Self::new_with_runtime_diagnostics( + TraceConfig::File(config), + Some(format!( + "opentelemetry.file_sinks[{file_sink_index}].output_directory" + )), + ) + } + pub(crate) fn new_from_automatic_configuration_for_plugin() -> Result { Self::new_with_runtime_diagnostics( - OpenTelemetryConfig::from_automatic_configuration(), + TraceConfig::Endpoint(OpenTelemetryConfig::from_automatic_configuration()), Some("opentelemetry.automatic.traces".to_string()), ) } fn new_with_runtime_diagnostics( - mut config: OpenTelemetryConfig, + mut config: TraceConfig, diagnostic_field: Option, ) -> Result { - if !config.automatic && config.endpoint.trim().is_empty() { - return Err(OpenTelemetryError::ExporterBuild( - "endpoint must be a nonblank string".to_string(), - )); - } - if config.automatic { - let endpoint = automatic_signal_endpoint("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") - .ok_or_else(|| { - OpenTelemetryError::ExporterBuild( - "automatic trace exporter requires a nonblank OTLP endpoint".to_string(), - ) - })?; - validate_trace_endpoint(&endpoint)?; - } else { - validate_trace_endpoint(&config.endpoint)?; - } - if config.completed_span_context_ttl.is_zero() { + let shared = config.shared(); + if shared.completed_span_context_ttl.is_zero() { return Err(OpenTelemetryError::ExporterBuild( "completed_span_context_ttl must be greater than 0".to_string(), )); } - validate_attribute_mappings(&config.attribute_mappings) + validate_attribute_mappings(&shared.attribute_mappings) .map_err(OpenTelemetryError::InvalidAttributeMappings)?; - validate_metadata_promotion_prefixes(&config.promote_metadata_prefixes) + validate_metadata_promotion_prefixes(&shared.promote_metadata_prefixes) .map_err(OpenTelemetryError::InvalidMetadataPromotionPrefixes)?; - validate_metadata_promotion_prefixes(&config.promote_resource_metadata_prefixes) + validate_metadata_promotion_prefixes(&shared.promote_resource_metadata_prefixes) .map_err(OpenTelemetryError::InvalidMetadataPromotionPrefixes)?; - if !config.automatic { - validate_telemetry_sdk_resource_attributes(&config.resource_attributes)?; - reject_global_header_environment()?; - validate_headers(&config.headers)?; - validate_header_files(&config.headers, &config.header_env, &config.header_file) + // An automatic endpoint takes its settings from the environment, so the + // checks below do not apply to it. A file sink has no endpoint at all. + // Resource attributes apply to either destination. An automatic + // endpoint takes them from the environment, so it is exempt. + let automatic = matches!(&config, TraceConfig::Endpoint(endpoint) if endpoint.automatic); + if !automatic { + validate_telemetry_sdk_resource_attributes(&shared.resource_attributes)?; + } + if let TraceConfig::Endpoint(endpoint) = &mut config { + if endpoint.automatic { + let resolved = automatic_signal_endpoint("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + .ok_or_else(|| { + OpenTelemetryError::ExporterBuild( + "automatic trace exporter requires a nonblank OTLP endpoint" + .to_string(), + ) + })?; + validate_trace_endpoint(&resolved)?; + } else { + if endpoint.endpoint.trim().is_empty() { + return Err(OpenTelemetryError::ExporterBuild( + "endpoint must be a nonblank string".to_string(), + )); + } + validate_trace_endpoint(&endpoint.endpoint)?; + reject_global_header_environment()?; + validate_headers(&endpoint.headers)?; + validate_header_files( + &endpoint.headers, + &endpoint.header_env, + &endpoint.header_file, + ) .map_err(OpenTelemetryError::ExporterBuild)?; - config.headers = resolve_header_env(&config.headers, &config.header_env)?; - validate_headers(&config.headers)?; + endpoint.headers = resolve_header_env(&endpoint.headers, &endpoint.header_env)?; + validate_headers(&endpoint.headers)?; + } } let runtime_diagnostics = SignalRuntimeDiagnostics::new(diagnostic_field); let (provider, runtime) = build_owned_tracer_provider(config.clone(), runtime_diagnostics.clone())?; - let owned_config = config.clone(); + let shared = config.shared().clone(); Ok(Self::from_tracer_provider_with_scope_and_type( provider, - config.instrumentation_scope, - config.otel_type, - config.mark_projection, - config.mark_exclude_names, - config.attribute_mappings, - config.promote_metadata_prefixes, - config.completed_span_context_ttl, + shared.instrumentation_scope, + shared.otel_type, + shared.mark_projection, + shared.mark_exclude_names, + shared.attribute_mappings, + shared.promote_metadata_prefixes, + shared.completed_span_context_ttl, Some(runtime), - Some(owned_config), + Some(config), )) } @@ -831,7 +970,7 @@ impl OpenTelemetrySubscriber { promote_metadata_prefixes: Vec, completed_span_context_ttl: Duration, runtime: Option, - owned_config: Option, + owned_config: Option, ) -> Self { let runtime_diagnostics = runtime .as_ref() @@ -1007,7 +1146,7 @@ fn shutdown_trace_providers( } fn build_owned_tracer_provider( - config: OpenTelemetryConfig, + config: TraceConfig, runtime_diagnostics: SignalRuntimeDiagnostics, ) -> Result<(SdkTracerProvider, ExporterRuntime)> { let resource_attributes = configured_resource_attributes(&config); @@ -1015,7 +1154,7 @@ fn build_owned_tracer_provider( } fn build_owned_tracer_provider_with_resource( - config: OpenTelemetryConfig, + config: TraceConfig, runtime_diagnostics: SignalRuntimeDiagnostics, resource_attributes: Vec, ) -> Result<(SdkTracerProvider, ExporterRuntime)> { @@ -1108,89 +1247,118 @@ fn build_tracer_provider( config: &OpenTelemetryConfig, runtime_diagnostics: SignalRuntimeDiagnostics, ) -> Result { - build_tracer_provider_with_resource( - config, - runtime_diagnostics, - configured_resource_attributes(config), - ) + let config = TraceConfig::Endpoint(config.clone()); + let attributes = configured_resource_attributes(&config); + build_tracer_provider_with_resource(&config, runtime_diagnostics, attributes) } fn build_tracer_provider_with_resource( - config: &OpenTelemetryConfig, + config: &TraceConfig, runtime_diagnostics: SignalRuntimeDiagnostics, resource_attributes: Vec, ) -> Result { - let exporter = if config.automatic { + match config { + TraceConfig::File(file) => provider_with_exporter( + OtlpFileSpanExporter::new( + &file.sink.output_directory, + &file.sink.path, + file.sink.format, + file.sink.append, + ) + .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))?, + config, + runtime_diagnostics, + resource_attributes, + ), + TraceConfig::Endpoint(endpoint) => provider_with_exporter( + otlp_span_exporter(endpoint)?, + config, + runtime_diagnostics, + resource_attributes, + ), + } +} + +/// Builds the OTLP network exporter for an endpoint destination. +fn otlp_span_exporter(settings: &OpenTelemetryConfig) -> Result { + if settings.automatic { let builder = OtlpSpanExporter::builder(); - if automatic_protocol_is_unset("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") { + return if automatic_protocol_is_unset("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") { builder .with_http() .with_protocol(Protocol::HttpBinary) .with_http_client(automatic_otlp_http_client()?) .build() - .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))? + .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string())) } else if automatic_protocol_is_grpc("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") { builder .with_tonic() .build() - .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))? + .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string())) } else { builder .with_http() .with_http_client(automatic_otlp_http_client()?) .build() - .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))? + .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string())) + }; + } + match settings.transport { + OtlpTransport::HttpBinary => { + let client = reqwest::Client::builder() + .timeout(settings.timeout) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))?; + let mut builder = OtlpSpanExporter::builder() + .with_http() + .with_protocol(Protocol::HttpBinary) + .with_timeout(settings.timeout); + if !settings.header_file.is_empty() { + builder = builder.with_http_client(HeaderFileHttpClient::new( + client, + HeaderFileResolver::new(settings.header_file.clone()), + )); + } else { + builder = builder.with_http_client(client); + } + builder = + builder.with_endpoint(resolve_http_trace_endpoint(&settings.endpoint).into_owned()); + if !settings.headers.is_empty() { + builder = builder.with_headers(settings.headers.clone()); + } + builder + .build() + .map_err(|e| OpenTelemetryError::ExporterBuild(e.to_string())) } - } else { - match config.transport { - OtlpTransport::HttpBinary => { - let client = reqwest::Client::builder() - .timeout(config.timeout) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|error| OpenTelemetryError::ExporterBuild(error.to_string()))?; - let mut builder = OtlpSpanExporter::builder() - .with_http() - .with_protocol(Protocol::HttpBinary) - .with_timeout(config.timeout); - if !config.header_file.is_empty() { - builder = builder.with_http_client(HeaderFileHttpClient::new( - client, - HeaderFileResolver::new(config.header_file.clone()), - )); - } else { - builder = builder.with_http_client(client); - } - builder = builder - .with_endpoint(resolve_http_trace_endpoint(&config.endpoint).into_owned()); - if !config.headers.is_empty() { - builder = builder.with_headers(config.headers.clone()); - } - builder - .build() - .map_err(|e| OpenTelemetryError::ExporterBuild(e.to_string()))? + OtlpTransport::Grpc => { + let mut builder = OtlpSpanExporter::builder() + .with_tonic() + .with_protocol(Protocol::Grpc) + .with_timeout(settings.timeout); + builder = builder.with_endpoint(settings.endpoint.clone()); + if !settings.headers.is_empty() { + builder = builder.with_metadata(build_grpc_metadata(&settings.headers)?); } - OtlpTransport::Grpc => { - let mut builder = OtlpSpanExporter::builder() - .with_tonic() - .with_protocol(Protocol::Grpc) - .with_timeout(config.timeout); - builder = builder.with_endpoint(config.endpoint.clone()); - if !config.headers.is_empty() { - builder = builder.with_metadata(build_grpc_metadata(&config.headers)?); - } - if !config.header_file.is_empty() { - builder = builder.with_interceptor(HeaderFileInterceptor::new( - HeaderFileResolver::new(config.header_file.clone()), - )); - } - builder - .build() - .map_err(|e| OpenTelemetryError::ExporterBuild(e.to_string()))? + if !settings.header_file.is_empty() { + builder = builder.with_interceptor(HeaderFileInterceptor::new( + HeaderFileResolver::new(settings.header_file.clone()), + )); } + builder + .build() + .map_err(|e| OpenTelemetryError::ExporterBuild(e.to_string())) } - }; + } +} +/// Wraps any span exporter in the shared provider, batching, and diagnostics. +fn provider_with_exporter( + exporter: E, + config: &TraceConfig, + runtime_diagnostics: SignalRuntimeDiagnostics, + resource_attributes: Vec, +) -> Result { // Disable per-span attribute caps. Consumers may emit large attribute // sets on long-running spans; the OTel SDK default (128) silently drops // attributes added last in the span's lifecycle. @@ -1204,41 +1372,42 @@ fn build_tracer_provider_with_resource( // Relay's diagnostic processor serializes export diagnostics, so retain // its single-export execution model instead of inheriting this setting. batch_config = batch_config.with_max_concurrent_exports(1); - if let Some(max_queue_size) = config.max_queue_size { + if let Some(max_queue_size) = config.shared().max_queue_size { batch_config = batch_config.with_max_queue_size(max_queue_size); } - if let Some(max_export_batch_size) = config.max_export_batch_size { + if let Some(max_export_batch_size) = config.shared().max_export_batch_size { batch_config = batch_config.with_max_export_batch_size(max_export_batch_size); } - if let Some(scheduled_delay) = config.scheduled_delay { + if let Some(scheduled_delay) = config.shared().scheduled_delay { batch_config = batch_config.with_scheduled_delay(scheduled_delay); } let processor = DiagnosticBatchSpanProcessor::new_with_batch_config_and_retry_timeout( exporter, - config.endpoint.clone(), + config.label(), runtime_diagnostics, batch_config.build(), - config.timeout, + config.channel_full_retry_budget(), ); Ok(builder.with_span_processor(processor).build()) } -fn configured_resource_attributes(config: &OpenTelemetryConfig) -> Vec { - if config.automatic { +fn configured_resource_attributes(config: &TraceConfig) -> Vec { + if matches!(config, TraceConfig::Endpoint(endpoint) if endpoint.automatic) { return Vec::new(); } + let shared = config.shared(); let mut attributes = Vec::new(); - if let Some(service_name) = &config.service_name { + if let Some(service_name) = &shared.service_name { attributes.push(KeyValue::new("service.name", service_name.clone())); } - if let Some(namespace) = &config.service_namespace { + if let Some(namespace) = &shared.service_namespace { attributes.push(KeyValue::new("service.namespace", namespace.clone())); } - if let Some(version) = &config.service_version { + if let Some(version) = &shared.service_version { attributes.push(KeyValue::new("service.version", version.clone())); } attributes.extend( - config + shared .resource_attributes .iter() .map(|(key, value)| KeyValue::new(key.clone(), value.clone())), @@ -1532,7 +1701,7 @@ pub(super) struct OtelEventProcessor { promote_metadata_prefixes: Vec, resource_metadata_prefixes: Vec, resource_metadata_protected_keys: HashSet, - owned_config: Option, + owned_config: Option, dynamic_pipelines: Arc>>, invalid_metric_count: u64, completed_span_context_ttl: Duration, @@ -1554,7 +1723,7 @@ struct DynamicTracePipeline { struct ResourcePipelineRequest { key: String, - config: OpenTelemetryConfig, + config: TraceConfig, attributes: Vec, instrumentation_scope: String, runtime_diagnostics: SignalRuntimeDiagnostics, @@ -1743,7 +1912,7 @@ impl OtelEventProcessor { promote_metadata_prefixes: Vec, completed_span_context_ttl: Duration, runtime_diagnostics: SignalRuntimeDiagnostics, - owned_config: Option, + owned_config: Option, dynamic_pipelines: Arc>>, ) -> Self { let tracer = provider.tracer(instrumentation_scope.clone()); @@ -1751,7 +1920,7 @@ impl OtelEventProcessor { .as_ref() .map(|config| { ( - config.promote_resource_metadata_prefixes.clone(), + config.shared().promote_resource_metadata_prefixes.clone(), configured_resource_attributes(config) .into_iter() .map(|attribute| attribute.key.as_str().to_string()) @@ -1808,14 +1977,18 @@ impl OtelEventProcessor { return None; } let config = self.owned_config.as_ref()?; - if config.promote_resource_metadata_prefixes.is_empty() { + if config + .shared() + .promote_resource_metadata_prefixes + .is_empty() + { return None; } let mut attributes = configured_resource_attributes(config); let promotion = promote_event_metadata_attributes( &mut attributes, event, - &config.promote_resource_metadata_prefixes, + &config.shared().promote_resource_metadata_prefixes, &self.resource_metadata_protected_keys, ); self.record_metadata_promotion_issues(promotion.issues, "resource_metadata"); @@ -1837,7 +2010,11 @@ impl OtelEventProcessor { let Some(config) = self.owned_config.as_ref() else { return self.tracer.clone(); }; - if config.promote_resource_metadata_prefixes.is_empty() { + if config + .shared() + .promote_resource_metadata_prefixes + .is_empty() + { return self.tracer.clone(); } @@ -1845,7 +2022,7 @@ impl OtelEventProcessor { promote_event_metadata_attributes( &mut attributes, event, - &config.promote_resource_metadata_prefixes, + &config.shared().promote_resource_metadata_prefixes, &self.resource_metadata_protected_keys, ); let key = canonical_resource_key(&attributes); diff --git a/crates/core/src/observability/otel_file.rs b/crates/core/src/observability/otel_file.rs new file mode 100644 index 000000000..06beabe33 --- /dev/null +++ b/crates/core/src/observability/otel_file.rs @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OTLP file export for NeMo Relay Core. +//! +//! [`OtlpFileSpanExporter`] writes the same `ExportTraceServiceRequest` the +//! network exporters put on the wire to a local file. [`OtlpFileFormat`] +//! selects between the OpenTelemetry file-exporter specification's JSON lines +//! and the Collector's length-delimited protobuf. + +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::Duration; + +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; +use opentelemetry_proto::transform::common::tonic::ResourceAttributesWithSchema; +use opentelemetry_proto::transform::trace::tonic::group_spans_by_resource_and_scope; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::error::{OTelSdkError, OTelSdkResult}; +use opentelemetry_sdk::trace::{SpanData, SpanExporter}; +use prost::Message; +use serde::{Deserialize, Serialize}; + +use super::private_file::{create_private_dir_all, open_private}; + +/// Result type for the OTLP file exporter. +pub type Result = std::result::Result; + +/// Errors produced while configuring or operating the OTLP file exporter. +#[derive(Debug, thiserror::Error)] +pub enum OtlpFileExporterError { + /// Failed to create the directory containing the output file. + #[error("failed to create OTLP output directory {path:?}: {source}")] + CreateDirectory { + /// Directory that could not be created. + path: PathBuf, + /// Underlying I/O error. + source: std::io::Error, + }, + /// Failed to open the output file. + #[error("failed to open OTLP output file {path:?}: {source}")] + OpenFile { + /// Output path that failed to open. + path: PathBuf, + /// Underlying I/O error. + source: std::io::Error, + }, +} + +/// On-disk encoding used by [`OtlpFileSpanExporter`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum OtlpFileFormat { + /// One OTLP/JSON-encoded `ExportTraceServiceRequest` per line: the + /// serialization the OpenTelemetry file-exporter specification describes, + /// and the default for that reason. + #[default] + JsonLines, + /// Length-delimited OTLP protobuf: a big-endian `u32` byte count followed + /// by that many bytes of encoded `ExportTraceServiceRequest`. + Proto, +} + +impl OtlpFileFormat { + /// Returns the conventional file extension for the format. + pub fn extension(self) -> &'static str { + match self { + Self::JsonLines => "jsonl", + Self::Proto => "otlp.pb", + } + } +} + +/// Writes exported spans to a local file as OTLP. +/// +/// Every export is flushed before it returns, so a batch the SDK reports as +/// delivered is durable and a run that dies mid-stream leaves a readable +/// prefix. +#[derive(Debug)] +pub struct OtlpFileSpanExporter { + path: PathBuf, + format: OtlpFileFormat, + writer: Mutex>>, + resource: ResourceAttributesWithSchema, +} + +impl OtlpFileSpanExporter { + /// Creates an exporter writing `path` in `format`. + /// + /// `root` confines the output and the file is owner-only, as for the ATOF + /// and ATIF sinks: a trajectory carries prompt and response content. + pub fn new(root: &Path, path: &Path, format: OtlpFileFormat, append: bool) -> Result { + if let Some(parent) = path.parent() { + create_private_dir_all(parent).map_err(|source| { + OtlpFileExporterError::CreateDirectory { + path: parent.to_path_buf(), + source, + } + })?; + } + let file = + open_private(root, path, append).map_err(|source| OtlpFileExporterError::OpenFile { + path: path.to_path_buf(), + source, + })?; + Ok(Self { + path: path.to_path_buf(), + format, + writer: Mutex::new(Some(BufWriter::new(file))), + resource: ResourceAttributesWithSchema::default(), + }) + } + + /// Returns the path this exporter writes to. + pub fn path(&self) -> &Path { + &self.path + } + + /// Encodes one export request in the configured format. + /// + /// Fully encoded before anything reaches the writer, so a serialization + /// failure cannot leave a partial record behind. + fn encode(&self, request: &ExportTraceServiceRequest) -> std::result::Result, String> { + match self.format { + OtlpFileFormat::JsonLines => { + // Compact, never pretty: one record per line, so an embedded + // newline would split a record. + let mut line = serde_json::to_vec(request) + .map_err(|error| format!("failed to serialize OTLP/JSON: {error}"))?; + line.push(b'\n'); + Ok(line) + } + OtlpFileFormat::Proto => { + let payload = request.encode_to_vec(); + let length = u32::try_from(payload.len()).map_err(|_| { + format!( + "OTLP export of {} bytes exceeds the length-delimited frame maximum", + payload.len() + ) + })?; + let mut frame = Vec::with_capacity(payload.len() + 4); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) + } + } + } +} + +impl SpanExporter for OtlpFileSpanExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + // An empty record is indistinguishable from a lost one. + if batch.is_empty() { + return Ok(()); + } + let resource_spans = group_spans_by_resource_and_scope(batch, &self.resource); + let record = self + .encode(&ExportTraceServiceRequest { resource_spans }) + .map_err(OTelSdkError::InternalFailure)?; + + let mut guard = self + .writer + .lock() + .map_err(|_| OTelSdkError::InternalFailure(lock_poisoned(&self.path)))?; + let writer = guard.as_mut().ok_or(OTelSdkError::AlreadyShutdown)?; + writer + .write_all(&record) + .and_then(|()| writer.flush()) + .map_err(|error| { + OTelSdkError::InternalFailure(format!( + "failed to write OTLP export to {:?}: {error}", + self.path + )) + }) + } + + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { + let mut guard = self + .writer + .lock() + .map_err(|_| OTelSdkError::InternalFailure(lock_poisoned(&self.path)))?; + let Some(mut writer) = guard.take() else { + return Err(OTelSdkError::AlreadyShutdown); + }; + writer.flush().map_err(|error| { + OTelSdkError::InternalFailure(format!( + "failed to flush OTLP output file {:?}: {error}", + self.path + )) + }) + } + + fn force_flush(&self) -> OTelSdkResult { + let mut guard = self + .writer + .lock() + .map_err(|_| OTelSdkError::InternalFailure(lock_poisoned(&self.path)))?; + // Every accepted record is already durable, so there is nothing to fail on. + let Some(writer) = guard.as_mut() else { + return Ok(()); + }; + writer.flush().map_err(|error| { + OTelSdkError::InternalFailure(format!( + "failed to flush OTLP output file {:?}: {error}", + self.path + )) + }) + } + + fn set_resource(&mut self, resource: &Resource) { + self.resource = resource.into(); + } +} + +fn lock_poisoned(path: &Path) -> String { + format!("the OTLP file exporter state lock for {path:?} was poisoned") +} + +// --------------------------------------------------------------------------- + +#[cfg(test)] +#[path = "../../tests/unit/observability/otel_file_tests.rs"] +mod tests; diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index e422ea4e7..395eab579 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -35,6 +35,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json}; use uuid::Uuid; +use super::otel_file::OtlpFileFormat; use super::private_file::atomic_private_write; use crate::api::event::{Event, LogSeverity, ScopeCategory, ValidatedMetricMeasurement}; use crate::api::runtime::{EventSubscriberFn, current_scope_stack, global_context}; @@ -55,8 +56,9 @@ use crate::observability::atof::{ AtofSinkConfig as CoreAtofSinkConfig, AtofStreamSinkConfig, }; use crate::observability::otel::{ - OpenTelemetryConfig as CoreOpenTelemetryConfig, OpenTelemetrySubscriber, OtlpTransport, - resolve_http_trace_endpoint, + OpenTelemetryConfig as CoreOpenTelemetryConfig, + OpenTelemetryFileSinkConfig as CoreOpenTelemetryFileSinkConfig, OpenTelemetrySubscriber, + OtlpFileSinkSettings, OtlpTransport, resolve_http_trace_endpoint, }; use crate::observability::otel_logs::{ OpenTelemetryLogConfig as CoreOpenTelemetryLogConfig, OpenTelemetryLogSubscriber, @@ -81,6 +83,7 @@ use crate::plugin::{ register_builtin_plugin, }; use crate::plugin::{RuntimeDiagnostic, record_active_plugin_runtime_diagnostic}; +use chrono::Utc; /// The plugin kind registered by the core crate. pub const OBSERVABILITY_PLUGIN_KIND: &str = "observability"; @@ -182,6 +185,9 @@ pub struct OpenTelemetrySectionConfig { skip_serializing_if = "Vec::is_empty" )] pub endpoints: Vec, + /// Local file destinations that receive the same projected spans. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub file_sinks: Vec, /// Optional OTLP log pipeline sourced from non-metric marks. #[serde(default, skip_serializing_if = "Option::is_none")] pub logs: Option, @@ -386,6 +392,77 @@ pub struct OpenTelemetryEndpointConfig { pub completed_span_context_ttl_millis: Option, } +/// One local file destination for projected OTLP spans. +/// +/// A file sink writes the same `ExportTraceServiceRequest` an OTLP endpoint +/// would receive, so a consumer that reads trajectories as artifacts needs no +/// collector. It shares the projection and batching settings with +/// [`OpenTelemetryEndpointConfig`] and drops the fields that only a network +/// destination has: endpoint, transport, headers, and timeout. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct OpenTelemetryFileSinkConfig { + /// Semantic projection written to this file. + #[serde(rename = "type", default)] + pub otel_type: OpenTelemetryType, + /// Directory containing the output file. + pub output_directory: PathBuf, + /// Output filename. Defaults to a timestamped name for the chosen format. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filename: Option, + /// On-disk encoding: `json_lines` or `proto`. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "otlp_file_format_schema"))] + pub format: OtlpFileFormat, + /// File open mode: `append` or `overwrite`. + #[serde(default = "default_otlp_file_sink_mode")] + #[cfg_attr(feature = "schema", schemars(schema_with = "atof_mode_schema"))] + pub mode: String, + /// Representation used for point-in-time marks. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] + pub mark_projection: MarkProjection, + /// Mark names excluded from tool projection. + #[serde(default = "default_mark_exclude_names")] + pub mark_exclude_names: Vec, + /// Projected attributes copied to aliases. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attribute_mappings: Vec, + /// Literal Event metadata prefixes copied to top-level OTLP attributes. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub promote_metadata_prefixes: Vec, + /// Literal root-scope Event metadata prefixes copied to OTLP resource attributes. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub promote_resource_metadata_prefixes: Vec, + /// Extra resource attributes. + #[serde(default)] + pub resource_attributes: HashMap, + /// `service.name` resource attribute. + #[serde(default = "default_otel_service_name")] + pub service_name: String, + /// Optional `service.namespace` resource attribute. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_namespace: Option, + /// Optional `service.version` resource attribute. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_version: Option, + /// Instrumentation scope name. + #[serde(default = "default_otel_instrumentation_scope")] + pub instrumentation_scope: String, + /// Maximum completed spans buffered before the sink drops new spans. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_queue_size: Option, + /// Maximum spans written in one batch. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_export_batch_size: Option, + /// Maximum delay before writing a non-full batch, in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled_delay_millis: Option, + /// How long completed scopes retain trace context for late marks, in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_span_context_ttl_millis: Option, +} + /// Multi-sink ATOF JSONL exporter config. /// /// When enabled, this section wraps @@ -667,6 +744,7 @@ crate::editor_config! { impl OpenTelemetrySectionConfig { enabled => { label: "enabled", kind: Boolean }, endpoints => { label: "traces", kind: List, name: "traces", list: &OPENTELEMETRY_ENDPOINT_LIST }, + file_sinks => { label: "file_sinks", kind: List, list: &OPENTELEMETRY_FILE_SINK_LIST }, logs => { label: "logs", kind: Section, @@ -837,6 +915,84 @@ impl EditorConfig for OpenTelemetryEndpointConfig { } } +impl EditorConfig for OpenTelemetryFileSinkConfig { + fn editor_schema() -> &'static EditorSchema { + static SCHEMA: EditorSchema = EditorSchema { + fields: &[ + otel_editor_field( + "type", + EditorFieldKind::Enum, + &["full", "gen_ai", "openinference"], + false, + ), + otel_editor_field("output_directory", EditorFieldKind::String, &[], false), + otel_editor_field("filename", EditorFieldKind::String, &[], true), + otel_editor_field( + "format", + EditorFieldKind::Enum, + &["json_lines", "proto"], + false, + ), + otel_editor_field( + "mode", + EditorFieldKind::Enum, + &["append", "overwrite"], + false, + ), + otel_editor_field( + "mark_projection", + EditorFieldKind::Enum, + &["inherit", "event", "tool"], + false, + ), + otel_list_editor_field( + "mark_exclude_names", + false, + &crate::config_editor::STRING_LIST_ITEM, + ), + otel_editor_field("attribute_mappings", EditorFieldKind::List, &[], false), + otel_editor_field( + "promote_metadata_prefixes", + EditorFieldKind::List, + &[], + true, + ), + otel_editor_field( + "promote_resource_metadata_prefixes", + EditorFieldKind::List, + &[], + true, + ), + otel_editor_field("service_name", EditorFieldKind::String, &[], false), + otel_editor_field("service_namespace", EditorFieldKind::String, &[], true), + otel_editor_field("service_version", EditorFieldKind::String, &[], true), + otel_editor_field("instrumentation_scope", EditorFieldKind::String, &[], false), + otel_editor_field("max_queue_size", EditorFieldKind::Integer, &[], true), + otel_editor_field("max_export_batch_size", EditorFieldKind::Integer, &[], true), + otel_editor_field( + "scheduled_delay_millis", + EditorFieldKind::Integer, + &[], + true, + ), + otel_editor_field( + "completed_span_context_ttl_millis", + EditorFieldKind::Integer, + &[], + true, + ), + otel_editor_field( + "resource_attributes", + EditorFieldKind::StringMap, + &[], + false, + ), + ], + }; + &SCHEMA + } +} + impl EditorConfig for OpenTelemetrySignalEndpointConfig { fn editor_schema() -> &'static EditorSchema { static SCHEMA: EditorSchema = EditorSchema { @@ -891,6 +1047,26 @@ static OPENTELEMETRY_ENDPOINT_LIST: EditorListItemSpec = EditorListItemSpec { list_item: None, }; +fn default_opentelemetry_file_sink_editor_value() -> Json { + serde_json::json!({ + "type": "full", + "output_directory": "", + "format": "json_lines", + "mode": "overwrite", + "service_name": "unknown_service", + "instrumentation_scope": "opentelemetry", + "resource_attributes": {}, + }) +} + +static OPENTELEMETRY_FILE_SINK_LIST: EditorListItemSpec = EditorListItemSpec { + kind: EditorFieldKind::Section, + schema: Some(::editor_schema), + default: Some(default_opentelemetry_file_sink_editor_value), + tagged_union: None, + list_item: None, +}; + fn default_opentelemetry_signal_endpoint_editor_value() -> Json { serde_json::json!({ "endpoint": "", @@ -1143,6 +1319,13 @@ fn otlp_transport_schema( string_enum_schema(generator, &["http_binary", "grpc"], Some("http_binary")) } +#[cfg(feature = "schema")] +fn otlp_file_format_schema( + generator: &mut schemars::r#gen::SchemaGenerator, +) -> schemars::schema::Schema { + string_enum_schema(generator, &["json_lines", "proto"], Some("json_lines")) +} + #[cfg(feature = "schema")] fn mark_projection_schema( generator: &mut schemars::r#gen::SchemaGenerator, @@ -1541,19 +1724,21 @@ fn register_opentelemetry( ) -> PluginResult<()> { let OpenTelemetrySectionConfig { endpoints, + file_sinks, logs, metrics, .. } = section; let logs = logs.filter(|section| section.enabled); let metrics = metrics.filter(|section| section.enabled); - if endpoints.is_empty() && logs.is_none() && metrics.is_none() { + if endpoints.is_empty() && file_sinks.is_empty() && logs.is_none() && metrics.is_none() { return Err(PluginError::InvalidConfig( - "enabled OpenTelemetry section requires at least one endpoint or an enabled log/metric signal" + "enabled OpenTelemetry section requires at least one endpoint, one file sink, or an enabled log/metric signal" .to_string(), )); } validate_distinct_opentelemetry_destinations(&endpoints)?; + validate_distinct_opentelemetry_file_sinks(&file_sinks)?; let log_resolution = logs .as_ref() .map(|section| resolve_signal_endpoints("logs", section.endpoints.as_ref(), &endpoints)) @@ -1568,7 +1753,11 @@ fn register_opentelemetry( if let Some(resolution) = &metric_resolution { warn_skipped_signal_endpoints("metrics", &resolution.endpoints); } - let trace_subscribers = build_opentelemetry_subscribers(endpoints)?; + let mut trace_subscribers = build_opentelemetry_subscribers(endpoints)?; + // File sinks join the trace fan-out: they receive the same projected spans + // an endpoint would, so logs and metrics derive from endpoints only. + let file_sink_subscribers = + build_opentelemetry_file_sink_subscribers(file_sinks, trace_subscribers.len())?; let signal_subscribers = build_opentelemetry_signal_subscribers( logs, log_resolution, @@ -1576,6 +1765,8 @@ fn register_opentelemetry( metric_resolution, &trace_subscribers, )?; + trace_subscribers.extend(file_sink_subscribers); + let trace_subscribers = trace_subscribers; let log_subscribers = signal_subscribers.logs; let metric_subscribers = signal_subscribers.metrics; if !has_active_opentelemetry_resource(&trace_subscribers) @@ -2043,6 +2234,43 @@ fn build_opentelemetry_subscribers( Ok(subscribers) } +fn build_opentelemetry_file_sink_subscribers( + file_sinks: Vec, + index_offset: usize, +) -> PluginResult>>> { + let mut subscribers = Vec::with_capacity(file_sinks.len()); + for (index, file_sink) in file_sinks.into_iter().enumerate() { + let subscriber = build_otel_file_config(index, file_sink).and_then(|config| { + OpenTelemetrySubscriber::new_for_plugin_file_sink(config, index) + .map_err(observability_registration_error) + }); + // Offset so a file sink and an endpoint never share a fan-out index, + // while the diagnostic field above still reports the configured slot. + let index = index_offset + index; + match subscriber { + Ok(value) => subscribers.push(IndexedOpenTelemetryResource { + index, + value: OpenTelemetryResource::Active(Arc::new(value)), + }), + Err(error) => { + log::warn!( + target: "nemo_relay.plugin", + event = "opentelemetry_file_sink_skipped", + plugin_kind = OBSERVABILITY_PLUGIN_KIND, + resource_kind = "otlp_file_sink", + resource_index = index; + "OpenTelemetry file sink was skipped during activation; delivery continues to valid destinations: {error}" + ); + subscribers.push(IndexedOpenTelemetryResource { + index, + value: OpenTelemetryResource::Skipped(error.to_string()), + }); + } + } + } + Ok(subscribers) +} + fn resolve_signal_endpoints( signal: &'static str, explicit: Option<&Vec>, @@ -3552,6 +3780,131 @@ fn build_otel_config( Ok(config) } +fn build_otel_file_config( + index: usize, + section: OpenTelemetryFileSinkConfig, +) -> PluginResult { + if section.output_directory.as_os_str().is_empty() { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].output_directory must be a nonblank path" + ))); + } + let append = match section.mode.as_str() { + "append" => true, + "overwrite" => false, + other => { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].mode must be 'append' or 'overwrite', got {other:?}" + ))); + } + }; + validate_otel_file_sink_batch_config(index, §ion)?; + let filename = match section.filename { + Some(filename) => { + validate_otel_file_sink_filename(index, &filename)?; + filename + } + None => default_otlp_file_sink_filename(section.format), + }; + + let settings = OtlpFileSinkSettings { + path: section.output_directory.join(&filename), + output_directory: section.output_directory, + format: section.format, + append, + }; + let mut config = CoreOpenTelemetryFileSinkConfig::new(section.otel_type, settings) + .with_service_name(section.service_name) + .with_instrumentation_scope(section.instrumentation_scope) + .with_mark_projection(section.mark_projection) + .with_mark_exclude_names(section.mark_exclude_names) + .with_attribute_mappings(section.attribute_mappings) + .with_promote_metadata_prefixes(section.promote_metadata_prefixes) + .with_promote_resource_metadata_prefixes(section.promote_resource_metadata_prefixes); + if let Some(max_queue_size) = section.max_queue_size { + config = config.with_max_queue_size(max_queue_size); + } + if let Some(max_export_batch_size) = section.max_export_batch_size { + config = config.with_max_export_batch_size(max_export_batch_size); + } + if let Some(scheduled_delay_millis) = section.scheduled_delay_millis { + config = config.with_scheduled_delay(Duration::from_millis(scheduled_delay_millis)); + } + if let Some(completed_span_context_ttl_millis) = section.completed_span_context_ttl_millis { + config = config.with_completed_span_context_ttl(Duration::from_millis( + completed_span_context_ttl_millis, + )); + } + if let Some(namespace) = section.service_namespace { + config = config.with_service_namespace(namespace); + } + if let Some(version) = section.service_version { + config = config.with_service_version(version); + } + for (key, value) in section.resource_attributes { + config = config.with_resource_attribute(key, value); + } + Ok(config) +} + +/// Rejects a filename that would escape `output_directory`. +/// +/// The exporter confines its writes as well, but a path component here is a +/// configuration mistake worth naming rather than an I/O error at export time. +fn validate_otel_file_sink_filename(index: usize, filename: &str) -> PluginResult<()> { + if filename.trim().is_empty() || filename.trim() != filename { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].filename must be nonblank and unpadded" + ))); + } + if Path::new(filename).components().count() != 1 + || matches!( + Path::new(filename).components().next(), + Some(Component::ParentDir) | Some(Component::RootDir) | Some(Component::Prefix(_)) + ) + { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].filename must be a single path component" + ))); + } + Ok(()) +} + +fn validate_otel_file_sink_batch_config( + index: usize, + section: &OpenTelemetryFileSinkConfig, +) -> PluginResult<()> { + for (field, value) in [ + ("max_queue_size", section.max_queue_size), + ("max_export_batch_size", section.max_export_batch_size), + ] { + if value == Some(0) { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].{field} must be greater than 0" + ))); + } + } + if section.scheduled_delay_millis == Some(0) { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].scheduled_delay_millis must be greater than 0" + ))); + } + if section.completed_span_context_ttl_millis == Some(0) { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].completed_span_context_ttl_millis must be greater than 0" + ))); + } + if matches!( + (section.max_export_batch_size, section.max_queue_size), + (Some(batch), Some(queue)) if batch > queue + ) { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{index}].max_export_batch_size must be less than or equal to max_queue_size" + ))); + } + Ok(()) +} + fn validate_otel_batch_config( index: usize, section: &OpenTelemetryEndpointConfig, @@ -4067,6 +4420,7 @@ fn validate_opentelemetry_section( .is_some_and(|signal| signal.enabled); if section.enabled && section.endpoints.is_empty() + && section.file_sinks.is_empty() && !has_enabled_signal && automatic_otlp_signals().is_none() { @@ -4076,10 +4430,35 @@ fn validate_opentelemetry_section( "observability.unsupported_value", Some("opentelemetry".to_string()), Some("endpoints".to_string()), - "enabled OpenTelemetry section requires at least one endpoint or an enabled log/metric signal" + "enabled OpenTelemetry section requires at least one endpoint, one file sink, or an enabled log/metric signal" .to_string(), ); } + for (index, file_sink) in section.file_sinks.iter().enumerate() { + if file_sink.output_directory.as_os_str().is_empty() { + push_policy_diag( + diagnostics, + policy.unsupported_value, + "observability.unsupported_value", + Some("opentelemetry".to_string()), + Some(format!("file_sinks[{index}].output_directory")), + "OpenTelemetry file sink output_directory must be a nonblank path".to_string(), + ); + } + if !matches!(file_sink.mode.as_str(), "append" | "overwrite") { + push_policy_diag( + diagnostics, + policy.unsupported_value, + "observability.unsupported_value", + Some("opentelemetry".to_string()), + Some(format!("file_sinks[{index}].mode")), + format!( + "OpenTelemetry file sink mode must be 'append' or 'overwrite', got {:?}", + file_sink.mode + ), + ); + } + } for (index, endpoint) in section.endpoints.iter().enumerate() { if endpoint.endpoint.trim().is_empty() { push_policy_diag( @@ -4505,6 +4884,30 @@ fn validate_distinct_opentelemetry_destinations( Ok(()) } +/// Rejects two file sinks writing the same path. +/// +/// Two exporters appending to one file interleave their records; two +/// overwriting it race. Either way the trace that survives is not the one +/// either sink was configured to produce. +fn validate_distinct_opentelemetry_file_sinks( + file_sinks: &[OpenTelemetryFileSinkConfig], +) -> PluginResult<()> { + let mut seen: HashMap = HashMap::new(); + for (index, file_sink) in file_sinks.iter().enumerate() { + let filename = file_sink + .filename + .clone() + .unwrap_or_else(|| default_otlp_file_sink_filename(file_sink.format)); + let path = file_sink.output_directory.join(filename); + if let Some(other_index) = seen.insert(path.clone(), index) { + return Err(PluginError::InvalidConfig(format!( + "OpenTelemetry file_sinks[{other_index}] and file_sinks[{index}] write the same path {path:?}; each sink requires its own file" + ))); + } + } + Ok(()) +} + fn opentelemetry_destination_collision_errors( endpoints: &[OpenTelemetryEndpointConfig], ) -> Vec { @@ -5443,6 +5846,18 @@ fn default_otlp_transport() -> String { "http_binary".to_string() } +fn default_otlp_file_sink_mode() -> String { + "overwrite".to_string() +} + +fn default_otlp_file_sink_filename(format: OtlpFileFormat) -> String { + format!( + "nemo-relay-otlp-{}.{}", + Utc::now().format("%Y-%m-%d-%H.%M.%S"), + format.extension() + ) +} + fn default_otel_service_name() -> String { "unknown_service".to_string() } diff --git a/crates/core/tests/unit/observability/otel_file_tests.rs b/crates/core/tests/unit/observability/otel_file_tests.rs new file mode 100644 index 000000000..25a4058e5 --- /dev/null +++ b/crates/core/tests/unit/observability/otel_file_tests.rs @@ -0,0 +1,626 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the OTLP file exporter. +//! +//! The round-trip assertions decode with `prost` and `serde_json` directly +//! rather than through this module's own encoder, so a writer and reader that +//! agree only with each other cannot pass. + +use super::*; +use opentelemetry::KeyValue; +use opentelemetry::trace::{Tracer, TracerProvider}; +use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider}; +use serde_json::Value as Json; +use std::fs; + +fn sample_spans(names: &[&str]) -> Vec { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let tracer = provider.tracer("nemo-relay-otlp-file-tests"); + for name in names { + tracer.in_span(name.to_string(), |_cx| {}); + } + provider.force_flush().unwrap(); + exporter.get_finished_spans().unwrap() +} + +fn exporter_at( + directory: &Path, + filename: &str, + format: OtlpFileFormat, + append: bool, +) -> OtlpFileSpanExporter { + OtlpFileSpanExporter::new(directory, &directory.join(filename), format, append).unwrap() +} + +/// Reads length-delimited protobuf frames without using the exporter's encoder. +fn read_proto_frames(path: &Path) -> Vec { + let bytes = fs::read(path).unwrap(); + let mut requests = Vec::new(); + let mut offset = 0usize; + while offset < bytes.len() { + let length = u32::from_be_bytes(bytes[offset..offset + 4].try_into().unwrap()) as usize; + offset += 4; + let frame = &bytes[offset..offset + length]; + offset += length; + requests.push(ExportTraceServiceRequest::decode(frame).unwrap()); + } + requests +} + +fn read_json_lines(path: &Path) -> Vec { + fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn span_names(request: &ExportTraceServiceRequest) -> Vec { + request + .resource_spans + .iter() + .flat_map(|resource| resource.scope_spans.iter()) + .flat_map(|scope| scope.spans.iter()) + .map(|span| span.name.clone()) + .collect() +} + +#[tokio::test] +async fn proto_export_round_trips_through_an_independent_decoder() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.otlp.pb", + OtlpFileFormat::Proto, + false, + ); + let spans = sample_spans(&["outer", "inner"]); + + exporter.export(spans).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let requests = read_proto_frames(&directory.path().join("trace.otlp.pb")); + assert_eq!(requests.len(), 1, "one export is one frame"); + let mut names = span_names(&requests[0]); + names.sort(); + assert_eq!(names, vec!["inner".to_string(), "outer".to_string()]); +} + +#[tokio::test] +async fn proto_frames_are_length_prefixed_big_endian() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.otlp.pb", + OtlpFileFormat::Proto, + false, + ); + exporter.export(sample_spans(&["only"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let bytes = fs::read(directory.path().join("trace.otlp.pb")).unwrap(); + let declared = u32::from_be_bytes(bytes[..4].try_into().unwrap()) as usize; + assert_eq!( + declared, + bytes.len() - 4, + "the prefix must describe exactly the bytes that follow it" + ); +} + +#[tokio::test] +async fn each_proto_export_appends_one_frame() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.otlp.pb", + OtlpFileFormat::Proto, + false, + ); + exporter.export(sample_spans(&["first"])).await.unwrap(); + exporter.export(sample_spans(&["second"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let requests = read_proto_frames(&directory.path().join("trace.otlp.pb")); + assert_eq!(requests.len(), 2); + assert_eq!(span_names(&requests[0]), vec!["first".to_string()]); + assert_eq!(span_names(&requests[1]), vec!["second".to_string()]); +} + +#[tokio::test] +async fn json_lines_export_is_one_json_object_per_export() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter.export(sample_spans(&["first"])).await.unwrap(); + exporter.export(sample_spans(&["second"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let lines = read_json_lines(&directory.path().join("trace.jsonl")); + assert_eq!( + lines.len(), + 2, + "one line per export, per the file-exporter spec" + ); + assert!(lines.iter().all(|line| line.is_object())); +} + +#[tokio::test] +async fn json_lines_encode_span_identifiers_as_hex() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter.export(sample_spans(&["only"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let lines = read_json_lines(&directory.path().join("trace.jsonl")); + let span = lines[0] + .pointer("/resourceSpans/0/scopeSpans/0/spans/0") + .expect("OTLP/JSON uses camelCase member names"); + let trace_id = span.pointer("/traceId").unwrap().as_str().unwrap(); + let span_id = span.pointer("/spanId").unwrap().as_str().unwrap(); + // OTLP/JSON encodes these two fields as hex rather than the base64 the + // protobuf JSON mapping would otherwise give a bytes field. + assert_eq!(trace_id.len(), 32); + assert_eq!(span_id.len(), 16); + assert!(trace_id.chars().all(|c| c.is_ascii_hexdigit())); + assert!(span_id.chars().all(|c| c.is_ascii_hexdigit())); +} + +#[tokio::test] +async fn json_lines_records_never_contain_an_embedded_newline() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + // A span name carrying a newline would split the record if the encoder + // ever pretty-printed. + let exporter_spans = sample_spans(&["outer\nnewline"]); + exporter.export(exporter_spans).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let contents = fs::read_to_string(directory.path().join("trace.jsonl")).unwrap(); + assert_eq!(contents.matches('\n').count(), 1); + assert!(contents.ends_with('\n')); +} + +#[tokio::test] +async fn an_empty_batch_writes_no_record() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter.export(Vec::new()).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let contents = fs::read_to_string(directory.path().join("trace.jsonl")).unwrap(); + assert!( + contents.is_empty(), + "an empty record would be indistinguishable from a lost one" + ); +} + +#[tokio::test] +async fn resource_attributes_reach_the_written_record() { + let directory = tempfile::tempdir().unwrap(); + let mut exporter = exporter_at( + directory.path(), + "trace.otlp.pb", + OtlpFileFormat::Proto, + false, + ); + let resource = Resource::builder_empty() + .with_attributes(vec![KeyValue::new("service.name", "relay-file-sink")]) + .build(); + exporter.set_resource(&resource); + exporter.export(sample_spans(&["only"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let requests = read_proto_frames(&directory.path().join("trace.otlp.pb")); + let attributes = &requests[0].resource_spans[0] + .resource + .as_ref() + .unwrap() + .attributes; + assert!( + attributes + .iter() + .any(|attribute| attribute.key == "service.name"), + "set_resource must reach the encoded request" + ); +} + +#[tokio::test] +async fn exporting_after_shutdown_is_rejected() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let error = exporter.export(sample_spans(&["late"])).await.unwrap_err(); + assert!(matches!(error, OTelSdkError::AlreadyShutdown)); +} + +#[test] +fn shutting_down_twice_is_rejected() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let error = exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap_err(); + assert!(matches!(error, OTelSdkError::AlreadyShutdown)); +} + +#[test] +fn flushing_after_shutdown_succeeds() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + // Everything the exporter accepted is already durable, so there is nothing + // left for a flush to fail on. + exporter.force_flush().unwrap(); +} + +#[tokio::test] +async fn each_export_is_durable_before_it_returns() { + let directory = tempfile::tempdir().unwrap(); + let exporter = exporter_at( + directory.path(), + "trace.otlp.pb", + OtlpFileFormat::Proto, + false, + ); + exporter.export(sample_spans(&["only"])).await.unwrap(); + + // Deliberately no shutdown: a run that dies between batches must still + // leave the batches it already delivered on disk. + let requests = read_proto_frames(&directory.path().join("trace.otlp.pb")); + assert_eq!(requests.len(), 1); +} + +#[tokio::test] +async fn overwrite_mode_replaces_an_existing_file() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("trace.jsonl"); + fs::write(&path, b"stale\n").unwrap(); + + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + exporter.export(sample_spans(&["fresh"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let contents = fs::read_to_string(&path).unwrap(); + assert!(!contents.contains("stale")); + assert_eq!(contents.lines().count(), 1); +} + +#[tokio::test] +async fn append_mode_preserves_existing_records() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("trace.jsonl"); + fs::write(&path, b"{\"resourceSpans\":[]}\n").unwrap(); + + let exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + true, + ); + exporter.export(sample_spans(&["fresh"])).await.unwrap(); + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap(); + + let lines = read_json_lines(&path); + assert_eq!(lines.len(), 2); +} + +#[test] +fn a_missing_output_directory_is_created() { + let directory = tempfile::tempdir().unwrap(); + let nested = directory.path().join("nested").join("deeper"); + let exporter = OtlpFileSpanExporter::new( + directory.path(), + &nested.join("trace.jsonl"), + OtlpFileFormat::JsonLines, + false, + ) + .unwrap(); + + assert!(nested.is_dir()); + assert_eq!(exporter.path(), nested.join("trace.jsonl")); +} + +#[test] +fn an_unwritable_directory_reports_the_path() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("trace.jsonl"); + fs::create_dir(&path).unwrap(); + + let error = + OtlpFileSpanExporter::new(directory.path(), &path, OtlpFileFormat::JsonLines, false) + .unwrap_err(); + assert!( + error.to_string().contains("trace.jsonl"), + "the error must name the path that failed: {error}" + ); +} + +#[cfg(unix)] +#[test] +fn the_output_file_is_readable_only_by_its_owner() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let _exporter = exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + ); + + let mode = fs::metadata(directory.path().join("trace.jsonl")) + .unwrap() + .permissions() + .mode(); + // A trajectory carries prompt and response content, so it is created with + // the same owner-only permissions as the ATOF and ATIF sinks. + assert_eq!( + mode & 0o077, + 0, + "unexpected group or other permissions: {mode:o}" + ); +} + +#[test] +fn each_format_names_its_conventional_extension() { + assert_eq!(OtlpFileFormat::JsonLines.extension(), "jsonl"); + assert_eq!(OtlpFileFormat::Proto.extension(), "otlp.pb"); + assert_eq!(OtlpFileFormat::default(), OtlpFileFormat::JsonLines); +} + +// --- File sink config ------------------------------------------------------- +// +// Endpoint options are absent from `OpenTelemetryFileSinkConfig` rather than +// rejected, so the cases a runtime check used to cover are now compile errors +// and have no test. + +fn file_sink_config(directory: &Path) -> crate::observability::otel::OpenTelemetryFileSinkConfig { + crate::observability::otel::OpenTelemetryFileSinkConfig::new( + crate::observability::OpenTelemetryType::Full, + crate::observability::otel::OtlpFileSinkSettings { + output_directory: directory.to_path_buf(), + path: directory.join("trace.jsonl"), + format: OtlpFileFormat::JsonLines, + append: false, + }, + ) +} + +#[test] +fn a_file_sink_config_keeps_the_sink_it_was_given() { + let directory = tempfile::tempdir().unwrap(); + let config = file_sink_config(directory.path()); + + assert_eq!(config.sink().path, directory.path().join("trace.jsonl")); + assert_eq!(config.sink().output_directory, directory.path()); + assert!(!config.sink().append); +} + +#[test] +fn a_file_sink_config_builds_a_subscriber_and_opens_its_file() { + let directory = tempfile::tempdir().unwrap(); + let subscriber = crate::observability::otel::OpenTelemetrySubscriber::new_file_sink( + file_sink_config(directory.path()), + ) + .expect("a file sink config builds"); + + assert!(directory.path().join("trace.jsonl").is_file()); + subscriber.shutdown().unwrap(); +} + +#[test] +fn a_file_sink_carries_the_shared_trace_options() { + let directory = tempfile::tempdir().unwrap(); + let config = file_sink_config(directory.path()) + .with_service_name("file-sink-agent") + .with_service_namespace("agents") + .with_completed_span_context_ttl(Duration::from_secs(30)); + + // The same builders an endpoint config offers, applied to a file sink. + assert_eq!(config.completed_span_context_ttl(), Duration::from_secs(30)); + let subscriber = crate::observability::otel::OpenTelemetrySubscriber::new_file_sink(config) + .expect("shared options apply to a file sink"); + subscriber.shutdown().unwrap(); +} + +#[test] +fn a_file_sink_ignores_process_global_otlp_headers() { + const CHILD_MARKER: &str = "NEMO_RELAY_TEST_FILE_SINK_GLOBAL_HEADER_CHILD"; + if std::env::var(CHILD_MARKER).is_ok() { + let directory = tempfile::tempdir().unwrap(); + // `OTEL_EXPORTER_OTLP_HEADERS` cannot reach a file, so a process that + // sets it for some other exporter must not break this one. + crate::observability::otel::OpenTelemetrySubscriber::new_file_sink(file_sink_config( + directory.path(), + )) + .expect("global OTLP headers do not apply to a file sink") + .shutdown() + .unwrap(); + return; + } + + // Set in a child so the variable cannot leak into sibling tests. + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "--nocapture", "--test-threads", "1"]) + .arg("observability::otel_file::tests::a_file_sink_ignores_process_global_otlp_headers") + .env(CHILD_MARKER, "1") + .env("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer token") + .status() + .unwrap(); + assert!(status.success(), "child run failed: {status}"); +} + +#[test] +fn a_directory_that_cannot_be_created_reports_the_directory() { + let directory = tempfile::tempdir().unwrap(); + // A file where a parent directory is expected: `create_dir_all` fails + // before the output file is ever opened. + let blocker = directory.path().join("not-a-directory"); + fs::write(&blocker, b"").unwrap(); + + let error = OtlpFileSpanExporter::new( + directory.path(), + &blocker.join("trace.jsonl"), + OtlpFileFormat::JsonLines, + false, + ) + .unwrap_err(); + + assert!( + matches!(error, OtlpFileExporterError::CreateDirectory { .. }), + "expected a directory-creation error, got {error}" + ); + assert!(error.to_string().contains("not-a-directory")); +} + +#[tokio::test] +async fn a_poisoned_writer_lock_is_reported_by_every_entry_point() { + let directory = tempfile::tempdir().unwrap(); + let exporter = std::sync::Arc::new(exporter_at( + directory.path(), + "trace.jsonl", + OtlpFileFormat::JsonLines, + false, + )); + + let poisoner = std::sync::Arc::clone(&exporter); + let _ = std::thread::spawn(move || { + let _guard = poisoner.writer.lock().unwrap(); + panic!("poison the writer lock"); + }) + .join(); + + // Every path that takes the lock reports the poisoning rather than + // unwrapping into a second panic. + for message in [ + exporter.export(sample_spans(&["late"])).await.unwrap_err(), + exporter + .shutdown_with_timeout(Duration::from_secs(1)) + .unwrap_err(), + exporter.force_flush().unwrap_err(), + ] { + assert!( + message.to_string().contains("poisoned"), + "unexpected error: {message}" + ); + } +} + +// --- Endpoint exporter construction ----------------------------------------- +// +// These exercise the endpoint branch of `otlp_span_exporter`, which the file +// sink work moved out of `build_tracer_provider_with_resource`. + +#[test] +fn an_endpoint_with_a_header_file_builds_on_both_transports() { + use crate::observability::otel::{OpenTelemetryConfig, OpenTelemetrySubscriber, OtlpTransport}; + + let directory = tempfile::tempdir().unwrap(); + let header_path = directory.path().join("token"); + fs::write(&header_path, b"Bearer file-token").unwrap(); + + for transport in [OtlpTransport::HttpBinary, OtlpTransport::Grpc] { + let config = OpenTelemetryConfig::new( + crate::observability::OpenTelemetryType::Full, + "https://collector.example/v1/traces", + ) + .with_transport(transport) + .with_header_file("authorization", header_path.display().to_string()); + + let subscriber = OpenTelemetrySubscriber::new(config) + .unwrap_or_else(|error| panic!("{transport:?} with a header file: {error}")); + subscriber.shutdown().unwrap(); + } +} + +#[test] +fn an_endpoint_without_a_header_file_builds_on_both_transports() { + use crate::observability::otel::{OpenTelemetryConfig, OpenTelemetrySubscriber, OtlpTransport}; + + for transport in [OtlpTransport::HttpBinary, OtlpTransport::Grpc] { + let config = OpenTelemetryConfig::new( + crate::observability::OpenTelemetryType::Full, + "https://collector.example/v1/traces", + ) + .with_transport(transport) + .with_header("authorization", "Bearer inline"); + + OpenTelemetrySubscriber::new(config) + .unwrap() + .shutdown() + .unwrap(); + } +} diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index c0ec89770..2abec95e7 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -94,14 +94,14 @@ fn provider_errors_identify_their_telemetry_signal() { fn default_trace_config_leaves_service_name_to_sdk_resource_detection() { let config = OpenTelemetryConfig::new(OpenTelemetryType::Full, "http://localhost:4318"); assert!( - configured_resource_attributes(&config) + configured_resource_attributes(&TraceConfig::Endpoint(config.clone())) .iter() .all(|attribute| attribute.key.as_str() != "service.name") ); let configured = config.with_service_name("relay-configured-service"); assert!( - configured_resource_attributes(&configured) + configured_resource_attributes(&TraceConfig::Endpoint(configured.clone())) .iter() .any(|attribute| { attribute.key.as_str() == "service.name" @@ -1492,39 +1492,43 @@ fn config_defaults_and_builder_overrides_are_applied() { } fn assert_config_builder_overrides(config: &OpenTelemetryConfig) { - assert_eq!(config.transport, OtlpTransport::HttpBinary); - assert_eq!(config.endpoint, "http://localhost:4318/v1/traces"); + let settings = config; + assert_eq!(settings.transport, OtlpTransport::HttpBinary); + assert_eq!(settings.endpoint, "http://localhost:4318/v1/traces"); assert_eq!( - config.headers.get("authorization"), + settings.headers.get("authorization"), Some(&"Bearer token".into()) ); assert_eq!( - config.header_env.get("x-api-key"), + settings.header_env.get("x-api-key"), Some(&"NEMO_RELAY_TEST_API_KEY".into()) ); assert_eq!( - config.resource_attributes.get("deployment.environment"), + config + .shared + .resource_attributes + .get("deployment.environment"), Some(&"test".into()) ); - assert_eq!(config.service_name.as_deref(), Some("demo-agent")); - assert_eq!(config.service_namespace.as_deref(), Some("agents")); - assert_eq!(config.service_version.as_deref(), Some("1.2.3")); - assert_eq!(config.instrumentation_scope, "demo-scope"); - assert_eq!(config.mark_projection, MarkProjection::Tool); - assert_eq!(config.mark_exclude_names, vec!["notification"]); - assert_eq!(config.attribute_mappings.len(), 1); + assert_eq!(config.shared.service_name.as_deref(), Some("demo-agent")); + assert_eq!(config.shared.service_namespace.as_deref(), Some("agents")); + assert_eq!(config.shared.service_version.as_deref(), Some("1.2.3")); + assert_eq!(config.shared.instrumentation_scope, "demo-scope"); + assert_eq!(config.shared.mark_projection, MarkProjection::Tool); + assert_eq!(config.shared.mark_exclude_names, vec!["notification"]); + assert_eq!(config.shared.attribute_mappings.len(), 1); assert_eq!(config.timeout, Duration::from_millis(1250)); } fn assert_config_defaults(defaults: &OpenTelemetryConfig) { assert_eq!(defaults.transport, OtlpTransport::HttpBinary); - assert_eq!(defaults.service_name, None); - assert_eq!(defaults.instrumentation_scope, "opentelemetry"); - assert_eq!(defaults.mark_projection, MarkProjection::Inherit); - assert_eq!(defaults.mark_exclude_names, vec!["llm.chunk"]); + assert_eq!(defaults.shared.service_name, None); + assert_eq!(defaults.shared.instrumentation_scope, "opentelemetry"); + assert_eq!(defaults.shared.mark_projection, MarkProjection::Inherit); + assert_eq!(defaults.shared.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(defaults.timeout, Duration::from_secs(3)); assert!(defaults.headers.is_empty()); - assert!(defaults.resource_attributes.is_empty()); + assert!(defaults.shared.resource_attributes.is_empty()); } #[test] @@ -4104,11 +4108,9 @@ fn http_trace_exports_do_not_follow_redirects() { write!(stream, "HTTP/1.1 {status} Redirect\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); request }); - let mut config = - OpenTelemetryConfig::new(otel_type, endpoint).with_timeout(Duration::from_secs(1)); - config - .headers - .insert("x-collector-key".into(), "test-secret".into()); + let config = OpenTelemetryConfig::new(otel_type, endpoint) + .with_timeout(Duration::from_secs(1)) + .with_header("x-collector-key", "test-secret"); let subscriber = OpenTelemetrySubscriber::new(config).unwrap(); let callback = subscriber.subscriber(); let uuid = Uuid::now_v7(); diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index ae16b512b..b9ea38e41 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -1072,6 +1072,7 @@ fn default_config_and_component_conversion_cover_public_shape() { let otel = OpenTelemetrySectionConfig { enabled: true, + file_sinks: Vec::new(), endpoints: vec![OpenTelemetryEndpointConfig { otel_type: OpenTelemetryType::Full, endpoint: "http://localhost:4318/v1/traces".to_string(), @@ -1497,6 +1498,7 @@ fn validate_opentelemetry_section_reports_empty_and_malformed_endpoints() { &policy, &OpenTelemetrySectionConfig { enabled: true, + file_sinks: Vec::new(), endpoints: Vec::new(), logs: None, metrics: None, @@ -1522,6 +1524,7 @@ fn validate_opentelemetry_section_reports_empty_and_malformed_endpoints() { &policy, &OpenTelemetrySectionConfig { enabled: true, + file_sinks: Vec::new(), endpoints: vec![endpoint], logs: None, metrics: None, @@ -1551,6 +1554,7 @@ fn validate_opentelemetry_section_reports_empty_and_malformed_endpoints() { &policy, &OpenTelemetrySectionConfig { enabled: true, + file_sinks: Vec::new(), endpoints: vec![endpoint], logs: None, metrics: None, @@ -1660,6 +1664,7 @@ fn opentelemetry_registration_rejects_an_empty_endpoint_list() { let error = register_opentelemetry( OpenTelemetrySectionConfig { enabled: true, + file_sinks: Vec::new(), endpoints: Vec::new(), logs: None, metrics: None, @@ -5922,3 +5927,420 @@ fn atif_filename_helpers_cover_metadata_resolution_and_rejection_paths() { .is_err() ); } + +// --- OpenTelemetry file sinks ---------------------------------------------- + +fn file_sink_section(output_directory: &Path) -> OpenTelemetryFileSinkConfig { + OpenTelemetryFileSinkConfig { + otel_type: OpenTelemetryType::Full, + output_directory: output_directory.to_path_buf(), + filename: Some("trace.jsonl".to_string()), + format: OtlpFileFormat::JsonLines, + mode: default_otlp_file_sink_mode(), + mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), + attribute_mappings: Vec::new(), + promote_metadata_prefixes: Vec::new(), + promote_resource_metadata_prefixes: Vec::new(), + resource_attributes: HashMap::new(), + service_name: default_otel_service_name(), + service_namespace: None, + service_version: None, + instrumentation_scope: default_otel_instrumentation_scope(), + max_queue_size: None, + max_export_batch_size: None, + scheduled_delay_millis: None, + completed_span_context_ttl_millis: None, + } +} + +#[test] +fn a_file_sink_section_parses_with_only_an_output_directory() { + let section: OpenTelemetryFileSinkConfig = + toml::from_str("output_directory = \"/tmp/relay-traces\"").unwrap(); + + assert_eq!(section.format, OtlpFileFormat::JsonLines); + assert_eq!(section.mode, "overwrite"); + assert_eq!(section.otel_type, OpenTelemetryType::Full); + assert!(section.filename.is_none()); +} + +#[test] +fn a_file_sink_section_parses_the_proto_format() { + let section: OpenTelemetryFileSinkConfig = + toml::from_str("output_directory = \"/tmp/relay-traces\"\nformat = \"proto\"").unwrap(); + + assert_eq!(section.format, OtlpFileFormat::Proto); +} + +#[test] +fn a_file_sink_config_resolves_its_path_under_the_output_directory() { + let directory = tempfile::tempdir().unwrap(); + let config = build_otel_file_config(0, file_sink_section(directory.path())).unwrap(); + + let settings = config.sink(); + assert_eq!(settings.path, directory.path().join("trace.jsonl")); + assert_eq!(settings.output_directory, directory.path()); + assert!(!settings.append, "overwrite mode must not append"); +} + +#[test] +fn a_file_sink_without_a_filename_names_the_file_after_its_format() { + let directory = tempfile::tempdir().unwrap(); + for (format, extension) in [ + (OtlpFileFormat::JsonLines, ".jsonl"), + (OtlpFileFormat::Proto, ".otlp.pb"), + ] { + let mut section = file_sink_section(directory.path()); + section.filename = None; + section.format = format; + let config = build_otel_file_config(0, section).unwrap(); + + let settings = config.sink(); + let path = settings.path.display().to_string(); + assert!( + path.ends_with(extension), + "unexpected default filename {path}" + ); + } +} + +#[test] +fn a_file_sink_rejects_a_blank_output_directory() { + let mut section = file_sink_section(Path::new("")); + section.filename = None; + let error = build_otel_file_config(3, section).unwrap_err(); + + assert!(error.to_string().contains("file_sinks[3].output_directory")); +} + +#[test] +fn a_file_sink_rejects_an_unknown_mode() { + let directory = tempfile::tempdir().unwrap(); + let mut section = file_sink_section(directory.path()); + section.mode = "truncate".to_string(); + let error = build_otel_file_config(1, section).unwrap_err(); + + assert!( + error + .to_string() + .contains("must be 'append' or 'overwrite'") + ); +} + +#[test] +fn a_file_sink_accepts_append_mode() { + let directory = tempfile::tempdir().unwrap(); + let mut section = file_sink_section(directory.path()); + section.mode = "append".to_string(); + let config = build_otel_file_config(0, section).unwrap(); + + let settings = config.sink(); + assert!(settings.append); +} + +#[test] +fn a_file_sink_rejects_a_filename_that_leaves_its_directory() { + let directory = tempfile::tempdir().unwrap(); + for filename in ["../escape.jsonl", "nested/trace.jsonl", "/absolute.jsonl"] { + let mut section = file_sink_section(directory.path()); + section.filename = Some(filename.to_string()); + let error = build_otel_file_config(2, section).unwrap_err(); + + assert!( + error.to_string().contains("single path component"), + "{filename} should be rejected, got {error}" + ); + } +} + +#[test] +fn a_file_sink_rejects_a_blank_or_padded_filename() { + let directory = tempfile::tempdir().unwrap(); + for filename in ["", " ", " trace.jsonl"] { + let mut section = file_sink_section(directory.path()); + section.filename = Some(filename.to_string()); + let error = build_otel_file_config(0, section).unwrap_err(); + + assert!(error.to_string().contains("nonblank and unpadded")); + } +} + +#[test] +fn a_file_sink_rejects_zero_and_inverted_batch_settings() { + let directory = tempfile::tempdir().unwrap(); + type MutateFileSink = Box; + let cases: Vec<(MutateFileSink, &str)> = vec![ + (Box::new(|s| s.max_queue_size = Some(0)), "max_queue_size"), + ( + Box::new(|s| s.max_export_batch_size = Some(0)), + "max_export_batch_size", + ), + ( + Box::new(|s| s.scheduled_delay_millis = Some(0)), + "scheduled_delay_millis", + ), + ( + Box::new(|s| s.completed_span_context_ttl_millis = Some(0)), + "completed_span_context_ttl_millis", + ), + ( + Box::new(|s| { + s.max_queue_size = Some(1); + s.max_export_batch_size = Some(2); + }), + "must be less than or equal to max_queue_size", + ), + ]; + for (mutate, expected) in cases { + let mut section = file_sink_section(directory.path()); + mutate(&mut section); + let error = build_otel_file_config(0, section).unwrap_err(); + assert!(error.to_string().contains(expected), "got {error}"); + } +} + +#[test] +fn two_file_sinks_writing_one_path_are_rejected() { + let directory = tempfile::tempdir().unwrap(); + let sinks = vec![ + file_sink_section(directory.path()), + file_sink_section(directory.path()), + ]; + let error = validate_distinct_opentelemetry_file_sinks(&sinks).unwrap_err(); + + assert!(error.to_string().contains("write the same path")); +} + +#[test] +fn file_sinks_writing_distinct_paths_are_accepted() { + let directory = tempfile::tempdir().unwrap(); + let mut second = file_sink_section(directory.path()); + second.filename = Some("other.jsonl".to_string()); + let sinks = vec![file_sink_section(directory.path()), second]; + + validate_distinct_opentelemetry_file_sinks(&sinks).unwrap(); +} + +#[test] +fn a_file_sink_config_builds_a_subscriber_and_opens_its_file() { + let directory = tempfile::tempdir().unwrap(); + let config = build_otel_file_config(0, file_sink_section(directory.path())).unwrap(); + + // Endpoint validation must not apply: this destination has no endpoint. + let subscriber = + crate::observability::otel::OpenTelemetrySubscriber::new_file_sink(config).unwrap(); + assert!(directory.path().join("trace.jsonl").is_file()); + subscriber.shutdown().unwrap(); +} + +#[test] +fn opentelemetry_registration_accepts_a_section_with_only_file_sinks() { + let _guard = crate::observability::test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let directory = tempfile::tempdir().unwrap(); + let mut context = PluginRegistrationContext::new(); + + register_opentelemetry( + OpenTelemetrySectionConfig { + enabled: true, + file_sinks: vec![file_sink_section(directory.path())], + endpoints: Vec::new(), + logs: None, + metrics: None, + }, + &mut context, + ) + .unwrap(); + + assert!(directory.path().join("trace.jsonl").is_file()); + crate::plugin::rollback_registrations(&mut context.into_registrations()); +} + +#[test] +fn opentelemetry_registration_rejects_an_empty_endpoint_and_file_sink_list() { + let mut context = PluginRegistrationContext::new(); + let error = register_opentelemetry( + OpenTelemetrySectionConfig { + enabled: true, + file_sinks: Vec::new(), + endpoints: Vec::new(), + logs: None, + metrics: None, + }, + &mut context, + ) + .unwrap_err(); + + assert!(error.to_string().contains("one file sink")); +} + +#[test] +fn file_sinks_are_offered_by_the_config_editor() { + let schema = OpenTelemetrySectionConfig::editor_schema(); + let file_sinks = schema.field("file_sinks").expect("file_sinks editor field"); + assert_eq!(file_sinks.kind, EditorFieldKind::List); + + let item_schema = (file_sinks + .list_item + .expect("file_sinks list metadata") + .schema + .expect("file_sinks item schema"))(); + let format = item_schema.field("format").expect("format editor field"); + assert_eq!(format.enum_values, &["json_lines", "proto"]); + assert!( + item_schema.field("endpoint").is_none(), + "a file sink has no endpoint to configure" + ); + assert!( + !item_schema + .field("output_directory") + .expect("output_directory editor field") + .optional, + "a file sink cannot default its output directory" + ); +} + +#[test] +fn the_file_sink_editor_default_is_a_valid_section() { + let default = default_opentelemetry_file_sink_editor_value(); + let section: OpenTelemetryFileSinkConfig = serde_json::from_value(default).unwrap(); + + assert_eq!(section.format, OtlpFileFormat::JsonLines); + assert_eq!(section.mode, "overwrite"); +} + +#[test] +fn validate_opentelemetry_section_accepts_a_file_sink_as_the_only_destination() { + let directory = tempfile::tempdir().unwrap(); + let policy = ConfigPolicy::default(); + let mut diagnostics = Vec::new(); + + validate_opentelemetry_section( + &mut diagnostics, + &policy, + &OpenTelemetrySectionConfig { + enabled: true, + file_sinks: vec![file_sink_section(directory.path())], + endpoints: Vec::new(), + logs: None, + metrics: None, + }, + ); + + // Static validation runs before activation, so a section it rejects never + // reaches `register_opentelemetry` at all. + assert!( + diagnostics.is_empty(), + "a file sink is a destination: {diagnostics:?}" + ); +} + +#[test] +fn validate_opentelemetry_section_reports_malformed_file_sinks() { + let directory = tempfile::tempdir().unwrap(); + let policy = ConfigPolicy::default(); + let mut blank_directory = file_sink_section(directory.path()); + blank_directory.output_directory = PathBuf::new(); + let mut bad_mode = file_sink_section(directory.path()); + bad_mode.mode = "truncate".to_string(); + let mut diagnostics = Vec::new(); + + validate_opentelemetry_section( + &mut diagnostics, + &policy, + &OpenTelemetrySectionConfig { + enabled: true, + file_sinks: vec![blank_directory, bad_mode], + endpoints: Vec::new(), + logs: None, + metrics: None, + }, + ); + + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic.field.as_deref() == Some("file_sinks[0].output_directory") + })); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.field.as_deref() == Some("file_sinks[1].mode")) + ); +} + +#[test] +fn a_file_sink_applies_every_optional_setting() { + let directory = tempfile::tempdir().unwrap(); + let mut section = file_sink_section(directory.path()); + section.max_queue_size = Some(4096); + section.max_export_batch_size = Some(512); + section.scheduled_delay_millis = Some(1000); + section.completed_span_context_ttl_millis = Some(30_000); + section.service_namespace = Some("agents".to_string()); + section.service_version = Some("1.2.3".to_string()); + section + .resource_attributes + .insert("deployment.environment".to_string(), "test".to_string()); + + let config = build_otel_file_config(0, section).unwrap(); + + assert_eq!( + config.batch_overrides(), + (Some(4096), Some(512), Some(Duration::from_millis(1000))) + ); + assert_eq!( + config.completed_span_context_ttl(), + Duration::from_millis(30_000) + ); + crate::observability::otel::OpenTelemetrySubscriber::new_file_sink(config) + .unwrap() + .shutdown() + .unwrap(); +} + +#[test] +fn an_invalid_file_sink_is_skipped_while_the_others_register() { + // Registration touches the global subscriber registry. + let _guard = crate::observability::test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + let directory = tempfile::tempdir().unwrap(); + let mut broken = file_sink_section(directory.path()); + broken.mode = "truncate".to_string(); + broken.filename = Some("broken.jsonl".to_string()); + let mut working = file_sink_section(directory.path()); + working.filename = Some("working.jsonl".to_string()); + + let mut context = PluginRegistrationContext::new(); + register_opentelemetry( + OpenTelemetrySectionConfig { + enabled: true, + file_sinks: vec![broken, working], + endpoints: Vec::new(), + logs: None, + metrics: None, + }, + &mut context, + ) + .unwrap(); + + // One bad sink does not stop delivery to the rest. + assert!(!directory.path().join("broken.jsonl").exists()); + assert!(directory.path().join("working.jsonl").is_file()); + crate::plugin::rollback_registrations(&mut context.into_registrations()); +} + +#[test] +fn file_sinks_without_filenames_collide_on_the_generated_name() { + let directory = tempfile::tempdir().unwrap(); + let mut first = file_sink_section(directory.path()); + first.filename = None; + let mut second = file_sink_section(directory.path()); + second.filename = None; + + // Both fall back to the same format-derived default. + let error = validate_distinct_opentelemetry_file_sinks(&[first, second]).unwrap_err(); + + assert!(error.to_string().contains("write the same path")); +} diff --git a/crates/ffi/nemo_relay.h b/crates/ffi/nemo_relay.h index 1672abdbf..4f3e3dda9 100644 --- a/crates/ffi/nemo_relay.h +++ b/crates/ffi/nemo_relay.h @@ -1769,6 +1769,30 @@ NemoRelayStatus nemo_relay_otel_subscriber_create(const char *otel_type, uint64_t timeout_millis, struct FfiOpenTelemetrySubscriber **out); +/** + * Creates one typed OpenTelemetry exporter subscriber that writes OTLP to a file. + * + * `otel_type` must be `full`, `gen_ai`, or `openinference`. `output_directory` is + * required. `filename` may be null to use a default name for the format. + * `format` is `json_lines` (the OpenTelemetry file-exporter specification's + * serialization, and the default when null) or `proto`. `mode` is `overwrite` + * (the default when null) or `append`. + * + * # Safety + * Any non-null C strings must be valid and `out` must be non-null. + */ +NemoRelayStatus nemo_relay_otel_subscriber_create_file_sink(const char *otel_type, + const char *output_directory, + const char *filename, + const char *format, + const char *mode, + const char *resource_attributes_json, + const char *service_name, + const char *service_namespace, + const char *service_version, + const char *instrumentation_scope, + struct FfiOpenTelemetrySubscriber **out); + /** * Creates one typed OpenTelemetry exporter subscriber with projection controls. * diff --git a/crates/ffi/src/api/observability.rs b/crates/ffi/src/api/observability.rs index f79819465..6c6d74c50 100644 --- a/crates/ffi/src/api/observability.rs +++ b/crates/ffi/src/api/observability.rs @@ -13,6 +13,7 @@ type AtofExporterConfig = nemo_relay::observability::atof::AtofExporterConfig; type AtofExporterError = nemo_relay::observability::atof::AtofExporterError; type AtofExporterMode = nemo_relay::observability::atof::AtofExporterMode; type OpenTelemetryConfig = nemo_relay::observability::otel::OpenTelemetryConfig; +type OpenTelemetryFileSinkConfig = nemo_relay::observability::otel::OpenTelemetryFileSinkConfig; type OpenTelemetrySubscriber = nemo_relay::observability::otel::OpenTelemetrySubscriber; type OpenTelemetryLogConfig = nemo_relay::observability::otel_logs::OpenTelemetryLogConfig; type OpenTelemetryLogSubscriber = nemo_relay::observability::otel_logs::OpenTelemetryLogSubscriber; @@ -725,6 +726,16 @@ fn otel_config_for_transport( .with_service_name(service_name)) } +fn create_otel_file_sink_subscriber( + config: OpenTelemetryFileSinkConfig, +) -> Result { + let _runtime_guard = tokio_runtime().enter(); + OpenTelemetrySubscriber::new_file_sink(config).map_err(|error| { + set_last_error(&error.to_string()); + NemoRelayStatus::Internal + }) +} + fn create_otel_subscriber( config: OpenTelemetryConfig, ) -> Result { @@ -875,6 +886,157 @@ pub unsafe extern "C" fn nemo_relay_otel_subscriber_create( NemoRelayStatus::Ok } +/// Creates one typed OpenTelemetry exporter subscriber that writes OTLP to a file. +/// +/// `otel_type` must be `full`, `gen_ai`, or `openinference`. `output_directory` is +/// required. `filename` may be null to use a default name for the format. +/// `format` is `json_lines` (the OpenTelemetry file-exporter specification's +/// serialization, and the default when null) or `proto`. `mode` is `overwrite` +/// (the default when null) or `append`. +/// +/// # Safety +/// Any non-null C strings must be valid and `out` must be non-null. +#[allow(clippy::too_many_arguments)] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nemo_relay_otel_subscriber_create_file_sink( + otel_type: *const c_char, + output_directory: *const c_char, + filename: *const c_char, + format: *const c_char, + mode: *const c_char, + resource_attributes_json: *const c_char, + service_name: *const c_char, + service_namespace: *const c_char, + service_version: *const c_char, + instrumentation_scope: *const c_char, + out: *mut *mut FfiOpenTelemetrySubscriber, +) -> NemoRelayStatus { + clear_last_error(); + if let Err(status) = required_out_ptr(out) { + return status; + } + let subscriber = match build_otel_file_sink_subscriber( + otel_type, + output_directory, + filename, + format, + mode, + resource_attributes_json, + service_name, + service_namespace, + service_version, + instrumentation_scope, + ) { + Ok(subscriber) => subscriber, + Err(status) => return status, + }; + unsafe { *out = Box::into_raw(Box::new(FfiOpenTelemetrySubscriber(subscriber))) }; + NemoRelayStatus::Ok +} + +/// Builds the subscriber behind [`nemo_relay_otel_subscriber_create_file_sink`]. +/// +/// Split out so each argument can propagate its own parse failure with `?` +/// instead of a `match` arm per option. +#[allow(clippy::too_many_arguments)] +fn build_otel_file_sink_subscriber( + otel_type: *const c_char, + output_directory: *const c_char, + filename: *const c_char, + format: *const c_char, + mode: *const c_char, + resource_attributes_json: *const c_char, + service_name: *const c_char, + service_namespace: *const c_char, + service_version: *const c_char, + instrumentation_scope: *const c_char, +) -> Result { + let otel_type = parse_otel_type(otel_type)?; + let settings = parse_ffi_file_sink_settings(output_directory, filename, format, mode)?; + let mut config = OpenTelemetryFileSinkConfig::new(otel_type, settings); + config = apply_optional_string( + config, + service_name, + OpenTelemetryFileSinkConfig::with_service_name, + )?; + config = apply_optional_string( + config, + service_namespace, + OpenTelemetryFileSinkConfig::with_service_namespace, + )?; + config = apply_optional_string( + config, + service_version, + OpenTelemetryFileSinkConfig::with_service_version, + )?; + config = apply_optional_string( + config, + instrumentation_scope, + OpenTelemetryFileSinkConfig::with_instrumentation_scope, + )?; + config = apply_string_map( + config, + resource_attributes_json, + "resource_attributes", + OpenTelemetryFileSinkConfig::with_resource_attribute, + )?; + create_otel_file_sink_subscriber(config) +} + +fn parse_ffi_file_sink_settings( + output_directory: *const c_char, + filename: *const c_char, + format: *const c_char, + mode: *const c_char, +) -> Result { + let output_directory = match parse_optional_string(output_directory)? { + Some(value) if !value.trim().is_empty() => value, + _ => { + set_last_error("output_directory is required"); + return Err(NemoRelayStatus::InvalidArg); + } + }; + let format = match parse_optional_string(format)?.as_deref() { + None | Some("json_lines") => { + nemo_relay::observability::otel_file::OtlpFileFormat::JsonLines + } + Some("proto") => nemo_relay::observability::otel_file::OtlpFileFormat::Proto, + Some(other) => { + set_last_error(&format!( + "format must be 'json_lines' or 'proto', got {other:?}" + )); + return Err(NemoRelayStatus::InvalidArg); + } + }; + let append = match parse_optional_string(mode)?.as_deref() { + None | Some("overwrite") => false, + Some("append") => true, + Some(other) => { + set_last_error(&format!( + "mode must be 'append' or 'overwrite', got {other:?}" + )); + return Err(NemoRelayStatus::InvalidArg); + } + }; + let filename = match parse_optional_string(filename)? { + Some(filename) => { + if std::path::Path::new(&filename).components().count() != 1 { + set_last_error("filename must be a single path component"); + return Err(NemoRelayStatus::InvalidArg); + } + filename + } + None => format!("nemo-relay-otlp.{}", format.extension()), + }; + let output_directory = std::path::PathBuf::from(output_directory); + Ok(nemo_relay::observability::otel::OtlpFileSinkSettings { + path: output_directory.join(filename), + output_directory, + format, + append, + }) +} + /// Creates one typed OpenTelemetry exporter subscriber with projection controls. /// /// The JSON arrays use `mark_exclude_names: ["llm.chunk"]` and diff --git a/crates/ffi/tests/unit/api/coverage_sweeps_tests.rs b/crates/ffi/tests/unit/api/coverage_sweeps_tests.rs index 4f8dd2c22..d57dd294c 100644 --- a/crates/ffi/tests/unit/api/coverage_sweeps_tests.rs +++ b/crates/ffi/tests/unit/api/coverage_sweeps_tests.rs @@ -3821,3 +3821,254 @@ fn test_ffi_otel_signal_subscribers_apply_all_typed_options() { ); } } + +#[test] +fn otel_file_sink_subscriber_create_covers_success_and_rejection() { + let directory = tempfile::tempdir().unwrap(); + let output_directory = cstring(&directory.path().display().to_string()); + let otel_type = cstring("full"); + + unsafe { + let filename = cstring("ffi-trace.jsonl"); + let mut subscriber = ptr::null_mut(); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + filename.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::Ok + ); + assert!(directory.path().join("ffi-trace.jsonl").is_file()); + assert_status!( + nemo_relay_otel_subscriber_shutdown(subscriber), + NemoRelayStatus::Ok + ); + types::nemo_relay_otel_subscriber_free(subscriber); + + // A null format defaults to the specification's JSON lines, and a null + // filename is derived from it. + let mut defaulted = ptr::null_mut(); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut defaulted, + ), + NemoRelayStatus::Ok + ); + assert!(directory.path().join("nemo-relay-otlp.jsonl").is_file()); + assert_status!( + nemo_relay_otel_subscriber_shutdown(defaulted), + NemoRelayStatus::Ok + ); + types::nemo_relay_otel_subscriber_free(defaulted); + + let bad_format = cstring("yaml"); + let mut rejected = ptr::null_mut(); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + bad_format.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut rejected, + ), + NemoRelayStatus::InvalidArg + ); + + let blank = cstring(""); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + blank.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut rejected, + ), + NemoRelayStatus::InvalidArg + ); + } +} + +#[test] +fn otel_file_sink_subscriber_create_covers_every_rejection_arm() { + let directory = tempfile::tempdir().unwrap(); + let output_directory = cstring(&directory.path().display().to_string()); + let otel_type = cstring("full"); + let invalid_utf8 = [0xffu8, 0]; + let bad_string = invalid_utf8.as_ptr() as *const c_char; + + // Every argument the entry point parses, refused one at a time. Each arm + // returns before the subscriber is built, so `out` stays untouched. + unsafe { + let mut subscriber = ptr::null_mut(); + + // A null `out` is rejected before anything is parsed. + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null_mut(), + ), + NemoRelayStatus::NullPointer + ); + + let bad_type = cstring("unsupported"); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + bad_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::InvalidArg + ); + + let bad_mode = cstring("truncate"); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + bad_mode.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::InvalidArg + ); + + let nested = cstring("nested/trace.jsonl"); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + nested.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::InvalidArg + ); + + // Resource attributes must be a JSON object of strings. + let bad_attributes = cstring(r#"{"env": 1}"#); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + bad_attributes.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::InvalidArg + ); + + // Each optional string is decoded, so invalid UTF-8 is refused wherever + // it appears. + for position in 0..4 { + let mut args: [*const c_char; 4] = [ptr::null(); 4]; + args[position] = bad_string; + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + args[0], + args[1], + args[2], + args[3], + &mut subscriber, + ), + NemoRelayStatus::InvalidUtf8 + ); + } + + // Proto format with append mode: the arms the success case misses. + let proto = cstring("proto"); + let append = cstring("append"); + assert_status!( + nemo_relay_otel_subscriber_create_file_sink( + otel_type.as_ptr(), + output_directory.as_ptr(), + ptr::null(), + proto.as_ptr(), + append.as_ptr(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + ptr::null(), + &mut subscriber, + ), + NemoRelayStatus::Ok + ); + assert_status!( + nemo_relay_otel_subscriber_shutdown(subscriber), + NemoRelayStatus::Ok + ); + types::nemo_relay_otel_subscriber_free(subscriber); + } +} diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index 4e333c48a..dde204a16 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -124,9 +124,37 @@ export interface OpenTelemetryMetricSectionConfig { cardinality_limit?: number; } +export interface OpenTelemetryFileSinkConfig { + type?: 'full' | 'gen_ai' | 'openinference'; + output_directory: string; + filename?: string; + /** + * `json_lines` follows the OpenTelemetry Protocol File Exporter + * specification: one OTLP/JSON record per line. `proto` writes each record + * length-delimited. + */ + format?: 'json_lines' | 'proto'; + mode?: 'append' | 'overwrite'; + mark_projection?: 'inherit' | 'event' | 'tool'; + mark_exclude_names?: string[]; + attribute_mappings?: Array<{ key: string; alias: string }>; + promote_metadata_prefixes?: string[]; + promote_resource_metadata_prefixes?: string[]; + resource_attributes?: Record; + service_name?: string; + service_namespace?: string; + service_version?: string; + instrumentation_scope?: string; + max_queue_size?: number; + max_export_batch_size?: number; + scheduled_delay_millis?: number; + completed_span_context_ttl_millis?: number; +} + export interface OpenTelemetrySectionConfig { enabled?: boolean; endpoints?: OpenTelemetryEndpointConfig[]; + file_sinks?: OpenTelemetryFileSinkConfig[]; logs?: OpenTelemetryLogSectionConfig; metrics?: OpenTelemetryMetricSectionConfig; } @@ -157,6 +185,7 @@ export declare function atofConfig(config?: AtofConfig): AtofConfig; export declare function atifConfig(config?: AtifConfig): AtifConfig; /** Create one typed OpenTelemetry endpoint. */ export declare function openTelemetryEndpoint(config: OpenTelemetryEndpointConfig): OpenTelemetryEndpointConfig; +export declare function openTelemetryFileSink(config: OpenTelemetryFileSinkConfig): OpenTelemetryFileSinkConfig; /** Create one signal-specific OpenTelemetry endpoint for logs or metrics. */ export declare function openTelemetrySignalEndpoint( config: OpenTelemetrySignalEndpointConfig, diff --git a/crates/node/observability.js b/crates/node/observability.js index 4a4cce7c7..714bc0bc3 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -79,6 +79,42 @@ function openTelemetryEndpoint(config) { }; } +/** + * Create one local file destination for projected OTLP spans. + * + * `json_lines` is the OpenTelemetry file-exporter specification's serialization: + * one OTLP/JSON record per line. `proto` writes each record length-delimited. + * + * @param {object} config - File sink settings including required `output_directory`. + * @returns {object} A normalized OpenTelemetry file sink. + */ +function openTelemetryFileSink(config) { + if (!config || typeof config !== 'object') { + throw new TypeError('OpenTelemetry file sink config is required'); + } + if (typeof config.output_directory !== 'string' || config.output_directory.trim() === '') { + throw new TypeError('OpenTelemetry file sink output_directory must be a nonblank string'); + } + if (config.format !== undefined && !['json_lines', 'proto'].includes(config.format)) { + throw new TypeError('OpenTelemetry file sink format must be "json_lines" or "proto"'); + } + if (config.mode !== undefined && !['append', 'overwrite'].includes(config.mode)) { + throw new TypeError('OpenTelemetry file sink mode must be "append" or "overwrite"'); + } + return { + type: 'full', + format: 'json_lines', + mode: 'overwrite', + service_name: 'unknown_service', + instrumentation_scope: 'opentelemetry', + completed_span_context_ttl_millis: DEFAULT_COMPLETED_SPAN_CONTEXT_TTL_MILLIS, + resource_attributes: {}, + promote_metadata_prefixes: [], + promote_resource_metadata_prefixes: [], + ...config, + }; +} + /** * Create one signal-specific OpenTelemetry endpoint for logs or metrics. * @@ -149,6 +185,7 @@ function openTelemetryConfig(config = {}) { return { enabled: false, endpoints: [], + file_sinks: [], ...config, }; } @@ -172,6 +209,7 @@ module.exports = { atofConfig, atifConfig, openTelemetryEndpoint, + openTelemetryFileSink, openTelemetrySignalEndpoint, openTelemetryLogConfig, openTelemetryMetricConfig, diff --git a/crates/node/src/api/mod.rs b/crates/node/src/api/mod.rs index 0c51b2975..8ae36c45a 100644 --- a/crates/node/src/api/mod.rs +++ b/crates/node/src/api/mod.rs @@ -301,6 +301,50 @@ fn parse_attribute_mappings( Ok(mappings) } +fn parse_otel_file_sink( + output_directory: &str, + filename: Option<&str>, + format: Option<&str>, + mode: Option<&str>, +) -> napi::Result { + let format = match format.unwrap_or("json_lines") { + "json_lines" => nemo_relay::observability::otel_file::OtlpFileFormat::JsonLines, + "proto" => nemo_relay::observability::otel_file::OtlpFileFormat::Proto, + other => { + return Err(napi::Error::from_reason(format!( + "format must be 'json_lines' or 'proto', got {other:?}" + ))); + } + }; + let append = match mode.unwrap_or("overwrite") { + "overwrite" => false, + "append" => true, + other => { + return Err(napi::Error::from_reason(format!( + "mode must be 'append' or 'overwrite', got {other:?}" + ))); + } + }; + let filename = match filename { + Some(filename) => { + if std::path::Path::new(filename).components().count() != 1 { + return Err(napi::Error::from_reason( + "filename must be a single path component", + )); + } + filename.to_string() + } + None => format!("nemo-relay-otlp.{}", format.extension()), + }; + let output_directory = std::path::PathBuf::from(output_directory); + Ok(nemo_relay::observability::otel::OtlpFileSinkSettings { + path: output_directory.join(filename), + output_directory, + format, + append, + }) +} + fn build_otel_config( options: OpenTelemetryConfig, ) -> napi::Result { @@ -345,9 +389,9 @@ fn build_otel_config( let mut config = nemo_relay::observability::otel::OpenTelemetryConfig::new(otel_type, endpoint) .with_transport(transport) + .with_timeout(std::time::Duration::from_millis(timeout_millis.into())) .with_service_name(service_name) .with_instrumentation_scope(instrumentation_scope) - .with_timeout(std::time::Duration::from_millis(timeout_millis.into())) .with_completed_span_context_ttl(std::time::Duration::from_millis( completed_span_context_ttl_millis, )); @@ -5251,6 +5295,78 @@ pub struct OpenTelemetryConfig { pub promote_resource_metadata_prefixes: Option>, } +/// Configuration for a subscriber that writes OTLP to a local file. +/// +/// Carries no endpoint, transport, headers, or timeout: a file destination has +/// no use for them. +#[napi(object)] +#[derive(Default)] +pub struct OpenTelemetryFileSinkConfig { + /// `"full"`, `"gen_ai"`, or `"openinference"`. + #[napi(ts_type = "\"full\" | \"gen_ai\" | \"openinference\"")] + pub r#type: String, + /// Directory containing the output file. + pub output_directory: String, + /// Output filename. Defaults to a name derived from `format`. + pub filename: Option, + /// `"json_lines"` (default) or `"proto"`. + #[napi(ts_type = "\"json_lines\" | \"proto\"")] + pub format: Option, + /// `"overwrite"` (default) or `"append"`. + #[napi(ts_type = "\"append\" | \"overwrite\"")] + pub mode: Option, + /// Extra OpenTelemetry resource attributes as string key/value pairs. + pub resource_attributes: Option, + /// `service.name` resource attribute. Defaults to `"unknown_service"`. + pub service_name: Option, + /// Optional `service.namespace` resource attribute. + pub service_namespace: Option, + /// Optional `service.version` resource attribute. + pub service_version: Option, + /// Instrumentation scope name. Defaults to `"opentelemetry"`. + pub instrumentation_scope: Option, +} + +fn build_otel_file_sink_config( + options: OpenTelemetryFileSinkConfig, +) -> napi::Result { + let otel_type = parse_otel_type(&options.r#type)?; + let directory = options.output_directory.trim(); + if directory.is_empty() { + return Err(napi::Error::from_reason( + "outputDirectory must be a nonblank string", + )); + } + let sink = parse_otel_file_sink( + directory, + options.filename.as_deref(), + options.format.as_deref(), + options.mode.as_deref(), + )?; + let mut config = + nemo_relay::observability::otel::OpenTelemetryFileSinkConfig::new(otel_type, sink) + .with_service_name( + options + .service_name + .unwrap_or_else(|| "unknown_service".to_string()), + ) + .with_instrumentation_scope( + options + .instrumentation_scope + .unwrap_or_else(|| "opentelemetry".to_string()), + ); + if let Some(namespace) = options.service_namespace { + config = config.with_service_namespace(namespace); + } + if let Some(version) = options.service_version { + config = config.with_service_version(version); + } + for (key, value) in parse_string_map(options.resource_attributes, "resourceAttributes")? { + config = config.with_resource_attribute(key, value); + } + Ok(config) +} + /// OpenTelemetry-backed event subscriber. #[napi] pub struct OpenTelemetrySubscriber { @@ -5269,6 +5385,16 @@ impl OpenTelemetrySubscriber { Ok(Self { inner }) } + /// Create a subscriber that writes OTLP to a local file. + #[napi(factory)] + pub fn file_sink(config: OpenTelemetryFileSinkConfig) -> napi::Result { + let inner = nemo_relay::observability::otel::OpenTelemetrySubscriber::new_file_sink( + build_otel_file_sink_config(config)?, + ) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + Ok(Self { inner }) + } + /// Register this subscriber globally with the given name. #[napi] pub fn register(&self, name: String) -> napi::Result<()> { diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index b3ca29543..421a6e519 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -35,6 +35,7 @@ describe('observability plugin helpers', () => { assert.deepEqual(observability.openTelemetryConfig(), { enabled: false, endpoints: [], + file_sinks: [], }); assert.deepEqual( observability.openTelemetryEndpoint({ @@ -115,6 +116,7 @@ describe('observability plugin helpers', () => { assert.deepEqual(observability.openTelemetryConfig({ enabled: true, logs, metrics }), { enabled: true, endpoints: [], + file_sinks: [], logs, metrics, }); @@ -346,3 +348,55 @@ describe('observability plugin helpers', () => { assert.doesNotMatch(secondPayload, /node-nested-agent/); }); }); + +describe('opentelemetry file sinks', () => { + it('normalizes a file sink and validates its fields', () => { + const directory = tempDir('otlp-file-sink'); + const sink = observability.openTelemetryFileSink({ output_directory: directory }); + + assert.equal(sink.type, 'full'); + assert.equal(sink.format, 'json_lines'); + assert.equal(sink.mode, 'overwrite'); + assert.equal(sink.output_directory, directory); + // A file sink has no network destination to configure. + assert.equal(sink.endpoint, undefined); + assert.equal(sink.transport, undefined); + + assert.throws(() => observability.openTelemetryFileSink(), /config is required/); + assert.throws(() => observability.openTelemetryFileSink({ output_directory: ' ' }), /nonblank/); + assert.throws( + () => observability.openTelemetryFileSink({ output_directory: directory, format: 'yaml' }), + /"json_lines" or "proto"/, + ); + assert.throws( + () => observability.openTelemetryFileSink({ output_directory: directory, mode: 'truncate' }), + /"append" or "overwrite"/, + ); + }); + + it('writes a trace file for a file-sink-only section', async () => { + const directory = tempDir('otlp-file-sink-write'); + const config = { + version: 4, + opentelemetry: observability.openTelemetryConfig({ + enabled: true, + file_sinks: [ + observability.openTelemetryFileSink({ + output_directory: directory, + filename: 'node-trace.jsonl', + }), + ], + }), + }; + + await pluginHost.initialize({ + version: 1, + components: [observability.ComponentSpec(config)], + }); + try { + assert.deepEqual(readdirSync(directory), ['node-trace.jsonl']); + } finally { + await pluginHost.close(); + } + }); +}); diff --git a/crates/python/Cargo.toml b/crates/python/Cargo.toml index c9b8296a6..9ca47ccdd 100644 --- a/crates/python/Cargo.toml +++ b/crates/python/Cargo.toml @@ -36,3 +36,4 @@ chrono = "0.4" [dev-dependencies] nemo-relay = { workspace = true, features = ["__test-plugin-host"] } +tempfile = "3" diff --git a/crates/python/src/py_types/mod.rs b/crates/python/src/py_types/mod.rs index 1da1b1cc4..1168922b9 100644 --- a/crates/python/src/py_types/mod.rs +++ b/crates/python/src/py_types/mod.rs @@ -186,6 +186,7 @@ fn register_observability_types(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/python/src/py_types/observability.rs b/crates/python/src/py_types/observability.rs index 3324399d2..6fd116185 100644 --- a/crates/python/src/py_types/observability.rs +++ b/crates/python/src/py_types/observability.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use pyo3::prelude::*; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tokio::runtime::{Handle, Runtime}; use super::{ @@ -499,6 +499,155 @@ pub struct PyOpenTelemetryConfig { pub(crate) promote_resource_metadata_prefixes: Vec, } +/// Configuration for an OpenTelemetry subscriber writing OTLP to a local file. +/// +/// Example: +/// ```python +/// config = OpenTelemetryFileSinkConfig("full", "/tmp/relay-traces") +/// ``` +#[pyclass(name = "OpenTelemetryFileSinkConfig", skip_from_py_object)] +#[derive(Clone)] +pub(crate) struct PyOtlpFileSink { + #[pyo3(get, set, name = "type")] + pub(crate) otel_type: String, + #[pyo3(get, set)] + pub(crate) service_name: String, + #[pyo3(get, set)] + pub(crate) service_namespace: Option, + #[pyo3(get, set)] + pub(crate) service_version: Option, + #[pyo3(get, set)] + pub(crate) instrumentation_scope: String, + #[pyo3(get, set)] + pub(crate) output_directory: String, + #[pyo3(get, set)] + pub(crate) filename: Option, + #[pyo3(get, set)] + pub(crate) format: String, + #[pyo3(get, set)] + pub(crate) mode: String, + pub(crate) resource_attributes: HashMap, +} + +#[pymethods] +impl PyOtlpFileSink { + #[new] + #[pyo3(signature = (otel_type, output_directory, filename=None, format="json_lines".to_string(), mode="overwrite".to_string()))] + pub(crate) fn new( + otel_type: String, + output_directory: String, + filename: Option, + format: String, + mode: String, + ) -> Self { + Self { + otel_type, + service_name: "unknown_service".to_string(), + service_namespace: None, + service_version: None, + instrumentation_scope: "opentelemetry".to_string(), + output_directory, + filename, + format, + mode, + resource_attributes: HashMap::new(), + } + } + + /// Add an OpenTelemetry resource attribute. + pub(crate) fn set_resource_attribute(&mut self, key: String, value: String) { + self.resource_attributes.insert(key, value); + } + + pub(crate) fn __repr__(&self) -> String { + format!( + "", + self.output_directory, self.format + ) + } +} + +impl PyOtlpFileSink { + pub(crate) fn to_rust_config( + &self, + ) -> PyResult { + let otel_type = parse_py_otel_type(&self.otel_type)?; + let mut config = nemo_relay::observability::otel::OpenTelemetryFileSinkConfig::new( + otel_type, + self.to_settings()?, + ) + .with_service_name(self.service_name.clone()) + .with_instrumentation_scope(self.instrumentation_scope.clone()); + if let Some(namespace) = &self.service_namespace { + config = config.with_service_namespace(namespace.clone()); + } + if let Some(version) = &self.service_version { + config = config.with_service_version(version.clone()); + } + for (key, value) in &self.resource_attributes { + config = config.with_resource_attribute(key.clone(), value.clone()); + } + Ok(config) + } +} + +fn parse_py_otel_type(value: &str) -> PyResult { + match value { + "full" => Ok(nemo_relay::observability::OpenTelemetryType::Full), + "gen_ai" => Ok(nemo_relay::observability::OpenTelemetryType::GenAi), + "openinference" => Ok(nemo_relay::observability::OpenTelemetryType::OpenInference), + other => Err(pyo3::exceptions::PyValueError::new_err(format!( + "type must be 'full', 'gen_ai', or 'openinference', got {other:?}" + ))), + } +} + +impl PyOtlpFileSink { + fn to_settings(&self) -> PyResult { + let format = match self.format.as_str() { + "json_lines" => nemo_relay::observability::otel_file::OtlpFileFormat::JsonLines, + "proto" => nemo_relay::observability::otel_file::OtlpFileFormat::Proto, + other => { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "format must be 'json_lines' or 'proto', got {other:?}" + ))); + } + }; + let append = match self.mode.as_str() { + "append" => true, + "overwrite" => false, + other => { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "mode must be 'append' or 'overwrite', got {other:?}" + ))); + } + }; + if self.output_directory.trim().is_empty() { + return Err(pyo3::exceptions::PyValueError::new_err( + "output_directory must be a nonblank path", + )); + } + let output_directory = PathBuf::from(&self.output_directory); + let filename = match &self.filename { + Some(filename) => { + if Path::new(filename).components().count() != 1 { + return Err(pyo3::exceptions::PyValueError::new_err( + "filename must be a single path component", + )); + } + filename.clone() + } + None => format!("nemo-relay-otlp.{}", format.extension()), + }; + Ok(nemo_relay::observability::otel::OtlpFileSinkSettings { + path: output_directory.join(filename), + output_directory, + format, + append, + }) + } +} + impl PyOpenTelemetryConfig { pub(crate) fn to_rust_config( &self, @@ -520,12 +669,13 @@ impl PyOpenTelemetryConfig { self.endpoint.clone(), ) .with_transport(transport) - .with_service_name(self.service_name.clone()) - .with_instrumentation_scope(self.instrumentation_scope.clone()) - .with_timeout(Duration::from_millis(self.timeout_millis)) - .with_completed_span_context_ttl(Duration::from_millis( - self.completed_span_context_ttl_millis, - )); + .with_timeout(Duration::from_millis(self.timeout_millis)); + config = config + .with_service_name(self.service_name.clone()) + .with_instrumentation_scope(self.instrumentation_scope.clone()) + .with_completed_span_context_ttl(Duration::from_millis( + self.completed_span_context_ttl_millis, + )); if let Some(namespace) = &self.service_namespace { config = config.with_service_namespace(namespace.clone()); @@ -690,7 +840,25 @@ pub struct PyOpenTelemetrySubscriber { #[pymethods] impl PyOpenTelemetrySubscriber { #[new] - pub(crate) fn new(config: PyRef<'_, PyOpenTelemetryConfig>) -> PyResult { + pub(crate) fn new(config: &Bound<'_, PyAny>) -> PyResult { + if let Ok(file_sink) = config.extract::>() { + let rust_config = file_sink.to_rust_config()?; + let inner = nemo_relay::observability::otel::OpenTelemetrySubscriber::new_file_sink( + rust_config, + ) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + return Ok(Self { + inner, + owned_runtime: None, + }); + } + let config = config + .extract::>() + .map_err(|_| { + pyo3::exceptions::PyTypeError::new_err( + "config must be an OpenTelemetryConfig or OpenTelemetryFileSinkConfig", + ) + })?; let rust_config = config.to_rust_config()?; let needs_owned_runtime = config.transport == "grpc" && Handle::try_current().is_err(); if needs_owned_runtime { diff --git a/crates/python/tests/coverage/py_types_coverage_tests.rs b/crates/python/tests/coverage/py_types_coverage_tests.rs index ea908aca8..bcf64c9b7 100644 --- a/crates/python/tests/coverage/py_types_coverage_tests.rs +++ b/crates/python/tests/coverage/py_types_coverage_tests.rs @@ -486,7 +486,7 @@ fn test_open_telemetry_config_and_subscriber_cover_lifecycle() { ); let config = pyo3::Py::new(py, config).unwrap(); - let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).borrow()).unwrap(); + let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()).unwrap(); let subscriber_name = format!("py_otel_{}", Uuid::now_v7().simple()); subscriber.register(subscriber_name.clone()).unwrap(); assert!(subscriber.deregister(subscriber_name.clone()).unwrap()); @@ -497,6 +497,91 @@ fn test_open_telemetry_config_and_subscriber_cover_lifecycle() { }); } +#[test] +fn test_open_telemetry_file_sink_config_writes_a_trace_file() { + let _python = crate::test_support::init_python_test(); + let directory = tempfile::tempdir().unwrap(); + Python::attach(|py| { + let config = PyOtlpFileSink::new( + "full".into(), + directory.path().display().to_string(), + Some("py-trace.jsonl".into()), + "json_lines".into(), + "overwrite".into(), + ); + let config = pyo3::Py::new(py, config).unwrap(); + let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()).unwrap(); + assert!(directory.path().join("py-trace.jsonl").is_file()); + subscriber.shutdown(py).unwrap(); + }); +} + +#[test] +fn test_open_telemetry_file_sink_defaults_name_the_file_after_the_format() { + let _python = crate::test_support::init_python_test(); + let directory = tempfile::tempdir().unwrap(); + Python::attach(|py| { + let config = PyOtlpFileSink::new( + "full".into(), + directory.path().display().to_string(), + None, + "proto".into(), + "overwrite".into(), + ); + let config = pyo3::Py::new(py, config).unwrap(); + let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()).unwrap(); + assert!(directory.path().join("nemo-relay-otlp.otlp.pb").is_file()); + subscriber.shutdown(py).unwrap(); + }); +} + +#[test] +fn test_open_telemetry_file_sink_rejects_invalid_inputs() { + let _python = crate::test_support::init_python_test(); + let directory = tempfile::tempdir().unwrap(); + Python::attach(|py| { + for (filename, format, mode, expected) in [ + (None, "yaml", "overwrite", "format must be"), + (None, "proto", "truncate", "mode must be"), + ( + Some("../escape.jsonl"), + "proto", + "append", + "single path component", + ), + ] { + let config = PyOtlpFileSink::new( + "full".into(), + directory.path().display().to_string(), + filename.map(str::to_string), + format.into(), + mode.into(), + ); + let config = pyo3::Py::new(py, config).unwrap(); + let Err(error) = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()) else { + panic!("expected {expected:?} to be rejected"); + }; + assert!( + error.to_string().contains(expected), + "expected {expected:?}, got {error}" + ); + } + + let config = PyOtlpFileSink::new( + "full".into(), + String::new(), + None, + "json_lines".into(), + "overwrite".into(), + ); + let config = pyo3::Py::new(py, config).unwrap(); + let Err(error) = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()) else { + panic!("a blank output_directory must be rejected"); + }; + assert!(error.to_string().contains("output_directory")); + }); +} + #[test] fn test_open_telemetry_config_rejects_invalid_inputs() { let _python = crate::test_support::init_python_test(); @@ -552,7 +637,7 @@ fn test_openinference_typed_otel_config_and_subscriber_cover_lifecycle() { ); let config = pyo3::Py::new(py, config).unwrap(); - let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).borrow()).unwrap(); + let subscriber = PyOpenTelemetrySubscriber::new(config.bind(py).as_any()).unwrap(); let subscriber_name = format!("py_openinference_{}", Uuid::now_v7().simple()); subscriber.register(subscriber_name.clone()).unwrap(); assert!(subscriber.deregister(subscriber_name.clone()).unwrap()); diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index d923d1f86..5c4d7ca5d 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -288,6 +288,56 @@ are configured. gRPC does not use HTTP redirects. | `promote_metadata_prefixes` | `[]` | Literal prefixes that select sanitized Event metadata to copy to top-level span attributes. | | `promote_resource_metadata_prefixes` | `[]` | Literal prefixes that select root Scope-start metadata to copy to OTLP resource attributes. Each unique effective resource retains an exporter pipeline for the subscriber lifetime. | +## Trace File Sinks + +A file sink writes spans to a local file instead of exporting them to a +collector. File sinks and endpoints can be configured together, and each +destination receives the same projected spans. + +```toml +[[components.config.opentelemetry.file_sinks]] +type = "full" +output_directory = "/var/log/nemo-relay" +format = "json_lines" +``` + +Two formats are available. `json_lines` implements the [OpenTelemetry Protocol +File Exporter specification](https://opentelemetry.io/docs/specs/otel/protocol/file-exporter/) +and is the default; use it for interoperability with other OTLP tooling. +`proto` writes the same records as length-delimited protobuf, matching the +OpenTelemetry Collector file exporter's `format: proto`; use it when file size +or parse cost matters. + +Each export is flushed before it is reported as delivered, so a run that exits +between batches still leaves a readable prefix. Output files are created with +owner-only permissions and confined to `output_directory`, the same as the ATOF +and ATIF file sinks. + +Two file sinks cannot write the same path; the configuration is rejected at +activation. + +| Field | Default | Notes | +|---|---|---| +| `type` | `full` | `full`, `gen_ai`, or `openinference`. | +| `output_directory` | Required | Directory containing the output file. Created if absent. | +| `filename` | Timestamped | Single path component. Defaults to `nemo-relay-otlp-.jsonl` or `.otlp.pb` for the chosen format. | +| `format` | `json_lines` | `json_lines` or `proto`. | +| `mode` | `overwrite` | `append` or `overwrite`. | +| `service_name` | `unknown_service` | `service.name` resource attribute. | +| `service_namespace` | Omitted | Optional `service.namespace`. | +| `service_version` | Omitted | Optional `service.version`. | +| `instrumentation_scope` | `opentelemetry` | Instrumentation scope name. | +| `max_queue_size` | Environment or `2048` | Maximum completed spans buffered before this sink drops new spans. | +| `max_export_batch_size` | Environment or `512` | Maximum spans written in one batch; capped at the effective queue size. | +| `scheduled_delay_millis` | Environment or `5000` ms | Maximum delay before this sink writes a non-full batch. | +| `completed_span_context_ttl_millis` | `60000` | Positive duration for retaining completed scopes' trace context for late marks. | +| `resource_attributes` | `{}` | String-to-string resource attributes. | +| `mark_projection` | `inherit` | Mark representation for `full` and `openinference`: `inherit`, `event`, or `tool`. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `full` and `openinference` projection. | +| `attribute_mappings` | `[]` | `{ key, alias }` copies applied by `full` and `openinference` projection. | +| `promote_metadata_prefixes` | `[]` | Literal prefixes that select sanitized Event metadata to copy to top-level span attributes. | +| `promote_resource_metadata_prefixes` | `[]` | Literal prefixes that select root Scope-start metadata to copy to OTLP resource attributes. | + ## Event Metadata Promotion Set `promote_metadata_prefixes` on a trace endpoint to copy selected keys from diff --git a/go/nemo_relay/nemo-relay-events-2026-09-21-15.24.52.jsonl b/go/nemo_relay/nemo-relay-events-2026-09-21-15.24.52.jsonl new file mode 100644 index 000000000..e69de29bb diff --git a/go/nemo_relay/nemo-relay-events-2026-09-21-15.25.33.jsonl b/go/nemo_relay/nemo-relay-events-2026-09-21-15.25.33.jsonl new file mode 100644 index 000000000..e69de29bb diff --git a/go/nemo_relay/nemo_relay.go b/go/nemo_relay/nemo_relay.go index 9efa93788..adc72f11e 100644 --- a/go/nemo_relay/nemo_relay.go +++ b/go/nemo_relay/nemo_relay.go @@ -287,6 +287,7 @@ extern int32_t nemo_relay_otel_subscriber_create_with_projection_options(const c extern int32_t nemo_relay_otel_subscriber_create_with_projection_options_v2(const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, uint64_t, const char*, const char*, const char*, const char*, void**); extern int32_t nemo_relay_otel_subscriber_create_with_projection_options_v3(const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, uint64_t, const char*, const char*, const char*, const char*, uint64_t, void**); extern int32_t nemo_relay_otel_subscriber_create_with_projection_options_v4(const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, uint64_t, const char*, const char*, const char*, const char*, uint64_t, void**); +extern int32_t nemo_relay_otel_subscriber_create_file_sink(const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, const char*, void**); extern int32_t nemo_relay_otel_subscriber_register(const void*, const char*); extern int32_t nemo_relay_otel_subscriber_deregister(const char*); extern int32_t nemo_relay_otel_subscriber_force_flush(const void*); @@ -2659,6 +2660,108 @@ func NewOpenTelemetrySubscriber(config OpenTelemetryConfig) (*OpenTelemetrySubsc return &OpenTelemetrySubscriber{ptr: ptr}, nil } +// OpenTelemetryFileSinkFormat selects the on-disk encoding for a file sink. +type OpenTelemetryFileSinkFormat string + +const ( + // OpenTelemetryFileSinkFormatJSONLines writes one OTLP/JSON-encoded + // ExportTraceServiceRequest per line, the serialization described by the + // OpenTelemetry Protocol File Exporter specification. + OpenTelemetryFileSinkFormatJSONLines OpenTelemetryFileSinkFormat = "json_lines" + // OpenTelemetryFileSinkFormatProto writes each request length-delimited. + OpenTelemetryFileSinkFormatProto OpenTelemetryFileSinkFormat = "proto" +) + +// OpenTelemetryFileSinkMode selects how an existing output file is opened. +type OpenTelemetryFileSinkMode string + +const ( + // OpenTelemetryFileSinkModeOverwrite truncates an existing file. + OpenTelemetryFileSinkModeOverwrite OpenTelemetryFileSinkMode = "overwrite" + // OpenTelemetryFileSinkModeAppend appends to an existing file. + OpenTelemetryFileSinkModeAppend OpenTelemetryFileSinkMode = "append" +) + +// OpenTelemetryFileSinkConfig configures a subscriber that writes OTLP to a +// local file instead of exporting it to a collector. It carries no endpoint, +// transport, headers, or timeout: those apply only to a network destination. +type OpenTelemetryFileSinkConfig struct { + Type OpenTelemetryType + OutputDirectory string + Filename string + Format OpenTelemetryFileSinkFormat + Mode OpenTelemetryFileSinkMode + ResourceAttributes map[string]string + ServiceName string + ServiceNamespace string + ServiceVersion string + InstrumentationScope string +} + +// NewOpenTelemetryFileSinkSubscriber creates a subscriber that writes projected +// spans to a local file. +func NewOpenTelemetryFileSinkSubscriber(config OpenTelemetryFileSinkConfig) (*OpenTelemetrySubscriber, error) { + if config.Type == "" { + config.Type = OpenTelemetryTypeFull + } + if config.ServiceName == "" { + config.ServiceName = "unknown_service" + } + if config.InstrumentationScope == "" { + config.InstrumentationScope = "opentelemetry" + } + // A nil map marshals to null, which the FFI boundary rejects. + if config.ResourceAttributes == nil { + config.ResourceAttributes = map[string]string{} + } + + cType := C.CString(string(config.Type)) + defer C.free(unsafe.Pointer(cType)) + cOutputDirectory := C.CString(config.OutputDirectory) + defer C.free(unsafe.Pointer(cOutputDirectory)) + cFilename := optionalCString(config.Filename) + defer C.free(unsafe.Pointer(cFilename)) + cFormat := optionalCString(string(config.Format)) + defer C.free(unsafe.Pointer(cFormat)) + cMode := optionalCString(string(config.Mode)) + defer C.free(unsafe.Pointer(cMode)) + + resourceAttrsJSON, err := jsonMarshal(config.ResourceAttributes) + if err != nil { + return nil, err + } + cResourceAttrsJSON := C.CString(string(resourceAttrsJSON)) + defer C.free(unsafe.Pointer(cResourceAttrsJSON)) + + cServiceName := C.CString(config.ServiceName) + defer C.free(unsafe.Pointer(cServiceName)) + cServiceNamespace := optionalCString(config.ServiceNamespace) + defer C.free(unsafe.Pointer(cServiceNamespace)) + cServiceVersion := optionalCString(config.ServiceVersion) + defer C.free(unsafe.Pointer(cServiceVersion)) + cInstrumentationScope := C.CString(config.InstrumentationScope) + defer C.free(unsafe.Pointer(cInstrumentationScope)) + + var ptr unsafe.Pointer + status := C.nemo_relay_otel_subscriber_create_file_sink( + cType, + cOutputDirectory, + cFilename, + cFormat, + cMode, + cResourceAttrsJSON, + cServiceName, + cServiceNamespace, + cServiceVersion, + cInstrumentationScope, + &ptr, + ) + if err := checkStatus(status); err != nil { + return nil, err + } + return &OpenTelemetrySubscriber{ptr: ptr}, nil +} + // Register registers the subscriber globally with the given name. func (s *OpenTelemetrySubscriber) Register(name string) error { cName := C.CString(name) diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 115317475..854fae1bf 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -23,10 +23,36 @@ type ObservabilityConfig struct { type ObservabilityOpenTelemetryConfig struct { Enabled bool `json:"enabled,omitempty"` Endpoints []ObservabilityOpenTelemetryEndpointConfig `json:"endpoints,omitempty"` + FileSinks []ObservabilityOpenTelemetryFileSinkConfig `json:"file_sinks,omitempty"` Logs *ObservabilityOpenTelemetryLogConfig `json:"logs,omitempty"` Metrics *ObservabilityOpenTelemetryMetricConfig `json:"metrics,omitempty"` } +// ObservabilityOpenTelemetryFileSinkConfig configures one local file destination for +// projected OTLP spans. Format "json_lines" follows the OpenTelemetry Protocol File +// Exporter specification: one OTLP/JSON record per line. Format "proto" writes each +// record length-delimited. +type ObservabilityOpenTelemetryFileSinkConfig struct { + Type OpenTelemetryType `json:"type,omitempty"` + OutputDirectory string `json:"output_directory"` + Filename string `json:"filename,omitempty"` + Format string `json:"format,omitempty"` + Mode string `json:"mode,omitempty"` + MarkProjection string `json:"mark_projection,omitempty"` + MarkExcludeNames []string `json:"mark_exclude_names,omitempty"` + AttributeMappings []OtlpAttributeMapping `json:"attribute_mappings,omitempty"` + PromoteMetadataPrefixes []string `json:"promote_metadata_prefixes,omitempty"` + ResourceAttributes map[string]string `json:"resource_attributes,omitempty"` + ServiceName string `json:"service_name,omitempty"` + ServiceNamespace string `json:"service_namespace,omitempty"` + ServiceVersion string `json:"service_version,omitempty"` + InstrumentationScope string `json:"instrumentation_scope,omitempty"` + MaxQueueSize *uint64 `json:"max_queue_size,omitempty"` + MaxExportBatchSize *uint64 `json:"max_export_batch_size,omitempty"` + ScheduledDelayMillis *uint64 `json:"scheduled_delay_millis,omitempty"` + CompletedSpanContextTTLMillis *uint64 `json:"completed_span_context_ttl_millis,omitempty"` +} + // ObservabilityOpenTelemetrySignalEndpointConfig configures one log or metric OTLP destination. type ObservabilityOpenTelemetrySignalEndpointConfig struct { Endpoint string `json:"endpoint"` diff --git a/go/nemo_relay/otel_test.go b/go/nemo_relay/otel_test.go index fcb3281ef..c47a58de3 100644 --- a/go/nemo_relay/otel_test.go +++ b/go/nemo_relay/otel_test.go @@ -7,10 +7,12 @@ import ( "bytes" "encoding/binary" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "testing" "time" @@ -375,3 +377,170 @@ func TestOpenTelemetrySubscriberExportsGenAIAgentProjection(t *testing.T) { t.Fatal("timed out waiting for OTLP request") } } + +// TestObservabilityOpenTelemetryFileSinkConfigSerializes checks the file-sink section +// marshals to the keys the Rust plugin config deserializes, and that optional fields +// stay absent so core defaults apply. +func TestObservabilityOpenTelemetryFileSinkConfigSerializes(t *testing.T) { + config := ObservabilityOpenTelemetryConfig{ + Enabled: true, + FileSinks: []ObservabilityOpenTelemetryFileSinkConfig{{ + Type: OpenTelemetryTypeFull, + OutputDirectory: "/var/log/nemo-relay", + Format: "proto", + }}, + } + + encoded, err := json.Marshal(config) + if err != nil { + t.Fatalf("marshal file sink config: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal file sink config: %v", err) + } + sinks, ok := decoded["file_sinks"].([]any) + if !ok || len(sinks) != 1 { + t.Fatalf("expected one file_sinks entry, got %v", decoded["file_sinks"]) + } + sink, ok := sinks[0].(map[string]any) + if !ok { + t.Fatalf("expected a file sink object, got %T", sinks[0]) + } + if sink["output_directory"] != "/var/log/nemo-relay" { + t.Errorf("unexpected output_directory %v", sink["output_directory"]) + } + if sink["format"] != "proto" { + t.Errorf("unexpected format %v", sink["format"]) + } + // A file sink has no endpoint, and unset optionals must not be emitted: + // an empty mode would otherwise override the core default. + for _, absent := range []string{"endpoint", "transport", "filename", "mode"} { + if _, present := sink[absent]; present { + t.Errorf("unexpected %q in a file sink section", absent) + } + } +} + +// TestOpenTelemetryFileSinkSubscriberWritesTraceFile exercises the file-sink FFI +// entry point: a subscriber built from a file-sink config opens its output file, +// and the endpoint-only parameters are absent from the config entirely. +func TestOpenTelemetryFileSinkSubscriberWritesTraceFile(t *testing.T) { + dir := t.TempDir() + subscriber, err := NewOpenTelemetryFileSinkSubscriber(OpenTelemetryFileSinkConfig{ + Type: OpenTelemetryTypeFull, + OutputDirectory: dir, + Filename: "go-trace.jsonl", + ServiceName: "go-file-sink", + }) + if err != nil { + t.Fatalf("create file sink subscriber: %v", err) + } + defer func() { + if err := subscriber.Shutdown(); err != nil { + t.Errorf("shutdown: %v", err) + } + }() + + if _, err := os.Stat(filepath.Join(dir, "go-trace.jsonl")); err != nil { + t.Fatalf("expected the trace file to exist: %v", err) + } +} + +// TestOpenTelemetryFileSinkSubscriberDefaultsFilenameToFormat checks that an +// omitted filename is derived from the chosen format. +func TestOpenTelemetryFileSinkSubscriberDefaultsFilenameToFormat(t *testing.T) { + dir := t.TempDir() + subscriber, err := NewOpenTelemetryFileSinkSubscriber(OpenTelemetryFileSinkConfig{ + OutputDirectory: dir, + Format: OpenTelemetryFileSinkFormatProto, + Mode: OpenTelemetryFileSinkModeAppend, + }) + if err != nil { + t.Fatalf("create file sink subscriber: %v", err) + } + defer subscriber.Shutdown() //nolint:errcheck // shutdown result is asserted elsewhere + + if _, err := os.Stat(filepath.Join(dir, "nemo-relay-otlp.otlp.pb")); err != nil { + t.Fatalf("expected the default proto filename: %v", err) + } +} + +// TestOpenTelemetryFileSinkSubscriberRejectsInvalidConfig covers the rejection +// paths: each is refused rather than silently defaulted. +func TestOpenTelemetryFileSinkSubscriberRejectsInvalidConfig(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + config OpenTelemetryFileSinkConfig + }{ + {"blank output directory", OpenTelemetryFileSinkConfig{OutputDirectory: ""}}, + {"unknown format", OpenTelemetryFileSinkConfig{OutputDirectory: dir, Format: "yaml"}}, + {"unknown mode", OpenTelemetryFileSinkConfig{OutputDirectory: dir, Mode: "truncate"}}, + {"filename escapes directory", OpenTelemetryFileSinkConfig{OutputDirectory: dir, Filename: "../escape.jsonl"}}, + {"unknown type", OpenTelemetryFileSinkConfig{OutputDirectory: dir, Type: "unsupported"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + subscriber, err := NewOpenTelemetryFileSinkSubscriber(tc.config) + if err == nil { + subscriber.Shutdown() //nolint:errcheck // cleanup for an unexpected success + t.Fatalf("expected %s to be rejected", tc.name) + } + }) + } +} + +// TestOpenTelemetryFileSinkSubscriberAppliesOptionalSettings covers the optional +// fields the minimal cases leave unset, so every branch of the FFI conversion is +// exercised from Go. +func TestOpenTelemetryFileSinkSubscriberAppliesOptionalSettings(t *testing.T) { + dir := t.TempDir() + subscriber, err := NewOpenTelemetryFileSinkSubscriber(OpenTelemetryFileSinkConfig{ + Type: OpenTelemetryTypeFull, + OutputDirectory: dir, + Filename: "full.otlp.pb", + Format: OpenTelemetryFileSinkFormatProto, + Mode: OpenTelemetryFileSinkModeAppend, + ServiceName: "go-file-sink", + ServiceNamespace: "agents", + ServiceVersion: "1.2.3", + InstrumentationScope: "go-scope", + ResourceAttributes: map[string]string{"deployment.environment": "test"}, + }) + if err != nil { + t.Fatalf("create file sink subscriber: %v", err) + } + defer func() { + if err := subscriber.Shutdown(); err != nil { + t.Errorf("shutdown: %v", err) + } + }() + + if _, err := os.Stat(filepath.Join(dir, "full.otlp.pb")); err != nil { + t.Fatalf("expected the trace file to exist: %v", err) + } +} + +// TestOpenTelemetryFileSinkSubscriberRegisters checks the subscriber reaches the +// shared registration path rather than only being constructed. +func TestOpenTelemetryFileSinkSubscriberRegisters(t *testing.T) { + dir := t.TempDir() + subscriber, err := NewOpenTelemetryFileSinkSubscriber(OpenTelemetryFileSinkConfig{ + OutputDirectory: dir, + Filename: "registered.jsonl", + }) + if err != nil { + t.Fatalf("create file sink subscriber: %v", err) + } + defer subscriber.Shutdown() //nolint:errcheck // asserted by the sibling test + + name := fmt.Sprintf("go_file_sink_%d", time.Now().UnixNano()) + if err := subscriber.Register(name); err != nil { + t.Fatalf("register: %v", err) + } + if err := subscriber.Deregister(name); err != nil { + t.Fatalf("deregister: %v", err) + } +} diff --git a/python/nemo_relay/__init__.py b/python/nemo_relay/__init__.py index 70e123c0b..8c202af91 100644 --- a/python/nemo_relay/__init__.py +++ b/python/nemo_relay/__init__.py @@ -115,6 +115,7 @@ async def main(): MetricTemporality, MetricValueType, OpenTelemetryConfig, + OpenTelemetryFileSinkConfig, OpenTelemetryLogConfig, OpenTelemetryLogSubscriber, OpenTelemetryMetricConfig, @@ -755,6 +756,7 @@ def worker() -> None: "AtofExporterConfig", "AtofExporter", "OpenTelemetryConfig", + "OpenTelemetryFileSinkConfig", "OpenTelemetrySubscriber", "OpenTelemetryRuntimeDiagnostic", "OpenTelemetryRuntimeDiagnostics", diff --git a/python/nemo_relay/__init__.pyi b/python/nemo_relay/__init__.pyi index 62d6baec1..b15b53143 100644 --- a/python/nemo_relay/__init__.pyi +++ b/python/nemo_relay/__init__.pyi @@ -110,6 +110,9 @@ from nemo_relay._native import ( from nemo_relay._native import ( OpenTelemetryConfig as OpenTelemetryConfig, ) +from nemo_relay._native import ( + OpenTelemetryFileSinkConfig as OpenTelemetryFileSinkConfig, +) from nemo_relay._native import ( OpenTelemetryLogConfig as OpenTelemetryLogConfig, ) diff --git a/python/nemo_relay/_native.pyi b/python/nemo_relay/_native.pyi index 3a87b0c85..6fc2c892a 100644 --- a/python/nemo_relay/_native.pyi +++ b/python/nemo_relay/_native.pyi @@ -1221,6 +1221,40 @@ class OpenTelemetryConfig: """Set one OpenTelemetry resource attribute key/value pair.""" ... +class OpenTelemetryFileSinkConfig: + """Configuration for a subscriber that writes OTLP to a local file. + + Description: + Carries no endpoint, transport, headers, or timeout: a file + destination has no use for them. ``json_lines`` is the OpenTelemetry + file-exporter specification's serialization; ``proto`` writes each + record length-delimited. + """ + + type: Literal["full", "gen_ai", "openinference"] + output_directory: str + filename: Optional[str] + format: Literal["json_lines", "proto"] + mode: Literal["append", "overwrite"] + service_name: str + service_namespace: Optional[str] + service_version: Optional[str] + instrumentation_scope: str + + def __init__( + self, + otel_type: Literal["full", "gen_ai", "openinference"], + output_directory: str, + filename: Optional[str] = None, + format: Literal["json_lines", "proto"] = "json_lines", + mode: Literal["append", "overwrite"] = "overwrite", + ) -> None: + """Create a file sink config.""" + ... + def set_resource_attribute(self, key: str, value: str) -> None: + """Add an OpenTelemetry resource attribute.""" + ... + class OpenTelemetrySubscriber: """OpenTelemetry-backed NeMo Relay event subscriber. diff --git a/python/tests/test_types.py b/python/tests/test_types.py index ada4de72b..03f5d1158 100644 --- a/python/tests/test_types.py +++ b/python/tests/test_types.py @@ -26,6 +26,7 @@ MetricTemporality, MetricValueType, OpenTelemetryConfig, + OpenTelemetryFileSinkConfig, OpenTelemetryLogConfig, OpenTelemetryLogSubscriber, OpenTelemetryMetricConfig, @@ -803,6 +804,78 @@ def test_config_defaults_mutation_and_repr(self) -> None: assert config.promote_resource_metadata_prefixes == ["deployment."] assert "OpenTelemetryConfig" in repr(config) + def test_file_sink_config_writes_a_trace_file(self, tmp_path) -> None: + config = OpenTelemetryFileSinkConfig("full", str(tmp_path), "py-trace.jsonl") + + subscriber = OpenTelemetrySubscriber(config) + try: + assert (tmp_path / "py-trace.jsonl").is_file() + finally: + subscriber.shutdown() + + def test_file_sink_defaults_name_the_file_after_the_format(self, tmp_path) -> None: + subscriber = OpenTelemetrySubscriber(OpenTelemetryFileSinkConfig("full", str(tmp_path), format="proto")) + try: + assert (tmp_path / "nemo-relay-otlp.otlp.pb").is_file() + finally: + subscriber.shutdown() + + @pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({"format": "yaml"}, "format must be"), + ({"mode": "truncate"}, "mode must be"), + ({"filename": "../escape.jsonl"}, "single path component"), + ], + ) + def test_file_sink_rejects_invalid_inputs(self, tmp_path, kwargs, expected) -> None: + config = OpenTelemetryFileSinkConfig("full", str(tmp_path), **kwargs) + + with pytest.raises(ValueError, match=expected): + OpenTelemetrySubscriber(config) + + def test_file_sink_applies_every_optional_setting(self, tmp_path) -> None: + config = OpenTelemetryFileSinkConfig("full", str(tmp_path), "full.jsonl", mode="append") + config.service_name = "py-file-sink" + config.service_namespace = "agents" + config.service_version = "1.2.3" + config.instrumentation_scope = "py-scope" + config.set_resource_attribute("deployment.environment", "test") + + assert config.service_namespace == "agents" + assert config.service_version == "1.2.3" + assert config.mode == "append" + assert "OpenTelemetryFileSinkConfig" in repr(config) + + subscriber = OpenTelemetrySubscriber(config) + try: + assert (tmp_path / "full.jsonl").is_file() + finally: + subscriber.shutdown() + + def test_file_sink_rejects_an_unknown_projection_type(self, tmp_path) -> None: + config = OpenTelemetryFileSinkConfig("unsupported", str(tmp_path)) + + with pytest.raises(ValueError, match="type must be"): + OpenTelemetrySubscriber(config) + + def test_file_sink_rejects_a_blank_output_directory(self) -> None: + config = OpenTelemetryFileSinkConfig("full", " ") + + with pytest.raises(ValueError, match="output_directory"): + OpenTelemetrySubscriber(config) + + def test_file_sink_has_no_endpoint_only_attributes(self, tmp_path) -> None: + config = OpenTelemetryFileSinkConfig("full", str(tmp_path)) + + # The options are absent from the type rather than rejected at runtime. + for attribute in ("endpoint", "transport", "timeout_millis", "set_header"): + assert not hasattr(config, attribute) + + def test_subscriber_rejects_an_unrelated_object(self) -> None: + with pytest.raises(TypeError, match="OpenTelemetryConfig"): + OpenTelemetrySubscriber(object()) + def test_config_rejects_invalid_map_values(self) -> None: config = OpenTelemetryConfig("full", "http://localhost:4318/v1/traces")