diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaRequestBuilder.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaRequestBuilder.cs index 7fe59fc8aec0..a1e800cf4e1b 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaRequestBuilder.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaRequestBuilder.cs @@ -55,7 +55,7 @@ WebRequest ILambdaExtensionRequest.GetEndInvocationRequest(CallTargetState state if (span.Context.TraceContext is { } traceContext) { - var samplingPriority = traceContext.GetOrMakeSamplingDecision(span); + var samplingPriority = traceContext.GetOrMakeSamplingDecision(); request.Headers.Set(HttpHeaderNames.SamplingPriority, SamplingPriorityValues.ToString(samplingPriority)); } diff --git a/tracer/src/Datadog.Trace/Propagators/OtelTraceStateHelpers.cs b/tracer/src/Datadog.Trace/Propagators/OtelTraceStateHelpers.cs new file mode 100644 index 000000000000..5a9822aac01a --- /dev/null +++ b/tracer/src/Datadog.Trace/Propagators/OtelTraceStateHelpers.cs @@ -0,0 +1,288 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Text; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Propagators +{ + /// + /// String-surgery helpers over the raw content of the W3C tracestate "ot=" list-member + /// (OpenTelemetry consistent-probability-sampling sub-keys "rv"/"th"). The value is never + /// decoded into a typed struct: these helpers are the only code that inspects or + /// rewrites the "rv"/"th" sub-keys; every other sub-key (recognized or not) round-trips + /// byte-for-byte through in its original order. + /// + internal static class OtelTraceStateHelpers + { + private const int MaxHexDigits = 14; + private const ulong MaxOtelTraceStateValue = (1UL << (MaxHexDigits * 4)) - 1; + + /// + /// Finds the "rv" item in the raw "ot=" value (items separated by ';', key/value by ':') + /// and returns its value parsed as exactly 14 lowercase hex digits, or null if absent or malformed. + /// Never throws. + /// + internal static ulong? ExtractRv(string? raw) + { + if (StringUtil.IsNullOrEmpty(raw)) + { + return null; + } + + var remaining = raw!.AsSpan(); + + while (true) + { + var separatorIndex = remaining.IndexOf(';'); + var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex); + var colonIndex = item.IndexOf(':'); + + if (colonIndex > 0 && colonIndex < item.Length - 1 && item.Slice(0, colonIndex).Equals("rv".AsSpan(), StringComparison.Ordinal)) + { + var rvSlice = item.Slice(colonIndex + 1); + return (rvSlice.Length != MaxHexDigits) ? null : TryParseLowercaseHex(rvSlice, out var rv) ? rv : null; + } + + if (separatorIndex < 0) + { + break; + } + + remaining = remaining.Slice(separatorIndex + 1); + } + + return null; + } + + /// + /// Removes malformed "rv" and "th" items while preserving valid and unknown items. + /// Returns the original string when no rewrite is needed, and null when nothing remains. + /// + internal static string? Normalize(string? raw) + { + if (StringUtil.IsNullOrEmpty(raw)) + { + return null; + } + + var remaining = raw!.AsSpan(); + + while (true) + { + var separatorIndex = remaining.IndexOf(';'); + var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex); + + if (IsInvalidRvOrTh(item)) + { + return RemoveInvalidRvTh(raw); + } + + if (separatorIndex < 0) + { + return raw; + } + + remaining = remaining.Slice(separatorIndex + 1); + } + } + + /// + /// Drops any existing "rv"/"th" items from (whether well-formed + /// or not), then emits "rv:<14-hex-digits>" (if is non-null) + /// followed by "th:<hex, trailing zero nibbles trimmed>" (if + /// is non-null), followed by every other item from in its original + /// order. Returns null when nothing is left to emit. + /// + internal static string? SetRvTh(string? raw, ulong? rv, ulong? th) + { + if (rv is > MaxOtelTraceStateValue) + { + throw new ArgumentOutOfRangeException(nameof(rv)); + } + + var sb = StringBuilderCache.Acquire(); + + try + { + if (rv is { } rvValue) + { + AppendRandomValueHex(sb, rvValue); + } + + if (th is { } thValue) + { + if (sb.Length > 0) + { + sb.Append(';'); + } + + AppendThresholdHex(sb, thValue); + } + + if (!StringUtil.IsNullOrEmpty(raw)) + { + var remaining = raw!.AsSpan(); + + while (true) + { + var separatorIndex = remaining.IndexOf(';'); + var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex); + var colonIndex = item.IndexOf(':'); + var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item; + + if (!key.Equals("rv".AsSpan(), StringComparison.Ordinal) && !key.Equals("th".AsSpan(), StringComparison.Ordinal)) + { + if (sb.Length > 0) + { + sb.Append(';'); + } + + sb.Append(item); + } + + if (separatorIndex < 0) + { + break; + } + + remaining = remaining.Slice(separatorIndex + 1); + } + } + + return sb.Length == 0 ? null : sb.ToString(); + } + finally + { + StringBuilderCache.Release(sb); + } + } + + private static void AppendRandomValueHex(StringBuilder sb, ulong rv) + { + sb.Append("rv:"); +#if NETCOREAPP3_1_OR_GREATER + Span buffer = stackalloc char[MaxHexDigits]; + _ = rv.TryFormat(buffer, out _, "x14"); + sb.Append(buffer); +#else + sb.Append(rv.ToString("x14")); +#endif + } + + private static void AppendThresholdHex(StringBuilder sb, ulong th) + { + // Format as 14 hex digits, then trim trailing zero nibbles. + // A fully-zero threshold trims to the empty string; represent it as a single "0". + sb.Append("th:"); +#if NETCOREAPP3_1_OR_GREATER + Span buffer = stackalloc char[MaxHexDigits]; + _ = th.TryFormat(buffer, out var written, "x14"); + var trimmed = buffer.Slice(0, written).TrimEnd('0'); + + if (trimmed.IsEmpty) + { + sb.Append('0'); + } + else + { + sb.Append(trimmed); + } +#else + var hex = th.ToString("x14"); + var trimmed = hex.TrimEnd('0'); + sb.Append(trimmed.Length == 0 ? "0" : trimmed); +#endif + } + + private static string? RemoveInvalidRvTh(string raw) + { + var sb = StringBuilderCache.Acquire(); + + try + { + var remaining = raw.AsSpan(); + + while (true) + { + var separatorIndex = remaining.IndexOf(';'); + var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex); + + if (!IsInvalidRvOrTh(item)) + { + if (sb.Length > 0) + { + sb.Append(';'); + } + + sb.Append(item); + } + + if (separatorIndex < 0) + { + break; + } + + remaining = remaining.Slice(separatorIndex + 1); + } + + return sb.Length == 0 ? null : sb.ToString(); + } + finally + { + StringBuilderCache.Release(sb); + } + } + + private static bool IsInvalidRvOrTh(ReadOnlySpan item) + { + var colonIndex = item.IndexOf(':'); + var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item; + + if (key.Equals("rv".AsSpan(), StringComparison.Ordinal)) + { + var value = colonIndex > 0 ? item.Slice(colonIndex + 1) : default; + return value.Length != MaxHexDigits || !TryParseLowercaseHex(value, out _); + } + + if (key.Equals("th".AsSpan(), StringComparison.Ordinal)) + { + var value = colonIndex > 0 ? item.Slice(colonIndex + 1) : default; + return value.Length is < 1 or > MaxHexDigits || !TryParseLowercaseHex(value, out _); + } + + return false; + } + + private static bool TryParseLowercaseHex(ReadOnlySpan value, out ulong result) + { + result = 0; + foreach (var character in value) + { + int digit; + + if (character is >= '0' and <= '9') + { + digit = character - '0'; + } + else if (character is >= 'a' and <= 'f') + { + digit = character - 'a' + 10; + } + else + { + return false; + } + + result = (result << 4) | (uint)digit; + } + + return true; + } + } +} diff --git a/tracer/src/Datadog.Trace/Propagators/SpanContextPropagator.cs b/tracer/src/Datadog.Trace/Propagators/SpanContextPropagator.cs index dda426ef29d0..0df41edb02fd 100644 --- a/tracer/src/Datadog.Trace/Propagators/SpanContextPropagator.cs +++ b/tracer/src/Datadog.Trace/Propagators/SpanContextPropagator.cs @@ -228,6 +228,7 @@ private static void MergeExtractedW3CSpanContext(SpanContext cumulativeSpanConte if (cumulativeSpanContext.RawTraceId == extractedSpanContext.RawTraceId) { cumulativeSpanContext.AdditionalW3CTraceState += extractedSpanContext.AdditionalW3CTraceState; + cumulativeSpanContext.OtelTraceState = extractedSpanContext.OtelTraceState; if (cumulativeSpanContext.RawSpanId != extractedSpanContext.RawSpanId) { diff --git a/tracer/src/Datadog.Trace/Propagators/W3CTraceContextPropagator.cs b/tracer/src/Datadog.Trace/Propagators/W3CTraceContextPropagator.cs index c94bbc831ffa..51a46d737e00 100644 --- a/tracer/src/Datadog.Trace/Propagators/W3CTraceContextPropagator.cs +++ b/tracer/src/Datadog.Trace/Propagators/W3CTraceContextPropagator.cs @@ -199,16 +199,39 @@ internal static string CreateTraceStateHeader(SpanContext context) sb.Length--; } - var additionalState = context.AdditionalW3CTraceState; - - if (!string.IsNullOrWhiteSpace(additionalState)) + var otelTraceState = context.OtelTraceState; + var additionalTraceState = context.AdditionalW3CTraceState; + ExtractMember( + additionalTraceState.AsSpan(), + "ot=", + out var originalOtelTraceState, + out var precedingMembers, + out var succeedingMembers, + out var hasOriginalOtelTraceState); + + var hasOtelTraceState = !string.IsNullOrWhiteSpace(otelTraceState); + var preserveOriginalOtelTraceState = hasOriginalOtelTraceState && + hasOtelTraceState && + originalOtelTraceState.Equals(otelTraceState.AsSpan(), StringComparison.Ordinal); + + if (preserveOriginalOtelTraceState) + { + AppendTraceStateMembers(sb, additionalTraceState.AsSpan()); + } + else { - if (sb.Length > 0) + if (hasOtelTraceState) { - sb.Append(TraceStateHeaderValuesSeparator); + if (sb.Length > 0) + { + sb.Append(TraceStateHeaderValuesSeparator); + } + + sb.Append("ot=").Append(otelTraceState); } - sb.Append(additionalState); + AppendTraceStateMembers(sb, precedingMembers); + AppendTraceStateMembers(sb, succeedingMembers); } return StringBuilderCache.GetStringAndRelease(sb); @@ -311,29 +334,38 @@ internal static W3CTraceState ParseTraceState(string? header) // header format: "[*,]dd=s:1;o:rum;t.dm:-4;t.usr.id:12345[,*]" if (string.IsNullOrWhiteSpace(header)) { - return new W3CTraceState(samplingPriority: null, origin: null, lastParent: ZeroLastParent, propagatedTags: null, additionalValues: null); + return new W3CTraceState(samplingPriority: null, origin: null, lastParent: ZeroLastParent, propagatedTags: null, additionalValues: null, otTraceState: null); } - SplitTraceStateValues( - header!.AsSpan().Trim(), + var traceState = header!.AsSpan().Trim(); + ExtractMember( + traceState, + "dd=", out var ddValues, out var precedingMembers, out var succeedingMembers, out _); + ExtractMember( + traceState, + "ot=", + out var otTraceState, + out _, + out _, + out var hasOtTraceState); var additionalValues = GetAdditionalValues(precedingMembers, succeedingMembers); - return ParseDdMember(ddValues, additionalValues); + return ParseDdMember(ddValues, additionalValues, hasOtTraceState ? otTraceState.ToString() : null); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static W3CTraceState ParseDdMember(ReadOnlySpan ddValues, string? additionalValues) + private static W3CTraceState ParseDdMember(ReadOnlySpan ddValues, string? additionalValues, string? otTraceState) { if (ddValues.Length < 3) { // "dd" section not found or it is too short // shortest valid length is 3 as in "a:b" ("dd=" prefix already stripped) // note for this case the p will be viewed as 0 if added as a span tag - return new W3CTraceState(samplingPriority: null, origin: null, lastParent: ZeroLastParent, propagatedTags: null, additionalValues); + return new W3CTraceState(samplingPriority: null, origin: null, lastParent: ZeroLastParent, propagatedTags: null, additionalValues, otTraceState); } int? samplingPriority = null; @@ -392,7 +424,7 @@ private static W3CTraceState ParseDdMember(ReadOnlySpan ddValues, string? propagatedTags = null; } - return new W3CTraceState(samplingPriority, origin.IsEmpty ? null : origin.ToString(), lastParent.IsEmpty ? ZeroLastParent : lastParent.ToString(), propagatedTags, additionalValues); + return new W3CTraceState(samplingPriority, origin.IsEmpty ? null : origin.ToString(), lastParent.IsEmpty ? ZeroLastParent : lastParent.ToString(), propagatedTags, additionalValues, otTraceState); } finally { @@ -417,16 +449,6 @@ private static bool ExtractKeyValue(ReadOnlySpan source, out ReadOnlySpan< return true; } - private static void SplitTraceStateValues( - ReadOnlySpan header, - out ReadOnlySpan ddValues, - out ReadOnlySpan precedingMembers, - out ReadOnlySpan succeedingMembers, - out bool hasDdValues) - { - ExtractMember(header, "dd=", out ddValues, out precedingMembers, out succeedingMembers, out hasDdValues); - } - private static void ExtractMember( ReadOnlySpan header, string prefix, @@ -473,20 +495,18 @@ private static void ExtractMember( succeedingMembers = endIndex == header.Length ? default : header.Slice(endIndex + 1); } - private static string? GetAdditionalValues( - ReadOnlySpan precedingMembers, - ReadOnlySpan succeedingMembers) + private static string? GetAdditionalValues(ReadOnlySpan precedingMembers, ReadOnlySpan succeedingMembers) { if (precedingMembers.IsEmpty) { return succeedingMembers.IsEmpty ? null : succeedingMembers.ToString(); } - if (succeedingMembers.IsEmpty) - { - return precedingMembers.ToString(); - } + return succeedingMembers.IsEmpty ? precedingMembers.ToString() : CombineMembers(precedingMembers, succeedingMembers); + } + private static string CombineMembers(ReadOnlySpan precedingMembers, ReadOnlySpan succeedingMembers) + { var sb = StringBuilderCache.Acquire(precedingMembers.Length + succeedingMembers.Length + 1); sb.Append(precedingMembers) .Append(TraceStateHeaderValuesSeparator) @@ -494,6 +514,21 @@ private static void ExtractMember( return StringBuilderCache.GetStringAndRelease(sb); } + private static void AppendTraceStateMembers(StringBuilder sb, ReadOnlySpan members) + { + if (members.Trim().IsEmpty) + { + return; + } + + if (sb.Length > 0) + { + sb.Append(TraceStateHeaderValuesSeparator); + } + + sb.Append(members); + } + private static int? SamplingPriorityToInt32(ReadOnlySpan samplingPriority) { return samplingPriority.Length switch @@ -569,6 +604,7 @@ public bool TryExtract( spanContext.PropagatedTags = traceTags; spanContext.AdditionalW3CTraceState = traceState.AdditionalValues; + spanContext.OtelTraceState = OtelTraceStateHelpers.Normalize(traceState.OtTraceState); spanContext.LastParentId = traceState.LastParent; context = new PropagationContext(spanContext, baggage: null); diff --git a/tracer/src/Datadog.Trace/Propagators/W3CTraceState.cs b/tracer/src/Datadog.Trace/Propagators/W3CTraceState.cs index 1f7d4436f57e..fc957c6fb44a 100644 --- a/tracer/src/Datadog.Trace/Propagators/W3CTraceState.cs +++ b/tracer/src/Datadog.Trace/Propagators/W3CTraceState.cs @@ -21,12 +21,19 @@ internal readonly struct W3CTraceState // the string left in "tracestate" after removing "dd=*" public readonly string? AdditionalValues; - public W3CTraceState(int? samplingPriority, string? origin, string? lastParent, string? propagatedTags, string? additionalValues) + /// + /// Raw content of the inbound "ot=" tracestate list-member (no "ot=" prefix), + /// captured verbatim with no sub-key parsing. Null if no "ot=" member was present. + /// + public readonly string? OtTraceState; + + public W3CTraceState(int? samplingPriority, string? origin, string? lastParent, string? propagatedTags, string? additionalValues, string? otTraceState = null) { SamplingPriority = samplingPriority; Origin = origin; LastParent = lastParent; PropagatedTags = propagatedTags; AdditionalValues = additionalValues; + OtTraceState = otTraceState; } } diff --git a/tracer/src/Datadog.Trace/Sampling/SamplingDecision.cs b/tracer/src/Datadog.Trace/Sampling/SamplingDecision.cs index 0f55b455ba43..5ff9aeeb73ee 100644 --- a/tracer/src/Datadog.Trace/Sampling/SamplingDecision.cs +++ b/tracer/src/Datadog.Trace/Sampling/SamplingDecision.cs @@ -18,7 +18,8 @@ internal readonly struct SamplingDecision priority: SamplingPriorityValues.Default, mechanism: SamplingMechanism.Default, rate: null, - limiterRate: null); + limiterRate: null, + sample: null); public readonly int Priority; @@ -28,12 +29,21 @@ internal readonly struct SamplingDecision public readonly float? LimiterRate; - public SamplingDecision(int priority, string? mechanism, float? rate, float? limiterRate) + /// + /// The raw probability keep/drop outcome (before any rate-limiter demotion), or null + /// when no probability mechanism made this decision (e.g. ). + /// Used to derive the OTel "ot.rv"/"ot.th" tracestate values in + /// — never affects . + /// + public readonly bool? KeptByProbabilitySampling; + + public SamplingDecision(int priority, string? mechanism, float? rate, float? limiterRate, bool? sample = null) { Priority = priority; Mechanism = mechanism; Rate = rate; LimiterRate = limiterRate; + KeptByProbabilitySampling = sample; } public void Deconstruct(out int priority, out string? mechanism, out float? rate, out float? limiterRate) diff --git a/tracer/src/Datadog.Trace/Sampling/TraceSampler.cs b/tracer/src/Datadog.Trace/Sampling/TraceSampler.cs index 269a3c719232..cf74da6a5a81 100644 --- a/tracer/src/Datadog.Trace/Sampling/TraceSampler.cs +++ b/tracer/src/Datadog.Trace/Sampling/TraceSampler.cs @@ -97,7 +97,7 @@ private SamplingDecision MakeSamplingDecision(Span span, float rate, string mech } } - return new SamplingDecision(priority, mechanism, rate, limiterRate); + return new SamplingDecision(priority, mechanism, rate, limiterRate, sample); } public sealed class Builder(IRateLimiter limiter) diff --git a/tracer/src/Datadog.Trace/SpanContext.cs b/tracer/src/Datadog.Trace/SpanContext.cs index 0eb636bd7800..00d3c2a41e06 100644 --- a/tracer/src/Datadog.Trace/SpanContext.cs +++ b/tracer/src/Datadog.Trace/SpanContext.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -57,7 +57,7 @@ public sealed partial class SpanContext : ISpanContext, IReadOnlyDictionary /// Initializes a new instance of the class @@ -271,18 +271,43 @@ internal string RawSpanId /// /// Gets or sets additional key/value pairs from an upstream "tracestate" W3C header that we will propagate downstream. /// This value will _not_ include the "dd" key, which is parsed out into other individual values - /// (e.g. sampling priority, origin, propagates tags, etc). + /// (e.g. sampling priority, origin, propagates tags, etc), but may include the "ot" key. /// internal string AdditionalW3CTraceState { - get => TraceContext?.AdditionalW3CTraceState ?? _additionalW3CTraceState; + get => TraceContext?.AdditionalW3CTraceState ?? _remoteW3CTraceState?.AdditionalW3CTraceState; set { - _additionalW3CTraceState = value; + if (TraceContext is { } traceContext) + { + traceContext.AdditionalW3CTraceState = value; + } + else if (_remoteW3CTraceState is not null || value is not null) + { + (_remoteW3CTraceState ??= new()).AdditionalW3CTraceState = value; + } + } + } - if (TraceContext is not null) + /// + /// Gets or sets the raw content of the inbound "ot=" W3C tracestate member + /// (OpenTelemetry consistent-probability-sampling sub-keys). Null if none was + /// present on extraction and nothing has derived one locally. + /// + [MaybeNull] + [AllowNull] + internal string OtelTraceState + { + get => TraceContext?.OtelTraceState ?? _remoteW3CTraceState?.OtelTraceState; + set + { + if (TraceContext is { } traceContext) + { + traceContext.OtelTraceState = value; + } + else if (_remoteW3CTraceState is not null || value is not null) { - TraceContext.AdditionalW3CTraceState = value; + (_remoteW3CTraceState ??= new()).OtelTraceState = value; } } } @@ -518,6 +543,13 @@ internal void ManuallySetPathwayContextToPairMessages(PathwayContext? pathwayCon PathwayContext = pathwayContext; } + private sealed class RemoteW3CTraceState + { + public string AdditionalW3CTraceState { get; set; } + + public string OtelTraceState { get; set; } + } + internal static class Keys { private const string Prefix = "__DistributedKey-"; diff --git a/tracer/src/Datadog.Trace/TraceContext.cs b/tracer/src/Datadog.Trace/TraceContext.cs index 15cbe00faaad..d594b001e1ac 100644 --- a/tracer/src/Datadog.Trace/TraceContext.cs +++ b/tracer/src/Datadog.Trace/TraceContext.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -20,6 +20,7 @@ using Datadog.Trace.FeatureFlags; using Datadog.Trace.Iast; using Datadog.Trace.Logging; +using Datadog.Trace.Propagators; using Datadog.Trace.Sampling; using Datadog.Trace.SourceGenerators; using Datadog.Trace.Tagging; @@ -40,6 +41,8 @@ internal sealed class TraceContext private IastRequestContext? _iastRequestContext; private AppSecRequestContext? _appSecRequestContext; + private string? _otelTraceState; + private bool _containsLocallyGeneratedOtelRandomValue; // Lazily created on the first feature-flag evaluation for this trace; null until then, so // traces that never evaluate a flag pay nothing. State dies with the TraceContext. @@ -102,7 +105,7 @@ public Span? RootSpan public string? SamplingMechanism { get; set; } - public float? AppliedSamplingRate { get; set; } + public double? AppliedSamplingRate { get; set; } public float? RateLimiterRate { get; set; } @@ -111,10 +114,27 @@ public Span? RootSpan /// /// Gets or sets additional key/value pairs from upstream "tracestate" header that we will propagate downstream. /// This value will _not_ include the "dd" key, which is parsed out into other individual values - /// (e.g. sampling priority, origin, propagates tags, etc). + /// (e.g. sampling priority, origin, propagates tags, etc), but may include the "ot" key. /// internal string? AdditionalW3CTraceState { get; set; } + /// + /// Gets or sets the raw content of the inbound/rewritten W3C tracestate "ot=" member + /// (OpenTelemetry consistent-probability-sampling sub-keys), with no "ot=" prefix. + /// Null means there is nothing to emit. Never decoded into typed fields — see + /// for the only code that inspects + /// or rewrites its "rv"/"th" sub-keys. + /// + internal string? OtelTraceState + { + get => _otelTraceState; + set + { + _otelTraceState = value; + _containsLocallyGeneratedOtelRandomValue = false; + } + } + /// Gets the IAST context internal IastRequestContext? IastRequestContext => _iastRequestContext; @@ -313,12 +333,7 @@ public int GetOrMakeSamplingDecision() return samplingPriority; } - return GetOrMakeSamplingDecision(_rootSpan); - } - - public int GetOrMakeSamplingDecision(Span? span) - { - if (span is null) + if (_rootSpan is null) { // we can't make a sampling decision without a root span because: // - we need a trace id, and for now trace id lives in SpanContext, not in TraceContext @@ -330,14 +345,15 @@ public int GetOrMakeSamplingDecision(Span? span) } var samplingDecision = CurrentTraceSettings?.TraceSampler is { } sampler - ? sampler.MakeSamplingDecision(span) + ? sampler.MakeSamplingDecision(_rootSpan) : SamplingDecision.Default; SetSamplingPriority( samplingDecision.Priority, samplingDecision.Mechanism, samplingDecision.Rate, - samplingDecision.LimiterRate); + samplingDecision.LimiterRate, + sample: samplingDecision.KeptByProbabilitySampling); return samplingDecision.Priority; } @@ -345,15 +361,18 @@ public int GetOrMakeSamplingDecision(Span? span) public void SetSamplingPriority( int? priority, string? mechanism = null, - float? rate = null, + double? rate = null, float? limiterRate = null, - bool notifyDistributedTracer = true) + bool notifyDistributedTracer = true, + bool? sample = null) { if (priority is not { } p) { return; } + var isLocalRoot = SamplingPriority is null; + // priority (keep/drop) can change (manually, ASM, etc) SamplingPriority = priority; @@ -376,15 +395,61 @@ public void SetSamplingPriority( Tags.RemoveTag(Trace.Tags.Propagated.DecisionMaker); } - // set Knuth sampling rate as a propagated tag for agent and rule-based sampling. - // use TryAddTag to preserve the original rate, consistent with AppliedSamplingRate ??= rate above. - if (rate is { } samplingRate && mechanism is Sampling.SamplingMechanism.AgentRate - or Sampling.SamplingMechanism.LocalTraceSamplingRule - or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule - or Sampling.SamplingMechanism.RemoteUserSamplingRule) + if (rate is { } samplingRate && samplingRate is >= 0f and <= 1f) + { + // set Knuth sampling rate as a propagated tag for agent and rule-based sampling only: + // "Default" means no agent-configured rate has been received yet (client-side fallback), + // and must not propagate as _dd.p.ksr, to stay consistent with other tracers. + if (mechanism is Sampling.SamplingMechanism.AgentRate + or Sampling.SamplingMechanism.LocalTraceSamplingRule + or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule + or Sampling.SamplingMechanism.RemoteUserSamplingRule) + { + // Format with up to 6 decimal digits and no trailing zeros. + Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("0.######", CultureInfo.InvariantCulture)); + } + + // (for OTel interop) derive/erase the "ot=" tracestate rv/th sub-keys for W3C injection on every root + // probability decision, including the "Default" mechanism fallback rate. + if (isLocalRoot && IsW3CTraceContextInjectionEnabled() && sample is { } didSample && RootSpan is { } rootSpan + && mechanism is Sampling.SamplingMechanism.AgentRate + or Sampling.SamplingMechanism.LocalTraceSamplingRule + or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule + or Sampling.SamplingMechanism.RemoteUserSamplingRule + or Sampling.SamplingMechanism.Default) + { + var rv = SamplingHelpers.ComputeOtelTraceStateRandomValue(rootSpan.TraceId128.Lower); + var th = SamplingHelpers.ComputeOtelTraceStateThreshold(samplingRate); + + // Ensure (rv, th) agrees with DD's actual keep/drop decision. + // This is due to floating point imprecision when converting to otel format + if (didSample && rv < th) + { + rv = th; + } + else if (!didSample && rv >= th) + { + rv = th > 0 ? th - 1 : 0; + } + + var rateLimiterRejected = didSample && SamplingPriorityValues.IsDrop(p); + if (rateLimiterRejected) + { + var inheritedRv = _containsLocallyGeneratedOtelRandomValue ? null : OtelTraceStateHelpers.ExtractRv(_otelTraceState); + _otelTraceState = OtelTraceStateHelpers.SetRvTh(_otelTraceState, inheritedRv ?? rv, th: null); + _containsLocallyGeneratedOtelRandomValue = inheritedRv is null; + } + else + { + _otelTraceState = OtelTraceStateHelpers.SetRvTh(_otelTraceState, rv, th); + _containsLocallyGeneratedOtelRandomValue = true; + } + } + } + else if (mechanism is Sampling.SamplingMechanism.Manual or Sampling.SamplingMechanism.Asm) { - // format with up to 6 decimal digits, no trailing zeros (per RFC) - Tags.TryAddTag(Trace.Tags.Propagated.KnuthSamplingRate, samplingRate.ToString("0.######", CultureInfo.InvariantCulture)); + var inheritedRv = _containsLocallyGeneratedOtelRandomValue ? null : OtelTraceStateHelpers.ExtractRv(_otelTraceState); + OtelTraceState = OtelTraceStateHelpers.SetRvTh(_otelTraceState, inheritedRv, th: null); } if (notifyDistributedTracer) @@ -393,6 +458,20 @@ or Sampling.SamplingMechanism.RemoteAdaptiveSamplingRule } } + private bool IsW3CTraceContextInjectionEnabled() + { + foreach (var style in Tracer.Settings.PropagationStyleInject) + { + if (string.Equals(style, ContextPropagationHeaderStyle.W3CTraceContext, StringComparison.OrdinalIgnoreCase) || + string.Equals(style, ContextPropagationHeaderStyle.Deprecated.W3CTraceContext, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + private void RunSpanSampler(in SpanCollection spans) { if (CurrentTraceSettings?.SpanSampler is null) diff --git a/tracer/src/Datadog.Trace/Tracer.cs b/tracer/src/Datadog.Trace/Tracer.cs index abb35acec096..da8450bc655a 100644 --- a/tracer/src/Datadog.Trace/Tracer.cs +++ b/tracer/src/Datadog.Trace/Tracer.cs @@ -333,6 +333,7 @@ internal SpanContext CreateSpanContext(ISpanContext parent = null, string servic traceContext.SetSamplingPriority(samplingPriority); traceContext.Origin = parentSpanContext.Origin; traceContext.AdditionalW3CTraceState = parentSpanContext.AdditionalW3CTraceState; + traceContext.OtelTraceState = parentSpanContext.OtelTraceState; } // if the parent is a remote context, set the last parent id that came from the distributed header diff --git a/tracer/src/Datadog.Trace/Util/SamplingHelpers.cs b/tracer/src/Datadog.Trace/Util/SamplingHelpers.cs index fc2910ac4a0f..59323a827b94 100644 --- a/tracer/src/Datadog.Trace/Util/SamplingHelpers.cs +++ b/tracer/src/Datadog.Trace/Util/SamplingHelpers.cs @@ -11,6 +11,8 @@ namespace Datadog.Trace.Util internal static class SamplingHelpers { private const ulong KnuthFactor = 1_111_111_111_111_111_111; + private const ulong OtelTraceStateValueRange = 1UL << 56; + private const ulong OtelTraceStateMaxValue = OtelTraceStateValueRange - 1; /// /// Determines if a trace should be kept based on its trace id and the given sampling rate. @@ -41,7 +43,20 @@ internal static bool SampleByRate(ulong id, double rate) return false; } - return (id * KnuthFactor) <= (rate * ulong.MaxValue); + return ComputeKnuthHash(id) <= (rate * ulong.MaxValue); + } + + internal static ulong ComputeKnuthHash(ulong id) => id * KnuthFactor; + + internal static ulong ComputeOtelTraceStateRandomValue(ulong traceId) => + (~ComputeKnuthHash(traceId)) >> 8; + + internal static ulong ComputeOtelTraceStateThreshold(double samplingRate) + { + var threshold = (ulong)Math.Round( + (1.0 - samplingRate) * OtelTraceStateValueRange, + MidpointRounding.AwayFromZero); + return Math.Min(threshold, OtelTraceStateMaxValue); } internal static bool IsKeptBySamplingPriority(in SpanCollection trace) diff --git a/tracer/test/Datadog.Trace.Tests/Propagators/MultiSpanContextPropagatorTests.cs b/tracer/test/Datadog.Trace.Tests/Propagators/MultiSpanContextPropagatorTests.cs index e93bb8cac067..8c17db09692a 100644 --- a/tracer/test/Datadog.Trace.Tests/Propagators/MultiSpanContextPropagatorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Propagators/MultiSpanContextPropagatorTests.cs @@ -467,6 +467,24 @@ public void Extract_Behavior_Restart() opts => opts.ExcludingMissingMembers()); } + [Fact] + public void Extract_Behavior_Restart_DoesNotCarryOverOtelTraceState() + { + var headers = new Mock(); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-000000000000000000000000075bcd15-000000003ade68b1-01" }); + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { "dd=s:1,ot=rv:ef284ace7a91e1;th:e6666666666668" }); + + var names = new[] { ContextPropagationHeaderStyle.W3CTraceContext }; + var restartPropagator = SpanContextPropagatorFactory.GetSpanContextPropagator(names, names, propagationExtractFirst: true, ExtractBehavior.Restart); + var result = restartPropagator.Extract(headers.Object); + + result.SpanContext.Should().BeNull(); + result.Links.Should().ContainSingle().Which.Context.OtelTraceState.Should().Be("rv:ef284ace7a91e1;th:e6666666666668"); + } + [Fact] public void Extract_B3SingleHeader_IHeadersCollection() { @@ -747,7 +765,7 @@ public void TraceContextPrecedence_Respected_WhenHavingMatchingTraceIds(bool ext headers.Setup(h => h.GetValues("traceparent")) .Returns(new[] { "00-11111111111111110000000000000001-000000003ade68b1-01" }); headers.Setup(h => h.GetValues("tracestate")) - .Returns(new[] { "dd=s:2;o:rum;p:0123456789abcdef;t.tid:1111111111111111,foo=1" }); + .Returns(new[] { "dd=s:2;o:rum;p:0123456789abcdef;t.tid:1111111111111111,ot=rv:ef284ace7a91e1;th:e6666666666668,foo=1" }); headers.Setup(h => h.GetValues("x-datadog-trace-id")) .Returns(new[] { "1" }); headers.Setup(h => h.GetValues("x-datadog-parent-id")) @@ -791,6 +809,7 @@ public void TraceContextPrecedence_Respected_WhenHavingMatchingTraceIds(bool ext }, opts => opts.ExcludingMissingMembers()); + result.SpanContext!.OtelTraceState.Should().Be(!extractFirst || w3CHeaderFirst ? "rv:ef284ace7a91e1;th:e6666666666668" : null); result.Baggage.Should().BeNull(); result.Links.Should().BeNullOrEmpty(); } diff --git a/tracer/test/Datadog.Trace.Tests/Propagators/OtelTraceStateHelpersTests.cs b/tracer/test/Datadog.Trace.Tests/Propagators/OtelTraceStateHelpersTests.cs new file mode 100644 index 000000000000..9e0b4c12b48e --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Propagators/OtelTraceStateHelpersTests.cs @@ -0,0 +1,88 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using Datadog.Trace.Propagators; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Propagators +{ + public class OtelTraceStateHelpersTests + { + [Theory] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData("th:e6666666666668", null)] + [InlineData("rv:ef284ace7a91e1", 0xef284ace7a91e1UL)] + [InlineData("rv:ef284ace7a91e1;th:e6666666666668", 0xef284ace7a91e1UL)] + [InlineData("th:e6666666666668;rv:ef284ace7a91e1", 0xef284ace7a91e1UL)] + [InlineData("foo:bar;rv:1;baz:qux", null)] + [InlineData("rv:zzzzzz", null)] + // rv must contain exactly 14 lowercase hexadecimal digits. + [InlineData("rv:ef284ace7a91e", null)] + [InlineData("rv:123456789abcdef1", null)] + [InlineData("rv:", null)] + public void ExtractRv_ReturnsValueOrNull(string? raw, ulong? expected) + { + OtelTraceStateHelpers.ExtractRv(raw).Should().Be(expected); + } + + [Theory] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData("rv:ef284ace7a91e1", "rv:ef284ace7a91e1")] + [InlineData("th:0", "th:0")] + [InlineData("th:e6666666666668", "th:e6666666666668")] + [InlineData("unknownkey:whatever", "unknownkey:whatever")] + [InlineData("rv:zz;th:zz", null)] + [InlineData("rv:ef284ace7a91e1;th:zz", "rv:ef284ace7a91e1")] + [InlineData("rv:zz;th:e6666666666668", "th:e6666666666668")] + [InlineData("foo:bar;rv:zz;th:e6666666666668;baz:qux", "foo:bar;th:e6666666666668;baz:qux")] + // rv values use lowercase hexadecimal digits. + [InlineData("rv:EF284ACE7A91E1;foo:bar", "foo:bar")] + // th must contain no more than 14 lowercase hexadecimal digits. + [InlineData("th:123456789abcdef;foo:bar", "foo:bar")] + [InlineData("rv:;th;foo:bar", "foo:bar")] + public void Normalize_RemovesOnlyMalformedRvAndTh(string? raw, string? expected) + { + OtelTraceStateHelpers.Normalize(raw).Should().Be(expected); + } + + [Theory] + [InlineData("rv:ef284ace7a91e1;th:e6666666666668")] + [InlineData("unknownkey:whatever")] + public void Normalize_ValidContentReturnsOriginalInstance(string raw) + { + OtelTraceStateHelpers.Normalize(raw).Should().BeSameAs(raw); + } + + [Theory] + [InlineData(null, null, null, null)] + [InlineData("", null, null, null)] + [InlineData(null, 0x1UL, null, "rv:00000000000001")] + [InlineData(null, 0xef284ace7a91e1UL, null, "rv:ef284ace7a91e1")] + [InlineData(null, null, 0xe6666666666668UL, "th:e6666666666668")] + [InlineData(null, 0xef284ace7a91e1UL, 0xe6666666666668UL, "rv:ef284ace7a91e1;th:e6666666666668")] + [InlineData(null, null, 0x100UL, "th:000000000001")] + [InlineData(null, null, 0UL, "th:0")] + [InlineData("foo:bar;rv:1;th:2", 0xef284ace7a91e1UL, 0xe6666666666668UL, "rv:ef284ace7a91e1;th:e6666666666668;foo:bar")] + [InlineData("rv:zzzz;th:2;foo:bar", 0xef284ace7a91e1UL, null, "rv:ef284ace7a91e1;foo:bar")] + [InlineData("foo:bar", null, null, "foo:bar")] + public void SetRvTh_RewritesRvAndThInPlace(string? raw, ulong? rv, ulong? th, string? expected) + { + OtelTraceStateHelpers.SetRvTh(raw, rv, th).Should().Be(expected); + } + + [Fact] + public void SetRvTh_ThrowsWhenRvExceeds56Bits() + { + FluentActions.Invoking(() => OtelTraceStateHelpers.SetRvTh(null, 1UL << 56, null)) + .Should().Throw(); + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Propagators/W3CTraceContextPropagatorTests.cs b/tracer/test/Datadog.Trace.Tests/Propagators/W3CTraceContextPropagatorTests.cs index d2f28b478728..7edefecea4b0 100644 --- a/tracer/test/Datadog.Trace.Tests/Propagators/W3CTraceContextPropagatorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Propagators/W3CTraceContextPropagatorTests.cs @@ -9,6 +9,7 @@ using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Headers; using Datadog.Trace.Propagators; +using Datadog.Trace.Sampling; using Datadog.Trace.Tagging; using Datadog.Trace.Tests.Util; using FluentAssertions; @@ -231,6 +232,32 @@ public void CreateTraceStateHeader_With128Bit_TraceId() tracestate.Should().Be("dd=s:2;p:0000000000000002"); } + [Fact] + public void CreateTraceStateHeader_EmitsOtRightAfterDd_WhenOtelTraceStateIsSet() + { + var traceContext = new TraceContext(new StubDatadogTracer()); + var spanContext = new SpanContext(parent: SpanContext.None, traceContext, serviceName: null, traceId: (TraceId)1, spanId: 2) + { + OtelTraceState = "rv:ef284ace7a91e1;th:e6666666666668", + AdditionalW3CTraceState = "congo=t61rcWkgMzE" + }; + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(spanContext); + + tracestate.Should().Be("dd=s:1;p:0000000000000002,ot=rv:ef284ace7a91e1;th:e6666666666668,congo=t61rcWkgMzE"); + } + + [Fact] + public void CreateTraceStateHeader_OmitsOtMember_WhenOtelTraceStateIsNull() + { + var traceContext = new TraceContext(new StubDatadogTracer()); + var spanContext = new SpanContext(parent: SpanContext.None, traceContext, serviceName: null, traceId: (TraceId)1, spanId: 2); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(spanContext); + + tracestate.Should().NotContain("ot="); + } + [Fact] public void Inject_IHeadersCollection() { @@ -407,6 +434,19 @@ public void ParseTraceStateWithLastParent() traceState.Should().BeEquivalentTo(expected); } + [Theory] + [InlineData("dd=s:1;o:rum", null, null)] + [InlineData("dd=s:1,ot=rv:ef284ace7a91e1;th:e6666666666668", "rv:ef284ace7a91e1;th:e6666666666668", "ot=rv:ef284ace7a91e1;th:e6666666666668")] + [InlineData("ot=th:e6666666666668,dd=s:1", "th:e6666666666668", "ot=th:e6666666666668")] + [InlineData("foo=bar,dd=s:1,ot=rv:1,baz=qux", "rv:1", "foo=bar,ot=rv:1,baz=qux")] + [InlineData("dd=s:1,ot=", "", "ot=")] + public void ParseTraceState_CapturesOtelTraceState(string header, string expectedOtTraceState, string expectedAdditionalValues) + { + var traceState = W3CTraceContextPropagator.ParseTraceState(header); + traceState.OtTraceState.Should().Be(expectedOtTraceState); + traceState.AdditionalValues.Should().Be(expectedAdditionalValues); + } + [Fact] public void MissingLastParentId_ShouldBe_Zeroes() { @@ -803,6 +843,139 @@ public void Extract_MatchingSampled1_UsesTracestateSamplingPriority(int sampling opts => opts.ExcludingMissingMembers()); } + [Theory] + [InlineData(null)] + [InlineData("rv:ef284ace7a91e1;th:e6666666666668")] + [InlineData("th:e6666666666668")] + public void Continuation_RoundTripsOtelTraceState(string inboundOtelTraceState) + { + var headers = new Mock(MockBehavior.Strict); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-00000000000000000000000000000001-0000000000000001-01" }); + + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { inboundOtelTraceState is null ? "dd=s:1" : $"dd=s:1,ot={inboundOtelTraceState}" }); + + var result = W3CPropagator.Extract(headers.Object); + + result.SpanContext!.OtelTraceState.Should().Be(inboundOtelTraceState); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(result.SpanContext); + if (inboundOtelTraceState is null) + { + tracestate.Should().NotContain("ot="); + } + else + { + tracestate.Split(',').Should().Contain($"ot={inboundOtelTraceState}"); + } + } + + [Fact] + public void Continuation_RemovesMalformedKnownOtSubkeys() + { + const string InboundOtelTraceState = "th:zz;rv:ef284ace7a91e1"; + const string ExpectedOtelTraceState = "rv:ef284ace7a91e1"; + const string ExpectedAdditionalTraceState = "foo=bar,something=else"; + var headers = new Mock(MockBehavior.Strict); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-00000000000000000000000000000001-0000000000000001-01" }); + + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { $"foo=bar,dd=s:1,ot={InboundOtelTraceState},something=else" }); + + var result = W3CPropagator.Extract(headers.Object); + + result.SpanContext!.OtelTraceState.Should().Be(ExpectedOtelTraceState); + result.SpanContext.AdditionalW3CTraceState.Should().Be($"foo=bar,ot={InboundOtelTraceState},something=else"); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(result.SpanContext!); + tracestate.Should().Be($"dd=s:1;p:0000000000000001,ot={ExpectedOtelTraceState},{ExpectedAdditionalTraceState}"); + } + + [Fact] + public void Continuation_UnknownOtContent_RoundTripsByteForByte() + { + const string UnknownOtelTraceState = "unknownkey:whatever"; + var headers = new Mock(MockBehavior.Strict); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-00000000000000000000000000000001-0000000000000001-01" }); + + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { $"dd=s:1,ot={UnknownOtelTraceState}" }); + + var result = W3CPropagator.Extract(headers.Object); + + result.SpanContext!.OtelTraceState.Should().Be(UnknownOtelTraceState); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(result.SpanContext); + tracestate.Should().Contain($"ot={UnknownOtelTraceState}"); + } + + [Fact] + public void Continuation_UnmodifiedOtMember_PreservesTracestateOrder() + { + const string InboundTraceState = "dd=s:1,foo=bar,ot=rv:6e6d1a75832a2f,something=else"; + const string ExpectedAdditionalTraceState = "foo=bar,ot=rv:6e6d1a75832a2f,something=else"; + var headers = new Mock(MockBehavior.Strict); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-00000000000000000000000000000001-0000000000000001-01" }); + + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { InboundTraceState }); + + var result = W3CPropagator.Extract(headers.Object); + + result.SpanContext!.AdditionalW3CTraceState.Should().Be(ExpectedAdditionalTraceState); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(result.SpanContext!); + tracestate.Should().Be($"dd=s:1;p:0000000000000001,{ExpectedAdditionalTraceState}"); + } + + [Fact] + public void Continuation_MultiVendorTracestate_RoundTripsFully() + { + const string InboundTraceState = "foo1=bar1,dd=s:1,ot=rv:ef284ace7a91e1;th:e6666666666668;unknownsubkey:x,congo=t61rcWkgMzE"; + var headers = new Mock(MockBehavior.Strict); + + headers.Setup(h => h.GetValues("traceparent")) + .Returns(new[] { "00-00000000000000000000000000000001-0000000000000001-01" }); + + headers.Setup(h => h.GetValues("tracestate")) + .Returns(new[] { InboundTraceState }); + + var result = W3CPropagator.Extract(headers.Object); + + result.SpanContext!.OtelTraceState.Should().Be("rv:ef284ace7a91e1;th:e6666666666668;unknownsubkey:x"); + result.SpanContext.AdditionalW3CTraceState.Should().Be("foo1=bar1,ot=rv:ef284ace7a91e1;th:e6666666666668;unknownsubkey:x,congo=t61rcWkgMzE"); + + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(result.SpanContext); + tracestate.Should().Be("dd=s:1;p:0000000000000001,foo1=bar1,ot=rv:ef284ace7a91e1;th:e6666666666668;unknownsubkey:x,congo=t61rcWkgMzE"); + } + + [Fact] + public void RootTrace_ProbabilityKeepAtKnownRate_EmitsRfcWorkedExampleOtelTraceState() + { + // Simulates a brand-new root trace (no incoming ot=) sampled at rate=0.1 + // with trace_id_low64 = 0xfff972474538efff. + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: 0xfff972474538efff); + + traceContext.SetSamplingPriority( + priority: SamplingPriorityValues.UserKeep, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: 0.1, + sample: true); + + var spanContext = traceContext.RootSpan!.Context; + var tracestate = W3CTraceContextPropagator.CreateTraceStateHeader(spanContext); + + tracestate.Should().Contain("ot=rv:ef284ace7a91e1;th:e6666666666668"); + } + [Theory] [InlineData(SamplingPriorityValues.AutoReject)] [InlineData(SamplingPriorityValues.UserReject)] diff --git a/tracer/test/Datadog.Trace.Tests/Sampling/SamplingHelpersTests.cs b/tracer/test/Datadog.Trace.Tests/Sampling/SamplingHelpersTests.cs index 1b387b838acf..7857d7f6c09e 100644 --- a/tracer/test/Datadog.Trace.Tests/Sampling/SamplingHelpersTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Sampling/SamplingHelpersTests.cs @@ -43,4 +43,25 @@ public void SampleByRate(ulong traceId, double rate, bool expected) { SamplingHelpers.SampleByRate(traceId, rate).Should().Be(expected); } + + [Fact] + public void ComputeOtelTraceStateRandomValue_ReturnsLower56BitsOfInvertedKnuthHash() + { + const ulong traceId = 0xfff972474538efffUL; + var knuthHash = SamplingHelpers.ComputeKnuthHash(traceId); + + knuthHash.Should().Be(0x10d7b531856e1e39UL); + SamplingHelpers.ComputeOtelTraceStateRandomValue(traceId).Should().Be((~knuthHash) >> 8); + } + + [Theory] + [InlineData(0d, 0xffffffffffffffUL)] + [InlineData(0.1d, 0xe6666666666668UL)] + [InlineData(1d, 0UL)] + public void ComputeOtelTraceStateThreshold_ReturnsClampedThreshold( + double samplingRate, + ulong expected) + { + SamplingHelpers.ComputeOtelTraceStateThreshold(samplingRate).Should().Be(expected); + } } diff --git a/tracer/test/Datadog.Trace.Tests/Sampling/TraceSamplerTests.cs b/tracer/test/Datadog.Trace.Tests/Sampling/TraceSamplerTests.cs index f79d28eada8a..dff482eb6c74 100644 --- a/tracer/test/Datadog.Trace.Tests/Sampling/TraceSamplerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Sampling/TraceSamplerTests.cs @@ -173,6 +173,25 @@ public async Task Choose_Between_Sampling_Mechanisms() mechanism2.Should().Be(SamplingMechanism.AgentRate); } + [Fact] + public async Task MakeSamplingDecision_ReturnsSampleField_MatchingKnuthOutcome() + { + var settings = TracerSettings.Create(new() { { ConfigurationKeys.ServiceName, ServiceName } }); + await using var tracer = TracerHelper.CreateWithFakeAgent(settings); + + using var scope = (Scope)tracer.StartActive(OperationName); + scope.Span.Context.TraceContext.Environment = Env; + + var builder = new TraceSampler.Builder(new NoLimits()); + builder.RegisterAgentSamplingRule(new AgentSamplingRule()); + var sampler = builder.Build(); + sampler.SetDefaultSampleRates(new Dictionary { { $"service:{ServiceName},env:{Env}", 1f } }); + + var decision = sampler.MakeSamplingDecision(scope.Span); + + decision.KeptByProbabilitySampling.Should().BeTrue(); + } + private async Task RunSamplerTest( ITraceSampler sampler, int iterations, diff --git a/tracer/test/Datadog.Trace.Tests/Tagging/ActivityTagsTests.cs b/tracer/test/Datadog.Trace.Tests/Tagging/ActivityTagsTests.cs index d486b9d62e51..796710d17b22 100644 --- a/tracer/test/Datadog.Trace.Tests/Tagging/ActivityTagsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Tagging/ActivityTagsTests.cs @@ -132,4 +132,32 @@ public async Task ArrayedTags_ShouldBe_PlacedInMeta(string tagKey, object tagVal } } } + + [Fact] + public async Task ActivityLink_PreservesTraceStateWithoutParsingOtelTraceState() + { + var traceId = new Mock(); + traceId.Setup(x => x.TraceId).Returns("0af7651916cd43dd8448eb211c80319c"); + var spanId = new Mock(); + spanId.Setup(x => x.SpanId).Returns("00f067aa0ba902b7"); + var context = new Mock(); + context.Setup(x => x.TraceId).Returns(traceId.Object); + context.Setup(x => x.SpanId).Returns(spanId.Object); + context.Setup(x => x.TraceState).Returns("dd=s:1,ot=rv:ef284ace7a91e1;th:e6666666666668"); + var link = new Mock(); + link.Setup(x => x.Context).Returns(context.Object); + + var activity = new Mock(); + activity.Setup(x => x.Kind).Returns(ActivityKind.Producer); + activity.Setup(x => x.Links).Returns(new object[] { link.Object }); + + await using var tracer = TracerHelper.CreateWithFakeAgent(); + using var span = tracer.StartSpan("operation", new OpenTelemetryTags()); + + OtlpHelpers.UpdateSpanFromActivity(activity.Object, span); + + var spanLinkContext = span.SpanLinks.Should().ContainSingle().Which.Context; + spanLinkContext.AdditionalW3CTraceState.Should().Be("ot=rv:ef284ace7a91e1;th:e6666666666668"); + spanLinkContext.OtelTraceState.Should().BeNull(); + } } diff --git a/tracer/test/Datadog.Trace.Tests/TraceContextTests.cs b/tracer/test/Datadog.Trace.Tests/TraceContextTests.cs index 961395c2b130..d25741c3071e 100644 --- a/tracer/test/Datadog.Trace.Tests/TraceContextTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TraceContextTests.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Configuration; +using Datadog.Trace.Propagators; using Datadog.Trace.Sampling; using Datadog.Trace.TestHelpers; using Datadog.Trace.TestHelpers.TestTracer; @@ -20,6 +21,27 @@ namespace Datadog.Trace.Tests { public class TraceContextTests { + // Values shared by the OpenTelemetry trace-state sampling tests below. + private const ulong OtelTraceStateExampleTraceIdLower = 0xfff972474538efff; + private const ulong OtelTraceStateImprecisionClampTraceIdLower = 0x03a93ee8b1999f00; + private const ulong OtelTraceStateMinimumTraceIdLower = 1; + private const double OtelTraceStateExampleSamplingRate = 0.1; + private const double OtelTraceStateImprecisionClampSamplingRate = 0.1; + private const float OtelTraceStateRateLimiterRate = 0.05f; + private const float NeverSampleRate = 0f; + private const float AlwaysSampleRate = 1f; + + // Expected values generated from the fixed trace ID at the sampling rate. + private const string OtelTraceStateExampleRandomValue = "ef284ace7a91e1"; + private const string OtelTraceStateExampleThreshold = "e6666666666668"; + private const string OtelTraceStateMaximumThreshold = "ffffffffffffff"; + private const string OtelTraceStateMinimumThreshold = "0"; + private const string OtelTraceStateUnrelatedValue = "foo:bar"; + private const string OtelTraceStateExample = "rv:" + OtelTraceStateExampleRandomValue + ";th:" + OtelTraceStateExampleThreshold; + private const string OtelTraceStateExampleWithoutThreshold = "rv:" + OtelTraceStateExampleRandomValue; + private const string OtelTraceStateExampleWithUnrelatedValue = OtelTraceStateExample + ";" + OtelTraceStateUnrelatedValue; + private const string OtelTraceStateExampleWithoutThresholdWithUnrelatedValue = OtelTraceStateExampleWithoutThreshold + ";" + OtelTraceStateUnrelatedValue; + private readonly StubDatadogTracer _tracerMock = new(); [Fact] @@ -217,5 +239,174 @@ public async Task Null_Service_Names_Dont_Throw() span.SetService(null, null); span.Finish(); // should not throw } + + [Fact] + public void SetSamplingPriority_RootProbabilityKeep_DerivesRvTh_MatchesRfcWorkedExample() + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateExampleTraceIdLower); + + traceContext.SetSamplingPriority( + priority: SamplingPriorityValues.UserKeep, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: OtelTraceStateExampleSamplingRate, + sample: true); + + traceContext.OtelTraceState.Should().Be(OtelTraceStateExample); + } + + [Fact] + public void SetSamplingPriority_RootProbabilityDrop_StillEmitsTh() + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateExampleTraceIdLower); + + traceContext.SetSamplingPriority( + priority: SamplingPriorityValues.UserReject, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: OtelTraceStateExampleSamplingRate, + sample: false); + + traceContext.OtelTraceState.Should().Contain("th:" + OtelTraceStateExampleThreshold); + } + + [Fact] + public void SetSamplingPriority_WithoutW3CInjection_DoesNotDeriveOtelTraceState() + { + var settings = TracerSettings.Create(new() { { ConfigurationKeys.PropagationStyleInject, ContextPropagationHeaderStyle.Datadog } }); + var traceContext = new TraceContext(new StubDatadogTracer(settings)); + var spanContext = new SpanContext(parent: SpanContext.None, traceContext, serviceName: null, traceId: (TraceId)OtelTraceStateMinimumTraceIdLower, spanId: RandomIdGenerator.Shared.NextSpanId()); + traceContext.AddSpan(new Span(spanContext, DateTimeOffset.UtcNow)); + + traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, SamplingMechanism.LocalTraceSamplingRule, rate: OtelTraceStateExampleSamplingRate, sample: true); + + traceContext.OtelTraceState.Should().BeNull(); + } + + [Fact] + public void SetSamplingPriority_ImprecisionClamp_ForcesAgreementWithDdDecision() + { + // The Datadog decision and the 56-bit OTel representation can land on + // opposite sides of a sampling boundary after rate conversion. + var sample = SamplingHelpers.SampleByRate(OtelTraceStateImprecisionClampTraceIdLower, OtelTraceStateImprecisionClampSamplingRate); + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(OtelTraceStateImprecisionClampTraceIdLower); + + traceContext.SetSamplingPriority( + priority: sample ? SamplingPriorityValues.UserKeep : SamplingPriorityValues.UserReject, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: (float)OtelTraceStateImprecisionClampSamplingRate, + sample: sample); + + var rv = OtelTraceStateHelpers.ExtractRv(traceContext.OtelTraceState)!.Value; + var th = ParseThForTest(traceContext.OtelTraceState); + (rv >= th).Should().Be(sample); + } + + [Theory] + [InlineData(SamplingMechanism.Manual)] + [InlineData(SamplingMechanism.Asm)] + public void SetSamplingPriority_NonProbabilityOverride_RemovesLocallyGeneratedRv(string mechanism) + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateMinimumTraceIdLower); + traceContext.SetSamplingPriority( + SamplingPriorityValues.UserKeep, + SamplingMechanism.LocalTraceSamplingRule, + rate: OtelTraceStateExampleSamplingRate, + sample: true); + + traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, mechanism); + + traceContext.OtelTraceState.Should().BeNull(); + } + + [Fact] + public void SetSamplingPriority_RateLimiterDemotesKeep_StripsThButKeepsLocallyGeneratedRv() + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateExampleTraceIdLower); + + traceContext.SetSamplingPriority( + priority: SamplingPriorityValues.UserReject, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: OtelTraceStateExampleSamplingRate, + limiterRate: OtelTraceStateRateLimiterRate, + sample: true); + + traceContext.OtelTraceState.Should().Be(OtelTraceStateExampleWithoutThreshold); + } + + [Fact] + public void TraceSampler_LimiterDemotesKeep_KeepsRv_ViaGetOrMakeSamplingDecision() + { + var builder = new TraceSampler.Builder(new TracerRateLimiter(maxTracesPerInterval: 0, intervalMilliseconds: null)); + builder.RegisterRule(new GlobalSamplingRateRule(AlwaysSampleRate)); + var sampler = builder.Build(); + + var tracer = new StubDatadogTracer(sampler); + var rootSpan = new Span(new SpanContext(OtelTraceStateExampleTraceIdLower, RandomIdGenerator.Shared.NextSpanId()), DateTimeOffset.UtcNow); + var traceContext = new TraceContext(tracer); + traceContext.AddSpan(rootSpan); + + traceContext.GetOrMakeSamplingDecision(); + + traceContext.OtelTraceState.Should().Be(OtelTraceStateExampleWithoutThreshold); + } + + [Fact] + public void SetSamplingPriority_RateLimiterDemotesKeep_StripsInheritedThButKeepsRvAndUnknownItems() + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateExampleTraceIdLower); + traceContext.OtelTraceState = OtelTraceStateExampleWithUnrelatedValue; + + traceContext.SetSamplingPriority( + priority: SamplingPriorityValues.UserReject, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: OtelTraceStateExampleSamplingRate, + limiterRate: OtelTraceStateRateLimiterRate, + sample: true); + + traceContext.OtelTraceState.Should().Be(OtelTraceStateExampleWithoutThresholdWithUnrelatedValue); + } + + [Theory] + // `th:ffffffffffffff` encodes a zero sampling rate, so the trace must drop. + [InlineData(NeverSampleRate, OtelTraceStateExampleRandomValue, OtelTraceStateMaximumThreshold)] + // `th:0` encodes a 100% sampling rate, so the trace must keep; `rv:0` is + // valid, but also represents a keep decision at this threshold. + [InlineData(AlwaysSampleRate, OtelTraceStateExampleRandomValue, OtelTraceStateMinimumThreshold)] + public void SetSamplingPriority_BoundaryRate_ProducesValidOtelTraceState(double rate, string expectedRandomValue, string expectedThreshold) + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateExampleTraceIdLower); + var sample = SamplingHelpers.SampleByRate(OtelTraceStateExampleTraceIdLower, rate); + + traceContext.SetSamplingPriority( + priority: sample ? SamplingPriorityValues.UserKeep : SamplingPriorityValues.UserReject, + mechanism: SamplingMechanism.LocalTraceSamplingRule, + rate: rate, + sample: sample); + + traceContext.OtelTraceState.Should().Be($"rv:{expectedRandomValue};th:{expectedThreshold}"); + } + + [Fact] + public void SetSamplingPriority_ManualOverride_StripsInheritedThButKeepsRv() + { + var traceContext = TraceContextTestHelpers.CreateTraceContextWithRootSpan(traceIdLower: OtelTraceStateMinimumTraceIdLower); + traceContext.OtelTraceState = OtelTraceStateExample; + + traceContext.SetSamplingPriority(SamplingPriorityValues.UserKeep, SamplingMechanism.Manual); + + traceContext.OtelTraceState.Should().Be(OtelTraceStateExampleWithoutThreshold); + } + + private static ulong ParseThForTest(string otelTraceState) + { + foreach (var item in otelTraceState.Split(';')) + { + if (item.StartsWith("th:", StringComparison.Ordinal)) + { + return Convert.ToUInt64(item.Substring(3), 16); + } + } + + throw new InvalidOperationException("no th found"); + } } } diff --git a/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs b/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs index ee8ab7ba170a..ec1e6ddf5864 100644 --- a/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs +++ b/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs @@ -3,12 +3,15 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +#nullable enable + using System; using System.Collections.Generic; using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.Configuration.Schema; using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Sampling; namespace Datadog.Trace.Tests.Util; @@ -20,11 +23,21 @@ public StubDatadogTracer() } public StubDatadogTracer(TracerSettings settings) + : this(settings, traceSampler: null) + { + } + + public StubDatadogTracer(ITraceSampler traceSampler) + : this(new TracerSettings(NullConfigurationSource.Instance), traceSampler) + { + } + + public StubDatadogTracer(TracerSettings settings, ITraceSampler? traceSampler) { DefaultServiceName = "stub-service"; Settings = settings; var namingSchema = new NamingSchema(SchemaVersion.V0, false, false, DefaultServiceName, null, null); - PerTraceSettings = new PerTraceSettings(null, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings, new ConfigurationTelemetry())); + PerTraceSettings = new PerTraceSettings(traceSampler, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings, new ConfigurationTelemetry())); } public string DefaultServiceName { get; } diff --git a/tracer/test/Datadog.Trace.Tests/Util/TraceContextTestHelpers.cs b/tracer/test/Datadog.Trace.Tests/Util/TraceContextTestHelpers.cs new file mode 100644 index 000000000000..554c85cb56cf --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Util/TraceContextTestHelpers.cs @@ -0,0 +1,21 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Tests.Util; + +internal static class TraceContextTestHelpers +{ + public static TraceContext CreateTraceContextWithRootSpan(ulong traceIdLower) + { + var traceContext = new TraceContext(new StubDatadogTracer()); + var spanContext = new SpanContext(parent: SpanContext.None, traceContext, serviceName: null, traceId: (TraceId)traceIdLower, spanId: RandomIdGenerator.Shared.NextSpanId()); + var rootSpan = new Span(spanContext, DateTimeOffset.UtcNow); + traceContext.AddSpan(rootSpan); + return traceContext; + } +}