Skip to content

feat(otel): Optimize processing of OpenTelemetry consistent probability sampling #9184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zacharycmontoya wants to merge 4 commits into from
+371 −227
Open

feat(otel): Optimize processing of OpenTelemetry consistent probability sampling #9184

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1fa19d6
Attempt to optimize the OTel tracestate handling by storing the relev…
zacharycmontoya Sep 4, 2026
c92ea62
Apply an additional bug fix identified by agent
zacharycmontoya Sep 4, 2026
e84dbe2
Fix an regression introduced by this change: When setting TraceContex…
zacharycmontoya Sep 4, 2026
50715e4
PR feedback:
zacharycmontoya Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions tracer/src/Datadog.Trace/OtelTraceState.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// <copyright file="OtelTraceState.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

#nullable enable

using System;
using Datadog.Trace.Propagators;

namespace Datadog.Trace;

internal sealed class OtelTraceState
{
internal OtelTraceState(string? headerString)
{
CachedHeaderString = headerString;
}

/// <summary>
/// Initializes a new instance of the <see cref="OtelTraceState"/> class holding the same values as
/// <paramref name="other"/>. A single extracted <see cref="SpanContext"/> can start more than one
/// trace, and <see cref="TraceContext.SetSamplingPriority"/> mutates this object in place, so each
/// <see cref="TraceContext"/> must take its own copy rather than alias the extracted one.
/// The copy is inherited state by definition, so <see cref="LocallyGeneratedOtelRandomValue"/>
/// deliberately starts out false.
/// </summary>
internal OtelTraceState(OtelTraceState other)
{
CachedHeaderString = other.CachedHeaderString;
RandomValue = other.RandomValue;
Threshold = other.Threshold;
IsModified = other.IsModified;
}

public ulong? Threshold { get; set; }

public ulong? RandomValue { get; set; }

public string? CachedHeaderString { get; }

public bool IsModified { get; set; }

public bool LocallyGeneratedOtelRandomValue { get; set; }

/// <summary>
/// Converts the original header into a TraceState object and stores valid "rv" and "th" items
/// into their first-class properties.
/// Unknown items remain present in the cached string.
/// Returns the original string when no rewrite is needed, and null when nothing remains.
/// </summary>
internal static OtelTraceState Parse(string? raw)
{
var traceState = new OtelTraceState(raw);
if (StringUtil.IsNullOrEmpty(raw))
{
return traceState;
}

var remaining = raw!.AsSpan();

while (true)
{
var separatorIndex = remaining.IndexOf(';');
var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex);
var colonIndex = item.IndexOf(':');
var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item;
var value = colonIndex > 0 ? item.Slice(colonIndex + 1) : default;

if (key.Equals("rv".AsSpan(), StringComparison.Ordinal))
{
if (value.Length != OtelTraceStateHelpers.MaxHexDigits || !OtelTraceStateHelpers.TryParseLowercaseHex(value, out ulong randomValue))
{
// Do not store the randomValue
// Instead, keep it null and report that the object is modified from the original header
traceState.IsModified = true;
}
else
{
traceState.RandomValue = randomValue;
}
}

if (key.Equals("th".AsSpan(), StringComparison.Ordinal))
{
if (value.Length is < 1 or > OtelTraceStateHelpers.MaxHexDigits || !OtelTraceStateHelpers.TryParseLowercaseHex(value, out ulong threshold))
{
// Do not store the threshold
// Instead, keep it null and report that the object is modified from the original header
traceState.IsModified = true;
}
else
{
traceState.Threshold = threshold;
}
}

if (separatorIndex < 0)
{
return traceState;
}

remaining = remaining.Slice(separatorIndex + 1);
}
}
}
172 changes: 37 additions & 135 deletions tracer/src/Datadog.Trace/Propagators/OtelTraceStateHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ namespace Datadog.Trace.Propagators
/// </summary>
internal static class OtelTraceStateHelpers
{
private const int MaxHexDigits = 14;
private const ulong MaxOtelTraceStateValue = (1UL << (MaxHexDigits * 4)) - 1;
internal const int MaxHexDigits = 14;
internal const ulong MaxOtelTraceStateValue = (1UL << (MaxHexDigits * 4)) - 1;

/// <summary>
/// Finds the "rv" item in the raw "ot=" value (items separated by ';', key/value by ':')
Expand Down Expand Up @@ -60,110 +60,71 @@ internal static class OtelTraceStateHelpers
return null;
}

/// <summary>
/// Removes malformed "rv" and "th" items while preserving valid and unknown items.
/// Returns the original string when no rewrite is needed, and null when nothing remains.
/// </summary>
internal static string? Normalize(string? raw)
{
if (StringUtil.IsNullOrEmpty(raw))
{
return null;
}

var remaining = raw!.AsSpan();

while (true)
{
var separatorIndex = remaining.IndexOf(';');
var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex);

if (IsInvalidRvOrTh(item))
{
return RemoveInvalidRvTh(raw);
}

if (separatorIndex < 0)
{
return raw;
}

remaining = remaining.Slice(separatorIndex + 1);
}
}

/// <summary>
/// Drops any existing "rv"/"th" items from <paramref name="raw"/> (whether well-formed
/// or not), then emits "rv:&lt;14-hex-digits&gt;" (if <paramref name="rv"/> is non-null)
/// followed by "th:&lt;hex, trailing zero nibbles trimmed&gt;" (if <paramref name="th"/>
/// is non-null), followed by every other item from <paramref name="raw"/> in its original
/// order. Returns null when nothing is left to emit.
/// </summary>
internal static string? SetRvTh(string? raw, ulong? rv, ulong? th)
internal static void SetRvTh(StringBuilder sb, string? raw, ulong? rv, ulong? th)
{
if (rv is > MaxOtelTraceStateValue)
{
throw new ArgumentOutOfRangeException(nameof(rv));
}

var sb = StringBuilderCache.Acquire();
// "sb" already holds the preceding tracestate members (e.g. "dd=...,ot="), so item
// separators must be relative to where this member's content starts, not to the whole builder.
var startLength = sb.Length;

if (rv is { } rvValue)
{
AppendRandomValueHex(sb, rvValue);
}

try
if (th is { } thValue)
{
if (rv is { } rvValue)
if (sb.Length > startLength)
{
AppendRandomValueHex(sb, rvValue);
sb.Append(';');
}

if (th is { } thValue)
{
if (sb.Length > 0)
{
sb.Append(';');
}
AppendThresholdHex(sb, thValue);
}

AppendThresholdHex(sb, thValue);
}
if (!StringUtil.IsNullOrEmpty(raw))
{
var remaining = raw!.AsSpan();

if (!StringUtil.IsNullOrEmpty(raw))
while (true)
{
var remaining = raw!.AsSpan();
var separatorIndex = remaining.IndexOf(';');
var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex);
var colonIndex = item.IndexOf(':');
var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item;

while (true)
if (!key.Equals("rv".AsSpan(), StringComparison.Ordinal) && !key.Equals("th".AsSpan(), StringComparison.Ordinal))
{
var separatorIndex = remaining.IndexOf(';');
var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex);
var colonIndex = item.IndexOf(':');
var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item;

if (!key.Equals("rv".AsSpan(), StringComparison.Ordinal) && !key.Equals("th".AsSpan(), StringComparison.Ordinal))
if (sb.Length > startLength)
{
if (sb.Length > 0)
{
sb.Append(';');
}

sb.Append(item);
sb.Append(';');
}

if (separatorIndex < 0)
{
break;
}
sb.Append(item);
}

remaining = remaining.Slice(separatorIndex + 1);
if (separatorIndex < 0)
{
break;
}
}

return sb.Length == 0 ? null : sb.ToString();
}
finally
{
StringBuilderCache.Release(sb);
remaining = remaining.Slice(separatorIndex + 1);
}
}
}

private static void AppendRandomValueHex(StringBuilder sb, ulong rv)
internal static void AppendRandomValueHex(StringBuilder sb, ulong rv)
{
sb.Append("rv:");
#if NETCOREAPP3_1_OR_GREATER
Expand All @@ -175,7 +136,7 @@ private static void AppendRandomValueHex(StringBuilder sb, ulong rv)
#endif
}

private static void AppendThresholdHex(StringBuilder sb, ulong th)
internal static void AppendThresholdHex(StringBuilder sb, ulong th)
{
// Format as 14 hex digits, then trim trailing zero nibbles.
// A fully-zero threshold trims to the empty string; represent it as a single "0".
Expand All @@ -200,66 +161,7 @@ private static void AppendThresholdHex(StringBuilder sb, ulong th)
#endif
}

private static string? RemoveInvalidRvTh(string raw)
{
var sb = StringBuilderCache.Acquire();

try
{
var remaining = raw.AsSpan();

while (true)
{
var separatorIndex = remaining.IndexOf(';');
var item = separatorIndex < 0 ? remaining : remaining.Slice(0, separatorIndex);

if (!IsInvalidRvOrTh(item))
{
if (sb.Length > 0)
{
sb.Append(';');
}

sb.Append(item);
}

if (separatorIndex < 0)
{
break;
}

remaining = remaining.Slice(separatorIndex + 1);
}

return sb.Length == 0 ? null : sb.ToString();
}
finally
{
StringBuilderCache.Release(sb);
}
}

private static bool IsInvalidRvOrTh(ReadOnlySpan<char> item)
{
var colonIndex = item.IndexOf(':');
var key = colonIndex > 0 ? item.Slice(0, colonIndex) : item;

if (key.Equals("rv".AsSpan(), StringComparison.Ordinal))
{
var value = colonIndex > 0 ? item.Slice(colonIndex + 1) : default;
return value.Length != MaxHexDigits || !TryParseLowercaseHex(value, out _);
}

if (key.Equals("th".AsSpan(), StringComparison.Ordinal))
{
var value = colonIndex > 0 ? item.Slice(colonIndex + 1) : default;
return value.Length is < 1 or > MaxHexDigits || !TryParseLowercaseHex(value, out _);
}

return false;
}

private static bool TryParseLowercaseHex(ReadOnlySpan<char> value, out ulong result)
internal static bool TryParseLowercaseHex(ReadOnlySpan<char> value, out ulong result)
{
result = 0;
foreach (var character in value)
Expand Down
Loading
Loading