From 29055f768aa282efc2516b97f452ed949a70f63b Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 16:03:10 +0100 Subject: [PATCH 01/29] Subscribe StatsAggregator to settings changes --- .../Datadog.Trace/Agent/ClientStatsPayload.cs | 20 ++++++++++++----- .../Datadog.Trace/Agent/StatsAggregator.cs | 14 +++++++++--- tracer/src/Datadog.Trace/Agent/StatsBuffer.cs | 7 +++--- .../Datadog.Trace.Tests/Agent/ApiTests.cs | 10 ++++----- .../Agent/StatsBufferTests.cs | 22 ++++++++++++++----- .../DataStreamsMessagePackFormatterTests.cs | 2 +- 6 files changed, 51 insertions(+), 24 deletions(-) diff --git a/tracer/src/Datadog.Trace/Agent/ClientStatsPayload.cs b/tracer/src/Datadog.Trace/Agent/ClientStatsPayload.cs index b4fc00955f74..5d0cac05a354 100644 --- a/tracer/src/Datadog.Trace/Agent/ClientStatsPayload.cs +++ b/tracer/src/Datadog.Trace/Agent/ClientStatsPayload.cs @@ -4,21 +4,29 @@ // using System.Threading; +using Datadog.Trace.Configuration; namespace Datadog.Trace.Agent { - internal class ClientStatsPayload + internal class ClientStatsPayload(MutableSettings settings) { + private AppSettings _settings = CreateSettings(settings); private long _sequence; - public string HostName { get; set; } + public string HostName { get; init; } - public string Environment { get; set; } + public AppSettings Details => _settings; - public string Version { get; set; } - - public string ProcessTags { get; set; } + public string ProcessTags { get; init; } public long GetSequenceNumber() => Interlocked.Increment(ref _sequence); + + public void UpdateDetails(MutableSettings settings) + => Interlocked.Exchange(ref _settings, CreateSettings(settings)); + + private static AppSettings CreateSettings(MutableSettings settings) + => new(settings.Environment, settings.ServiceVersion); + + internal record AppSettings(string Environment, string Version); } } diff --git a/tracer/src/Datadog.Trace/Agent/StatsAggregator.cs b/tracer/src/Datadog.Trace/Agent/StatsAggregator.cs index d46c8c264ad7..39cc4be2264a 100644 --- a/tracer/src/Datadog.Trace/Agent/StatsAggregator.cs +++ b/tracer/src/Datadog.Trace/Agent/StatsAggregator.cs @@ -43,6 +43,7 @@ internal class StatsAggregator : IStatsAggregator private readonly ErrorSampler _errorSampler; private readonly RareSampler _rareSampler; private readonly AnalyticsEventsSampler _analyticsEventSampler; + private readonly IDisposable _settingSubscription; private int _currentBuffer; @@ -63,13 +64,19 @@ internal StatsAggregator(IApi api, TracerSettings settings, IDiscoveryService di _rareSampler = new RareSampler(settings); _analyticsEventSampler = new AnalyticsEventsSampler(); - var header = new ClientStatsPayload + // Create with the initial mutable settings, but be aware that this could change later + var header = new ClientStatsPayload(settings.Manager.InitialMutableSettings) { - Environment = settings.MutableSettings.Environment, - Version = settings.MutableSettings.ServiceVersion, HostName = HostMetadata.Instance.Hostname, ProcessTags = settings.PropagateProcessTags ? ProcessTags.SerializedTags : null }; + _settingSubscription = settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + header.UpdateDetails(mutable); + } + }); for (int i = 0; i < _buffers.Length; i++) { @@ -101,6 +108,7 @@ public Task DisposeAsync() { _discoveryService.RemoveSubscription(HandleConfigUpdate); _processExit.TrySetResult(true); + _settingSubscription.Dispose(); return _flushTask; } diff --git a/tracer/src/Datadog.Trace/Agent/StatsBuffer.cs b/tracer/src/Datadog.Trace/Agent/StatsBuffer.cs index 3f2ad976e44c..bd16b4b96f17 100644 --- a/tracer/src/Datadog.Trace/Agent/StatsBuffer.cs +++ b/tracer/src/Datadog.Trace/Agent/StatsBuffer.cs @@ -16,7 +16,7 @@ internal class StatsBuffer { private readonly List _keysToRemove; - private readonly ClientStatsPayload _header; + private ClientStatsPayload _header; public StatsBuffer(ClientStatsPayload header) { @@ -70,11 +70,12 @@ public void Serialize(Stream stream, long bucketDuration) MessagePackBinary.WriteString(stream, "Hostname"); MessagePackBinary.WriteString(stream, _header.HostName ?? string.Empty); + var details = _header.Details; MessagePackBinary.WriteString(stream, "Env"); - MessagePackBinary.WriteString(stream, _header.Environment ?? string.Empty); + MessagePackBinary.WriteString(stream, details.Environment ?? string.Empty); MessagePackBinary.WriteString(stream, "Version"); - MessagePackBinary.WriteString(stream, _header.Version ?? string.Empty); + MessagePackBinary.WriteString(stream, details.Version ?? string.Empty); if (!string.IsNullOrEmpty(_header.ProcessTags)) { diff --git a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs index 272e7d0d79d2..870b6897dd6e 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; +using Datadog.Trace.Configuration; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.Newtonsoft.Json; using FluentAssertions; @@ -167,12 +168,9 @@ public async Task SendStatsAsync_200OK_AllGood() var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); - var statsBuffer = new StatsBuffer(new ClientStatsPayload + var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), [])) { - Environment = "myEnv", - HostName = "myHost", - ProcessTags = "tag.a:b,tag.c:d", - Version = "myVersion" + ProcessTags = "tag.a:b,tag.c:d" }); await api.SendStatsAsync(statsBuffer, 1); @@ -196,7 +194,7 @@ public async Task SendStatsAsync_500_ErrorIsCaught() var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); - var statsBuffer = new StatsBuffer(new ClientStatsPayload()); + var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), []))); await api.SendStatsAsync(statsBuffer, 1); diff --git a/tracer/test/Datadog.Trace.Tests/Agent/StatsBufferTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/StatsBufferTests.cs index 4d6a2cfb9e8c..531176a49ffb 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/StatsBufferTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/StatsBufferTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using Datadog.Trace.Agent; +using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers.Stats; using FluentAssertions; using MessagePack; @@ -29,7 +30,18 @@ public void Serialization() { const long expectedDuration = 42; - var payload = new ClientStatsPayload { Environment = "Env", HostName = "Hostname", Version = "v99.99", ProcessTags = "a.b:c_d,x.y:z" }; + var settings = MutableSettings.CreateForTesting( + new(), + new() + { + { ConfigurationKeys.Environment, "Env" }, + { ConfigurationKeys.ServiceVersion, "v99.99" }, + }); + var payload = new ClientStatsPayload(settings) + { + HostName = "Hostname", + ProcessTags = "a.b:c_d,x.y:z", + }; var buffer = new StatsBuffer(payload); @@ -50,8 +62,8 @@ public void Serialization() var result = MessagePackSerializer.Deserialize(stream.ToArray()); result.Hostname.Should().Be(payload.HostName); - result.Env.Should().Be(payload.Environment); - result.Version.Should().Be(payload.Version); + result.Env.Should().Be(payload.Details.Environment); + result.Version.Should().Be(payload.Details.Version); result.ProcessTags.Should().Be(payload.ProcessTags); result.Lang.Should().Be(TracerConstants.Language); result.TracerVersion.Should().Be(TracerConstants.AssemblyVersion); @@ -75,7 +87,7 @@ public void Serialization() [Fact] public void Reset() { - var buffer = new StatsBuffer(new ClientStatsPayload()); + var buffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), []))); var key1 = new StatsAggregationKey("resource1", "service1", "operation1", "type1", 1, false); var key2 = new StatsAggregationKey("resource2", "service2", "operation2", "type2", 2, false); @@ -111,7 +123,7 @@ public void Reset() [Fact] public void IncrementSequence() { - var buffer = new StatsBuffer(new ClientStatsPayload()); + var buffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), []))); var key = new StatsAggregationKey("resource1", "service1", "operation1", "type1", 1, false); var statsBucket = new StatsBucket(key) { Duration = 1, Errors = 11, Hits = 111, TopLevelHits = 10 }; diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs index c07dcbe86fe0..07c8fb5ee800 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs @@ -189,7 +189,7 @@ public void ProcessTagsGetWritten() { var bucketDuration = 10_000_000_000; var settings = TracerSettings.Create(new Dictionary { { ConfigurationKeys.Environment, "my-env" }, { ConfigurationKeys.PropagateProcessTags, "true" } }); - var formatter = new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled), "service=name"); + var formatter = new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled)); using var ms = new MemoryStream(); formatter.Serialize(ms, bucketDuration, [], []); From b9b97a027e65ebde383ae9916299477cf66c7f81 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 16:03:20 +0100 Subject: [PATCH 02/29] Update CI Vis use of mutable settings --- .../Ci/Agent/MessagePack/CIEventMessagePackFormatter.cs | 5 +++-- tracer/src/Datadog.Trace/Ci/Net/TestOptimizationClient.cs | 7 ++++--- tracer/src/Datadog.Trace/Ci/TestOptimization.cs | 4 ++-- .../Configuration/TestOptimizationSettingsTests.cs | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tracer/src/Datadog.Trace/Ci/Agent/MessagePack/CIEventMessagePackFormatter.cs b/tracer/src/Datadog.Trace/Ci/Agent/MessagePack/CIEventMessagePackFormatter.cs index 766bea3256c9..d5b866564e89 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/MessagePack/CIEventMessagePackFormatter.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/MessagePack/CIEventMessagePackFormatter.cs @@ -40,9 +40,10 @@ internal class CIEventMessagePackFormatter : EventMessagePackFormatter Date: Wed, 22 Oct 2025 17:38:24 +0100 Subject: [PATCH 03/29] Update DSM usages --- .../DataStreamsMessagePackFormatter.cs | 36 ++++++++++++---- .../DataStreamsManager.cs | 42 +++++++++++++------ .../DataStreamsWriter.cs | 7 ++-- .../Transport/DataStreamsApi.cs | 41 ++++++++++++++---- .../src/Datadog.Trace/TracerManagerFactory.cs | 2 +- .../DataStreamsAggregatorTests.cs | 2 +- .../DataStreamsApiTests.cs | 5 ++- .../DataStreamsManagerTests.cs | 15 ++++--- .../DataStreamsMessagePackFormatterTests.cs | 6 +-- .../DataStreamsMonitoringTransportTests.cs | 11 +++-- .../DataStreamsWriterTests.cs | 8 +++- .../SpanContextDataStreamsManagerTests.cs | 16 +++---- .../TracerManagerFactoryTests.cs | 2 +- 13 files changed, 130 insertions(+), 63 deletions(-) diff --git a/tracer/src/Datadog.Trace/DataStreamsMonitoring/Aggregation/DataStreamsMessagePackFormatter.cs b/tracer/src/Datadog.Trace/DataStreamsMonitoring/Aggregation/DataStreamsMessagePackFormatter.cs index 40ba1ae96c94..f001c1ef08c5 100644 --- a/tracer/src/Datadog.Trace/DataStreamsMonitoring/Aggregation/DataStreamsMessagePackFormatter.cs +++ b/tracer/src/Datadog.Trace/DataStreamsMonitoring/Aggregation/DataStreamsMessagePackFormatter.cs @@ -7,7 +7,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Threading; using Datadog.Trace.Configuration; using Datadog.Trace.ContinuousProfiler; using Datadog.Trace.Vendors.Datadog.Sketches; @@ -18,14 +20,11 @@ namespace Datadog.Trace.DataStreamsMonitoring.Aggregation internal class DataStreamsMessagePackFormatter { private readonly byte[] _environmentBytes = StringEncoding.UTF8.GetBytes("Env"); - private readonly byte[] _environmentValueBytes; private readonly byte[] _serviceBytes = StringEncoding.UTF8.GetBytes("Service"); private readonly long _productMask; private readonly bool _isInDefaultState; private readonly bool _writeProcessTags; - private readonly byte[] _serviceValueBytes; - // private readonly byte[] _primaryTagBytes = StringEncoding.UTF8.GetBytes("PrimaryTag"); // private readonly byte[] _primaryTagValueBytes; private readonly byte[] _statsBytes = StringEncoding.UTF8.GetBytes("Stats"); @@ -54,18 +53,37 @@ internal class DataStreamsMessagePackFormatter private readonly byte[] _processTagsBytes = StringEncoding.UTF8.GetBytes("ProcessTags"); private readonly byte[] _isInDefaultStateBytes = StringEncoding.UTF8.GetBytes("IsInDefaultState"); - public DataStreamsMessagePackFormatter(TracerSettings tracerSettings, ProfilerSettings profilerSettings, string defaultServiceName) + private byte[] _environmentValueBytes; + private byte[] _serviceValueBytes; + + public DataStreamsMessagePackFormatter(TracerSettings tracerSettings, ProfilerSettings profilerSettings) { - var env = tracerSettings.MutableSettings.Environment; // .NET tracer doesn't yet support primary tag // _primaryTagValueBytes = Array.Empty(); - _environmentValueBytes = string.IsNullOrEmpty(env) - ? [] - : StringEncoding.UTF8.GetBytes(env); - _serviceValueBytes = StringEncoding.UTF8.GetBytes(defaultServiceName); + UpdateSettings(tracerSettings.Manager.InitialMutableSettings); + // Not disposing the subscription on the basis this is never cleaned up + tracerSettings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateSettings(mutable); + } + }); + _productMask = GetProductsMask(tracerSettings, profilerSettings); _isInDefaultState = tracerSettings.IsDataStreamsMonitoringInDefaultState; _writeProcessTags = tracerSettings.PropagateProcessTags; + + [MemberNotNull(nameof(_environmentValueBytes))] + [MemberNotNull(nameof(_serviceValueBytes))] + void UpdateSettings(MutableSettings settings) + { + var env = StringUtil.IsNullOrEmpty(settings.Environment) ? [] : StringEncoding.UTF8.GetBytes(settings.Environment); + Interlocked.Exchange(ref _environmentValueBytes!, env); + + var service = StringUtil.IsNullOrEmpty(settings.DefaultServiceName) ? [] : StringEncoding.UTF8.GetBytes(settings.DefaultServiceName); + Interlocked.Exchange(ref _serviceValueBytes!, service); + } } // should be the same across all languages diff --git a/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsManager.cs b/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsManager.cs index 7f20c383b41c..a275d693f097 100644 --- a/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsManager.cs +++ b/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsManager.cs @@ -27,23 +27,39 @@ internal class DataStreamsManager private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); private static readonly AsyncLocal LastConsumePathway = new(); // saves the context on consume checkpointing only private readonly ConcurrentDictionary _schemaRateLimiters = new(); - private readonly NodeHashBase _nodeHashBase; + private readonly IDisposable _updateSubscription; + private long _nodeHashBase; // note that this actually represents a `ulong` that we have done an unsafe cast for private bool _isEnabled; private bool _isInDefaultState; private IDataStreamsWriter? _writer; public DataStreamsManager( - string? env, - string defaultServiceName, + TracerSettings tracerSettings, IDataStreamsWriter? writer, - bool isInDefaultState, string? processTags) { - // We don't support primary tag in .NET yet - _nodeHashBase = HashHelper.CalculateNodeHashBase(defaultServiceName, env, primaryTag: null, processTags); + UpdateNodeHash(tracerSettings.Manager.InitialMutableSettings); _isEnabled = writer is not null; _writer = writer; - _isInDefaultState = isInDefaultState; + _isInDefaultState = tracerSettings.IsDataStreamsMonitoringInDefaultState; + _updateSubscription = tracerSettings.Manager.SubscribeToChanges(updates => + { + if (updates.UpdatedMutable is { } updated) + { + UpdateNodeHash(updated); + } + }); + + void UpdateNodeHash(MutableSettings settings) + { + // We don't yet support primary tag in .NET yet + var value = HashHelper.CalculateNodeHashBase(settings.DefaultServiceName, settings.Environment, primaryTag: null, processTags); + // Working around the fact we can't do Interlocked.Exchange with the struct + // and also that we can't do Interlocked.Exchange with a ulong in < .NET 5 + Interlocked.Exchange( + ref _nodeHashBase, + unchecked((long)value.Value)); // reinterpret as a long + } } public bool IsEnabled => Volatile.Read(ref _isEnabled); @@ -53,18 +69,18 @@ public DataStreamsManager( public static DataStreamsManager Create( TracerSettings settings, ProfilerSettings profilerSettings, - IDiscoveryService discoveryService, - string defaultServiceName) + IDiscoveryService discoveryService) { var writer = settings.IsDataStreamsMonitoringEnabled - ? DataStreamsWriter.Create(settings, profilerSettings, discoveryService, defaultServiceName) + ? DataStreamsWriter.Create(settings, profilerSettings, discoveryService) : null; - return new DataStreamsManager(settings.MutableSettings.Environment, defaultServiceName, writer, settings.IsDataStreamsMonitoringInDefaultState, settings.PropagateProcessTags ? ProcessTags.SerializedTags : null); + return new DataStreamsManager(settings, writer, settings.PropagateProcessTags ? ProcessTags.SerializedTags : null); } public async Task DisposeAsync() { + _updateSubscription.Dispose(); Volatile.Write(ref _isEnabled, false); var writer = Interlocked.Exchange(ref _writer, null); @@ -201,7 +217,9 @@ public void InjectPathwayContextAsBase64String(PathwayContext? context var edgeStartNs = previousContext == null && timeInQueueMs > 0 ? nowNs - (timeInQueueMs * 1_000_000) : nowNs; var pathwayStartNs = previousContext?.PathwayStart ?? edgeStartNs; - var nodeHash = HashHelper.CalculateNodeHash(_nodeHashBase, edgeTags); + // Don't blame me, blame the fact we can't do Volatile.Read with a ulong in .NET FX... + var nodeHashBase = new NodeHashBase(unchecked((ulong)Volatile.Read(ref _nodeHashBase))); + var nodeHash = HashHelper.CalculateNodeHash(nodeHashBase, edgeTags); var parentHash = previousContext?.Hash ?? default; var pathwayHash = HashHelper.CalculatePathwayHash(nodeHash, parentHash); diff --git a/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsWriter.cs b/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsWriter.cs index 6c5c47b154d1..a966ed87c750 100644 --- a/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsWriter.cs +++ b/tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsWriter.cs @@ -74,14 +74,13 @@ public DataStreamsWriter( public static DataStreamsWriter Create( TracerSettings settings, ProfilerSettings profilerSettings, - IDiscoveryService discoveryService, - string defaultServiceName) + IDiscoveryService discoveryService) => new( settings, new DataStreamsAggregator( - new DataStreamsMessagePackFormatter(settings, profilerSettings, defaultServiceName), + new DataStreamsMessagePackFormatter(settings, profilerSettings), bucketDurationMs: DataStreamsConstants.DefaultBucketDurationMs), - new DataStreamsApi(DataStreamsTransportStrategy.GetAgentIntakeFactory(settings.Exporter)), + new DataStreamsApi(settings.Manager, DataStreamsTransportStrategy.GetAgentIntakeFactory), bucketDurationMs: DataStreamsConstants.DefaultBucketDurationMs, discoveryService); diff --git a/tracer/src/Datadog.Trace/DataStreamsMonitoring/Transport/DataStreamsApi.cs b/tracer/src/Datadog.Trace/DataStreamsMonitoring/Transport/DataStreamsApi.cs index b00850d405ba..31814297cb0d 100644 --- a/tracer/src/Datadog.Trace/DataStreamsMonitoring/Transport/DataStreamsApi.cs +++ b/tracer/src/Datadog.Trace/DataStreamsMonitoring/Transport/DataStreamsApi.cs @@ -6,9 +6,12 @@ #nullable enable using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; +using Datadog.Trace.Configuration; using Datadog.Trace.Logging; namespace Datadog.Trace.DataStreamsMonitoring.Transport; @@ -16,22 +19,40 @@ namespace Datadog.Trace.DataStreamsMonitoring.Transport; internal class DataStreamsApi : IDataStreamsApi { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private readonly IApiRequestFactory _requestFactory; - private readonly Uri _endpoint; + private RequestDetails _config; - public DataStreamsApi(IApiRequestFactory apiRequestFactory) + public DataStreamsApi( + TracerSettings.SettingsManager settings, + Func factory) { - _requestFactory = apiRequestFactory; - _endpoint = _requestFactory.GetEndpoint(DataStreamsConstants.IntakePath); - Log.Debug("Using data streams intake endpoint {DataStreamsIntakeEndpoint}", _endpoint.ToString()); + UpdateFactory(settings.InitialExporterSettings); + settings.SubscribeToChanges(changes => + { + if (changes.UpdatedExporter is not null) + { + UpdateFactory(changes.UpdatedExporter); + } + }); + + [MemberNotNull(nameof(_config))] + void UpdateFactory(ExporterSettings exporter) + { + var requestFactory = factory(exporter); + var endpoint = requestFactory.GetEndpoint(DataStreamsConstants.IntakePath); + Interlocked.Exchange(ref _config!, new(requestFactory, endpoint)); + + Log.Debug("Using data streams intake endpoint {DataStreamsIntakeEndpoint}", endpoint); + } } public async Task SendAsync(ArraySegment bytes) { + var config = Volatile.Read(ref _config); + var requestFactory = config.RequestFactory; try { Log.Debug("Sending {Count} bytes to the data streams intake", bytes.Count); - var request = _requestFactory.Create(_endpoint); + var request = requestFactory.Create(config.Endpoint); using var response = await request.PostAsync(bytes, MimeTypes.MsgPack, "gzip").ConfigureAwait(false); if (response.StatusCode is >= 200 and < 300) @@ -41,13 +62,15 @@ public async Task SendAsync(ArraySegment bytes) } var responseContent = await response.ReadAsStringAsync().ConfigureAwait(false); - Log.Warning("Error sending data streams monitoring data to '{Endpoint}' {StatusCode} {Content}", _requestFactory.Info(_endpoint), response.StatusCode, responseContent); + Log.Warning("Error sending data streams monitoring data to '{Endpoint}' {StatusCode} {Content}", requestFactory.Info(config.Endpoint), response.StatusCode, responseContent); return false; } catch (Exception ex) { - Log.Warning(ex, "Error sending data streams monitoring data to '{Endpoint}'", _requestFactory.Info(_endpoint)); + Log.Warning(ex, "Error sending data streams monitoring data to '{Endpoint}'", requestFactory.Info(config.Endpoint)); return false; } } + + private record RequestDetails(IApiRequestFactory RequestFactory, Uri Endpoint); } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 7bab6e42eb59..e9e6861d406e 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -176,7 +176,7 @@ internal TracerManager CreateTracerManager( telemetry.RecordProfilerSettings(profiler); telemetry.ProductChanged(TelemetryProductType.Profiler, enabled: profiler.Status.IsProfilerReady, error: null); - dataStreamsManager ??= DataStreamsManager.Create(settings, profiler.Settings, discoveryService, defaultServiceName); + dataStreamsManager ??= DataStreamsManager.Create(settings, profiler.Settings, discoveryService); if (ShouldEnableRemoteConfiguration(settings)) { diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsAggregatorTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsAggregatorTests.cs index 6dcdbb4b6d39..d3fef2c365cc 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsAggregatorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsAggregatorTests.cs @@ -130,7 +130,7 @@ public async Task Aggregator_FlushesStats() private static DataStreamsAggregator CreateAggregatorWithData(Tracer tracer, long t1, long t2) { var aggregator = new DataStreamsAggregator( - new DataStreamsMessagePackFormatter(tracer.Settings, new ProfilerSettings(ProfilerState.Disabled), "service"), + new DataStreamsMessagePackFormatter(tracer.Settings, new ProfilerSettings(ProfilerState.Disabled)), BucketDurationMs); aggregator.Add( diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsApiTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsApiTests.cs index 3ee048c45573..bd214d26d23d 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsApiTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsApiTests.cs @@ -5,6 +5,7 @@ using System; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.DataStreamsMonitoring.Transport; using Datadog.Trace.TestHelpers.TransportHelpers; using FluentAssertions; @@ -18,7 +19,7 @@ public class DataStreamsApiTests public async Task SendAsync_When200_ReturnsTrue() { var factory = new TestRequestFactory(x => new TestApiRequest(x)); - var api = new DataStreamsApi(factory); + var api = new DataStreamsApi(new TracerSettings().Manager, _ => factory); var result = await api.SendAsync(new ArraySegment(new byte[64])); factory.RequestsSent.Should().HaveCount(1); @@ -33,7 +34,7 @@ public async Task SendAsync_When200_ReturnsTrue() public async Task SendAsync_WhenError_ReturnsFalse_AndDoesntRetry(int statusCode) { var factory = new TestRequestFactory(x => new TestApiRequest(x, statusCode)); - var api = new DataStreamsApi(factory); + var api = new DataStreamsApi(new TracerSettings().Manager, _ => factory); var result = await api.SendAsync(new ArraySegment(new byte[64])); factory.RequestsSent.Should().HaveCount(1); diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsManagerTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsManagerTests.cs index 56690e2c6baa..9b05863662fc 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsManagerTests.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.DataStreamsMonitoring; using Datadog.Trace.DataStreamsMonitoring.Aggregation; using Datadog.Trace.DataStreamsMonitoring.Hashes; @@ -312,12 +313,14 @@ public async Task WhenDisposedTwice_DisposesWriterOnce() private static DataStreamsManager GetDataStreamManager(bool enabled, out DataStreamsWriterMock writer) { writer = enabled ? new DataStreamsWriterMock() : null; - return new DataStreamsManager( - env: "foo", - defaultServiceName: "bar", - writer, - isInDefaultState: false, - processTags: null); + var settings = TracerSettings.Create( + new() + { + { ConfigurationKeys.Environment, "foo" }, + { ConfigurationKeys.ServiceName, "bar" }, + { ConfigurationKeys.DataStreamsMonitoring.Enabled, enabled.ToString() }, + }); + return new DataStreamsManager(settings, writer, processTags: null); } internal class DataStreamsWriterMock : IDataStreamsWriter diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs index 07c8fb5ee800..76f9a5c016eb 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMessagePackFormatterTests.cs @@ -29,8 +29,8 @@ public void CanRoundTripMessagePackFormat() var service = "service=name"; var bucketDuration = 10_000_000_000; var edgeTags = new[] { "edge-1" }; - var settings = TracerSettings.Create(new() { { ConfigurationKeys.Environment, "my-env" } }); - var formatter = new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled), service); + var settings = TracerSettings.Create(new() { { ConfigurationKeys.Environment, "my-env" }, { ConfigurationKeys.ServiceName, service } }); + var formatter = new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled)); var timeNs = DateTimeOffset.UtcNow.ToUnixTimeNanoseconds(); @@ -109,7 +109,7 @@ public void CanRoundTripMessagePackFormat() var expected = new MockDataStreamsPayload { - Env = settings.MutableSettings.Environment, + Env = settings.Manager.InitialMutableSettings.Environment, Service = service, Lang = "dotnet", TracerVersion = TracerConstants.AssemblyVersion, diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMonitoringTransportTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMonitoringTransportTests.cs index 15dd023c8683..bfea29f05633 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMonitoringTransportTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsMonitoringTransportTests.cs @@ -57,14 +57,13 @@ public async Task TransportsWorkCorrectly(Enum transport) // That ensures we only get a single payload var bucketDurationMs = (int)TimeSpan.FromMinutes(60).TotalMilliseconds; var tracerSettings = GetSettings(agent); - var api = new DataStreamsApi( - DataStreamsTransportStrategy.GetAgentIntakeFactory(tracerSettings.Exporter)); + var api = new DataStreamsApi(tracerSettings.Manager, DataStreamsTransportStrategy.GetAgentIntakeFactory); var discovery = new DiscoveryServiceMock(); var writer = new DataStreamsWriter( tracerSettings, new DataStreamsAggregator( - new DataStreamsMessagePackFormatter(tracerSettings, new ProfilerSettings(ProfilerState.Disabled), "service"), + new DataStreamsMessagePackFormatter(tracerSettings, new ProfilerSettings(ProfilerState.Disabled)), bucketDurationMs), api, bucketDurationMs: bucketDurationMs, @@ -130,10 +129,10 @@ private MockTracerAgent Create(TracesTransportType transportType) private TracerSettings GetSettings(MockTracerAgent agent) => agent switch { - MockTracerAgent.TcpUdpAgent x => TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, $"http://localhost:{x.Port}" }, { ConfigurationKeys.Environment, "env" } }), - MockTracerAgent.NamedPipeAgent x => TracerSettings.Create(new() { { ConfigurationKeys.TracesPipeName, x.TracesWindowsPipeName }, { ConfigurationKeys.Environment, "env" } }), + MockTracerAgent.TcpUdpAgent x => TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, $"http://localhost:{x.Port}" }, { ConfigurationKeys.Environment, "env" }, { ConfigurationKeys.ServiceName, "service" } }), + MockTracerAgent.NamedPipeAgent x => TracerSettings.Create(new() { { ConfigurationKeys.TracesPipeName, x.TracesWindowsPipeName }, { ConfigurationKeys.Environment, "env" }, { ConfigurationKeys.ServiceName, "service" } }), #if NETCOREAPP3_1_OR_GREATER - MockTracerAgent.UdsAgent x => TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, ExporterSettings.UnixDomainSocketPrefix + x.TracesUdsPath }, { ConfigurationKeys.Environment, "env" } }), + MockTracerAgent.UdsAgent x => TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, ExporterSettings.UnixDomainSocketPrefix + x.TracesUdsPath }, { ConfigurationKeys.Environment, "env" }, { ConfigurationKeys.ServiceName, "service" } }), #endif _ => throw new InvalidOperationException("Unknown agent type " + agent.GetType()), }; diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsWriterTests.cs index e0a97214274c..d17450feb4ca 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/DataStreamsWriterTests.cs @@ -342,11 +342,15 @@ private static DataStreamsWriter CreateWriter( int bucketDurationMs = BucketDurationMs) { discoveryService = new DiscoveryServiceMock(); - var settings = TracerSettings.Create(new() { { ConfigurationKeys.Environment, Environment } }); + var settings = TracerSettings.Create(new() + { + { ConfigurationKeys.Environment, Environment }, + { ConfigurationKeys.ServiceName, Service }, + }); return new DataStreamsWriter( settings, new DataStreamsAggregator( - new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled), Service), + new DataStreamsMessagePackFormatter(settings, new ProfilerSettings(ProfilerState.Disabled)), bucketDurationMs), stubApi, bucketDurationMs: bucketDurationMs, diff --git a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/SpanContextDataStreamsManagerTests.cs b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/SpanContextDataStreamsManagerTests.cs index 5adb68a7861b..150280b0b687 100644 --- a/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/SpanContextDataStreamsManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DataStreamsMonitoring/SpanContextDataStreamsManagerTests.cs @@ -3,6 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +using Datadog.Trace.Configuration; using Datadog.Trace.DataStreamsMonitoring; using Datadog.Trace.DataStreamsMonitoring.Hashes; using Datadog.Trace.TestHelpers.TransportHelpers; @@ -70,12 +71,13 @@ public void SetCheckpoint_BatchConsumeDoesNotCreateChain() private static DataStreamsManager GetEnabledDataStreamManager() { - var dsm = new DataStreamsManager( - env: "env", - defaultServiceName: "service", - new Mock().Object, - isInDefaultState: false, - processTags: null); - return dsm; + var settings = TracerSettings.Create( + new() + { + { ConfigurationKeys.Environment, "env" }, + { ConfigurationKeys.ServiceName, "service" }, + { ConfigurationKeys.DataStreamsMonitoring.Enabled, "1" }, + }); + return new DataStreamsManager(settings, new Mock().Object, processTags: null); } } diff --git a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs index a0f530a11e2c..b5eab9e53537 100644 --- a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs @@ -138,7 +138,7 @@ private static TracerManager CreateTracerManager(TracerSettings settings) BuildLogSubmissionManager(), Mock.Of(), Mock.Of(), - new DataStreamsManager("env", "service", Mock.Of(), isInDefaultState: false, processTags: null), + new DataStreamsManager(settings, Mock.Of(), processTags: null), remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, From 402bad6a1fa7a2938cd6abaa86deed0febbcee7d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 Oct 2025 17:47:50 +0100 Subject: [PATCH 04/29] Fix manual instrumentation --- .../PopulateDictionaryIntegration.cs | 43 ++++--- .../Tracer/CtorIntegration.cs | 36 +++--- .../SettingsInstrumentationTests.cs | 114 +++++++++--------- 3 files changed, 100 insertions(+), 93 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Configuration/TracerSettings/PopulateDictionaryIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Configuration/TracerSettings/PopulateDictionaryIntegration.cs index bee8997e603f..cc38f3704ef9 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Configuration/TracerSettings/PopulateDictionaryIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Configuration/TracerSettings/PopulateDictionaryIntegration.cs @@ -33,8 +33,11 @@ public class PopulateDictionaryIntegration { internal static CallTargetState OnMethodBegin(Dictionary values, bool useDefaultSources) { + // This creates a "one off", "throw away" instance of the tracer settings, which ignores previous configuration etc + // However, we _can_ reuse the existing "global" tracer settings if they use default sources, and just use the "initial" + // settings for the "mutable" settings var settings = useDefaultSources - ? Trace.Configuration.TracerSettings.FromDefaultSourcesInternal() + ? Datadog.Trace.Tracer.Instance.Settings : new Trace.Configuration.TracerSettings(null, new ConfigurationTelemetry(), new OverrideErrorLog()); PopulateSettings(values, settings); @@ -46,31 +49,33 @@ internal static CallTargetState OnMethodBegin(Dictionary values, Trace.Configuration.TracerSettings settings) { // record all the settings in the dictionary - values[TracerSettingKeyConstants.AgentUriKey] = settings.Exporter.AgentUri; + var mutableSettings = settings.Manager.InitialMutableSettings; + var exporterSettings = settings.Manager.InitialExporterSettings; + values[TracerSettingKeyConstants.AgentUriKey] = exporterSettings.AgentUri; #pragma warning disable CS0618 // Type or member is obsolete - values[TracerSettingKeyConstants.AnalyticsEnabledKey] = settings.MutableSettings.AnalyticsEnabled; + values[TracerSettingKeyConstants.AnalyticsEnabledKey] = mutableSettings.AnalyticsEnabled; #pragma warning restore CS0618 // Type or member is obsolete - values[TracerSettingKeyConstants.CustomSamplingRules] = settings.MutableSettings.CustomSamplingRules; + values[TracerSettingKeyConstants.CustomSamplingRules] = mutableSettings.CustomSamplingRules; values[TracerSettingKeyConstants.DiagnosticSourceEnabledKey] = GlobalSettings.Instance.DiagnosticSourceEnabled; - values[TracerSettingKeyConstants.DisabledIntegrationNamesKey] = settings.MutableSettings.DisabledIntegrationNames; - values[TracerSettingKeyConstants.EnvironmentKey] = settings.MutableSettings.Environment; - values[TracerSettingKeyConstants.GlobalSamplingRateKey] = settings.MutableSettings.GlobalSamplingRate; - values[TracerSettingKeyConstants.GrpcTags] = settings.MutableSettings.GrpcTags; - values[TracerSettingKeyConstants.HeaderTags] = settings.MutableSettings.HeaderTags; - values[TracerSettingKeyConstants.KafkaCreateConsumerScopeEnabledKey] = settings.MutableSettings.KafkaCreateConsumerScopeEnabled; + values[TracerSettingKeyConstants.DisabledIntegrationNamesKey] = mutableSettings.DisabledIntegrationNames; + values[TracerSettingKeyConstants.EnvironmentKey] = mutableSettings.Environment; + values[TracerSettingKeyConstants.GlobalSamplingRateKey] = mutableSettings.GlobalSamplingRate; + values[TracerSettingKeyConstants.GrpcTags] = mutableSettings.GrpcTags; + values[TracerSettingKeyConstants.HeaderTags] = mutableSettings.HeaderTags; + values[TracerSettingKeyConstants.KafkaCreateConsumerScopeEnabledKey] = mutableSettings.KafkaCreateConsumerScopeEnabled; #pragma warning disable DD0002 // This API is only for public usage and should not be called internally (there's no internal version currently) - values[TracerSettingKeyConstants.LogsInjectionEnabledKey] = settings.MutableSettings.LogsInjectionEnabled; + values[TracerSettingKeyConstants.LogsInjectionEnabledKey] = mutableSettings.LogsInjectionEnabled; #pragma warning restore DD0002 - values[TracerSettingKeyConstants.MaxTracesSubmittedPerSecondKey] = settings.MutableSettings.MaxTracesSubmittedPerSecond; - values[TracerSettingKeyConstants.ServiceNameKey] = settings.MutableSettings.ServiceName; - values[TracerSettingKeyConstants.ServiceVersionKey] = settings.MutableSettings.ServiceVersion; - values[TracerSettingKeyConstants.StartupDiagnosticLogEnabledKey] = settings.MutableSettings.StartupDiagnosticLogEnabled; + values[TracerSettingKeyConstants.MaxTracesSubmittedPerSecondKey] = mutableSettings.MaxTracesSubmittedPerSecond; + values[TracerSettingKeyConstants.ServiceNameKey] = mutableSettings.ServiceName; + values[TracerSettingKeyConstants.ServiceVersionKey] = mutableSettings.ServiceVersion; + values[TracerSettingKeyConstants.StartupDiagnosticLogEnabledKey] = mutableSettings.StartupDiagnosticLogEnabled; values[TracerSettingKeyConstants.StatsComputationEnabledKey] = settings.StatsComputationEnabled; - values[TracerSettingKeyConstants.TraceEnabledKey] = settings.MutableSettings.TraceEnabled; - values[TracerSettingKeyConstants.TracerMetricsEnabledKey] = settings.MutableSettings.TracerMetricsEnabled; + values[TracerSettingKeyConstants.TraceEnabledKey] = mutableSettings.TraceEnabled; + values[TracerSettingKeyConstants.TracerMetricsEnabledKey] = mutableSettings.TracerMetricsEnabled; - values[TracerSettingKeyConstants.GlobalTagsKey] = settings.MutableSettings.GlobalTags; - values[TracerSettingKeyConstants.IntegrationSettingsKey] = BuildIntegrationSettings(settings.MutableSettings.Integrations); + values[TracerSettingKeyConstants.GlobalTagsKey] = mutableSettings.GlobalTags; + values[TracerSettingKeyConstants.IntegrationSettingsKey] = BuildIntegrationSettings(mutableSettings.Integrations); } private static Dictionary? BuildIntegrationSettings(IntegrationSettingsCollection settings) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs index 792d9ec4c817..b6d61da88e43 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs @@ -45,33 +45,35 @@ internal static CallTargetState OnMethodBegin(TTarget instance, object? internal static void PopulateSettings(Dictionary values, TracerSettings settings) { // record all the settings in the dictionary + var mutableSettings = settings.Manager.InitialMutableSettings; + var exporterSettings = settings.Manager.InitialExporterSettings; // This key is used to detect if the settings have been populated _at all_, so should always be sent - values[TracerSettingKeyConstants.AgentUriKey] = settings.Exporter.AgentUri; + values[TracerSettingKeyConstants.AgentUriKey] = exporterSettings.AgentUri; #pragma warning disable CS0618 // Type or member is obsolete - values[TracerSettingKeyConstants.AnalyticsEnabledKey] = settings.MutableSettings.AnalyticsEnabled; + values[TracerSettingKeyConstants.AnalyticsEnabledKey] = mutableSettings.AnalyticsEnabled; #pragma warning restore CS0618 // Type or member is obsolete - values[TracerSettingKeyConstants.CustomSamplingRules] = settings.MutableSettings.CustomSamplingRules; + values[TracerSettingKeyConstants.CustomSamplingRules] = mutableSettings.CustomSamplingRules; values[TracerSettingKeyConstants.DiagnosticSourceEnabledKey] = GlobalSettings.Instance.DiagnosticSourceEnabled; - values[TracerSettingKeyConstants.EnvironmentKey] = settings.MutableSettings.Environment; - values[TracerSettingKeyConstants.GlobalSamplingRateKey] = settings.MutableSettings.GlobalSamplingRate; - values[TracerSettingKeyConstants.KafkaCreateConsumerScopeEnabledKey] = settings.MutableSettings.KafkaCreateConsumerScopeEnabled; + values[TracerSettingKeyConstants.EnvironmentKey] = mutableSettings.Environment; + values[TracerSettingKeyConstants.GlobalSamplingRateKey] = mutableSettings.GlobalSamplingRate; + values[TracerSettingKeyConstants.KafkaCreateConsumerScopeEnabledKey] = mutableSettings.KafkaCreateConsumerScopeEnabled; #pragma warning disable DD0002 // This API is only for public usage and should not be called internally (there's no internal version currently) - values[TracerSettingKeyConstants.LogsInjectionEnabledKey] = settings.MutableSettings.LogsInjectionEnabled; + values[TracerSettingKeyConstants.LogsInjectionEnabledKey] = mutableSettings.LogsInjectionEnabled; #pragma warning restore DD0002 - values[TracerSettingKeyConstants.MaxTracesSubmittedPerSecondKey] = settings.MutableSettings.MaxTracesSubmittedPerSecond; - values[TracerSettingKeyConstants.ServiceNameKey] = settings.MutableSettings.ServiceName; - values[TracerSettingKeyConstants.ServiceVersionKey] = settings.MutableSettings.ServiceVersion; - values[TracerSettingKeyConstants.StartupDiagnosticLogEnabledKey] = settings.MutableSettings.StartupDiagnosticLogEnabled; + values[TracerSettingKeyConstants.MaxTracesSubmittedPerSecondKey] = mutableSettings.MaxTracesSubmittedPerSecond; + values[TracerSettingKeyConstants.ServiceNameKey] = mutableSettings.ServiceName; + values[TracerSettingKeyConstants.ServiceVersionKey] = mutableSettings.ServiceVersion; + values[TracerSettingKeyConstants.StartupDiagnosticLogEnabledKey] = mutableSettings.StartupDiagnosticLogEnabled; values[TracerSettingKeyConstants.StatsComputationEnabledKey] = settings.StatsComputationEnabled; - values[TracerSettingKeyConstants.TraceEnabledKey] = settings.MutableSettings.TraceEnabled; - values[TracerSettingKeyConstants.TracerMetricsEnabledKey] = settings.MutableSettings.TracerMetricsEnabled; + values[TracerSettingKeyConstants.TraceEnabledKey] = mutableSettings.TraceEnabled; + values[TracerSettingKeyConstants.TracerMetricsEnabledKey] = mutableSettings.TracerMetricsEnabled; // probably don't _have_ to copy these dictionaries, but playing it safe - values[TracerSettingKeyConstants.GlobalTagsKey] = new ConcurrentDictionary(settings.MutableSettings.GlobalTags); - values[TracerSettingKeyConstants.GrpcTags] = new ConcurrentDictionary(settings.MutableSettings.GrpcTags); - values[TracerSettingKeyConstants.HeaderTags] = new ConcurrentDictionary(settings.MutableSettings.HeaderTags); + values[TracerSettingKeyConstants.GlobalTagsKey] = new ConcurrentDictionary(mutableSettings.GlobalTags); + values[TracerSettingKeyConstants.GrpcTags] = new ConcurrentDictionary(mutableSettings.GrpcTags); + values[TracerSettingKeyConstants.HeaderTags] = new ConcurrentDictionary(mutableSettings.HeaderTags); - values[TracerSettingKeyConstants.IntegrationSettingsKey] = BuildIntegrationSettings(settings.MutableSettings.Integrations); + values[TracerSettingKeyConstants.IntegrationSettingsKey] = BuildIntegrationSettings(mutableSettings.Integrations); } private static Dictionary? BuildIntegrationSettings(IntegrationSettingsCollection settings) diff --git a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs index b51d0bf6333d..997babf0f896 100644 --- a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs @@ -207,15 +207,15 @@ public void ManualToAutomatic_CustomSettingsAreTransferredCorrectly(bool useLega var automatic = new TracerSettings(configSource); AssertEquivalent(manual, automatic); - automatic.MutableSettings.ServiceNameMappings.Should().Equal(mappings); - automatic.MutableSettings.HttpClientErrorStatusCodes.Should().Equal(MutableSettings.ParseHttpCodesToArray(string.Join(",", clientErrors))); - automatic.MutableSettings.HttpServerErrorStatusCodes.Should().Equal(MutableSettings.ParseHttpCodesToArray(string.Join(",", serverErrors))); - - automatic.MutableSettings.Integrations[IntegrationId.OpenTelemetry].Enabled.Should().BeFalse(); - automatic.MutableSettings.Integrations[IntegrationId.Kafka].Enabled.Should().BeFalse(); - automatic.MutableSettings.Integrations[IntegrationId.Aerospike].Enabled.Should().BeFalse(); - automatic.MutableSettings.Integrations[IntegrationId.Grpc].AnalyticsEnabled.Should().BeTrue(); - automatic.MutableSettings.Integrations[IntegrationId.Couchbase].AnalyticsSampleRate.Should().Be(0.5); + automatic.Manager.InitialMutableSettings.ServiceNameMappings.Should().Equal(mappings); + automatic.Manager.InitialMutableSettings.HttpClientErrorStatusCodes.Should().Equal(MutableSettings.ParseHttpCodesToArray(string.Join(",", clientErrors))); + automatic.Manager.InitialMutableSettings.HttpServerErrorStatusCodes.Should().Equal(MutableSettings.ParseHttpCodesToArray(string.Join(",", serverErrors))); + + automatic.Manager.InitialMutableSettings.Integrations[IntegrationId.OpenTelemetry].Enabled.Should().BeFalse(); + automatic.Manager.InitialMutableSettings.Integrations[IntegrationId.Kafka].Enabled.Should().BeFalse(); + automatic.Manager.InitialMutableSettings.Integrations[IntegrationId.Aerospike].Enabled.Should().BeFalse(); + automatic.Manager.InitialMutableSettings.Integrations[IntegrationId.Grpc].AnalyticsEnabled.Should().BeTrue(); + automatic.Manager.InitialMutableSettings.Integrations[IntegrationId.Couchbase].AnalyticsSampleRate.Should().Be(0.5); } [Fact] @@ -230,26 +230,26 @@ public void AutomaticToManual_ImmutableSettingsAreTransferredCorrectly() manual.AgentUri.Should().Be(automatic.Exporter.AgentUri); manual.Exporter.AgentUri.Should().Be(automatic.Exporter.AgentUri); - manual.AnalyticsEnabled.Should().Be(automatic.MutableSettings.AnalyticsEnabled); - manual.CustomSamplingRules.Should().Be(automatic.MutableSettings.CustomSamplingRules); - manual.Environment.Should().Be(automatic.MutableSettings.Environment); - manual.GlobalSamplingRate.Should().Be(automatic.MutableSettings.GlobalSamplingRate); - manual.GlobalTags.Should().BeEquivalentTo(automatic.MutableSettings.GlobalTags); - manual.HeaderTags.Should().BeEquivalentTo(automatic.MutableSettings.HeaderTags); + manual.AnalyticsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.AnalyticsEnabled); + manual.CustomSamplingRules.Should().Be(automatic.Manager.InitialMutableSettings.CustomSamplingRules); + manual.Environment.Should().Be(automatic.Manager.InitialMutableSettings.Environment); + manual.GlobalSamplingRate.Should().Be(automatic.Manager.InitialMutableSettings.GlobalSamplingRate); + manual.GlobalTags.Should().BeEquivalentTo(automatic.Manager.InitialMutableSettings.GlobalTags); + manual.HeaderTags.Should().BeEquivalentTo(automatic.Manager.InitialMutableSettings.HeaderTags); // force fluent assertions to just compare the properties, not use the `Equals` implementation manual.Integrations.Settings.Should() .BeEquivalentTo( - automatic.MutableSettings.Integrations.Settings.ToDictionary(x => x.IntegrationName, x => x), + automatic.Manager.InitialMutableSettings.Integrations.Settings.ToDictionary(x => x.IntegrationName, x => x), options => options.ComparingByMembers(typeof(IntegrationSettings))); - manual.KafkaCreateConsumerScopeEnabled.Should().Be(automatic.MutableSettings.KafkaCreateConsumerScopeEnabled); - manual.LogsInjectionEnabled.Should().Be(automatic.MutableSettings.LogsInjectionEnabled); - manual.MaxTracesSubmittedPerSecond.Should().Be(automatic.MutableSettings.MaxTracesSubmittedPerSecond); - manual.ServiceName.Should().Be(automatic.MutableSettings.ServiceName); - manual.ServiceVersion.Should().Be(automatic.MutableSettings.ServiceVersion); - manual.StartupDiagnosticLogEnabled.Should().Be(automatic.MutableSettings.StartupDiagnosticLogEnabled); + manual.KafkaCreateConsumerScopeEnabled.Should().Be(automatic.Manager.InitialMutableSettings.KafkaCreateConsumerScopeEnabled); + manual.LogsInjectionEnabled.Should().Be(automatic.Manager.InitialMutableSettings.LogsInjectionEnabled); + manual.MaxTracesSubmittedPerSecond.Should().Be(automatic.Manager.InitialMutableSettings.MaxTracesSubmittedPerSecond); + manual.ServiceName.Should().Be(automatic.Manager.InitialMutableSettings.ServiceName); + manual.ServiceVersion.Should().Be(automatic.Manager.InitialMutableSettings.ServiceVersion); + manual.StartupDiagnosticLogEnabled.Should().Be(automatic.Manager.InitialMutableSettings.StartupDiagnosticLogEnabled); manual.StatsComputationEnabled.Should().Be(automatic.StatsComputationEnabled); - manual.TraceEnabled.Should().Be(automatic.MutableSettings.TraceEnabled); - manual.TracerMetricsEnabled.Should().Be(automatic.MutableSettings.TracerMetricsEnabled); + manual.TraceEnabled.Should().Be(automatic.Manager.InitialMutableSettings.TraceEnabled); + manual.TracerMetricsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.TracerMetricsEnabled); manual.Integrations[nameof(IntegrationId.OpenTelemetry)].Enabled.Should().BeFalse(); manual.Integrations[nameof(IntegrationId.Kafka)].Enabled.Should().BeFalse(); @@ -264,26 +264,26 @@ private static void AssertEquivalent(ManualSettings manual, TracerSettings autom GetTransformedAgentUri(manual.AgentUri).Should().Be(automatic.Exporter.AgentUri); GetTransformedAgentUri(manual.Exporter.AgentUri).Should().Be(automatic.Exporter.AgentUri); - manual.AnalyticsEnabled.Should().Be(automatic.MutableSettings.AnalyticsEnabled); - manual.CustomSamplingRules.Should().Be(automatic.MutableSettings.CustomSamplingRules); + manual.AnalyticsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.AnalyticsEnabled); + manual.CustomSamplingRules.Should().Be(automatic.Manager.InitialMutableSettings.CustomSamplingRules); manual.DiagnosticSourceEnabled.Should().Be(GlobalSettings.Instance.DiagnosticSourceEnabled); - manual.Environment.Should().Be(automatic.MutableSettings.Environment); - manual.GlobalSamplingRate.Should().Be(automatic.MutableSettings.GlobalSamplingRate); - manual.GlobalTags.Should().BeEquivalentTo(automatic.MutableSettings.GlobalTags); + manual.Environment.Should().Be(automatic.Manager.InitialMutableSettings.Environment); + manual.GlobalSamplingRate.Should().Be(automatic.Manager.InitialMutableSettings.GlobalSamplingRate); + manual.GlobalTags.Should().BeEquivalentTo(automatic.Manager.InitialMutableSettings.GlobalTags); // These _aren't_ necessarily equivalent because of DisabledIntegrations. // If you add an integration to the manual.DisabledIntegrations, then the automatic.Integrations // will include it, but the original manual.Integrations _won't_. All a bit of a mess, but // essentially due to the legacy design of the TracerSettings object // manual.Integrations.Settings.Should().BeEquivalentTo(automatic.Integrations.Settings.ToDictionary(x => x.IntegrationName, x => x)); - manual.KafkaCreateConsumerScopeEnabled.Should().Be(automatic.MutableSettings.KafkaCreateConsumerScopeEnabled); - manual.LogsInjectionEnabled.Should().Be(automatic.MutableSettings.LogsInjectionEnabled); - manual.MaxTracesSubmittedPerSecond.Should().Be(automatic.MutableSettings.MaxTracesSubmittedPerSecond); - manual.ServiceName.Should().Be(automatic.MutableSettings.ServiceName); - manual.ServiceVersion.Should().Be(automatic.MutableSettings.ServiceVersion); - manual.StartupDiagnosticLogEnabled.Should().Be(automatic.MutableSettings.StartupDiagnosticLogEnabled); + manual.KafkaCreateConsumerScopeEnabled.Should().Be(automatic.Manager.InitialMutableSettings.KafkaCreateConsumerScopeEnabled); + manual.LogsInjectionEnabled.Should().Be(automatic.Manager.InitialMutableSettings.LogsInjectionEnabled); + manual.MaxTracesSubmittedPerSecond.Should().Be(automatic.Manager.InitialMutableSettings.MaxTracesSubmittedPerSecond); + manual.ServiceName.Should().Be(automatic.Manager.InitialMutableSettings.ServiceName); + manual.ServiceVersion.Should().Be(automatic.Manager.InitialMutableSettings.ServiceVersion); + manual.StartupDiagnosticLogEnabled.Should().Be(automatic.Manager.InitialMutableSettings.StartupDiagnosticLogEnabled); manual.StatsComputationEnabled.Should().Be(automatic.StatsComputationEnabled); - manual.TraceEnabled.Should().Be(automatic.MutableSettings.TraceEnabled); - manual.TracerMetricsEnabled.Should().Be(automatic.MutableSettings.TracerMetricsEnabled); + manual.TraceEnabled.Should().Be(automatic.Manager.InitialMutableSettings.TraceEnabled); + manual.TracerMetricsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.TracerMetricsEnabled); Uri GetTransformedAgentUri(Uri agentUri) => ExporterSettings.Create(new() @@ -347,29 +347,29 @@ private static TracerSettings GetAndAssertAutomaticTracerSettings() }); // verify that all the settings are as expected - automatic.MutableSettings.AnalyticsEnabled.Should().Be(true); - automatic.MutableSettings.CustomSamplingRules.Should().Be("""[{"sample_rate":0.3, "service":"shopping-cart.*"}]"""); + automatic.Manager.InitialMutableSettings.AnalyticsEnabled.Should().Be(true); + automatic.Manager.InitialMutableSettings.CustomSamplingRules.Should().Be("""[{"sample_rate":0.3, "service":"shopping-cart.*"}]"""); GlobalSettings.Instance.DiagnosticSourceEnabled.Should().Be(true); - automatic.MutableSettings.DisabledIntegrationNames.Should().BeEquivalentTo(["something", "OpenTelemetry", "Kafka"]); - automatic.MutableSettings.Environment.Should().Be("my-test-env"); - automatic.MutableSettings.GlobalSamplingRate.Should().Be(0.5); - automatic.MutableSettings.GlobalTags.Should().BeEquivalentTo(new Dictionary { { "tag1", "value" } }); - automatic.MutableSettings.GrpcTags.Should().BeEquivalentTo(new Dictionary { { "grpc1", "grpc-value" } }); - automatic.MutableSettings.HeaderTags.Should().BeEquivalentTo(new Dictionary { { "header1", "header-value" } }); - automatic.MutableSettings.KafkaCreateConsumerScopeEnabled.Should().Be(false); - automatic.MutableSettings.LogsInjectionEnabled.Should().Be(true); - automatic.MutableSettings.MaxTracesSubmittedPerSecond.Should().Be(50); - automatic.MutableSettings.ServiceName.Should().Be("my-test-service"); - automatic.MutableSettings.ServiceVersion.Should().Be("1.2.3"); - automatic.MutableSettings.StartupDiagnosticLogEnabled.Should().Be(false); + automatic.Manager.InitialMutableSettings.DisabledIntegrationNames.Should().BeEquivalentTo(["something", "OpenTelemetry", "Kafka"]); + automatic.Manager.InitialMutableSettings.Environment.Should().Be("my-test-env"); + automatic.Manager.InitialMutableSettings.GlobalSamplingRate.Should().Be(0.5); + automatic.Manager.InitialMutableSettings.GlobalTags.Should().BeEquivalentTo(new Dictionary { { "tag1", "value" } }); + automatic.Manager.InitialMutableSettings.GrpcTags.Should().BeEquivalentTo(new Dictionary { { "grpc1", "grpc-value" } }); + automatic.Manager.InitialMutableSettings.HeaderTags.Should().BeEquivalentTo(new Dictionary { { "header1", "header-value" } }); + automatic.Manager.InitialMutableSettings.KafkaCreateConsumerScopeEnabled.Should().Be(false); + automatic.Manager.InitialMutableSettings.LogsInjectionEnabled.Should().Be(true); + automatic.Manager.InitialMutableSettings.MaxTracesSubmittedPerSecond.Should().Be(50); + automatic.Manager.InitialMutableSettings.ServiceName.Should().Be("my-test-service"); + automatic.Manager.InitialMutableSettings.ServiceVersion.Should().Be("1.2.3"); + automatic.Manager.InitialMutableSettings.StartupDiagnosticLogEnabled.Should().Be(false); automatic.StatsComputationEnabled.Should().Be(true); - automatic.MutableSettings.TraceEnabled.Should().Be(false); - automatic.MutableSettings.TracerMetricsEnabled.Should().Be(true); + automatic.Manager.InitialMutableSettings.TraceEnabled.Should().Be(false); + automatic.Manager.InitialMutableSettings.TracerMetricsEnabled.Should().Be(true); automatic.Exporter.AgentUri.Should().Be(new Uri("http://127.0.0.1:1234")); - automatic.MutableSettings.Integrations[nameof(IntegrationId.Aerospike)].Enabled.Should().Be(false); - automatic.MutableSettings.Integrations[nameof(IntegrationId.Grpc)].AnalyticsEnabled.Should().Be(true); - automatic.MutableSettings.Integrations[nameof(IntegrationId.Couchbase)].AnalyticsSampleRate.Should().Be(0.5); - automatic.MutableSettings.Integrations[nameof(IntegrationId.Kafka)].Enabled.Should().BeFalse(); + automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Aerospike)].Enabled.Should().Be(false); + automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Grpc)].AnalyticsEnabled.Should().Be(true); + automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Couchbase)].AnalyticsSampleRate.Should().Be(0.5); + automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Kafka)].Enabled.Should().BeFalse(); return automatic; } From fe78c6864994c81db37b092da3c7c94f9cf6a4a2 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 10:58:56 +0100 Subject: [PATCH 05/29] Update direct log submission to use settings manager Also: - slight refactor of LogFormatter to reduce some allocation - ignore "previous" when creating DirectLogSubmissionManager (seeing as that won't be a thing soon) --- .../DirectLogSubmissionManager.cs | 16 +- .../Formatting/LogFormatter.cs | 168 ++++++++++-------- .../src/Datadog.Trace/TracerManagerFactory.cs | 4 - .../ILogger/DirectSubmissionLoggerTests.cs | 2 +- .../LogSettingsHelper.cs | 28 +-- .../DirectLogSubmissionSettingsTests.cs | 4 +- .../Formatting/LogFormatterTests.cs | 6 +- .../TracerManagerFactoryTests.cs | 11 +- 8 files changed, 124 insertions(+), 115 deletions(-) diff --git a/tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionManager.cs b/tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionManager.cs index e2b439d324ef..1dde43ec9504 100644 --- a/tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionManager.cs +++ b/tracer/src/Datadog.Trace/Logging/DirectSubmission/DirectLogSubmissionManager.cs @@ -30,16 +30,12 @@ private DirectLogSubmissionManager(DirectLogSubmissionSettings settings, IDirect public LogFormatter Formatter { get; } public static DirectLogSubmissionManager Create( - DirectLogSubmissionManager? previous, TracerSettings settings, DirectLogSubmissionSettings directLogSettings, ImmutableAzureAppServiceSettings? azureAppServiceSettings, - string serviceName, - string env, - string serviceVersion, IGitMetadataTagsProvider gitMetadataTagsProvider) { - var formatter = new LogFormatter(settings, directLogSettings, azureAppServiceSettings, serviceName, env, serviceVersion, gitMetadataTagsProvider); + var formatter = new LogFormatter(settings, directLogSettings, azureAppServiceSettings, gitMetadataTagsProvider); #if NETCOREAPP3_1_OR_GREATER if (settings.OpenTelemetryLogsEnabled is true) @@ -48,14 +44,6 @@ public static DirectLogSubmissionManager Create( } #endif - if (previous is not null) - { - // Only the formatter uses settings that are configurable in code. - // If that ever changes, need to update the log-shipping integrations that - // currently cache the sink/settings instances - return new DirectLogSubmissionManager(previous.Settings, previous.Sink, formatter); - } - if (!directLogSettings.IsEnabled) { return new DirectLogSubmissionManager(directLogSettings, new NullDirectSubmissionLogSink(), formatter); @@ -76,6 +64,8 @@ public async Task DisposeAsync() { await sink.DisposeAsync().ConfigureAwait(false); } + + Formatter.Dispose(); } catch (Exception ex) { diff --git a/tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs b/tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs index eaae44b7cea9..40275a4b82da 100644 --- a/tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs +++ b/tracer/src/Datadog.Trace/Logging/DirectSubmission/Formatting/LogFormatter.cs @@ -6,9 +6,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Text; +using System.Threading; using Datadog.Trace.Ci.Tags; using Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging; using Datadog.Trace.Configuration; @@ -17,7 +19,7 @@ namespace Datadog.Trace.Logging.DirectSubmission.Formatting { - internal class LogFormatter + internal class LogFormatter : IDisposable { private const char KeyValueTagSeparator = ':'; private const char TagSeparator = ','; @@ -28,50 +30,90 @@ internal class LogFormatter private const string EnvPropertyName = "dd_env"; private const string VersionPropertyName = "dd_version"; + private readonly object _lock = new(); + private readonly IDisposable _settingSub; private readonly string? _source; - private readonly string? _service; private readonly string? _host; - private readonly string? _env; - private readonly string? _version; private readonly IGitMetadataTagsProvider _gitMetadataTagsProvider; private readonly bool _use128Bits; - private bool _gitMetadataAdded; + private string? _gitMetadataTags; private string? _ciVisibilityDdTags; + private ServiceTags _serviceTags; public LogFormatter( TracerSettings settings, DirectLogSubmissionSettings directLogSettings, ImmutableAzureAppServiceSettings? aasSettings, - string serviceName, - string env, - string version, IGitMetadataTagsProvider gitMetadataTagsProvider) { _source = string.IsNullOrEmpty(directLogSettings.Source) ? null : directLogSettings.Source; - _service = string.IsNullOrEmpty(serviceName) ? null : serviceName; _host = string.IsNullOrEmpty(directLogSettings.Host) ? null : directLogSettings.Host; - _env = string.IsNullOrEmpty(env) ? null : env; - _version = string.IsNullOrEmpty(version) ? null : version; _gitMetadataTagsProvider = gitMetadataTagsProvider; _use128Bits = settings.TraceId128BitLoggingEnabled; - var globalTags = directLogSettings.GlobalTags is { Count: > 0 } ? directLogSettings.GlobalTags : settings.MutableSettings.GlobalTags; - Tags = EnrichTagsWithAasMetadata(StringifyGlobalTags(globalTags), aasSettings); + UpdateServiceTags(settings.Manager.InitialMutableSettings); + _settingSub = settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateServiceTags(mutable); + } + }); + + [MemberNotNull(nameof(_serviceTags))] + void UpdateServiceTags(MutableSettings mutableSettings) + { + // we take a lock here to handle the case where we're running + // concurrently with EnrichTagsStringWithGitMetadata + lock (_lock) + { + var service = mutableSettings.DefaultServiceName; + var env = string.IsNullOrEmpty(mutableSettings.Environment) ? null : mutableSettings.Environment; + var version = string.IsNullOrEmpty(mutableSettings.ServiceVersion) ? null : mutableSettings.ServiceVersion; + var tagDictionary = directLogSettings.GlobalTags is { Count: > 0 } ? directLogSettings.GlobalTags : mutableSettings.GlobalTags; + var globalTags = StringifyGlobalTags(tagDictionary, aasSettings); + var gitMetadataTags = _gitMetadataTags; + _serviceTags = new ServiceTags(service, env, version, JoinTags(globalTags, gitMetadataTags)); + } + } } internal delegate LogPropertyRenderingDetails FormatDelegate(JsonTextWriter writer, in T state); - internal string? Tags { get; private set; } + // Internal for testing only + internal string? Tags => Volatile.Read(ref _serviceTags).Tags; - private static string StringifyGlobalTags(IReadOnlyDictionary globalTags) + private static string StringifyGlobalTags( + IReadOnlyDictionary globalTags, + ImmutableAzureAppServiceSettings? aasSettings) { - if (globalTags.Count == 0) + var hasResourceId = !string.IsNullOrEmpty(aasSettings?.ResourceId); + var hasSiteKind = !string.IsNullOrEmpty(aasSettings?.SiteKind); + if (globalTags.Count == 0 && !hasResourceId && !hasSiteKind) { return string.Empty; } var sb = StringBuilderCache.Acquire(); + + // AAS tags + if (hasResourceId) + { + sb.Append(Trace.Tags.AzureAppServicesResourceId) + .Append(KeyValueTagSeparator) + .Append(aasSettings?.ResourceId) + .Append(TagSeparator); + } + + if (hasSiteKind) + { + sb.Append(Trace.Tags.AzureAppServicesSiteKind) + .Append(KeyValueTagSeparator) + .Append(aasSettings?.SiteKind) + .Append(TagSeparator); + } + foreach (var tagPair in globalTags) { sb.Append(tagPair.Key) @@ -100,53 +142,19 @@ private static string RemoveScheme(string url) return url; } - private static string EnrichTagsWithAasMetadata(string globalTags, ImmutableAzureAppServiceSettings? aasSettings) + private static string? JoinTags(string? globalTags, string? gitMetadataTags) { - if (aasSettings is null) + if (StringUtil.IsNullOrEmpty(gitMetadataTags)) { return globalTags; } - var hasResourceId = !string.IsNullOrEmpty(aasSettings.ResourceId); - var hasSiteKind = !string.IsNullOrEmpty(aasSettings.SiteKind); - - if (!hasResourceId && !hasSiteKind) - { - return globalTags; - } - - var sb = StringBuilderCache.Acquire(); - - if (hasResourceId) - { - sb.Append(Trace.Tags.AzureAppServicesResourceId) - .Append(KeyValueTagSeparator) - .Append(aasSettings.ResourceId) - .Append(TagSeparator); - } - - if (hasSiteKind) - { - sb.Append(Trace.Tags.AzureAppServicesSiteKind) - .Append(KeyValueTagSeparator) - .Append(aasSettings.SiteKind) - .Append(TagSeparator); - } - - // remove final joiner - sb.Remove(sb.Length - 1, length: 1); - if (!string.IsNullOrEmpty(globalTags)) - { - sb.Append(TagSeparator) - .Append(globalTags); - } - - return StringBuilderCache.GetStringAndRelease(sb); + return StringUtil.IsNullOrEmpty(globalTags) ? gitMetadataTags : $"{globalTags}{TagSeparator}{gitMetadataTags}"; } private void EnrichTagsStringWithGitMetadata() { - if (_gitMetadataAdded) + if (_gitMetadataTags is not null) { return; } @@ -159,11 +167,19 @@ private void EnrichTagsStringWithGitMetadata() if (gitMetadata != GitMetadata.Empty) { + // we take a lock here to handle the case where we're running concurrently with a settings update var gitMetadataTags = $"{CommonTags.GitCommit}{KeyValueTagSeparator}{gitMetadata.CommitSha},{CommonTags.GitRepository}{KeyValueTagSeparator}{RemoveScheme(gitMetadata.RepositoryUrl)}"; - Tags = string.IsNullOrEmpty(Tags) ? gitMetadataTags : $"{Tags}{TagSeparator}{gitMetadataTags}"; + Volatile.Write(ref _gitMetadataTags, gitMetadataTags); + lock (_lock) + { + var currentServiceTags = _serviceTags; + _serviceTags = currentServiceTags with { Tags = JoinTags(currentServiceTags.Tags, gitMetadataTags) }; + } + } + else + { + Volatile.Write(ref _gitMetadataTags, string.Empty); // to signal that we extracted it but it was missing } - - _gitMetadataAdded = true; } internal static bool IsSourceProperty(string? propertyName) => @@ -327,22 +343,25 @@ internal void FormatLog( writer.WriteValue(_source); } - if (_service is not null && !renderingDetails.HasRenderedService) + EnrichTagsStringWithGitMetadata(); + var serviceTags = _serviceTags; + + if (serviceTags.Service is not null && !renderingDetails.HasRenderedService) { writer.WritePropertyName(ServicePropertyName, escape: false); - writer.WriteValue(_service); + writer.WriteValue(serviceTags.Service); } - if (_env is not null && !renderingDetails.HasRenderedEnv) + if (serviceTags.Env is not null && !renderingDetails.HasRenderedEnv) { writer.WritePropertyName(EnvPropertyName, escape: false); - writer.WriteValue(_env); + writer.WriteValue(serviceTags.Env); } - if (_version is not null && !renderingDetails.HasRenderedVersion) + if (serviceTags.Version is not null && !renderingDetails.HasRenderedVersion) { writer.WritePropertyName(VersionPropertyName, escape: false); - writer.WriteValue(_version); + writer.WriteValue(serviceTags.Version); } if (_host is not null && !renderingDetails.HasRenderedHost) @@ -351,12 +370,10 @@ internal void FormatLog( writer.WriteValue(_host); } - EnrichTagsStringWithGitMetadata(); - - if (!StringUtil.IsNullOrEmpty(Tags) && !renderingDetails.HasRenderedTags) + if (!StringUtil.IsNullOrEmpty(serviceTags.Tags) && !renderingDetails.HasRenderedTags) { writer.WritePropertyName(TagsPropertyName, escape: false); - writer.WriteValue(Tags); + writer.WriteValue(serviceTags.Tags); } writer.WriteEndObject(); @@ -391,21 +408,22 @@ internal void FormatCIVisibilityLog(StringBuilder sb, string source, string? log writer.WriteValue(message); EnrichTagsStringWithGitMetadata(); + var serviceTags = _serviceTags; - var env = _env ?? string.Empty; + var env = serviceTags.Env ?? string.Empty; var ddTags = _ciVisibilityDdTags; if (ddTags is null) { - ddTags = GetCIVisiblityDDTagsString(env); + ddTags = GetCIVisiblityDDTagsString(serviceTags, env); _ciVisibilityDdTags = ddTags; } - var service = _service; + var service = serviceTags.Service; if (span is not null) { if (span.GetTag(Trace.Tags.Env) is { } spanEnv && spanEnv != env) { - ddTags = GetCIVisiblityDDTagsString(spanEnv); + ddTags = GetCIVisiblityDDTagsString(serviceTags, spanEnv); } if (!string.IsNullOrEmpty(span.ServiceName)) @@ -450,19 +468,23 @@ internal void FormatCIVisibilityLog(StringBuilder sb, string source, string? log writer.WriteEndObject(); } - private string GetCIVisiblityDDTagsString(string environment) + private string GetCIVisiblityDDTagsString(ServiceTags serviceTags, string environment) { // spaces are not allowed inside ddtags environment = environment.Replace(" ", string.Empty); environment = environment.Replace(":", string.Empty); var ddtags = $"env:{environment},datadog.product:citest"; - if (Tags is { Length: > 0 } globalTags) + if (serviceTags.Tags is { Length: > 0 } globalTags) { ddtags += "," + globalTags; } return ddtags; } + + public void Dispose() => _settingSub.Dispose(); + + private record ServiceTags(string? Service, string? Env, string? Version, string? Tags); } } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index e9e6861d406e..4931ec1c824a 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -156,13 +156,9 @@ internal TracerManager CreateTracerManager( var gitMetadataTagsProvider = GetGitMetadataTagsProvider(settings, settings.Manager.InitialMutableSettings, scopeManager, telemetry); logSubmissionManager = DirectLogSubmissionManager.Create( - logSubmissionManager, settings, settings.LogSubmissionSettings, settings.AzureAppServiceMetadata, - defaultServiceName, - settings.MutableSettings.Environment, - settings.MutableSettings.ServiceVersion, gitMetadataTagsProvider); telemetry.RecordTracerSettings(settings, defaultServiceName); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.Managed.Tests/AutoInstrumentation/Logging/ILogger/DirectSubmissionLoggerTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.Managed.Tests/AutoInstrumentation/Logging/ILogger/DirectSubmissionLoggerTests.cs index 8a56af8c3d3f..20a703984e10 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.Managed.Tests/AutoInstrumentation/Logging/ILogger/DirectSubmissionLoggerTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.Managed.Tests/AutoInstrumentation/Logging/ILogger/DirectSubmissionLoggerTests.cs @@ -100,7 +100,7 @@ private static DirectSubmissionLogger GetLogger(TestSink sink) name: "TestLogger", logEventCreator: new DatadogLogEventCreator(LogSettingsHelper.GetFormatter(), scopeProvider: new NullScopeProvider()), sink: sink, - minimumLogLevel: settings.MinimumLevel); + minimumLogLevel: settings.LogSubmissionSettings.MinimumLevel); } internal class TestSink : IDirectSubmissionLogSink diff --git a/tracer/test/Datadog.Trace.TestHelpers/LogSettingsHelper.cs b/tracer/test/Datadog.Trace.TestHelpers/LogSettingsHelper.cs index 6999d02ff2aa..b1648413199a 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/LogSettingsHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/LogSettingsHelper.cs @@ -13,20 +13,23 @@ namespace Datadog.Trace.TestHelpers { internal class LogSettingsHelper { - public static LogFormatter GetFormatter() => new( - new TracerSettings(null, Configuration.Telemetry.NullConfigurationTelemetry.Instance, new OverrideErrorLog()), - GetValidSettings(), - aasSettings: null, - serviceName: "MyTestService", - env: "integration_tests", - version: "1.0.0", - gitMetadataTagsProvider: new NullGitMetadataProvider()); - - public static DirectLogSubmissionSettings GetValidSettings() + public static LogFormatter GetFormatter() { - var tracerSettings = TracerSettings.Create(new() + var tracerSettings = GetValidSettings(); + return new( + tracerSettings, + tracerSettings.LogSubmissionSettings, + aasSettings: null, + gitMetadataTagsProvider: new NullGitMetadataProvider()); + } + + public static TracerSettings GetValidSettings() + => TracerSettings.Create(new() { { ConfigurationKeys.ApiKey, "abcdef" }, + { ConfigurationKeys.Environment, "integration_tests" }, + { ConfigurationKeys.ServiceName, "MyTestService" }, + { ConfigurationKeys.ServiceVersion, "1.0.0" }, { ConfigurationKeys.DirectLogSubmission.Host, "some_host" }, { ConfigurationKeys.DirectLogSubmission.Source, "csharp" }, { ConfigurationKeys.DirectLogSubmission.Url, "https://localhost:1234" }, @@ -36,8 +39,5 @@ public static DirectLogSubmissionSettings GetValidSettings() { ConfigurationKeys.DirectLogSubmission.BatchPeriodSeconds, "2" }, { ConfigurationKeys.DirectLogSubmission.QueueSizeLimit, "100000" } }); - - return tracerSettings.LogSubmissionSettings; - } } } diff --git a/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/DirectLogSubmissionSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/DirectLogSubmissionSettingsTests.cs index da713d836562..eeaaeb8ad566 100644 --- a/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/DirectLogSubmissionSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/DirectLogSubmissionSettingsTests.cs @@ -212,7 +212,7 @@ public void ApiKey(string value, string expected) [Fact] public void ValidSettingsAreValid() { - var settings = LogSettingsHelper.GetValidSettings(); + var settings = LogSettingsHelper.GetValidSettings().LogSubmissionSettings; settings.IsEnabled.Should().BeTrue(); } @@ -376,7 +376,7 @@ public void WhenLogsInjectionIsExplicitlySetViaEnvironmentThenValueIsUsed(bool l var tracerSettings = new TracerSettings(new NameValueConfigurationSource(config)); tracerSettings.LogSubmissionSettings.IsEnabled.Should().Be(directLogSubmissionEnabled); - tracerSettings.MutableSettings.LogsInjectionEnabled.Should().Be(logsInjectionEnabled); + tracerSettings.Manager.InitialMutableSettings.LogsInjectionEnabled.Should().Be(logsInjectionEnabled); } } } diff --git a/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/Formatting/LogFormatterTests.cs b/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/Formatting/LogFormatterTests.cs index 5aeb241e1309..f34dbdfba1be 100644 --- a/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/Formatting/LogFormatterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Logging/DirectSubmission/Formatting/LogFormatterTests.cs @@ -33,6 +33,9 @@ public class LogFormatterTests : IDisposable private static readonly NameValueCollection Defaults = new() { { ConfigurationKeys.ApiKey, "some_value" }, + { ConfigurationKeys.Environment, Env }, + { ConfigurationKeys.ServiceName, Service }, + { ConfigurationKeys.ServiceVersion, Version }, { ConfigurationKeys.DirectLogSubmission.Host, Host }, { ConfigurationKeys.DirectLogSubmission.Source, Source }, { ConfigurationKeys.DirectLogSubmission.Url, "http://localhost" }, @@ -159,9 +162,6 @@ public void WritesLogFormatCorrectly( _tracerSettings, _directLogSettings, aasSettings: aasSettings, - serviceName: Service, - env: Env, - version: Version, gitMetadataTagsProvider: new NullGitMetadataProvider()); formatter.FormatLog(sb, state, timestamp, message, eventId: null, logLevel, exception: null, RenderProperties); diff --git a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs index b5eab9e53537..e16542a50129 100644 --- a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs @@ -146,13 +146,14 @@ private static TracerManager CreateTracerManager(TracerSettings settings) static DirectLogSubmissionManager BuildLogSubmissionManager() => DirectLogSubmissionManager.Create( - previous: null, - settings: new TracerSettings(NullConfigurationSource.Instance), + settings: TracerSettings.Create(new() + { + { ConfigurationKeys.Environment, "test" }, + { ConfigurationKeys.ServiceName, "test" }, + { ConfigurationKeys.ServiceVersion, "test" }, + }), directLogSettings: new TracerSettings().LogSubmissionSettings, azureAppServiceSettings: null, - serviceName: "test", - env: "test", - serviceVersion: "test", gitMetadataTagsProvider: Mock.Of()); static RuntimeMetricsWriter BuildRuntimeMetrics() From 57e28cf1de37d857de31e854baeb770f46ca8104 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 12:00:48 +0100 Subject: [PATCH 06/29] Fix OTLP usages --- .../OpenTelemetry/Logs/OtlpExporter.cs | 33 +++++++++++---- .../OpenTelemetry/Logs/OtlpLogsSerializer.cs | 34 +++++++++++----- .../Metrics/OtlpMetricsSerializer.cs | 40 +++++++++++++------ 3 files changed, 76 insertions(+), 31 deletions(-) diff --git a/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpExporter.cs b/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpExporter.cs index 85d7bdad65cc..07bcac1aabe4 100644 --- a/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpExporter.cs +++ b/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpExporter.cs @@ -7,8 +7,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Configuration; using Datadog.Trace.Logging; @@ -30,19 +32,25 @@ internal class OtlpExporter : IOtlpExporter private readonly HttpClient _httpClient; private readonly OtlpGrpcExportClient? _grpcClient; private readonly OtlpHttpExportClient? _httpExportClient; - private readonly Uri _endpoint; private readonly IReadOnlyDictionary _headers; private readonly int _timeoutMs; private readonly OtlpProtocol _protocol; - private readonly TracerSettings _settings; + private OtlpLogsSerializer.ResourceTags _resourceTags; public OtlpExporter(TracerSettings settings) { - _settings = settings; - _endpoint = settings.OtlpLogsEndpoint; + var endpoint = settings.OtlpLogsEndpoint; _headers = settings.OtlpLogsHeaders; _timeoutMs = settings.OtlpLogsTimeoutMs; _protocol = settings.OtlpLogsProtocol; + UpdateResourceTags(settings.Manager.InitialMutableSettings); + settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateResourceTags(mutable); + } + }); _httpClient = CreateHttpClient(); @@ -50,7 +58,7 @@ public OtlpExporter(TracerSettings settings) { var opt = new OtlpExporterOptions { - Endpoint = _endpoint, + Endpoint = endpoint, TimeoutMilliseconds = _timeoutMs, Protocol = (OtlpExportProtocol)_protocol, AppendSignalPathToEndpoint = true @@ -68,7 +76,7 @@ public OtlpExporter(TracerSettings settings) { var opt = new OtlpExporterOptions { - Endpoint = _endpoint, + Endpoint = endpoint, TimeoutMilliseconds = _timeoutMs, Protocol = (OtlpExportProtocol)_protocol, AppendSignalPathToEndpoint = false // HTTP endpoint already includes /v1/logs @@ -82,6 +90,17 @@ public OtlpExporter(TracerSettings settings) const string logsHttpPath = "v1/logs"; _httpExportClient = new OtlpHttpExportClient(opt, _httpClient, logsHttpPath); } + + [MemberNotNull(nameof(_resourceTags))] + void UpdateResourceTags(MutableSettings mutable) + { + var newTags = new OtlpLogsSerializer.ResourceTags( + serviceName: mutable.DefaultServiceName, + environment: mutable.Environment, + serviceVersion: mutable.ServiceVersion, + globalTags: mutable.GlobalTags); + Interlocked.Exchange(ref _resourceTags, newTags); + } } /// @@ -166,7 +185,7 @@ private async Task SendOtlpRequest(IReadOnlyList logs) // For gRPC, reserve 5 bytes at the start for the frame header (added later) // For HTTP, start at position 0 var startPosition = _protocol == OtlpProtocol.Grpc ? 5 : 0; - var otlpPayload = OtlpLogsSerializer.SerializeLogs(logs, _settings, startPosition); + var otlpPayload = OtlpLogsSerializer.SerializeLogs(logs, _resourceTags, startPosition); return _protocol switch { diff --git a/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpLogsSerializer.cs b/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpLogsSerializer.cs index 23b6b5f2c30c..282f5a283b0c 100644 --- a/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpLogsSerializer.cs +++ b/tracer/src/Datadog.Trace/OpenTelemetry/Logs/OtlpLogsSerializer.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using Datadog.Trace.Configuration; using Datadog.Trace.Vendors.OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Serializer; using static Datadog.Trace.Vendors.OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Serializer.ProtobufOtlpCommonFieldNumberConstants; @@ -28,7 +29,7 @@ internal static class OtlpLogsSerializer /// /// Serializes logs to OTLP LogsData binary format using vendored protobuf serializer /// - public static byte[] SerializeLogs(IReadOnlyList logs, TracerSettings? settings, int startPosition = 0) + public static byte[] SerializeLogs(IReadOnlyList logs, ResourceTags settings, int startPosition = 0) { if (logs.Count == 0) { @@ -42,7 +43,7 @@ public static byte[] SerializeLogs(IReadOnlyList logs, TracerSettings? int resourceLogsLengthPosition = writePosition; writePosition += ReserveSizeForLength; - writePosition = WriteResourceLogs(buffer, writePosition, logs, settings!); + writePosition = WriteResourceLogs(buffer, writePosition, logs, settings); ProtobufSerializer.WriteReservedLength(buffer, resourceLogsLengthPosition, writePosition - (resourceLogsLengthPosition + ReserveSizeForLength)); @@ -51,7 +52,7 @@ public static byte[] SerializeLogs(IReadOnlyList logs, TracerSettings? return result; } - private static int WriteResourceLogs(byte[] buffer, int writePosition, IReadOnlyList logs, TracerSettings settings) + private static int WriteResourceLogs(byte[] buffer, int writePosition, IReadOnlyList logs, ResourceTags settings) { writePosition = ProtobufSerializer.WriteTag(buffer, writePosition, ResourceLogs_Resource, ProtobufWireType.LEN); int resourceLengthPosition = writePosition; @@ -90,19 +91,19 @@ private static int WriteResourceLogs(byte[] buffer, int writePosition, IReadOnly return writePosition; } - private static int WriteResource(byte[] buffer, int writePosition, TracerSettings settings) + private static int WriteResource(byte[] buffer, int writePosition, ResourceTags settings) { - var serviceName = settings.MutableSettings.ServiceName ?? "unknown_service:dotnet"; + var serviceName = settings.ServiceName ?? "unknown_service:dotnet"; writePosition = WriteResourceAttribute(buffer, writePosition, "service.name", serviceName); - if (!StringUtil.IsNullOrEmpty(settings.MutableSettings.ServiceVersion)) + if (!StringUtil.IsNullOrEmpty(settings.ServiceVersion)) { - writePosition = WriteResourceAttribute(buffer, writePosition, "service.version", settings.MutableSettings.ServiceVersion!); + writePosition = WriteResourceAttribute(buffer, writePosition, "service.version", settings.ServiceVersion); } - if (!StringUtil.IsNullOrEmpty(settings.MutableSettings.Environment)) + if (!StringUtil.IsNullOrEmpty(settings.Environment)) { - writePosition = WriteResourceAttribute(buffer, writePosition, "deployment.environment", settings.MutableSettings.Environment!); + writePosition = WriteResourceAttribute(buffer, writePosition, "deployment.environment", settings.Environment); } // Write telemetry SDK attributes @@ -110,9 +111,9 @@ private static int WriteResource(byte[] buffer, int writePosition, TracerSetting writePosition = WriteResourceAttribute(buffer, writePosition, "telemetry.sdk.language", "dotnet"); writePosition = WriteResourceAttribute(buffer, writePosition, "telemetry.sdk.version", TracerConstants.AssemblyVersion); - if (settings.MutableSettings.GlobalTags.Count > 0) + if (settings.GlobalTags.Count > 0) { - foreach (var tag in settings.MutableSettings.GlobalTags) + foreach (var tag in settings.GlobalTags) { if (IsHandledResourceAttribute(tag.Key)) { @@ -278,5 +279,16 @@ private static bool IsHandledResourceAttribute(string tagKey) tagKey.Equals("deployment.environment", StringComparison.OrdinalIgnoreCase) || tagKey.Equals("service.version", StringComparison.OrdinalIgnoreCase); } + + internal class ResourceTags(string serviceName, string? environment, string? serviceVersion, ReadOnlyDictionary globalTags) + { + public string ServiceName { get; } = serviceName; + + public string? Environment { get; } = environment; + + public string? ServiceVersion { get; } = serviceVersion; + + public ReadOnlyDictionary GlobalTags { get; } = globalTags; + } } #endif diff --git a/tracer/src/Datadog.Trace/OpenTelemetry/Metrics/OtlpMetricsSerializer.cs b/tracer/src/Datadog.Trace/OpenTelemetry/Metrics/OtlpMetricsSerializer.cs index 47bdd2f9e99a..971c37d97d40 100644 --- a/tracer/src/Datadog.Trace/OpenTelemetry/Metrics/OtlpMetricsSerializer.cs +++ b/tracer/src/Datadog.Trace/OpenTelemetry/Metrics/OtlpMetricsSerializer.cs @@ -7,8 +7,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; +using System.Threading; using Datadog.Trace.Configuration; #nullable enable @@ -27,12 +29,24 @@ internal class OtlpMetricsSerializer private const int LengthDelimited = 2; private readonly TracerSettings _settings; - private readonly byte[] _cachedResourceData; + private byte[] _cachedResourceData; public OtlpMetricsSerializer(TracerSettings settings) { _settings = settings; - _cachedResourceData = SerializeResource(settings); + UpdateCachedResourceData(settings.Manager.InitialMutableSettings); + settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateCachedResourceData(mutable); + } + }); + [MemberNotNull(nameof(_cachedResourceData))] + void UpdateCachedResourceData(MutableSettings mutable) + { + Interlocked.Exchange(ref _cachedResourceData, SerializeResource(mutable)); + } } /// @@ -74,8 +88,9 @@ private byte[] SerializeResourceMetrics(IReadOnlyList metrics) using var writer = new BinaryWriter(buffer, Encoding.UTF8); WriteTag(writer, FieldNumbers.Resource, LengthDelimited); - WriteVarInt(writer, _cachedResourceData.Length); - writer.Write(_cachedResourceData); + var data = Volatile.Read(ref _cachedResourceData); + WriteVarInt(writer, data.Length); + writer.Write(data); // Group metrics by meter identity (name + version + tags) var meterGroups = new Dictionary>(); @@ -104,7 +119,7 @@ private byte[] SerializeResourceMetrics(IReadOnlyList metrics) return buffer.ToArray(); } - private byte[] SerializeResource(TracerSettings settings) + private byte[] SerializeResource(MutableSettings settings) { using var buffer = new MemoryStream(); using var writer = new BinaryWriter(buffer, Encoding.UTF8); @@ -124,31 +139,30 @@ private byte[] SerializeResource(TracerSettings settings) WriteVarInt(writer, sdkVersionAttr.Length); writer.Write(sdkVersionAttr); - var serviceName = settings.MutableSettings.ServiceName ?? "unknown_service:dotnet"; - var serviceNameAttr = SerializeKeyValue("service.name", serviceName); + var serviceNameAttr = SerializeKeyValue("service.name", settings.DefaultServiceName); WriteTag(writer, FieldNumbers.Attributes, LengthDelimited); WriteVarInt(writer, serviceNameAttr.Length); writer.Write(serviceNameAttr); - if (!string.IsNullOrEmpty(settings.MutableSettings.Environment)) + if (!string.IsNullOrEmpty(settings.Environment)) { - var envAttr = SerializeKeyValue("deployment.environment.name", settings.MutableSettings.Environment); + var envAttr = SerializeKeyValue("deployment.environment.name", settings.Environment); WriteTag(writer, FieldNumbers.Attributes, LengthDelimited); WriteVarInt(writer, envAttr.Length); writer.Write(envAttr); } - if (!string.IsNullOrEmpty(settings.MutableSettings.ServiceVersion)) + if (!string.IsNullOrEmpty(settings.ServiceVersion)) { - var versionAttr = SerializeKeyValue("service.version", settings.MutableSettings.ServiceVersion); + var versionAttr = SerializeKeyValue("service.version", settings.ServiceVersion); WriteTag(writer, FieldNumbers.Attributes, LengthDelimited); WriteVarInt(writer, versionAttr.Length); writer.Write(versionAttr); } - if (settings.MutableSettings.GlobalTags.Count > 0) + if (settings.GlobalTags.Count > 0) { - foreach (var tag in settings.MutableSettings.GlobalTags) + foreach (var tag in settings.GlobalTags) { if (IsHandledResourceAttribute(tag.Key)) { From 71b7b90a7f0e4ff2614cd790ea922ca68dfb9e25 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 12:34:52 +0100 Subject: [PATCH 07/29] "Fix" debugger - this only uses the initial settings though, and doesn't respond to changes I left it like this because the debugger already doesn't respond to changes like other services do --- tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs | 2 +- tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs | 6 +++--- .../Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs | 4 ++-- .../Debugger/Upload/DebuggerUploadApiBase.cs | 9 ++++++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs index 055eef530874..3d36aacd574a 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs @@ -39,7 +39,7 @@ internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoverySe var diagnosticsUploader = CreateDiagnosticsUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), diagnosticsSink); var lineProbeResolver = LineProbeResolver.Create(debuggerSettings.ThirdPartyDetectionExcludes, debuggerSettings.ThirdPartyDetectionIncludes); var probeStatusPoller = ProbeStatusPoller.Create(diagnosticsSink, debuggerSettings); - var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.MutableSettings.Environment, tracerSettings.MutableSettings.ServiceVersion); + var configurationUpdater = ConfigurationUpdater.Create(tracerSettings.Manager.InitialMutableSettings.Environment, tracerSettings.Manager.InitialMutableSettings.ServiceVersion); var statsd = GetDogStatsd(tracerSettings, serviceName); diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs index b04b3207b5d2..99ef47f0f23b 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs @@ -88,12 +88,12 @@ private string GetServiceName(TracerSettings tracerSettings) { try { - return TraceUtil.NormalizeTag(tracerSettings.MutableSettings.DefaultServiceName); + return TraceUtil.NormalizeTag(tracerSettings.Manager.InitialMutableSettings.DefaultServiceName); } catch (Exception e) { Log.Error(e, "Could not set `DynamicInstrumentationHelper.ServiceName`."); - return tracerSettings.MutableSettings.DefaultServiceName; + return tracerSettings.Manager.InitialMutableSettings.DefaultServiceName; } } @@ -185,7 +185,7 @@ private void OneTimeSetup(TracerSettings tracerSettings) LifetimeManager.Instance.AddShutdownTask(ShutdownTasks); SetGeneralConfig(tracerSettings, DebuggerSettings); - if (tracerSettings.MutableSettings.StartupDiagnosticLogEnabled) + if (tracerSettings.Manager.InitialMutableSettings.StartupDiagnosticLogEnabled) { _ = Task.Run(WriteStartupDebuggerDiagnosticLog); } diff --git a/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs b/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs index bb9361d0cf39..6f56667706a2 100644 --- a/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs +++ b/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolsUploader.cs @@ -63,8 +63,8 @@ private SymbolsUploader( { _symDbEndpoint = null; _alreadyProcessed = new HashSet(); - _environment = tracerSettings.MutableSettings.Environment; - _serviceVersion = tracerSettings.MutableSettings.ServiceVersion; + _environment = tracerSettings.Manager.InitialMutableSettings.Environment; + _serviceVersion = tracerSettings.Manager.InitialMutableSettings.ServiceVersion; _serviceName = serviceName; _discoveryService = discoveryService; _api = api; diff --git a/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiBase.cs b/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiBase.cs index b12413d84b1f..029dc96072bd 100644 --- a/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiBase.cs +++ b/tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiBase.cs @@ -74,13 +74,16 @@ private string GetDefaultTagsMergedWithGlobalTags() try { - var environment = TraceUtil.NormalizeTag(Tracer.Instance.Settings.MutableSettings.Environment); + // TODO: this only gets the original values, before any updates from remote config or config in code + // this should be refactored to subscribe to changes instead + var mutableSettings = Tracer.Instance.Settings.Manager.InitialMutableSettings; + var environment = TraceUtil.NormalizeTag(mutableSettings.Environment); if (!string.IsNullOrEmpty(environment)) { sb.Append($"env:{environment},"); } - var version = Tracer.Instance.Settings.MutableSettings.ServiceVersion; + var version = mutableSettings.ServiceVersion; if (!string.IsNullOrEmpty(version)) { sb.Append($"version:{version},"); @@ -106,7 +109,7 @@ private string GetDefaultTagsMergedWithGlobalTags() sb.Append($"{CommonTags.GitCommit}:{gitMetadata.CommitSha},"); } - foreach (var kvp in Tracer.Instance.Settings.MutableSettings.GlobalTags) + foreach (var kvp in mutableSettings.GlobalTags) { sb.Append($"{kvp.Key}:{kvp.Value},"); } From 92cf95b9f26b974f4dfb0b0d5eb1a7de89d230c6 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 13:02:27 +0100 Subject: [PATCH 08/29] Fix remote config to subscribe to setting changes --- .../Protocol/RcmClientTracer.cs | 49 +++++++- .../RemoteConfigurationManager.cs | 113 +++++++++--------- .../RemoteConfigurationSettings.cs | 7 -- .../RemoteConfigurationApiFactory.cs | 2 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 10 +- .../RemoteConfigurationSettingsTests.cs | 18 --- .../RemoteConfigurationApiTests.cs | 6 +- 7 files changed, 107 insertions(+), 98 deletions(-) diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Protocol/RcmClientTracer.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Protocol/RcmClientTracer.cs index 30b77081a667..8ce1f8ad4623 100644 --- a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Protocol/RcmClientTracer.cs +++ b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Protocol/RcmClientTracer.cs @@ -3,7 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +#nullable enable + using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; using Datadog.Trace.Vendors.Newtonsoft.Json; #nullable enable @@ -12,6 +16,8 @@ namespace Datadog.Trace.RemoteConfigurationManagement.Protocol { internal class RcmClientTracer { + // Don't change this constructor - it's used by Newtonsoft.JSON for deserialization + // and that can mean the provided properties are not _really_ nullable, even though we "require" them to be public RcmClientTracer(string runtimeId, string tracerVersion, string service, string env, string? appVersion, List tags, List? processTags) { RuntimeId = runtimeId; @@ -21,20 +27,23 @@ public RcmClientTracer(string runtimeId, string tracerVersion, string service, s ProcessTags = processTags; Env = env; AppVersion = appVersion; - Tags = tags; + Tags = tags ?? []; } + [JsonIgnore] + public bool IsGitMetadataAddedToRequestTags { get; set; } + [JsonProperty("runtime_id")] - public string RuntimeId { get; } + public string? RuntimeId { get; } [JsonProperty("language")] public string Language { get; } [JsonProperty("tracer_version")] - public string TracerVersion { get; } + public string? TracerVersion { get; } [JsonProperty("service")] - public string Service { get; } + public string? Service { get; } [JsonProperty("process_tags")] public List? ProcessTags { get; } @@ -43,12 +52,42 @@ public RcmClientTracer(string runtimeId, string tracerVersion, string service, s public string[]? ExtraServices { get; set; } [JsonProperty("env")] - public string Env { get; } + public string? Env { get; } [JsonProperty("app_version")] public string? AppVersion { get; } [JsonProperty("tags")] public List Tags { get; } + + public static RcmClientTracer Create(string runtimeId, string tracerVersion, string service, string env, string? appVersion, ReadOnlyDictionary globalTags, List? processTags) + => new(runtimeId, tracerVersion, service, env, appVersion, GetTags(env, service, globalTags), processTags); + + private static List GetTags(string? environment, string? serviceVersion, ReadOnlyDictionary? globalTags) + { + var tags = globalTags?.Count > 0 + ? globalTags.Select(pair => pair.Key + ":" + pair.Value).ToList() + : []; + + if (!string.IsNullOrEmpty(environment)) + { + tags.Add($"env:{environment}"); + } + + if (!string.IsNullOrEmpty(serviceVersion)) + { + tags.Add($"version:{serviceVersion}"); + } + + tags.Add($"tracer_version:{TracerConstants.ThreePartVersion}"); + + var hostName = PlatformHelpers.HostMetadata.Instance?.Hostname; + if (!string.IsNullOrEmpty(hostName)) + { + tags.Add($"host_name:{hostName}"); + } + + return tags; + } } } diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationManager.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationManager.cs index 8124bb1d6935..9b7fcb74059e 100644 --- a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationManager.cs +++ b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationManager.cs @@ -7,7 +7,7 @@ using System; using System.Collections.Generic; -using System.Linq; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent.DiscoveryService; @@ -24,86 +24,85 @@ internal class RemoteConfigurationManager : IRemoteConfigurationManager { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(RemoteConfigurationManager)); - private readonly RcmClientTracer _rcmTracer; private readonly IDiscoveryService _discoveryService; - private readonly IRemoteConfigurationApi _remoteConfigurationApi; private readonly IGitMetadataTagsProvider _gitMetadataTagsProvider; private readonly TimeSpan _pollInterval; private readonly IRcmSubscriptionManager _subscriptionManager; - + private readonly IDisposable _settingSubscription; private readonly TaskCompletionSource _processExit = new(); + private IRemoteConfigurationApi _remoteConfigurationApi; + private RcmClientTracer _rcmTracer; private int _isPollingStarted; private bool _isRcmEnabled; - private bool _gitMetadataAddedToRequestTags; private RemoteConfigurationManager( IDiscoveryService discoveryService, - IRemoteConfigurationApi remoteConfigurationApi, - RcmClientTracer rcmTracer, + TracerSettings settings, TimeSpan pollInterval, IGitMetadataTagsProvider gitMetadataTagsProvider, - IRcmSubscriptionManager subscriptionManager) + IRcmSubscriptionManager subscriptionManager, + List? processTags) { _discoveryService = discoveryService; - _remoteConfigurationApi = remoteConfigurationApi; - _rcmTracer = rcmTracer; _pollInterval = pollInterval; _gitMetadataTagsProvider = gitMetadataTagsProvider; _subscriptionManager = subscriptionManager; discoveryService.SubscribeToChanges(SetRcmEnabled); + UpdateRcmApi(settings.Manager.InitialExporterSettings); + UpdateRcmClientTracer(settings.Manager.InitialMutableSettings); + _settingSubscription = settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } updated) + { + UpdateRcmClientTracer(updated); + } + + if (changes.UpdatedExporter is { } exporter) + { + UpdateRcmApi(exporter); + } + }); + + [MemberNotNull(nameof(_rcmTracer))] + void UpdateRcmClientTracer(MutableSettings mutable) + { + var rcmTracer = RcmClientTracer.Create( + runtimeId: Util.RuntimeId.Get(), + tracerVersion: TracerConstants.ThreePartVersion, + // Service Name must be lowercase, otherwise the agent will not be able to find the service + service: TraceUtil.NormalizeTag(mutable.DefaultServiceName), + env: TraceUtil.NormalizeTag(mutable.Environment), + appVersion: mutable.ServiceVersion, + globalTags: mutable.GlobalTags, + processTags: processTags); + Interlocked.Exchange(ref _rcmTracer!, rcmTracer); + } + + [MemberNotNull(nameof(_remoteConfigurationApi))] + void UpdateRcmApi(ExporterSettings exporter) + { + var rcmApi = RemoteConfigurationApiFactory.Create(exporter, discoveryService); + + Interlocked.Exchange(ref _remoteConfigurationApi!, rcmApi); + } } public static RemoteConfigurationManager Create( IDiscoveryService discoveryService, - IRemoteConfigurationApi remoteConfigurationApi, RemoteConfigurationSettings settings, - string serviceName, TracerSettings tracerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IRcmSubscriptionManager subscriptionManager) { - var tags = GetTags(settings, tracerSettings); - return new RemoteConfigurationManager( discoveryService, - remoteConfigurationApi, - new RcmClientTracer(settings.RuntimeId, settings.TracerVersion, serviceName, TraceUtil.NormalizeTag(tracerSettings.MutableSettings.Environment), tracerSettings.MutableSettings.ServiceVersion, tags, tracerSettings.PropagateProcessTags ? ProcessTags.TagsList : null), + tracerSettings, pollInterval: settings.PollInterval, gitMetadataTagsProvider, - subscriptionManager); - } - - private static List GetTags(RemoteConfigurationSettings rcmSettings, TracerSettings tracerSettings) - { - var tags = tracerSettings.MutableSettings.GlobalTags?.Select(pair => pair.Key + ":" + pair.Value).ToList() ?? new List(); - - var environment = TraceUtil.NormalizeTag(tracerSettings.MutableSettings.Environment); - if (!string.IsNullOrEmpty(environment)) - { - tags.Add($"env:{environment}"); - } - - var serviceVersion = tracerSettings.MutableSettings.ServiceVersion; - if (!string.IsNullOrEmpty(serviceVersion)) - { - tags.Add($"version:{serviceVersion}"); - } - - var tracerVersion = rcmSettings.TracerVersion; - if (!string.IsNullOrEmpty(tracerVersion)) - { - tags.Add($"tracer_version:{tracerVersion}"); - } - - var hostName = PlatformHelpers.HostMetadata.Instance?.Hostname; - if (!string.IsNullOrEmpty(hostName)) - { - tags.Add($"host_name:{hostName}"); - } - - return tags; + subscriptionManager, + tracerSettings.PropagateProcessTags ? ProcessTags.TagsList : null); } public void Start() @@ -117,6 +116,7 @@ public void Dispose() if (_processExit.TrySetResult(true)) { _discoveryService.RemoveSubscription(SetRcmEnabled); + _settingSubscription.Dispose(); } else { @@ -164,18 +164,19 @@ private async Task StartPollingAsync() private Task Poll() { - return _subscriptionManager.SendRequest(_rcmTracer, request => + var rcm = Volatile.Read(ref _rcmTracer); + return _subscriptionManager.SendRequest(rcm, request => { - EnrichTagsWithGitMetadata(request.Client.ClientTracer.Tags); + EnrichTagsWithGitMetadata(request.Client.ClientTracer); request.Client.ClientTracer.ExtraServices = ExtraServicesProvider.Instance.GetExtraServices(); - return _remoteConfigurationApi.GetConfigs(request); + return Volatile.Read(ref _remoteConfigurationApi).GetConfigs(request); }); } - private void EnrichTagsWithGitMetadata(List tags) + private void EnrichTagsWithGitMetadata(RcmClientTracer details) { - if (_gitMetadataAddedToRequestTags) + if (details.IsGitMetadataAddedToRequestTags) { return; } @@ -188,11 +189,11 @@ private void EnrichTagsWithGitMetadata(List tags) if (gitMetadata != GitMetadata.Empty) { - tags.Add($"{CommonTags.GitCommit}:{gitMetadata.CommitSha}"); - tags.Add($"{CommonTags.GitRepository}:{gitMetadata.RepositoryUrl}"); + details.Tags.Add($"{CommonTags.GitCommit}:{gitMetadata.CommitSha}"); + details.Tags.Add($"{CommonTags.GitRepository}:{gitMetadata.RepositoryUrl}"); } - _gitMetadataAddedToRequestTags = true; + details.IsGitMetadataAddedToRequestTags = true; } private void SetRcmEnabled(AgentConfiguration c) diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationSettings.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationSettings.cs index a9d7ca01ad0a..1affa82929fe 100644 --- a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationSettings.cs +++ b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/RemoteConfigurationSettings.cs @@ -20,9 +20,6 @@ public RemoteConfigurationSettings(IConfigurationSource? configurationSource, IC { configurationSource ??= NullConfigurationSource.Instance; - RuntimeId = Util.RuntimeId.Get(); - TracerVersion = TracerConstants.ThreePartVersion; - var pollInterval = new ConfigurationBuilder(configurationSource, telemetry) #pragma warning disable CS0618 .WithKeys(ConfigurationKeys.Rcm.PollInterval, ConfigurationKeys.Rcm.PollIntervalInternal) @@ -32,10 +29,6 @@ public RemoteConfigurationSettings(IConfigurationSource? configurationSource, IC PollInterval = TimeSpan.FromSeconds(pollInterval.Value); } - public string RuntimeId { get; } - - public string TracerVersion { get; } - public TimeSpan PollInterval { get; } public static RemoteConfigurationSettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry) diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApiFactory.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApiFactory.cs index f2602aeaed80..a8f54f36b86d 100644 --- a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApiFactory.cs +++ b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApiFactory.cs @@ -14,7 +14,7 @@ namespace Datadog.Trace.RemoteConfigurationManagement.Transport { internal class RemoteConfigurationApiFactory { - public static IRemoteConfigurationApi Create(ExporterSettings exporterSettings, RemoteConfigurationSettings remoteConfigurationSettings, IDiscoveryService discoveryService) + public static IRemoteConfigurationApi Create(ExporterSettings exporterSettings, IDiscoveryService discoveryService) { var apiRequestFactory = AgentTransportStrategy.Get( exporterSettings, diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 4931ec1c824a..bde0b99ddb41 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -180,18 +180,10 @@ internal TracerManager CreateTracerManager( { var sw = Stopwatch.StartNew(); - var rcmSettings = RemoteConfigurationSettings.FromDefaultSource(); - var rcmApi = RemoteConfigurationApiFactory.Create(settings.Exporter, rcmSettings, discoveryService); - - // Service Name must be lowercase, otherwise the agent will not be able to find the service - var serviceName = TraceUtil.NormalizeTag(defaultServiceName); - remoteConfigurationManager = RemoteConfigurationManager.Create( discoveryService, - rcmApi, - rcmSettings, - serviceName, + RemoteConfigurationSettings.FromDefaultSource(), settings, gitMetadataTagsProvider, RcmSubscriptionManager.Instance); diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/RemoteConfigurationSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/RemoteConfigurationSettingsTests.cs index 04f3d3fb6d3e..3a5edf544906 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/RemoteConfigurationSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/RemoteConfigurationSettingsTests.cs @@ -19,24 +19,6 @@ namespace Datadog.Trace.Tests.Configuration { public class RemoteConfigurationSettingsTests : SettingsTestsBase { - [Fact] - public void RuntimeId() - { - var source = CreateConfigurationSource(); - var settings = new RemoteConfigurationSettings(source, NullConfigurationTelemetry.Instance); - - settings.RuntimeId.Should().Be(Datadog.Trace.Util.RuntimeId.Get()); - } - - [Fact] - public void TracerVersion() - { - var source = CreateConfigurationSource(); - var settings = new RemoteConfigurationSettings(source, NullConfigurationTelemetry.Instance); - - settings.TracerVersion.Should().Be(TracerConstants.ThreePartVersion); - } - [Theory] [InlineData(null, null, RemoteConfigurationSettings.DefaultPollIntervalSeconds)] [InlineData("", null, RemoteConfigurationSettings.DefaultPollIntervalSeconds)] diff --git a/tracer/test/Datadog.Trace.Tests/RemoteConfigurationManagement/RemoteConfigurationApiTests.cs b/tracer/test/Datadog.Trace.Tests/RemoteConfigurationManagement/RemoteConfigurationApiTests.cs index 93e5e5de2b5a..cc58c2f01b23 100644 --- a/tracer/test/Datadog.Trace.Tests/RemoteConfigurationManagement/RemoteConfigurationApiTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RemoteConfigurationManagement/RemoteConfigurationApiTests.cs @@ -4,6 +4,7 @@ // using System; +using System.Collections.Generic; using System.Threading.Tasks; using Datadog.Trace.Agent.Transports; using Datadog.Trace.RemoteConfigurationManagement; @@ -11,6 +12,7 @@ using Datadog.Trace.RemoteConfigurationManagement.Transport; using Datadog.Trace.TestHelpers.TransportHelpers; using Datadog.Trace.Tests.Agent; +using Datadog.Trace.Util; using Datadog.Trace.Vendors.Newtonsoft.Json; using FluentAssertions; using Xunit; @@ -237,13 +239,13 @@ private static GetRcmRequest GetRequest(string backendClientStage = null) { // We don't really care about this being "real" data, as long as it has the right shape, // but we'll keep it reasonable here for the sake of it - var tracer = new RcmClientTracer( + var tracer = RcmClientTracer.Create( runtimeId: Guid.NewGuid().ToString(), tracerVersion: TracerConstants.ThreePartVersion, service: nameof(RemoteConfigurationApiTests), env: "RCM Test", appVersion: "1.0.0", - tags: [], + globalTags: ReadOnlyDictionary.Empty, processTags: ["a.b:c", "x.y:z"]); var state = new RcmClientState( From 79a77b3cc77662d0dac9d6dd4cdac91da9e8c96e Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 13:32:53 +0100 Subject: [PATCH 09/29] Fix telemetry usages --- .../ApplicationTelemetryCollector.cs | 75 +++++++++++-------- .../IntegrationTelemetryCollector.cs | 6 +- .../Telemetry/ITelemetryController.cs | 6 -- .../Telemetry/TelemetryController.cs | 26 ++++--- .../Telemetry/TelemetryFactory.cs | 4 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 1 - .../Helpers/TelemetryHelperTests.cs | 2 +- .../ApplicationTelemetryCollectorTests.cs | 6 +- .../IntegrationTelemetryCollectorTests.cs | 22 +++--- .../Telemetry/TelemetryControllerTests.cs | 20 ++--- 10 files changed, 92 insertions(+), 76 deletions(-) diff --git a/tracer/src/Datadog.Trace/Telemetry/Collectors/ApplicationTelemetryCollector.cs b/tracer/src/Datadog.Trace/Telemetry/Collectors/ApplicationTelemetryCollector.cs index 888d586d4e7f..d1674abb5d67 100644 --- a/tracer/src/Datadog.Trace/Telemetry/Collectors/ApplicationTelemetryCollector.cs +++ b/tracer/src/Datadog.Trace/Telemetry/Collectors/ApplicationTelemetryCollector.cs @@ -17,40 +17,19 @@ internal class ApplicationTelemetryCollector private ApplicationTelemetryData? _applicationData = null; private HostTelemetryData? _hostData = null; - public void RecordTracerSettings( - TracerSettings tracerSettings, - string defaultServiceName) + public void RecordTracerSettings(TracerSettings tracerSettings) { - // Try to retrieve config based Git Info - // If explicitly provided, these values take precedence - GitMetadata? gitMetadata = _gitMetadata; - if (tracerSettings.GitMetadataEnabled && !string.IsNullOrEmpty(tracerSettings.MutableSettings.GitCommitSha) && !string.IsNullOrEmpty(tracerSettings.MutableSettings.GitRepositoryUrl)) - { - gitMetadata = new GitMetadata(tracerSettings.MutableSettings.GitCommitSha!, tracerSettings.MutableSettings.GitRepositoryUrl!); - Interlocked.Exchange(ref _gitMetadata, gitMetadata); - } - string? processTags = null; - if (tracerSettings.PropagateProcessTags && !string.IsNullOrEmpty(ProcessTags.SerializedTags)) + if (tracerSettings.PropagateProcessTags) { - processTags = ProcessTags.SerializedTags; + var pTags = ProcessTags.SerializedTags; + if (!string.IsNullOrEmpty(processTags)) + { + processTags = pTags; + } } - var frameworkDescription = FrameworkDescription.Instance; - var application = new ApplicationTelemetryData( - serviceName: defaultServiceName, - env: tracerSettings.MutableSettings.Environment ?? string.Empty, // required, but we don't have it - serviceVersion: tracerSettings.MutableSettings.ServiceVersion ?? string.Empty, // required, but we don't have it - tracerVersion: TracerConstants.AssemblyVersion, - languageName: TracerConstants.Language, - languageVersion: frameworkDescription.ProductVersion, - runtimeName: frameworkDescription.Name, - runtimeVersion: frameworkDescription.ProductVersion, - commitSha: gitMetadata?.CommitSha, - repositoryUrl: gitMetadata?.RepositoryUrl, - processTags: processTags); - - Interlocked.Exchange(ref _applicationData, application); + RecordMutableSettings(tracerSettings, tracerSettings.Manager.InitialMutableSettings, processTags); // The host properties can't change, so only need to set them the first time if (Volatile.Read(ref _hostData) is not null) @@ -58,6 +37,7 @@ public void RecordTracerSettings( return; } + var frameworkDescription = FrameworkDescription.Instance; var host = HostMetadata.Instance; var osDescription = frameworkDescription.OSArchitecture == "x86" ? $"{frameworkDescription.OSDescription} (32bit)" @@ -75,6 +55,41 @@ public void RecordTracerSettings( }; } + public void RecordMutableSettings(TracerSettings tracerSettings, MutableSettings mutableSettings) + => RecordMutableSettings(tracerSettings, mutableSettings, null); + + private void RecordMutableSettings(TracerSettings tracerSettings, MutableSettings mutableSettings, string? processTags) + { + // Try to retrieve config based Git Info + GitMetadata? gitMetadata; + // If explicitly provided, these values take precedence + if (tracerSettings.GitMetadataEnabled && !StringUtil.IsNullOrEmpty(mutableSettings.GitCommitSha) && !StringUtil.IsNullOrEmpty(mutableSettings.GitRepositoryUrl)) + { + gitMetadata = new GitMetadata(mutableSettings.GitCommitSha, mutableSettings.GitRepositoryUrl); + Interlocked.Exchange(ref _gitMetadata, gitMetadata); + } + else + { + gitMetadata = Volatile.Read(ref _gitMetadata); + } + + var frameworkDescription = FrameworkDescription.Instance; + var application = new ApplicationTelemetryData( + serviceName: mutableSettings.DefaultServiceName, + env: mutableSettings.Environment ?? string.Empty, // required, but we don't have it + serviceVersion: mutableSettings.ServiceVersion ?? string.Empty, // required, but we don't have it + tracerVersion: TracerConstants.AssemblyVersion, + languageName: TracerConstants.Language, + languageVersion: frameworkDescription.ProductVersion, + runtimeName: frameworkDescription.Name, + runtimeVersion: frameworkDescription.ProductVersion, + commitSha: gitMetadata?.CommitSha, + repositoryUrl: gitMetadata?.RepositoryUrl, + processTags: processTags ?? Volatile.Read(ref _applicationData)?.ProcessTags); + + Interlocked.Exchange(ref _applicationData, application); + } + public void RecordGitMetadata(GitMetadata gitMetadata) { if (gitMetadata.IsEmpty) @@ -91,7 +106,7 @@ public void RecordGitMetadata(GitMetadata gitMetadata) while (true) { - var original = _applicationData; + var original = Volatile.Read(ref _applicationData); var application = new ApplicationTelemetryData( serviceName: original.ServiceName, env: original.Env, diff --git a/tracer/src/Datadog.Trace/Telemetry/Collectors/IntegrationTelemetryCollector.cs b/tracer/src/Datadog.Trace/Telemetry/Collectors/IntegrationTelemetryCollector.cs index 9f9ce546563a..2d81cc3d85fa 100644 --- a/tracer/src/Datadog.Trace/Telemetry/Collectors/IntegrationTelemetryCollector.cs +++ b/tracer/src/Datadog.Trace/Telemetry/Collectors/IntegrationTelemetryCollector.cs @@ -30,11 +30,11 @@ public IntegrationTelemetryCollector() } } - public void RecordTracerSettings(TracerSettings settings) + public void RecordTracerSettings(MutableSettings settings) { - for (var i = 0; i < settings.MutableSettings.Integrations.Settings.Length; i++) + for (var i = 0; i < settings.Integrations.Settings.Length; i++) { - var integration = settings.MutableSettings.Integrations.Settings[i]; + var integration = settings.Integrations.Settings[i]; if (integration.Enabled == false) { _integrationsById[i].WasExplicitlyDisabled = 1; diff --git a/tracer/src/Datadog.Trace/Telemetry/ITelemetryController.cs b/tracer/src/Datadog.Trace/Telemetry/ITelemetryController.cs index 73168bdcc38f..64a2f60ba58c 100644 --- a/tracer/src/Datadog.Trace/Telemetry/ITelemetryController.cs +++ b/tracer/src/Datadog.Trace/Telemetry/ITelemetryController.cs @@ -31,12 +31,6 @@ internal interface ITelemetryController /// void IntegrationDisabledDueToError(IntegrationId integrationId, string error); - /// - /// Called when a tracer is initialized to record the tracer's settings - /// Only the first tracer registered is recorded - /// - void RecordTracerSettings(TracerSettings settings, string defaultServiceName); - /// /// Called to record profiler-related telemetry /// diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs index 29e25f9dd04c..a59549de91b4 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs @@ -45,12 +45,14 @@ internal class TelemetryController : ITelemetryController private readonly TagBuilder _logTagBuilder = new(); private readonly Task _flushTask; private readonly Scheduler _scheduler; + private readonly IDisposable _settingsSubscription; private TelemetryTransportManager _transportManager; private bool _sendTelemetry; private bool _isStarted; private string? _namingVersion; internal TelemetryController( + TracerSettings tracerSettings, IConfigurationTelemetry configuration, IDependencyTelemetryCollector dependencies, IMetricsTelemetryCollector metrics, @@ -85,26 +87,29 @@ internal TelemetryController( Log.Warning(ex, "Unable to register a callback to the AppDomain.AssemblyLoad event. Telemetry collection of loaded assemblies will be disabled."); } + RecordTracerSettings(tracerSettings); + _settingsSubscription = tracerSettings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } updated) + { + _application.RecordMutableSettings(tracerSettings, updated); + _integrations.RecordTracerSettings(updated); + } + }); + _flushTask = Task.Run(PushTelemetryLoopAsync); _flushTask.ContinueWith(t => Log.Error(t.Exception, "Error in telemetry flush task"), TaskContinuationOptions.OnlyOnFaulted); } - public void RecordTracerSettings(TracerSettings settings, string defaultServiceName) + public void RecordTracerSettings(TracerSettings settings) { // Note that this _doesn't_ clear the configuration held by ImmutableTracerSettings // that's necessary because users could reconfigure the tracer to re-use an old // ImmutableTracerSettings, at which point that config would become "current", so we // need to keep it around settings.Telemetry.CopyTo(_configuration); - // if the mutable settings have changed since the start, re-record them - // to ensure they have the correct values. This is a temporary measure before - // we fully extract mutable settings - if (!ReferenceEquals(settings.MutableSettings, settings.Manager.InitialMutableSettings)) - { - settings.MutableSettings.Telemetry.CopyTo(_configuration); - } - - _application.RecordTracerSettings(settings, defaultServiceName); + _application.RecordTracerSettings(settings); + _integrations.RecordTracerSettings(settings.Manager.InitialMutableSettings); _namingVersion = ((int)settings.MetadataSchemaVersion).ToString(); _logTagBuilder.Update(settings); _queue.Enqueue(new WorkItem(WorkItem.ItemType.EnableSending, null)); @@ -161,6 +166,7 @@ public void IntegrationDisabledDueToError(IntegrationId integrationId, string er public async Task DisposeAsync() { + _settingsSubscription.Dispose(); TerminateLoop(); await _flushTask.ConfigureAwait(false); } diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs index afff18053a04..ec7cd09e9203 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs @@ -110,7 +110,7 @@ public ITelemetryController CreateTelemetryController(TracerSettings tracerSetti } log.Debug("Creating telemetry controller v2"); - return CreateController(telemetryTransports, settings, discoveryService); + return CreateController(tracerSettings, telemetryTransports, settings, discoveryService); } catch (Exception ex) { @@ -155,6 +155,7 @@ private static void DisableConfigCollector() } private ITelemetryController CreateController( + TracerSettings tracerSettings, TelemetryTransports telemetryTransports, TelemetrySettings settings, IDiscoveryService discoveryService) @@ -171,6 +172,7 @@ private ITelemetryController CreateController( lock (_sync) { _controller ??= new TelemetryController( + tracerSettings, Config, _dependencies!, Metrics, diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index bde0b99ddb41..dbbf97cc89ed 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -161,7 +161,6 @@ internal TracerManager CreateTracerManager( settings.AzureAppServiceMetadata, gitMetadataTagsProvider); - telemetry.RecordTracerSettings(settings, defaultServiceName); TelemetryFactory.Metrics.SetWafAndRulesVersion(Security.Instance.DdlibWafVersion, Security.Instance.WafRuleFileVersion); ErrorData? initError = !string.IsNullOrEmpty(Security.Instance.InitializationError) ? new ErrorData(TelemetryErrorCode.AppsecConfigurationError, Security.Instance.InitializationError) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelperTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelperTests.cs index 79040b2ade83..aff3ed45afd3 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelperTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelperTests.cs @@ -76,7 +76,7 @@ public void AssertIntegration_HandlesMultipleTelemetryPushes() { ConfigurationKeys.DisabledIntegrations, $"{nameof(IntegrationId.Kafka)};{nameof(IntegrationId.Msmq)}" } }); - collector.RecordTracerSettings(tracerSettings); + collector.RecordTracerSettings(tracerSettings.Manager.InitialMutableSettings); metricsCollector.AggregateMetrics(); telemetryData.Add(BuildTelemetryData(collector.GetData(), metrics: metricsCollector.GetMetrics(), sendAppClosing: true)); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/ApplicationTelemetryCollectorTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/ApplicationTelemetryCollectorTests.cs index 8192b7183815..c4d98aa04551 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/ApplicationTelemetryCollectorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/ApplicationTelemetryCollectorTests.cs @@ -42,7 +42,7 @@ public void ApplicationDataShouldIncludeExpectedValues() collector.GetApplicationData().Should().BeNull(); - collector.RecordTracerSettings(settings, ServiceName); + collector.RecordTracerSettings(settings); // calling twice should give same results AssertData(collector.GetApplicationData()); @@ -85,7 +85,7 @@ public void ApplicationWithNoGitDataShouldIncludeExpectedValues() collector.GetApplicationData().Should().BeNull(); - collector.RecordTracerSettings(settings, ServiceName); + collector.RecordTracerSettings(settings); // calling twice should give same results AssertData(collector.GetApplicationData()); @@ -130,7 +130,7 @@ public void HostDataShouldIncludeExpectedValues() collector.GetHostData().Should().BeNull(); - collector.RecordTracerSettings(settings, ServiceName); + collector.RecordTracerSettings(settings); // calling twice should give same results AssertData(collector.GetHostData()); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/IntegrationTelemetryCollectorTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/IntegrationTelemetryCollectorTests.cs index 9cbeb580d27f..7cfc29479fbd 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/IntegrationTelemetryCollectorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/IntegrationTelemetryCollectorTests.cs @@ -23,7 +23,7 @@ public class IntegrationTelemetryCollectorTests public void HasChangesWhenNewIntegrationRunning() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -36,7 +36,7 @@ public void HasChangesWhenNewIntegrationRunning() public void DoesNotHaveChangesWhenSameIntegrationRunning() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -53,7 +53,7 @@ public void DoesNotHaveChangesWhenSameIntegrationRunning() public void HasChangesWhenNewIntegrationGeneratedSpan() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -66,7 +66,7 @@ public void HasChangesWhenNewIntegrationGeneratedSpan() public void DoesNotHaveChangesWhenSameIntegrationGeneratedSpan() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -83,7 +83,7 @@ public void DoesNotHaveChangesWhenSameIntegrationGeneratedSpan() public void HasChangesWhenNewIntegrationDisabled() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -96,7 +96,7 @@ public void HasChangesWhenNewIntegrationDisabled() public void DoesNotHaveChangesWhenSameIntegrationDisabled() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.GetData(); collector.HasChanges().Should().BeFalse(); @@ -113,7 +113,7 @@ public void DoesNotHaveChangesWhenSameIntegrationDisabled() public void WhenIntegrationRunsSuccessfullyHasExpectedValues() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.IntegrationRunning(IntegrationId); collector.IntegrationGeneratedSpan(IntegrationId); @@ -130,7 +130,7 @@ public void WhenIntegrationRunsSuccessfullyHasExpectedValues() public void WhenIntegrationRunsButDoesNotGenerateSpanHasExpectedValues() { var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.IntegrationRunning(IntegrationId); @@ -147,7 +147,7 @@ public void WhenIntegrationErrorsHasExpectedValues() { const string error = "Some error"; var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.IntegrationRunning(IntegrationId); collector.IntegrationDisabledDueToError(IntegrationId, error); @@ -165,7 +165,7 @@ public void WhenIntegrationRunsThenErrorsHasExpectedValues() { const string error = "Some error"; var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); collector.IntegrationRunning(IntegrationId); collector.IntegrationGeneratedSpan(IntegrationId); @@ -184,7 +184,7 @@ public void OnlyIncludesChangedValues() { const string error = "Some error"; var collector = new IntegrationTelemetryCollector(); - collector.RecordTracerSettings(new TracerSettings()); + collector.RecordTracerSettings(new TracerSettings().Manager.InitialMutableSettings); // first call collector.GetData().Should().NotBeEmpty(); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs index 7bdb7803518b..493fa8447a4e 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs @@ -38,6 +38,7 @@ public async Task TelemetryControllerShouldSendTelemetry() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -45,7 +46,6 @@ public async Task TelemetryControllerShouldSendTelemetry() transportManager, _flushInterval); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); var data = await WaitForRequestStarted(transport, _timeout); @@ -59,6 +59,7 @@ public async Task TelemetryControllerShouldSendGitMetadataWithTelemetry() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -69,7 +70,6 @@ public async Task TelemetryControllerShouldSendGitMetadataWithTelemetry() var sha = "testCommitSha"; var repo = "testRepositoryUrl"; controller.RecordGitMetadata(new GitMetadata(sha, repo)); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); var data = await WaitForRequestStarted(transport, _timeout); @@ -108,6 +108,7 @@ public async Task TelemetryControllerShouldUpdateGitMetadataWithTelemetry() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -115,7 +116,6 @@ public async Task TelemetryControllerShouldUpdateGitMetadataWithTelemetry() transportManager, _flushInterval); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); var data = await WaitForRequestStarted(transport, _timeout); @@ -142,7 +142,9 @@ public async Task TelemetryControllerRecordsConfigurationFromTracerSettings() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var collector = new ConfigurationTelemetry(); + var settings = new TracerSettings(); var controller = new TelemetryController( + settings, collector, new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -150,9 +152,6 @@ public async Task TelemetryControllerRecordsConfigurationFromTracerSettings() transportManager, _flushInterval); - var settings = new TracerSettings(); - controller.RecordTracerSettings(settings, "DefaultServiceName"); - // Just basic check that we have the same number of config values var configCount = settings.Telemetry.Should() .BeOfType() @@ -174,6 +173,7 @@ public async Task TelemetryControllerCanBeDisposedTwice() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -192,6 +192,7 @@ public async Task TelemetrySendsHeartbeatAlongWithData() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -199,7 +200,6 @@ public async Task TelemetrySendsHeartbeatAlongWithData() transportManager, _flushInterval); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); var requiredHeartbeats = 10; @@ -237,6 +237,7 @@ public async Task TelemetryControllerAddsAllAssembliesToCollector() // creating a new controller so we have the same list of assemblies var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -244,7 +245,6 @@ public async Task TelemetryControllerAddsAllAssembliesToCollector() transportManager, _flushInterval); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); var allData = await WaitForRequestStarted(transport, _timeout); @@ -276,6 +276,7 @@ public async Task TelemetryControllerRecordsAppEndpoints() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -283,7 +284,6 @@ public async Task TelemetryControllerRecordsAppEndpoints() transportManager, _flushInterval); - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.RecordAppEndpoints(new List { new("GET", "/api/test"), @@ -322,6 +322,7 @@ public async Task TelemetryControllerDumpsAllTelemetryToFile() var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( + new TracerSettings(), new ConfigurationTelemetry(), new DependencyTelemetryCollector(), new NullMetricsTelemetryCollector(), @@ -335,7 +336,6 @@ public async Task TelemetryControllerDumpsAllTelemetryToFile() File.ReadAllText(tempFile).Should().BeNullOrEmpty(); // after starting telemetry - controller.RecordTracerSettings(new TracerSettings(), "DefaultServiceName"); controller.Start(); await WaitForRequestStarted(transport, _timeout); From 0d48128b51e0c35aaa757dc95d1b8d309afceada Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 23 Oct 2025 14:05:21 +0100 Subject: [PATCH 10/29] Fix Tracer and TracerManager --- .../ClrProfiler/Instrumentation.cs | 10 +++-- tracer/src/Datadog.Trace/Tracer.cs | 4 +- tracer/src/Datadog.Trace/TracerManager.cs | 42 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index 7beb27571025..d43a42ec14a4 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -84,6 +84,8 @@ private static void PropagateStableConfiguration() Log.Debug("Setting Stable Configuration in Continuous Profiler native library."); var tracer = Tracer.Instance; var tracerSettings = tracer.Settings; + var mutableSettings = tracerSettings.Manager.InitialMutableSettings; + NativeInterop.SharedConfig config = new NativeInterop.SharedConfig { ProfilingEnabled = profilerSettings.ProfilerState switch @@ -93,14 +95,14 @@ private static void PropagateStableConfiguration() _ => NativeInterop.ProfilingEnabled.Disabled }, - TracingEnabled = tracerSettings.MutableSettings.TraceEnabled, + TracingEnabled = mutableSettings.TraceEnabled, IastEnabled = Iast.Iast.Instance.Settings.Enabled, RaspEnabled = Security.Instance.Settings.RaspEnabled, DynamicInstrumentationEnabled = false, // TODO: find where to get this value from but for the other native p/invoke call RuntimeId = RuntimeId.Get(), - Environment = tracerSettings.MutableSettings.Environment, - ServiceName = tracer.DefaultServiceName, - Version = tracerSettings.MutableSettings.ServiceVersion + Environment = mutableSettings.Environment, + ServiceName = mutableSettings.DefaultServiceName, + Version = mutableSettings.ServiceVersion }; // Make sure nothing bubbles up, even if there are issues diff --git a/tracer/src/Datadog.Trace/Tracer.cs b/tracer/src/Datadog.Trace/Tracer.cs index 6d2c142badae..8c726376bd7d 100644 --- a/tracer/src/Datadog.Trace/Tracer.cs +++ b/tracer/src/Datadog.Trace/Tracer.cs @@ -416,12 +416,12 @@ internal Span StartSpan(string operationName, ITags tags = null, ISpanContext pa }; // Apply any global tags - if (Settings.MutableSettings.GlobalTags.Count > 0) + if (CurrentTraceSettings.Settings.GlobalTags is { Count: > 0 } globalTags) { // if DD_TAGS contained "env", "version", "git.commit.sha", or "git.repository.url", they were used to set // ImmutableTracerSettings.Environment, ImmutableTracerSettings.ServiceVersion, ImmutableTracerSettings.GitCommitSha, and ImmutableTracerSettings.GitRepositoryUrl // and removed from Settings.GlobalTags - foreach (var entry in Settings.MutableSettings.GlobalTags) + foreach (var entry in globalTags) { span.SetTag(entry.Key, entry.Value); } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 488ecab65c0b..61dd6fdc188a 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -52,6 +52,8 @@ internal class TracerManager private static bool _globalInstanceInitialized; private static object _globalInstanceLock = new(); + private readonly IDisposable _settingSubscription; + private PerTraceSettings _perTraceSettings; private volatile bool _isClosing = false; public TracerManager( @@ -101,10 +103,21 @@ public TracerManager( TracerFlareManager = tracerFlareManager; SpanEventsManager = new SpanEventsManager(discoveryService); - var schema = new NamingSchema(settings.MetadataSchemaVersion, settings.PeerServiceTagsEnabled, settings.RemoveClientServiceNamesEnabled, settings.MutableSettings.DefaultServiceName, settings.MutableSettings.ServiceNameMappings, settings.PeerServiceNameMappings); - PerTraceSettings = new(traceSampler, spanSampler, schema, settings.MutableSettings); - SpanContextPropagator = SpanContextPropagatorFactory.GetSpanContextPropagator(settings.PropagationStyleInject, settings.PropagationStyleExtract, settings.PropagationExtractFirstOnly, settings.PropagationBehaviorExtract); + UpdatePerTraceSettings(settings.Manager.InitialMutableSettings); + _settingSubscription = settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdatePerTraceSettings(mutable); + } + }); + + void UpdatePerTraceSettings(MutableSettings mutableSettings) + { + var schema = new NamingSchema(settings.MetadataSchemaVersion, settings.PeerServiceTagsEnabled, settings.RemoveClientServiceNamesEnabled, mutableSettings.DefaultServiceName, mutableSettings.ServiceNameMappings, settings.PeerServiceNameMappings); + Interlocked.Exchange(ref _perTraceSettings, new(traceSampler, spanSampler, schema, mutableSettings)); + } } /// @@ -163,7 +176,7 @@ public static TracerManager Instance public ISpanEventsManager SpanEventsManager { get; } - public PerTraceSettings PerTraceSettings { get; } + public PerTraceSettings PerTraceSettings => Volatile.Read(ref _perTraceSettings); public SpanContextPropagator SpanContextPropagator { get; } @@ -233,6 +246,7 @@ private static async Task CleanUpOldTracerManager(TracerManager oldManager, Trac { try { + oldManager._settingSubscription.Dispose(); var agentWriterReplaced = false; if (oldManager.AgentWriter != newManager.AgentWriter && oldManager.AgentWriter is not null) { @@ -687,25 +701,6 @@ private static void OneTimeSetup(TracerSettings tracerSettings) // Record the service discovery metadata ServiceDiscoveryHelper.StoreTracerMetadata(tracerSettings, tracerSettings.Manager.InitialMutableSettings); - - // Register for rebuilding the settings on changes - // TODO: This is only temporary, we want to _stop_ rebuilding everything whenever settings change in the future - // We also don't bother to dispose this because we never unsubscribe - tracerSettings.Manager.SubscribeToChanges(updatedSettings => - { - var newSettings = updatedSettings switch - { - { UpdatedExporter: { } e, UpdatedMutable: { } m } => Tracer.Instance.Settings with { Exporter = e, MutableSettings = m }, - { UpdatedExporter: { } e } => Tracer.Instance.Settings with { Exporter = e }, - { UpdatedMutable: { } m } => Tracer.Instance.Settings with { MutableSettings = m }, - _ => null, - }; - if (newSettings != null) - { - // Update the global instance - Trace.Tracer.Configure(newSettings); - } - }); } private static Task RunShutdownTasksAsync(Exception ex) => RunShutdownTasksAsync(_instance, _heartbeatTimer); @@ -726,6 +721,7 @@ private static async Task RunShutdownTasksAsync(TracerManager instance, Timer he if (instance is not null) { + instance._settingSubscription.Dispose(); Log.Debug("Disposing DynamicConfigurationManager"); instance.DynamicConfigurationManager.Dispose(); Log.Debug("Disposing TracerFlareManager"); From 0c39f4282f1578490757b571ef65ff0d282aec29 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 24 Oct 2025 10:08:33 +0100 Subject: [PATCH 11/29] Update statsd handling in runtime metrics - Move statsd instance creation to separate factory - Create a StatsdManager to handle automatic updating in response to setting changes - Always create a statsd instance, as it's hard to know if we're _ever_ going to need one, and reduces some of the compexity --- .../Datadog.Trace/Debugger/DebuggerFactory.cs | 9 +- .../src/Datadog.Trace/DogStatsd/NoOpStatsd.cs | 2 + .../Datadog.Trace/DogStatsd/StatsdFactory.cs | 99 ++++++++++++ .../Datadog.Trace/DogStatsd/StatsdManager.cs | 143 ++++++++++++++++++ .../src/Datadog.Trace/TracerManagerFactory.cs | 37 ++--- .../Datadog.Trace.Tests/DogStatsDTests.cs | 27 +++- .../DogStatsd/StatsdManagerTests.cs | 100 ++++++++++++ 7 files changed, 387 insertions(+), 30 deletions(-) create mode 100644 tracer/src/Datadog.Trace/DogStatsd/StatsdFactory.cs create mode 100644 tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs create mode 100644 tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs index 3d36aacd574a..906ef05d6391 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs @@ -63,11 +63,16 @@ private static IDogStatsd GetDogStatsd(TracerSettings tracerSettings, string ser && tracerSettings.Exporter.MetricsTransport == TransportType.UDS) { Log.Information("Metric probes are not supported on Windows when transport type is UDS"); - statsd = new NoOpStatsd(); + statsd = NoOpStatsd.Instance; } else { - statsd = TracerManagerFactory.CreateDogStatsdClient(tracerSettings, serviceName, constantTags: null, prefix: DebuggerSettings.DebuggerMetricPrefix, telemtryFlushInterval: null); + // TODO: use StatsdManager to get automatic updating on exporter and other setting changes + statsd = StatsdFactory.CreateDogStatsdClient( + tracerSettings.Manager.InitialMutableSettings, + tracerSettings.Exporter, + includeDefaultTags: false, + prefix: DebuggerSettings.DebuggerMetricPrefix); } return statsd; diff --git a/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs b/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs index cbb0372bcada..98d840752d80 100644 --- a/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs +++ b/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs @@ -10,6 +10,8 @@ namespace Datadog.Trace.DogStatsd { internal class NoOpStatsd : IDogStatsd { + public static readonly NoOpStatsd Instance = new(); + public ITelemetryCounters TelemetryCounters => null; public void Configure(StatsdConfig config) diff --git a/tracer/src/Datadog.Trace/DogStatsd/StatsdFactory.cs b/tracer/src/Datadog.Trace/DogStatsd/StatsdFactory.cs new file mode 100644 index 000000000000..37f25162c6b2 --- /dev/null +++ b/tracer/src/Datadog.Trace/DogStatsd/StatsdFactory.cs @@ -0,0 +1,99 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; +using System.Collections.Generic; +using Datadog.Trace.Configuration; +using Datadog.Trace.Logging; +using Datadog.Trace.Processors; +using Datadog.Trace.Vendors.StatsdClient; +using Datadog.Trace.Vendors.StatsdClient.Transport; + +namespace Datadog.Trace.DogStatsd; + +internal static class StatsdFactory +{ + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(StatsdFactory)); + + internal static IDogStatsd CreateDogStatsdClient(MutableSettings settings, ExporterSettings exporter, bool includeDefaultTags, string? prefix = null) + { + if (includeDefaultTags) + { + var customTagCount = settings.GlobalTags.Count; + var constantTags = new List(5 + customTagCount) + { + "lang:.NET", + $"lang_interpreter:{FrameworkDescription.Instance.Name}", + $"lang_version:{FrameworkDescription.Instance.ProductVersion}", + $"tracer_version:{TracerConstants.AssemblyVersion}", + $"{Tags.RuntimeId}:{Tracer.RuntimeId}" + }; + + if (customTagCount > 0) + { + var tagProcessor = new TruncatorTagsProcessor(); + foreach (var kvp in settings.GlobalTags) + { + var key = kvp.Key; + var value = kvp.Value; + tagProcessor.ProcessMeta(ref key, ref value); + constantTags.Add($"{key}:{value}"); + } + } + + return CreateDogStatsdClient(settings, exporter, constantTags, prefix); + } + + return CreateDogStatsdClient(settings, exporter, constantTags: null, prefix); + } + + private static IDogStatsd CreateDogStatsdClient(MutableSettings settings, ExporterSettings exporter, List? constantTags, string? prefix = null) + { + try + { + var statsd = new DogStatsdService(); + var config = new StatsdConfig + { + ConstantTags = constantTags is not null ? [..constantTags] : [], + Prefix = prefix, + // note that if these are null, statsd tries to grab them directly from the environment, which could be unsafe + ServiceName = NormalizerTraceProcessor.NormalizeService(settings.DefaultServiceName), + Environment = settings.Environment, + ServiceVersion = settings.ServiceVersion, + // Force flush interval to null to avoid ever sending telemetry, as these are recorded as custom metrics + Advanced = { TelemetryFlushInterval = null }, + }; + + switch (exporter.MetricsTransport) + { + case TransportType.NamedPipe: + config.PipeName = exporter.MetricsPipeName; + Log.Information("Using windows named pipes for metrics transport: {PipeName}", config.PipeName); + break; +#if NETCOREAPP3_1_OR_GREATER + case TransportType.UDS: + config.StatsdServerName = $"{ExporterSettings.UnixDomainSocketPrefix}{exporter.MetricsUnixDomainSocketPath}"; + Log.Information("Using unix domain sockets for metrics transport: {Socket}", config.StatsdServerName); + break; +#endif + case TransportType.UDP: + default: + config.StatsdServerName = exporter.MetricsHostname; + config.StatsdPort = exporter.DogStatsdPort; + Log.Information("Using UDP for metrics transport: {Hostname}:{Port}", config.StatsdServerName, config.StatsdPort); + break; + } + + statsd.Configure(config); + return statsd; + } + catch (Exception ex) + { + Log.Error(ex, "Unable to instantiate StatsD client"); + return new NoOpStatsd(); + } + } +} diff --git a/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs new file mode 100644 index 000000000000..999a4bd95246 --- /dev/null +++ b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs @@ -0,0 +1,143 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Configuration; +using Datadog.Trace.Logging; +using Datadog.Trace.Vendors.StatsdClient; + +namespace Datadog.Trace.DogStatsd; + +/// +/// This acts as a wrapper around a "real" service or a client, +/// but which responds to changes in settings caused by remote config or configuration in code. +/// +internal sealed class StatsdManager : IDogStatsd +{ + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private readonly object _lock = new(); + private readonly IDisposable _settingSubscription; + private IDogStatsd _current; + private bool _isDisposed; + + public StatsdManager(TracerSettings tracerSettings, bool includeDefaultTags) + { + _current = StatsdFactory.CreateDogStatsdClient( + tracerSettings.Manager.InitialMutableSettings, + tracerSettings.Manager.InitialExporterSettings, + includeDefaultTags); + + _settingSubscription = tracerSettings.Manager.SubscribeToChanges(c => + { + // To avoid expensive unnecessary replacements, we only rebuild the statsd instance + // if something changes that could impact it. In other words, if StatsdFactory uses + // a value then we should check if it's changed here + if (!HasImpactingChanges(c)) + { + return; + } + + IDogStatsd previous; + + lock (_lock) + { + if (_isDisposed) + { + return; + } + + previous = _current; + _current = StatsdFactory.CreateDogStatsdClient( + c.UpdatedMutable ?? c.PreviousMutable, + c.UpdatedExporter ?? c.PreviousExporter, + includeDefaultTags); + } + + if (previous is DogStatsdService dogStatsdService) + { + // Kick off disposal in the background after a delay to make sure everything is flushed + // There's a risk that something could have grabbed the instance, and if something + // tries to write to the client, it will throw an exception. + Task.Run(async () => + { + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + dogStatsdService.Flush(); + dogStatsdService.Dispose(); + }) + .ContinueWith(t => Log.Error(t.Exception, "There was an error disposing the statsd client"), TaskContinuationOptions.OnlyOnFaulted); + } + }); + } + + // Delegated implementation + public ITelemetryCounters TelemetryCounters => Volatile.Read(ref _current).TelemetryCounters; + + public void Configure(StatsdConfig config) => Volatile.Read(ref _current).Configure(config); + + public void Counter(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Counter(statName, value, sampleRate, tags); + + public void Decrement(string statName, int value = 1, double sampleRate = 1, params string[] tags) => Volatile.Read(ref _current).Decrement(statName, value, sampleRate, tags); + + public void Event(string title, string text, string? alertType = null, string? aggregationKey = null, string? sourceType = null, int? dateHappened = null, string? priority = null, string? hostname = null, string[]? tags = null) => Volatile.Read(ref _current).Event(title, text, alertType, aggregationKey, sourceType, dateHappened, priority, hostname, tags); + + public void Gauge(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Gauge(statName, value, sampleRate, tags); + + public void Histogram(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Histogram(statName, value, sampleRate, tags); + + public void Distribution(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Distribution(statName, value, sampleRate, tags); + + public void Increment(string statName, int value = 1, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Increment(statName, value, sampleRate, tags); + + public void Set(string statName, T value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Set(statName, value, sampleRate, tags); + + public void Set(string statName, string value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Set(statName, value, sampleRate, tags); + + public IDisposable StartTimer(string name, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).StartTimer(name, sampleRate, tags); + + public void Time(Action action, string statName, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Time(action, statName, sampleRate, tags); + + public T Time(Func func, string statName, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Time(func, statName, sampleRate, tags); + + public void Timer(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Timer(statName, value, sampleRate, tags); + + public void ServiceCheck(string name, Status status, int? timestamp = null, string? hostname = null, string[]? tags = null, string? message = null) => Volatile.Read(ref _current).ServiceCheck(name, status, timestamp, hostname, tags, message); + + public void Dispose() + { + IDogStatsd previous; + lock (_lock) + { + _isDisposed = true; + _settingSubscription.Dispose(); + previous = _current; + _current = NoOpStatsd.Instance; + } + + // Given we're shutting down at this point, it doesn't seem like it's worth actually disposing + // the instance (and risking an error) so we just flush to make sure everything is sent + if (previous is DogStatsdService dogStatsdService) + { + dogStatsdService.Flush(); + } + } + + // Internal for testing + internal static bool HasImpactingChanges(TracerSettings.SettingsManager.SettingChanges changes) + { + var hasChanges = changes.UpdatedExporter is not null // relying on this to only be non null if _anything_ changed + || (changes.UpdatedMutable is { } updated + && !( + string.Equals(updated.Environment, changes.PreviousMutable.Environment, StringComparison.Ordinal) + && string.Equals(updated.ServiceVersion, changes.PreviousMutable.ServiceVersion, StringComparison.Ordinal) + // The service name comparison isn't _strictly_ correct, because we normalize it further, but this is probably good enough + && string.Equals(updated.DefaultServiceName, changes.PreviousMutable.DefaultServiceName, StringComparison.OrdinalIgnoreCase) + && updated.GlobalTags.SequenceEqual(changes.PreviousMutable.GlobalTags))); + return hasChanges; + } +} diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index dbbf97cc89ed..449caf4d1e57 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -130,28 +130,19 @@ internal TracerManager CreateTracerManager( discoveryService ??= GetDiscoveryService(settings); - bool runtimeMetricsEnabled = settings.RuntimeMetricsEnabled && !DistributedTracer.Instance.IsChildTracer; + // Technically we don't _always_ need a dogstatsd instance, because we only need it if runtime metrics + // are enabled _or_ tracer metrics are enabled. However, tracer metrics can be enabled and disabled dynamically + // at runtime, which makes managing the lifetime of the statsd instance more complex than we'd like, so + // for simplicity, we _always_ create a new statsd instance + statsd ??= new StatsdManager(settings, includeDefaultTags: true); + runtimeMetrics ??= settings.RuntimeMetricsEnabled && !DistributedTracer.Instance.IsChildTracer + ? new RuntimeMetricsWriter(statsd, TimeSpan.FromSeconds(10), settings.IsRunningInAzureAppService) + : null; - statsd = (settings.MutableSettings.TracerMetricsEnabled || runtimeMetricsEnabled) - ? (statsd ?? CreateDogStatsdClient(settings, defaultServiceName)) - : null; sampler ??= GetSampler(settings); agentWriter ??= GetAgentWriter(settings, settings.MutableSettings.TracerMetricsEnabled ? statsd : null, rates => sampler.SetDefaultSampleRates(rates), discoveryService); scopeManager ??= new AsyncLocalScopeManager(); - if (runtimeMetricsEnabled && runtimeMetrics is { }) - { - runtimeMetrics.UpdateStatsd(statsd); - } - else if (runtimeMetricsEnabled) - { - runtimeMetrics = new RuntimeMetricsWriter(statsd, TimeSpan.FromSeconds(10), settings.IsRunningInAzureAppService); - } - else - { - runtimeMetrics = null; - } - telemetry ??= CreateTelemetryController(settings, discoveryService); var gitMetadataTagsProvider = GetGitMetadataTagsProvider(settings, settings.Manager.InitialMutableSettings, scopeManager, telemetry); @@ -449,6 +440,12 @@ private static string GetUrl(TracerSettings settings) } } + // internal for testing + internal virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) + => settings.AgentFeaturePollingEnabled ? + DiscoveryService.Create(settings.Exporter) : + NullDiscoveryService.Instance; + internal static IDogStatsd CreateDogStatsdClient(TracerSettings settings, string serviceName, List constantTags, string prefix = null, TimeSpan? telemtryFlushInterval = null) { try @@ -495,12 +492,6 @@ internal static IDogStatsd CreateDogStatsdClient(TracerSettings settings, string } } - // internal for testing - internal virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) - => settings.AgentFeaturePollingEnabled ? - DiscoveryService.Create(settings.Exporter) : - NullDiscoveryService.Instance; - private static IDogStatsd CreateDogStatsdClient(TracerSettings settings, string serviceName) { var customTagCount = settings.MutableSettings.GlobalTags.Count; diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs index 1d6c3a842087..24cbadb17667 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs @@ -48,7 +48,8 @@ public async Task Do_not_send_metrics_when_disabled() Assert.True(spans.Count == 1, AssertionFailureMessage(1, spans)); - // no methods should be called on the IStatsd + // no methods should be called on the IStatsd other than dispose + statsd.Verify(s => s.Dispose(), Times.Once); statsd.VerifyNoOtherCalls(); } @@ -134,7 +135,11 @@ public void CanCreateDogStatsD_UDP_FromTraceAgentSettings(string agentUri, strin // No guarantees it's actually using the _right_ config here, but it's better than nothing using var agent = MockTracerAgent.Create(_output, useStatsd: true, requestedStatsDPort: expectedPort); - var dogStatsD = TracerManagerFactory.CreateDogStatsdClient(settings, "test service", null); + var dogStatsD = StatsdFactory.CreateDogStatsdClient( + settings.Manager.InitialMutableSettings, + settings.Manager.InitialExporterSettings, + includeDefaultTags: true, + prefix: null); // If there's an error during configuration, we get a no-op instance, so using this as a test dogStatsD.Should() @@ -159,7 +164,11 @@ public void CanCreateDogStatsD_NamedPipes_FromTraceAgentSettings() // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing - var dogStatsD = TracerManagerFactory.CreateDogStatsdClient(settings, "test service", null); + var dogStatsD = StatsdFactory.CreateDogStatsdClient( + settings.Manager.InitialMutableSettings, + settings.Manager.InitialExporterSettings, + includeDefaultTags: true, + prefix: null); // If there's an error during configuration, we get a no-op instance, so using this as a test dogStatsD.Should() @@ -189,7 +198,11 @@ public void CanCreateDogStatsD_UDS_FromTraceAgentSettings() // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing - var dogStatsD = TracerManagerFactory.CreateDogStatsdClient(settings, "test service", null); + var dogStatsD = StatsdFactory.CreateDogStatsdClient( + settings.Manager.InitialMutableSettings, + settings.Manager.InitialExporterSettings, + includeDefaultTags: true, + prefix: null); // If there's an error during configuration, we get a no-op instance, so using this as a test dogStatsD.Should() @@ -217,7 +230,11 @@ public void CanCreateDogStatsD_UDS_FallsBackToUdp_FromTraceAgentSettings() // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing - var dogStatsD = TracerManagerFactory.CreateDogStatsdClient(settings, "test service", null); + var dogStatsD = StatsdFactory.CreateDogStatsdClient( + settings.Manager.InitialMutableSettings, + settings.Manager.InitialExporterSettings, + includeDefaultTags: true, + prefix: null); // If there's an error during configuration, we get a no-op instance, so using this as a test dogStatsD.Should() diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs new file mode 100644 index 000000000000..e5bfe5096fc3 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs @@ -0,0 +1,100 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.DogStatsd; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.DogStatsd; + +public class StatsdManagerTests +{ + private static readonly TracerSettings TracerSettings = new(); + private static readonly MutableSettings PreviousMutable = MutableSettings.CreateForTesting(TracerSettings, []); + private static readonly ExporterSettings PreviousExporter = new ExporterSettings(null); + + [Fact] + public void HasImpactingChanges_WhenNoChanges() + { + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: null, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeFalse(); + } + + [Fact] + public void HasImpactingChanges_WhenNoChanges2() + { + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: PreviousMutable, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeFalse(); + } + + [Fact] + public void HasImpactingChanges_WhenExporterChanges() + { + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: null, + updatedExporter: PreviousExporter, // We don't check for "real" differences, assume all changes matter + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); + } + + [Fact] + public void HasImpactingChanges_WhenMutableChangesEnv() + { + var newSettings = MutableSettings.CreateForTesting(TracerSettings, new() { { ConfigurationKeys.Environment, "new" } }); + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: newSettings, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); + } + + [Fact] + public void HasImpactingChanges_WhenMutableChangesServiceName() + { + var newSettings = MutableSettings.CreateForTesting(TracerSettings, new() { { ConfigurationKeys.ServiceName, "service" } }); + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: newSettings, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); + } + + [Fact] + public void HasImpactingChanges_WhenMutableChangesServiceVersion() + { + var newSettings = MutableSettings.CreateForTesting(TracerSettings, new() { { ConfigurationKeys.ServiceVersion, "1.0.0" } }); + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: newSettings, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); + } + + [Fact] + public void HasImpactingChanges_WhenMutableChangesGlobalTags() + { + var newSettings = MutableSettings.CreateForTesting(TracerSettings, new() { { ConfigurationKeys.GlobalTags, "a:b" } }); + var changes = new TracerSettings.SettingsManager.SettingChanges( + updatedMutable: newSettings, + updatedExporter: null, + PreviousMutable, + PreviousExporter); + StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); + } +} From 64849a7b2f25817dedddb42d6047a5b2dd0ff0f7 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 24 Oct 2025 11:19:03 +0100 Subject: [PATCH 12/29] Update ITraceSampler to respond to setting changes --- .../Sampling/ManagedTraceSampler.cs | 159 ++++++++++++++++++ .../src/Datadog.Trace/TracerManagerFactory.cs | 74 +------- 2 files changed, 163 insertions(+), 70 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs diff --git a/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs b/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs new file mode 100644 index 000000000000..d29d5930537a --- /dev/null +++ b/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs @@ -0,0 +1,159 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using Datadog.Trace.Configuration; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Sampling; + +/// +/// An implementation of that handles rebuilding the trace sampler on changes. +/// +/// +internal sealed class ManagedTraceSampler : ITraceSampler +{ + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private readonly object _lock = new(); + private IReadOnlyDictionary? _defaultSampleRates; + private TraceSampler _current; + + public ManagedTraceSampler(TracerSettings settings) + { + _current = CreateSampler(settings.Manager.InitialMutableSettings, settings.CustomSamplingRulesFormat); + // Sampler lifetime is same as app lifetime, so don't bother worrying about disposal. + settings.Manager.SubscribeToChanges(changes => + { + // only update the sampling rules if there are changes to things we care about + if (changes.UpdatedMutable is { } updated + && (updated.MaxTracesSubmittedPerSecond != changes.PreviousMutable.MaxTracesSubmittedPerSecond + || updated.CustomSamplingRulesIsRemote != changes.PreviousMutable.CustomSamplingRulesIsRemote + || !string.Equals(updated.CustomSamplingRules, changes.PreviousMutable.CustomSamplingRules) + || AreDifferent(updated.GlobalSamplingRate, changes.PreviousMutable.GlobalSamplingRate))) + { + var newSampler = CreateSampler(changes.UpdatedMutable, settings.CustomSamplingRulesFormat); + // Use a lock to avoid edge cases with setting the default sample rates + lock (_lock) + { + if (_defaultSampleRates is { } rates) + { + newSampler.SetDefaultSampleRates(rates); + } + + _current = newSampler; + } + } + }); + + static bool AreDifferent(double? a, double? b) + { + if (a is null && b is null) + { + return false; + } + + if (a is null || b is null) + { + return true; + } + + // Absolute comparisons of floating points are bad, so use a tolerance + return Math.Abs(a.Value - b.Value) > 0.00001f; + } + } + + public bool HasResourceBasedSamplingRule => Volatile.Read(ref _current).HasResourceBasedSamplingRule; + + public void SetDefaultSampleRates(IReadOnlyDictionary sampleRates) + { + // lock to avoid missed sample rate updates when updating inner sample + lock (_lock) + { + // save the rates for if/when we rebuild on setting changes + _defaultSampleRates = sampleRates; + _current.SetDefaultSampleRates(sampleRates); + } + } + + public SamplingDecision MakeSamplingDecision(Span span) => Volatile.Read(ref _current).MakeSamplingDecision(span); + + private static TraceSampler CreateSampler(MutableSettings settings, string customSamplingRulesFormat) + { + // ISamplingRule is used to implement, in order of precedence: + // - custom sampling rules + // - remote custom rules (provenance: "customer") + // - remote dynamic rules (provenance: "dynamic") + // - local custom rules (provenance: "local"/none) = DD_TRACE_SAMPLING_RULES + // - global sampling rate + // - remote + // - local = DD_TRACE_SAMPLE_RATE + // - agent sampling rates (as a single rule) + + // Note: the order that rules are registered is important, as they are evaluated in order. + // The first rule that matches will be used to determine the sampling rate. + + var sampler = new TraceSampler.Builder(new TracerRateLimiter(maxTracesPerInterval: settings.MaxTracesSubmittedPerSecond, intervalMilliseconds: null)); + + // sampling rules (remote value overrides local value) + var samplingRulesJson = settings.CustomSamplingRules; + + // check if the rules are remote or local because they have different JSON schemas + if (settings.CustomSamplingRulesIsRemote) + { + // remote sampling rules + if (!StringUtil.IsNullOrWhiteSpace(samplingRulesJson)) + { + var remoteSamplingRules = + RemoteCustomSamplingRule.BuildFromConfigurationString( + samplingRulesJson, + RegexBuilder.DefaultTimeout); + + sampler.RegisterRules(remoteSamplingRules); + } + } + else + { + // local sampling rules + var patternFormatIsValid = SamplingRulesFormat.IsValid(customSamplingRulesFormat, out var samplingRulesFormat); + + if (patternFormatIsValid && !StringUtil.IsNullOrWhiteSpace(samplingRulesJson)) + { + var localSamplingRules = + LocalCustomSamplingRule.BuildFromConfigurationString( + samplingRulesJson, + samplingRulesFormat, + RegexBuilder.DefaultTimeout); + + sampler.RegisterRules(localSamplingRules); + } + } + + // global sampling rate (remote value overrides local value) + if (settings.GlobalSamplingRate is { } globalSamplingRate) + { + if (globalSamplingRate is < 0f or > 1f) + { + Log.Warning( + "{ConfigurationKey} configuration of {ConfigurationValue} is out of range", + ConfigurationKeys.GlobalSamplingRate, + settings.GlobalSamplingRate); + } + else + { + sampler.RegisterRule(new GlobalSamplingRateRule((float)globalSamplingRate)); + } + } + + // AgentSamplingRule handles all sampling rates received from the agent as a single "rule". + // This rule is always present, even if the agent has not yet provided any sampling rates. + sampler.RegisterAgentSamplingRule(new AgentSamplingRule()); + + return sampler.Build(); + } +} diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 449caf4d1e57..362e3eb2003f 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -254,19 +254,9 @@ protected virtual TracerManager CreateTracerManagerFrom( protected virtual ITraceSampler GetSampler(TracerSettings settings) { - // ISamplingRule is used to implement, in order of precedence: - // - custom sampling rules - // - remote custom rules (provenance: "customer") - // - remote dynamic rules (provenance: "dynamic") - // - local custom rules (provenance: "local"/none) = DD_TRACE_SAMPLING_RULES - // - global sampling rate - // - remote - // - local = DD_TRACE_SAMPLE_RATE - // - agent sampling rates (as a single rule) - - // Note: the order that rules are registered is important, as they are evaluated in order. - // The first rule that matches will be used to determine the sampling rate. - + // TODO: This may need to be updated to be dynamic, and to handle changes to enablement + // e.g. AppSec can be enabled/disabled dynamically, which could change this flag at runtime, + // leaving us with the wrong sampler in place if (settings.ApmTracingEnabled == false && (Security.Instance.Settings.AppsecEnabled || Iast.Iast.Instance.Settings.Enabled)) { @@ -276,63 +266,7 @@ protected virtual ITraceSampler GetSampler(TracerSettings settings) return samplerStandalone.Build(); } - var sampler = new TraceSampler.Builder(new TracerRateLimiter(maxTracesPerInterval: settings.MutableSettings.MaxTracesSubmittedPerSecond, intervalMilliseconds: null)); - - // sampling rules (remote value overrides local value) - var samplingRulesJson = settings.MutableSettings.CustomSamplingRules; - - // check if the rules are remote or local because they have different JSON schemas - if (settings.MutableSettings.CustomSamplingRulesIsRemote) - { - // remote sampling rules - if (!string.IsNullOrWhiteSpace(samplingRulesJson)) - { - var remoteSamplingRules = - RemoteCustomSamplingRule.BuildFromConfigurationString( - samplingRulesJson, - RegexBuilder.DefaultTimeout); - - sampler.RegisterRules(remoteSamplingRules); - } - } - else - { - // local sampling rules - var patternFormatIsValid = SamplingRulesFormat.IsValid(settings.CustomSamplingRulesFormat, out var samplingRulesFormat); - - if (patternFormatIsValid && !string.IsNullOrWhiteSpace(samplingRulesJson)) - { - var localSamplingRules = - LocalCustomSamplingRule.BuildFromConfigurationString( - samplingRulesJson, - samplingRulesFormat, - RegexBuilder.DefaultTimeout); - - sampler.RegisterRules(localSamplingRules); - } - } - - // global sampling rate (remote value overrides local value) - if (settings.MutableSettings.GlobalSamplingRate is { } globalSamplingRate) - { - if (globalSamplingRate is < 0f or > 1f) - { - Log.Warning( - "{ConfigurationKey} configuration of {ConfigurationValue} is out of range", - ConfigurationKeys.GlobalSamplingRate, - settings.MutableSettings.GlobalSamplingRate); - } - else - { - sampler.RegisterRule(new GlobalSamplingRateRule((float)globalSamplingRate)); - } - } - - // AgentSamplingRule handles all sampling rates received from the agent as a single "rule". - // This rule is always present, even if the agent has not yet provided any sampling rates. - sampler.RegisterAgentSamplingRule(new AgentSamplingRule()); - - return sampler.Build(); + return new ManagedTraceSampler(settings); } protected virtual ISpanSampler GetSpanSampler(TracerSettings settings) From 672d23a1ab5cd3e0a66f6f5d1362897af95f452b Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 24 Oct 2025 16:07:25 +0100 Subject: [PATCH 13/29] Update Api and AgentWriter to handle changes in exporter and settings --- tracer/src/Datadog.Trace/Agent/AgentWriter.cs | 29 ++- tracer/src/Datadog.Trace/Agent/Api.cs | 24 ++- tracer/src/Datadog.Trace/Agent/ManagedApi.cs | 60 ++++++ .../Datadog.Trace/Ci/Agent/ApmAgentWriter.cs | 7 +- .../TestOptimizationTracerManagerFactory.cs | 12 +- .../DataPipeline/ManagedTraceExporter.cs | 179 +++++++++++++++++ .../Telemetry/TelemetryFactory.cs | 8 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 184 ++---------------- .../LibDatadog/TraceExporterTests.cs | 10 +- .../Agent/AgentWriterTests.cs | 4 +- .../Datadog.Trace.Tests/Agent/ApiTests.cs | 22 +-- .../Benchmarks.Trace/AgentWriterBenchmark.cs | 8 +- 12 files changed, 335 insertions(+), 212 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Agent/ManagedApi.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/DataPipeline/ManagedTraceExporter.cs diff --git a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs index 09c7281ecea5..e4c718bbe9c2 100644 --- a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs +++ b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs @@ -65,17 +65,27 @@ internal class AgentWriter : IAgentWriter private long _droppedTraces; + private bool _traceMetricsEnabled; + public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, TracerSettings settings) - : this(api, statsAggregator, statsd, maxBufferSize: settings.TraceBufferSize, batchInterval: settings.TraceBatchInterval, apmTracingEnabled: settings.ApmTracingEnabled) + : this(api, statsAggregator, statsd, maxBufferSize: settings.TraceBufferSize, batchInterval: settings.TraceBatchInterval, apmTracingEnabled: settings.ApmTracingEnabled, initialTracerMetricsEnabled: settings.Manager.InitialMutableSettings.TracerMetricsEnabled) { + settings.Manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable + && mutable.TracerMetricsEnabled != changes.PreviousMutable.TracerMetricsEnabled) + { + Volatile.Write(ref _traceMetricsEnabled, mutable.TracerMetricsEnabled); + } + }); } - public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, bool automaticFlush = true, int maxBufferSize = 1024 * 1024 * 10, int batchInterval = 100, bool apmTracingEnabled = true) - : this(api, statsAggregator, statsd, MovingAverageKeepRateCalculator.CreateDefaultKeepRateCalculator(), automaticFlush, maxBufferSize, batchInterval, apmTracingEnabled) + public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, bool automaticFlush = true, int maxBufferSize = 1024 * 1024 * 10, int batchInterval = 100, bool apmTracingEnabled = true, bool initialTracerMetricsEnabled = false) + : this(api, statsAggregator, statsd, MovingAverageKeepRateCalculator.CreateDefaultKeepRateCalculator(), automaticFlush, maxBufferSize, batchInterval, apmTracingEnabled, initialTracerMetricsEnabled) { } - internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, IKeepRateCalculator traceKeepRateCalculator, bool automaticFlush, int maxBufferSize, int batchInterval, bool apmTracingEnabled) + internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, IKeepRateCalculator traceKeepRateCalculator, bool automaticFlush, int maxBufferSize, int batchInterval, bool apmTracingEnabled, bool initialTracerMetricsEnabled) { _statsAggregator = statsAggregator; @@ -92,6 +102,9 @@ internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd stat _backBuffer = new SpanBuffer(maxBufferSize, formatterResolver); _activeBuffer = _frontBuffer; + _apmTracingEnabled = apmTracingEnabled; + _traceMetricsEnabled = initialTracerMetricsEnabled; + _serializationTask = automaticFlush ? Task.Factory.StartNew(SerializeTracesLoop, TaskCreationOptions.LongRunning) : Task.CompletedTask; _serializationTask.ContinueWith(t => Log.Error(t.Exception, "Error in serialization task"), TaskContinuationOptions.OnlyOnFaulted); @@ -99,8 +112,6 @@ internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd stat _flushTask.ContinueWith(t => Log.Error(t.Exception, "Error in flush task"), TaskContinuationOptions.OnlyOnFaulted); _backBufferFlushTask = _frontBufferFlushTask = Task.CompletedTask; - - _apmTracingEnabled = apmTracingEnabled; } internal event Action Flushed; @@ -138,7 +149,7 @@ public void WriteTrace(ArraySegment trace) } } - if (_statsd != null) + if (Volatile.Read(ref _traceMetricsEnabled)) { _statsd.Increment(TracerMetricNames.Queue.EnqueuedTraces); _statsd.Increment(TracerMetricNames.Queue.EnqueuedSpans, trace.Count); @@ -314,7 +325,7 @@ async Task InternalBufferFlush() try { - if (_statsd != null) + if (Volatile.Read(ref _traceMetricsEnabled)) { _statsd.Increment(TracerMetricNames.Queue.DequeuedTraces, buffer.TraceCount); _statsd.Increment(TracerMetricNames.Queue.DequeuedSpans, buffer.SpanCount); @@ -512,7 +523,7 @@ private void DropTrace(ArraySegment spans) TelemetryFactory.Metrics.RecordCountSpanDropped(MetricTags.DropReason.OverfullBuffer, spans.Count); TelemetryFactory.Metrics.RecordCountTraceChunkDropped(MetricTags.DropReason.OverfullBuffer); - if (_statsd != null) + if (Volatile.Read(ref _traceMetricsEnabled)) { _statsd.Increment(TracerMetricNames.Queue.DroppedTraces); _statsd.Increment(TracerMetricNames.Queue.DroppedSpans, spans.Count); diff --git a/tracer/src/Datadog.Trace/Agent/Api.cs b/tracer/src/Datadog.Trace/Agent/Api.cs index 385459f78001..627760214edf 100644 --- a/tracer/src/Datadog.Trace/Agent/Api.cs +++ b/tracer/src/Datadog.Trace/Agent/Api.cs @@ -5,8 +5,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Sockets; +using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent.Transports; using Datadog.Trace.DogStatsd; @@ -32,7 +34,7 @@ internal class Api : IApi private readonly IDatadogLogger _log; private readonly IApiRequestFactory _apiRequestFactory; - private readonly IDogStatsd _statsd; + private readonly IDogStatsd _originalStatsd; private readonly string _containerId; private readonly string _entityId; private readonly Uri _tracesEndpoint; @@ -41,6 +43,7 @@ internal class Api : IApi private readonly bool _partialFlushEnabled; private readonly SendCallback _sendStats; private readonly SendCallback _sendTraces; + private IDogStatsd _statsd; private string _cachedResponse; private string _agentVersion; @@ -49,6 +52,7 @@ public Api( IDogStatsd statsd, Action> updateSampleRates, bool partialFlushEnabled, + bool healthMetricsEnabled, IDatadogLogger log = null) { // optionally injecting a log instance in here for testing purposes @@ -57,7 +61,8 @@ public Api( _sendStats = SendStatsAsyncImpl; _sendTraces = SendTracesAsyncImpl; _updateSampleRates = updateSampleRates; - _statsd = statsd; + _originalStatsd = statsd; + ToggleTracerHealthMetrics(healthMetricsEnabled); _containerId = ContainerMetadata.GetContainerId(); _entityId = ContainerMetadata.GetEntityId(); _apiRequestFactory = apiRequestFactory; @@ -77,6 +82,12 @@ private enum SendResult Failed_DontRetry, } + [MemberNotNull(nameof(_statsd))] + public void ToggleTracerHealthMetrics(bool enabled) + { + Interlocked.Exchange(ref _statsd, enabled ? _originalStatsd : null); + } + public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) { _log.Debug("Sending stats to the Datadog Agent."); @@ -299,10 +310,11 @@ private async Task SendTracesAsyncImpl(IApiRequest request, bool fin try { + var healthStats = Volatile.Read(ref _statsd); try { TelemetryFactory.Metrics.RecordCountTraceApiRequests(); - _statsd?.Increment(TracerMetricNames.Api.Requests); + healthStats?.Increment(TracerMetricNames.Api.Requests); response = await request.PostAsync(traces, MimeTypes.MsgPack).ConfigureAwait(false); } catch (Exception ex) @@ -311,17 +323,17 @@ private async Task SendTracesAsyncImpl(IApiRequest request, bool fin // (which are handled below) var tag = ex is TimeoutException ? MetricTags.ApiError.Timeout : MetricTags.ApiError.NetworkError; TelemetryFactory.Metrics.RecordCountTraceApiErrors(tag); - _statsd?.Increment(TracerMetricNames.Api.Errors); + healthStats?.Increment(TracerMetricNames.Api.Errors); throw; } - if (_statsd != null) + if (healthStats != null) { // don't bother creating the tags array if trace metrics are disabled string[] tags = { $"status:{response.StatusCode}" }; // count every response, grouped by status code - _statsd?.Increment(TracerMetricNames.Api.Responses, tags: tags); + healthStats?.Increment(TracerMetricNames.Api.Responses, tags: tags); } TelemetryFactory.Metrics.RecordCountTraceApiResponses(response.GetTelemetryStatusCodeMetricTag()); diff --git a/tracer/src/Datadog.Trace/Agent/ManagedApi.cs b/tracer/src/Datadog.Trace/Agent/ManagedApi.cs new file mode 100644 index 000000000000..635e64831e80 --- /dev/null +++ b/tracer/src/Datadog.Trace/Agent/ManagedApi.cs @@ -0,0 +1,60 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Configuration; +using Datadog.Trace.Vendors.StatsdClient; + +namespace Datadog.Trace.Agent; + +/// +/// A managed version of that is rebuilt whenever the exporter settings change +/// +internal class ManagedApi : IApi +{ + private Api _api; + + public ManagedApi( + TracerSettings.SettingsManager settings, + IDogStatsd statsd, + Action> updateSampleRates, + bool partialFlushEnabled) + { + UpdateApi(settings.InitialExporterSettings, settings.InitialMutableSettings.TracerMetricsEnabled); + // ManagedApi lifetime matches application lifetime, so we don't bother to dispose the subscription + settings.SubscribeToChanges(changes => + { + if (changes.UpdatedExporter is { } exporter) + { + var mutable = changes.UpdatedMutable ?? changes.PreviousMutable; + UpdateApi(exporter, mutable.TracerMetricsEnabled); + } + else if (changes.UpdatedMutable is { } mutable && mutable.TracerMetricsEnabled != changes.PreviousMutable.TracerMetricsEnabled) + { + _api.ToggleTracerHealthMetrics(mutable.TracerMetricsEnabled); + } + }); + + [MemberNotNull(nameof(_api))] + void UpdateApi(ExporterSettings exporterSettings, bool healthMetricsEnabled) + { + var apiRequestFactory = TracesTransportStrategy.Get(exporterSettings); + var api = new Api(apiRequestFactory, statsd, updateSampleRates, partialFlushEnabled, healthMetricsEnabled); + Interlocked.Exchange(ref _api!, api); + } + } + + public Task SendTracesAsync(ArraySegment traces, int numberOfTraces, bool statsComputationEnabled, long numberOfDroppedP0Traces, long numberOfDroppedP0Spans, bool apmTracingEnabled = true) + => Volatile.Read(ref _api).SendTracesAsync(traces, numberOfTraces, statsComputationEnabled, numberOfDroppedP0Traces, numberOfDroppedP0Spans, apmTracingEnabled); + + public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) + => Volatile.Read(ref _api).SendStatsAsync(stats, bucketDuration); +} diff --git a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs index 95f8d4d9ce24..5bfe1e082428 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs @@ -29,13 +29,14 @@ public ApmAgentWriter(TracerSettings settings, Action> { var partialFlushEnabled = settings.PartialFlushEnabled; var apiRequestFactory = TracesTransportStrategy.Get(settings.Exporter); - var api = new Api(apiRequestFactory, null, updateSampleRates, partialFlushEnabled); + var api = new Api(apiRequestFactory, null, updateSampleRates, partialFlushEnabled, healthMetricsEnabled: false); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); - _agentWriter = new AgentWriter(api, statsAggregator, null, maxBufferSize: maxBufferSize, apmTracingEnabled: settings.ApmTracingEnabled); + _agentWriter = new AgentWriter(api, statsAggregator, null, maxBufferSize: maxBufferSize, apmTracingEnabled: settings.ApmTracingEnabled, initialTracerMetricsEnabled: settings.Manager.InitialMutableSettings.TracerMetricsEnabled); } - public ApmAgentWriter(IApi api, int maxBufferSize = DefaultMaxBufferSize) + // Internal for testing + internal ApmAgentWriter(IApi api, int maxBufferSize = DefaultMaxBufferSize) { _agentWriter = new AgentWriter(api, null, null, maxBufferSize: maxBufferSize); } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs index 53798be38d6f..4fbd1cd4f834 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs @@ -64,9 +64,15 @@ protected override TracerManager CreateTracerManagerFrom( return new TestOptimizationTracerManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager, spanEventsManager); } - protected override ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) + protected override TelemetrySettings CreateTelemetrySettings(TracerSettings settings) { - return TelemetryFactory.Instance.CreateCiVisibilityTelemetryController(settings, discoveryService, isAgentAvailable: !_settings.Agentless); + var isAgentAvailable = !_settings.Agentless; + return TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, TelemetryFactory.Config, settings, isAgentAvailable); + } + + protected override ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) + { + return TelemetryFactory.Instance.CreateCiVisibilityTelemetryController(settings, telemetrySettings: telemetrySettings, discoveryService); } protected override IGitMetadataTagsProvider GetGitMetadataTagsProvider(TracerSettings settings, MutableSettings initialMutableSettings, IScopeManager scopeManager, ITelemetryController telemetry) @@ -81,7 +87,7 @@ protected override ITraceSampler GetSampler(TracerSettings settings) protected override bool ShouldEnableRemoteConfiguration(TracerSettings settings) => false; - protected override IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService) + protected override IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) { // Check for agentless scenario if (_settings.Agentless) diff --git a/tracer/src/Datadog.Trace/LibDatadog/DataPipeline/ManagedTraceExporter.cs b/tracer/src/Datadog.Trace/LibDatadog/DataPipeline/ManagedTraceExporter.cs new file mode 100644 index 000000000000..432eb066227d --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/DataPipeline/ManagedTraceExporter.cs @@ -0,0 +1,179 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Configuration; +using Datadog.Trace.Logging; +using Datadog.Trace.PlatformHelpers; +using Datadog.Trace.Telemetry; +using Datadog.Trace.Util; + +namespace Datadog.Trace.LibDatadog.DataPipeline; + +/// +/// A "managed" version of that responds to changes in settings by replacing the trace exporter +/// +internal class ManagedTraceExporter : IApi, IDisposable +{ + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private readonly IDisposable _settingSubscription; + private TraceExporter? _current; + + private ManagedTraceExporter( + TraceExporter traceExporter, + TracerSettings settings, + Action> updateSampleRates, + TelemetrySettings telemetrySettings) + { + _current = traceExporter; + _settingSubscription = settings.Manager.SubscribeToChanges(changes => + { + // avoid changes if we don't need them + // only care about exporter and service/version/env + if (changes.UpdatedExporter is not null + || (changes.UpdatedMutable is { } mutable + && (!string.Equals(mutable.DefaultServiceName, changes.PreviousMutable.DefaultServiceName) + || !string.Equals(mutable.Environment, changes.PreviousMutable.Environment) + || !string.Equals(mutable.ServiceVersion, changes.PreviousMutable.ServiceVersion)))) + { + var exporter = CreateTraceExporter( + settings, + changes.UpdatedMutable ?? changes.PreviousMutable, + changes.UpdatedExporter ?? changes.PreviousExporter, + updateSampleRates, + telemetrySettings); + + var previous = Interlocked.Exchange(ref _current, exporter); + // Disposing immediately here has the potential to cause problems if there's an in-flight request to SendTracesAsync(). + // However, SendTracesAsync() has a lot of try-catch around, and the _caller_ is also expected to handle if SendTracesAsync() + // throws, so it should have a very small risk. We _could_ introduce a flag to allow blocking until it's "safe" to dispose + // but overall I think the added complexity there is likely not worth the risk. Obviously if there's any risk of + // actual _Crashes_ then we should go to the effort, but I don't think that's the case. The same pattern exists in Dispose(). + previous?.Dispose(); + } + }); + } + + public void Dispose() + { + _settingSubscription.Dispose(); + Interlocked.Exchange(ref _current, null)?.Dispose(); + } + + public Task SendTracesAsync(ArraySegment traces, int numberOfTraces, bool statsComputationEnabled, long numberOfDroppedP0Traces, long numberOfDroppedP0Spans, bool apmTracingEnabled = true) + { + // Handle shutdown scenario where api is null + return Volatile.Read(ref _current)?.SendTracesAsync(traces, numberOfTraces, statsComputationEnabled, numberOfDroppedP0Traces, numberOfDroppedP0Spans, apmTracingEnabled) + ?? Task.FromResult(false); + } + + public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) + { + // Handle shutdown scenario where api is null + return Volatile.Read(ref _current)?.SendStatsAsync(stats, bucketDuration) ?? Task.FromResult(false); + } + + // Internal for testing + internal static bool TryCreateTraceExporter( + TracerSettings settings, + Action> updateSampleRates, + TelemetrySettings telemetrySettings, + [NotNullWhen(true)]out ManagedTraceExporter? traceExporter) + { + try + { + // We try to create the "initial" version up front, because if it _doesn't_ work, then + // we assume + var initialExporter = CreateTraceExporter( + settings, + settings.Manager.InitialMutableSettings, + settings.Manager.InitialExporterSettings, + updateSampleRates, + telemetrySettings); + traceExporter = new ManagedTraceExporter(initialExporter, settings, updateSampleRates, telemetrySettings); + return true; + } + catch (Exception ex) + { + Log.Error(ex, "Failed to create native Trace Exporter, falling back to managed API"); + traceExporter = null; + return false; + } + } + + private static TraceExporter CreateTraceExporter( + TracerSettings settings, + MutableSettings mutableSettings, + ExporterSettings exporterSettings, + Action> updateSampleRates, + TelemetrySettings telemetrySettings) + { + // If file logging is enabled, then enable logging in libdatadog + // We assume that we can't go from pipeline enabled -> disabled, so we should never need to call logger.Disable() + // Note that this _could_ fail if there's an issue in libdatadog, but we continue to _Try_ to initialize the exporter anyway + // If this was previously initialized, it will be re-initialized with the new settings, which is fine + if (Log.FileLoggingConfiguration is { } fileConfig) + { + var logger = LibDatadog.Logging.Logger.Instance; + logger.Enable(fileConfig, DomainMetadata.Instance); + + // hacky to use the global setting, but about the only option we have atm + logger.SetLogLevel(GlobalSettings.Instance.DebugEnabled); + } + + TelemetryClientConfiguration? telemetryClientConfiguration = null; + + // We don't know how to handle telemetry in Agentless mode yet + // so we disable telemetry in this case + if (telemetrySettings.TelemetryEnabled && telemetrySettings.Agentless == null) + { + telemetryClientConfiguration = new TelemetryClientConfiguration + { + Interval = (ulong)telemetrySettings.HeartbeatInterval.TotalMilliseconds, + RuntimeId = new CharSlice(Tracer.RuntimeId), + DebugEnabled = telemetrySettings.DebugEnabled + }; + } + + // When APM is disabled, we don't want to compute stats at all + // A common use case is in Application Security Monitoring (ASM) scenarios: + // when APM is disabled but ASM is enabled. + var clientComputedStats = !settings.StatsComputationEnabled && !settings.ApmTracingEnabled; + + var frameworkDescription = FrameworkDescription.Instance; + using var configuration = new TraceExporterConfiguration + { + Url = GetUrl(exporterSettings), + TraceVersion = TracerConstants.AssemblyVersion, + Env = mutableSettings.Environment, + Version = mutableSettings.ServiceVersion, + Service = mutableSettings.DefaultServiceName, + Hostname = HostMetadata.Instance.Hostname, + Language = TracerConstants.Language, + LanguageVersion = frameworkDescription.ProductVersion, + LanguageInterpreter = frameworkDescription.Name, + ComputeStats = settings.StatsComputationEnabled, + TelemetryClientConfiguration = telemetryClientConfiguration, + ClientComputedStats = clientComputedStats, + ConnectionTimeoutMs = 15_000 + }; + + return new TraceExporter(configuration, updateSampleRates); + + static string GetUrl(ExporterSettings exporterSettings) => + exporterSettings.TracesTransport switch + { + TracesTransportType.WindowsNamedPipe => $"windows://./pipe/{exporterSettings.TracesPipeName}", + TracesTransportType.UnixDomainSocket => $"unix://{exporterSettings.TracesUnixDomainSocketPath}", + _ => exporterSettings.AgentUri.ToString() + }; + } +} diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs index ec7cd09e9203..2cbc604800e6 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs @@ -59,11 +59,11 @@ internal static IConfigurationTelemetry SetConfigForTesting(IConfigurationTeleme /// public static TelemetryFactory CreateFactory() => new(); - public ITelemetryController CreateTelemetryController(TracerSettings tracerSettings, IDiscoveryService discoveryService) - => CreateTelemetryController(tracerSettings, TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, Config, tracerSettings, isAgentAvailable: null), discoveryService, useCiVisibilityTelemetry: false); + public ITelemetryController CreateTelemetryController(TracerSettings tracerSettings, TelemetrySettings telemetrySettings, IDiscoveryService discoveryService) + => CreateTelemetryController(tracerSettings, telemetrySettings, discoveryService, useCiVisibilityTelemetry: false); - public ITelemetryController CreateCiVisibilityTelemetryController(TracerSettings tracerSettings, IDiscoveryService discoveryService, bool isAgentAvailable) - => CreateTelemetryController(tracerSettings, TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, Config, tracerSettings, isAgentAvailable), discoveryService, useCiVisibilityTelemetry: true); + public ITelemetryController CreateCiVisibilityTelemetryController(TracerSettings tracerSettings, TelemetrySettings telemetrySettings, IDiscoveryService discoveryService) + => CreateTelemetryController(tracerSettings, telemetrySettings, discoveryService, useCiVisibilityTelemetry: true); public ITelemetryController CreateTelemetryController(TracerSettings tracerSettings, TelemetrySettings settings, IDiscoveryService discoveryService, bool useCiVisibilityTelemetry) { diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 362e3eb2003f..1dc33c42d95e 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -140,11 +140,9 @@ internal TracerManager CreateTracerManager( : null; sampler ??= GetSampler(settings); - agentWriter ??= GetAgentWriter(settings, settings.MutableSettings.TracerMetricsEnabled ? statsd : null, rates => sampler.SetDefaultSampleRates(rates), discoveryService); + agentWriter ??= GetAgentWriter(settings, statsd, rates => sampler.SetDefaultSampleRates(rates), discoveryService, telemetrySettings); scopeManager ??= new AsyncLocalScopeManager(); - telemetry ??= CreateTelemetryController(settings, discoveryService); - var gitMetadataTagsProvider = GetGitMetadataTagsProvider(settings, settings.Manager.InitialMutableSettings, scopeManager, telemetry); logSubmissionManager = DirectLogSubmissionManager.Create( settings, @@ -217,10 +215,15 @@ internal TracerManager CreateTracerManager( spanEventsManager); } - protected virtual ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService) - { - return TelemetryFactory.Instance.CreateTelemetryController(settings, discoveryService); - } + protected virtual TelemetrySettings CreateTelemetrySettings(TracerSettings settings) => + TelemetrySettings.FromSource( + GlobalConfigurationSource.Instance, + TelemetryFactory.Config, + settings, + isAgentAvailable: null); + + protected virtual ITelemetryController CreateTelemetryController(TracerSettings settings, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) + => TelemetryFactory.Instance.CreateTelemetryController(settings, telemetrySettings, discoveryService); protected virtual IGitMetadataTagsProvider GetGitMetadataTagsProvider(TracerSettings settings, MutableSettings initialMutableSettings, IScopeManager scopeManager, ITelemetryController telemetry) { @@ -279,178 +282,21 @@ protected virtual ISpanSampler GetSpanSampler(TracerSettings settings) return new SpanSampler(SpanSamplingRule.BuildFromConfigurationString(settings.SpanSamplingRules, RegexBuilder.DefaultTimeout)); } - protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService) + protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) { - var apiRequestFactory = TracesTransportStrategy.Get(settings.Exporter); - var api = GetApi(settings, statsd, updateSampleRates, apiRequestFactory); + // Currently we assume this _can't_ toggle at runtime, may need to revisit this if that changes + IApi api = settings.DataPipelineEnabled && ManagedTraceExporter.TryCreateTraceExporter(settings, updateSampleRates, telemetrySettings, out var traceExporter) + ? traceExporter + : new ManagedApi(settings.Manager, statsd, updateSampleRates, settings.PartialFlushEnabled); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); return new AgentWriter(api, statsAggregator, statsd, settings); } - [TestingAndPrivateOnly] - internal static IApi GetApi(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IApiRequestFactory apiRequestFactory) - { - // Currently we assume this _can't_ toggle at runtime, may need to revisit this if that changes - if (settings.DataPipelineEnabled) - { - try - { - // If file logging is enabled, then enable logging in libdatadog - // We assume that we can't go from pipeline enabled -> disabled, so we should never need to call logger.Disable() - // Note that this _could_ fail if there's an issue in libdatadog, but we continue to _Try_ to initialize the exporter anyway - // If this was previously initialized, it will be re-initialized with the new settings, which is fine - if (Log.FileLoggingConfiguration is { } fileConfig) - { - var logger = LibDatadog.Logging.Logger.Instance; - logger.Enable(fileConfig, DomainMetadata.Instance); - - // hacky to use the global setting, but about the only option we have atm - logger.SetLogLevel(GlobalSettings.Instance.DebugEnabled); - } - - // TODO: we should refactor this so that we're not re-building the telemetry settings, and instead using the existing ones - var telemetrySettings = TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, TelemetryFactory.Config, settings, isAgentAvailable: null); - TelemetryClientConfiguration? telemetryClientConfiguration = null; - - // We don't know how to handle telemetry in Agentless mode yet - // so we disable telemetry in this case - if (telemetrySettings.TelemetryEnabled && telemetrySettings.Agentless == null) - { - telemetryClientConfiguration = new TelemetryClientConfiguration - { - Interval = (ulong)telemetrySettings.HeartbeatInterval.TotalMilliseconds, - RuntimeId = new CharSlice(Tracer.RuntimeId), - DebugEnabled = telemetrySettings.DebugEnabled - }; - } - - // When APM is disabled, we don't want to compute stats at all - // A common use case is in Application Security Monitoring (ASM) scenarios: - // when APM is disabled but ASM is enabled. - var clientComputedStats = !settings.StatsComputationEnabled && !settings.ApmTracingEnabled; - - var frameworkDescription = FrameworkDescription.Instance; - using var configuration = new TraceExporterConfiguration - { - Url = GetUrl(settings), - TraceVersion = TracerConstants.AssemblyVersion, - Env = settings.MutableSettings.Environment, - Version = settings.MutableSettings.ServiceVersion, - Service = settings.MutableSettings.DefaultServiceName, - Hostname = HostMetadata.Instance.Hostname, - Language = TracerConstants.Language, - LanguageVersion = frameworkDescription.ProductVersion, - LanguageInterpreter = frameworkDescription.Name, - ComputeStats = settings.StatsComputationEnabled, - TelemetryClientConfiguration = telemetryClientConfiguration, - ClientComputedStats = clientComputedStats, - ConnectionTimeoutMs = 15_000 - }; - - return new TraceExporter(configuration, updateSampleRates); - } - catch (Exception ex) - { - Log.Error(ex, "Failed to create native Trace Exporter, falling back to managed API"); - } - } - - return new Api(apiRequestFactory, statsd, updateSampleRates, settings.PartialFlushEnabled); - } - - private static string GetUrl(TracerSettings settings) - { - switch (settings.Exporter.TracesTransport) - { - case TracesTransportType.WindowsNamedPipe: - return $"windows://./pipe/{settings.Exporter.TracesPipeName}"; - case TracesTransportType.UnixDomainSocket: - return $"unix://{settings.Exporter.TracesUnixDomainSocketPath}"; - case TracesTransportType.Default: - default: - return settings.Exporter.AgentUri.ToString(); - } - } - - // internal for testing internal virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) => settings.AgentFeaturePollingEnabled ? DiscoveryService.Create(settings.Exporter) : NullDiscoveryService.Instance; - - internal static IDogStatsd CreateDogStatsdClient(TracerSettings settings, string serviceName, List constantTags, string prefix = null, TimeSpan? telemtryFlushInterval = null) - { - try - { - var statsd = new DogStatsdService(); - var config = new StatsdConfig - { - ConstantTags = constantTags?.ToArray(), - Prefix = prefix, - // note that if these are null, statsd tries to grab them directly from the environment, which could be unsafe - ServiceName = NormalizerTraceProcessor.NormalizeService(serviceName), - Environment = settings.MutableSettings.Environment, - ServiceVersion = settings.MutableSettings.ServiceVersion, - Advanced = { TelemetryFlushInterval = telemtryFlushInterval } - }; - - switch (settings.Exporter.MetricsTransport) - { - case MetricsTransportType.NamedPipe: - config.PipeName = settings.Exporter.MetricsPipeName; - Log.Information("Using windows named pipes for metrics transport: {PipeName}.", config.PipeName); - break; -#if NETCOREAPP3_1_OR_GREATER - case MetricsTransportType.UDS: - config.StatsdServerName = $"{ExporterSettings.UnixDomainSocketPrefix}{settings.Exporter.MetricsUnixDomainSocketPath}"; - Log.Information("Using unix domain sockets for metrics transport: {Socket}.", config.StatsdServerName); - break; -#endif - case MetricsTransportType.UDP: - default: - config.StatsdServerName = settings.Exporter.MetricsHostname; - config.StatsdPort = settings.Exporter.DogStatsdPort; - Log.Information("Using UDP for metrics transport: {Hostname}:{Port}.", config.StatsdServerName, config.StatsdPort); - break; - } - - statsd.Configure(config); - return statsd; - } - catch (Exception ex) - { - Log.Error(ex, "Unable to instantiate StatsD client."); - return new NoOpStatsd(); - } - } - - private static IDogStatsd CreateDogStatsdClient(TracerSettings settings, string serviceName) - { - var customTagCount = settings.MutableSettings.GlobalTags.Count; - var constantTags = new List(5 + customTagCount) - { - "lang:.NET", - $"lang_interpreter:{FrameworkDescription.Instance.Name}", - $"lang_version:{FrameworkDescription.Instance.ProductVersion}", - $"tracer_version:{TracerConstants.AssemblyVersion}", - $"{Tags.RuntimeId}:{Tracer.RuntimeId}" - }; - - if (customTagCount > 0) - { - var tagProcessor = new TruncatorTagsProcessor(); - foreach (var kvp in settings.MutableSettings.GlobalTags) - { - var key = kvp.Key; - var value = kvp.Value; - tagProcessor.ProcessMeta(ref key, ref value); - constantTags.Add($"{key}:{value}"); - } - } - - return CreateDogStatsdClient(settings, serviceName, constantTags); - } } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index 62eb76c3bb7f..0c06d0a6afe7 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -12,9 +12,11 @@ using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.AppSec.Rasp; using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.DogStatsd; using Datadog.Trace.LibDatadog; using Datadog.Trace.LibDatadog.DataPipeline; +using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; @@ -77,12 +79,12 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) var statsd = new NoOpStatsd(); // We have to replace the agent writer so that we can intercept the sample rate responses - var exporter = TracerManagerFactory.GetApi( + ManagedTraceExporter.TryCreateTraceExporter( tracerSettings, - statsd, rates => sampleRateResponses.Enqueue(rates), - new Mock().Object); - exporter.Should().BeOfType(); + TelemetrySettings.FromSource(NullConfigurationSource.Instance, new ConfigurationTelemetry(), tracerSettings, isAgentAvailable: null), + out var exporter).Should().BeTrue(); + exporter.Should().NotBeNull(); var agentWriter = new AgentWriter(exporter, new NullStatsAggregator(), statsd, tracerSettings); diff --git a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs index e195acd0db87..dcce90ce9b7c 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs @@ -347,7 +347,7 @@ public void DropTraces() var sizeOfTrace = ComputeSize(CreateTraceChunk(1)); // Make the buffer size big enough for a single trace - var agent = new AgentWriter(Mock.Of(), statsAggregator: null, statsd.Object, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1); + var agent = new AgentWriter(Mock.Of(), statsAggregator: null, statsd.Object, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, initialTracerMetricsEnabled: true); // Fill the two buffers agent.WriteTrace(CreateTraceChunk(1)); @@ -438,7 +438,7 @@ public async Task AddsTraceKeepRateMetricToRootSpan() // Make the buffer size big enough for a single trace var api = new MockApi(); - var agent = new AgentWriter(api, statsAggregator: null, statsd: null, calculator, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, batchInterval: 100, apmTracingEnabled: true); + var agent = new AgentWriter(api, statsAggregator: null, statsd: null, calculator, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, batchInterval: 100, apmTracingEnabled: true, initialTracerMetricsEnabled: false); // Fill both buffers agent.WriteTrace(spans); diff --git a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs index 870b6897dd6e..0498c15736fe 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs @@ -39,7 +39,7 @@ public async Task SendTraceAsync_200OK_AllGood() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -67,7 +67,7 @@ public async Task SendTracesAsync_ShouldNotRetry_ForSpecificResponses(int status factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -92,7 +92,7 @@ public async Task SendTracesAsync_ShouldSendFiveTimes_ForFailedResponses(int sta factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -123,7 +123,7 @@ public async Task SendTracesAsync_ShouldSendThreeTimes_ForFailedResponseThenSucc factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -145,7 +145,7 @@ public async Task SendTracesAsync_500_ErrorIsCaught() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -166,7 +166,7 @@ public async Task SendStatsAsync_200OK_AllGood() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), [])) { @@ -192,7 +192,7 @@ public async Task SendStatsAsync_500_ErrorIsCaught() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), []))); @@ -217,7 +217,7 @@ public async Task StatsHeader(bool statsComputationEnabled) factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, statsComputationEnabled, 0, 0); @@ -243,7 +243,7 @@ public async Task ExtractAgentVersionHeaderAndLogsWarning() var logMock = new Mock(); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: true, log: logMock.Object); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: true, log: logMock.Object, healthMetricsEnabled: false); // First time should write the warning await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -284,7 +284,7 @@ public async Task SetsDefaultSamplingRates() var ratesWereSet = false; Action> updateSampleRates = _ => ratesWereSet = true; - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: updateSampleRates, partialFlushEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: updateSampleRates, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); ratesWereSet.Should().BeTrue(); @@ -311,7 +311,7 @@ public void LogPartialFlushWarning(string agentVersion, bool partialFlushEnabled factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: partialFlushEnabled); + var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: partialFlushEnabled, healthMetricsEnabled: false); // First call depends on the parameters of the test api.LogPartialFlushWarningIfRequired(agentVersion).Should().Be(expectedResult); diff --git a/tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs b/tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs index a4ff3f2ba228..12cf9d913c01 100644 --- a/tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs +++ b/tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs @@ -8,6 +8,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; using Datadog.Trace.Configuration; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Util; namespace Benchmarks.Trace @@ -47,7 +48,12 @@ public void GlobalSetup() var sources = new CompositeConfigurationSource(new[] { overrides, GlobalConfigurationSource.Instance }); var settings = new TracerSettings(sources); - var api = new Api(new FakeApiRequestFactory(settings.Exporter.AgentUri), statsd: null, updateSampleRates: null, partialFlushEnabled: false); + var api = new Api( + new FakeApiRequestFactory(settings.Manager.InitialExporterSettings.AgentUri), + statsd: new StatsdManager(settings, (_, _) => null!), + updateSampleRates: null, + partialFlushEnabled: false, + healthMetricsEnabled: false); _agentWriter = new AgentWriter(api, statsAggregator: null, statsd: null, automaticFlush: false); From 92012071ceeaac31c23ab4f8cbdf87f8dbf0da3a Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 24 Oct 2025 16:17:06 +0100 Subject: [PATCH 14/29] Fix remaining usages of direct access to Mutable Settings --- .../Configuration/TracerSettings.cs | 3 - .../src/Datadog.Trace/TracerManagerFactory.cs | 15 +--- .../StatsTests.cs | 4 +- .../Configuration/ConfigurationSourceTests.cs | 82 +++++++++---------- .../DynamicConfigurationTests.cs | 37 ++------- .../Configuration/MutableSettingsTests.cs | 2 +- .../TracerSettingsSettingManagerTests.cs | 12 --- 7 files changed, 53 insertions(+), 102 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index afbe9197c420..a4c62283d1e0 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -730,7 +730,6 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = // Move the creation of these settings inside SettingsManager? var initialMutableSettings = MutableSettings.CreateInitialMutableSettings(source, telemetry, errorLog, this); Manager = new(this, initialMutableSettings, Exporter); - MutableSettings = initialMutableSettings; } internal bool IsRunningInCiVisibility { get; } @@ -743,8 +742,6 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = internal IConfigurationTelemetry Telemetry => _telemetry; - internal MutableSettings MutableSettings { get; init; } - internal string FallbackApplicationName => _fallbackApplicationName.Value; /// diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 1dc33c42d95e..e6e80cb391dc 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -5,26 +5,20 @@ using System; using System.Collections.Generic; -using System.Reflection; using Datadog.Trace.Agent; using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.AppSec; using Datadog.Trace.ClrProfiler; using Datadog.Trace.Configuration; -using Datadog.Trace.Configuration.ConfigurationSources; using Datadog.Trace.ContinuousProfiler; using Datadog.Trace.DataStreamsMonitoring; using Datadog.Trace.DogStatsd; -using Datadog.Trace.Iast; using Datadog.Trace.LibDatadog; using Datadog.Trace.LibDatadog.DataPipeline; using Datadog.Trace.LibDatadog.HandsOffConfiguration; using Datadog.Trace.Logging; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.Logging.TracerFlare; -using Datadog.Trace.PlatformHelpers; -using Datadog.Trace.Processors; -using Datadog.Trace.Propagators; using Datadog.Trace.RemoteConfigurationManagement; using Datadog.Trace.RemoteConfigurationManagement.Transport; using Datadog.Trace.RuntimeMetrics; @@ -34,8 +28,6 @@ using Datadog.Trace.Telemetry.Metrics; using Datadog.Trace.Util; using Datadog.Trace.Vendors.StatsdClient; -using ConfigurationKeys = Datadog.Trace.Configuration.ConfigurationKeys; -using MetricsTransportType = Datadog.Trace.Vendors.StatsdClient.Transport.TransportType; using NativeInterop = Datadog.Trace.ContinuousProfiler.NativeInterop; using Stopwatch = System.Diagnostics.Stopwatch; @@ -123,12 +115,9 @@ internal TracerManager CreateTracerManager( Log.Warning(libdatadogAvailaibility.Exception, "An exception occurred while checking if libdatadog is available"); } - // TODO: Update anything that accesses tracerSettings.MutableSettings or tracerSettings.Manager.InitialTracerSettings - // to subscribe to changes, once we stop creating a new TracerManager whenever there's a config change - - var defaultServiceName = settings.MutableSettings.DefaultServiceName; - discoveryService ??= GetDiscoveryService(settings); + var telemetrySettings = CreateTelemetrySettings(settings); + telemetry ??= CreateTelemetryController(settings, discoveryService, telemetrySettings); // Technically we don't _always_ need a dogstatsd instance, because we only need it if runtime metrics // are enabled _or_ tracer metrics are enabled. However, tracer metrics can be enabled and disabled dynamically diff --git a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs index 0f49220b5ba0..06e3563d3aa0 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs @@ -583,9 +583,9 @@ void AssertTraces(IReadOnlyList payload, bool expectStats) void AssertStats(MockClientStatsPayload stats, Span span, long totalDuration) { - stats.Env.Should().Be(settings.MutableSettings.Environment); + stats.Env.Should().Be(settings.Manager.InitialMutableSettings.Environment); stats.Hostname.Should().Be(HostMetadata.Instance.Hostname); - stats.Version.Should().Be(settings.MutableSettings.ServiceVersion); + stats.Version.Should().Be(settings.Manager.InitialMutableSettings.ServiceVersion); stats.TracerVersion.Should().Be(TracerConstants.AssemblyVersion); stats.AgentAggregation.Should().Be(null); stats.Lang.Should().Be(TracerConstants.Language); diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs index 8d6f65654588..fb2ec8137011 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs @@ -71,23 +71,23 @@ public ConfigurationSourceTests() public static IEnumerable<(Func SettingGetter, object ExpectedValue)> GetDefaultTestData() { - yield return (s => s.MutableSettings.TraceEnabled, true); + yield return (s => s.Manager.InitialMutableSettings.TraceEnabled, true); yield return (s => s.Exporter.AgentUri, new Uri("http://127.0.0.1:8126/")); - yield return (s => s.MutableSettings.Environment, null); - yield return (s => s.MutableSettings.ServiceName, null); - yield return (s => s.MutableSettings.DisabledIntegrationNames.Count, 1); // The OpenTelemetry integration is disabled by default - yield return (s => s.MutableSettings.LogsInjectionEnabled, true); - yield return (s => s.MutableSettings.GlobalTags.Count, 0); + yield return (s => s.Manager.InitialMutableSettings.Environment, null); + yield return (s => s.Manager.InitialMutableSettings.ServiceName, null); + yield return (s => s.Manager.InitialMutableSettings.DisabledIntegrationNames.Count, 1); // The OpenTelemetry integration is disabled by default + yield return (s => s.Manager.InitialMutableSettings.LogsInjectionEnabled, true); + yield return (s => s.Manager.InitialMutableSettings.GlobalTags.Count, 0); #pragma warning disable 618 // App analytics is deprecated but supported - yield return (s => s.MutableSettings.AnalyticsEnabled, false); + yield return (s => s.Manager.InitialMutableSettings.AnalyticsEnabled, false); #pragma warning restore 618 - yield return (s => s.MutableSettings.CustomSamplingRules, null); - yield return (s => s.MutableSettings.MaxTracesSubmittedPerSecond, 100); - yield return (s => s.MutableSettings.TracerMetricsEnabled, false); + yield return (s => s.Manager.InitialMutableSettings.CustomSamplingRules, null); + yield return (s => s.Manager.InitialMutableSettings.MaxTracesSubmittedPerSecond, 100); + yield return (s => s.Manager.InitialMutableSettings.TracerMetricsEnabled, false); yield return (s => s.Exporter.DogStatsdPort, 8125); yield return (s => s.PropagationStyleInject, new[] { "Datadog", "tracecontext", "baggage" }); yield return (s => s.PropagationStyleExtract, new[] { "Datadog", "tracecontext", "baggage" }); - yield return (s => s.MutableSettings.ServiceNameMappings, new string[0]); + yield return (s => s.Manager.InitialMutableSettings.ServiceNameMappings, new string[0]); yield return (s => s.TraceId128BitGenerationEnabled, true); yield return (s => s.TraceId128BitLoggingEnabled, true); @@ -97,58 +97,58 @@ public ConfigurationSourceTests() public static IEnumerable<(string Key, string Value, Func Getter, object Expected)> GetBreakingChangeTestData() { // Test edge cases that expose various discrepenacies with the Agent DD_TAGS parsing algorithm that we would like to support - yield return (ConfigurationKeys.GlobalTags, "k1:v1 k2:v2", s => s.MutableSettings.GlobalTags, TagsK1V1K2V2); - yield return (ConfigurationKeys.GlobalTags, "key1,key2", s => s.MutableSettings.GlobalTags, TagsKey1Key2); - yield return (ConfigurationKeys.GlobalTags, "key1,key2:", s => s.MutableSettings.GlobalTags, TagsKey1Key2); - yield return (ConfigurationKeys.GlobalTags, "key :val, aKey : aVal bKey:bVal cKey:", s => s.MutableSettings.GlobalTags, TagsWithSpacesInValue); + yield return (ConfigurationKeys.GlobalTags, "k1:v1 k2:v2", s => s.Manager.InitialMutableSettings.GlobalTags, TagsK1V1K2V2); + yield return (ConfigurationKeys.GlobalTags, "key1,key2", s => s.Manager.InitialMutableSettings.GlobalTags, TagsKey1Key2); + yield return (ConfigurationKeys.GlobalTags, "key1,key2:", s => s.Manager.InitialMutableSettings.GlobalTags, TagsKey1Key2); + yield return (ConfigurationKeys.GlobalTags, "key :val, aKey : aVal bKey:bVal cKey:", s => s.Manager.InitialMutableSettings.GlobalTags, TagsWithSpacesInValue); } public static IEnumerable<(string Key, string Value, Func Getter, object Expected)> GetTestData() { - yield return (ConfigurationKeys.TraceEnabled, "true", s => s.MutableSettings.TraceEnabled, true); - yield return (ConfigurationKeys.TraceEnabled, "false", s => s.MutableSettings.TraceEnabled, false); + yield return (ConfigurationKeys.TraceEnabled, "true", s => s.Manager.InitialMutableSettings.TraceEnabled, true); + yield return (ConfigurationKeys.TraceEnabled, "false", s => s.Manager.InitialMutableSettings.TraceEnabled, false); yield return (ConfigurationKeys.AgentHost, "test-host", s => s.Exporter.AgentUri, new Uri("http://test-host:8126/")); yield return (ConfigurationKeys.AgentPort, "9000", s => s.Exporter.AgentUri, new Uri("http://127.0.0.1:9000/")); - yield return (ConfigurationKeys.Environment, "staging", s => s.MutableSettings.Environment, "staging"); + yield return (ConfigurationKeys.Environment, "staging", s => s.Manager.InitialMutableSettings.Environment, "staging"); - yield return (ConfigurationKeys.ServiceVersion, "1.0.0", s => s.MutableSettings.ServiceVersion, "1.0.0"); + yield return (ConfigurationKeys.ServiceVersion, "1.0.0", s => s.Manager.InitialMutableSettings.ServiceVersion, "1.0.0"); - yield return (ConfigurationKeys.ServiceName, "web-service", s => s.MutableSettings.ServiceName, "web-service"); - yield return ("DD_SERVICE_NAME", "web-service", s => s.MutableSettings.ServiceName, "web-service"); + yield return (ConfigurationKeys.ServiceName, "web-service", s => s.Manager.InitialMutableSettings.ServiceName, "web-service"); + yield return ("DD_SERVICE_NAME", "web-service", s => s.Manager.InitialMutableSettings.ServiceName, "web-service"); - yield return (ConfigurationKeys.DisabledIntegrations, "integration1;integration2;;INTEGRATION2", s => s.MutableSettings.DisabledIntegrationNames.Count, 3); // The OpenTelemetry integration is disabled by defau)t + yield return (ConfigurationKeys.DisabledIntegrations, "integration1;integration2;;INTEGRATION2", s => s.Manager.InitialMutableSettings.DisabledIntegrationNames.Count, 3); // The OpenTelemetry integration is disabled by defau)t - yield return (ConfigurationKeys.GlobalTags, "k1:v1, k2:v2", s => s.MutableSettings.GlobalTags, TagsK1V1K2V2); - yield return (ConfigurationKeys.GlobalTags, "keyonly:,nocolon,:,:valueonly,k2:v2", s => s.MutableSettings.GlobalTags, TagsK2V2); - yield return ("DD_TRACE_GLOBAL_TAGS", "k1:v1, k2:v2", s => s.MutableSettings.GlobalTags, TagsK1V1K2V2); - yield return (ConfigurationKeys.GlobalTags, "k1:v1,k1:v2", s => s.MutableSettings.GlobalTags.Count, 1); - yield return (ConfigurationKeys.GlobalTags, "k1:v1, k2:v2:with:colons, :leading:colon:bad, trailing:colon:good:", s => s.MutableSettings.GlobalTags, TagsWithColonsInValue); + yield return (ConfigurationKeys.GlobalTags, "k1:v1, k2:v2", s => s.Manager.InitialMutableSettings.GlobalTags, TagsK1V1K2V2); + yield return (ConfigurationKeys.GlobalTags, "keyonly:,nocolon,:,:valueonly,k2:v2", s => s.Manager.InitialMutableSettings.GlobalTags, TagsK2V2); + yield return ("DD_TRACE_GLOBAL_TAGS", "k1:v1, k2:v2", s => s.Manager.InitialMutableSettings.GlobalTags, TagsK1V1K2V2); + yield return (ConfigurationKeys.GlobalTags, "k1:v1,k1:v2", s => s.Manager.InitialMutableSettings.GlobalTags.Count, 1); + yield return (ConfigurationKeys.GlobalTags, "k1:v1, k2:v2:with:colons, :leading:colon:bad, trailing:colon:good:", s => s.Manager.InitialMutableSettings.GlobalTags, TagsWithColonsInValue); // Test edge cases that expose various discrepenacies with the Agent DD_TAGS parsing algorithm that we would like to support - yield return (ConfigurationKeys.GlobalTags, "k1:v1 k2:v2", s => s.MutableSettings.GlobalTags, new Dictionary() { { "k1", "v1 k2:v2" } }); - yield return (ConfigurationKeys.GlobalTags, "key1,key2", s => s.MutableSettings.GlobalTags.Count, 0); - yield return (ConfigurationKeys.GlobalTags, "key1,key2:", s => s.MutableSettings.GlobalTags.Count, 0); - yield return (ConfigurationKeys.GlobalTags, "key :val, aKey : aVal bKey:bVal cKey:", s => s.MutableSettings.GlobalTags, TagsWithSpacesInValue); + yield return (ConfigurationKeys.GlobalTags, "k1:v1 k2:v2", s => s.Manager.InitialMutableSettings.GlobalTags, new Dictionary() { { "k1", "v1 k2:v2" } }); + yield return (ConfigurationKeys.GlobalTags, "key1,key2", s => s.Manager.InitialMutableSettings.GlobalTags.Count, 0); + yield return (ConfigurationKeys.GlobalTags, "key1,key2:", s => s.Manager.InitialMutableSettings.GlobalTags.Count, 0); + yield return (ConfigurationKeys.GlobalTags, "key :val, aKey : aVal bKey:bVal cKey:", s => s.Manager.InitialMutableSettings.GlobalTags, TagsWithSpacesInValue); #pragma warning disable 618 // App Analytics is deprecated but still supported - yield return (ConfigurationKeys.GlobalAnalyticsEnabled, "true", s => s.MutableSettings.AnalyticsEnabled, true); - yield return (ConfigurationKeys.GlobalAnalyticsEnabled, "false", s => s.MutableSettings.AnalyticsEnabled, false); + yield return (ConfigurationKeys.GlobalAnalyticsEnabled, "true", s => s.Manager.InitialMutableSettings.AnalyticsEnabled, true); + yield return (ConfigurationKeys.GlobalAnalyticsEnabled, "false", s => s.Manager.InitialMutableSettings.AnalyticsEnabled, false); #pragma warning restore 618 - yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header2:Content-Type,header3: Content-Type ,header4:C!!!ont_____ent----tYp!/!e,header6:9invalidtagname,:invalidtagonly,invalidheaderonly:,validheaderwithoutcolon,:", s => s.MutableSettings.HeaderTags, HeaderTagsWithOptionalMappings); - yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header2:tag1", s => s.MutableSettings.HeaderTags, HeaderTagsSameTag); - yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header1:tag2", s => s.MutableSettings.HeaderTags.Count, 1); - yield return (ConfigurationKeys.HeaderTags, "header3:my.header.with.dot,my.new.header.with.dot", s => s.MutableSettings.HeaderTags, HeaderTagsWithDots); + yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header2:Content-Type,header3: Content-Type ,header4:C!!!ont_____ent----tYp!/!e,header6:9invalidtagname,:invalidtagonly,invalidheaderonly:,validheaderwithoutcolon,:", s => s.Manager.InitialMutableSettings.HeaderTags, HeaderTagsWithOptionalMappings); + yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header2:tag1", s => s.Manager.InitialMutableSettings.HeaderTags, HeaderTagsSameTag); + yield return (ConfigurationKeys.HeaderTags, "header1:tag1,header1:tag2", s => s.Manager.InitialMutableSettings.HeaderTags.Count, 1); + yield return (ConfigurationKeys.HeaderTags, "header3:my.header.with.dot,my.new.header.with.dot", s => s.Manager.InitialMutableSettings.HeaderTags, HeaderTagsWithDots); - yield return (ConfigurationKeys.ServiceNameMappings, "elasticsearch:custom-name", s => s.MutableSettings.ServiceNameMappings["elasticsearch"], "custom-name"); + yield return (ConfigurationKeys.ServiceNameMappings, "elasticsearch:custom-name", s => s.Manager.InitialMutableSettings.ServiceNameMappings["elasticsearch"], "custom-name"); } // JsonConfigurationSource needs to be tested with JSON data, which cannot be used with the other IConfigurationSource implementations. public static IEnumerable<(string Value, Func Getter, object Expected)> GetJsonTestData() { - yield return new(@"{ ""DD_TRACE_GLOBAL_TAGS"": { ""k1"":""v1"", ""k2"": ""v2""} }", s => s.MutableSettings.GlobalTags, TagsK1V1K2V2); + yield return new(@"{ ""DD_TRACE_GLOBAL_TAGS"": { ""k1"":""v1"", ""k2"": ""v2""} }", s => s.Manager.InitialMutableSettings.GlobalTags, TagsK1V1K2V2); } public static IEnumerable GetBadJsonTestData1() @@ -166,7 +166,7 @@ public static IEnumerable GetBadJsonTestData2() public static IEnumerable<(string Value, Func Getter, object Expected)> GetBadJsonTestData3() { // Json doesn't represent dictionary of string to string - yield return (@"{ ""DD_TRACE_GLOBAL_TAGS"": { ""name1"": { ""name2"": [ ""vers"" ] } } }", s => s.MutableSettings.GlobalTags.Count, 0); + yield return (@"{ ""DD_TRACE_GLOBAL_TAGS"": { ""name1"": { ""name2"": [ ""vers"" ] } } }", s => s.Manager.InitialMutableSettings.GlobalTags.Count, 0); } public void Dispose() @@ -301,7 +301,7 @@ public void TestHeaderTagsNormalization(bool headerTagsNormalizationFixEnabled, IConfigurationSource source = new NameValueConfigurationSource(collection); var settings = new TracerSettings(source); - Assert.Equal(expectedValue, settings.MutableSettings.HeaderTags); + Assert.Equal(expectedValue, settings.Manager.InitialMutableSettings.HeaderTags); } private void AssertNameValueConfigurationSource(IEnumerable<(string Key, string Value, Func Getter, object Expected)> testData, string setExperimentalFeaturesEnabled = "") diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs index 6f4cfa623028..b355e26eece8 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs @@ -63,7 +63,6 @@ public void ApplyTagsToDirectLogs() // emulate the one-time subscribe that TracerManager.Instance does var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); - using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); @@ -83,7 +82,6 @@ public void DoesNotOverrideDirectLogsTags() { ConfigurationKeys.GlobalTags, "key2:value2" }, }); var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); - using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); tracerManager.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1"); @@ -102,7 +100,6 @@ public void DoesNotReplaceRuntimeMetricsWriter() { ConfigurationKeys.GlobalTags, "key1:value1" }, }); var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); - using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); var previousRuntimeMetrics = tracerManager.RuntimeMetrics; @@ -117,18 +114,17 @@ public void EnableTracing() { var tracerSettings = new TracerSettings(); var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); - using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); // tracing is enabled by default - tracerManager.Settings.MutableSettings.TraceEnabled.Should().BeTrue(); + tracerManager.PerTraceSettings.Settings.TraceEnabled.Should().BeTrue(); // disable "remotely" DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", false)), tracerSettings); - tracerManager.Settings.MutableSettings.TraceEnabled.Should().BeFalse(); + tracerManager.PerTraceSettings.Settings.TraceEnabled.Should().BeFalse(); // re-enable "remotely" DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_enabled", true)), tracerSettings); - tracerManager.Settings.MutableSettings.TraceEnabled.Should().BeTrue(); + tracerManager.PerTraceSettings.Settings.TraceEnabled.Should().BeTrue(); } [Fact] @@ -148,10 +144,9 @@ public void SetSamplingRules() }); var tracerManager = TracerManagerFactory.Instance.CreateTracerManager(tracerSettings, null); - using var sub = tracerSettings.Manager.SubscribeToChanges(x => tracerManager = UpdateTracerManager(x, tracerSettings, tracerManager)); - tracerManager.Settings.MutableSettings.CustomSamplingRules.Should().Be(localSamplingRulesJson); - tracerManager.Settings.MutableSettings.CustomSamplingRulesIsRemote.Should().BeFalse(); + tracerManager.PerTraceSettings.Settings.CustomSamplingRules.Should().Be(localSamplingRulesJson); + tracerManager.PerTraceSettings.Settings.CustomSamplingRulesIsRemote.Should().BeFalse(); var rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); @@ -181,8 +176,8 @@ public void SetSamplingRules() DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder, tracerSettings); var remoteSamplingRulesJson = JsonConvert.SerializeObject(remoteSamplingRulesConfig); - tracerManager.Settings.MutableSettings.CustomSamplingRules.Should().Be(remoteSamplingRulesJson); - tracerManager.Settings.MutableSettings.CustomSamplingRulesIsRemote.Should().BeTrue(); + tracerManager.PerTraceSettings.Settings.CustomSamplingRules.Should().Be(remoteSamplingRulesJson); + tracerManager.PerTraceSettings.Settings.CustomSamplingRulesIsRemote.Should().BeTrue(); rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); @@ -278,23 +273,5 @@ private static DynamicConfigConfigurationSource CreateConfig(params (string Key, return new DynamicConfigConfigurationSource(configObj, ConfigurationOrigins.RemoteConfig); } - - private static TracerManager UpdateTracerManager(TracerSettings.SettingsManager.SettingChanges updates, TracerSettings settings, TracerManager tracerManager) - { - var newSettings = updates switch - { - { UpdatedExporter: { } e, UpdatedMutable: { } m } => settings with { Exporter = e, MutableSettings = m }, - { UpdatedExporter: { } e } => settings with { Exporter = e }, - { UpdatedMutable: { } m } => settings with { MutableSettings = m }, - _ => null, - }; - - if (newSettings != null) - { - tracerManager = TracerManagerFactory.Instance.CreateTracerManager(newSettings, tracerManager); - } - - return tracerManager; - } } } diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/MutableSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/MutableSettingsTests.cs index 1813303ef479..e9693c59234f 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/MutableSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/MutableSettingsTests.cs @@ -172,7 +172,7 @@ public async Task TraceEnabled(string value, string otelValue, bool areTracesEna var errorLog = new OverrideErrorLog(); var tracerSettings = new TracerSettings(new NameValueConfigurationSource(settings), NullConfigurationTelemetry.Instance, errorLog); - Assert.Equal(areTracesEnabled, tracerSettings.MutableSettings.TraceEnabled); + Assert.Equal(areTracesEnabled, tracerSettings.Manager.InitialMutableSettings.TraceEnabled); errorLog.ShouldHaveExpectedOtelMetric(metric, ConfigurationKeys.OpenTelemetry.TracesExporter.ToLowerInvariant(), ConfigurationKeys.TraceEnabled.ToLowerInvariant()); _writerMock.Invocations.Clear(); diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs index 0306b7a55bc5..bdf52cd7bd0e 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs @@ -16,16 +16,4 @@ namespace Datadog.Trace.Tests.Configuration; public class TracerSettingsSettingManagerTests { - [Fact] - public void UpdatingTracerSettingsDoesNotReplaceSettingsManager() - { - var tracerSettings = TracerSettings.Create([]); - tracerSettings.MutableSettings.Should().BeSameAs(tracerSettings.Manager.InitialMutableSettings); - - var originalManager = tracerSettings.Manager; - var newSettings = tracerSettings with { MutableSettings = MutableSettings.CreateForTesting(tracerSettings, []) }; - - newSettings.MutableSettings.Should().NotBeSameAs(newSettings.Manager.InitialMutableSettings); - newSettings.Manager.Should().BeSameAs(originalManager); - } } From eaac50df02d4faa5b2746e1b5ec6eedac1197480 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 24 Oct 2025 16:52:59 +0100 Subject: [PATCH 15/29] minor fixes --- .../Debugger/DynamicInstrumentationTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs index 411fe407641b..fbde9d1e1e4d 100644 --- a/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Debugger/DynamicInstrumentationTests.cs @@ -16,6 +16,7 @@ using Datadog.Trace.Debugger.Models; using Datadog.Trace.Debugger.ProbeStatuses; using Datadog.Trace.Debugger.Sink; +using Datadog.Trace.DogStatsd; using Datadog.Trace.RemoteConfigurationManagement; using Datadog.Trace.RemoteConfigurationManagement.Protocol; using FluentAssertions; @@ -41,7 +42,7 @@ public async Task DynamicInstrumentationEnabled_ServicesCalled() var probeStatusPoller = new ProbeStatusPollerMock(); var updater = ConfigurationUpdater.Create("env", "version"); - var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, new DogStatsd.NoOpStatsd()); + var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance); debugger.Initialize(); // Wait for async initialization to complete @@ -78,7 +79,7 @@ public void DynamicInstrumentationDisabled_ServicesNotCalled() var probeStatusPoller = new ProbeStatusPollerMock(); var updater = ConfigurationUpdater.Create(string.Empty, string.Empty); - var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, new DogStatsd.NoOpStatsd()); + var debugger = new DynamicInstrumentation(settings, discoveryService, rcmSubscriptionManagerMock, lineProbeResolver, snapshotUploader, logUploader, diagnosticsUploader, probeStatusPoller, updater, NoOpStatsd.Instance); debugger.Initialize(); lineProbeResolver.Called.Should().BeFalse(); probeStatusPoller.Called.Should().BeFalse(); From c7c34adc22b688e903400c127051336db3c59fb6 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 27 Oct 2025 11:33:04 +0000 Subject: [PATCH 16/29] Update CI Vis usages - these don't need to respond to changes because reconfiguration is not allowed --- tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs | 3 ++- .../Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs | 6 ++++-- tracer/src/Datadog.Trace/Ci/TestOptimization.cs | 2 +- .../Datadog.Trace/Ci/TestOptimizationTracerManagement.cs | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs index 5bfe1e082428..4188a4259f15 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs @@ -28,7 +28,8 @@ internal class ApmAgentWriter : IEventWriter public ApmAgentWriter(TracerSettings settings, Action> updateSampleRates, IDiscoveryService discoveryService, int maxBufferSize = DefaultMaxBufferSize) { var partialFlushEnabled = settings.PartialFlushEnabled; - var apiRequestFactory = TracesTransportStrategy.Get(settings.Exporter); + // CI Vis doesn't allow reconfiguration, so don't need to subscribe to changes + var apiRequestFactory = TracesTransportStrategy.Get(settings.Manager.InitialExporterSettings); var api = new Api(apiRequestFactory, null, updateSampleRates, partialFlushEnabled, healthMetricsEnabled: false); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); diff --git a/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs b/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs index 8cca067a17fe..fe06c8e5ffda 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs @@ -151,7 +151,9 @@ private void EnsureUrl() else { // Use Agent EVP Proxy - switch (_settings.TracerSettings.Exporter.TracesTransport) + // CI Visibility doesn't allow re-configuration, so only need to use the "initial" settings + var exporterSettings = _settings.TracerSettings.Manager.InitialExporterSettings; + switch (exporterSettings.TracesTransport) { case TracesTransportType.WindowsNamedPipe: case TracesTransportType.UnixDomainSocket: @@ -159,7 +161,7 @@ private void EnsureUrl() break; case TracesTransportType.Default: default: - builder = new UriBuilder(_settings.TracerSettings.Exporter.AgentUri); + builder = new UriBuilder(exporterSettings.AgentUri); break; } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs index 29163da9450e..b969a4e459d5 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs @@ -247,7 +247,7 @@ public void Initialize() TracerManagement = new TestOptimizationTracerManagement( settings: Settings, getDiscoveryServiceFunc: static s => DiscoveryService.Create( - s.TracerSettings.Exporter, + s.TracerSettings.Manager.InitialExporterSettings, tcpTimeout: TimeSpan.FromSeconds(5), initialRetryDelayMs: 100, maxRetryDelayMs: 1000, diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagement.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagement.cs index 3a509250d10b..4ee53e8e643b 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagement.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagement.cs @@ -159,7 +159,7 @@ public IApiRequestFactory GetRequestFactory(TracerSettings settings) public IApiRequestFactory GetRequestFactory(TracerSettings tracerSettings, TimeSpan timeout) { IApiRequestFactory? factory; - var exporterSettings = tracerSettings.Exporter; + var exporterSettings = tracerSettings.Manager.InitialExporterSettings; if (exporterSettings.TracesTransport != TracesTransportType.Default) { factory = AgentTransportStrategy.Get( @@ -181,7 +181,7 @@ public IApiRequestFactory GetRequestFactory(TracerSettings tracerSettings, TimeS timeout: timeout); #else Log.Information("TestOptimizationTracerManagement: Using {FactoryType} for trace transport.", nameof(ApiWebRequestFactory)); - factory = new ApiWebRequestFactory(tracerSettings.Exporter.AgentUri, AgentHttpHeaderNames.DefaultHeaders, timeout: timeout); + factory = new ApiWebRequestFactory(exporterSettings.AgentUri, AgentHttpHeaderNames.DefaultHeaders, timeout: timeout); #endif if (!string.IsNullOrWhiteSpace(_settings.ProxyHttps)) { From 8eef65907c9a7a2cb0cc8b3c3eb4a352e8f05d1d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 27 Oct 2025 14:49:43 +0000 Subject: [PATCH 17/29] Update Telemetry to handle changes to agent configuration --- .../Telemetry/TelemetryFactory.cs | 6 +- .../Telemetry/TelemetryTransportManager.cs | 101 +++++++++++++----- .../Transports/TelemetryTransportFactory.cs | 33 ++++-- .../Transports/TelemetryTransports.cs | 22 ---- .../TelemetryTransportTests.cs | 14 ++- .../Telemetry/TelemetryControllerTests.cs | 18 ++-- .../TelemetryTransportManagerTests.cs | 55 +++++----- .../TelemetryTransportFactoryTests.cs | 4 +- 8 files changed, 145 insertions(+), 108 deletions(-) delete mode 100644 tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransports.cs diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs index 2cbc604800e6..b53b198fe044 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs @@ -80,7 +80,7 @@ public ITelemetryController CreateTelemetryController(TracerSettings tracerSetti try { - var telemetryTransports = TelemetryTransportFactory.Create(settings, tracerSettings.Exporter); + var telemetryTransports = new TelemetryTransportFactory(settings); if (!telemetryTransports.HasTransports) { @@ -156,11 +156,11 @@ private static void DisableConfigCollector() private ITelemetryController CreateController( TracerSettings tracerSettings, - TelemetryTransports telemetryTransports, + TelemetryTransportFactory telemetryTransports, TelemetrySettings settings, IDiscoveryService discoveryService) { - var transportManager = new TelemetryTransportManager(telemetryTransports, discoveryService); + var transportManager = new TelemetryTransportManager(tracerSettings.Manager, telemetryTransports, discoveryService); // The telemetry controller must be a singleton, so we initialize once // Note that any dependencies initialized inside the controller are also singletons (by design) // Initialized once so if we create a new controller from this factory we get the same collector instances. diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryTransportManager.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryTransportManager.cs index d4e96ca8a9e2..66acdf366ef6 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryTransportManager.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryTransportManager.cs @@ -6,8 +6,10 @@ #nullable enable using System; using System.Diagnostics; +using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.Configuration; using Datadog.Trace.Logging; using Datadog.Trace.SourceGenerators; using Datadog.Trace.Telemetry.Transports; @@ -21,30 +23,53 @@ internal class TelemetryTransportManager : IDisposable internal const int MaxFatalErrors = 2; internal const int MaxTransientErrors = 5; - private readonly TelemetryTransports _transports; private readonly IDiscoveryService _discoveryService; + private readonly IDisposable? _settingSubscription; + private readonly ITelemetryTransport? _agentlessTransport; + private ITelemetryTransport? _currentAgentTransport; private ITelemetryTransport _currentTransport; + private bool _agentTransportUpdated; private bool? _canSendToAgent = null; - public TelemetryTransportManager(TelemetryTransports transports, IDiscoveryService discoveryService) + public TelemetryTransportManager(TracerSettings.SettingsManager settings, TelemetryTransportFactory transports, IDiscoveryService discoveryService) { - _transports = transports; - _discoveryService = discoveryService; - if (!_transports.HasTransports) + if (!transports.HasTransports) { throw new ArgumentException("Must have at least one transport", nameof(transports)); } + _agentlessTransport = transports.AgentlessTransport; + _discoveryService = discoveryService; + discoveryService.SubscribeToChanges(HandleAgentDiscoveryUpdate); + if (transports.AgentTransportFactory is { } agentFactory) + { + _currentAgentTransport = agentFactory(settings.InitialExporterSettings); + _settingSubscription = settings.SubscribeToChanges(changes => + { + if (changes.UpdatedExporter is { } exporter) + { + var newTransport = agentFactory(exporter); + Interlocked.Exchange(ref _currentAgentTransport, newTransport); + Volatile.Write(ref _agentTransportUpdated, true); + if (Log.IsEnabled(LogEventLevel.Debug)) + { + Log.Debug("Telemetry AgentProxy updated {TransportInfo}", newTransport.GetTransportInfo()); + } + } + }); + } + + // use agent if we know it's available, use agentless if we know it's not, and use agent as fallback _currentTransport = GetNextTransport(null); if (Log.IsEnabled(LogEventLevel.Debug)) { Log.Debug( "Telemetry AgentProxy enabled: {AgentProxyEnabled}, Agentless enabled: {AgentlessEnabled}, Agent proxy available {AgentProxyAvailable}. Initial Transport {TransportInfo}", - _transports.AgentTransport is not null, - _transports.AgentlessTransport is not null, + _currentAgentTransport is not null, + _agentlessTransport is not null, _canSendToAgent switch { true => "Available", false => "Unavailable", _ => "Unknown" }, _currentTransport.GetTransportInfo()); } @@ -52,12 +77,15 @@ public TelemetryTransportManager(TelemetryTransports transports, IDiscoveryServi public void Dispose() { + _settingSubscription?.Dispose(); _discoveryService.RemoveSubscription(HandleAgentDiscoveryUpdate); } public async Task TryPushTelemetry(TelemetryData telemetryData) { - var pushResult = await _currentTransport.PushTelemetry(telemetryData).ConfigureAwait(false); + RefreshAgentTransportIfRequired(); + var currentTransport = _currentTransport; + var pushResult = await currentTransport.PushTelemetry(telemetryData).ConfigureAwait(false); if (pushResult == TelemetryPushResult.Success) { @@ -65,13 +93,13 @@ public async Task TryPushTelemetry(TelemetryData telemetryData) return true; } - var previousTransport = _currentTransport; + var previousTransport = currentTransport; _currentTransport = GetNextTransport(previousTransport); Log.Debug( "Telemetry transport {FailedTransportInfo} failed. Enabling next transport {NextTransportInfo}", previousTransport.GetTransportInfo(), - _currentTransport.GetTransportInfo()); + currentTransport.GetTransportInfo()); return false; } @@ -82,29 +110,22 @@ public async Task TryPushTelemetry(TelemetryData telemetryData) internal ITelemetryTransport GetNextTransport(ITelemetryTransport? currentTransport) { // use agent if we know it's available, use agentless if we know it's not, and use agent as fallback + var agentProxy = Volatile.Read(ref _currentAgentTransport); + var agentless = _agentlessTransport; if (currentTransport is null) { - return _transports switch - { - { AgentTransport: { } t } when _canSendToAgent ?? true => t, - { AgentlessTransport: { } t } => t, - { AgentTransport: { } t } => t, - _ => throw new Exception("Must have at least one transport"), - }; + return agentProxy is not null && (_canSendToAgent ?? true) + ? agentProxy + : agentless ?? agentProxy ?? throw new Exception("Must have at least one transport"); } - var agentProxy = _transports.AgentTransport; - var agentless = _transports.AgentlessTransport; - - // If only one transport is configured, continue to use it - // If we're using agentProxy, and agentless is configured, use that - // If we're using agentless, and agentProxy is configured, and we don't _know_ that we can't use it, use that - // If no transports are available, - if (currentTransport == agentProxy) + // - If only one transport is configured, continue to use it + // - If we're using agentProxy, and agentless is configured, use that + // - If we're using agentless, and agentProxy is configured, and we don't _know_ that we can't use it, use that + if (currentTransport != agentless) { - return agentless is null - ? agentProxy // nothing else available, keep using it - : agentless; // switch from agent to agentless + // Switch from agent to agentless if available, otherwise stick with the same transport, + return agentless ?? currentTransport; } Debug.Assert(agentless is not null, "If current transport is not agent, it must be agentless"); @@ -120,6 +141,30 @@ internal ITelemetryTransport GetNextTransport(ITelemetryTransport? currentTransp : agentless!; // the agent is not available, so stick to agentless } + private void RefreshAgentTransportIfRequired() + { + if (!Volatile.Read(ref _agentTransportUpdated)) + { + return; + } + + // Refresh required, reset the flag + Volatile.Write(ref _agentTransportUpdated, false); + + // if we're currently using the agentless transport, just keep using that + // we also ignore the case where we don't have an agent transport, because this method shouldn't be called in that case + var currentTransport = _currentTransport; + var currentAgentTransport = _currentAgentTransport; + if (currentTransport == _agentlessTransport || currentAgentTransport is null) + { + // nothing to do + return; + } + + // otherwise, replace the current transport + _currentTransport = currentAgentTransport; + } + private void HandleAgentDiscoveryUpdate(AgentConfiguration config) { _canSendToAgent = !string.IsNullOrWhiteSpace(config.TelemetryProxyEndpoint); diff --git a/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransportFactory.cs b/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransportFactory.cs index 581b5de9ecf0..40a740a8c746 100644 --- a/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransportFactory.cs +++ b/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransportFactory.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -12,19 +12,34 @@ namespace Datadog.Trace.Telemetry.Transports { internal class TelemetryTransportFactory { - public static TelemetryTransports Create(TelemetrySettings telemetrySettings, ExporterSettings exporterSettings) + public TelemetryTransportFactory(TelemetrySettings telemetrySettings) { - var agentProxy = telemetrySettings is { AgentProxyEnabled: true } - ? GetAgentFactory(exporterSettings, telemetrySettings) - : null; + AgentTransportFactory = telemetrySettings switch + { + { AgentProxyEnabled: true } => e => GetAgentFactory(e, telemetrySettings), + _ => null, + }; - var agentless = telemetrySettings is { Agentless: { } a } - ? GetAgentlessFactory(a, telemetrySettings) - : null; + AgentlessTransport = telemetrySettings is { Agentless: { } a } + ? GetAgentlessFactory(a, telemetrySettings) + : null; + } - return new TelemetryTransports(agentProxy, agentless); + // Internal for testing + internal TelemetryTransportFactory( + Func? agentTransportFactory, + ITelemetryTransport? agentlessTransport) + { + AgentTransportFactory = agentTransportFactory; + AgentlessTransport = agentlessTransport; } + public Func? AgentTransportFactory { get; } + + public ITelemetryTransport? AgentlessTransport { get; } + + public bool HasTransports => AgentTransportFactory is not null || AgentlessTransport is not null; + private static ITelemetryTransport GetAgentFactory(ExporterSettings exporterSettings, TelemetrySettings telemetrySettings) => new AgentTelemetryTransport( TelemetryTransportStrategy.GetAgentIntakeFactory(exporterSettings), diff --git a/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransports.cs b/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransports.cs deleted file mode 100644 index e46483c493da..000000000000 --- a/tracer/src/Datadog.Trace/Telemetry/Transports/TelemetryTransports.cs +++ /dev/null @@ -1,22 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#nullable enable -namespace Datadog.Trace.Telemetry.Transports; - -internal class TelemetryTransports -{ - public TelemetryTransports(ITelemetryTransport? agentTransport, ITelemetryTransport? agentlessTransport) - { - AgentTransport = agentTransport; - AgentlessTransport = agentlessTransport; - } - - public ITelemetryTransport? AgentTransport { get; } - - public ITelemetryTransport? AgentlessTransport { get; } - - public bool HasTransports => AgentTransport is not null || AgentlessTransport is not null; -} diff --git a/tracer/test/Datadog.Trace.IntegrationTests/TelemetryTransportTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/TelemetryTransportTests.cs index 3bd8cd45085c..b2c1ca94e3c8 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/TelemetryTransportTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/TelemetryTransportTests.cs @@ -174,20 +174,18 @@ private static TelemetryData GetSampleData() => private static ITelemetryTransport GetAgentOnlyTransport(Uri telemetryUri, string compressionMethod) { - var transport = TelemetryTransportFactory.Create( - new TelemetrySettings(telemetryEnabled: true, configurationError: null, agentlessSettings: null, agentProxyEnabled: true, heartbeatInterval: HeartbeatInterval, dependencyCollectionEnabled: true, metricsEnabled: false, debugEnabled: false, compressionMethod: compressionMethod), - ExporterSettings.Create(new() { { ConfigurationKeys.AgentUri, telemetryUri } })); - transport.AgentTransport.Should().NotBeNull().And.BeOfType(); - return transport.AgentTransport; + var transport = new TelemetryTransportFactory( + new TelemetrySettings(telemetryEnabled: true, configurationError: null, agentlessSettings: null, agentProxyEnabled: true, heartbeatInterval: HeartbeatInterval, dependencyCollectionEnabled: true, metricsEnabled: false, debugEnabled: false, compressionMethod: compressionMethod)); + transport.AgentTransportFactory.Should().NotBeNull(); + return transport.AgentTransportFactory!(ExporterSettings.Create(new() { { ConfigurationKeys.AgentUri, telemetryUri } })); } private static ITelemetryTransport GetAgentlessOnlyTransport(Uri telemetryUri, string apiKey, TelemetrySettings.AgentlessSettings.CloudSettings cloudSettings) { var agentlessSettings = new TelemetrySettings.AgentlessSettings(telemetryUri, apiKey, cloudSettings); - var transport = TelemetryTransportFactory.Create( - new TelemetrySettings(telemetryEnabled: true, configurationError: null, agentlessSettings, agentProxyEnabled: false, heartbeatInterval: HeartbeatInterval, dependencyCollectionEnabled: true, metricsEnabled: false, debugEnabled: false, compressionMethod: GzipCompression), - new ExporterSettings()); + var transport = new TelemetryTransportFactory( + new TelemetrySettings(telemetryEnabled: true, configurationError: null, agentlessSettings, agentProxyEnabled: false, heartbeatInterval: HeartbeatInterval, dependencyCollectionEnabled: true, metricsEnabled: false, debugEnabled: false, compressionMethod: GzipCompression)); transport.AgentlessTransport.Should().NotBeNull().And.BeOfType(); return transport.AgentlessTransport; diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs index 493fa8447a4e..c99ac504757c 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerTests.cs @@ -35,7 +35,7 @@ public class TelemetryControllerTests public async Task TelemetryControllerShouldSendTelemetry() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -56,7 +56,7 @@ public async Task TelemetryControllerShouldSendTelemetry() public async Task TelemetryControllerShouldSendGitMetadataWithTelemetry() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -105,7 +105,7 @@ public async Task TelemetryControllerShouldSendGitMetadataWithTelemetry() public async Task TelemetryControllerShouldUpdateGitMetadataWithTelemetry() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -139,7 +139,7 @@ public async Task TelemetryControllerShouldUpdateGitMetadataWithTelemetry() public async Task TelemetryControllerRecordsConfigurationFromTracerSettings() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var collector = new ConfigurationTelemetry(); var settings = new TracerSettings(); @@ -170,7 +170,7 @@ public async Task TelemetryControllerRecordsConfigurationFromTracerSettings() public async Task TelemetryControllerCanBeDisposedTwice() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -189,7 +189,7 @@ public async Task TelemetryControllerCanBeDisposedTwice() public async Task TelemetrySendsHeartbeatAlongWithData() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -227,7 +227,7 @@ public async Task TelemetrySendsHeartbeatAlongWithData() public async Task TelemetryControllerAddsAllAssembliesToCollector() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var currentAssemblyNames = AppDomain.CurrentDomain .GetAssemblies() @@ -273,7 +273,7 @@ public async Task TelemetryControllerAddsAllAssembliesToCollector() public async Task TelemetryControllerRecordsAppEndpoints() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), @@ -319,7 +319,7 @@ public async Task TelemetryControllerRecordsAppEndpoints() public async Task TelemetryControllerDumpsAllTelemetryToFile() { var transport = new TestTelemetryTransport(pushResult: TelemetryPushResult.Success); - var transportManager = new TelemetryTransportManager(new TelemetryTransports(transport, null), NullDiscoveryService.Instance); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, new TelemetryTransportFactory(_ => transport, null), NullDiscoveryService.Instance); var controller = new TelemetryController( new TracerSettings(), diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryTransportManagerTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryTransportManagerTests.cs index 3bc845e5f428..39c8194e5f51 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryTransportManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryTransportManagerTests.cs @@ -6,6 +6,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.Telemetry; using Datadog.Trace.Telemetry.Transports; using Datadog.Trace.Tests.Agent; @@ -21,8 +22,8 @@ public async Task WhenHaveSuccess_ReturnsTrue() { const int requestCount = 10; var telemetryPushResults = Enumerable.Repeat(TelemetryPushResult.Success, requestCount).ToArray(); - var transports = new TelemetryTransports(new TestTransport(telemetryPushResults), null); - var transportManager = new TelemetryTransportManager(transports, new DiscoveryServiceMock()); + var transports = new TelemetryTransportFactory(_ => new TestTransport(telemetryPushResults), null); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, transports, new DiscoveryServiceMock()); for (var i = 0; i < requestCount; i++) { @@ -38,8 +39,8 @@ public async Task WhenHaveFailure_ReturnsFalse(int result) { const int requestCount = 10; var telemetryPushResults = Enumerable.Repeat((TelemetryPushResult)result, requestCount).ToArray(); - var transports = new TelemetryTransports(new TestTransport(telemetryPushResults), null); - var transportManager = new TelemetryTransportManager(transports, new DiscoveryServiceMock()); + var transports = new TelemetryTransportFactory(_ => new TestTransport(telemetryPushResults), null); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, transports, new DiscoveryServiceMock()); for (var i = 0; i < requestCount; i++) { @@ -52,8 +53,8 @@ public async Task WhenHaveFailure_ReturnsFalse(int result) public async Task TestTransport_ThrowsIfCalledTooManyTimes() { var transport1 = new TestTransport(TelemetryPushResult.TransientFailure); - var transports = new TelemetryTransports(transport1, null); - var transportManager = new TelemetryTransportManager(transports, new DiscoveryServiceMock()); + var transports = new TelemetryTransportFactory(_ => transport1, null); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, transports, new DiscoveryServiceMock()); await transportManager.TryPushTelemetry(null!); @@ -66,8 +67,8 @@ public async Task WhenHaveFailure_SwitchesTransport() { var transport1 = new TestTransport(TelemetryPushResult.TransientFailure, TelemetryPushResult.Success); var transport2 = new TestTransport(TelemetryPushResult.TransientFailure); - var transports = new TelemetryTransports(transport1, transport2); - var transportManager = new TelemetryTransportManager(transports, new DiscoveryServiceMock()); + var transports = new TelemetryTransportFactory(_ => transport1, transport2); + var transportManager = new TelemetryTransportManager(new TracerSettings().Manager, transports, new DiscoveryServiceMock()); var fail1 = await transportManager.TryPushTelemetry(null!); var fail2 = await transportManager.TryPushTelemetry(null!); @@ -84,11 +85,11 @@ public async Task WhenHaveFailure_SwitchesTransport() [InlineData(false)] public void WhenOnlyAgentAvailable_AlwaysUsesAgent(bool? initiallyAvailableInDiscovery) { - var transports = new TelemetryTransports( - agentTransport: new TestTransport(), + var transports = new TelemetryTransportFactory( + agentTransportFactory: _ => new TestTransport(), agentlessTransport: null); var discoveryService = new DiscoveryServiceMock(); - var manager = new TelemetryTransportManager(transports, discoveryService); + var manager = new TelemetryTransportManager(new TracerSettings().Manager, transports, discoveryService); if (initiallyAvailableInDiscovery == true) { @@ -101,21 +102,21 @@ public void WhenOnlyAgentAvailable_AlwaysUsesAgent(bool? initiallyAvailableInDis // initial value var nextTransport = manager.GetNextTransport(null); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes // on error nextTransport = manager.GetNextTransport(nextTransport); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes // agent no longer available discoveryService.TriggerChange(telemetryProxyEndpoint: null); nextTransport = manager.GetNextTransport(nextTransport); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes // agent available again discoveryService.TriggerChange(); nextTransport = manager.GetNextTransport(nextTransport); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes } [Theory] @@ -124,9 +125,9 @@ public void WhenOnlyAgentAvailable_AlwaysUsesAgent(bool? initiallyAvailableInDis [InlineData(false)] public void WhenOnlyAgentlessAvailable_AlwaysUsesAgentless(bool? initiallyAvailableInDiscovery) { - var transports = new TelemetryTransports(agentTransport: null, agentlessTransport: new TestTransport()); + var transports = new TelemetryTransportFactory(agentTransportFactory: _ => null, agentlessTransport: new TestTransport()); var discoveryService = new DiscoveryServiceMock(); - var manager = new TelemetryTransportManager(transports, discoveryService); + var manager = new TelemetryTransportManager(new TracerSettings().Manager, transports, discoveryService); if (initiallyAvailableInDiscovery == true) { @@ -161,9 +162,9 @@ public void WhenOnlyAgentlessAvailable_AlwaysUsesAgentless(bool? initiallyAvaila [InlineData(true)] public void WhenBothAvailable_AndInitiallyAvailableOrUnknownDiscovery_UsesAgent(bool notifyAvailable) { - var transports = new TelemetryTransports(agentTransport: new TestTransport(), agentlessTransport: new TestTransport()); + var transports = new TelemetryTransportFactory(agentTransportFactory: _ => new TestTransport(), agentlessTransport: new TestTransport()); var discoveryService = new DiscoveryServiceMock(); - var manager = new TelemetryTransportManager(transports, discoveryService); + var manager = new TelemetryTransportManager(new TracerSettings().Manager, transports, discoveryService); if (notifyAvailable) { @@ -172,15 +173,15 @@ public void WhenBothAvailable_AndInitiallyAvailableOrUnknownDiscovery_UsesAgent( // initial value var nextTransport = manager.GetNextTransport(null); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes } [Fact] public void WhenBothAvailable_AndInitiallyUnAvailable_UsesAgentless() { - var transports = new TelemetryTransports(agentTransport: new TestTransport(), agentlessTransport: new TestTransport()); + var transports = new TelemetryTransportFactory(agentTransportFactory: _ => new TestTransport(), agentlessTransport: new TestTransport()); var discoveryService = new DiscoveryServiceMock(); - var manager = new TelemetryTransportManager(transports, discoveryService); + var manager = new TelemetryTransportManager(new TracerSettings().Manager, transports, discoveryService); discoveryService.TriggerChange(telemetryProxyEndpoint: null); @@ -192,13 +193,13 @@ public void WhenBothAvailable_AndInitiallyUnAvailable_UsesAgentless() [Fact] public void WhenBothAvailable_UsesNextExpectedTransport() { - var transports = new TelemetryTransports(agentTransport: new TestTransport(), agentlessTransport: new TestTransport()); + var transports = new TelemetryTransportFactory(agentTransportFactory: _ => new TestTransport(), agentlessTransport: new TestTransport()); var discoveryService = new DiscoveryServiceMock(); - var manager = new TelemetryTransportManager(transports, discoveryService); + var manager = new TelemetryTransportManager(new TracerSettings().Manager, transports, discoveryService); // initial value var nextTransport = manager.GetNextTransport(null); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // Equivalent to "should be agent" but accounting for the fact we respond to config changes // we now know agent is available, but it failed, so switch to agentless discoveryService.TriggerChange(); @@ -207,7 +208,7 @@ public void WhenBothAvailable_UsesNextExpectedTransport() // agentless failed, and agent is available, so switch to agent nextTransport = manager.GetNextTransport(nextTransport); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // agent failed, so switch back to agentless nextTransport = manager.GetNextTransport(nextTransport); @@ -225,7 +226,7 @@ public void WhenBothAvailable_UsesNextExpectedTransport() // Agent is available again, so switch to agent discoveryService.TriggerChange(); nextTransport = manager.GetNextTransport(nextTransport); - nextTransport.Should().Be(transports.AgentTransport); + nextTransport.Should().NotBe(transports.AgentlessTransport); // And we're back to the starting point again nextTransport = manager.GetNextTransport(nextTransport); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/Transports/TelemetryTransportFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/Telemetry/Transports/TelemetryTransportFactoryTests.cs index 97d1153e1c45..8686f1ee6966 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/Transports/TelemetryTransportFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/Transports/TelemetryTransportFactoryTests.cs @@ -34,12 +34,12 @@ public void UsesCorrectTransports(bool agentProxyEnabled, bool agentlessEnabled) var exporterSettings = new ExporterSettings(); - var transports = TelemetryTransportFactory.Create(telemetrySettings, exporterSettings); + var transports = new TelemetryTransportFactory(telemetrySettings); using var s = new AssertionScope(); if (agentProxyEnabled) { - transports.AgentTransport.Should().NotBeNull().And.BeOfType(); + transports.AgentTransportFactory?.Invoke(exporterSettings).Should().NotBeNull().And.BeOfType(); } if (agentlessEnabled) From 7e0ea487b8ecfdbf721ea6f6d044472812a43c41 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 27 Oct 2025 15:58:17 +0000 Subject: [PATCH 18/29] Update dynamic config usages --- tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs | 3 +++ .../Configuration/DynamicConfigurationTests.cs | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs b/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs index d29d5930537a..ed81894803bd 100644 --- a/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs +++ b/tracer/src/Datadog.Trace/Sampling/ManagedTraceSampler.cs @@ -83,6 +83,9 @@ public void SetDefaultSampleRates(IReadOnlyDictionary sampleRates public SamplingDecision MakeSamplingDecision(Span span) => Volatile.Read(ref _current).MakeSamplingDecision(span); + // used for testing + internal IReadOnlyList GetRules() => Volatile.Read(ref _current).GetRules(); + private static TraceSampler CreateSampler(MutableSettings settings, string customSamplingRulesFormat) { // ISamplingRule is used to implement, in order of precedence: diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs index b355e26eece8..3f987f0b6458 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs @@ -45,11 +45,12 @@ public void ApplyConfigurationTwice() { var tracer = TracerManager.Instance; + // We no longer replace tracer instances when reconfiguring the tracer DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)), tracer.Settings); var newTracer = TracerManager.Instance; - newTracer.Should().NotBeSameAs(tracer); + newTracer.Should().BeSameAs(tracer); DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(CreateConfig(("tracing_sampling_rate", 0.4)), tracer.Settings); @@ -148,7 +149,7 @@ public void SetSamplingRules() tracerManager.PerTraceSettings.Settings.CustomSamplingRules.Should().Be(localSamplingRulesJson); tracerManager.PerTraceSettings.Settings.CustomSamplingRulesIsRemote.Should().BeFalse(); - var rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); + var rules = ((ManagedTraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); rules.Should() .BeEquivalentTo( @@ -179,7 +180,7 @@ public void SetSamplingRules() tracerManager.PerTraceSettings.Settings.CustomSamplingRules.Should().Be(remoteSamplingRulesJson); tracerManager.PerTraceSettings.Settings.CustomSamplingRulesIsRemote.Should().BeTrue(); - rules = ((TraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); + rules = ((ManagedTraceSampler)tracerManager.PerTraceSettings.TraceSampler)!.GetRules(); // new list should include the remote rules, not the local rules rules.Should() From 9a922764492370bea8e3884cf0d956a8bf3e7ee8 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 27 Oct 2025 16:51:30 +0000 Subject: [PATCH 19/29] Update debugger ExporterSettings - this only uses the initial settings though, and doesn't respond to changes --- tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs | 6 +++--- .../ExceptionAutoInstrumentation/ExceptionReplay.cs | 3 ++- tracer/src/Datadog.Trace/TracerManager.cs | 8 +++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs index 906ef05d6391..93c7b717a057 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs @@ -60,7 +60,7 @@ private static IDogStatsd GetDogStatsd(TracerSettings tracerSettings, string ser { IDogStatsd statsd; if (FrameworkDescription.Instance.IsWindows() - && tracerSettings.Exporter.MetricsTransport == TransportType.UDS) + && tracerSettings.Manager.InitialExporterSettings.MetricsTransport == TransportType.UDS) { Log.Information("Metric probes are not supported on Windows when transport type is UDS"); statsd = NoOpStatsd.Instance; @@ -70,7 +70,7 @@ private static IDogStatsd GetDogStatsd(TracerSettings tracerSettings, string ser // TODO: use StatsdManager to get automatic updating on exporter and other setting changes statsd = StatsdFactory.CreateDogStatsdClient( tracerSettings.Manager.InitialMutableSettings, - tracerSettings.Exporter, + tracerSettings.Manager.InitialExporterSettings, includeDefaultTags: false, prefix: DebuggerSettings.DebuggerMetricPrefix); } @@ -119,7 +119,7 @@ private static IApiRequestFactory GetApiFactory(TracerSettings tracerSettings, b { // TODO: we need to be able to update the tracer settings dynamically return AgentTransportStrategy.Get( - tracerSettings.Exporter, + tracerSettings.Manager.InitialExporterSettings, productName: "debugger", tcpTimeout: TimeSpan.FromSeconds(15), AgentHttpHeaderNames.MinimalHeaders, diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs index 2c05f287c3fc..952e32175c47 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplay.cs @@ -65,8 +65,9 @@ private void InitSnapshotsSink() // Set up the snapshots sink. var snapshotSlicer = SnapshotSlicer.Create(debuggerSettings); _snapshotSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); + // TODO: respond to changes in exporter settings var apiFactory = AgentTransportStrategy.Get( - tracer.Settings.Exporter, + tracer.Settings.Manager.InitialExporterSettings, productName: "debugger", tcpTimeout: TimeSpan.FromSeconds(15), AgentHttpHeaderNames.MinimalHeaders, diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 61dd6fdc188a..7088e489578e 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -326,6 +326,8 @@ private static async Task WriteDiagnosticLog(TracerManager instance) string agentError = null; var instanceSettings = instance.Settings; var mutableSettings = instance.PerTraceSettings.Settings; + // TODO: this only writes the initial settings - we should make sure to record an "update" log on reconfiguration + var exporterSettings = instanceSettings.Manager.InitialExporterSettings; // In AAS, the trace agent is deployed alongside the tracer and managed by the tracer // Disable this check as it may hit the trace agent before it is ready to receive requests and give false negatives @@ -403,10 +405,10 @@ void WriteDictionary(IReadOnlyDictionary dictionary) writer.WriteValue(mutableSettings.DefaultServiceName); writer.WritePropertyName("agent_url"); - writer.WriteValue(instanceSettings.Exporter.TraceAgentUriBase); + writer.WriteValue(exporterSettings.TraceAgentUriBase); writer.WritePropertyName("agent_transport"); - writer.WriteValue(instanceSettings.Exporter.TracesTransport.ToString()); + writer.WriteValue(exporterSettings.TracesTransport.ToString()); writer.WritePropertyName("debug"); writer.WriteValue(GlobalSettings.Instance.DebugEnabled); @@ -520,7 +522,7 @@ void WriteDictionary(IReadOnlyDictionary dictionary) writer.WritePropertyName("exporter_settings_warning"); writer.WriteStartArray(); - foreach (var warning in instanceSettings.Exporter.ValidationWarnings) + foreach (var warning in exporterSettings.ValidationWarnings) { writer.WriteValue(warning); } From 6208fa5a934f6edf1b93b4dd9f8e32c65237de50 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Mon, 27 Oct 2025 17:23:27 +0000 Subject: [PATCH 20/29] Update DiscoveryService --- .../src/Datadog.Trace.Tools.Runner/Utils.cs | 8 +- .../DiscoveryService/DiscoveryService.cs | 76 +++++++++++++++---- .../src/Datadog.Trace/Ci/TestOptimization.cs | 2 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 2 +- .../LibDatadog/TraceExporterTests.cs | 2 +- .../StatsTests.cs | 6 +- 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/tracer/src/Datadog.Trace.Tools.Runner/Utils.cs b/tracer/src/Datadog.Trace.Tools.Runner/Utils.cs index d4754c55a7b4..94a778ad6642 100644 --- a/tracer/src/Datadog.Trace.Tools.Runner/Utils.cs +++ b/tracer/src/Datadog.Trace.Tools.Runner/Utils.cs @@ -417,9 +417,9 @@ public static async Task CheckAgentConnectionAsync(string ag var settings = new TracerSettings(configurationSource, new ConfigurationTelemetry(), new OverrideErrorLog()); - Log.Debug("Creating DiscoveryService for: {AgentUri}", settings.Exporter.AgentUri); - var discoveryService = DiscoveryService.Create( - settings.Exporter, + Log.Debug("Creating DiscoveryService for: {AgentUri}", settings.Manager.InitialExporterSettings.AgentUri); + var discoveryService = DiscoveryService.CreateUnmanaged( + settings.Manager.InitialExporterSettings, tcpTimeout: TimeSpan.FromSeconds(5), initialRetryDelayMs: 200, maxRetryDelayMs: 1000, @@ -433,7 +433,7 @@ public static async Task CheckAgentConnectionAsync(string ag using (cts.Token.Register( () => { - WriteError($"Error connecting to the Datadog Agent at {settings.Exporter.AgentUri}."); + WriteError($"Error connecting to the Datadog Agent at {settings.Manager.InitialExporterSettings.AgentUri}."); tcs.TrySetResult(null); })) { diff --git a/tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs b/tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs index 23186cf276be..97a04e6f4e1b 100644 --- a/tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs +++ b/tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs @@ -35,7 +35,6 @@ internal class DiscoveryService : IDiscoveryService private const string SupportedTracerFlareEndpoint = "tracer_flare/v1"; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private readonly IApiRequestFactory _apiRequestFactory; private readonly int _initialRetryDelayMs; private readonly int _maxRetryDelayMs; private readonly int _recheckIntervalMs; @@ -43,8 +42,29 @@ internal class DiscoveryService : IDiscoveryService private readonly List> _agentChangeCallbacks = new(); private readonly object _lock = new(); private readonly Task _discoveryTask; + private readonly IDisposable? _settingSubscription; + private IApiRequestFactory _apiRequestFactory; private AgentConfiguration? _configuration; + public DiscoveryService( + TracerSettings.SettingsManager settings, + TimeSpan tcpTimeout, + int initialRetryDelayMs, + int maxRetryDelayMs, + int recheckIntervalMs) + : this(CreateApiRequestFactory(settings.InitialExporterSettings, tcpTimeout), initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs) + { + // Create as a "managed" service that can update the request factory + _settingSubscription = settings.SubscribeToChanges(changes => + { + if (changes.UpdatedExporter is { } exporter) + { + var newFactory = CreateApiRequestFactory(exporter, tcpTimeout); + Interlocked.Exchange(ref _apiRequestFactory!, newFactory); + } + }); + } + /// /// Initializes a new instance of the class. /// Public for testing purposes @@ -82,28 +102,39 @@ public DiscoveryService( SupportedTracerFlareEndpoint, }; - public static DiscoveryService Create(ExporterSettings exporterSettings) - => Create( + /// + /// Create a instance that responds to runtime changes in settings + /// + public static DiscoveryService CreateManaged(TracerSettings settings) + => new( + settings.Manager, + tcpTimeout: TimeSpan.FromSeconds(15), + initialRetryDelayMs: 500, + maxRetryDelayMs: 5_000, + recheckIntervalMs: 30_000); + + /// + /// Create a instance that does _not_ respond to runtime changes in settings + /// + public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings) + => CreateUnmanaged( exporterSettings, tcpTimeout: TimeSpan.FromSeconds(15), initialRetryDelayMs: 500, maxRetryDelayMs: 5_000, recheckIntervalMs: 30_000); - public static DiscoveryService Create( + /// + /// Create a instance that does _not_ respond to runtime changes in settings + /// + public static DiscoveryService CreateUnmanaged( ExporterSettings exporterSettings, TimeSpan tcpTimeout, int initialRetryDelayMs, int maxRetryDelayMs, int recheckIntervalMs) => new( - AgentTransportStrategy.Get( - exporterSettings, - productName: "discovery", - tcpTimeout: tcpTimeout, - AgentHttpHeaderNames.MinimalHeaders, - () => new MinimalAgentHeaderHelper(), - uri => uri), + CreateApiRequestFactory(exporterSettings, tcpTimeout), initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs); @@ -169,7 +200,8 @@ private void NotifySubscribers(AgentConfiguration newConfig) private async Task FetchConfigurationLoopAsync() { - var uri = _apiRequestFactory.GetEndpoint("info"); + var requestFactory = _apiRequestFactory; + var uri = requestFactory.GetEndpoint("info"); int? sleepDuration = null; @@ -177,7 +209,15 @@ private async Task FetchConfigurationLoopAsync() { try { - var api = _apiRequestFactory.Create(uri); + // If the exporter settings have been updated, refresh the endpoint + var updatedFactory = Volatile.Read(ref _apiRequestFactory); + if (requestFactory != updatedFactory) + { + requestFactory = updatedFactory; + uri = requestFactory.GetEndpoint("info"); + } + + var api = requestFactory.Create(uri); using var response = await api.GetAsync().ConfigureAwait(false); if (response.StatusCode is >= 200 and < 300) @@ -323,6 +363,7 @@ private async Task ProcessDiscoveryResponse(IApiResponse response) public Task DisposeAsync() { + _settingSubscription?.Dispose(); if (!_processExit.TrySetResult(true)) { // Double dispose in prod shouldn't happen, and should be avoided, so logging for follow-up @@ -331,5 +372,14 @@ public Task DisposeAsync() return _discoveryTask; } + + private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings, TimeSpan tcpTimeout) + => AgentTransportStrategy.Get( + exporterSettings, + productName: "discovery", + tcpTimeout: tcpTimeout, + AgentHttpHeaderNames.MinimalHeaders, + () => new MinimalAgentHeaderHelper(), + uri => uri); } } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs index b969a4e459d5..6340c18ee598 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs @@ -246,7 +246,7 @@ public void Initialize() // In case we are running using the agent, check if the event platform proxy is supported. TracerManagement = new TestOptimizationTracerManagement( settings: Settings, - getDiscoveryServiceFunc: static s => DiscoveryService.Create( + getDiscoveryServiceFunc: static s => DiscoveryService.CreateUnmanaged( s.TracerSettings.Manager.InitialExporterSettings, tcpTimeout: TimeSpan.FromSeconds(5), initialRetryDelayMs: 100, diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index e6e80cb391dc..084e8fe57c1b 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -285,7 +285,7 @@ protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStats internal virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) => settings.AgentFeaturePollingEnabled ? - DiscoveryService.Create(settings.Exporter) : + DiscoveryService.CreateManaged(settings) : NullDiscoveryService.Instance; } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index 0c06d0a6afe7..d86e2be379cc 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -75,7 +75,7 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) var sampleRateResponses = new ConcurrentQueue>(); - var discovery = DiscoveryService.Create(tracerSettings.Exporter); + var discovery = DiscoveryService.CreateUnmanaged(tracerSettings.Manager.InitialExporterSettings); var statsd = new NoOpStatsd(); // We have to replace the agent writer so that we can intercept the sample rate responses diff --git a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs index 06e3563d3aa0..ff2ee9b74b08 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs @@ -58,7 +58,7 @@ public async Task SendsStatsWithProcessing_Normalizer() { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); - var discovery = DiscoveryService.Create(settings.Exporter); + var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings); // Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point // and this was easiest to retrofit without changing the test structure too much. var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery); @@ -205,7 +205,7 @@ public async Task SendsStatsWithProcessing_Obfuscator() { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); - var discovery = DiscoveryService.Create(settings.Exporter); + var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings); // Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point // and this was easiest to retrofit without changing the test structure too much. var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery); @@ -366,7 +366,7 @@ private async Task SendStatsHelper(bool statsComputationEnabled, bool expectStat { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, })); - var discovery = DiscoveryService.Create(settings.Exporter); + var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings); // Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point // and this was easiest to retrofit without changing the test structure too much. var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery); From c10eca5187e41df765be58ea3e891a6b2a27ba66 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 28 Oct 2025 10:28:28 +0000 Subject: [PATCH 21/29] Update TracerFlareApi --- .../Logging/TracerFlare/TracerFlareApi.cs | 54 ++++++++++++------- .../src/Datadog.Trace/TracerManagerFactory.cs | 2 +- .../TracerFlare/TracerFlareApiTests.cs | 20 ++++--- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/tracer/src/Datadog.Trace/Logging/TracerFlare/TracerFlareApi.cs b/tracer/src/Datadog.Trace/Logging/TracerFlare/TracerFlareApi.cs index 79c03dbe2cb1..60cbc03761dd 100644 --- a/tracer/src/Datadog.Trace/Logging/TracerFlare/TracerFlareApi.cs +++ b/tracer/src/Datadog.Trace/Logging/TracerFlare/TracerFlareApi.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; @@ -23,35 +24,38 @@ internal class TracerFlareApi private const string TracerFlareEndpoint = "tracer_flare/v1"; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private readonly IApiRequestFactory _requestFactory; - private readonly Uri _endpoint; + private Api _api; + // Internal for testing public TracerFlareApi(IApiRequestFactory requestFactory) { - _requestFactory = requestFactory; - _endpoint = _requestFactory.GetEndpoint(TracerFlareEndpoint); + _api = new(requestFactory, requestFactory.GetEndpoint(TracerFlareEndpoint)); } - public static TracerFlareApi Create(ExporterSettings exporterSettings) + public TracerFlareApi(TracerSettings.SettingsManager settings) + : this(CreateApiRequestFactory(settings.InitialExporterSettings)) { - var requestFactory = AgentTransportStrategy.Get( - exporterSettings, - productName: "tracer_flare", - tcpTimeout: TimeSpan.FromSeconds(30), - AgentHttpHeaderNames.MinimalHeaders, - () => new MinimalAgentHeaderHelper(), - uri => uri); - - return new TracerFlareApi(requestFactory); + settings.SubscribeToChanges(changes => + { + if (changes.UpdatedExporter is { } exporter) + { + var requestFactory = CreateApiRequestFactory(exporter); + var api = new Api(requestFactory, requestFactory.GetEndpoint(TracerFlareEndpoint)); + Interlocked.Exchange(ref _api, api); + } + }); } + public static TracerFlareApi CreateManaged(TracerSettings.SettingsManager settings) => new(settings); + public async Task> SendTracerFlare(Func writeFlareToStreamFunc, string caseId, string hostname, string email) { + var api = Volatile.Read(ref _api); try { - Log.Debug("Sending tracer flare to {Endpoint}", _endpoint); + Log.Debug("Sending tracer flare to {Endpoint}", api.Endpoint); - var request = _requestFactory.Create(_endpoint); + var request = api.RequestFactory.Create(api.Endpoint); using var response = await request.PostAsync( stream => TracerFlareRequestFactory.WriteRequestBody(stream, writeFlareToStreamFunc, caseId, hostname: hostname, email: email), MimeTypes.MultipartFormData, @@ -80,13 +84,27 @@ public static TracerFlareApi Create(ExporterSettings exporterSettings) Log.Warning(e, "Error parsing {StatusCode} response from tracer flare endpoint: {ResponseContent}", response.StatusCode, responseContent); } - Log.Warning("Error sending tracer flare to '{Endpoint}' {StatusCode} ", _requestFactory.Info(_endpoint), response.StatusCode); + Log.Warning("Error sending tracer flare to '{Endpoint}' {StatusCode} ", api.RequestFactory.Info(api.Endpoint), response.StatusCode); return new(false, error); } catch (Exception ex) { - Log.Information(ex, "Error sending tracer flare to '{Endpoint}'", _requestFactory.Info(_endpoint)); + Log.Information(ex, "Error sending tracer flare to '{Endpoint}'", api.RequestFactory.Info(api.Endpoint)); return new(false, null); } } + + private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings) + { + var requestFactory = AgentTransportStrategy.Get( + exporterSettings, + productName: "tracer_flare", + tcpTimeout: TimeSpan.FromSeconds(30), + AgentHttpHeaderNames.MinimalHeaders, + () => new MinimalAgentHeaderHelper(), + uri => uri); + return requestFactory; + } + + private record Api(IApiRequestFactory RequestFactory, Uri Endpoint); } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 084e8fe57c1b..85dc4b7404fe 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -169,7 +169,7 @@ internal TracerManager CreateTracerManager( } dynamicConfigurationManager ??= new DynamicConfigurationManager(RcmSubscriptionManager.Instance); - tracerFlareManager ??= new TracerFlareManager(discoveryService, RcmSubscriptionManager.Instance, telemetry, TracerFlareApi.Create(settings.Exporter)); + tracerFlareManager ??= new TracerFlareManager(discoveryService, RcmSubscriptionManager.Instance, telemetry, TracerFlareApi.CreateManaged(settings.Manager)); spanEventsManager ??= new SpanEventsManager(discoveryService); } else diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs index 4553fc8b1bc9..df13bf3339a8 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs @@ -32,7 +32,7 @@ public async Task CanSendToAgent_Tcp() { using var agent = MockTracerAgent.Create(output); var agentPath = new Uri($"http://localhost:{agent.Port}"); - var settings = ExporterSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); + var settings = TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); await RunTest(settings, agent); } @@ -45,8 +45,7 @@ public async Task CanSendToAgent_UDS() { using var agent = MockTracerAgent.Create(output, new UnixDomainSocketConfig(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()), null)); var agentPath = agent.TracesUdsPath; - var settings = new ExporterSettings( - new NameValueConfigurationSource(new() { { "DD_APM_RECEIVER_SOCKET", agentPath } })); + var settings = TracerSettings.Create(new() { { "DD_APM_RECEIVER_SOCKET", agentPath } }); await RunTest(settings, agent); } @@ -82,8 +81,7 @@ async Task RunNamedPipesTest() { using var agent = MockTracerAgent.Create(output, new WindowsPipesConfig($"trace-{Guid.NewGuid()}", null)); var pipeName = agent.TracesWindowsPipeName; - var settings = new ExporterSettings( - new NameValueConfigurationSource(new() { { "DD_TRACE_PIPE_NAME", pipeName } })); + var settings = TracerSettings.Create(new() { { "DD_TRACE_PIPE_NAME", pipeName } }); await RunTest(settings, agent); } @@ -96,12 +94,12 @@ public async Task ReturnsFalseWhenSendFails() { using var agent = MockTracerAgent.Create(output); var agentPath = new Uri($"http://localhost:{agent.Port}"); - var settings = ExporterSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); + var settings = TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); var invalidJson = "{meep"; agent.CustomResponses[MockTracerResponseType.TracerFlare] = new MockTracerResponse(invalidJson, 500); - var api = TracerFlareApi.Create(settings); + var api = TracerFlareApi.CreateManaged(settings.Manager); var result = await api.SendTracerFlare(WriteFlareToStreamFunc, CaseId, Hostname, Email); @@ -118,12 +116,12 @@ public async Task ReturnsErrorMessageWhenSendFails() { using var agent = MockTracerAgent.Create(output); var agentPath = new Uri($"http://localhost:{agent.Port}"); - var settings = ExporterSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); + var settings = TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, agentPath } }); var somethingWentWrong = "Something went wrong"; agent.CustomResponses[MockTracerResponseType.TracerFlare] = new MockTracerResponse($$"""{ "error": "{{somethingWentWrong}}" }""", 500); - var api = TracerFlareApi.Create(settings); + var api = TracerFlareApi.CreateManaged(settings.Manager); var result = await api.SendTracerFlare(WriteFlareToStreamFunc, CaseId, Hostname, Email); @@ -133,9 +131,9 @@ public async Task ReturnsErrorMessageWhenSendFails() result.Value.Should().Be(somethingWentWrong); } - private async Task RunTest(ExporterSettings settings, MockTracerAgent agent) + private async Task RunTest(TracerSettings settings, MockTracerAgent agent) { - var api = TracerFlareApi.Create(settings); + var api = TracerFlareApi.CreateManaged(settings.Manager); var result = await api.SendTracerFlare(WriteFlareToStreamFunc, CaseId, Hostname, Email); From 1d6ad9f45e9aee120a73bccf94c02b883f0ff17d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 28 Oct 2025 12:11:14 +0000 Subject: [PATCH 22/29] Remove Exporter property from TracerSettings --- .../Configuration/TracerSettings.cs | 29 +++++-------------- .../Configuration/ConfigurationSourceTests.cs | 8 ++--- .../Configuration/TracerSettingsTests.cs | 6 ++-- .../Datadog.Trace.Tests/DogStatsDTests.cs | 10 +++---- .../SettingsInstrumentationTests.cs | 10 +++---- 5 files changed, 23 insertions(+), 40 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index a4c62283d1e0..6fee833ca806 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -125,7 +125,7 @@ internal TracerSettings(IConfigurationSource? source, IConfigurationTelemetry te .AsBoolResult() .OverrideWith(in otelActivityListenerEnabled, ErrorLog, defaultValue: false); - Exporter = new ExporterSettings(source, _telemetry); + var exporter = new ExporterSettings(source, _telemetry); PeerServiceTagsEnabled = config .WithKeys(ConfigurationKeys.PeerServiceDefaultsEnabled) @@ -353,7 +353,11 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = // Windows supports UnixDomainSocket https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ // but tokio hasn't added support for it yet https://github.com/tokio-rs/tokio/issues/2201 - if (Exporter.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) + // There's an issue here, in that technically a user can initially be configured to send over TCP/named pipes, + // and so we allow and enable the datapipeline. Later, they could configure the app in code to send over UDS. + // This is a problem, as we currently don't support toggling the data pipeline at runtime, so we explicitly block + // this scenario in the public API. + if (exporter.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) { DataPipelineEnabled = false; Log.Warning( @@ -729,7 +733,7 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = // Move the creation of these settings inside SettingsManager? var initialMutableSettings = MutableSettings.CreateInitialMutableSettings(source, telemetry, errorLog, this); - Manager = new(this, initialMutableSettings, Exporter); + Manager = new(this, initialMutableSettings, exporter); } internal bool IsRunningInCiVisibility { get; } @@ -889,11 +893,6 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = /// internal string[] DisabledActivitySources { get; } - /// - /// Gets the transport settings that dictate how the tracer connects to the agent. - /// - public ExporterSettings Exporter { get; init; } - /// /// Gets a value indicating the format for custom trace sampling rules ("regex" or "glob"). /// If the value is not recognized, trace sampling rules are disabled. @@ -1355,19 +1354,5 @@ internal static TracerSettings Create(Dictionary settings) internal static TracerSettings Create(Dictionary settings, LibDatadogAvailableResult isLibDatadogAvailable) => new(new DictionaryConfigurationSource(settings.ToDictionary(x => x.Key, x => x.Value?.ToString()!)), new ConfigurationTelemetry(), new(), isLibDatadogAvailable); - - internal void CollectTelemetry(IConfigurationTelemetry destination) - { - // copy the current settings into telemetry - _telemetry.CopyTo(destination); - - // If ExporterSettings has been replaced, it will have its own telemetry collector - // so we need to record those values too. - if (Exporter.Telemetry is { } exporterTelemetry - && exporterTelemetry != _telemetry) - { - exporterTelemetry.CopyTo(destination); - } - } } } diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs index fb2ec8137011..af87926a762e 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/ConfigurationSourceTests.cs @@ -72,7 +72,7 @@ public ConfigurationSourceTests() public static IEnumerable<(Func SettingGetter, object ExpectedValue)> GetDefaultTestData() { yield return (s => s.Manager.InitialMutableSettings.TraceEnabled, true); - yield return (s => s.Exporter.AgentUri, new Uri("http://127.0.0.1:8126/")); + yield return (s => s.Manager.InitialExporterSettings.AgentUri, new Uri("http://127.0.0.1:8126/")); yield return (s => s.Manager.InitialMutableSettings.Environment, null); yield return (s => s.Manager.InitialMutableSettings.ServiceName, null); yield return (s => s.Manager.InitialMutableSettings.DisabledIntegrationNames.Count, 1); // The OpenTelemetry integration is disabled by default @@ -84,7 +84,7 @@ public ConfigurationSourceTests() yield return (s => s.Manager.InitialMutableSettings.CustomSamplingRules, null); yield return (s => s.Manager.InitialMutableSettings.MaxTracesSubmittedPerSecond, 100); yield return (s => s.Manager.InitialMutableSettings.TracerMetricsEnabled, false); - yield return (s => s.Exporter.DogStatsdPort, 8125); + yield return (s => s.Manager.InitialExporterSettings.DogStatsdPort, 8125); yield return (s => s.PropagationStyleInject, new[] { "Datadog", "tracecontext", "baggage" }); yield return (s => s.PropagationStyleExtract, new[] { "Datadog", "tracecontext", "baggage" }); yield return (s => s.Manager.InitialMutableSettings.ServiceNameMappings, new string[0]); @@ -108,8 +108,8 @@ public ConfigurationSourceTests() yield return (ConfigurationKeys.TraceEnabled, "true", s => s.Manager.InitialMutableSettings.TraceEnabled, true); yield return (ConfigurationKeys.TraceEnabled, "false", s => s.Manager.InitialMutableSettings.TraceEnabled, false); - yield return (ConfigurationKeys.AgentHost, "test-host", s => s.Exporter.AgentUri, new Uri("http://test-host:8126/")); - yield return (ConfigurationKeys.AgentPort, "9000", s => s.Exporter.AgentUri, new Uri("http://127.0.0.1:9000/")); + yield return (ConfigurationKeys.AgentHost, "test-host", s => s.Manager.InitialExporterSettings.AgentUri, new Uri("http://test-host:8126/")); + yield return (ConfigurationKeys.AgentPort, "9000", s => s.Manager.InitialExporterSettings.AgentUri, new Uri("http://127.0.0.1:9000/")); yield return (ConfigurationKeys.Environment, "staging", s => s.Manager.InitialMutableSettings.Environment, "staging"); diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsTests.cs index a3406ba739dc..25f8a5348cbf 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsTests.cs @@ -36,7 +36,7 @@ public void ReplaceLocalhost(string original, string expected) var tracerSettings = new TracerSettings(new NameValueConfigurationSource(settings)); - Assert.Equal(expected, tracerSettings.Exporter.AgentUri.ToString()); + Assert.Equal(expected, tracerSettings.Manager.InitialExporterSettings.AgentUri.ToString()); } [Theory] @@ -681,9 +681,7 @@ public void IsRemoteConfigurationAvailable_AzureAppService(bool? overrideValue, public void RecordsTelemetryAboutTfm() { var tracerSettings = new TracerSettings(NullConfigurationSource.Instance); - var collector = new ConfigurationTelemetry(); - tracerSettings.CollectTelemetry(collector); - var data = collector.GetData(); + var data = tracerSettings.Telemetry.GetData(); var value = data .GroupBy(x => x.Name) .Should() diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs index 24cbadb17667..487b80ffcb86 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs @@ -128,8 +128,8 @@ public void CanCreateDogStatsD_UDP_FromTraceAgentSettings(string agentUri, strin { ConfigurationKeys.DogStatsdPort, port }, })); - settings.Exporter.MetricsTransport.Should().Be(TransportType.UDP); - var expectedPort = settings.Exporter.DogStatsdPort; + settings.Manager.InitialExporterSettings.MetricsTransport.Should().Be(TransportType.UDP); + var expectedPort = settings.Manager.InitialExporterSettings.DogStatsdPort; // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing @@ -160,7 +160,7 @@ public void CanCreateDogStatsD_NamedPipes_FromTraceAgentSettings() { ConfigurationKeys.MetricsPipeName, agent.StatsWindowsPipeName }, })); - settings.Exporter.MetricsTransport.Should().Be(TransportType.NamedPipe); + settings.Manager.InitialExporterSettings.MetricsTransport.Should().Be(TransportType.NamedPipe); // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing @@ -194,7 +194,7 @@ public void CanCreateDogStatsD_UDS_FromTraceAgentSettings() { ConfigurationKeys.MetricsUnixDomainSocketPath, $"unix://{metricsPath}" }, })); - settings.Exporter.MetricsTransport.Should().Be(TransportType.UDS); + settings.Manager.InitialExporterSettings.MetricsTransport.Should().Be(TransportType.UDS); // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing @@ -226,7 +226,7 @@ public void CanCreateDogStatsD_UDS_FallsBackToUdp_FromTraceAgentSettings() // If we're not using the "default" UDS path, then we fallback to UDP for stats // Should fallback to the "default" stats location - settings.Exporter.MetricsTransport.Should().Be(TransportType.UDP); + settings.Manager.InitialExporterSettings.MetricsTransport.Should().Be(TransportType.UDP); // Dogstatsd tries to actually contact the agent during creation, so need to have something listening // No guarantees it's actually using the _right_ config here, but it's better than nothing diff --git a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs index 997babf0f896..985c48fd3696 100644 --- a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs @@ -228,8 +228,8 @@ public void AutomaticToManual_ImmutableSettingsAreTransferredCorrectly() var manual = new ImmutableManualSettings(serializedSettings); - manual.AgentUri.Should().Be(automatic.Exporter.AgentUri); - manual.Exporter.AgentUri.Should().Be(automatic.Exporter.AgentUri); + manual.AgentUri.Should().Be(automatic.Manager.InitialExporterSettings.AgentUri); + manual.Exporter.AgentUri.Should().Be(automatic.Manager.InitialExporterSettings.AgentUri); manual.AnalyticsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.AnalyticsEnabled); manual.CustomSamplingRules.Should().Be(automatic.Manager.InitialMutableSettings.CustomSamplingRules); manual.Environment.Should().Be(automatic.Manager.InitialMutableSettings.Environment); @@ -261,8 +261,8 @@ public void AutomaticToManual_ImmutableSettingsAreTransferredCorrectly() private static void AssertEquivalent(ManualSettings manual, TracerSettings automatic) { // AgentUri gets transformed in exporter settings, so hacking around that here - GetTransformedAgentUri(manual.AgentUri).Should().Be(automatic.Exporter.AgentUri); - GetTransformedAgentUri(manual.Exporter.AgentUri).Should().Be(automatic.Exporter.AgentUri); + GetTransformedAgentUri(manual.AgentUri).Should().Be(automatic.Manager.InitialExporterSettings.AgentUri); + GetTransformedAgentUri(manual.Exporter.AgentUri).Should().Be(automatic.Manager.InitialExporterSettings.AgentUri); manual.AnalyticsEnabled.Should().Be(automatic.Manager.InitialMutableSettings.AnalyticsEnabled); manual.CustomSamplingRules.Should().Be(automatic.Manager.InitialMutableSettings.CustomSamplingRules); @@ -365,7 +365,7 @@ private static TracerSettings GetAndAssertAutomaticTracerSettings() automatic.StatsComputationEnabled.Should().Be(true); automatic.Manager.InitialMutableSettings.TraceEnabled.Should().Be(false); automatic.Manager.InitialMutableSettings.TracerMetricsEnabled.Should().Be(true); - automatic.Exporter.AgentUri.Should().Be(new Uri("http://127.0.0.1:1234")); + automatic.Manager.InitialExporterSettings.AgentUri.Should().Be(new Uri("http://127.0.0.1:1234")); automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Aerospike)].Enabled.Should().Be(false); automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Grpc)].AnalyticsEnabled.Should().Be(true); automatic.Manager.InitialMutableSettings.Integrations[nameof(IntegrationId.Couchbase)].AnalyticsSampleRate.Should().Be(0.5); From 73dd290d060c7c2e34bd18d7cfa2ee6e8c4a6fba Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 29 Oct 2025 15:11:41 +0000 Subject: [PATCH 23/29] Remove EnableSending and DisableSending from TelemetryController This isn't necessary with the current design, and it causes issues today --- .../Telemetry/TelemetryController.cs | 25 +------------------ .../Telemetry/TelemetryFactory.cs | 1 - 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs index a59549de91b4..d7b455b892ce 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs @@ -47,7 +47,6 @@ internal class TelemetryController : ITelemetryController private readonly Scheduler _scheduler; private readonly IDisposable _settingsSubscription; private TelemetryTransportManager _transportManager; - private bool _sendTelemetry; private bool _isStarted; private string? _namingVersion; @@ -112,7 +111,6 @@ public void RecordTracerSettings(TracerSettings settings) _integrations.RecordTracerSettings(settings.Manager.InitialMutableSettings); _namingVersion = ((int)settings.MetadataSchemaVersion).ToString(); _logTagBuilder.Update(settings); - _queue.Enqueue(new WorkItem(WorkItem.ItemType.EnableSending, null)); } public void RecordGitMetadata(GitMetadata gitMetadata) @@ -171,11 +169,6 @@ public async Task DisposeAsync() await _flushTask.ConfigureAwait(false); } - public void DisableSending() - { - _queue.Enqueue(new WorkItem(WorkItem.ItemType.DisableSending, null)); - } - public void SetTransportManager(TelemetryTransportManager manager) { _queue.Enqueue(new WorkItem(WorkItem.ItemType.SetTransportManager, manager)); @@ -274,12 +267,6 @@ private async Task PushTelemetryLoopAsync() case WorkItem.ItemType.SetTracerStarted: _isStarted = true; break; - case WorkItem.ItemType.EnableSending: - _sendTelemetry = true; - break; - case WorkItem.ItemType.DisableSending: - _sendTelemetry = false; - break; case WorkItem.ItemType.SetFlushInterval: _scheduler.SetFlushInterval((TimeSpan)item.State!); break; @@ -293,7 +280,7 @@ private async Task PushTelemetryLoopAsync() await _metrics.DisposeAsync().ConfigureAwait(false); } - if (_isStarted && _sendTelemetry && _scheduler.ShouldFlushTelemetry) + if (_isStarted && _scheduler.ShouldFlushTelemetry) { await PushTelemetry(includeLogs: _scheduler.ShouldFlushRedactedErrorLogs, sendAppClosing: isFinalPush).ConfigureAwait(false); } @@ -317,14 +304,6 @@ private async Task PushTelemetry(bool includeLogs, bool sendAppClosing) // need to make sure we clear the buffers. If we don't we could get overflows. // We will lose these metrics if the endpoint errors, but better than growing too much. MetricResults? metrics = _metrics.GetMetrics(); - - if (!_sendTelemetry) - { - // sending is currently disabled, so don't fetch the other data or attempt to send - Log.Debug("Telemetry pushing currently disabled, skipping"); - return; - } - var application = _application.GetApplicationData(); var host = _application.GetHostData(); if (application is null || host is null) @@ -376,8 +355,6 @@ public enum ItemType { SetTransportManager, SetFlushInterval, - EnableSending, - DisableSending, SetTracerStarted } diff --git a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs index b53b198fe044..8c83171e7824 100644 --- a/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs +++ b/tracer/src/Datadog.Trace/Telemetry/TelemetryFactory.cs @@ -182,7 +182,6 @@ private ITelemetryController CreateController( } } - _controller.DisableSending(); // disable sending until fully configured _controller.SetTransportManager(transportManager); _controller.SetFlushInterval(settings.HeartbeatInterval); From 6591ff1a75b6aa390147ae0ea86b22d30df62535 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 30 Oct 2025 09:50:51 +0000 Subject: [PATCH 24/29] Fix OTel metrics test - I don't know why this changed, but it looks better? --- .../OpenTelemetrySdkTests.cs | 4 +- ...dkTests.SubmitsOtlpMetrics_DD.verified.txt | 239 ++++++++++++++++++ ...ests.SubmitsOtlpMetrics_OTEL.verified.txt} | 0 ...pMetrics_up_to_1_5_0.NET_6_DD.verified.txt | 2 +- ...etrics_up_to_1_7_0.NET_7_8_DD.verified.txt | 205 +++++++++++++++ ...ics_up_to_1_7_0.NET_7_8_OTEL.verified.txt} | 0 6 files changed, 448 insertions(+), 2 deletions(-) create mode 100644 tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_DD.verified.txt rename tracer/test/snapshots/{OpenTelemetrySdkTests.SubmitsOtlpMetrics.verified.txt => OpenTelemetrySdkTests.SubmitsOtlpMetrics_OTEL.verified.txt} (100%) create mode 100644 tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_DD.verified.txt rename tracer/test/snapshots/{OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8.verified.txt => OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_OTEL.verified.txt} (100%) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs index 768aa2531e7b..6d793cd68231 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs @@ -225,12 +225,14 @@ public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetric var snapshotName = runtimeMajor switch { - 6 when parsedVersion >= new Version("1.3.2") && parsedVersion < new Version("1.5.0") => otelMetricsEnabled.Equals("true") ? ".NET_6_OTEL" : ".NET_6_DD", + 6 when parsedVersion >= new Version("1.3.2") && parsedVersion < new Version("1.5.0") => ".NET_6", 7 or 8 when parsedVersion >= new Version("1.5.1") && parsedVersion < new Version("1.10.0") => ".NET_7_8", >= 9 when parsedVersion >= new Version("1.10.0") => string.Empty, _ => throw new SkipException($"Skipping test due to irrelevant runtime and OTel versions mix: .NET {runtimeMajor} & Otel v{parsedVersion}") }; + snapshotName = otelMetricsEnabled.Equals("true") ? $"{snapshotName}_OTEL" : $"{snapshotName}_DD"; + var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "localhost"; var otlpPort = protocol == "grpc" ? 4317 : 4318; diff --git a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_DD.verified.txt b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_DD.verified.txt new file mode 100644 index 000000000000..fb5bebe60a03 --- /dev/null +++ b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_DD.verified.txt @@ -0,0 +1,239 @@ +[ + { + "resource_metrics": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.name", + "value": { + "string_value": "sdk-name" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "string_value": "dotnet" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "string_value": "sdk-version" + } + }, + { + "key": "service.name", + "value": { + "string_value": "Samples.OpenTelemetrySdk" + } + } + ] + }, + "scope_metrics": [ + { + "scope": { + "name": "OpenTelemetryMetricsMeter", + "version": "1.0", + "attributes": [ + { + "key": "MeterTagKey", + "value": { + "string_value": "MeterTagValue" + } + } + ] + }, + "metrics": [ + { + "name": "example.async.counter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "22" + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_DELTA", + "is_monotonic": true + } + }, + { + "name": "example.async.gauge", + "gauge": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_double": 88.0 + } + ] + } + }, + { + "name": "example.async.upDownCounter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "66" + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE" + } + }, + { + "name": "example.counter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "11", + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ] + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_DELTA", + "is_monotonic": true + } + }, + { + "name": "example.gauge", + "gauge": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_double": 77.0, + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ] + } + ] + } + }, + { + "name": "example.histogram", + "histogram": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "count": "1", + "sum": 33.0, + "bucket_counts": [ + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0" + ], + "explicit_bounds": [ + 0.0, + 5.0, + 10.0, + 25.0, + 50.0, + 75.0, + 100.0, + 250.0, + 500.0, + 750.0, + 1000.0, + 2500.0, + 5000.0, + 7500.0, + 10000.0 + ], + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ], + "min": 33.0, + "max": 33.0 + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_DELTA" + } + }, + { + "name": "example.upDownCounter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "55", + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ] + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE" + } + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics.verified.txt b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_OTEL.verified.txt similarity index 100% rename from tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics.verified.txt rename to tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_OTEL.verified.txt diff --git a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_5_0.NET_6_DD.verified.txt b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_5_0.NET_6_DD.verified.txt index dfcac5252416..039616fd9a2a 100644 --- a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_5_0.NET_6_DD.verified.txt +++ b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_5_0.NET_6_DD.verified.txt @@ -25,7 +25,7 @@ { "key": "service.name", "value": { - "string_value": "unknown_service:dotnet" + "string_value": "Samples.OpenTelemetrySdk" } } ] diff --git a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_DD.verified.txt b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_DD.verified.txt new file mode 100644 index 000000000000..f0fed94decdf --- /dev/null +++ b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_DD.verified.txt @@ -0,0 +1,205 @@ +[ + { + "resource_metrics": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.name", + "value": { + "string_value": "sdk-name" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "string_value": "dotnet" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "string_value": "sdk-version" + } + }, + { + "key": "service.name", + "value": { + "string_value": "Samples.OpenTelemetrySdk" + } + } + ] + }, + "scope_metrics": [ + { + "scope": { + "name": "OpenTelemetryMetricsMeter", + "version": "1.0" + }, + "metrics": [ + { + "name": "example.async.counter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "22" + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "is_monotonic": true + } + }, + { + "name": "example.async.gauge", + "gauge": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_double": 88.0 + } + ] + } + }, + { + "name": "example.async.upDownCounter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "66" + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE" + } + }, + { + "name": "example.counter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "11", + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ] + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "is_monotonic": true + } + }, + { + "name": "example.histogram", + "histogram": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "count": "1", + "sum": 33.0, + "bucket_counts": [ + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0" + ], + "explicit_bounds": [ + 0.0, + 5.0, + 10.0, + 25.0, + 50.0, + 75.0, + 100.0, + 250.0, + 500.0, + 750.0, + 1000.0, + 2500.0, + 5000.0, + 7500.0, + 10000.0 + ], + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ], + "min": 33.0, + "max": 33.0 + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE" + } + }, + { + "name": "example.upDownCounter", + "sum": { + "data_points": [ + { + "start_time_unix_nano": "0", + "time_unix_nano": "0", + "as_int": "55", + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + }, + { + "key": "rid", + "value": { + "string_value": "1234567890" + } + } + ] + } + ], + "aggregation_temporality": "AGGREGATION_TEMPORALITY_CUMULATIVE" + } + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8.verified.txt b/tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_OTEL.verified.txt similarity index 100% rename from tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8.verified.txt rename to tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpMetrics_up_to_1_7_0.NET_7_8_OTEL.verified.txt From 7efac2d40ba43db2da7634471e36c805e931d372 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 30 Oct 2025 11:01:41 +0000 Subject: [PATCH 25/29] Fix manual instrumentation not reporting the correct settings --- .../Tracer/CtorIntegration.cs | 12 +++++++----- ...tUpdatedImmutableTracerSettingsIntegration.cs | 16 +++++++++++++--- .../SettingsInstrumentationTests.cs | 7 +++++-- .../benchmarks/Benchmarks.Trace/SpanBenchmark.cs | 2 +- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs index b6d61da88e43..5e00a3b17494 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/CtorIntegration.cs @@ -36,17 +36,19 @@ internal static CallTargetState OnMethodBegin(TTarget instance, object? // but in 3.7.0 we don't need to instrument them if (automaticTracer is Datadog.Trace.Tracer tracer) { - PopulateSettings(values, tracer.Settings); + PopulateSettings(values, tracer); } return CallTargetState.GetDefault(); } - internal static void PopulateSettings(Dictionary values, TracerSettings settings) + internal static void PopulateSettings(Dictionary values, Datadog.Trace.Tracer tracer) { // record all the settings in the dictionary - var mutableSettings = settings.Manager.InitialMutableSettings; - var exporterSettings = settings.Manager.InitialExporterSettings; + var mutableSettings = tracer.CurrentTraceSettings.Settings; + // TODO: This doesn't get the "current" exporter settings, if they've been changed for code. + // We don't currently provide a way to do that without subscribing to all changes, which would be overkill here. + var exporterSettings = tracer.Settings.Manager.InitialExporterSettings; // This key is used to detect if the settings have been populated _at all_, so should always be sent values[TracerSettingKeyConstants.AgentUriKey] = exporterSettings.AgentUri; #pragma warning disable CS0618 // Type or member is obsolete @@ -64,7 +66,7 @@ internal static void PopulateSettings(Dictionary values, Tracer values[TracerSettingKeyConstants.ServiceNameKey] = mutableSettings.ServiceName; values[TracerSettingKeyConstants.ServiceVersionKey] = mutableSettings.ServiceVersion; values[TracerSettingKeyConstants.StartupDiagnosticLogEnabledKey] = mutableSettings.StartupDiagnosticLogEnabled; - values[TracerSettingKeyConstants.StatsComputationEnabledKey] = settings.StatsComputationEnabled; + values[TracerSettingKeyConstants.StatsComputationEnabledKey] = tracer.Settings.StatsComputationEnabled; values[TracerSettingKeyConstants.TraceEnabledKey] = mutableSettings.TraceEnabled; values[TracerSettingKeyConstants.TracerMetricsEnabledKey] = mutableSettings.TracerMetricsEnabled; diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/GetUpdatedImmutableTracerSettingsIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/GetUpdatedImmutableTracerSettingsIntegration.cs index 891ba2069004..3a4ade5226b4 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/GetUpdatedImmutableTracerSettingsIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/ManualInstrumentation/Tracer/GetUpdatedImmutableTracerSettingsIntegration.cs @@ -29,12 +29,22 @@ public class GetUpdatedImmutableTracerSettingsIntegration { internal static CallTargetState OnMethodBegin(TTarget instance, ref object? automaticTracer, ref object? automaticSettings) { + // In previous versions of Datadog.Trace (<3.30.0), we would replace the entire TracerSettings + // object whenever the settings changed, and use that to track whether we need to update the manual + // side. Now, TracerSettings is immutable for the lifetime of the application, and we instead + // update the MutableSettings and ExporterSettings whenever something changes. To keep roughly the same + // compatible behaviour here, we now store the current MutableSettings in `automaticSettings` to track + // whether things need to update. Note that this isn't _strictly_ correct, because if the customer updates + // only the exporter settings, we won't track that it's changed here. However, in PopulateSettings we _also_ + // don't populate the latest exporter settings there, so that's ok! Setting the exporter settings in code is + // deprecated (as it's problematic for a bunch of reasons), but it's still possible, so this is a half-way + // house way to handle it. if (automaticTracer is Datadog.Trace.Tracer tracer - && (automaticSettings is null || !ReferenceEquals(tracer.Settings, automaticSettings))) + && (automaticSettings is null || !ReferenceEquals(tracer.CurrentTraceSettings.Settings, automaticSettings))) { - automaticSettings = tracer.Settings; + automaticSettings = tracer.CurrentTraceSettings.Settings; var dict = new Dictionary(); - CtorIntegration.PopulateSettings(dict, tracer.Settings); + CtorIntegration.PopulateSettings(dict, tracer); return new CallTargetState(scope: null, state: dict); } diff --git a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs index 985c48fd3696..705d0b00855f 100644 --- a/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs +++ b/tracer/test/Datadog.Trace.Tests/ManualInstrumentation/SettingsInstrumentationTests.cs @@ -8,12 +8,14 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation; using Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.Configuration.TracerSettings; using Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.Tracer; using Datadog.Trace.Configuration; using Datadog.Trace.Configuration.ConfigurationSources; using Datadog.Trace.Telemetry.Metrics; +using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; using CtorIntegration = Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.Tracer.CtorIntegration; @@ -219,12 +221,13 @@ public void ManualToAutomatic_CustomSettingsAreTransferredCorrectly(bool useLega } [Fact] - public void AutomaticToManual_ImmutableSettingsAreTransferredCorrectly() + public async Task AutomaticToManual_ImmutableSettingsAreTransferredCorrectly() { var automatic = GetAndAssertAutomaticTracerSettings(); + await using var tracer = TracerHelper.Create(automatic); Dictionary serializedSettings = new(); - CtorIntegration.PopulateSettings(serializedSettings, automatic); + CtorIntegration.PopulateSettings(serializedSettings, tracer); var manual = new ImmutableManualSettings(serializedSettings); diff --git a/tracer/test/benchmarks/Benchmarks.Trace/SpanBenchmark.cs b/tracer/test/benchmarks/Benchmarks.Trace/SpanBenchmark.cs index aa55bd5a709c..435b1947414d 100644 --- a/tracer/test/benchmarks/Benchmarks.Trace/SpanBenchmark.cs +++ b/tracer/test/benchmarks/Benchmarks.Trace/SpanBenchmark.cs @@ -42,7 +42,7 @@ public void GlobalSetup() // Create the manual integration Dictionary manualSettings = new(); - CtorIntegration.PopulateSettings(manualSettings, _tracer.Settings); + CtorIntegration.PopulateSettings(manualSettings, _tracer); // Constructor is private, so create using reflection _manualTracer = (ManualTracer)typeof(ManualTracer) From 9e87ab3de17bef0fda2d9b1fbf3216740ffd0d4d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 30 Oct 2025 11:17:01 +0000 Subject: [PATCH 26/29] Fix telemetry recording and log configuration when settings change --- .../Configuration/MutableSettings.cs | 12 ++------ .../Configuration/SettingsManager.cs | 17 ++++------- tracer/src/Datadog.Trace/TracerManager.cs | 22 +++++++++----- .../src/Datadog.Trace/TracerManagerFactory.cs | 29 +++++++++++-------- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs index 7881d83c2199..ca354b771dcd 100644 --- a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs @@ -61,8 +61,7 @@ private MutableSettings( ReadOnlyDictionary serviceNameMappings, string? gitRepositoryUrl, string? gitCommitSha, - OverrideErrorLog errorLog, - IConfigurationTelemetry telemetry) + OverrideErrorLog errorLog) { IsInitialSettings = isInitialSettings; TraceEnabled = traceEnabled; @@ -92,7 +91,6 @@ private MutableSettings( GitRepositoryUrl = gitRepositoryUrl; GitCommitSha = gitCommitSha; ErrorLog = errorLog; - Telemetry = telemetry; } // Settings that can be set via remote config @@ -263,8 +261,6 @@ private MutableSettings( internal OverrideErrorLog ErrorLog { get; } - internal IConfigurationTelemetry Telemetry { get; } - internal static ReadOnlyDictionary? InitializeHeaderTags(ConfigurationBuilder config, string key, bool headerTagsNormalizationFixEnabled) => InitializeHeaderTags( config.WithKeys(key).AsDictionaryResult(allowOptionalMappings: true), @@ -737,8 +733,7 @@ public static MutableSettings CreateUpdatedMutableSettings( serviceNameMappings: serviceNameMappings, gitRepositoryUrl: gitRepositoryUrl, gitCommitSha: gitCommitSha, - errorLog: errorLog, - telemetry: telemetry); + errorLog: errorLog); static ReadOnlyDictionary GetHeaderTagsResult( ConfigurationBuilder.ClassConfigurationResultWithKey> result, @@ -1071,8 +1066,7 @@ public static MutableSettings CreateInitialMutableSettings( serviceNameMappings: serviceNameMappings, gitRepositoryUrl: gitRepositoryUrl, gitCommitSha: gitCommitSha, - errorLog: errorLog, - telemetry: telemetry); + errorLog: errorLog); } /// diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs index 8a0005207c0a..6e8dcc6a8b26 100644 --- a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -131,7 +131,7 @@ private bool UpdateSettings( internal SettingChanges? BuildNewSettings( IConfigurationSource dynamicConfigSource, ManualInstrumentationConfigurationSourceBase manualSource, - IConfigurationTelemetry centralTelemetry) + IConfigurationTelemetry telemetry) { var initialSettings = manualSource.UseDefaultSources ? InitialMutableSettings @@ -141,14 +141,14 @@ private bool UpdateSettings( var currentMutable = current?.UpdatedMutable ?? current?.PreviousMutable ?? InitialMutableSettings; var currentExporter = current?.UpdatedExporter ?? current?.PreviousExporter ?? InitialExporterSettings; - var telemetry = new ConfigurationTelemetry(); + var overrideErrorLog = new OverrideErrorLog(); var newMutableSettings = MutableSettings.CreateUpdatedMutableSettings( dynamicConfigSource, manualSource, initialSettings, _tracerSettings, telemetry, - new OverrideErrorLog()); // TODO: We'll later report these + overrideErrorLog); // TODO: We'll later report these // The only exporter setting we currently _allow_ to change is the AgentUri, but if that does change, // it can mean that _everything_ about the exporter settings changes. To minimize the work to do, and @@ -156,11 +156,10 @@ private bool UpdateSettings( // set, or unchanged, there's no need to update the exporter settings. // We only technically need to do this today if _manual_ config changes, not if remote config changes, // but for simplicity we don't distinguish currently. - var exporterTelemetry = new ConfigurationTelemetry(); var newRawExporterSettings = ExporterSettings.Raw.CreateUpdatedFromManualConfig( currentExporter.RawSettings, manualSource, - exporterTelemetry, + telemetry, manualSource.UseDefaultSources); var isSameMutableSettings = currentMutable.Equals(newMutableSettings); @@ -169,18 +168,12 @@ private bool UpdateSettings( if (isSameMutableSettings && isSameExporterSettings) { Log.Debug("No changes detected in the new configuration"); - // Even though there were no "real" changes, there may be _effective_ changes in telemetry that - // need to be recorded (e.g. the customer set the value in code, but it was already set via - // env vars). We _should_ record exporter settings too, but that introduces a bunch of complexity - // which we'll resolve later anyway, so just have that gap for now (it's very niche). - // If there are changes, they're recorded automatically in ConfigureInternal - telemetry.CopyTo(centralTelemetry); return null; } Log.Information("Notifying consumers of new settings"); var updatedMutableSettings = isSameMutableSettings ? null : newMutableSettings; - var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, exporterTelemetry); + var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, telemetry); return new SettingChanges(updatedMutableSettings, updatedExporterSettings, currentMutable, currentExporter); } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 7088e489578e..8b099ab472f3 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -30,6 +30,7 @@ using Datadog.Trace.Sampling; using Datadog.Trace.SourceGenerators; using Datadog.Trace.Telemetry; +using Datadog.Trace.Util; using Datadog.Trace.Util.Http; using Datadog.Trace.Vendors.Newtonsoft.Json; using Datadog.Trace.Vendors.StatsdClient; @@ -314,7 +315,7 @@ private static async Task CleanUpOldTracerManager(TracerManager oldManager, Trac } } - private static async Task WriteDiagnosticLog(TracerManager instance) + private static async Task WriteDiagnosticLog(TracerManager instance, MutableSettings mutableSettings, ExporterSettings exporterSettings) { try { @@ -325,9 +326,6 @@ private static async Task WriteDiagnosticLog(TracerManager instance) string agentError = null; var instanceSettings = instance.Settings; - var mutableSettings = instance.PerTraceSettings.Settings; - // TODO: this only writes the initial settings - we should make sure to record an "update" log on reconfiguration - var exporterSettings = instanceSettings.Manager.InitialExporterSettings; // In AAS, the trace agent is deployed alongside the tracer and managed by the tracer // Disable this check as it may hit the trace agent before it is ready to receive requests and give false negatives @@ -612,7 +610,8 @@ void WriteDictionary(IReadOnlyDictionary dictionary) Log.Information("DATADOG TRACER CONFIGURATION - {Configuration}", stringWriter.ToString()); OverrideErrorLog.Instance.ProcessAndClearActions(Log, TelemetryFactory.Metrics); // global errors, only logged once - instanceSettings.ErrorLog.ProcessAndClearActions(Log, TelemetryFactory.Metrics); // global errors, only logged once + instanceSettings.ErrorLog.ProcessAndClearActions(Log, TelemetryFactory.Metrics); + mutableSettings.ErrorLog.ProcessAndClearActions(Log, TelemetryFactory.Metrics); } catch (Exception ex) { @@ -685,11 +684,20 @@ private static TracerManager CreateInitializedTracer(TracerSettings settings, Tr OneTimeSetup(newManager.Settings); } - if (newManager.PerTraceSettings.Settings.StartupDiagnosticLogEnabled) + if (newManager.Settings.Manager is { InitialMutableSettings: { StartupDiagnosticLogEnabled: true } mutable, InitialExporterSettings: { } exporter }) { - _ = Task.Run(() => WriteDiagnosticLog(newManager)); + _ = Task.Run(() => WriteDiagnosticLog(newManager, mutable, exporter)); } + newManager.Settings.Manager.SubscribeToChanges(changes => + { + var mutable = changes.UpdatedMutable ?? changes.PreviousMutable; + if (mutable.StartupDiagnosticLogEnabled) + { + _ = Task.Run(() => WriteDiagnosticLog(newManager, mutable, changes.UpdatedExporter ?? changes.PreviousExporter)); + } + }); + return newManager; } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 85dc4b7404fe..6b07399ac883 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -63,21 +63,26 @@ internal TracerManager CreateTracerManager(TracerSettings settings, TracerManage tracerFlareManager: null, spanEventsManager: null); - try + tracer.Settings.Manager.SubscribeToChanges(changes => { - if (Profiler.Instance.Status.IsProfilerReady) + if (changes.UpdatedMutable is { } mutableSettings) { - var mutableSettings = tracer.PerTraceSettings.Settings; - NativeInterop.SetApplicationInfoForAppDomain(RuntimeId.Get(), mutableSettings.DefaultServiceName, mutableSettings.Environment, mutableSettings.ServiceVersion); + try + { + if (Profiler.Instance.Status.IsProfilerReady) + { + NativeInterop.SetApplicationInfoForAppDomain(RuntimeId.Get(), mutableSettings.DefaultServiceName, mutableSettings.Environment, mutableSettings.ServiceVersion); + } + } + catch (Exception ex) + { + // We failed to retrieve the runtime from native this can be because: + // - P/Invoke issue (unknown dll, unknown entrypoint...) + // - We are running in a partial trust environment + Log.Warning(ex, "Failed to set the service name for native."); + } } - } - catch (Exception ex) - { - // We failed to retrieve the runtime from native this can be because: - // - P/Invoke issue (unknown dll, unknown entrypoint...) - // - We are running in a partial trust environment - Log.Warning(ex, "Failed to set the service name for native."); - } + }); return tracer; } From 48e7faaa1e2330f699e9d2e7265b27013aaedbf0 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 31 Oct 2025 18:01:59 +0000 Subject: [PATCH 27/29] Rework the StatsDManager Make sure we can't dispose a stats consumer that's in use (as it will throw) Rework to use a "lease" mechanism to track usages Make passing in a statsmanager required --- tracer/missing-nullability-files.csv | 2 - tracer/src/Datadog.Trace/Agent/AgentWriter.cs | 52 +- tracer/src/Datadog.Trace/Agent/Api.cs | 18 +- .../src/Datadog.Trace/Agent/IAgentWriter.cs | 2 + tracer/src/Datadog.Trace/Agent/ManagedApi.cs | 3 +- .../Datadog.Trace/Ci/Agent/ApmAgentWriter.cs | 10 +- .../Ci/TestOptimizationTracerManager.cs | 5 +- .../TestOptimizationTracerManagerFactory.cs | 5 +- .../Datadog.Trace/DogStatsd/IStatsdManager.cs | 26 + .../src/Datadog.Trace/DogStatsd/NoOpStatsd.cs | 2 +- .../Datadog.Trace/DogStatsd/StatsdConsumer.cs | 20 + .../Datadog.Trace/DogStatsd/StatsdManager.cs | 303 ++++++-- .../AzureAppServicePerformanceCounters.cs | 33 +- .../RuntimeMetrics/IRuntimeMetricsListener.cs | 2 - .../RuntimeMetrics/MemoryMappedCounters.cs | 38 +- .../PerformanceCountersListener.cs | 46 +- .../RuntimeMetrics/RuntimeEventListener.cs | 50 +- .../RuntimeMetrics/RuntimeMetricsWriter.cs | 43 +- tracer/src/Datadog.Trace/Tracer.cs | 3 +- tracer/src/Datadog.Trace/TracerManager.cs | 7 +- .../src/Datadog.Trace/TracerManagerFactory.cs | 14 +- .../CI/Agent/ApmAgentWriterTests.cs | 5 +- .../LibDatadog/TraceExporterTests.cs | 3 +- .../OriginTagSendTraces.cs | 3 +- .../SpanTagTests.cs | 3 +- .../Tagging/AASTagsTests.cs | 7 +- .../Tagging/ProcessTagsTests.cs | 3 +- ...Tests_MultipleChunksWithUpstreamService.cs | 3 +- ...ts_MultipleChunksWithoutUpstreamService.cs | 3 +- .../Tagging/TraceContextPropertyTests.cs | 3 +- .../Tagging/TraceTagTests.cs | 3 +- .../Tagging/TraceTags.cs | 3 +- .../EventTrackingSdkTests.cs | 13 +- .../Stats/TestStatsdManager.cs | 26 + .../TestTracer/ScopedTracer.cs | 36 +- .../Agent/AgentWriterTests.cs | 29 +- .../Datadog.Trace.Tests/Agent/ApiTests.cs | 23 +- .../SpanMessagePackFormatterTests.cs | 7 +- .../DogStatsd/StatsdManagerTests.cs | 715 ++++++++++++++++++ .../AzurePerformanceCountersListenerTests.cs | 3 +- .../MemoryMappedCountersTests.cs | 3 +- .../PerformanceCountersListenerTests.cs | 8 +- .../RuntimeEventListenerTests.cs | 29 +- .../RuntimeMetricsWriterTests.cs | 11 +- .../Tagging/TagsListTests.cs | 3 +- .../TracerManagerFactoryTests.cs | 5 +- .../Util/RandomIdGeneratorTests.cs | 7 +- 47 files changed, 1347 insertions(+), 294 deletions(-) create mode 100644 tracer/src/Datadog.Trace/DogStatsd/IStatsdManager.cs create mode 100644 tracer/src/Datadog.Trace/DogStatsd/StatsdConsumer.cs create mode 100644 tracer/test/Datadog.Trace.TestHelpers/Stats/TestStatsdManager.cs diff --git a/tracer/missing-nullability-files.csv b/tracer/missing-nullability-files.csv index 5c89e56bf127..32684c4f0b51 100644 --- a/tracer/missing-nullability-files.csv +++ b/tracer/missing-nullability-files.csv @@ -34,10 +34,8 @@ src/Datadog.Trace/Tracer.cs src/Datadog.Trace/TracerConstants.cs src/Datadog.Trace/TracerManager.cs src/Datadog.Trace/TracerManagerFactory.cs -src/Datadog.Trace/Agent/AgentWriter.cs src/Datadog.Trace/Agent/Api.cs src/Datadog.Trace/Agent/ClientStatsPayload.cs -src/Datadog.Trace/Agent/IAgentWriter.cs src/Datadog.Trace/Agent/IApi.cs src/Datadog.Trace/Agent/IApiRequest.cs src/Datadog.Trace/Agent/IApiRequestFactory.cs diff --git a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs index e4c718bbe9c2..a6cb16135053 100644 --- a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs +++ b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +#nullable enable + using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -12,10 +14,8 @@ using Datadog.Trace.Configuration; using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; -using Datadog.Trace.Tagging; using Datadog.Trace.Telemetry; using Datadog.Trace.Telemetry.Metrics; -using Datadog.Trace.Vendors.StatsdClient; namespace Datadog.Trace.Agent { @@ -25,10 +25,10 @@ internal class AgentWriter : IAgentWriter private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private static readonly ArraySegment EmptyPayload = new(new byte[] { 0x90 }); + private static readonly ArraySegment EmptyPayload = new([0x90]); private readonly ConcurrentQueue _pendingTraces = new ConcurrentQueue(); - private readonly IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private readonly Task _flushTask; private readonly Task _serializationTask; private readonly TaskCompletionSource _processExit = new TaskCompletionSource(); @@ -43,7 +43,7 @@ internal class AgentWriter : IAgentWriter private readonly int _batchInterval; private readonly IKeepRateCalculator _traceKeepRateCalculator; - private readonly IStatsAggregator _statsAggregator; + private readonly IStatsAggregator? _statsAggregator; private readonly bool _apmTracingEnabled; @@ -67,7 +67,7 @@ internal class AgentWriter : IAgentWriter private bool _traceMetricsEnabled; - public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, TracerSettings settings) + public AgentWriter(IApi api, IStatsAggregator? statsAggregator, IStatsdManager statsd, TracerSettings settings) : this(api, statsAggregator, statsd, maxBufferSize: settings.TraceBufferSize, batchInterval: settings.TraceBatchInterval, apmTracingEnabled: settings.ApmTracingEnabled, initialTracerMetricsEnabled: settings.Manager.InitialMutableSettings.TracerMetricsEnabled) { settings.Manager.SubscribeToChanges(changes => @@ -76,16 +76,17 @@ public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd && mutable.TracerMetricsEnabled != changes.PreviousMutable.TracerMetricsEnabled) { Volatile.Write(ref _traceMetricsEnabled, mutable.TracerMetricsEnabled); + _statsd.SetRequired(StatsdConsumer.AgentWriter, mutable.TracerMetricsEnabled); } }); } - public AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, bool automaticFlush = true, int maxBufferSize = 1024 * 1024 * 10, int batchInterval = 100, bool apmTracingEnabled = true, bool initialTracerMetricsEnabled = false) + public AgentWriter(IApi api, IStatsAggregator? statsAggregator, IStatsdManager statsd, bool automaticFlush = true, int maxBufferSize = 1024 * 1024 * 10, int batchInterval = 100, bool apmTracingEnabled = true, bool initialTracerMetricsEnabled = false) : this(api, statsAggregator, statsd, MovingAverageKeepRateCalculator.CreateDefaultKeepRateCalculator(), automaticFlush, maxBufferSize, batchInterval, apmTracingEnabled, initialTracerMetricsEnabled) { } - internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd statsd, IKeepRateCalculator traceKeepRateCalculator, bool automaticFlush, int maxBufferSize, int batchInterval, bool apmTracingEnabled, bool initialTracerMetricsEnabled) + internal AgentWriter(IApi api, IStatsAggregator? statsAggregator, IStatsdManager statsd, IKeepRateCalculator traceKeepRateCalculator, bool automaticFlush, int maxBufferSize, int batchInterval, bool apmTracingEnabled, bool initialTracerMetricsEnabled) { _statsAggregator = statsAggregator; @@ -104,6 +105,7 @@ internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd stat _apmTracingEnabled = apmTracingEnabled; _traceMetricsEnabled = initialTracerMetricsEnabled; + _statsd.SetRequired(StatsdConsumer.AgentWriter, initialTracerMetricsEnabled); _serializationTask = automaticFlush ? Task.Factory.StartNew(SerializeTracesLoop, TaskCreationOptions.LongRunning) : Task.CompletedTask; _serializationTask.ContinueWith(t => Log.Error(t.Exception, "Error in serialization task"), TaskContinuationOptions.OnlyOnFaulted); @@ -114,7 +116,7 @@ internal AgentWriter(IApi api, IStatsAggregator statsAggregator, IDogStatsd stat _backBufferFlushTask = _frontBufferFlushTask = Task.CompletedTask; } - internal event Action Flushed; + internal event Action? Flushed; internal SpanBuffer ActiveBuffer => _activeBuffer; @@ -151,8 +153,12 @@ public void WriteTrace(ArraySegment trace) if (Volatile.Read(ref _traceMetricsEnabled)) { - _statsd.Increment(TracerMetricNames.Queue.EnqueuedTraces); - _statsd.Increment(TracerMetricNames.Queue.EnqueuedSpans, trace.Count); + using var lease = _statsd.TryGetClientLease(); + if (lease.Client is { } statsd) + { + statsd.Increment(TracerMetricNames.Queue.EnqueuedTraces); + statsd.Increment(TracerMetricNames.Queue.EnqueuedSpans, trace.Count); + } } } @@ -255,7 +261,7 @@ private async Task FlushBuffersTaskLoopAsync() { tasks[2] = Task.Delay(TimeSpan.FromSeconds(1)); await Task.WhenAny(tasks).ConfigureAwait(false); - tasks[2] = null; + tasks[2] = null!; if (_forceFlush.Task.IsCompleted) { @@ -327,8 +333,12 @@ async Task InternalBufferFlush() { if (Volatile.Read(ref _traceMetricsEnabled)) { - _statsd.Increment(TracerMetricNames.Queue.DequeuedTraces, buffer.TraceCount); - _statsd.Increment(TracerMetricNames.Queue.DequeuedSpans, buffer.SpanCount); + using var lease = _statsd.TryGetClientLease(); + if (lease.Client is { } statsd) + { + statsd.Increment(TracerMetricNames.Queue.DequeuedTraces, buffer.TraceCount); + statsd.Increment(TracerMetricNames.Queue.DequeuedSpans, buffer.SpanCount); + } } var droppedTraces = Interlocked.Exchange(ref _droppedTraces, 0); @@ -347,7 +357,7 @@ async Task InternalBufferFlush() { droppedP0Traces = Interlocked.Exchange(ref _droppedP0Traces, 0); droppedP0Spans = Interlocked.Exchange(ref _droppedP0Spans, 0); - Log.Debug("Flushing {Spans} spans across {Traces} traces. CanComputeStats is enabled with {DroppedP0Traces} droppedP0Traces and {DroppedP0Spans} droppedP0Spans", buffer.SpanCount, buffer.TraceCount, droppedP0Traces, droppedP0Spans); + Log.Debug("Flushing {Spans} spans across {Traces} traces. CanComputeStats is enabled with {DroppedP0Traces} droppedP0Traces and {DroppedP0Spans} droppedP0Spans", buffer.SpanCount, buffer.TraceCount, droppedP0Traces, droppedP0Spans); // Metrics for unsampled traces/spans already recorded } else @@ -388,7 +398,7 @@ async Task InternalBufferFlush() private void SerializeTrace(ArraySegment spans) { // Declaring as inline method because only safe to invoke in the context of SerializeTrace - SpanBuffer SwapBuffers() + SpanBuffer? SwapBuffers() { if (_activeBuffer == _frontBuffer) { @@ -525,8 +535,12 @@ private void DropTrace(ArraySegment spans) if (Volatile.Read(ref _traceMetricsEnabled)) { - _statsd.Increment(TracerMetricNames.Queue.DroppedTraces); - _statsd.Increment(TracerMetricNames.Queue.DroppedSpans, spans.Count); + using var lease = _statsd.TryGetClientLease(); + if (lease.Client is { } statsd) + { + statsd.Increment(TracerMetricNames.Queue.DroppedTraces); + statsd.Increment(TracerMetricNames.Queue.DroppedSpans, spans.Count); + } } } @@ -589,7 +603,7 @@ private void SerializeTracesLoop() private readonly struct WorkItem { public readonly ArraySegment Trace; - public readonly Action Callback; + public readonly Action? Callback; public WorkItem(ArraySegment trace) { diff --git a/tracer/src/Datadog.Trace/Agent/Api.cs b/tracer/src/Datadog.Trace/Agent/Api.cs index 627760214edf..14bd63161281 100644 --- a/tracer/src/Datadog.Trace/Agent/Api.cs +++ b/tracer/src/Datadog.Trace/Agent/Api.cs @@ -34,7 +34,7 @@ internal class Api : IApi private readonly IDatadogLogger _log; private readonly IApiRequestFactory _apiRequestFactory; - private readonly IDogStatsd _originalStatsd; + private readonly IStatsdManager _statsd; private readonly string _containerId; private readonly string _entityId; private readonly Uri _tracesEndpoint; @@ -43,13 +43,13 @@ internal class Api : IApi private readonly bool _partialFlushEnabled; private readonly SendCallback _sendStats; private readonly SendCallback _sendTraces; - private IDogStatsd _statsd; private string _cachedResponse; private string _agentVersion; + private bool _healthMetricsEnabled; public Api( IApiRequestFactory apiRequestFactory, - IDogStatsd statsd, + IStatsdManager statsd, Action> updateSampleRates, bool partialFlushEnabled, bool healthMetricsEnabled, @@ -61,12 +61,13 @@ public Api( _sendStats = SendStatsAsyncImpl; _sendTraces = SendTracesAsyncImpl; _updateSampleRates = updateSampleRates; - _originalStatsd = statsd; + _statsd = statsd; ToggleTracerHealthMetrics(healthMetricsEnabled); _containerId = ContainerMetadata.GetContainerId(); _entityId = ContainerMetadata.GetEntityId(); _apiRequestFactory = apiRequestFactory; _partialFlushEnabled = partialFlushEnabled; + _healthMetricsEnabled = healthMetricsEnabled; _tracesEndpoint = _apiRequestFactory.GetEndpoint(TracesPath); _log.Debug("Using traces endpoint {TracesEndpoint}", _tracesEndpoint.ToString()); _statsEndpoint = _apiRequestFactory.GetEndpoint(StatsPath); @@ -85,7 +86,8 @@ private enum SendResult [MemberNotNull(nameof(_statsd))] public void ToggleTracerHealthMetrics(bool enabled) { - Interlocked.Exchange(ref _statsd, enabled ? _originalStatsd : null); + Volatile.Write(ref _healthMetricsEnabled, enabled); + _statsd.SetRequired(StatsdConsumer.TraceApi, enabled); } public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) @@ -310,7 +312,9 @@ private async Task SendTracesAsyncImpl(IApiRequest request, bool fin try { - var healthStats = Volatile.Read(ref _statsd); + var healthMetricsEnabled = Volatile.Read(ref _healthMetricsEnabled); + using var lease = healthMetricsEnabled ? _statsd.TryGetClientLease() : default; + var healthStats = healthMetricsEnabled ? lease.Client : null; try { TelemetryFactory.Metrics.RecordCountTraceApiRequests(); @@ -333,7 +337,7 @@ private async Task SendTracesAsyncImpl(IApiRequest request, bool fin string[] tags = { $"status:{response.StatusCode}" }; // count every response, grouped by status code - healthStats?.Increment(TracerMetricNames.Api.Responses, tags: tags); + healthStats.Increment(TracerMetricNames.Api.Responses, tags: tags); } TelemetryFactory.Metrics.RecordCountTraceApiResponses(response.GetTelemetryStatusCodeMetricTag()); diff --git a/tracer/src/Datadog.Trace/Agent/IAgentWriter.cs b/tracer/src/Datadog.Trace/Agent/IAgentWriter.cs index 2e3573ce6801..3c6563dca7da 100644 --- a/tracer/src/Datadog.Trace/Agent/IAgentWriter.cs +++ b/tracer/src/Datadog.Trace/Agent/IAgentWriter.cs @@ -3,6 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +#nullable enable + using System; using System.Threading.Tasks; diff --git a/tracer/src/Datadog.Trace/Agent/ManagedApi.cs b/tracer/src/Datadog.Trace/Agent/ManagedApi.cs index 635e64831e80..072e69839856 100644 --- a/tracer/src/Datadog.Trace/Agent/ManagedApi.cs +++ b/tracer/src/Datadog.Trace/Agent/ManagedApi.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Configuration; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Vendors.StatsdClient; namespace Datadog.Trace.Agent; @@ -24,7 +25,7 @@ internal class ManagedApi : IApi public ManagedApi( TracerSettings.SettingsManager settings, - IDogStatsd statsd, + IStatsdManager statsd, Action> updateSampleRates, bool partialFlushEnabled) { diff --git a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs index 4188a4259f15..e9442f5cfde4 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/ApmAgentWriter.cs @@ -11,6 +11,7 @@ using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.Ci.EventModel; using Datadog.Trace.Configuration; +using Datadog.Trace.DogStatsd; namespace Datadog.Trace.Ci.Agent; @@ -30,16 +31,17 @@ public ApmAgentWriter(TracerSettings settings, Action> var partialFlushEnabled = settings.PartialFlushEnabled; // CI Vis doesn't allow reconfiguration, so don't need to subscribe to changes var apiRequestFactory = TracesTransportStrategy.Get(settings.Manager.InitialExporterSettings); - var api = new Api(apiRequestFactory, null, updateSampleRates, partialFlushEnabled, healthMetricsEnabled: false); + var statsdManager = new StatsdManager(settings); + var api = new Api(apiRequestFactory, statsdManager, updateSampleRates, partialFlushEnabled, healthMetricsEnabled: false); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); - _agentWriter = new AgentWriter(api, statsAggregator, null, maxBufferSize: maxBufferSize, apmTracingEnabled: settings.ApmTracingEnabled, initialTracerMetricsEnabled: settings.Manager.InitialMutableSettings.TracerMetricsEnabled); + _agentWriter = new AgentWriter(api, statsAggregator, statsdManager, maxBufferSize: maxBufferSize, apmTracingEnabled: settings.ApmTracingEnabled, initialTracerMetricsEnabled: settings.Manager.InitialMutableSettings.TracerMetricsEnabled); } // Internal for testing - internal ApmAgentWriter(IApi api, int maxBufferSize = DefaultMaxBufferSize) + internal ApmAgentWriter(IApi api, IStatsdManager statsdManager, int maxBufferSize = DefaultMaxBufferSize) { - _agentWriter = new AgentWriter(api, null, null, maxBufferSize: maxBufferSize); + _agentWriter = new AgentWriter(api, null, statsdManager, maxBufferSize: maxBufferSize); } public void WriteEvent(IEvent @event) diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs index f7f894912998..3ddaae0dc31a 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManager.cs @@ -13,6 +13,7 @@ using Datadog.Trace.Ci.EventModel; using Datadog.Trace.Configuration; using Datadog.Trace.DataStreamsMonitoring; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.Logging.TracerFlare; @@ -32,7 +33,7 @@ public TestOptimizationTracerManager( TracerSettings settings, IAgentWriter agentWriter, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetricsWriter, DirectLogSubmissionManager logSubmissionManager, ITelemetryController telemetry, @@ -148,7 +149,7 @@ public LockedManager( TracerSettings settings, IAgentWriter agentWriter, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetricsWriter, DirectLogSubmissionManager logSubmissionManager, ITelemetryController telemetry, diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs index 4fbd1cd4f834..53be742ed6fb 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimizationTracerManagerFactory.cs @@ -14,6 +14,7 @@ using Datadog.Trace.Ci.Sampling; using Datadog.Trace.Configuration; using Datadog.Trace.DataStreamsMonitoring; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.Logging.TracerFlare; using Datadog.Trace.RemoteConfigurationManagement; @@ -41,7 +42,7 @@ protected override TracerManager CreateTracerManagerFrom( TracerSettings settings, IAgentWriter agentWriter, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetrics, DirectLogSubmissionManager logSubmissionManager, ITelemetryController telemetry, @@ -87,7 +88,7 @@ protected override ITraceSampler GetSampler(TracerSettings settings) protected override bool ShouldEnableRemoteConfiguration(TracerSettings settings) => false; - protected override IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) + protected override IAgentWriter GetAgentWriter(TracerSettings settings, IStatsdManager statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) { // Check for agentless scenario if (_settings.Agentless) diff --git a/tracer/src/Datadog.Trace/DogStatsd/IStatsdManager.cs b/tracer/src/Datadog.Trace/DogStatsd/IStatsdManager.cs new file mode 100644 index 000000000000..b479c4ca19bb --- /dev/null +++ b/tracer/src/Datadog.Trace/DogStatsd/IStatsdManager.cs @@ -0,0 +1,26 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; +using Datadog.Trace.Vendors.StatsdClient; + +namespace Datadog.Trace.DogStatsd; + +internal interface IStatsdManager : IDisposable +{ + /// + /// Obtain a for accessing a instance. + /// The lease must be disposed after all references to the client have gone. + /// + StatsdManager.StatsdClientLease TryGetClientLease(); + + /// + /// Called by users of to indicate that a "live" client is required. + /// Each unique consumer of should set a different + /// value. + /// + void SetRequired(StatsdConsumer consumer, bool enabled); +} diff --git a/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs b/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs index 98d840752d80..f866d98b604d 100644 --- a/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs +++ b/tracer/src/Datadog.Trace/DogStatsd/NoOpStatsd.cs @@ -8,7 +8,7 @@ namespace Datadog.Trace.DogStatsd { - internal class NoOpStatsd : IDogStatsd + internal sealed class NoOpStatsd : IDogStatsd { public static readonly NoOpStatsd Instance = new(); diff --git a/tracer/src/Datadog.Trace/DogStatsd/StatsdConsumer.cs b/tracer/src/Datadog.Trace/DogStatsd/StatsdConsumer.cs new file mode 100644 index 000000000000..3ee578ff2b9a --- /dev/null +++ b/tracer/src/Datadog.Trace/DogStatsd/StatsdConsumer.cs @@ -0,0 +1,20 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; + +namespace Datadog.Trace.DogStatsd; + +[Flags] +internal enum StatsdConsumer +{ + // None = 0, Must not use this + // Define bits per consumer: + RuntimeMetricsWriter = 1 << 0, + TraceApi = 1 << 1, + AgentWriter = 1 << 2, +} diff --git a/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs index 999a4bd95246..d2dd57a0d08e 100644 --- a/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs +++ b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs @@ -7,7 +7,6 @@ using System; using System.Threading; -using System.Threading.Tasks; using Datadog.Trace.Configuration; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.StatsdClient; @@ -18,20 +17,30 @@ namespace Datadog.Trace.DogStatsd; /// This acts as a wrapper around a "real" service or a client, /// but which responds to changes in settings caused by remote config or configuration in code. /// -internal sealed class StatsdManager : IDogStatsd +internal sealed class StatsdManager : IStatsdManager { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); private readonly object _lock = new(); private readonly IDisposable _settingSubscription; - private IDogStatsd _current; - private bool _isDisposed; + private int _isRequiredMask; + private StatsdClientHolder? _current; + private Func _factory; - public StatsdManager(TracerSettings tracerSettings, bool includeDefaultTags) + public StatsdManager(TracerSettings tracerSettings) + : this(tracerSettings, CreateClient) { - _current = StatsdFactory.CreateDogStatsdClient( + } + + // Internal for testing + internal StatsdManager(TracerSettings tracerSettings, Func statsdFactory) + { + // The initial factory, assuming there's no updates + _factory = () => statsdFactory( tracerSettings.Manager.InitialMutableSettings, - tracerSettings.Manager.InitialExporterSettings, - includeDefaultTags); + tracerSettings.Manager.InitialExporterSettings); + + // We don't create a new client unless we need one, and we rely on consumers of the manager to tell us when it's needed + _current = null; _settingSubscription = tracerSettings.Manager.SubscribeToChanges(c => { @@ -40,104 +49,256 @@ public StatsdManager(TracerSettings tracerSettings, bool includeDefaultTags) // a value then we should check if it's changed here if (!HasImpactingChanges(c)) { + Log.Debug("No impacting changes found for StatsdManager, ignoring settings update"); return; } - IDogStatsd previous; + // update the factory + Log.Debug("Updating statsdClient factory to use new configuration"); + Interlocked.Exchange( + ref _factory, + () => statsdFactory( + c.UpdatedMutable ?? c.PreviousMutable, + c.UpdatedExporter ?? c.PreviousExporter)); + + // check if we actually need to do an update or if noone is using the client yet + if (Volatile.Read(ref _isRequiredMask) != 0) + { + // Someone needs it, so create + EnsureClient(ensureCreated: true, forceRecreate: true); + } + }); + } - lock (_lock) + /// + public StatsdClientLease TryGetClientLease() + { + while (true) + { + var current = Volatile.Read(ref _current); + if (current == null) { - if (_isDisposed) + return default; + } + + if (current.TryRetain()) + { + return new StatsdClientLease(current); + } + + // The client was marked for closing, there should be a new one + // we can use instead, so loop around and grab that. + } + } + + /// + public void SetRequired(StatsdConsumer consumer, bool enabled) + { + var bitToSet = (int)consumer; + + if (enabled) + { + // Set the consumer bit; and check if there's been a change from before +#if NET6_0_OR_GREATER + var prev = Interlocked.Or(ref _isRequiredMask, bitToSet); +#else + // Can't use Interlocked.Or, so have to use a loop to be sure + int prev, updated; + do + { + prev = Volatile.Read(ref _isRequiredMask); + updated = prev | bitToSet; + if (prev == updated) { + // already set, nothing to do return; } - - previous = _current; - _current = StatsdFactory.CreateDogStatsdClient( - c.UpdatedMutable ?? c.PreviousMutable, - c.UpdatedExporter ?? c.PreviousExporter, - includeDefaultTags); } + while (Interlocked.CompareExchange(ref _isRequiredMask, updated, prev) != prev); +#endif + if (prev == 0) + { + // We transitioned from 0 -> non-zero: ensure client exists + EnsureClient(ensureCreated: true, forceRecreate: false); + } + } + else + { + // Atomically clear bit; clearMask is all ones, excluding the bit we're clearing + var clearMask = ~bitToSet; +#if NET6_0_OR_GREATER - if (previous is DogStatsdService dogStatsdService) + var prev = Interlocked.And(ref _isRequiredMask, clearMask); +#else + int prev, updated; + do + { + prev = Volatile.Read(ref _isRequiredMask); + updated = prev & clearMask; + if (prev == updated) + { + // already cleared, nothing to do + return; + } + } + while (Interlocked.CompareExchange(ref _isRequiredMask, updated, prev) != prev); +#endif + if (prev == bitToSet) { - // Kick off disposal in the background after a delay to make sure everything is flushed - // There's a risk that something could have grabbed the instance, and if something - // tries to write to the client, it will throw an exception. - Task.Run(async () => - { - await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); - dogStatsdService.Flush(); - dogStatsdService.Dispose(); - }) - .ContinueWith(t => Log.Error(t.Exception, "There was an error disposing the statsd client"), TaskContinuationOptions.OnlyOnFaulted); + // We transitioned from 1 -> 0: get rid of the client + EnsureClient(ensureCreated: false, forceRecreate: false); } - }); + } } - // Delegated implementation - public ITelemetryCounters TelemetryCounters => Volatile.Read(ref _current).TelemetryCounters; + public void Dispose() + { + _settingSubscription.Dispose(); + // We swap out the client to make sure we do any flushes. + EnsureClient(ensureCreated: false, forceRecreate: true); + } + + // Internal for testing + internal static bool HasImpactingChanges(TracerSettings.SettingsManager.SettingChanges changes) + { + var hasChanges = changes.UpdatedExporter is not null // relying on this to only be non null if _anything_ changed + || (changes.UpdatedMutable is { } updated + && !( + string.Equals(updated.Environment, changes.PreviousMutable.Environment, StringComparison.Ordinal) + && string.Equals(updated.ServiceVersion, changes.PreviousMutable.ServiceVersion, StringComparison.Ordinal) + // The service name comparison isn't _strictly_ correct, because we normalize it further, but this is probably good enough + && string.Equals(updated.DefaultServiceName, changes.PreviousMutable.DefaultServiceName, StringComparison.OrdinalIgnoreCase) + && updated.GlobalTags.SequenceEqual(changes.PreviousMutable.GlobalTags))); + return hasChanges; + } - public void Configure(StatsdConfig config) => Volatile.Read(ref _current).Configure(config); + private static IDogStatsd CreateClient(MutableSettings settings, ExporterSettings exporter) + => StatsdFactory.CreateDogStatsdClient(settings, exporter, includeDefaultTags: true); - public void Counter(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Counter(statName, value, sampleRate, tags); + private void EnsureClient(bool ensureCreated, bool forceRecreate) + { + StatsdClientHolder? previous; + Log.Debug("Recreating statsdClient: Create new client: {CreateClient}, Force recreate: {ForceRecreate}", ensureCreated, forceRecreate); - public void Decrement(string statName, int value = 1, double sampleRate = 1, params string[] tags) => Volatile.Read(ref _current).Decrement(statName, value, sampleRate, tags); + lock (_lock) + { + previous = _current; + if (ensureCreated && previous != null && !forceRecreate) + { + // Already created + return; + } - public void Event(string title, string text, string? alertType = null, string? aggregationKey = null, string? sourceType = null, int? dateHappened = null, string? priority = null, string? hostname = null, string[]? tags = null) => Volatile.Read(ref _current).Event(title, text, alertType, aggregationKey, sourceType, dateHappened, priority, hostname, tags); + _current = ensureCreated + ? new StatsdClientHolder(_factory()) + : null; + } - public void Gauge(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Gauge(statName, value, sampleRate, tags); + previous?.MarkClosing(); // will dispose when last lease releases + } - public void Histogram(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Histogram(statName, value, sampleRate, tags); + internal readonly struct StatsdClientLease(StatsdClientHolder? holder) : IDisposable + { + private readonly StatsdClientHolder? _holder = holder; - public void Distribution(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Distribution(statName, value, sampleRate, tags); + public IDogStatsd? Client => _holder?.Client; - public void Increment(string statName, int value = 1, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Increment(statName, value, sampleRate, tags); + public void Dispose() => _holder?.Release(); + } - public void Set(string statName, T value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Set(statName, value, sampleRate, tags); + internal sealed class StatsdClientHolder(IDogStatsd client) + { + private const int ClosingBit = 1 << 31; // sign bit = closing - public void Set(string statName, string value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Set(statName, value, sampleRate, tags); + // Logically, _state represents two values we need to check: + // - Was MarkClosing() called? + // - How many references does it have? + // We keep this all in the same variable to avoid race conditions that + // would occur if we had separate flag variables for count and closing + // high bit = closing, low 31 bits = refcount + private int _state; + private int _disposed; - public IDisposable StartTimer(string name, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).StartTimer(name, sampleRate, tags); + public IDogStatsd Client { get; } = client; - public void Time(Action action, string statName, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Time(action, statName, sampleRate, tags); + public bool TryRetain() + { + while (true) + { + var state = Volatile.Read(ref _state); + if ((state & ClosingBit) != 0) + { + // already closing; deny new leases + return false; + } - public T Time(Func func, string statName, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Time(func, statName, sampleRate, tags); + if ((state & int.MaxValue) == int.MaxValue) + { + // Guard against int.MaxValue retentions, won't happen, but play it safe + return false; + } - public void Timer(string statName, double value, double sampleRate = 1, string[]? tags = null) => Volatile.Read(ref _current).Timer(statName, value, sampleRate, tags); + // Conditionally bump ref count + if (Interlocked.CompareExchange(ref _state, state + 1, state) == state) + { + // ok, increment ref count + return true; + } - public void ServiceCheck(string name, Status status, int? timestamp = null, string? hostname = null, string[]? tags = null, string? message = null) => Volatile.Read(ref _current).ServiceCheck(name, status, timestamp, hostname, tags, message); + // The state of the client holder changed out from under us (someone else retained it, or it was closed), try again + } + } - public void Dispose() - { - IDogStatsd previous; - lock (_lock) + /// + /// Invoked by to indicate client is done with it + /// + public void Release() { - _isDisposed = true; - _settingSubscription.Dispose(); - previous = _current; - _current = NoOpStatsd.Instance; + var v = Interlocked.Decrement(ref _state); + if (v == ClosingBit) + { + // count hit zero and we're marked as closing + Dispose(); + } } - // Given we're shutting down at this point, it doesn't seem like it's worth actually disposing - // the instance (and risking an error) so we just flush to make sure everything is sent - if (previous is DogStatsdService dogStatsdService) + /// + /// Invoked by when swapping clients + /// + public void MarkClosing() { - dogStatsdService.Flush(); + // Set the closing bit to ensure no more retention of client +#if NET6_0_OR_GREATER + Interlocked.Or(ref _state, ClosingBit); +#else + // Interlocked.Or is not available in < .NET 6, so have to emulate it + int state; + do + { + state = Volatile.Read(ref _state); + } + while (Interlocked.CompareExchange(ref _state, state | ClosingBit, state) != state); +#endif + + // If ref count is 0 (i.e., state == ClosingBit), dispose now; else wait for Release() to reach 0 + if ((Volatile.Read(ref _state) & int.MaxValue) == 0) + { + Dispose(); + } } - } - // Internal for testing - internal static bool HasImpactingChanges(TracerSettings.SettingsManager.SettingChanges changes) - { - var hasChanges = changes.UpdatedExporter is not null // relying on this to only be non null if _anything_ changed - || (changes.UpdatedMutable is { } updated - && !( - string.Equals(updated.Environment, changes.PreviousMutable.Environment, StringComparison.Ordinal) - && string.Equals(updated.ServiceVersion, changes.PreviousMutable.ServiceVersion, StringComparison.Ordinal) - // The service name comparison isn't _strictly_ correct, because we normalize it further, but this is probably good enough - && string.Equals(updated.DefaultServiceName, changes.PreviousMutable.DefaultServiceName, StringComparison.OrdinalIgnoreCase) - && updated.GlobalTags.SequenceEqual(changes.PreviousMutable.GlobalTags))); - return hasChanges; + private void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + Log.Debug("Disposing DogStatsdService"); + if (Client is DogStatsdService dogStatsd) + { + dogStatsd.Flush(); + } + + Client.Dispose(); + } + } } } diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/AzureAppServicePerformanceCounters.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/AzureAppServicePerformanceCounters.cs index 460997f07c8e..cf00621a49df 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/AzureAppServicePerformanceCounters.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/AzureAppServicePerformanceCounters.cs @@ -7,6 +7,7 @@ using System; using System.Threading; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Util; using Datadog.Trace.Vendors.Newtonsoft.Json; @@ -20,14 +21,16 @@ internal class AzureAppServicePerformanceCounters : IRuntimeMetricsListener private const string GarbageCollectionMetrics = $"{MetricsNames.Gen0HeapSize}, {MetricsNames.Gen1HeapSize}, {MetricsNames.Gen2HeapSize}, {MetricsNames.LohSize}, {MetricsNames.Gen0CollectionsCount}, {MetricsNames.Gen1CollectionsCount}, {MetricsNames.Gen2CollectionsCount}"; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private int? _previousGen0Count; private int? _previousGen1Count; private int? _previousGen2Count; - public AzureAppServicePerformanceCounters(IDogStatsd statsd) + public AzureAppServicePerformanceCounters(IStatsdManager statsd) { + // We assume this is always used by RuntimeMetricsWriter, and therefore we hae already called SetRequired() + // If it's every used outside that context, we need to update to use SetRequired instead _statsd = statsd; } @@ -37,13 +40,20 @@ public void Dispose() public void Refresh() { + using var lease = _statsd.TryGetClientLease(); + if (lease.Client is not { } statsd) + { + // bail out, we have no client for some reason + return; + } + var rawValue = EnvironmentHelpers.GetEnvironmentVariable(EnvironmentVariableName); var value = JsonConvert.DeserializeObject(rawValue); - _statsd.Gauge(MetricsNames.Gen0HeapSize, value.Gen0Size); - _statsd.Gauge(MetricsNames.Gen1HeapSize, value.Gen1Size); - _statsd.Gauge(MetricsNames.Gen2HeapSize, value.Gen2Size); - _statsd.Gauge(MetricsNames.LohSize, value.LohSize); + statsd.Gauge(MetricsNames.Gen0HeapSize, value.Gen0Size); + statsd.Gauge(MetricsNames.Gen1HeapSize, value.Gen1Size); + statsd.Gauge(MetricsNames.Gen2HeapSize, value.Gen2Size); + statsd.Gauge(MetricsNames.LohSize, value.LohSize); var gen0 = GC.CollectionCount(0); var gen1 = GC.CollectionCount(1); @@ -51,17 +61,17 @@ public void Refresh() if (_previousGen0Count != null) { - _statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); + statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); } if (_previousGen1Count != null) { - _statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); + statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); } if (_previousGen2Count != null) { - _statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); + statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); } _previousGen0Count = gen0; @@ -71,11 +81,6 @@ public void Refresh() Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GarbageCollectionMetrics); } - public void UpdateStatsd(IDogStatsd statsd) - { - Interlocked.Exchange(ref _statsd, statsd); - } - private class PerformanceCountersValue { [JsonProperty("gen0HeapSize")] diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/IRuntimeMetricsListener.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/IRuntimeMetricsListener.cs index 9986aec5c2b3..68f65c30cf33 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/IRuntimeMetricsListener.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/IRuntimeMetricsListener.cs @@ -11,7 +11,5 @@ namespace Datadog.Trace.RuntimeMetrics internal interface IRuntimeMetricsListener : IDisposable { void Refresh(); - - void UpdateStatsd(IDogStatsd statsd); } } diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/MemoryMappedCounters.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/MemoryMappedCounters.cs index 4eb8cfa13aa1..7f710c9cdea5 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/MemoryMappedCounters.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/MemoryMappedCounters.cs @@ -9,6 +9,7 @@ using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; using System.Threading; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Util; using Datadog.Trace.Vendors.StatsdClient; @@ -23,7 +24,7 @@ internal class MemoryMappedCounters : IRuntimeMetricsListener private readonly int _processId; - private IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private int? _previousGen0Count; private int? _previousGen1Count; @@ -33,8 +34,10 @@ internal class MemoryMappedCounters : IRuntimeMetricsListener private MemoryMappedFile _file; private MemoryMappedViewAccessor _view; - public MemoryMappedCounters(IDogStatsd statsd) + public MemoryMappedCounters(IStatsdManager statsd) { + // We assume this is always used by RuntimeMetricsWriter, and therefore we hae already called SetRequired() + // If it's every used outside that context, we need to update to use SetRequired instead _statsd = statsd; ProcessHelpers.GetCurrentProcessInformation(out _, out _, out _processId); @@ -131,10 +134,15 @@ public void Refresh() throw new InvalidOperationException($"The PID in the IPC control block does not match (expected {_processId}, found {controlBlock.Perf.GC.ProcessID}"); } - _statsd.Gauge(MetricsNames.Gen0HeapSize, perf.GC.GenHeapSize0); - _statsd.Gauge(MetricsNames.Gen1HeapSize, perf.GC.GenHeapSize1); - _statsd.Gauge(MetricsNames.Gen2HeapSize, perf.GC.GenHeapSize2); - _statsd.Gauge(MetricsNames.LohSize, perf.GC.LargeObjSize); + // if we can't send stats (e.g. we're shutting down), there's not much point in + // running all this, but seeing as we update various state, play it safe and just do no-ops + using var lease = _statsd.TryGetClientLease(); + var statsd = lease.Client ?? NoOpStatsd.Instance; + + statsd.Gauge(MetricsNames.Gen0HeapSize, perf.GC.GenHeapSize0); + statsd.Gauge(MetricsNames.Gen1HeapSize, perf.GC.GenHeapSize1); + statsd.Gauge(MetricsNames.Gen2HeapSize, perf.GC.GenHeapSize2); + statsd.Gauge(MetricsNames.LohSize, perf.GC.LargeObjSize); var contentionCount = perf.LocksAndThreads.Contention; @@ -144,7 +152,7 @@ public void Refresh() } else { - _statsd.Counter(MetricsNames.ContentionCount, contentionCount - _lastContentionCount.Value); + statsd.Counter(MetricsNames.ContentionCount, contentionCount - _lastContentionCount.Value); _lastContentionCount = contentionCount; } @@ -154,29 +162,27 @@ public void Refresh() if (_previousGen0Count != null) { - _statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); + statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); } if (_previousGen1Count != null) { - _statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); + statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); } if (_previousGen2Count != null) { - _statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); + statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); } _previousGen0Count = gen0; _previousGen1Count = gen1; _previousGen2Count = gen2; - Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GarbageCollectionMetrics); - } - - public void UpdateStatsd(IDogStatsd statsd) - { - Interlocked.Exchange(ref _statsd, statsd); + if (statsd is not NoOpStatsd) + { + Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GarbageCollectionMetrics); + } } [StructLayout(LayoutKind.Sequential)] diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/PerformanceCountersListener.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/PerformanceCountersListener.cs index 1aa2f00a3654..cf0f0abaf7c1 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/PerformanceCountersListener.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/PerformanceCountersListener.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Util; using Datadog.Trace.Vendors.StatsdClient; @@ -27,7 +28,7 @@ internal class PerformanceCountersListener : IRuntimeMetricsListener private readonly string _processName; private readonly int _processId; - private IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private string _instanceName; private PerformanceCounterCategory _memoryCategory; @@ -47,8 +48,10 @@ internal class PerformanceCountersListener : IRuntimeMetricsListener private Task _initializationTask; - public PerformanceCountersListener(IDogStatsd statsd) + public PerformanceCountersListener(IStatsdManager statsd) { + // We assume this is always used by RuntimeMetricsWriter, and therefore we hae already called SetRequired() + // If it's every used outside that context, we need to update to use SetRequired instead _statsd = statsd; ProcessHelpers.GetCurrentProcessInformation(out _processName, out _, out _processId); @@ -83,12 +86,17 @@ public void Refresh() _instanceName = GetSimpleInstanceName(); } - TryUpdateGauge(MetricsNames.Gen0HeapSize, _gen0Size); - TryUpdateGauge(MetricsNames.Gen1HeapSize, _gen1Size); - TryUpdateGauge(MetricsNames.Gen2HeapSize, _gen2Size); - TryUpdateGauge(MetricsNames.LohSize, _lohSize); + // if we can't send stats (e.g. we're shutting down), there's not much point in + // running all this, but seeing as we update various state, play it safe and just do no-ops + using var lease = _statsd.TryGetClientLease(); + var statsd = lease.Client ?? NoOpStatsd.Instance; - TryUpdateCounter(MetricsNames.ContentionCount, _contentionCount, ref _lastContentionCount); + TryUpdateGauge(statsd, MetricsNames.Gen0HeapSize, _gen0Size); + TryUpdateGauge(statsd, MetricsNames.Gen1HeapSize, _gen1Size); + TryUpdateGauge(statsd, MetricsNames.Gen2HeapSize, _gen2Size); + TryUpdateGauge(statsd, MetricsNames.LohSize, _lohSize); + + TryUpdateCounter(statsd, MetricsNames.ContentionCount, _contentionCount, ref _lastContentionCount); var gen0 = GC.CollectionCount(0); var gen1 = GC.CollectionCount(1); @@ -96,29 +104,27 @@ public void Refresh() if (_previousGen0Count != null) { - _statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); + statsd.Increment(MetricsNames.Gen0CollectionsCount, gen0 - _previousGen0Count.Value); } if (_previousGen1Count != null) { - _statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); + statsd.Increment(MetricsNames.Gen1CollectionsCount, gen1 - _previousGen1Count.Value); } if (_previousGen2Count != null) { - _statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); + statsd.Increment(MetricsNames.Gen2CollectionsCount, gen2 - _previousGen2Count.Value); } _previousGen0Count = gen0; _previousGen1Count = gen1; _previousGen2Count = gen2; - Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GarbageCollectionMetrics); - } - - public void UpdateStatsd(IDogStatsd statsd) - { - Interlocked.Exchange(ref _statsd, statsd); + if (statsd is not NoOpStatsd) + { + Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GarbageCollectionMetrics); + } } protected virtual void InitializePerformanceCounters() @@ -152,17 +158,17 @@ protected virtual void InitializePerformanceCounters() } } - private void TryUpdateGauge(string path, PerformanceCounterWrapper counter) + private void TryUpdateGauge(IDogStatsd statsd, string path, PerformanceCounterWrapper counter) { var value = counter.GetValue(_instanceName); if (value != null) { - _statsd.Gauge(path, value.Value); + statsd.Gauge(path, value.Value); } } - private void TryUpdateCounter(string path, PerformanceCounterWrapper counter, ref double? lastValue) + private void TryUpdateCounter(IDogStatsd statsd, string path, PerformanceCounterWrapper counter, ref double? lastValue) { var value = counter.GetValue(_instanceName); @@ -177,7 +183,7 @@ private void TryUpdateCounter(string path, PerformanceCounterWrapper counter, re return; } - _statsd.Counter(path, value.Value - lastValue.Value); + statsd.Counter(path, value.Value - lastValue.Value); lastValue = value; } diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeEventListener.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeEventListener.cs index f035d5b45a45..3784ec12c5dd 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeEventListener.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeEventListener.cs @@ -9,6 +9,7 @@ using System.Collections.ObjectModel; using System.Diagnostics.Tracing; using System.Threading; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.StatsdClient; @@ -41,7 +42,7 @@ internal class RuntimeEventListener : EventListener, IRuntimeMetricsListener private readonly string _delayInSeconds; - private IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private long _contentionCount; @@ -61,7 +62,7 @@ static RuntimeEventListener() }; } - public RuntimeEventListener(IDogStatsd statsd, TimeSpan delay) + public RuntimeEventListener(IStatsdManager statsd, TimeSpan delay) { _statsd = statsd; _delayInSeconds = ((int)delay.TotalSeconds).ToString(); @@ -71,19 +72,22 @@ public RuntimeEventListener(IDogStatsd statsd, TimeSpan delay) public void Refresh() { + // if we can't send stats (e.g. we're shutting down), there's not much point in + // running all this, but seeing as we update various state, play it safe and just do no-ops + using var lease = _statsd.TryGetClientLease(); + var statsd = lease.Client ?? NoOpStatsd.Instance; + // Can't use a Timing because Dogstatsd doesn't support local aggregation // It means that the aggregations in the UI would be wrong - _statsd.Gauge(MetricsNames.ContentionTime, _contentionTime.Clear()); - _statsd.Counter(MetricsNames.ContentionCount, Interlocked.Exchange(ref _contentionCount, 0)); - - _statsd.Gauge(MetricsNames.ThreadPoolWorkersCount, ThreadPool.ThreadCount); + statsd?.Gauge(MetricsNames.ContentionTime, _contentionTime.Clear()); + statsd?.Counter(MetricsNames.ContentionCount, Interlocked.Exchange(ref _contentionCount, 0)); - Log.Debug("Sent the following metrics to the DD agent: {Metrics}", ThreadStatsMetrics); - } + statsd?.Gauge(MetricsNames.ThreadPoolWorkersCount, ThreadPool.ThreadCount); - public void UpdateStatsd(IDogStatsd statsd) - { - Interlocked.Exchange(ref _statsd, statsd); + if (statsd is not NoOpStatsd) + { + Log.Debug("Sent the following metrics to the DD agent: {Metrics}", ThreadStatsMetrics); + } } protected override void OnEventWritten(EventWrittenEventArgs eventData) @@ -99,9 +103,13 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) try { + using var lease = _statsd.TryGetClientLease(); + // We want to make sure we still refresh everything, so use a noop if not available + var client = lease.Client; + var statsd = client ?? NoOpStatsd.Instance; if (eventData.EventName == "EventCounters") { - ExtractCounters(eventData.Payload); + ExtractCounters(statsd, eventData.Payload); } else if (eventData.EventId == EventGcSuspendBegin) { @@ -113,7 +121,7 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) if (start != null) { - _statsd.Timer(MetricsNames.GcPauseTime, (eventData.TimeStamp - start.Value).TotalMilliseconds); + statsd.Timer(MetricsNames.GcPauseTime, (eventData.TimeStamp - start.Value).TotalMilliseconds); Log.Debug("Sent the following metrics to the DD agent: {Metrics}", MetricsNames.GcPauseTime); } } @@ -123,10 +131,10 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) { var stats = HeapStats.FromPayload(eventData.Payload); - _statsd.Gauge(MetricsNames.Gen0HeapSize, stats.Gen0Size); - _statsd.Gauge(MetricsNames.Gen1HeapSize, stats.Gen1Size); - _statsd.Gauge(MetricsNames.Gen2HeapSize, stats.Gen2Size); - _statsd.Gauge(MetricsNames.LohSize, stats.LohSize); + statsd.Gauge(MetricsNames.Gen0HeapSize, stats.Gen0Size); + statsd.Gauge(MetricsNames.Gen1HeapSize, stats.Gen1Size); + statsd.Gauge(MetricsNames.Gen2HeapSize, stats.Gen2Size); + statsd.Gauge(MetricsNames.LohSize, stats.LohSize); Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GcHeapStatsMetrics); } @@ -143,10 +151,10 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) if (heapHistory.MemoryLoad != null) { - _statsd.Gauge(MetricsNames.GcMemoryLoad, heapHistory.MemoryLoad.Value); + statsd.Gauge(MetricsNames.GcMemoryLoad, heapHistory.MemoryLoad.Value); } - _statsd.Increment(GcCountMetricNames[heapHistory.Generation], 1, tags: heapHistory.Compacting ? CompactingGcTags : NotCompactingGcTags); + statsd.Increment(GcCountMetricNames[heapHistory.Generation], 1, tags: heapHistory.Compacting ? CompactingGcTags : NotCompactingGcTags); Log.Debug("Sent the following metrics to the DD agent: {Metrics}", GcGlobalHeapMetrics); } } @@ -176,7 +184,7 @@ private void EnableEventSource(EventSource eventSource) } } - private void ExtractCounters(ReadOnlyCollection payload) + private void ExtractCounters(IDogStatsd statsd, ReadOnlyCollection payload) { for (int i = 0; i < payload.Count; ++i) { @@ -196,7 +204,7 @@ private void ExtractCounters(ReadOnlyCollection payload) { var value = (double)rawValue; - _statsd.Gauge(statName, value); + statsd.Gauge(statName, value); Log.Debug("Sent the following metrics to the DD agent: {Metrics}", statName); } else diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeMetricsWriter.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeMetricsWriter.cs index c3bdc357e0d9..62816b4962ea 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeMetricsWriter.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeMetricsWriter.cs @@ -9,6 +9,7 @@ using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Threading; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.StatsdClient; @@ -28,7 +29,7 @@ internal class RuntimeMetricsWriter : IDisposable private static readonly Version Windows81Version = new(6, 3, 9600); private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); - private static readonly Func InitializeListenerFunc = InitializeListener; + private static readonly Func InitializeListenerFunc = InitializeListener; [ThreadStatic] private static bool _inspectingFirstChanceException; @@ -51,7 +52,7 @@ internal class RuntimeMetricsWriter : IDisposable #endif private readonly ConcurrentDictionary _exceptionCounts = new ConcurrentDictionary(); - private IDogStatsd _statsd; + private readonly IStatsdManager _statsd; private int _outOfMemoryCount; // The time when the runtime metrics were last pushed @@ -61,15 +62,16 @@ internal class RuntimeMetricsWriter : IDisposable private TimeSpan _previousSystemCpu; private int _disposed; - public RuntimeMetricsWriter(IDogStatsd statsd, TimeSpan delay, bool inAzureAppServiceContext) + public RuntimeMetricsWriter(IStatsdManager statsd, TimeSpan delay, bool inAzureAppServiceContext) : this(statsd, delay, inAzureAppServiceContext, InitializeListenerFunc) { } - internal RuntimeMetricsWriter(IDogStatsd statsd, TimeSpan delay, bool inAzureAppServiceContext, Func initializeListener) + internal RuntimeMetricsWriter(IStatsdManager statsd, TimeSpan delay, bool inAzureAppServiceContext, Func initializeListener) { _delay = delay; _statsd = statsd; + _statsd.SetRequired(StatsdConsumer.RuntimeMetricsWriter, enabled: true); _lastUpdate = DateTime.UtcNow; try @@ -158,12 +160,6 @@ public void Dispose() _exceptionCounts.Clear(); } - internal void UpdateStatsd(IDogStatsd statsd) - { - Interlocked.Exchange(ref _statsd, statsd); - _listener?.UpdateStatsd(statsd); - } - internal void PushEvents() { if (Volatile.Read(ref _disposed) == 1) @@ -180,6 +176,10 @@ internal void PushEvents() _lastUpdate = now; _listener?.Refresh(); + // if we can't send stats (e.g. we're shutting down), there's not much point in + // running all this, but seeing as we update various state, play it safe and just do no-ops + using var lease = _statsd.TryGetClientLease(); + var statsd = lease.Client ?? NoOpStatsd.Instance; if (_enableProcessMetrics) { @@ -196,25 +196,28 @@ internal void PushEvents() var maximumCpu = Environment.ProcessorCount * elapsedSinceLastUpdate.TotalMilliseconds; var totalCpu = userCpu + systemCpu; - _statsd.Gauge(MetricsNames.ThreadsCount, threadCount); + statsd.Gauge(MetricsNames.ThreadsCount, threadCount); #if NETSTANDARD if (_enableProcessMemory) { - _statsd.Gauge(MetricsNames.CommittedMemory, memoryUsage); + statsd.Gauge(MetricsNames.CommittedMemory, memoryUsage); Log.Debug("Sent the following metrics to the DD agent: {Metrics}", MetricsNames.CommittedMemory); } #else - _statsd.Gauge(MetricsNames.CommittedMemory, memoryUsage); + statsd.Gauge(MetricsNames.CommittedMemory, memoryUsage); #endif // Get CPU time in milliseconds per second - _statsd.Gauge(MetricsNames.CpuUserTime, userCpu.TotalMilliseconds / elapsedSinceLastUpdate.TotalSeconds); - _statsd.Gauge(MetricsNames.CpuSystemTime, systemCpu.TotalMilliseconds / elapsedSinceLastUpdate.TotalSeconds); + statsd.Gauge(MetricsNames.CpuUserTime, userCpu.TotalMilliseconds / elapsedSinceLastUpdate.TotalSeconds); + statsd.Gauge(MetricsNames.CpuSystemTime, systemCpu.TotalMilliseconds / elapsedSinceLastUpdate.TotalSeconds); - _statsd.Gauge(MetricsNames.CpuPercentage, Math.Round(totalCpu.TotalMilliseconds * 100 / maximumCpu, 1, MidpointRounding.AwayFromZero)); + statsd.Gauge(MetricsNames.CpuPercentage, Math.Round(totalCpu.TotalMilliseconds * 100 / maximumCpu, 1, MidpointRounding.AwayFromZero)); - Log.Debug("Sent the following metrics to the DD agent: {Metrics}", ProcessMetrics); + if (statsd is not NoOpStatsd) + { + Log.Debug("Sent the following metrics to the DD agent: {Metrics}", ProcessMetrics); + } } bool sentExceptionCount = false; @@ -222,7 +225,7 @@ internal void PushEvents() if (Volatile.Read(ref _outOfMemoryCount) > 0) { var oomCount = Interlocked.Exchange(ref _outOfMemoryCount, 0); - _statsd.Increment(MetricsNames.ExceptionsCount, oomCount, tags: [$"exception_type:{OutOfMemoryExceptionName}"]); + statsd.Increment(MetricsNames.ExceptionsCount, oomCount, tags: [$"exception_type:{OutOfMemoryExceptionName}"]); sentExceptionCount = true; } @@ -230,7 +233,7 @@ internal void PushEvents() { foreach (var element in _exceptionCounts) { - _statsd.Increment(MetricsNames.ExceptionsCount, element.Value, tags: [$"exception_type:{element.Key}"]); + statsd.Increment(MetricsNames.ExceptionsCount, element.Value, tags: [$"exception_type:{element.Key}"]); } // There's a race condition where we could clear items that haven't been pushed @@ -273,7 +276,7 @@ internal void PushEvents() } } - private static IRuntimeMetricsListener InitializeListener(IDogStatsd statsd, TimeSpan delay, bool inAzureAppServiceContext) + private static IRuntimeMetricsListener InitializeListener(IStatsdManager statsd, TimeSpan delay, bool inAzureAppServiceContext) { #if NETCOREAPP return new RuntimeEventListener(statsd, delay); diff --git a/tracer/src/Datadog.Trace/Tracer.cs b/tracer/src/Datadog.Trace/Tracer.cs index 8c726376bd7d..796090f4b821 100644 --- a/tracer/src/Datadog.Trace/Tracer.cs +++ b/tracer/src/Datadog.Trace/Tracer.cs @@ -15,6 +15,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.Debugger; using Datadog.Trace.Debugger.SpanCodeOrigin; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging.TracerFlare; using Datadog.Trace.Sampling; using Datadog.Trace.SourceGenerators; @@ -51,7 +52,7 @@ public class Tracer : IDatadogTracer, IDatadogOpenTracingTracer /// Note that this API does NOT replace the global Tracer instance. /// The created will be scoped specifically to this instance. /// - internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ITraceSampler sampler, IScopeManager scopeManager, IDogStatsd statsd, ITelemetryController telemetry = null, IDiscoveryService discoveryService = null) + internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ITraceSampler sampler, IScopeManager scopeManager, IStatsdManager statsd, ITelemetryController telemetry = null, IDiscoveryService discoveryService = null) : this(TracerManagerFactory.Instance.CreateTracerManager(settings, agentWriter, sampler, scopeManager, statsd, runtimeMetrics: null, logSubmissionManager: null, telemetry: telemetry ?? NullTelemetryController.Instance, discoveryService ?? NullDiscoveryService.Instance, dataStreamsManager: null, remoteConfigurationManager: null, dynamicConfigurationManager: null, tracerFlareManager: null, spanEventsManager: null)) { } diff --git a/tracer/src/Datadog.Trace/TracerManager.cs b/tracer/src/Datadog.Trace/TracerManager.cs index 8b099ab472f3..9d3924008748 100644 --- a/tracer/src/Datadog.Trace/TracerManager.cs +++ b/tracer/src/Datadog.Trace/TracerManager.cs @@ -61,7 +61,7 @@ public TracerManager( TracerSettings settings, IAgentWriter agentWriter, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetricsWriter, DirectLogSubmissionManager directLogSubmission, ITelemetryController telemetry, @@ -155,7 +155,7 @@ public static TracerManager Instance /// Gets the global instance. public QueryStringManager QueryStringManager { get; } - public IDogStatsd Statsd { get; } + public IStatsdManager Statsd { get; } public ITraceProcessor[] TraceProcessors { get; } @@ -778,7 +778,8 @@ private static void HeartbeatCallback(object state) // send traces to the Agent if (_instance?.PerTraceSettings.Settings.TracerMetricsEnabled == true) { - _instance?.Statsd?.Gauge(TracerMetricNames.Health.Heartbeat, Tracer.LiveTracerCount); + using var lease = _instance.Statsd.TryGetClientLease(); + lease.Client?.Gauge(TracerMetricNames.Health.Heartbeat, Tracer.LiveTracerCount); } } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 6b07399ac883..67046b1638d6 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -89,14 +89,14 @@ internal TracerManager CreateTracerManager(TracerSettings settings, TracerManage /// /// Internal for use in tests that create "standalone" by - /// + /// /// internal TracerManager CreateTracerManager( TracerSettings settings, IAgentWriter agentWriter, ITraceSampler sampler, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetrics, DirectLogSubmissionManager logSubmissionManager, ITelemetryController telemetry, @@ -124,11 +124,7 @@ internal TracerManager CreateTracerManager( var telemetrySettings = CreateTelemetrySettings(settings); telemetry ??= CreateTelemetryController(settings, discoveryService, telemetrySettings); - // Technically we don't _always_ need a dogstatsd instance, because we only need it if runtime metrics - // are enabled _or_ tracer metrics are enabled. However, tracer metrics can be enabled and disabled dynamically - // at runtime, which makes managing the lifetime of the statsd instance more complex than we'd like, so - // for simplicity, we _always_ create a new statsd instance - statsd ??= new StatsdManager(settings, includeDefaultTags: true); + statsd ??= new StatsdManager(settings); runtimeMetrics ??= settings.RuntimeMetricsEnabled && !DistributedTracer.Instance.IsChildTracer ? new RuntimeMetricsWriter(statsd, TimeSpan.FromSeconds(10), settings.IsRunningInAzureAppService) : null; @@ -234,7 +230,7 @@ protected virtual TracerManager CreateTracerManagerFrom( TracerSettings settings, IAgentWriter agentWriter, IScopeManager scopeManager, - IDogStatsd statsd, + IStatsdManager statsd, RuntimeMetricsWriter runtimeMetrics, DirectLogSubmissionManager logSubmissionManager, ITelemetryController telemetry, @@ -276,7 +272,7 @@ protected virtual ISpanSampler GetSpanSampler(TracerSettings settings) return new SpanSampler(SpanSamplingRule.BuildFromConfigurationString(settings.SpanSamplingRules, RegexBuilder.DefaultTimeout)); } - protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) + protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IStatsdManager statsd, Action> updateSampleRates, IDiscoveryService discoveryService, TelemetrySettings telemetrySettings) { // Currently we assume this _can't_ toggle at runtime, may need to revisit this if that changes IApi api = settings.DataPipelineEnabled && ManagedTraceExporter.TryCreateTraceExporter(settings, updateSampleRates, telemetrySettings, out var traceExporter) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/Agent/ApmAgentWriterTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/Agent/ApmAgentWriterTests.cs index 03312a522610..540d397da347 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/Agent/ApmAgentWriterTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/Agent/ApmAgentWriterTests.cs @@ -9,6 +9,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Agent.MessagePack; using Datadog.Trace.Ci.Agent; +using Datadog.Trace.TestHelpers.Stats; using Moq; using Xunit; @@ -27,7 +28,7 @@ public ApmAgentWriterTests() _settings = Ci.Configuration.TestOptimizationSettings.FromDefaultSources().TracerSettings; _api = new Mock(); - _ciAgentWriter = new ApmAgentWriter(_api.Object); + _ciAgentWriter = new ApmAgentWriter(_api.Object, TestStatsdManager.NoOp); } [Fact] @@ -58,7 +59,7 @@ public async Task WriteTrace_2Traces_SendToApi() [Fact] public async Task FlushTwice() { - var w = new ApmAgentWriter(_api.Object); + var w = new ApmAgentWriter(_api.Object, TestStatsdManager.NoOp); await w.FlushAndCloseAsync(); await w.FlushAndCloseAsync(); } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index d86e2be379cc..3a5bc5a47dab 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -18,6 +18,7 @@ using Datadog.Trace.LibDatadog.DataPipeline; using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Moq; @@ -86,7 +87,7 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) out var exporter).Should().BeTrue(); exporter.Should().NotBeNull(); - var agentWriter = new AgentWriter(exporter, new NullStatsAggregator(), statsd, tracerSettings); + var agentWriter = new AgentWriter(exporter, new NullStatsAggregator(), new TestStatsdManager(statsd), tracerSettings); await using (var tracer = TracerHelper.Create(tracerSettings, agentWriter: agentWriter, statsd: statsd, discoveryService: discovery)) { diff --git a/tracer/test/Datadog.Trace.IntegrationTests/OriginTagSendTraces.cs b/tracer/test/Datadog.Trace.IntegrationTests/OriginTagSendTraces.cs index 995802ab1a34..f6a922afa739 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/OriginTagSendTraces.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/OriginTagSendTraces.cs @@ -9,6 +9,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -68,7 +69,7 @@ public async Task NormalOriginSpan() private ScopedTracer GetTracer() { var settings = new TracerSettings(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); return TracerHelper.Create(settings, agentWriter, null, null, null); } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/SpanTagTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/SpanTagTests.cs index 500be7eb1847..96a8c986c287 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/SpanTagTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/SpanTagTests.cs @@ -10,6 +10,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.Sampling; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -24,7 +25,7 @@ public class SpanTagTests public SpanTagTests() { _testApi = new MockApi(); - _writer = new AgentWriter(_testApi, statsAggregator: null, statsd: null); + _writer = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp); } [Fact] diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/AASTagsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/AASTagsTests.cs index 6b73272acbc0..4af181fb899e 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/AASTagsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/AASTagsTests.cs @@ -12,6 +12,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.PlatformHelpers; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -32,7 +33,7 @@ public async Task AasTagsShouldBeSerialized() { var source = GetMockVariables(); var settings = new TracerSettings(source); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null); using (tracer.StartActiveInternal("root")) @@ -49,7 +50,7 @@ public async Task AasTagsShouldBeSerialized() public async Task NoAasTagsIfNotInAASContext() { var settings = new TracerSettings(null); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null); using (tracer.StartActiveInternal("root")) @@ -73,7 +74,7 @@ public async Task AasTagsShouldBeSerializedOnLocalRootSpans() var source = GetMockVariables(); var settings = new TracerSettings(source); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null); ISpan span1; diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/ProcessTagsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/ProcessTagsTests.cs index 1b2dfd5a2094..ca242fbe5381 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/ProcessTagsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/ProcessTagsTests.cs @@ -9,6 +9,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -30,7 +31,7 @@ public ProcessTagsTests() public async Task ProcessTags_Only_In_First_Span(bool enabled) { var settings = new TracerSettings(new NameValueConfigurationSource(new NameValueCollection { { ConfigurationKeys.PropagateProcessTags, enabled ? "true" : "false" } })); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter); using (tracer.StartActiveInternal("A")) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithUpstreamService.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithUpstreamService.cs index 1d1a86641627..91582ef56bc4 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithUpstreamService.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithUpstreamService.cs @@ -8,6 +8,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -375,7 +376,7 @@ public async Task FourChunks_1_Span_Each() private ScopedTracer GetTracer() { var settings = new TracerSettings(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); return TracerHelper.Create(settings, agentWriter, null, null, null); } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithoutUpstreamService.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithoutUpstreamService.cs index b256e11455d6..544f0b8f6823 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithoutUpstreamService.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/SamplingPriorityTests_MultipleChunksWithoutUpstreamService.cs @@ -8,6 +8,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -363,7 +364,7 @@ public async Task FourChunks_1_Span_Each() private ScopedTracer GetTracer() { var settings = new TracerSettings(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp); return TracerHelper.Create(settings, agentWriter, null, null, null); } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceContextPropertyTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceContextPropertyTests.cs index beda77730fb6..467a2930acb7 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceContextPropertyTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceContextPropertyTests.cs @@ -9,6 +9,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -148,7 +149,7 @@ private Task AssertTag(string key, string value) private ScopedTracer GetTracer() { var settings = new TracerSettings(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); return TracerHelper.Create(settings, agentWriter, null, null, null); } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTagTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTagTests.cs index 54cf5b99d930..b2bcae6f53be 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTagTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTagTests.cs @@ -10,6 +10,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -73,7 +74,7 @@ public async Task SetTraceTagOnRootSpan() private ScopedTracer GetTracer() { var settings = new TracerSettings(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp); return TracerHelper.Create(settings, agentWriter, null, null, null); } } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTags.cs b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTags.cs index 7bcef0c5f1c3..60b2ce3df3de 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTags.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Tagging/TraceTags.cs @@ -10,6 +10,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.Sampling; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using FluentAssertions; using Xunit; @@ -31,7 +32,7 @@ public async Task SerializeSamplingMechanismTag(string samplingMechanism) var settings = TracerSettings.Create(new() { { ConfigurationKeys.GlobalSamplingRate, 0 } }); var testApi = new MockApi(); - var agentWriter = new AgentWriter(testApi, statsAggregator: null, statsd: null); + var agentWriter = new AgentWriter(testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp); await using var tracer = TracerHelper.Create(settings, agentWriter, null, null, null); using (var scope = tracer.StartActiveInternal("root")) diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/EventTrackingSdkTests.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/EventTrackingSdkTests.cs index 576873cb1e52..a9eae8068413 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/EventTrackingSdkTests.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/EventTrackingSdkTests.cs @@ -9,6 +9,7 @@ using Datadog.Trace.AppSec; using Datadog.Trace.Configuration; using Datadog.Trace.Sampling; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using Moq; using Xunit; @@ -23,7 +24,7 @@ public void TrackUserLoginSuccessEvent_OnRootSpanDirectly_ShouldSetOnTrace() var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); @@ -45,7 +46,7 @@ public void TrackUserLoginSuccessEvent_WithMeta_OnRootSpanDirectly_ShouldSetOnTr var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); @@ -79,7 +80,7 @@ public void TrackUserLoginFailureEvent_OnRootSpanDirectly_ShouldSetOnTrace() var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); @@ -102,7 +103,7 @@ public void TrackUserLoginFailureEvent_WithMeta_OnRootSpanDirectly_ShouldSetOnTr var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); @@ -136,7 +137,7 @@ public void TrackCustomEvent_OnRootSpanDirectly_ShouldSetOnTrace() var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); @@ -157,7 +158,7 @@ public void TrackCustomEvent_WithMeta_OnRootSpanDirectly_ShouldSetOnTrace() var scopeManager = new AsyncLocalScopeManager(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.StartupDiagnosticLogEnabled, false } }); - var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, Mock.Of()); + var tracer = new Tracer(settings, Mock.Of(), Mock.Of(), scopeManager, new TestStatsdManager(Mock.Of())); var rootTestScope = (Scope)tracer.StartActive("test.trace"); var childTestScope = (Scope)tracer.StartActive("test.trace.child"); diff --git a/tracer/test/Datadog.Trace.TestHelpers/Stats/TestStatsdManager.cs b/tracer/test/Datadog.Trace.TestHelpers/Stats/TestStatsdManager.cs new file mode 100644 index 000000000000..56d5b3b97e5e --- /dev/null +++ b/tracer/test/Datadog.Trace.TestHelpers/Stats/TestStatsdManager.cs @@ -0,0 +1,26 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.DogStatsd; +using Datadog.Trace.Vendors.StatsdClient; + +namespace Datadog.Trace.TestHelpers.Stats; + +/// +/// An instance of that returns the provided instance +/// +internal class TestStatsdManager(IDogStatsd client) : IStatsdManager +{ + public static TestStatsdManager NoOp => new(NoOpStatsd.Instance); + + public void Dispose() => client.Dispose(); + + public StatsdManager.StatsdClientLease TryGetClientLease() + => new(new StatsdManager.StatsdClientHolder(client)); + + public void SetRequired(StatsdConsumer consumer, bool enabled) + { + } +} diff --git a/tracer/test/Datadog.Trace.TestHelpers/TestTracer/ScopedTracer.cs b/tracer/test/Datadog.Trace.TestHelpers/TestTracer/ScopedTracer.cs index ea689cff0264..77da03d833a7 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/TestTracer/ScopedTracer.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/TestTracer/ScopedTracer.cs @@ -8,21 +8,39 @@ using Datadog.Trace.Agent; using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.Configuration; +using Datadog.Trace.DogStatsd; using Datadog.Trace.Sampling; using Datadog.Trace.Telemetry; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; namespace Datadog.Trace.TestHelpers.TestTracer; -internal class ScopedTracer( - TracerSettings settings = null, - IAgentWriter agentWriter = null, - ITraceSampler sampler = null, - IScopeManager scopeManager = null, - IDogStatsd statsd = null, - ITelemetryController telemetryController = null, - IDiscoveryService discoveryService = null) - : Tracer(settings, agentWriter, sampler, scopeManager, statsd, telemetry: telemetryController, discoveryService: discoveryService), IAsyncDisposable +internal class ScopedTracer : Tracer, IAsyncDisposable { + public ScopedTracer( + TracerSettings settings = null, + IAgentWriter agentWriter = null, + ITraceSampler sampler = null, + IScopeManager scopeManager = null, + IDogStatsd statsd = null, + ITelemetryController telemetryController = null, + IDiscoveryService discoveryService = null) + : this(settings, agentWriter, sampler, scopeManager, statsd is null ? null : new TestStatsdManager(statsd), telemetryController, discoveryService) + { + } + + public ScopedTracer( + TracerSettings settings, + IAgentWriter agentWriter, + ITraceSampler sampler, + IScopeManager scopeManager, + IStatsdManager statsdManager, + ITelemetryController telemetryController = null, + IDiscoveryService discoveryService = null) + : base(settings, agentWriter, sampler, scopeManager, statsdManager, telemetry: telemetryController, discoveryService: discoveryService) + { + } + public ValueTask DisposeAsync() => new(TracerManager.ShutdownAsync()); } diff --git a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs index dcce90ce9b7c..7eeaa76f8c52 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs @@ -14,6 +14,7 @@ using Datadog.Trace.DogStatsd; using Datadog.Trace.Sampling; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using Datadog.Trace.Vendors.Newtonsoft.Json; using Datadog.Trace.Vendors.StatsdClient; @@ -34,7 +35,7 @@ public AgentWriterTests(ITestOutputHelper output) { _output = output; _api = new Mock(); - _agentWriter = new AgentWriter(_api.Object, statsAggregator: null, statsd: null); + _agentWriter = new AgentWriter(_api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp); } [Fact] @@ -46,7 +47,7 @@ public async Task SpanSampling_CanComputeStats_ShouldNotSend_WhenSpanSamplingDoe statsAggregator.Setup(x => x.CanComputeStats).Returns(true); statsAggregator.Setup(x => x.ProcessTrace(It.IsAny>())).Returns>(x => x); statsAggregator.Setup(x => x.ShouldKeepTrace(It.IsAny>())).Returns(false); - var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: null, automaticFlush: false); + var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agent, sampler: null, scopeManager: null, statsd: null); @@ -75,7 +76,7 @@ public async Task SpanSampling_ShouldSend_SingleMatchedSpan_WhenStatsDrops() statsAggregator.Setup(x => x.ProcessTrace(It.IsAny>())).Returns>(x => x); statsAggregator.Setup(x => x.ShouldKeepTrace(It.IsAny>())).Returns(false); var settings = SpanSamplingRule("*", "*"); - var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: null, automaticFlush: false); + var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agent, sampler: null, scopeManager: null, statsd: null); var traceContext = new TraceContext(tracer); @@ -106,7 +107,7 @@ public async Task SpanSampling_ShouldSend_MultipleMatchedSpans_WhenStatsDrops() statsAggregator.Setup(x => x.ProcessTrace(It.IsAny>())).Returns>(x => x); statsAggregator.Setup(x => x.ShouldKeepTrace(It.IsAny>())).Returns(false); var settings = SpanSamplingRule("*", "*"); - var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: null, automaticFlush: false); + var agent = new AgentWriter(api.Object, statsAggregator.Object, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agent, sampler: null, scopeManager: null, statsd: null); var traceContext = new TraceContext(tracer); @@ -143,7 +144,7 @@ public async Task SpanSampling_ShouldSend_MultipleMatchedSpans_WhenStatsDropsOne statsAggregator.Setup(x => x.ShouldKeepTrace(It.IsAny>())).Returns(false); var settings = SpanSamplingRule("*", "operation"); - var agentWriter = new AgentWriter(api, statsAggregator.Object, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(api, statsAggregator.Object, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null); var traceContext = new TraceContext(tracer); @@ -191,7 +192,7 @@ public void PushStats() statsAggregator.Setup(x => x.ProcessTrace(spans)).Returns(spans); statsAggregator.Setup(x => x.CanComputeStats).Returns(true); - var agent = new AgentWriter(Mock.Of(), statsAggregator.Object, statsd: null, automaticFlush: false); + var agent = new AgentWriter(Mock.Of(), statsAggregator.Object, statsd: TestStatsdManager.NoOp, automaticFlush: false); agent.WriteTrace(spans); @@ -227,7 +228,7 @@ public async Task WriteTrace_2Traces_SendToApi() [Fact] public async Task FlushTwice() { - var w = new AgentWriter(_api.Object, statsAggregator: null, statsd: null); + var w = new AgentWriter(_api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp); await w.FlushAndCloseAsync(); await w.FlushAndCloseAsync(); } @@ -238,7 +239,7 @@ public async Task FaultyApi() // The flush thread should be able to recover from an error when calling the API // Also, it should free the faulty buffer var api = new Mock(); - var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: null, automaticFlush: false); + var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); api.Setup(a => a.SendTracesAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => throw new InvalidOperationException()); @@ -259,7 +260,7 @@ public Task SwitchBuffer() { // Make sure that the agent is able to switch to the secondary buffer when the primary is full/busy var api = new Mock(); - var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: null); + var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp); var barrier = new Barrier(2); @@ -317,7 +318,7 @@ public async Task FlushBothBuffers() var sizeOfTrace = ComputeSize(CreateTraceChunk(1)); // Make the buffer size big enough for a single trace - var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: null, automaticFlush: false, maxBufferSize: (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1); + var agent = new AgentWriter(api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false, maxBufferSize: (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1); agent.WriteTrace(CreateTraceChunk(1)); agent.WriteTrace(CreateTraceChunk(1)); @@ -347,7 +348,7 @@ public void DropTraces() var sizeOfTrace = ComputeSize(CreateTraceChunk(1)); // Make the buffer size big enough for a single trace - var agent = new AgentWriter(Mock.Of(), statsAggregator: null, statsd.Object, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, initialTracerMetricsEnabled: true); + var agent = new AgentWriter(Mock.Of(), statsAggregator: null, new TestStatsdManager(statsd.Object), automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, initialTracerMetricsEnabled: true); // Fill the two buffers agent.WriteTrace(CreateTraceChunk(1)); @@ -397,7 +398,7 @@ public void DropTraces() [Fact] public Task WakeUpSerializationTask() { - var agent = new AgentWriter(Mock.Of(), statsAggregator: null, statsd: null, batchInterval: 0); + var agent = new AgentWriter(Mock.Of(), statsAggregator: null, statsd: TestStatsdManager.NoOp, batchInterval: 0); // To reduce flakiness, first we make sure the serialization thread is started WaitForDequeue(agent); @@ -438,7 +439,7 @@ public async Task AddsTraceKeepRateMetricToRootSpan() // Make the buffer size big enough for a single trace var api = new MockApi(); - var agent = new AgentWriter(api, statsAggregator: null, statsd: null, calculator, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, batchInterval: 100, apmTracingEnabled: true, initialTracerMetricsEnabled: false); + var agent = new AgentWriter(api, statsAggregator: null, statsd: TestStatsdManager.NoOp, calculator, automaticFlush: false, (sizeOfTrace * 2) + SpanBuffer.HeaderSize - 1, batchInterval: 100, apmTracingEnabled: true, initialTracerMetricsEnabled: false); // Fill both buffers agent.WriteTrace(spans); @@ -470,7 +471,7 @@ public async Task AddsTraceKeepRateMetricToRootSpan() public void AgentWriterEnqueueFlushTasks() { var api = new Mock(); - var agentWriter = new AgentWriter(api.Object, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(api.Object, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); var flushTcs = new TaskCompletionSource(); int invocation = 0; diff --git a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs index 0498c15736fe..39a735f443de 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/ApiTests.cs @@ -13,6 +13,7 @@ using Datadog.Trace.Agent.Transports; using Datadog.Trace.Configuration; using Datadog.Trace.Logging; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.Newtonsoft.Json; using FluentAssertions; using Moq; @@ -39,7 +40,7 @@ public async Task SendTraceAsync_200OK_AllGood() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -67,7 +68,7 @@ public async Task SendTracesAsync_ShouldNotRetry_ForSpecificResponses(int status factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -92,7 +93,7 @@ public async Task SendTracesAsync_ShouldSendFiveTimes_ForFailedResponses(int sta factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -123,7 +124,7 @@ public async Task SendTracesAsync_ShouldSendThreeTimes_ForFailedResponseThenSucc factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var responseResult = await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -145,7 +146,7 @@ public async Task SendTracesAsync_500_ErrorIsCaught() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -166,7 +167,7 @@ public async Task SendStatsAsync_200OK_AllGood() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), [])) { @@ -192,7 +193,7 @@ public async Task SendStatsAsync_500_ErrorIsCaught() factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); var statsBuffer = new StatsBuffer(new ClientStatsPayload(MutableSettings.CreateForTesting(new(), []))); @@ -217,7 +218,7 @@ public async Task StatsHeader(bool statsComputationEnabled) factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, statsComputationEnabled, 0, 0); @@ -243,7 +244,7 @@ public async Task ExtractAgentVersionHeaderAndLogsWarning() var logMock = new Mock(); - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: true, log: logMock.Object, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: true, log: logMock.Object, healthMetricsEnabled: false); // First time should write the warning await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); @@ -284,7 +285,7 @@ public async Task SetsDefaultSamplingRates() var ratesWereSet = false; Action> updateSampleRates = _ => ratesWereSet = true; - var api = new Api(apiRequestFactory: factoryMock.Object, statsd: null, updateSampleRates: updateSampleRates, partialFlushEnabled: false, healthMetricsEnabled: false); + var api = new Api(apiRequestFactory: factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: updateSampleRates, partialFlushEnabled: false, healthMetricsEnabled: false); await api.SendTracesAsync(new ArraySegment(new byte[64]), 1, false, 0, 0, false); ratesWereSet.Should().BeTrue(); @@ -311,7 +312,7 @@ public void LogPartialFlushWarning(string agentVersion, bool partialFlushEnabled factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == TracesPath))).Returns(new Uri("http://localhost/traces")); factoryMock.Setup(x => x.GetEndpoint(It.Is(s => s == StatsPath))).Returns(new Uri("http://localhost/stats")); - var api = new Api(factoryMock.Object, statsd: null, updateSampleRates: null, partialFlushEnabled: partialFlushEnabled, healthMetricsEnabled: false); + var api = new Api(factoryMock.Object, statsd: TestStatsdManager.NoOp, updateSampleRates: null, partialFlushEnabled: partialFlushEnabled, healthMetricsEnabled: false); // First call depends on the parameters of the test api.LogPartialFlushWarningIfRequired(agentVersion).Should().Be(expectedResult); diff --git a/tracer/test/Datadog.Trace.Tests/Agent/MessagePack/SpanMessagePackFormatterTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/MessagePack/SpanMessagePackFormatterTests.cs index d1240bc04027..be235d2ddca0 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/MessagePack/SpanMessagePackFormatterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/MessagePack/SpanMessagePackFormatterTests.cs @@ -18,6 +18,7 @@ using Datadog.Trace.Tagging; using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.TestHelpers.TestTracer; using Datadog.Trace.Tests.Util; using Datadog.Trace.Util; @@ -221,7 +222,7 @@ public async Task SpanEvent_Tag_Serialization(bool? nativeSpanEventsEnabled) var discoveryService = new DiscoveryServiceMock(); var mockApi = new MockApi(); var settings = TracerSettings.Create(new()); - var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null, NullTelemetryController.Instance, discoveryService: discoveryService); tracer.TracerManager.Start(); @@ -450,7 +451,7 @@ public async Task TraceId128_PropagatedTag(bool generate128BitTraceId) { var mockApi = new MockApi(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.FeatureFlags.TraceId128BitGenerationEnabled, generate128BitTraceId } }); - var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null, NullTelemetryController.Instance, NullDiscoveryService.Instance); using (_ = tracer.StartActive("root")) @@ -491,7 +492,7 @@ public async Task LastParentId_Tag() { var mockApi = new MockApi(); var settings = TracerSettings.Create(new() { { ConfigurationKeys.FeatureFlags.TraceId128BitGenerationEnabled, false } }); - var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(mockApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); await using var tracer = TracerHelper.Create(settings, agentWriter, sampler: null, scopeManager: null, statsd: null, NullTelemetryController.Instance, NullDiscoveryService.Instance); using (var scope = tracer.StartActiveInternal("root")) diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs index e5bfe5096fc3..512d19b2fb5e 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs @@ -3,9 +3,18 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation; using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.ConfigurationSources; using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.DogStatsd; +using Datadog.Trace.Vendors.StatsdClient; using FluentAssertions; using Xunit; @@ -97,4 +106,710 @@ public void HasImpactingChanges_WhenMutableChangesGlobalTags() PreviousExporter); StatsdManager.HasImpactingChanges(changes).Should().BeTrue(); } + + [Fact] + public void InitialState_ClientNotCreated() + { + var clientCount = 0; + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + var lease = manager.TryGetClientLease(); + + lease.Client.Should().BeNull("client should not be created unless required"); + clientCount.Should().Be(0, "factory should not be called"); + } + + [Fact] + public void SetRequired_CreatesClient() + { + var clientCount = 0; + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var lease = manager.TryGetClientLease(); + + lease.Client.Should().NotBeNull("client should be created when required"); + clientCount.Should().Be(1, "factory should be called exactly once"); + } + + [Fact] + public void SetRequired_False_DisposesClient() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + using (manager.TryGetClientLease()) + { + } + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + + client.IsDisposed.Should().BeTrue("client should be disposed when no longer required and not in use"); + } + + [Fact] + public void MultipleConsumers_AllRequire_SingleClient() + { + var clientCount = 0; + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + manager.SetRequired(StatsdConsumer.TraceApi, true); + manager.SetRequired(StatsdConsumer.AgentWriter, true); + + clientCount.Should().Be(1, "only one client should be created for multiple consumers"); + } + + [Fact] + public void MultipleConsumers_PartialUnrequire_KeepsClient() + { + var clientCount = 0; + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + manager.SetRequired(StatsdConsumer.TraceApi, true); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + + clientCount.Should().Be(1, "only one client should be created for multiple consumers"); + client.IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); + } + + [Fact] + public void MultipleConsumers_AllUnrequire_DisposesClient() + { + var clientCount = 0; + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return client; + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + manager.SetRequired(StatsdConsumer.TraceApi, true); + + using (manager.TryGetClientLease()) + { + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + manager.SetRequired(StatsdConsumer.TraceApi, false); + + client.IsDisposed.Should().BeFalse("client should be disposed while it is leased"); + } + + client.IsDisposed.Should().BeTrue("client should be disposed when all consumers unrequire it"); + } + + [Fact] + public void MultipleConsumers_ReRequire_CreatesNewClient() + { + var clientCount = 0; + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + clientCount.Should().Be(3); + client.IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); + } + + [Fact] + public void Lease_ProvidesAccessToClient() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + using var lease = manager.TryGetClientLease(); + + lease.Client.Should().BeSameAs(client); + } + + [Fact] + public void MultipleLeasesSimultaneously_ShareSameClient() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + using var lease1 = manager.TryGetClientLease(); + using var lease2 = manager.TryGetClientLease(); + using var lease3 = manager.TryGetClientLease(); + + lease1.Client.Should().BeSameAs(client); + lease2.Client.Should().BeSameAs(client); + lease3.Client.Should().BeSameAs(client); + } + + [Fact] + public void DisposingLease_DoesNotDisposeClient_WhileOtherLeasesActive() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var lease1 = manager.TryGetClientLease(); + var lease2 = manager.TryGetClientLease(); + + lease1.Dispose(); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + + client.IsDisposed.Should().BeFalse("client should not be disposed while other leases are active"); + lease2.Dispose(); + client.IsDisposed.Should().BeTrue(); + } + + [Fact] + public void NeverReturnsDisposedClient() + { + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var lease1 = manager.TryGetClientLease(); + lease1.Dispose(); + + // Trigger client recreation + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + // Try to get a new lease + var lease2 = manager.TryGetClientLease(); + + lease2.Client.Should().NotBeNull(); + ((MockStatsdClient)lease2.Client).IsDisposed.Should().BeFalse("should never return a disposed client"); + + // Cleanup + lease2.Dispose(); + } + + [Fact] + public void ReferenceCountingPreventsDisposalWhileLeasesActive() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var lease = manager.TryGetClientLease(); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + + client.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); + + lease.Dispose(); + + client.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); + } + + [Fact] + public void Dispose_WithActiveLease_DisposesAfterLeaseReleased() + { + var client = new MockStatsdClient(); + var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var lease = manager.TryGetClientLease(); + manager.Dispose(); // note _manager_ disposed + + client.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); + + // Dispose the lease + lease.Dispose(); + + // Now it should be disposed + client.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); + } + + [Fact] + public void SettingsUpdate_RecreatesClient_WhenRequired() + { + var clientCount = 0; + var tracerSettings = new TracerSettings(); + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var lease1 = manager.TryGetClientLease(); + var client1 = lease1.Client; + lease1.Dispose(); + + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new Dictionary { { TracerSettingKeyConstants.EnvironmentKey, "test" } }, + useDefaultSources: true), + NullConfigurationTelemetry.Instance); + + var lease2 = manager.TryGetClientLease(); + + clientCount.Should().Be(2, "new client should be created after settings change"); + lease2.Client.Should().NotBeSameAs(client1, "should get a new client after settings update"); + + lease2.Dispose(); + } + + [Fact] + public void SettingsUpdate_OldLeaseContinuesWorkingWithOldClient() + { + var tracerSettings = new TracerSettings(); + using var manager = new StatsdManager(tracerSettings, (_, _) => new MockStatsdClient()); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var lease1 = manager.TryGetClientLease(); + var client1 = lease1.Client; + + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new Dictionary { { TracerSettingKeyConstants.EnvironmentKey, "test" } }, + useDefaultSources: true), + NullConfigurationTelemetry.Instance); + + lease1.Client.Should().BeSameAs(client1, "old lease should continue to reference old client"); + ((MockStatsdClient)client1).IsDisposed.Should().BeFalse("old client should not be disposed while lease is active"); + + // Get new lease + var lease2 = manager.TryGetClientLease(); + var client2 = lease2.Client; + lease2.Client.Should().NotBeSameAs(client1, "new lease should get new client"); + + // Cleanup + lease1.Dispose(); + ((MockStatsdClient)client1).IsDisposed.Should().BeTrue("old client should be disposed after lease is released"); + lease2.Dispose(); + ((MockStatsdClient)client2).IsDisposed.Should().BeFalse("new client is still in use"); + } + + [Fact] + public void SettingsUpdate_DoesNotRecreateClient_WhenNotRequired() + { + var tracerSettings = new TracerSettings(); + var clientCount = 0; + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + // Don't call SetRequired - no client should be created + + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new Dictionary { { TracerSettingKeyConstants.EnvironmentKey, "test" } }, + useDefaultSources: true), + NullConfigurationTelemetry.Instance); + + clientCount.Should().Be(0, "client should not be created for settings update when not required"); + } + + [Fact] + public void SettingsUpdate_DoesNotRecreateClient_WhenSettingsDontChange() + { + var tracerSettings = new TracerSettings(); + var clientCount = 0; + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + clientCount.Should().Be(1); + + // Same default settings source + + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource(new Dictionary(), useDefaultSources: true), + NullConfigurationTelemetry.Instance); + + clientCount.Should().Be(1, "client should not be recreated for settings update when no changes"); + } + + [Fact] + public void SettingsUpdate_DoesNotRecreateClient_WhenRelevantSettingsDontChange() + { + var tracerSettings = new TracerSettings(); + var clientCount = 0; + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + clientCount.Should().Be(1); + + // Header tags does not affect statsdclient + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new Dictionary { { TracerSettingKeyConstants.HeaderTags, "some-header" } }, + useDefaultSources: true), + NullConfigurationTelemetry.Instance); + + clientCount.Should().Be(1, "client should not be recreated for settings update when no changes"); + } + + [Fact] + public void ConcurrentLeaseAcquisition_AllSucceed() + { + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var leases = new ConcurrentQueue(); + Parallel.For(0, 100, _ => + { + var lease = manager.TryGetClientLease(); + leases.Enqueue(lease); + }); + + leases.Should().HaveCount(100); + leases.Should().AllSatisfy(lease => lease.Client.Should().BeSameAs(client)); + + // Cleanup + while (leases.TryDequeue(out var lease)) + { + lease.Dispose(); + } + } + + [Fact] + public async Task ConcurrentLeaseAcquisitionAndDisposal_ThreadSafe() + { + var clientCount = 0; + var client = new MockStatsdClient(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return client; + }); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var cts = new CancellationTokenSource(); + + var tasks = new[] + { + Task.Run(() => + { + while (!cts.Token.IsCancellationRequested) + { + var lease = manager.TryGetClientLease(); + lease.Dispose(); + } + }), + Task.Run(() => + { + while (!cts.Token.IsCancellationRequested) + { + var lease = manager.TryGetClientLease(); + lease.Dispose(); + } + }), + Task.Run(() => + { + while (!cts.Token.IsCancellationRequested) + { + var lease = manager.TryGetClientLease(); + lease.Dispose(); + } + }) + }; + + Thread.Sleep(100); + cts.Cancel(); + + await Task.WhenAll(tasks); + clientCount.Should().Be(1, "client should not be recreated for settings update when no changes"); + client.IsDisposed.Should().BeFalse("client should not be disposed while still required"); + } + + [Fact] + public void ConcurrentSetRequired_ThreadSafe() + { + var clientCount = 0; + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + // random toggling on an off + Parallel.For(0, 50, i => + { + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, i % 2 == 0); + }); + + Parallel.For(0, 50, i => + { + manager.SetRequired(StatsdConsumer.TraceApi, i % 3 == 0); + }); + + // The exact client count is non-deterministic, but should be reasonable + clientCount.Should().BeLessThan(50, "should not create excessive clients"); + } + + [Fact] + public async Task ConcurrentSettingsUpdateAndLeaseAcquisition_ThreadSafe() + { + var tracerSettings = new TracerSettings(); + var clientCount = 0; + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + Interlocked.Increment(ref clientCount); + return new MockStatsdClient(); + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var cts = new CancellationTokenSource(); + var disposedClientReturned = 0; + + var tasks = new[] + { + Task.Run(() => + { + // Update the environment continuously + var counter = 0; + while (!cts.Token.IsCancellationRequested) + { + tracerSettings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new Dictionary { { TracerSettingKeyConstants.EnvironmentKey, $"env{counter++}" } }, + useDefaultSources: true), + NullConfigurationTelemetry.Instance); + } + }), + + Task.Run(() => + { + while (!cts.Token.IsCancellationRequested) + { + using var lease = manager.TryGetClientLease(); + if (lease.Client != null) + { + // Check if client is disposed WHILE we hold the lease + if (((MockStatsdClient)lease.Client).IsDisposed) + { + Interlocked.Increment(ref disposedClientReturned); + } + } + } + }) + }; + + await Task.Delay(500); + cts.Cancel(); + + await Task.WhenAll(tasks); + disposedClientReturned.Should().Be(0, "should never return a disposed client while holding a lease"); + } + + [Fact] + public async Task ConcurrentLeaseDisposalDuringClientRecreation_ThreadSafe() + { + var clients = new ConcurrentQueue(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + var client = new MockStatsdClient(); + clients.Enqueue(client); + return client; + }); + + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + // Create a bunch of leases + var leases = Enumerable.Range(0, 10) + .Select(_ => manager.TryGetClientLease()) + .ToList(); + + var random = new Random(); + + var mutex = new CountdownEvent(leases.Count + 1); + var tasks = leases.Select(lease => Task.Run(() => + { + mutex.Signal(); // decrement + mutex.Wait(); // wait for count to hit zero + Thread.Sleep(random.Next(1, 10)); + lease.Dispose(); + })).ToList(); + + // Wait and then do everything at once + mutex.Signal(); + mutex.Wait(); + + // Trigger recreation while leases are being disposed + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + await Task.WhenAll(tasks); + + // We only recreated once + clients.Should().HaveCount(2); + clients.TryDequeue(out var client1).Should().BeTrue(); + client1.IsDisposed.Should().BeTrue("old client should be disposed"); + clients.TryDequeue(out var client2).Should().BeTrue(); + client2.IsDisposed.Should().BeFalse("latest client should not be disposed"); + } + + [Fact] + public void MultipleTransitionsBetweenRequiredAndNotRequired() + { + var clients = new List(); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + var client = new MockStatsdClient(); + clients.Add(client); + return client; + }); + + for (var i = 0; i < 5; i++) + { + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + var lease = manager.TryGetClientLease(); + lease.Dispose(); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + } + + clients.Count.Should().Be(5, "should create a new client for each transition"); + clients.Should().AllSatisfy(client => client.IsDisposed.Should().BeTrue("all old clients should be disposed")); + } + + [Fact] + public void Dispose_MultipleTimes_IsSafe() + { + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + manager.Dispose(); + manager.Dispose(); + manager.Dispose(); + } + + [Fact] + public void DefaultLease_CanDisposeSafely() + { + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + + var lease = manager.TryGetClientLease(); + + lease.Client.Should().BeNull(); + lease.Dispose(); + } + + [Fact] + public void DisposingLease_MultipleTimes_DoesNotDisposeStatsDMultipleTimes() + { + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); + + var lease = manager.TryGetClientLease(); + var client = lease.Client.Should().NotBeNull().And.BeOfType().Subject; + manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); + lease.Dispose(); + lease.Dispose(); + lease.Dispose(); + client.DisposeCount.Should().Be(1); + } + + private class MockStatsdClient : IDogStatsd + { + private int _disposeCount; + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public bool IsDisposed => DisposeCount > 0; + + public ITelemetryCounters TelemetryCounters => null; + + public void Configure(StatsdConfig config) + { + } + + public void Counter(string statName, double value, double sampleRate = 1, string[] tags = null) + { + } + + public void Decrement(string statName, int value = 1, double sampleRate = 1, params string[] tags) + { + } + + public void Event(string title, string text, string alertType = null, string aggregationKey = null, string sourceType = null, int? dateHappened = null, string priority = null, string hostname = null, string[] tags = null) + { + } + + public void Gauge(string statName, double value, double sampleRate = 1, string[] tags = null) + { + } + + public void Histogram(string statName, double value, double sampleRate = 1, string[] tags = null) + { + } + + public void Distribution(string statName, double value, double sampleRate = 1, string[] tags = null) + { + } + + public void Increment(string statName, int value = 1, double sampleRate = 1, string[] tags = null) + { + } + + public void Set(string statName, T value, double sampleRate = 1, string[] tags = null) + { + } + + public void Set(string statName, string value, double sampleRate = 1, string[] tags = null) + { + } + + public IDisposable StartTimer(string name, double sampleRate = 1, string[] tags = null) + { + return null; + } + + public void Time(Action action, string statName, double sampleRate = 1, string[] tags = null) + { + } + + public T Time(Func func, string statName, double sampleRate = 1, string[] tags = null) + { + return func(); + } + + public void Timer(string statName, double value, double sampleRate = 1, string[] tags = null) + { + } + + public void ServiceCheck(string name, Status status, int? timestamp = null, string hostname = null, string[] tags = null, string message = null) + { + } + + public void Dispose() + { + Interlocked.Increment(ref _disposeCount); + } + } } diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/AzurePerformanceCountersListenerTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/AzurePerformanceCountersListenerTests.cs index 98bf21c35159..2fdef369700f 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/AzurePerformanceCountersListenerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/AzurePerformanceCountersListenerTests.cs @@ -7,6 +7,7 @@ using System; using Datadog.Trace.RuntimeMetrics; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using Moq; using Xunit; @@ -29,7 +30,7 @@ public void PushEvents() var statsd = new Mock(); - using var listener = new AzureAppServicePerformanceCounters(statsd.Object); + using var listener = new AzureAppServicePerformanceCounters(new TestStatsdManager(statsd.Object)); listener.Refresh(); diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/MemoryMappedCountersTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/MemoryMappedCountersTests.cs index ca6fc3477f6d..2349e7d05e60 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/MemoryMappedCountersTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/MemoryMappedCountersTests.cs @@ -6,6 +6,7 @@ #if NETFRAMEWORK using Datadog.Trace.RuntimeMetrics; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using Moq; using Xunit; @@ -19,7 +20,7 @@ public void PushEvents() { var statsd = new Mock(); - using var listener = new MemoryMappedCounters(statsd.Object); + using var listener = new MemoryMappedCounters(new TestStatsdManager(statsd.Object)); listener.Refresh(); diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/PerformanceCountersListenerTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/PerformanceCountersListenerTests.cs index ab9b1819fce8..e0bfa929c2b1 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/PerformanceCountersListenerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/PerformanceCountersListenerTests.cs @@ -7,7 +7,9 @@ using System; using System.Threading; using System.Threading.Tasks; +using Datadog.Trace.DogStatsd; using Datadog.Trace.RuntimeMetrics; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using FluentAssertions; using Moq; @@ -22,7 +24,7 @@ public async Task PushEvents() { var statsd = new Mock(); - using var listener = new PerformanceCountersListener(statsd.Object); + using var listener = new PerformanceCountersListener(new TestStatsdManager(statsd.Object)); await listener.WaitForInitialization(); @@ -64,7 +66,7 @@ void Callback() var statsd = new Mock(); - using var listener = new TestPerformanceCounterListener(statsd.Object, Callback); + using var listener = new TestPerformanceCounterListener(new TestStatsdManager(statsd.Object), Callback); // The first SignalAndWait will deadlock if InitializePerformanceCounters is not called asynchronously barrier.SignalAndWait(); @@ -86,7 +88,7 @@ private class TestPerformanceCounterListener : PerformanceCountersListener // The field needs to be volatile because it's used concurrently from two threads private volatile Action _callback; - public TestPerformanceCounterListener(IDogStatsd statsd, Action callback) + public TestPerformanceCounterListener(IStatsdManager statsd, Action callback) : base(statsd) { _callback = callback; diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs index d0781983fcc6..d29846106a63 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs @@ -7,9 +7,16 @@ #if NET5_0_OR_GREATER using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Diagnostics.Tracing; using System.Threading; +using Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.ConfigurationSources; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.DogStatsd; using Datadog.Trace.RuntimeMetrics; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using Moq; using Xunit; @@ -25,7 +32,7 @@ public void PushEvents() { var statsd = new Mock(); - using var listener = new RuntimeEventListener(statsd.Object, TimeSpan.FromSeconds(10)); + using var listener = new RuntimeEventListener(new TestStatsdManager(statsd.Object), TimeSpan.FromSeconds(10)); listener.Refresh(); @@ -47,7 +54,7 @@ public void MonitorGarbageCollections() statsd.Setup(s => s.Timer(MetricsNames.GcPauseTime, It.IsAny(), It.IsAny(), null)) .Callback(() => mutex.Set()); - using var listener = new RuntimeEventListener(statsd.Object, TimeSpan.FromSeconds(10)); + using var listener = new RuntimeEventListener(new TestStatsdManager(statsd.Object), TimeSpan.FromSeconds(10)); statsd.Invocations.Clear(); @@ -98,7 +105,7 @@ public void PushEventCounters() }; var statsd = new Mock(); - using var listener = new RuntimeEventListener(statsd.Object, TimeSpan.FromSeconds(1)); + using var listener = new RuntimeEventListener(new TestStatsdManager(statsd.Object), TimeSpan.FromSeconds(1)); // Wait for the counters to be refreshed mutex.Wait(); @@ -141,12 +148,22 @@ public void UpdateStatsdOnReinitialization() var originalStatsd = new Mock(); var newStatsd = new Mock(); - using var listener = new RuntimeEventListener(originalStatsd.Object, TimeSpan.FromSeconds(1)); - using var writer = new RuntimeMetricsWriter(originalStatsd.Object, TimeSpan.FromSeconds(1), false); + var settings = TracerSettings.Create(new() { { ConfigurationKeys.ServiceName, "original" } }); + var statsdManager = new StatsdManager( + settings, + (m, e) => m.ServiceName == "original" ? originalStatsd.Object : newStatsd.Object); + + using var listener = new RuntimeEventListener(statsdManager, TimeSpan.FromSeconds(1)); + using var writer = new RuntimeMetricsWriter(statsdManager, TimeSpan.FromSeconds(1), false); mutex.Wait(); - writer.UpdateStatsd(newStatsd.Object); + // Updating the service name should trigger a new statsd client to be created + settings.Manager.UpdateManualConfigurationSettings( + new ManualInstrumentationConfigurationSource( + new ReadOnlyDictionary(new Dictionary { { TracerSettingKeyConstants.ServiceNameKey, "updated" } }), + useDefaultSources: true), + NullConfigurationTelemetry.Instance); mutex.Reset(); mutex.Wait(); diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs index 22dfd3967d21..4e783833d09c 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Datadog.Trace.RuntimeMetrics; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using FluentAssertions; using Moq; @@ -29,7 +30,7 @@ public void PushEvents() listener.Setup(l => l.Refresh()) .Callback(() => mutex.Set()); - using (new RuntimeMetricsWriter(Mock.Of(), TimeSpan.FromMilliseconds(10), false, (statsd, timeSpan, inAppContext) => listener.Object)) + using (new RuntimeMetricsWriter(new TestStatsdManager(Mock.Of()), TimeSpan.FromMilliseconds(10), false, (statsd, timeSpan, inAppContext) => listener.Object)) { Assert.True(mutex.Wait(10000), "Method Refresh() wasn't called on the listener"); } @@ -38,7 +39,7 @@ public void PushEvents() [Fact] public void ShouldSwallowFactoryExceptions() { - var writer = new RuntimeMetricsWriter(Mock.Of(), TimeSpan.FromMilliseconds(10), false, (statsd, timeSpan, inAppContext) => throw new InvalidOperationException("This exception should be caught")); + var writer = new RuntimeMetricsWriter(new TestStatsdManager(Mock.Of()), TimeSpan.FromMilliseconds(10), false, (statsd, timeSpan, inAppContext) => throw new InvalidOperationException("This exception should be caught")); writer.Dispose(); } @@ -48,7 +49,7 @@ public void ShouldCaptureFirstChanceExceptions() var statsd = new Mock(); var listener = new Mock(); - using (var writer = new RuntimeMetricsWriter(statsd.Object, TimeSpan.FromMilliseconds(Timeout.Infinite), false, (statsd, timeSpan, inAppContext) => listener.Object)) + using (var writer = new RuntimeMetricsWriter(new TestStatsdManager(statsd.Object), TimeSpan.FromMilliseconds(Timeout.Infinite), false, (statsd, timeSpan, inAppContext) => listener.Object)) { for (int i = 0; i < 10; i++) { @@ -113,7 +114,7 @@ public async Task ShouldCaptureProcessMetrics() var statsd = new Mock(); var listener = new Mock(); - using (new RuntimeMetricsWriter(statsd.Object, TimeSpan.FromSeconds(1), false, (_, _, _) => listener.Object)) + using (new RuntimeMetricsWriter(new TestStatsdManager(statsd.Object), TimeSpan.FromSeconds(1), false, (_, _, _) => listener.Object)) { var expectedNumberOfThreads = Process.GetCurrentProcess().Threads.Count; @@ -177,7 +178,7 @@ public void CleanupResources() var statsd = new Mock(); var listener = new Mock(); - var writer = new RuntimeMetricsWriter(statsd.Object, TimeSpan.FromMilliseconds(Timeout.Infinite), false, (statsd, timeSpan, inAppContext) => listener.Object); + var writer = new RuntimeMetricsWriter(new TestStatsdManager(statsd.Object), TimeSpan.FromMilliseconds(Timeout.Infinite), false, (statsd, timeSpan, inAppContext) => listener.Object); writer.Dispose(); #if NETFRAMEWORK diff --git a/tracer/test/Datadog.Trace.Tests/Tagging/TagsListTests.cs b/tracer/test/Datadog.Trace.Tests/Tagging/TagsListTests.cs index 71597c34b7d1..fbac0f382f45 100644 --- a/tracer/test/Datadog.Trace.Tests/Tagging/TagsListTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Tagging/TagsListTests.cs @@ -14,6 +14,7 @@ using Datadog.Trace.SourceGenerators; using Datadog.Trace.Tagging; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Util; using FluentAssertions; using Moq; @@ -30,7 +31,7 @@ public TagsListTests() { var settings = new TracerSettings(); _testApi = new MockApi(); - var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: null, automaticFlush: false); + var agentWriter = new AgentWriter(_testApi, statsAggregator: null, statsd: TestStatsdManager.NoOp, automaticFlush: false); _tracer = new Tracer(settings, agentWriter, sampler: null, scopeManager: null, statsd: null); } diff --git a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs index e16542a50129..408933104353 100644 --- a/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs +++ b/tracer/test/Datadog.Trace.Tests/TracerManagerFactoryTests.cs @@ -19,6 +19,7 @@ using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; using Datadog.Trace.TestHelpers.PlatformHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; using FluentAssertions; using Moq; @@ -133,7 +134,7 @@ private static TracerManager CreateTracerManager(TracerSettings settings) Mock.Of(), Mock.Of(), Mock.Of(), - Mock.Of(), + new TestStatsdManager(Mock.Of()), BuildRuntimeMetrics(), BuildLogSubmissionManager(), Mock.Of(), @@ -157,7 +158,7 @@ static DirectLogSubmissionManager BuildLogSubmissionManager() gitMetadataTagsProvider: Mock.Of()); static RuntimeMetricsWriter BuildRuntimeMetrics() - => new(Mock.Of(), TimeSpan.FromMinutes(1), inAzureAppServiceContext: false, (_, _, _) => Mock.Of()); + => new(new TestStatsdManager(Mock.Of()), TimeSpan.FromMinutes(1), inAzureAppServiceContext: false, (_, _, _) => Mock.Of()); } private static IConfigurationSource CreateConfigurationSource(params (string Key, string Value)[] values) diff --git a/tracer/test/Datadog.Trace.Tests/Util/RandomIdGeneratorTests.cs b/tracer/test/Datadog.Trace.Tests/Util/RandomIdGeneratorTests.cs index 6a8f71af92c4..a1db4e113165 100644 --- a/tracer/test/Datadog.Trace.Tests/Util/RandomIdGeneratorTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Util/RandomIdGeneratorTests.cs @@ -12,6 +12,7 @@ using Datadog.Trace.Sampling; using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; +using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Util; using Datadog.Trace.Vendors.StatsdClient; using FluentAssertions; @@ -159,7 +160,7 @@ public void Default_Is_128Bit_TraceId() Mock.Of(), Mock.Of(), new AsyncLocalScopeManager(), - Mock.Of(), + new TestStatsdManager(Mock.Of()), Mock.Of(), Mock.Of()); @@ -179,7 +180,7 @@ public void Configure_128Bit_TraceId_Disabled() Mock.Of(), Mock.Of(), new AsyncLocalScopeManager(), - Mock.Of(), + new TestStatsdManager(Mock.Of()), Mock.Of(), Mock.Of()); @@ -199,7 +200,7 @@ public void Configure_128Bit_TraceId_Enabled() Mock.Of(), Mock.Of(), new AsyncLocalScopeManager(), - Mock.Of(), + new TestStatsdManager(Mock.Of()), Mock.Of(), Mock.Of()); From d95a9af43129ac7bd99bf2200167b056a0e409c6 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 11 Nov 2025 18:11:11 +0000 Subject: [PATCH 28/29] Dispose statsd client in the background The statsd client does sync-over-async in the flush and dispose paths, which can lead to deadlocks and thread exhaustion. To work around that, we push the dispose to happen on a thread-pool thread instead, in the background --- .../Datadog.Trace/DogStatsd/StatsdManager.cs | 30 ++-- .../DogStatsd/StatsdManagerTests.cs | 163 ++++++++++-------- .../RuntimeEventListenerTests.cs | 2 +- 3 files changed, 109 insertions(+), 86 deletions(-) diff --git a/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs index d2dd57a0d08e..3d455cb149ab 100644 --- a/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs +++ b/tracer/src/Datadog.Trace/DogStatsd/StatsdManager.cs @@ -7,6 +7,7 @@ using System; using System.Threading; +using System.Threading.Tasks; using Datadog.Trace.Configuration; using Datadog.Trace.Logging; using Datadog.Trace.Vendors.StatsdClient; @@ -24,7 +25,7 @@ internal sealed class StatsdManager : IStatsdManager private readonly IDisposable _settingSubscription; private int _isRequiredMask; private StatsdClientHolder? _current; - private Func _factory; + private Func _factory; public StatsdManager(TracerSettings tracerSettings) : this(tracerSettings, CreateClient) @@ -32,7 +33,7 @@ public StatsdManager(TracerSettings tracerSettings) } // Internal for testing - internal StatsdManager(TracerSettings tracerSettings, Func statsdFactory) + internal StatsdManager(TracerSettings tracerSettings, Func statsdFactory) { // The initial factory, assuming there's no updates _factory = () => statsdFactory( @@ -172,8 +173,8 @@ internal static bool HasImpactingChanges(TracerSettings.SettingsManager.SettingC return hasChanges; } - private static IDogStatsd CreateClient(MutableSettings settings, ExporterSettings exporter) - => StatsdFactory.CreateDogStatsdClient(settings, exporter, includeDefaultTags: true); + private static StatsdClientHolder CreateClient(MutableSettings settings, ExporterSettings exporter) + => new(StatsdFactory.CreateDogStatsdClient(settings, exporter, includeDefaultTags: true)); private void EnsureClient(bool ensureCreated, bool forceRecreate) { @@ -189,9 +190,7 @@ private void EnsureClient(bool ensureCreated, bool forceRecreate) return; } - _current = ensureCreated - ? new StatsdClientHolder(_factory()) - : null; + _current = ensureCreated ? _factory() : null; } previous?.MarkClosing(); // will dispose when last lease releases @@ -221,6 +220,9 @@ internal sealed class StatsdClientHolder(IDogStatsd client) public IDogStatsd Client { get; } = client; + // Internal for testing + public bool IsDisposed => Volatile.Read(ref _disposed) == 1; + public bool TryRetain() { while (true) @@ -292,12 +294,18 @@ private void Dispose() if (Interlocked.Exchange(ref _disposed, 1) == 0) { Log.Debug("Disposing DogStatsdService"); - if (Client is DogStatsdService dogStatsd) + + // We push this all to a background thread to avoid the disposes running in-line + // the DogStatsdService does sync-over-async, and this can cause thread exhaustion + _ = Task.Run(() => { - dogStatsd.Flush(); - } + if (Client is DogStatsdService dogStatsd) + { + dogStatsd.Flush(); + } - Client.Dispose(); + Client.Dispose(); + }); } } } diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs index 512d19b2fb5e..4dc61ae82271 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsd/StatsdManagerTests.cs @@ -114,7 +114,7 @@ public void InitialState_ClientNotCreated() using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); var lease = manager.TryGetClientLease(); @@ -130,7 +130,7 @@ public void SetRequired_CreatesClient() using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -143,8 +143,8 @@ public void SetRequired_CreatesClient() [Fact] public void SetRequired_False_DisposesClient() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); using (manager.TryGetClientLease()) @@ -153,7 +153,7 @@ public void SetRequired_False_DisposesClient() manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); - client.IsDisposed.Should().BeTrue("client should be disposed when no longer required and not in use"); + holder.IsDisposed.Should().BeTrue("client should be disposed when no longer required and not in use"); } [Fact] @@ -163,7 +163,7 @@ public void MultipleConsumers_AllRequire_SingleClient() using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -177,11 +177,11 @@ public void MultipleConsumers_AllRequire_SingleClient() public void MultipleConsumers_PartialUnrequire_KeepsClient() { var clientCount = 0; - var client = new MockStatsdClient(); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return holder; }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -190,18 +190,18 @@ public void MultipleConsumers_PartialUnrequire_KeepsClient() manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); clientCount.Should().Be(1, "only one client should be created for multiple consumers"); - client.IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); + holder.IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); } [Fact] public void MultipleConsumers_AllUnrequire_DisposesClient() { var clientCount = 0; - var client = new MockStatsdClient(); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return client; + return holder; }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -212,21 +212,23 @@ public void MultipleConsumers_AllUnrequire_DisposesClient() manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); manager.SetRequired(StatsdConsumer.TraceApi, false); - client.IsDisposed.Should().BeFalse("client should be disposed while it is leased"); + holder.IsDisposed.Should().BeFalse("client should be disposed while it is leased"); } - client.IsDisposed.Should().BeTrue("client should be disposed when all consumers unrequire it"); + holder.IsDisposed.Should().BeTrue("client should be disposed when all consumers unrequire it"); } [Fact] public void MultipleConsumers_ReRequire_CreatesNewClient() { var clientCount = 0; - var client = new MockStatsdClient(); + StatsdManager.StatsdClientHolder holder = null; using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + var newClient = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + Interlocked.Exchange(ref holder, newClient); + return newClient; }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -236,42 +238,42 @@ public void MultipleConsumers_ReRequire_CreatesNewClient() manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); clientCount.Should().Be(3); - client.IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); + Volatile.Read(ref holder).IsDisposed.Should().BeFalse("client should remain when at least one consumer requires it"); } [Fact] public void Lease_ProvidesAccessToClient() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); using var lease = manager.TryGetClientLease(); - lease.Client.Should().BeSameAs(client); + lease.Client.Should().BeSameAs(holder.Client); } [Fact] public void MultipleLeasesSimultaneously_ShareSameClient() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); using var lease1 = manager.TryGetClientLease(); using var lease2 = manager.TryGetClientLease(); using var lease3 = manager.TryGetClientLease(); - lease1.Client.Should().BeSameAs(client); - lease2.Client.Should().BeSameAs(client); - lease3.Client.Should().BeSameAs(client); + lease1.Client.Should().BeSameAs(holder.Client); + lease2.Client.Should().BeSameAs(holder.Client); + lease3.Client.Should().BeSameAs(holder.Client); } [Fact] public void DisposingLease_DoesNotDisposeClient_WhileOtherLeasesActive() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var lease1 = manager.TryGetClientLease(); @@ -280,15 +282,21 @@ public void DisposingLease_DoesNotDisposeClient_WhileOtherLeasesActive() lease1.Dispose(); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); - client.IsDisposed.Should().BeFalse("client should not be disposed while other leases are active"); + holder.IsDisposed.Should().BeFalse("client should not be disposed while other leases are active"); lease2.Dispose(); - client.IsDisposed.Should().BeTrue(); + holder.IsDisposed.Should().BeTrue(); } [Fact] public void NeverReturnsDisposedClient() { - using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + StatsdManager.StatsdClientHolder holder = null; + using var manager = new StatsdManager(new TracerSettings(), (_, _) => + { + var newClient = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + Volatile.Write(ref holder, newClient); + return newClient; + }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -303,7 +311,7 @@ public void NeverReturnsDisposedClient() var lease2 = manager.TryGetClientLease(); lease2.Client.Should().NotBeNull(); - ((MockStatsdClient)lease2.Client).IsDisposed.Should().BeFalse("should never return a disposed client"); + holder.IsDisposed.Should().BeFalse("should never return a disposed client"); // Cleanup lease2.Dispose(); @@ -312,37 +320,37 @@ public void NeverReturnsDisposedClient() [Fact] public void ReferenceCountingPreventsDisposalWhileLeasesActive() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var lease = manager.TryGetClientLease(); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); - client.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); + holder.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); lease.Dispose(); - client.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); + holder.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); } [Fact] public void Dispose_WithActiveLease_DisposesAfterLeaseReleased() { - var client = new MockStatsdClient(); - var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var lease = manager.TryGetClientLease(); manager.Dispose(); // note _manager_ disposed - client.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); + holder.IsDisposed.Should().BeFalse("client should not be disposed while lease is active"); // Dispose the lease lease.Dispose(); // Now it should be disposed - client.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); + holder.IsDisposed.Should().BeTrue("client should be disposed after lease is released"); } [Fact] @@ -353,7 +361,7 @@ public void SettingsUpdate_RecreatesClient_WhenRequired() using var manager = new StatsdManager(tracerSettings, (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -379,11 +387,19 @@ public void SettingsUpdate_RecreatesClient_WhenRequired() public void SettingsUpdate_OldLeaseContinuesWorkingWithOldClient() { var tracerSettings = new TracerSettings(); - using var manager = new StatsdManager(tracerSettings, (_, _) => new MockStatsdClient()); + + StatsdManager.StatsdClientHolder holder = null; + using var manager = new StatsdManager(tracerSettings, (_, _) => + { + var newClient = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + Volatile.Write(ref holder, newClient); + return newClient; + }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var lease1 = manager.TryGetClientLease(); var client1 = lease1.Client; + var oldHolder = holder; tracerSettings.Manager.UpdateManualConfigurationSettings( new ManualInstrumentationConfigurationSource( @@ -392,18 +408,17 @@ public void SettingsUpdate_OldLeaseContinuesWorkingWithOldClient() NullConfigurationTelemetry.Instance); lease1.Client.Should().BeSameAs(client1, "old lease should continue to reference old client"); - ((MockStatsdClient)client1).IsDisposed.Should().BeFalse("old client should not be disposed while lease is active"); + Volatile.Read(ref holder)!.IsDisposed.Should().BeFalse("old client should not be disposed while lease is active"); // Get new lease var lease2 = manager.TryGetClientLease(); - var client2 = lease2.Client; lease2.Client.Should().NotBeSameAs(client1, "new lease should get new client"); // Cleanup lease1.Dispose(); - ((MockStatsdClient)client1).IsDisposed.Should().BeTrue("old client should be disposed after lease is released"); + oldHolder.IsDisposed.Should().BeTrue("old client should be disposed after lease is released"); lease2.Dispose(); - ((MockStatsdClient)client2).IsDisposed.Should().BeFalse("new client is still in use"); + Volatile.Read(ref holder)!.IsDisposed.Should().BeFalse("new client is still in use"); } [Fact] @@ -414,7 +429,7 @@ public void SettingsUpdate_DoesNotRecreateClient_WhenNotRequired() using var manager = new StatsdManager(tracerSettings, (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); // Don't call SetRequired - no client should be created @@ -436,7 +451,7 @@ public void SettingsUpdate_DoesNotRecreateClient_WhenSettingsDontChange() using var manager = new StatsdManager(tracerSettings, (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -459,7 +474,7 @@ public void SettingsUpdate_DoesNotRecreateClient_WhenRelevantSettingsDontChange( using var manager = new StatsdManager(tracerSettings, (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -478,8 +493,8 @@ public void SettingsUpdate_DoesNotRecreateClient_WhenRelevantSettingsDontChange( [Fact] public void ConcurrentLeaseAcquisition_AllSucceed() { - var client = new MockStatsdClient(); - using var manager = new StatsdManager(new TracerSettings(), (_, _) => client); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var leases = new ConcurrentQueue(); @@ -490,7 +505,7 @@ public void ConcurrentLeaseAcquisition_AllSucceed() }); leases.Should().HaveCount(100); - leases.Should().AllSatisfy(lease => lease.Client.Should().BeSameAs(client)); + leases.Should().AllSatisfy(lease => lease.Client.Should().BeSameAs(holder.Client)); // Cleanup while (leases.TryDequeue(out var lease)) @@ -503,11 +518,11 @@ public void ConcurrentLeaseAcquisition_AllSucceed() public async Task ConcurrentLeaseAcquisitionAndDisposal_ThreadSafe() { var clientCount = 0; - var client = new MockStatsdClient(); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return client; + return holder; }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var cts = new CancellationTokenSource(); @@ -545,7 +560,7 @@ public async Task ConcurrentLeaseAcquisitionAndDisposal_ThreadSafe() await Task.WhenAll(tasks); clientCount.Should().Be(1, "client should not be recreated for settings update when no changes"); - client.IsDisposed.Should().BeFalse("client should not be disposed while still required"); + holder.IsDisposed.Should().BeFalse("client should not be disposed while still required"); } [Fact] @@ -555,7 +570,7 @@ public void ConcurrentSetRequired_ThreadSafe() using var manager = new StatsdManager(new TracerSettings(), (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); // random toggling on an off @@ -581,7 +596,7 @@ public async Task ConcurrentSettingsUpdateAndLeaseAcquisition_ThreadSafe() using var manager = new StatsdManager(tracerSettings, (_, _) => { Interlocked.Increment(ref clientCount); - return new MockStatsdClient(); + return new(new MockStatsdClient()); }); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); @@ -612,7 +627,7 @@ public async Task ConcurrentSettingsUpdateAndLeaseAcquisition_ThreadSafe() if (lease.Client != null) { // Check if client is disposed WHILE we hold the lease - if (((MockStatsdClient)lease.Client).IsDisposed) + if (((MockStatsdClient)lease.Client).DisposeCount > 0) { Interlocked.Increment(ref disposedClientReturned); } @@ -631,11 +646,11 @@ public async Task ConcurrentSettingsUpdateAndLeaseAcquisition_ThreadSafe() [Fact] public async Task ConcurrentLeaseDisposalDuringClientRecreation_ThreadSafe() { - var clients = new ConcurrentQueue(); + var holders = new ConcurrentQueue(); using var manager = new StatsdManager(new TracerSettings(), (_, _) => { - var client = new MockStatsdClient(); - clients.Enqueue(client); + var client = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + holders.Enqueue(client); return client; }); @@ -668,21 +683,21 @@ public async Task ConcurrentLeaseDisposalDuringClientRecreation_ThreadSafe() await Task.WhenAll(tasks); // We only recreated once - clients.Should().HaveCount(2); - clients.TryDequeue(out var client1).Should().BeTrue(); + holders.Should().HaveCount(2); + holders.TryDequeue(out var client1).Should().BeTrue(); client1.IsDisposed.Should().BeTrue("old client should be disposed"); - clients.TryDequeue(out var client2).Should().BeTrue(); + holders.TryDequeue(out var client2).Should().BeTrue(); client2.IsDisposed.Should().BeFalse("latest client should not be disposed"); } [Fact] public void MultipleTransitionsBetweenRequiredAndNotRequired() { - var clients = new List(); + var holders = new List(); using var manager = new StatsdManager(new TracerSettings(), (_, _) => { - var client = new MockStatsdClient(); - clients.Add(client); + var client = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + holders.Add(client); return client; }); @@ -694,14 +709,14 @@ public void MultipleTransitionsBetweenRequiredAndNotRequired() manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); } - clients.Count.Should().Be(5, "should create a new client for each transition"); - clients.Should().AllSatisfy(client => client.IsDisposed.Should().BeTrue("all old clients should be disposed")); + holders.Count.Should().Be(5, "should create a new client for each transition"); + holders.Should().AllSatisfy(client => client.IsDisposed.Should().BeTrue("all old clients should be disposed")); } [Fact] public void Dispose_MultipleTimes_IsSafe() { - using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new(new MockStatsdClient())); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); manager.Dispose(); @@ -712,7 +727,7 @@ public void Dispose_MultipleTimes_IsSafe() [Fact] public void DefaultLease_CanDisposeSafely() { - using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => new(new MockStatsdClient())); var lease = manager.TryGetClientLease(); @@ -723,16 +738,18 @@ public void DefaultLease_CanDisposeSafely() [Fact] public void DisposingLease_MultipleTimes_DoesNotDisposeStatsDMultipleTimes() { - using var manager = new StatsdManager(new TracerSettings(), (_, _) => new MockStatsdClient()); + var holder = new StatsdManager.StatsdClientHolder(new MockStatsdClient()); + using var manager = new StatsdManager(new TracerSettings(), (_, _) => holder); manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, true); var lease = manager.TryGetClientLease(); - var client = lease.Client.Should().NotBeNull().And.BeOfType().Subject; + var statsdClient = lease.Client.Should().NotBeNull().And.BeOfType().Subject; manager.SetRequired(StatsdConsumer.RuntimeMetricsWriter, false); lease.Dispose(); lease.Dispose(); lease.Dispose(); - client.DisposeCount.Should().Be(1); + holder.IsDisposed.Should().BeTrue(); + statsdClient.DisposeCount.Should().BeLessThanOrEqualTo(1); // we dispose in the background, so may not have happened yet } private class MockStatsdClient : IDogStatsd @@ -741,8 +758,6 @@ private class MockStatsdClient : IDogStatsd public int DisposeCount => Volatile.Read(ref _disposeCount); - public bool IsDisposed => DisposeCount > 0; - public ITelemetryCounters TelemetryCounters => null; public void Configure(StatsdConfig config) diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs index d29846106a63..2701dc39d087 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeEventListenerTests.cs @@ -151,7 +151,7 @@ public void UpdateStatsdOnReinitialization() var settings = TracerSettings.Create(new() { { ConfigurationKeys.ServiceName, "original" } }); var statsdManager = new StatsdManager( settings, - (m, e) => m.ServiceName == "original" ? originalStatsd.Object : newStatsd.Object); + (m, e) => new(m.ServiceName == "original" ? originalStatsd.Object : newStatsd.Object)); using var listener = new RuntimeEventListener(statsdManager, TimeSpan.FromSeconds(1)); using var writer = new RuntimeMetricsWriter(statsdManager, TimeSpan.FromSeconds(1), false); From 9db67e456e293c648b91231db6573b1cc8fb1cdb Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 26 Nov 2025 11:10:31 +0000 Subject: [PATCH 29/29] Add fix for not re-reporting telemetry when there are "empty" dynamic config changes (#7796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary of changes A fix for #7724 to handle telemetry reporting in dynamic config "reset" scenarios ## Reason for change The system tests for #7724 were failing in some dynamic configuration scenarios. Specifically, the tests were sending remote config _without_ any configuration values "i.e. 'reset to use defaults'" and were waiting a telemetry update. However, we never sent it, because there was "no telemetry to record". Note that we _did_ correctly apply the new configuration, we just didn't report the telemetry correctly, primarily due to limitations in the telemetry protocol. This PR adds a fix for that, and will be merged into #7724. ## Implementation details The solution is to "remember" the telemetry from the default mutable configuration values, _without_ any dynamic sources, and "replay" this telemetry when we update telemetry. This feels kind of hacky, but it's something I suspected we might need to do, and had been avoiding up to this point because we do a "full reconfigure" anyway. ## Test coverage Added a specific unit test that mimics the behaviour of the system-test (i.e. an "empty" dynamic config response) and confirms the telemetry is recorded as expected ## Other details https://datadoghq.atlassian.net/browse/LANGPLAT-819 Part of a config stack - https://github.com/DataDog/dd-trace-dotnet/pull/7522 - https://github.com/DataDog/dd-trace-dotnet/pull/7525 - https://github.com/DataDog/dd-trace-dotnet/pull/7530 - https://github.com/DataDog/dd-trace-dotnet/pull/7532 - https://github.com/DataDog/dd-trace-dotnet/pull/7543 - https://github.com/DataDog/dd-trace-dotnet/pull/7544 - https://github.com/DataDog/dd-trace-dotnet/pull/7721 - https://github.com/DataDog/dd-trace-dotnet/pull/7722 - https://github.com/DataDog/dd-trace-dotnet/pull/7695 - https://github.com/DataDog/dd-trace-dotnet/pull/7723 - https://github.com/DataDog/dd-trace-dotnet/pull/7724 - https://github.com/DataDog/dd-trace-dotnet/pull/7796 👈 Unlike other PRs in the stack, I'll merge this directly into #7724 to fix the tests there, just thought I'd keep this separate for easier reviewing --- .../Configuration/MutableSettings.cs | 4 +- .../Configuration/SettingsManager.cs | 85 ++++++++++-- .../Configuration/TracerSettings.cs | 129 +++++++++--------- .../Transport/RemoteConfigurationApi.cs | 2 + .../EmptyDatadogTracer.cs | 3 +- .../Agent/AgentWriterTests.cs | 3 +- .../TracerSettingsSettingManagerTests.cs | 76 +++++++++++ .../Util/StubDatadogTracer.cs | 3 +- 8 files changed, 222 insertions(+), 83 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs index ca354b771dcd..86b5e076c796 100644 --- a/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/MutableSettings.cs @@ -1074,10 +1074,10 @@ public static MutableSettings CreateInitialMutableSettings( /// by excluding all the default sources. Effectively gives all the settings their default /// values. Should only be used with the manual instrumentation source /// - public static MutableSettings CreateWithoutDefaultSources(TracerSettings tracerSettings) + public static MutableSettings CreateWithoutDefaultSources(TracerSettings tracerSettings, ConfigurationTelemetry telemetry) => CreateInitialMutableSettings( NullConfigurationSource.Instance, - new ConfigurationTelemetry(), + telemetry, new OverrideErrorLog(), tracerSettings); diff --git a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs index 6e8dcc6a8b26..74bcd3e5ece4 100644 --- a/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs +++ b/tracer/src/Datadog.Trace/Configuration/SettingsManager.cs @@ -7,41 +7,56 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using Datadog.Trace.Configuration.ConfigurationSources; using Datadog.Trace.Configuration.ConfigurationSources.Telemetry; using Datadog.Trace.Configuration.Telemetry; -using Datadog.Trace.Logging; namespace Datadog.Trace.Configuration; public partial record TracerSettings { - internal class SettingsManager( - TracerSettings tracerSettings, - MutableSettings initialMutable, - ExporterSettings initialExporter) + internal class SettingsManager { - private readonly TracerSettings _tracerSettings = tracerSettings; + private readonly TracerSettings _tracerSettings; + private readonly ConfigurationTelemetry _initialTelemetry; private readonly List _subscribers = []; private IConfigurationSource _dynamicConfigurationSource = NullConfigurationSource.Instance; private ManualInstrumentationConfigurationSourceBase _manualConfigurationSource = new ManualInstrumentationConfigurationSource(new Dictionary(), useDefaultSources: true); + // We delay creating these, as we likely won't need them + private ConfigurationTelemetry? _noDefaultSettingsTelemetry; + private MutableSettings? _noDefaultSourcesSettings; + private SettingChanges? _latest; + public SettingsManager(IConfigurationSource source, TracerSettings tracerSettings, IConfigurationTelemetry telemetry, OverrideErrorLog errorLog) + { + // We record the telemetry for the initial settings in a dedicated ConfigurationTelemetry, + // because we need to be able to reapply this configuration on dynamic config updates + // We don't re-record error logs, so we just use the built-in for that + var initialTelemetry = new ConfigurationTelemetry(); + InitialMutableSettings = MutableSettings.CreateInitialMutableSettings(source, initialTelemetry, errorLog, tracerSettings); + InitialExporterSettings = new ExporterSettings(source, initialTelemetry); + _tracerSettings = tracerSettings; + _initialTelemetry = initialTelemetry; + initialTelemetry.CopyTo(telemetry); + } + /// /// Gets the initial . On app startup, these will be the values read from /// static sources. To subscribe to updates to these settings, from code or remote config, call . /// - public MutableSettings InitialMutableSettings { get; } = initialMutable; + public MutableSettings InitialMutableSettings { get; } /// /// Gets the initial . On app startup, these will be the values read from /// static sources. To subscribe to updates to these settings, from code or remote config, call . /// - public ExporterSettings InitialExporterSettings { get; } = initialExporter; + public ExporterSettings InitialExporterSettings { get; } /// /// Subscribe to changes in and/or . @@ -133,21 +148,45 @@ private bool UpdateSettings( ManualInstrumentationConfigurationSourceBase manualSource, IConfigurationTelemetry telemetry) { - var initialSettings = manualSource.UseDefaultSources - ? InitialMutableSettings - : MutableSettings.CreateWithoutDefaultSources(_tracerSettings); + // Set the correct default telemetry and initial settings depending + // on whether the manual config source explicitly disables using the default sources + ConfigurationTelemetry defaultTelemetry; + MutableSettings initialSettings; + if (manualSource.UseDefaultSources) + { + defaultTelemetry = _initialTelemetry; + initialSettings = InitialMutableSettings; + } + else + { + // We only need to initialize the "no default sources" settings once + // and we don't want to initialize them if we don't _need_ to + // so lazy-initialize here + if (_noDefaultSourcesSettings is null || _noDefaultSettingsTelemetry is null) + { + InitialiseNoDefaultSourceSettings(); + } + + defaultTelemetry = _noDefaultSettingsTelemetry; + initialSettings = _noDefaultSourcesSettings; + } var current = _latest; var currentMutable = current?.UpdatedMutable ?? current?.PreviousMutable ?? InitialMutableSettings; var currentExporter = current?.UpdatedExporter ?? current?.PreviousExporter ?? InitialExporterSettings; + // we create a temporary ConfigurationTelemetry object to hold the changes to settings + // if nothing is actually written, and nothing changes compared to the default, then we + // don't need to report it to the provided telemetry + var tempTelemetry = new ConfigurationTelemetry(); + var overrideErrorLog = new OverrideErrorLog(); var newMutableSettings = MutableSettings.CreateUpdatedMutableSettings( dynamicConfigSource, manualSource, initialSettings, _tracerSettings, - telemetry, + tempTelemetry, overrideErrorLog); // TODO: We'll later report these // The only exporter setting we currently _allow_ to change is the AgentUri, but if that does change, @@ -159,7 +198,7 @@ private bool UpdateSettings( var newRawExporterSettings = ExporterSettings.Raw.CreateUpdatedFromManualConfig( currentExporter.RawSettings, manualSource, - telemetry, + tempTelemetry, manualSource.UseDefaultSources); var isSameMutableSettings = currentMutable.Equals(newMutableSettings); @@ -171,6 +210,11 @@ private bool UpdateSettings( return null; } + // we have changes, so we need to report them + // First record the "default"/fallback values, then record the "new" values + defaultTelemetry.CopyTo(telemetry); + tempTelemetry.CopyTo(telemetry); + Log.Information("Notifying consumers of new settings"); var updatedMutableSettings = isSameMutableSettings ? null : newMutableSettings; var updatedExporterSettings = isSameExporterSettings ? null : new ExporterSettings(newRawExporterSettings, telemetry); @@ -178,6 +222,21 @@ private bool UpdateSettings( return new SettingChanges(updatedMutableSettings, updatedExporterSettings, currentMutable, currentExporter); } + [MemberNotNull(nameof(_noDefaultSettingsTelemetry))] + [MemberNotNull(nameof(_noDefaultSourcesSettings))] + private void InitialiseNoDefaultSourceSettings() + { + if (_noDefaultSourcesSettings is not null + && _noDefaultSettingsTelemetry is not null) + { + return; + } + + var telemetry = new ConfigurationTelemetry(); + _noDefaultSettingsTelemetry = telemetry; + _noDefaultSourcesSettings = MutableSettings.CreateWithoutDefaultSources(_tracerSettings, telemetry); + } + private void NotifySubscribers(SettingChanges settings) { _latest = settings; diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 6fee833ca806..c16f8e1b1878 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -125,8 +125,6 @@ internal TracerSettings(IConfigurationSource? source, IConfigurationTelemetry te .AsBoolResult() .OverrideWith(in otelActivityListenerEnabled, ErrorLog, defaultValue: false); - var exporter = new ExporterSettings(source, _telemetry); - PeerServiceTagsEnabled = config .WithKeys(ConfigurationKeys.PeerServiceDefaultsEnabled) .AsBool(defaultValue: false); @@ -335,66 +333,6 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = OpenTelemetryLogsEnabled = OpenTelemetryLogsEnabled && OtelLogsExporterEnabled; - DataPipelineEnabled = config - .WithKeys(ConfigurationKeys.TraceDataPipelineEnabled) - .AsBool(defaultValue: EnvironmentHelpers.IsUsingAzureAppServicesSiteExtension() && !EnvironmentHelpers.IsAzureFunctions()); - - if (DataPipelineEnabled) - { - // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter - // as it may lead to different stats results than the managed one. - if (StatsComputationEnabled) - { - DataPipelineEnabled = false; - Log.Warning( - $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but {ConfigurationKeys.StatsComputationEnabled} is enabled. Disabling data pipeline."); - _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); - } - - // Windows supports UnixDomainSocket https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ - // but tokio hasn't added support for it yet https://github.com/tokio-rs/tokio/issues/2201 - // There's an issue here, in that technically a user can initially be configured to send over TCP/named pipes, - // and so we allow and enable the datapipeline. Later, they could configure the app in code to send over UDS. - // This is a problem, as we currently don't support toggling the data pipeline at runtime, so we explicitly block - // this scenario in the public API. - if (exporter.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) - { - DataPipelineEnabled = false; - Log.Warning( - $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but TracesTransport is set to UnixDomainSocket which is not supported on Windows. Disabling data pipeline."); - _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); - } - - if (!isLibDatadogAvailable.IsAvailable) - { - DataPipelineEnabled = false; - if (isLibDatadogAvailable.Exception is not null) - { - Log.Warning( - isLibDatadogAvailable.Exception, - $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but libdatadog is not available. Disabling data pipeline."); - } - else - { - Log.Warning( - $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but libdatadog is not available. Disabling data pipeline."); - } - - _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); - } - - // SSI already utilizes libdatadog. To prevent unexpected behavior, - // we proactively disable the data pipeline when SSI is enabled. Theoretically, this should not cause any issues, - // but as a precaution, we are taking a conservative approach during the initial rollout phase. - if (!string.IsNullOrEmpty(EnvironmentHelpers.GetEnvironmentVariable("DD_INJECTION_ENABLED"))) - { - DataPipelineEnabled = false; - Log.Warning( - $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but SSI is enabled. Disabling data pipeline."); - _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); - } - } - // We should also be writing telemetry for OTEL_LOGS_EXPORTER similar to OTEL_METRICS_EXPORTER, but we don't have a corresponding Datadog config // When we do, we can insert that here CustomSamplingRulesFormat = config.WithKeys(ConfigurationKeys.CustomSamplingRulesFormat) @@ -731,9 +669,70 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = // We create a lazy here because this is kind of expensive, and we want to avoid calling it if we can _fallbackApplicationName = new(() => ApplicationNameHelpers.GetFallbackApplicationName(this)); - // Move the creation of these settings inside SettingsManager? - var initialMutableSettings = MutableSettings.CreateInitialMutableSettings(source, telemetry, errorLog, this); - Manager = new(this, initialMutableSettings, exporter); + // There's a circular dependency here because DataPipeline depends on ExporterSettings, + // but the settings manager depends on TracerSettings. Basically this is all fine as long + // as nothing in the MutableSettings or ExporterSettings depends on the value of DataPipelineEnabled! + Manager = new(source, this, telemetry, errorLog); + + DataPipelineEnabled = config + .WithKeys(ConfigurationKeys.TraceDataPipelineEnabled) + .AsBool(defaultValue: EnvironmentHelpers.IsUsingAzureAppServicesSiteExtension() && !EnvironmentHelpers.IsAzureFunctions()); + + if (DataPipelineEnabled) + { + // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter + // as it may lead to different stats results than the managed one. + if (StatsComputationEnabled) + { + DataPipelineEnabled = false; + Log.Warning( + $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but {ConfigurationKeys.StatsComputationEnabled} is enabled. Disabling data pipeline."); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + + // Windows supports UnixDomainSocket https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ + // but tokio hasn't added support for it yet https://github.com/tokio-rs/tokio/issues/2201 + // There's an issue here, in that technically a user can initially be configured to send over TCP/named pipes, + // and so we allow and enable the datapipeline. Later, they could configure the app in code to send over UDS. + // This is a problem, as we currently don't support toggling the data pipeline at runtime, so we explicitly block + // this scenario in the public API. + if (Manager.InitialExporterSettings.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) + { + DataPipelineEnabled = false; + Log.Warning( + $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but TracesTransport is set to UnixDomainSocket which is not supported on Windows. Disabling data pipeline."); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + + if (!isLibDatadogAvailable.IsAvailable) + { + DataPipelineEnabled = false; + if (isLibDatadogAvailable.Exception is not null) + { + Log.Warning( + isLibDatadogAvailable.Exception, + $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but libdatadog is not available. Disabling data pipeline."); + } + else + { + Log.Warning( + $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but libdatadog is not available. Disabling data pipeline."); + } + + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + + // SSI already utilizes libdatadog. To prevent unexpected behavior, + // we proactively disable the data pipeline when SSI is enabled. Theoretically, this should not cause any issues, + // but as a precaution, we are taking a conservative approach during the initial rollout phase. + if (!string.IsNullOrEmpty(EnvironmentHelpers.GetEnvironmentVariable("DD_INJECTION_ENABLED"))) + { + DataPipelineEnabled = false; + Log.Warning( + $"{ConfigurationKeys.TraceDataPipelineEnabled} is enabled, but SSI is enabled. Disabling data pipeline."); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + } } internal bool IsRunningInCiVisibility { get; } diff --git a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApi.cs b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApi.cs index ae46547bad31..ac0875bb5427 100644 --- a/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApi.cs +++ b/tracer/src/Datadog.Trace/RemoteConfigurationManagement/Transport/RemoteConfigurationApi.cs @@ -6,6 +6,7 @@ #nullable enable using System; +using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -15,6 +16,7 @@ using Datadog.Trace.Logging; using Datadog.Trace.PlatformHelpers; using Datadog.Trace.RemoteConfigurationManagement.Protocol; +using Datadog.Trace.Util.Streams; using Datadog.Trace.Vendors.Newtonsoft.Json; namespace Datadog.Trace.RemoteConfigurationManagement.Transport diff --git a/tracer/test/Datadog.Trace.Security.Unit.Tests/EmptyDatadogTracer.cs b/tracer/test/Datadog.Trace.Security.Unit.Tests/EmptyDatadogTracer.cs index c43533eeb3a8..30ba0ace8710 100644 --- a/tracer/test/Datadog.Trace.Security.Unit.Tests/EmptyDatadogTracer.cs +++ b/tracer/test/Datadog.Trace.Security.Unit.Tests/EmptyDatadogTracer.cs @@ -6,6 +6,7 @@ using System; using Datadog.Trace.Configuration; using Datadog.Trace.Configuration.Schema; +using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.Sampling; using Moq; @@ -23,7 +24,7 @@ public EmptyDatadogTracer() DefaultServiceName = "My Service Name"; Settings = new TracerSettings(NullConfigurationSource.Instance); var namingSchema = new NamingSchema(SchemaVersion.V0, false, false, DefaultServiceName, null, null); - PerTraceSettings = new PerTraceSettings(null, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings)); + PerTraceSettings = new PerTraceSettings(null, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings, new ConfigurationTelemetry())); } public string DefaultServiceName { get; } diff --git a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs index 7eeaa76f8c52..8eb464810838 100644 --- a/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs @@ -11,6 +11,7 @@ using Datadog.Trace.Agent; using Datadog.Trace.Agent.MessagePack; using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.DogStatsd; using Datadog.Trace.Sampling; using Datadog.Trace.TestHelpers; @@ -427,7 +428,7 @@ public async Task AddsTraceKeepRateMetricToRootSpan() var tracer = new Mock(); tracer.Setup(x => x.DefaultServiceName).Returns("Default"); - tracer.Setup(x => x.PerTraceSettings).Returns(new PerTraceSettings(null, null, null!, MutableSettings.CreateWithoutDefaultSources(new(NullConfigurationSource.Instance)))); + tracer.Setup(x => x.PerTraceSettings).Returns(new PerTraceSettings(null, null, null!, MutableSettings.CreateWithoutDefaultSources(new(NullConfigurationSource.Instance), new ConfigurationTelemetry()))); var traceContext = new TraceContext(tracer.Object); var rootSpanContext = new SpanContext(null, traceContext, null); var rootSpan = new Span(rootSpanContext, DateTimeOffset.UtcNow); diff --git a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs index bdf52cd7bd0e..1cfb0f2377d3 100644 --- a/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Configuration/TracerSettingsSettingManagerTests.cs @@ -16,4 +16,80 @@ namespace Datadog.Trace.Tests.Configuration; public class TracerSettingsSettingManagerTests { + [Fact] + public void UpdateSettings_HandlesDynamicConfigurationChanges() + { + var tracerSettings = new TracerSettings(); + SettingChanges settingChanges = null; + tracerSettings.Manager.SubscribeToChanges(changes => settingChanges = changes); + + // default is null, but it's recorded as "1.0" in telemetry + tracerSettings.Manager.InitialMutableSettings.GlobalSamplingRate.Should().Be(null); + var sampleRateConfig = GetLatestSampleRateTelemetry((ConfigurationTelemetry)tracerSettings.Telemetry); + + sampleRateConfig.Should().NotBeNull(); + sampleRateConfig.Origin.Should().Be(ConfigurationOrigins.Default); + sampleRateConfig.DoubleValue.Should().Be(1.0); + + var rawConfig1 = """{"action": "enable", "revision": 1698167126064, "service_target": {"service": "test_service", "env": "test_env"}, "lib_config": {"tracing_sampling_rate": 0.7, "log_injection_enabled": null, "tracing_header_tags": null, "runtime_metrics_enabled": null, "tracing_debug": null, "tracing_service_mapping": null, "tracing_sampling_rules": null, "data_streams_enabled": null, "dynamic_instrumentation_enabled": null, "exception_replay_enabled": null, "code_origin_enabled": null, "live_debugging_enabled": null}, "id": "-1796479631020605752"}"""; + var dynamicConfig = new DynamicConfigConfigurationSource(rawConfig1, ConfigurationOrigins.RemoteConfig); + var telemetry = new ConfigurationTelemetry(); + + var wasUpdated = tracerSettings.Manager.UpdateDynamicConfigurationSettings( + dynamicConfig, + centralTelemetry: telemetry); + + wasUpdated.Should().BeTrue(); + settingChanges.Should().NotBeNull(); + settingChanges!.UpdatedExporter.Should().BeNull(); + settingChanges!.UpdatedMutable.Should().NotBeNull(); + settingChanges!.UpdatedMutable.GlobalSamplingRate.Should().Be(0.7); + + sampleRateConfig = GetLatestSampleRateTelemetry(telemetry); + + sampleRateConfig.Should().NotBeNull(); + sampleRateConfig.Origin.Should().Be(ConfigurationOrigins.RemoteConfig); + sampleRateConfig.DoubleValue.Should().Be(0.7); + telemetry.Clear(); + + // reset to "default" + var rawConfig2 = """{"action": "enable", "revision": 1698167126064, "service_target": {"service": "test_service", "env": "test_env"}, "lib_config": {"tracing_sampling_rate": null, "log_injection_enabled": null, "tracing_header_tags": null, "runtime_metrics_enabled": null, "tracing_debug": null, "tracing_service_mapping": null, "tracing_sampling_rules": null, "data_streams_enabled": null, "dynamic_instrumentation_enabled": null, "exception_replay_enabled": null, "code_origin_enabled": null, "live_debugging_enabled": null}, "id": "5931732111467439992"}"""; + dynamicConfig = new DynamicConfigConfigurationSource(rawConfig2, ConfigurationOrigins.RemoteConfig); + + wasUpdated = tracerSettings.Manager.UpdateDynamicConfigurationSettings( + dynamicConfig, + centralTelemetry: telemetry); + + wasUpdated.Should().BeTrue(); + settingChanges.Should().NotBeNull(); + settingChanges!.UpdatedExporter.Should().BeNull(); + settingChanges!.UpdatedMutable.Should().NotBeNull(); + settingChanges!.UpdatedMutable.GlobalSamplingRate.Should().Be(null); + + sampleRateConfig = GetLatestSampleRateTelemetry(telemetry); + + sampleRateConfig.Should().NotBeNull(); + sampleRateConfig.Origin.Should().Be(ConfigurationOrigins.Default); + sampleRateConfig.DoubleValue.Should().Be(1.0); + telemetry.Clear(); + + // Send the same config again, and make sure we _don't_ record more telemetry, so what we already have is correct + var existingChanges = settingChanges; + wasUpdated = tracerSettings.Manager.UpdateDynamicConfigurationSettings( + dynamicConfig, + centralTelemetry: telemetry); + + wasUpdated.Should().BeFalse(); + existingChanges.Should().Be(settingChanges); + telemetry.GetQueueForTesting().Should().BeEmpty(); + } + + private static ConfigurationTelemetry.ConfigurationTelemetryEntry GetLatestSampleRateTelemetry(ConfigurationTelemetry telemetry) + { + return telemetry + .GetQueueForTesting() + .Where(x => x.Key == ConfigurationKeys.GlobalSamplingRate) + .OrderByDescending(x => x.SeqId) + .FirstOrDefault(); + } } diff --git a/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs b/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs index 595985f4c427..ed889f080d90 100644 --- a/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs +++ b/tracer/test/Datadog.Trace.Tests/Util/StubDatadogTracer.cs @@ -6,6 +6,7 @@ using System; using Datadog.Trace.Configuration; using Datadog.Trace.Configuration.Schema; +using Datadog.Trace.Configuration.Telemetry; namespace Datadog.Trace.Tests.Util; @@ -16,7 +17,7 @@ public StubDatadogTracer() DefaultServiceName = "stub-service"; Settings = new TracerSettings(NullConfigurationSource.Instance); var namingSchema = new NamingSchema(SchemaVersion.V0, false, false, DefaultServiceName, null, null); - PerTraceSettings = new PerTraceSettings(null, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings)); + PerTraceSettings = new PerTraceSettings(null, null, namingSchema, MutableSettings.CreateWithoutDefaultSources(Settings, new ConfigurationTelemetry())); } public string DefaultServiceName { get; }