Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logging sampling #5574

Merged
merged 67 commits into from
Feb 27, 2025
Merged
Changes from 1 commit
Commits
Show all changes
67 commits
Select commit Hold shift + click to select a range
1280d1b
initial proposal
Jun 5, 2024
f8b502c
update
Jul 3, 2024
ed7e0db
rebase
evgenyfedorov2 Sep 12, 2024
e15139d
use .net 9
evgenyfedorov2 Sep 30, 2024
12fd4b4
bufferin - initial
evgenyfedorov2 Oct 7, 2024
9fa8403
Remove junk
evgenyfedorov2 Oct 7, 2024
2ff5f78
buffer - renames
evgenyfedorov2 Oct 9, 2024
0cf2c50
.
evgenyfedorov2 Oct 9, 2024
6f8265e
.
evgenyfedorov2 Oct 9, 2024
cdb2c82
Sampling WIP with Global and HttpRequest samplers
evgenyfedorov2 Oct 14, 2024
cb1de9d
sampling
evgenyfedorov2 Oct 14, 2024
c4596a3
cosmetic
evgenyfedorov2 Oct 14, 2024
5e92d14
Prepare to update API proposal
evgenyfedorov2 Oct 15, 2024
8f760b1
.
evgenyfedorov2 Oct 15, 2024
57f902b
state at 23_10_2024 after updating Github proposal
evgenyfedorov2 Oct 23, 2024
610647c
Return Global and Http Request buffering options
evgenyfedorov2 Oct 24, 2024
3be3850
polish sampling
evgenyfedorov2 Oct 24, 2024
8ce40c1
add alwaysOnSampler
evgenyfedorov2 Oct 24, 2024
d288300
update namespaces
evgenyfedorov2 Oct 24, 2024
e120145
abstractions test
evgenyfedorov2 Oct 24, 2024
b443094
add some tests
evgenyfedorov2 Oct 25, 2024
b41fcdf
remove buffering
evgenyfedorov2 Oct 25, 2024
a5e1440
merge
evgenyfedorov2 Oct 25, 2024
cbd2a15
update tests
evgenyfedorov2 Oct 25, 2024
5580cff
remove sample app
evgenyfedorov2 Oct 25, 2024
07a2860
cosmetic changes
evgenyfedorov2 Oct 25, 2024
9ac7cab
update tests
evgenyfedorov2 Oct 25, 2024
0df498f
validate supplied probability
evgenyfedorov2 Oct 25, 2024
7b9479d
Fix warnings
evgenyfedorov2 Oct 25, 2024
4007ce1
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Oct 26, 2024
473bfca
more tests
evgenyfedorov2 Oct 26, 2024
b140e68
Address PR comments
evgenyfedorov2 Oct 30, 2024
32b5adc
Address PR comments
evgenyfedorov2 Oct 31, 2024
ebd4795
wip
evgenyfedorov2 Nov 4, 2024
901bc22
add config support to Ratio based Sampler
evgenyfedorov2 Nov 6, 2024
e33863e
fix warnings
evgenyfedorov2 Nov 6, 2024
d43f891
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Nov 6, 2024
0adbfc7
PR comments
evgenyfedorov2 Nov 7, 2024
e938600
PR Comments
evgenyfedorov2 Nov 7, 2024
d49c1f1
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Nov 12, 2024
1606acb
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Dec 3, 2024
a831d20
PR Comments
evgenyfedorov2 Dec 3, 2024
9263b04
PR Comments
evgenyfedorov2 Dec 3, 2024
cf30f0e
Add more tests
evgenyfedorov2 Dec 3, 2024
57f7080
Add log message state to SamplingParameters
evgenyfedorov2 Dec 3, 2024
10ad269
More tests
evgenyfedorov2 Dec 3, 2024
a8bdc6e
Use LogEntry and remove func based sampler
evgenyfedorov2 Feb 12, 2025
08a40d2
Rename Probability to Probabilistic
evgenyfedorov2 Feb 12, 2025
997deb2
ProbabilisticSamplerOptions to use IList
evgenyfedorov2 Feb 12, 2025
8a8ddc1
Add event name
evgenyfedorov2 Feb 12, 2025
3a240fb
Add an overload with Action delegate
evgenyfedorov2 Feb 12, 2025
f8e0bc1
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Feb 12, 2025
26db353
Rename Category to CategoryName to better match log filtering API
evgenyfedorov2 Feb 14, 2025
3b320aa
Merge branch 'evgenyfedorov2/log_sampling' of https://github.com/evge…
evgenyfedorov2 Feb 14, 2025
49459b6
Update
evgenyfedorov2 Feb 14, 2025
cfd544b
Minor changes
evgenyfedorov2 Feb 21, 2025
017a89c
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Feb 21, 2025
654d22f
Rename Probabilistic to RandomProbabilistic
evgenyfedorov2 Feb 21, 2025
64a348e
more tests
evgenyfedorov2 Feb 21, 2025
cb4de8b
Add options validation
evgenyfedorov2 Feb 24, 2025
97c26cc
Minor optimization
evgenyfedorov2 Feb 24, 2025
eb3b660
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Feb 25, 2025
c05e86e
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Feb 26, 2025
0b1f973
Cosmetic update
evgenyfedorov2 Feb 26, 2025
8868c9b
Add rule caching
evgenyfedorov2 Feb 26, 2025
e7c7fe0
PR comments
evgenyfedorov2 Feb 27, 2025
1e678e5
Merge branch 'main' into evgenyfedorov2/log_sampling
evgenyfedorov2 Feb 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Return Global and Http Request buffering options
evgenyfedorov2 committed Oct 24, 2024
commit 610647c628e294dd166fd76e16ab37999e1551db
Original file line number Diff line number Diff line change
@@ -16,12 +16,12 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Buffering;

internal class GlobalBuffer : BackgroundService, ILoggingBuffer
{
private readonly BufferingOptions _options;
private readonly GlobalBufferingOptions _options;
private readonly ConcurrentDictionary<IBufferedLogger, ConcurrentQueue<GlobalBufferedLogRecord>> _buffers;
private readonly TimeProvider _timeProvider = TimeProvider.System;
private DateTimeOffset _lastFlushTimestamp;

public GlobalBuffer(IOptions<BufferingOptions> options)
public GlobalBuffer(IOptions<GlobalBufferingOptions> options)
{
_options = options.Value;
_lastFlushTimestamp = _timeProvider.GetUtcNow();
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ public static class GlobalBufferingLoggerBuilderExtensions
public static ILoggingBuilder AddGlobalBuffering(
this ILoggingBuilder builder,
Func<string?, EventId?, LogLevel?, bool> filter,
Action<BufferingOptions>? options = null)
Action<GlobalBufferingOptions>? options = null)
{
_ = Throw.IfNull(builder);

@@ -51,17 +51,17 @@ public static ILoggingBuilder AddGlobalBufferProvider(this ILoggingBuilder build
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder"/> to add the buffer to.</param>
/// <param name="filter">The filter to be used to decide what to buffer.</param>
/// <param name="configureOptions">The delegate to configure <see cref="BufferingOptions"/>.</param>
/// <param name="configureOptions">The delegate to configure <see cref="GlobalBufferingOptions"/>.</param>
/// <returns>The <see cref="ILoggingBuilder"/> so that additional calls can be chained.</returns>
public static ILoggingBuilder ConfigureBuffering(
this ILoggingBuilder builder,
Func<string?, EventId?, LogLevel?, bool> filter,
Action<BufferingOptions>? configureOptions = null)
Action<GlobalBufferingOptions>? configureOptions = null)
{
_ = Throw.IfNull(builder);

_ = builder.Services.Configure(configureOptions ?? new Action<BufferingOptions>((_) => { }));
_ = builder.Services.Configure<BufferingOptions>(opts => opts.AddFilter(filter));
_ = builder.Services.Configure(configureOptions ?? new Action<GlobalBufferingOptions>((_) => { }));
_ = builder.Services.Configure<GlobalBufferingOptions>(opts => opts.AddFilter(filter));

return builder;
}
@@ -70,7 +70,7 @@ public static ILoggingBuilder ConfigureBuffering(
/// Adds a log buffer to the factory.
/// </summary>
public static void AddFilter(
this BufferingOptions options,
this GlobalBufferingOptions options,
Func<string?, EventId?, LogLevel?, bool> filter)
{
_ = Throw.IfNull(options);
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Buffering;
/// <summary>
/// The options for LoggerBuffer.
/// </summary>
public class BufferingOptions
public class GlobalBufferingOptions
{
/// <summary>
/// Gets or sets the time to suspend the buffer after flushing.
Original file line number Diff line number Diff line change
@@ -44,12 +44,16 @@ public bool TryEnqueue(
// probably don't need to limit buffer capacity?
// becase buffer is disposed when the respective HttpContext is disposed
// don't expect it to grow so much to cause a problem?
if (queue.Count >= _options.PerRequestCapacity)
{
_ = queue.TryDequeue(out HttpRequestBufferedLogRecord? _);
}

// having said that, I question the usefullness of the HTTP buffering.
// If I have 1000 RPS each with a buffer which is auto-disposed,
// then something bad happens and 900 requests out of 1000 failed,
// their HttpContext were disposed, as well as buffers,
// so at this point logs are lost and it is too late to call the Flusth() method
// so at this point logs are lost and it is too late to call the Flush() method

queue.Enqueue(record);

Original file line number Diff line number Diff line change
@@ -23,13 +23,13 @@ public static class HttpRequestBufferingLoggerBuilderExtensions
public static ILoggingBuilder AddHttpRequestBuffering(
this ILoggingBuilder builder,
Func<string?, EventId?, LogLevel?, bool> filter,
Action<BufferingOptions>? options = null)
Action<HttpRequestBufferingOptions>? options = null)
{
_ = Throw.IfNull(builder);

return builder
.AddHttpRequestBufferProvider()
.ConfigureBuffering(filter, options);
.ConfigureHttpRequestBuffering(filter, options);
}

/// <summary>
@@ -45,4 +45,45 @@ public static ILoggingBuilder AddHttpRequestBufferProvider(this ILoggingBuilder

return builder;
}

/// <summary>
/// Adds a log buffer to the factory.
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder"/> to add the buffer to.</param>
/// <param name="filter">The filter to be used to decide what to buffer.</param>
/// <param name="configureOptions">The delegate to configure <see cref="HttpRequestBufferingOptions"/>.</param>
/// <returns>The <see cref="ILoggingBuilder"/> so that additional calls can be chained.</returns>
public static ILoggingBuilder ConfigureHttpRequestBuffering(
this ILoggingBuilder builder,
Func<string?, EventId?, LogLevel?, bool> filter,
Action<HttpRequestBufferingOptions>? configureOptions = null)
{
_ = Throw.IfNull(builder);

_ = builder.Services.Configure(configureOptions ?? new Action<HttpRequestBufferingOptions>((_) => { }));
_ = builder.Services.Configure<HttpRequestBufferingOptions>(opts => opts.AddHttpRequestBufferingFilter(filter));
_ = builder.Services
.AddOptions<GlobalBufferingOptions>()
.Configure<HttpRequestBufferingOptions>((globalOpts, requestOpts) =>
{
globalOpts.Capacity = requestOpts.GlobalCapacity;
globalOpts.Filter = requestOpts.Filter;
globalOpts.SuspendAfterFlushDuration = requestOpts.SuspendAfterFlushDuration;
});

return builder;
}

/// <summary>
/// Adds a log buffer to the factory.
/// </summary>
public static void AddHttpRequestBufferingFilter(
this HttpRequestBufferingOptions options,
Func<string?, EventId?, LogLevel?, bool> filter)
{
_ = Throw.IfNull(options);
_ = Throw.IfNull(filter);

options.Filter = filter;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using Microsoft.Extensions.Logging;

namespace Microsoft.Extensions.Diagnostics.Logging.Buffering;

/// <summary>
/// The options for LoggerBuffer.
/// </summary>
public class HttpRequestBufferingOptions
{
/// <summary>
/// Gets or sets the time to suspend the buffer after flushing.
/// </summary>
/// <remarks>
/// Use this to temporarily suspend buffering after a flush, e.g. in case of an incident you may want all logs to be emitted immediately,
/// so the buffering will be suspended for the <see paramref="SuspendAfterFlushDuration"/> time.
/// </remarks>
public TimeSpan SuspendAfterFlushDuration { get; set; } = TimeSpan.FromSeconds(30);

/// <summary>
/// Gets or sets the size of the buffer for a request.
/// </summary>
public int PerRequestCapacity { get; set; } = 1_000;

/// <summary>
/// Gets or sets the size of the buffer for a request.
/// </summary>
public int GlobalCapacity { get; set; } = 1_000_000;

/// <summary>
/// Gets or sets the filter delegate to determine what to buffer.
/// </summary>
internal Func<string?, EventId?, LogLevel?, bool> Filter { get; set; } = (_, _, _) => false;
}