From bc8caf7ac6850ddf814343ee8ce8340360c3a15e Mon Sep 17 00:00:00 2001 From: Alexander Dinauer Date: Thu, 8 May 2025 12:28:51 +0200 Subject: [PATCH] Add batch processor for logs --- .../core/SessionTrackingIntegrationTest.kt | 5 + sentry/api/sentry.api | 22 ++++ .../main/java/io/sentry/ISentryClient.java | 3 + .../main/java/io/sentry/NoOpSentryClient.java | 6 ++ .../src/main/java/io/sentry/SentryClient.java | 53 +++++---- .../java/io/sentry/SentryEnvelopeItem.java | 2 +- .../main/java/io/sentry/SentryLogEvents.java | 4 + .../sentry/logger/ILoggerBatchProcessor.java | 10 ++ .../main/java/io/sentry/logger/LoggerApi.java | 4 +- .../sentry/logger/LoggerBatchProcessor.java | 101 ++++++++++++++++++ .../logger/NoOpLoggerBatchProcessor.java | 27 +++++ 11 files changed, 213 insertions(+), 24 deletions(-) create mode 100644 sentry/src/main/java/io/sentry/logger/ILoggerBatchProcessor.java create mode 100644 sentry/src/main/java/io/sentry/logger/LoggerBatchProcessor.java create mode 100644 sentry/src/main/java/io/sentry/logger/NoOpLoggerBatchProcessor.java diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SessionTrackingIntegrationTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SessionTrackingIntegrationTest.kt index 5545bbc163..e20e79b324 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SessionTrackingIntegrationTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SessionTrackingIntegrationTest.kt @@ -17,6 +17,7 @@ import io.sentry.Sentry import io.sentry.SentryEnvelope import io.sentry.SentryEvent import io.sentry.SentryLogEvent +import io.sentry.SentryLogEvents import io.sentry.SentryOptions import io.sentry.SentryReplayEvent import io.sentry.Session @@ -190,6 +191,10 @@ class SessionTrackingIntegrationTest { TODO("Not yet implemented") } + override fun captureBatchedLogEvents(logEvents: SentryLogEvents) { + TODO("Not yet implemented") + } + override fun getRateLimiter(): RateLimiter? { TODO("Not yet implemented") } diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index a976b404bf..8f6c5ea566 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -987,6 +987,7 @@ public abstract interface class io/sentry/IScopesStorage { } public abstract interface class io/sentry/ISentryClient { + public abstract fun captureBatchedLogEvents (Lio/sentry/SentryLogEvents;)V public abstract fun captureCheckIn (Lio/sentry/CheckIn;Lio/sentry/IScope;Lio/sentry/Hint;)Lio/sentry/protocol/SentryId; public fun captureEnvelope (Lio/sentry/SentryEnvelope;)Lio/sentry/protocol/SentryId; public abstract fun captureEnvelope (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Lio/sentry/protocol/SentryId; @@ -2718,6 +2719,7 @@ public final class io/sentry/SentryBaseEvent$Serializer { public final class io/sentry/SentryClient : io/sentry/ISentryClient { public fun (Lio/sentry/SentryOptions;)V + public fun captureBatchedLogEvents (Lio/sentry/SentryLogEvents;)V public fun captureCheckIn (Lio/sentry/CheckIn;Lio/sentry/IScope;Lio/sentry/Hint;)Lio/sentry/protocol/SentryId; public fun captureEnvelope (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Lio/sentry/protocol/SentryId; public fun captureEvent (Lio/sentry/SentryEvent;Lio/sentry/IScope;Lio/sentry/Hint;)Lio/sentry/protocol/SentryId; @@ -3071,6 +3073,7 @@ public final class io/sentry/SentryLogEventAttributeValue$JsonKeys { public final class io/sentry/SentryLogEvents : io/sentry/JsonSerializable, io/sentry/JsonUnknown { public fun (Ljava/util/List;)V + public fun getItems ()Ljava/util/List; public fun getUnknown ()Ljava/util/Map; public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setUnknown (Ljava/util/Map;)V @@ -4666,6 +4669,11 @@ public abstract interface class io/sentry/logger/ILoggerApi { public abstract fun warn (Ljava/lang/String;[Ljava/lang/Object;)V } +public abstract interface class io/sentry/logger/ILoggerBatchProcessor { + public abstract fun add (Lio/sentry/SentryLogEvent;)V + public abstract fun close (Z)V +} + public final class io/sentry/logger/LoggerApi : io/sentry/logger/ILoggerApi { public fun (Lio/sentry/Scopes;)V public fun debug (Ljava/lang/String;[Ljava/lang/Object;)V @@ -4678,6 +4686,14 @@ public final class io/sentry/logger/LoggerApi : io/sentry/logger/ILoggerApi { public fun warn (Ljava/lang/String;[Ljava/lang/Object;)V } +public final class io/sentry/logger/LoggerBatchProcessor : io/sentry/logger/ILoggerBatchProcessor { + public static final field FLUSH_AFTER_MS I + public static final field MAX_BATCH_SIZE I + public fun (Lio/sentry/SentryOptions;Lio/sentry/ISentryClient;)V + public fun add (Lio/sentry/SentryLogEvent;)V + public fun close (Z)V +} + public final class io/sentry/logger/NoOpLoggerApi : io/sentry/logger/ILoggerApi { public fun debug (Ljava/lang/String;[Ljava/lang/Object;)V public fun error (Ljava/lang/String;[Ljava/lang/Object;)V @@ -4690,6 +4706,12 @@ public final class io/sentry/logger/NoOpLoggerApi : io/sentry/logger/ILoggerApi public fun warn (Ljava/lang/String;[Ljava/lang/Object;)V } +public final class io/sentry/logger/NoOpLoggerBatchProcessor : io/sentry/logger/ILoggerBatchProcessor { + public fun add (Lio/sentry/SentryLogEvent;)V + public fun close (Z)V + public static fun getInstance ()Lio/sentry/logger/NoOpLoggerBatchProcessor; +} + public final class io/sentry/opentelemetry/OpenTelemetryUtil { public fun ()V public static fun applyIgnoredSpanOrigins (Lio/sentry/SentryOptions;)V diff --git a/sentry/src/main/java/io/sentry/ISentryClient.java b/sentry/src/main/java/io/sentry/ISentryClient.java index 36d154d9cd..c770be0b05 100644 --- a/sentry/src/main/java/io/sentry/ISentryClient.java +++ b/sentry/src/main/java/io/sentry/ISentryClient.java @@ -295,6 +295,9 @@ SentryId captureProfileChunk( @ApiStatus.Experimental void captureLog(@NotNull SentryLogEvent logEvent, @Nullable IScope scope, @Nullable Hint hint); + @ApiStatus.Experimental + void captureBatchedLogEvents(@NotNull SentryLogEvents logEvents); + @ApiStatus.Internal @Nullable RateLimiter getRateLimiter(); diff --git a/sentry/src/main/java/io/sentry/NoOpSentryClient.java b/sentry/src/main/java/io/sentry/NoOpSentryClient.java index 9a616132ab..1a1b378a43 100644 --- a/sentry/src/main/java/io/sentry/NoOpSentryClient.java +++ b/sentry/src/main/java/io/sentry/NoOpSentryClient.java @@ -84,6 +84,12 @@ public void captureLog( // do nothing } + @ApiStatus.Experimental + @Override + public void captureBatchedLogEvents(@NotNull SentryLogEvents logEvents) { + // do nothing + } + @Override public @Nullable RateLimiter getRateLimiter() { return null; diff --git a/sentry/src/main/java/io/sentry/SentryClient.java b/sentry/src/main/java/io/sentry/SentryClient.java index d7ef9562f8..f6dc924925 100644 --- a/sentry/src/main/java/io/sentry/SentryClient.java +++ b/sentry/src/main/java/io/sentry/SentryClient.java @@ -6,6 +6,9 @@ import io.sentry.hints.Backfillable; import io.sentry.hints.DiskFlushNotification; import io.sentry.hints.TransactionEnd; +import io.sentry.logger.ILoggerBatchProcessor; +import io.sentry.logger.LoggerBatchProcessor; +import io.sentry.logger.NoOpLoggerBatchProcessor; import io.sentry.protocol.Contexts; import io.sentry.protocol.DebugMeta; import io.sentry.protocol.SentryId; @@ -16,7 +19,6 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -36,6 +38,7 @@ public final class SentryClient implements ISentryClient { private final @NotNull SentryOptions options; private final @NotNull ITransport transport; private final @NotNull SortBreadcrumbsByDate sortBreadcrumbsByDate = new SortBreadcrumbsByDate(); + private final @NotNull ILoggerBatchProcessor loggerBatchProcessor; @Override public boolean isEnabled() { @@ -55,6 +58,11 @@ public SentryClient(final @NotNull SentryOptions options) { final RequestDetailsResolver requestDetailsResolver = new RequestDetailsResolver(options); transport = transportFactory.create(options, requestDetailsResolver.resolve()); + if (options.getExperimental().getLogs().isEnabled()) { + loggerBatchProcessor = new LoggerBatchProcessor(options, this); + } else { + loggerBatchProcessor = NoOpLoggerBatchProcessor.getInstance(); + } } private boolean shouldApplyScopeData( @@ -625,8 +633,7 @@ public void captureUserFeedback(final @NotNull UserFeedback userFeedback) { return new SentryEnvelope(envelopeHeader, envelopeItems); } - private @NotNull SentryEnvelope buildEnvelope( - final @NotNull SentryLogEvents logEvents, final @Nullable TraceContext traceContext) { + private @NotNull SentryEnvelope buildEnvelope(final @NotNull SentryLogEvents logEvents) { final List envelopeItems = new ArrayList<>(); final SentryEnvelopeItem logItem = @@ -634,7 +641,7 @@ public void captureUserFeedback(final @NotNull UserFeedback userFeedback) { envelopeItems.add(logItem); final SentryEnvelopeHeader envelopeHeader = - new SentryEnvelopeHeader(null, options.getSdkVersion(), traceContext); + new SentryEnvelopeHeader(null, options.getSdkVersion(), null); return new SentryEnvelope(envelopeHeader, envelopeItems); } @@ -1018,17 +1025,17 @@ public void captureLog( hint = new Hint(); } - @Nullable TraceContext traceContext = null; - if (scope != null) { - final @Nullable ITransaction transaction = scope.getTransaction(); - if (transaction != null) { - traceContext = transaction.traceContext(); - } else { - final @NotNull PropagationContext propagationContext = - TracingUtils.maybeUpdateBaggage(scope, options); - traceContext = propagationContext.traceContext(); - } - } + // @Nullable TraceContext traceContext = null; + // if (scope != null) { + // final @Nullable ITransaction transaction = scope.getTransaction(); + // if (transaction != null) { + // traceContext = transaction.traceContext(); + // } else { + // final @NotNull PropagationContext propagationContext = + // TracingUtils.maybeUpdateBaggage(scope, options); + // traceContext = propagationContext.traceContext(); + // } + // } if (logEvent != null) { logEvent = executeBeforeSendLog(logEvent, hint); @@ -1040,15 +1047,18 @@ public void captureLog( .recordLostEvent(DiscardReason.BEFORE_SEND, DataCategory.LogItem); return; } + + loggerBatchProcessor.add(logEvent); } - try { - final @NotNull SentryEnvelope envelope = - buildEnvelope(new SentryLogEvents(Arrays.asList(logEvent)), traceContext); + hint.clear(); + } - hint.clear(); - // TODO buffer - sendEnvelope(envelope, hint); + @Override + public void captureBatchedLogEvents(final @NotNull SentryLogEvents logEvents) { + try { + final @NotNull SentryEnvelope envelope = buildEnvelope(logEvents); + sendEnvelope(envelope, null); } catch (IOException e) { options.getLogger().log(SentryLevel.WARNING, e, "Capturing log failed."); } @@ -1307,6 +1317,7 @@ public void close(final boolean isRestarting) { options.getLogger().log(SentryLevel.INFO, "Closing SentryClient."); try { flush(isRestarting ? 0 : options.getShutdownTimeoutMillis()); + loggerBatchProcessor.close(isRestarting); transport.close(isRestarting); } catch (IOException e) { options diff --git a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java index 8c2ece51cc..d25a74c802 100644 --- a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java +++ b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java @@ -503,7 +503,7 @@ public static SentryEnvelopeItem fromLogs( null, null, null, - 1); + logEvents.getItems().size()); // avoid method refs on Android due to some issues with older AGP setups // noinspection Convert2MethodRef diff --git a/sentry/src/main/java/io/sentry/SentryLogEvents.java b/sentry/src/main/java/io/sentry/SentryLogEvents.java index 06a1f04ed3..432ba092f7 100644 --- a/sentry/src/main/java/io/sentry/SentryLogEvents.java +++ b/sentry/src/main/java/io/sentry/SentryLogEvents.java @@ -17,6 +17,10 @@ public SentryLogEvents(final @NotNull List items) { this.items = items; } + public @NotNull List getItems() { + return items; + } + // region json public static final class JsonKeys { public static final String ITEMS = "items"; diff --git a/sentry/src/main/java/io/sentry/logger/ILoggerBatchProcessor.java b/sentry/src/main/java/io/sentry/logger/ILoggerBatchProcessor.java new file mode 100644 index 0000000000..9dbc7c883e --- /dev/null +++ b/sentry/src/main/java/io/sentry/logger/ILoggerBatchProcessor.java @@ -0,0 +1,10 @@ +package io.sentry.logger; + +import io.sentry.SentryLogEvent; +import org.jetbrains.annotations.NotNull; + +public interface ILoggerBatchProcessor { + void add(@NotNull SentryLogEvent event); + + void close(boolean isRestarting); +} diff --git a/sentry/src/main/java/io/sentry/logger/LoggerApi.java b/sentry/src/main/java/io/sentry/logger/LoggerApi.java index b1a9eee562..20226f46b7 100644 --- a/sentry/src/main/java/io/sentry/logger/LoggerApi.java +++ b/sentry/src/main/java/io/sentry/logger/LoggerApi.java @@ -32,8 +32,8 @@ public LoggerApi(final @NotNull Scopes scopes) { @Override public void trace(final @Nullable String message, final @Nullable Object... args) { - // TODO SentryLevel.TRACE does not exists yet - // log(SentryLevel.TRACE, message, args); + // TODO SentryLevel.TRACE does not exists yet so we just report it as DEBUG for now + log(SentryLevel.DEBUG, message, args); } @Override diff --git a/sentry/src/main/java/io/sentry/logger/LoggerBatchProcessor.java b/sentry/src/main/java/io/sentry/logger/LoggerBatchProcessor.java new file mode 100644 index 0000000000..5f2ee9f11e --- /dev/null +++ b/sentry/src/main/java/io/sentry/logger/LoggerBatchProcessor.java @@ -0,0 +1,101 @@ +package io.sentry.logger; + +import io.sentry.ISentryClient; +import io.sentry.ISentryLifecycleToken; +import io.sentry.SentryLogEvent; +import io.sentry.SentryLogEvents; +import io.sentry.SentryOptions; +import io.sentry.util.AutoClosableReentrantLock; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public final class LoggerBatchProcessor implements ILoggerBatchProcessor { + + public static final int FLUSH_AFTER_MS = 5000; + public static final int MAX_BATCH_SIZE = 100; + + private final @NotNull SentryOptions options; + private final @NotNull ISentryClient client; + private final @NotNull Queue queue; + private volatile @Nullable Future scheduledFlush; + private static final @NotNull AutoClosableReentrantLock scheduleLock = + new AutoClosableReentrantLock(); + + public LoggerBatchProcessor( + final @NotNull SentryOptions options, final @NotNull ISentryClient client) { + this.options = options; + this.client = client; + this.queue = new ConcurrentLinkedQueue<>(); + } + + @Override + public void add(final @NotNull SentryLogEvent logEvent) { + queue.offer(logEvent); + maybeSchedule(false, false); + } + + @Override + public void close(final boolean isRestarting) { + if (isRestarting) { + maybeSchedule(true, true); + } else { + while (!queue.isEmpty()) { + flushBatch(); + } + } + } + + private void maybeSchedule(boolean forceSchedule, boolean immediately) { + try (final @NotNull ISentryLifecycleToken ignored = scheduleLock.acquire()) { + final @Nullable Future latestScheduledFlush = scheduledFlush; + if (forceSchedule + || latestScheduledFlush == null + || latestScheduledFlush.isDone() + || latestScheduledFlush.isCancelled()) { + final int flushAfterMs = immediately ? 0 : FLUSH_AFTER_MS; + scheduledFlush = options.getExecutorService().schedule(new BatchRunnable(), flushAfterMs); + } + } + } + + private void flush() { + flushInternal(); + try (final @NotNull ISentryLifecycleToken ignored = scheduleLock.acquire()) { + if (!queue.isEmpty()) { + maybeSchedule(true, false); + } + } + } + + private void flushInternal() { + flushBatch(); + if (queue.size() >= MAX_BATCH_SIZE) { + flushInternal(); + } + } + + private void flushBatch() { + final @NotNull List logEvents = new ArrayList<>(MAX_BATCH_SIZE); + do { + final @Nullable SentryLogEvent logEvent = queue.poll(); + if (logEvent != null) { + logEvents.add(logEvent); + } + } while (!queue.isEmpty() && logEvents.size() < MAX_BATCH_SIZE); + + client.captureBatchedLogEvents(new SentryLogEvents(logEvents)); + } + + private class BatchRunnable implements Runnable { + + @Override + public void run() { + flush(); + } + } +} diff --git a/sentry/src/main/java/io/sentry/logger/NoOpLoggerBatchProcessor.java b/sentry/src/main/java/io/sentry/logger/NoOpLoggerBatchProcessor.java new file mode 100644 index 0000000000..9568304444 --- /dev/null +++ b/sentry/src/main/java/io/sentry/logger/NoOpLoggerBatchProcessor.java @@ -0,0 +1,27 @@ +package io.sentry.logger; + +import io.sentry.SentryLogEvent; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +@ApiStatus.Experimental +public final class NoOpLoggerBatchProcessor implements ILoggerBatchProcessor { + + private static final NoOpLoggerBatchProcessor instance = new NoOpLoggerBatchProcessor(); + + private NoOpLoggerBatchProcessor() {} + + public static NoOpLoggerBatchProcessor getInstance() { + return instance; + } + + @Override + public void add(@NotNull SentryLogEvent event) { + // do nothing + } + + @Override + public void close(final boolean isRestarting) { + // do nothing + } +}