Logs: Integrate in Sentry Client (#2920)

Author: denrase

CHANGELOG.md

@@ -5,6 +5,8 @@
 ### Features
 
 - Logs: Models & Envelopes ([#2916](https://github.com/getsentry/sentry-dart/pull/2916))
+- Logs: Integrate in Sentry Client ([#2920](https://github.com/getsentry/sentry-dart/pull/2920))
+- [Structured Logs]: Buffering and Flushing of Logs ([#2930](https://github.com/getsentry/sentry-dart/pull/2930))
 
 ## 9.0.0-beta.2
 

dart/lib/src/noop_log_batcher.dart

@@ -0,0 +1,12 @@
+import 'dart:async';
+
+import 'sentry_log_batcher.dart';
+import 'protocol/sentry_log.dart';
+
+class NoopLogBatcher implements SentryLogBatcher {
+  @override
+  FutureOr<void> addLog(SentryLog log) {}
+
+  @override
+  Future<void> flush() async {}
+}

dart/lib/src/noop_sentry_client.dart

@@ -66,4 +66,7 @@ class NoOpSentryClient implements SentryClient {
   Future<SentryId> captureFeedback(SentryFeedback feedback,
           {Scope? scope, Hint? hint}) async =>
       SentryId.empty();
+
+  @override
+  Future<void> captureLog(SentryLog log, {Scope? scope}) async {}
 }

dart/lib/src/protocol/sentry_log.dart

@@ -10,14 +10,16 @@ class SentryLog {
   Map<String, SentryLogAttribute> attributes;
   int? severityNumber;
 
+  /// The traceId is initially an empty default value and is populated during event processing;
+  /// by the time processing completes, it is guaranteed to be a valid non-empty trace id.
   SentryLog({
     required this.timestamp,
-    required this.traceId,
+    SentryId? traceId,
     required this.level,
     required this.body,
     required this.attributes,
     this.severityNumber,
-  });
+  }) : traceId = traceId ?? SentryId.empty();
 
   Map<String, dynamic> toJson() {
     return {

dart/lib/src/protocol/sentry_log_attribute.dart

@@ -8,19 +8,19 @@ class SentryLogAttribute {
     return SentryLogAttribute._(value, 'string');
   }
 
-  factory SentryLogAttribute.boolean(bool value) {
+  factory SentryLogAttribute.bool(bool value) {
     return SentryLogAttribute._(value, 'boolean');
   }
 
-  factory SentryLogAttribute.integer(int value) {
+  factory SentryLogAttribute.int(int value) {
     return SentryLogAttribute._(value, 'integer');
   }
 
   factory SentryLogAttribute.double(double value) {
     return SentryLogAttribute._(value, 'double');
   }
 
-  // In the future the SDK will also support string[], boolean[], integer[], double[] values.
+  // In the future the SDK will also support List<String>, List<bool>, List<int>, List<double> values.
   Map<String, dynamic> toJson() {
     return {
       'value': value,

dart/lib/src/sentry_client.dart

@@ -27,6 +27,7 @@ import 'type_check_hint.dart';
 import 'utils/isolate_utils.dart';
 import 'utils/regex_utils.dart';
 import 'utils/stacktrace_utils.dart';
+import 'sentry_log_batcher.dart';
 import 'version.dart';
 
 /// Default value for [SentryUser.ipAddress]. It gets set when an event does not have
@@ -75,6 +76,9 @@ class SentryClient {
     if (enableFlutterSpotlight) {
       options.transport = SpotlightHttpTransport(options, options.transport);
     }
+    if (options.enableLogs) {
+      options.logBatcher = SentryLogBatcher(options);
+    }
     return SentryClient._(options);
   }
 
@@ -485,6 +489,73 @@ class SentryClient {
     );
   }
 
+  @internal
+  Future<void> captureLog(
+    SentryLog log, {
+    Scope? scope,
+  }) async {
+    if (!_options.enableLogs) {
+      return;
+    }
+
+    log.attributes['sentry.sdk.name'] = SentryLogAttribute.string(
+      _options.sdk.name,
+    );
+    log.attributes['sentry.sdk.version'] = SentryLogAttribute.string(
+      _options.sdk.version,
+    );
+    final environment = _options.environment;
+    if (environment != null) {
+      log.attributes['sentry.environment'] = SentryLogAttribute.string(
+        environment,
+      );
+    }
+    final release = _options.release;
+    if (release != null) {
+      log.attributes['sentry.release'] = SentryLogAttribute.string(
+        release,
+      );
+    }
+
+    final propagationContext = scope?.propagationContext;
+    if (propagationContext != null) {
+      log.traceId = propagationContext.traceId;
+    }
+    final span = scope?.span;
+    if (span != null) {
+      log.attributes['sentry.trace.parent_span_id'] = SentryLogAttribute.string(
+        span.context.spanId.toString(),
+      );
+    }
+
+    final beforeSendLog = _options.beforeSendLog;
+    SentryLog? processedLog = log;
+    if (beforeSendLog != null) {
+      try {
+        final callbackResult = beforeSendLog(log);
+
+        if (callbackResult is Future<SentryLog?>) {
+          processedLog = await callbackResult;
+        } else {
+          processedLog = callbackResult;
+        }
+      } catch (exception, stackTrace) {
+        _options.logger(
+          SentryLevel.error,
+          'The beforeSendLog callback threw an exception',
+          exception: exception,
+          stackTrace: stackTrace,
+        );
+        if (_options.automatedTestMode) {
+          rethrow;
+        }
+      }
+    }
+    if (processedLog != null) {
+      _options.logBatcher.addLog(processedLog);
+    }
+  }
+
   void close() {
     _options.httpClient.close();
   }

dart/lib/src/sentry_log_batcher.dart

@@ -0,0 +1,52 @@
+import 'dart:async';
+import 'sentry_envelope.dart';
+import 'sentry_options.dart';
+import 'protocol/sentry_log.dart';
+import 'package:meta/meta.dart';
+
+@internal
+class SentryLogBatcher {
+  SentryLogBatcher(this._options, {Duration? flushTimeout, int? maxBufferSize})
+      : _flushTimeout = flushTimeout ?? Duration(seconds: 5),
+        _maxBufferSize = maxBufferSize ?? 100;
+
+  final SentryOptions _options;
+  final Duration _flushTimeout;
+  final int _maxBufferSize;
+
+  final _logBuffer = <SentryLog>[];
+
+  Timer? _flushTimer;
+
+  void addLog(SentryLog log) {
+    _logBuffer.add(log);
+
+    _flushTimer?.cancel();
+
+    if (_logBuffer.length >= _maxBufferSize) {
+      return flush();
+    } else {
+      _flushTimer = Timer(_flushTimeout, flush);
+    }
+  }
+
+  void flush() {
+    _flushTimer?.cancel();
+    _flushTimer = null;
+
+    final logs = List<SentryLog>.from(_logBuffer);
+    _logBuffer.clear();
+
+    if (logs.isEmpty) {
+      return;
+    }
+
+    final envelope = SentryEnvelope.fromLogs(
+      logs,
+      _options.sdk,
+    );
+
+    // TODO: Make sure the Android SDK understands the log envelope type.
+    _options.transport.send(envelope);
+  }
+}

dart/lib/src/sentry_options.dart

@@ -15,6 +15,8 @@ import 'sentry_exception_factory.dart';
 import 'sentry_stack_trace_factory.dart';
 import 'transport/noop_transport.dart';
 import 'version.dart';
+import 'sentry_log_batcher.dart';
+import 'noop_log_batcher.dart';
 
 // TODO: shutdownTimeout, flushTimeoutMillis
 // https://api.dart.dev/stable/2.10.2/dart-io/HttpClient/close.html doesn't have a timeout param, we'd need to implement manually
@@ -198,6 +200,10 @@ class SentryOptions {
   /// Can return true to emit the metric, or false to drop it.
   BeforeMetricCallback? beforeMetricCallback;
 
+  /// This function is called right before a log is about to be sent.
+  /// Can return a modified log or null to drop the log.
+  BeforeSendLogCallback? beforeSendLog;
+
   /// Sets the release. SDK will try to automatically configure a release out of the box
   /// See [docs for further information](https://docs.sentry.io/platforms/flutter/configuration/releases/)
   String? release;
@@ -531,6 +537,14 @@ class SentryOptions {
   /// This is opt-in, as it can lead to existing exception beeing grouped as new ones.
   bool groupExceptions = false;
 
+  /// Enable to capture and send logs to Sentry.
+  ///
+  /// Disabled by default.
+  bool enableLogs = false;
+
+  @internal
+  SentryLogBatcher logBatcher = NoopLogBatcher();
+
   SentryOptions({String? dsn, Platform? platform, RuntimeChecker? checker}) {
     this.dsn = dsn;
     if (platform != null) {
@@ -660,6 +674,10 @@ typedef BeforeMetricCallback = bool Function(
   Map<String, String>? tags,
 });
 
+/// This function is called right before a log is about to be sent.
+/// Can return a modified log or null to drop the log.
+typedef BeforeSendLogCallback = FutureOr<SentryLog?> Function(SentryLog log);
+
 /// Used to provide timestamp for logging.
 typedef ClockProvider = DateTime Function();
 

dart/test/mocks/mock_log_batcher.dart

@@ -0,0 +1,19 @@
+import 'dart:async';
+
+import 'package:sentry/src/protocol/sentry_log.dart';
+import 'package:sentry/src/sentry_log_batcher.dart';
+
+class MockLogBatcher implements SentryLogBatcher {
+  final addLogCalls = <SentryLog>[];
+  final flushCalls = <void>[];
+
+  @override
+  FutureOr<void> addLog(SentryLog log) {
+    addLogCalls.add(log);
+  }
+
+  @override
+  Future<void> flush() async {
+    flushCalls.add(null);
+  }
+}

dart/test/mocks/mock_sentry_client.dart

@@ -8,8 +8,8 @@ class MockSentryClient with NoSuchMethodProvider implements SentryClient {
   List<CaptureMessageCall> captureMessageCalls = [];
   List<CaptureEnvelopeCall> captureEnvelopeCalls = [];
   List<CaptureTransactionCall> captureTransactionCalls = [];
-
   List<CaptureFeedbackCall> captureFeedbackCalls = [];
+  List<CaptureLogCall> captureLogCalls = [];
   int closeCalls = 0;
 
   @override
@@ -84,6 +84,11 @@ class MockSentryClient with NoSuchMethodProvider implements SentryClient {
     return SentryId.newId();
   }
 
+  @override
+  Future<void> captureLog(SentryLog log, {Scope? scope}) async {
+    captureLogCalls.add(CaptureLogCall(log, scope));
+  }
+
   @override
   void close() {
     closeCalls = closeCalls + 1;
@@ -173,3 +178,10 @@ class CaptureTransactionCall {
 
   CaptureTransactionCall(this.transaction, this.traceContext, this.hint);
 }
+
+class CaptureLogCall {
+  final SentryLog log;
+  final Scope? scope;
+
+  CaptureLogCall(this.log, this.scope);
+}

dart/test/mocks/mock_transport.dart

@@ -7,6 +7,7 @@ class MockTransport implements Transport {
   List<SentryEnvelope> envelopes = [];
   List<SentryEvent> events = [];
   List<String> statsdItems = [];
+  List<Map<String, dynamic>> logs = [];
 
   int _calls = 0;
   String _exceptions = '';
@@ -31,7 +32,7 @@ class MockTransport implements Transport {
     try {
       envelopes.add(envelope);
       if (parseFromEnvelope) {
-        await _eventFromEnvelope(envelope);
+        await _parseEnvelope(envelope);
       }
     } catch (e, stack) {
       _exceptions += '$e\n$stack\n\n';
@@ -41,14 +42,21 @@ class MockTransport implements Transport {
     return envelope.header.eventId ?? SentryId.empty();
   }
 
-  Future<void> _eventFromEnvelope(SentryEnvelope envelope) async {
+  Future<void> _parseEnvelope(SentryEnvelope envelope) async {
     final RegExp statSdRegex = RegExp('^(?!{).+@.+:.+\\|.+', multiLine: true);
 
     final envelopeItemData = await envelope.items.first.dataFactory();
     final envelopeItem = utf8.decode(envelopeItemData);
 
     if (statSdRegex.hasMatch(envelopeItem)) {
       statsdItems.add(envelopeItem);
+    } else if (envelopeItem.contains('items') &&
+        envelopeItem.contains('timestamp') &&
+        envelopeItem.contains('trace_id') &&
+        envelopeItem.contains('level') &&
+        envelopeItem.contains('body')) {
+      final envelopeItemJson = jsonDecode(envelopeItem) as Map<String, dynamic>;
+      logs.add(envelopeItemJson);
     } else {
       final envelopeItemJson = jsonDecode(envelopeItem) as Map<String, dynamic>;
       events.add(SentryEvent.fromJson(envelopeItemJson));

dart/test/protocol/sentry_log_attribute_test.dart

@@ -11,17 +11,17 @@ void main() {
     });
   });
 
-  test('$SentryLogAttribute boolean to json', () {
-    final attribute = SentryLogAttribute.boolean(true);
+  test('$SentryLogAttribute bool to json', () {
+    final attribute = SentryLogAttribute.bool(true);
     final json = attribute.toJson();
     expect(json, {
       'value': true,
       'type': 'boolean',
     });
   });
 
-  test('$SentryLogAttribute integer to json', () {
-    final attribute = SentryLogAttribute.integer(1);
+  test('$SentryLogAttribute int to json', () {
+    final attribute = SentryLogAttribute.int(1);
     final json = attribute.toJson();
 
     expect(json, {

dart/test/protocol/sentry_log_test.dart

@@ -13,8 +13,8 @@ void main() {
       body: 'fixture-body',
       attributes: {
         'test': SentryLogAttribute.string('fixture-test'),
-        'test2': SentryLogAttribute.boolean(true),
-        'test3': SentryLogAttribute.integer(9001),
+        'test2': SentryLogAttribute.bool(true),
+        'test3': SentryLogAttribute.int(9001),
         'test4': SentryLogAttribute.double(9000.1),
       },
       severityNumber: 1,