add log exporter and metric reader params to configure azure monitor

sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 1.8.4 (Unreleased)
 
 ### Features Added
+- Added ability to add additional Log Record Processors and Metric Readers via configure_azure_monitor
+  ([#44367](https://github.com/Azure/azure-sdk-for-python/pull/44367))
 
 ### Breaking Changes
 

sdk/monitor/azure-monitor-opentelemetry/README.md

@@ -68,6 +68,8 @@ You can use `configure_azure_monitor` to set up instrumentation for your app to
 | `resource` | Specifies the OpenTelemetry [Resource][ot_spec_resource] associated with your application. Passed in [Resource Attributes][ot_spec_resource_attributes] take priority over default attributes and those from [Resource Detectors][ot_python_resource_detectors]. | [OTEL_SERVICE_NAME][ot_spec_service_name], [OTEL_RESOURCE_ATTRIBUTES][ot_spec_resource_attributes], [OTEL_EXPERIMENTAL_RESOURCE_DETECTORS][ot_python_resource_detectors] |
 | `span_processors` | A list of [span processors][ot_span_processor] that will perform processing on each of your spans before they are exported. Useful for filtering/modifying telemetry. | `N/A` |
 | `views` | A list of [views][ot_view] that will be used to customize metrics exported by the SDK. | `N/A` |
+| `log_record_processors` | A list of [log record processors][ot_log_record_processor] that will process log records before they are exported. | `N/A` |
+| `metric_readers` | A list of [metric reader][ot_metric_reader] that will process metric readers before they are exported | `N/A` |
 | `traces_per_second` | Configures the Rate Limited sampler by specifying the maximum number of traces to sample per second. When set, this automatically enables the rate-limited sampler. Alternatively, you can configure sampling using the `OTEL_TRACES_SAMPLER` and `OTEL_TRACES_SAMPLER_ARG` environment variables as described in the table below. Please note that the sampling configuration via environment variables will have precedence over the sampling exporter/distro options. | `N/A`
 
 You can configure further with [OpenTelemetry environment variables][ot_env_vars].
@@ -231,6 +233,7 @@ contact [[email protected]](mailto:[email protected]) with any additio
 [ot_sdk_python]: https://github.com/open-telemetry/opentelemetry-python
 [ot_sdk_python_metric_reader]: https://opentelemetry-python.readthedocs.io/en/latest/sdk/metrics.export.html#opentelemetry.sdk.metrics.export.MetricReader
 [ot_span_processor]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#span-processor
+[ot_log_record_processor]: https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/logs/sdk.md#log-record-processor
 [ot_view]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#view
 [ot_sdk_python_view_examples]: https://github.com/open-telemetry/opentelemetry-python/tree/main/docs/examples/metrics/views
 [ot_instrumentation_django]: https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-django

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_configure.py

@@ -12,7 +12,7 @@
 )
 from opentelemetry.metrics import set_meter_provider
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader, MetricReader
 from opentelemetry.sdk.metrics.view import View
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
@@ -38,6 +38,8 @@
     SAMPLING_RATIO_ARG,
     SAMPLING_TRACES_PER_SECOND_ARG,
     SPAN_PROCESSORS_ARG,
+    LOG_RECORD_PROCESSORS_ARG,
+    METRIC_READERS_ARG,
     VIEWS_ARG,
     ENABLE_TRACE_BASED_SAMPLING_ARG,
 )
@@ -102,6 +104,10 @@ def configure_azure_monitor(**kwargs) -> None:  # pylint: disable=C4758
      Attributes take priority over default attributes and those from Resource Detectors.
     :keyword list[~opentelemetry.sdk.trace.SpanProcessor] span_processors: List of `SpanProcessor` objects
      to process every span prior to exporting. Will be run sequentially.
+    :keyword list[~opentelemetry.sdk._logs.LogRecordProcessor] log_record_processors: List of `LogRecordProcessor` 
+     objects to process every log record prior to exporting. Will be run sequentially.
+    :keyword list[~opentelemetry.sdk.metrics.MetricReader] metric_readers: List of MetricReader objects to read and
+     export metrics. Each reader can have its own exporter and collection interval.
     :keyword bool enable_live_metrics: Boolean value to determine whether to enable live metrics feature.
      Defaults to `False`.
     :keyword bool enable_performance_counters: Boolean value to determine whether to enable performance counters.
@@ -110,7 +116,7 @@ def configure_azure_monitor(**kwargs) -> None:  # pylint: disable=C4758
      `<tempfile.gettempdir()>/Microsoft/AzureMonitor/opentelemetry-python-<your-instrumentation-key>`.
     :keyword list[~opentelemetry.sdk.metrics.view.View] views: List of `View` objects to configure and filter
      metric output.
-    :keyword bool enable_trace_based_sampling_for_logs: Boolean value to determine whether to enable trace based 
+    :keyword bool enable_trace_based_sampling_for_logs: Boolean value to determine whether to enable trace based
      sampling for logs. Defaults to `False`
     :rtype: None
     """
@@ -216,6 +222,8 @@ def _setup_logging(configurations: Dict[str, ConfigurationValue]):
         enable_performance_counters_config = configurations[ENABLE_PERFORMANCE_COUNTERS_ARG]
         logger_provider = LoggerProvider(resource=resource)
         enable_trace_based_sampling_for_logs = configurations[ENABLE_TRACE_BASED_SAMPLING_ARG]
+        for log_record_processor in configurations[LOG_RECORD_PROCESSORS_ARG]:  # type: ignore
+            logger_provider.add_log_record_processor(log_record_processor)  # type: ignore
         if configurations.get(ENABLE_LIVE_METRICS_ARG):
             qlp = _QuickpulseLogRecordProcessor()
             logger_provider.add_log_record_processor(qlp)
@@ -274,11 +282,12 @@ def _setup_logging(configurations: Dict[str, ConfigurationValue]):
 def _setup_metrics(configurations: Dict[str, ConfigurationValue]):
     resource: Resource = configurations[RESOURCE_ARG]  # type: ignore
     views: List[View] = configurations[VIEWS_ARG]  # type: ignore
+    readers: list[MetricReader] = configurations[METRIC_READERS_ARG]  # type: ignore
     enable_performance_counters_config = configurations[ENABLE_PERFORMANCE_COUNTERS_ARG]
     metric_exporter = AzureMonitorMetricExporter(**configurations)
-    reader = PeriodicExportingMetricReader(metric_exporter)
+    readers.append(PeriodicExportingMetricReader(metric_exporter))
     meter_provider = MeterProvider(
-        metric_readers=[reader],
+        metric_readers=readers,
         resource=resource,
         views=views,
     )

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py

@@ -24,6 +24,8 @@
 RESOURCE_ARG = "resource"
 SAMPLING_RATIO_ARG = "sampling_ratio"
 SPAN_PROCESSORS_ARG = "span_processors"
+LOG_RECORD_PROCESSORS_ARG = "log_record_processors"
+METRIC_READERS_ARG = "metric_readers"
 VIEWS_ARG = "views"
 RATE_LIMITED_SAMPLER = "microsoft.rate_limited"
 FIXED_PERCENTAGE_SAMPLER = "microsoft.fixed.percentage"

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_utils/configurations.py

@@ -43,6 +43,8 @@
     SAMPLING_RATIO_ARG,
     SAMPLING_TRACES_PER_SECOND_ARG,
     SPAN_PROCESSORS_ARG,
+    LOG_RECORD_PROCESSORS_ARG,
+    METRIC_READERS_ARG,
     VIEWS_ARG,
     RATE_LIMITED_SAMPLER,
     FIXED_PERCENTAGE_SAMPLER,
@@ -77,6 +79,8 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]:
     _default_sampling_ratio(configurations)
     _default_instrumentation_options(configurations)
     _default_span_processors(configurations)
+    _default_log_record_processors(configurations)
+    _default_metric_readers(configurations)
     _default_enable_live_metrics(configurations)
     _default_enable_performance_counters(configurations)
     _default_views(configurations)
@@ -220,6 +224,12 @@ def _default_instrumentation_options(configurations):
 def _default_span_processors(configurations):
     configurations.setdefault(SPAN_PROCESSORS_ARG, [])
 
+def _default_log_record_processors(configurations):
+    configurations.setdefault(LOG_RECORD_PROCESSORS_ARG, [])
+
+
+def _default_metric_readers(configurations):
+    configurations.setdefault(METRIC_READERS_ARG, [])
 
 def _default_enable_live_metrics(configurations):
     configurations.setdefault(ENABLE_LIVE_METRICS_ARG, False)

sdk/monitor/azure-monitor-opentelemetry/tests/test_configure.py

@@ -524,6 +524,7 @@ def test_setup_logging(self, get_logger_mock, pclp_mock):
         logging_handler_mock.return_value = logging_handler_init_mock
         logger_mock = Mock()
         logger_mock.handlers = []
+        custom_lrp = Mock()
         get_logger_mock.return_value = logger_mock
         formatter_init_mock = Mock()
         elp_init_mock = Mock()
@@ -535,6 +536,7 @@ def test_setup_logging(self, get_logger_mock, pclp_mock):
             "enable_performance_counters": True,
             "logger_name": "test",
             "resource": TEST_RESOURCE,
+            "log_record_processors": [custom_lrp],
             "logging_formatter": formatter_init_mock,
             "enable_trace_based_sampling_for_logs": False,
         }
@@ -558,7 +560,9 @@ def test_setup_logging(self, get_logger_mock, pclp_mock):
         set_logger_provider_mock.assert_called_once_with(lp_init_mock)
         log_exporter_mock.assert_called_once_with(**configurations)
         blrp_mock.assert_called_once_with(log_exp_init_mock, {"enable_trace_based_sampling_for_logs": False})
-        self.assertEqual(lp_init_mock.add_log_record_processor.call_count, 2)
+        self.assertEqual(lp_init_mock.add_log_record_processor.call_count, 3)
+        lp_init_mock.add_log_record_processor.assert_has_calls([call(custom_lrp), call(pclp_init_mock), call(blrp_init_mock)])
+        self.assertEqual(lp_init_mock.add_log_record_processor.call_count, 3)
         lp_init_mock.add_log_record_processor.assert_has_calls([call(pclp_init_mock), call(blrp_init_mock)])
         logging_handler_mock.assert_called_once_with(logger_provider=lp_init_mock)
         logging_handler_init_mock.setFormatter.assert_called_once_with(formatter_init_mock)
@@ -591,7 +595,7 @@ def test_setup_logging_duplicate_logger(self, get_logger_mock, instance_mock, pc
 
         # Create a mock handler that looks like LoggingHandler
         logging_handler_init_mock = Mock()
-        
+
         # Set up the logger to already have a LoggingHandler
         logger_mock = Mock()
         logger_mock.handlers = [logging_handler_init_mock]
@@ -600,27 +604,33 @@ def test_setup_logging_duplicate_logger(self, get_logger_mock, instance_mock, pc
 
         elp_init_mock = Mock()
         elp_mock.return_value = elp_init_mock
-        
+
         configurations = {
             "connection_string": "test_cs",
             "enable_performance_counters": True,
             "logger_name": "test",
             "resource": TEST_RESOURCE,
+            "log_record_processors": [],
             "logging_formatter": None,
             "enable_trace_based_sampling_for_logs": True,
         }
-        
+
         # Patch all the necessary modules and imports
-        with patch.dict('sys.modules', {
-            'opentelemetry._logs': Mock(set_logger_provider=set_logger_provider_mock),
-            'opentelemetry.sdk._logs': Mock(LoggerProvider=lp_mock),
-            'azure.monitor.opentelemetry.exporter.export.logs._processor': Mock(_AzureBatchLogRecordProcessor=blrp_mock),
-            'azure.monitor.opentelemetry.exporter': Mock(AzureMonitorLogExporter=log_exporter_mock),
-            'opentelemetry._events': Mock(_set_event_logger_provider=set_elp_mock),
-            'opentelemetry.sdk._events': Mock(EventLoggerProvider=elp_mock)
-        }):
+        with patch.dict(
+            "sys.modules",
+            {
+                "opentelemetry._logs": Mock(set_logger_provider=set_logger_provider_mock),
+                "opentelemetry.sdk._logs": Mock(LoggerProvider=lp_mock),
+                "azure.monitor.opentelemetry.exporter.export.logs._processor": Mock(
+                    _AzureBatchLogRecordProcessor=blrp_mock
+                ),
+                "azure.monitor.opentelemetry.exporter": Mock(AzureMonitorLogExporter=log_exporter_mock),
+                "opentelemetry._events": Mock(_set_event_logger_provider=set_elp_mock),
+                "opentelemetry.sdk._events": Mock(EventLoggerProvider=elp_mock),
+            },
+        ):
             _setup_logging(configurations)
-        
+
         # Verify the correct behavior
         lp_mock.assert_called_once_with(resource=TEST_RESOURCE)
         set_logger_provider_mock.assert_called_once_with(lp_init_mock)
@@ -668,6 +678,7 @@ def test_setup_logging_disable_performance_counters(self, get_logger_mock, pclp_
             "enable_performance_counters": False,
             "logger_name": "test",
             "resource": TEST_RESOURCE,
+            "log_record_processors": [],
             "logging_formatter": formatter_init_mock,
             "enable_trace_based_sampling_for_logs": False,
         }
@@ -730,15 +741,20 @@ def test_setup_metrics(
         reader_init_mock = Mock()
         reader_mock.return_value = reader_init_mock
 
+        # Custom metric readers provided by user
+        custom_reader_1 = Mock()
+        custom_reader_2 = Mock()
+
         configurations = {
             "connection_string": "test_cs",
             "enable_performance_counters": True,
             "resource": TEST_RESOURCE,
+            "metric_readers": [custom_reader_1, custom_reader_2],
             "views": [],
         }
         _setup_metrics(configurations)
         mp_mock.assert_called_once_with(
-            metric_readers=[reader_init_mock],
+            metric_readers=[custom_reader_1, custom_reader_2, reader_init_mock],
             resource=TEST_RESOURCE,
             views=[],
         )
@@ -783,6 +799,7 @@ def test_setup_metrics_views(
             "connection_string": "test_cs",
             "enable_performance_counters": False,
             "resource": TEST_RESOURCE,
+            "metric_readers": [],
             "views": [view_mock],
         }
         _setup_metrics(configurations)
@@ -831,6 +848,7 @@ def test_setup_metrics_perf_counters_disabled(
             "connection_string": "test_cs",
             "enable_performance_counters": False,
             "resource": TEST_RESOURCE,
+            "metric_readers": [],
             "views": [],
         }
         _setup_metrics(configurations)

sdk/monitor/azure-monitor-opentelemetry/tests/utils/test_configurations.py

@@ -75,6 +75,8 @@ def test_get_configurations(self, resource_create_mock):
             views=["test_view"],
             logger_name="test_logger",
             span_processors=["test_processor"],
+            log_record_processors=["test_log_record_processor"],
+            metric_readers=["test_metric_reader"],
             enable_trace_based_sampling_for_logs=True,
         )
 
@@ -109,6 +111,8 @@ def test_get_configurations(self, resource_create_mock):
         self.assertEqual(configurations["views"], ["test_view"])
         self.assertEqual(configurations["logger_name"], "test_logger")
         self.assertEqual(configurations["span_processors"], ["test_processor"])
+        self.assertEqual(configurations["log_record_processors"], ["test_log_record_processor"])
+        self.assertEqual(configurations["metric_readers"], ["test_metric_reader"])
         self.assertEqual(configurations[ENABLE_TRACE_BASED_SAMPLING_ARG], True)
 
     @patch.dict("os.environ", {}, clear=True)
@@ -143,6 +147,8 @@ def test_get_configurations_defaults(self, resource_create_mock):
         self.assertEqual(configurations["enable_performance_counters"], True)
         self.assertEqual(configurations["logger_name"], "")
         self.assertEqual(configurations["span_processors"], [])
+        self.assertEqual(configurations["log_record_processors"], [])
+        self.assertEqual(configurations["metric_readers"], [])
         self.assertEqual(configurations["views"], [])
         self.assertEqual(configurations[ENABLE_TRACE_BASED_SAMPLING_ARG], False)
 
@@ -423,7 +429,7 @@ def test_get_configurations_env_vars_rate_limited(self, resource_create_mock):
         self.assertEqual(environ[OTEL_EXPERIMENTAL_RESOURCE_DETECTORS], "custom_resource_detector")
         resource_create_mock.assert_called_once_with()
         self.assertEqual(configurations["traces_per_second"], 0.5)
-    
+
     @patch.dict("os.environ", {}, clear=True)
     @patch("opentelemetry.sdk.resources.Resource.create", return_value=TEST_DEFAULT_RESOURCE)
     def test_get_configurations_rate_limited_sampler_param(self, resource_create_mock):
@@ -488,7 +494,7 @@ def test_get_configurations_env_vars_no_preference(self, resource_create_mock):
         self.assertEqual(environ[OTEL_EXPERIMENTAL_RESOURCE_DETECTORS], "custom_resource_detector")
         resource_create_mock.assert_called_once_with()
         self.assertEqual(configurations["sampling_ratio"], 1.0)
-    
+
     @patch.dict(
         "os.environ",
         {
@@ -526,7 +532,7 @@ def test_get_configurations_env_vars_check_default(self, resource_create_mock):
         self.assertEqual(environ[OTEL_EXPERIMENTAL_RESOURCE_DETECTORS], "custom_resource_detector")
         resource_create_mock.assert_called_once_with()
         self.assertEqual(configurations["sampling_ratio"], 1.0)
-    
+
     @patch.dict(
         "os.environ",
         {
@@ -564,4 +570,4 @@ def test_get_configurations_env_vars_fixed_percentage(self, resource_create_mock
         self.assertEqual(configurations["resource"].attributes, TEST_DEFAULT_RESOURCE.attributes)
         self.assertEqual(environ[OTEL_EXPERIMENTAL_RESOURCE_DETECTORS], "custom_resource_detector")
         resource_create_mock.assert_called_once_with()
-        self.assertEqual(configurations["sampling_ratio"], 0.9)
+        self.assertEqual(configurations["sampling_ratio"], 0.9)