Revert "Revert two prs related to OTel 1.37 upgrade (aws-observability#529)"

This reverts commit bf43ed0.

Author: lukeina2z

.github/actions/set_up/action.yml

@@ -27,7 +27,8 @@ runs:
 
     - name: Install tox
       shell: bash
-      run: pip install tox==3.27.1 tox-factor
+      run: |
+        pip install tox==4.30.3
 
     - name: Cache tox environment
       # Preserves .tox directory between runs for faster installs
@@ -41,4 +42,7 @@ runs:
     - name: Run unit tests/benchmarks with tox
       if: ${{ inputs.run_unit_tests == 'true' }}
       shell: bash
-      run: tox -f ${{ inputs.python_version }}-${{ inputs.package_name }} -- -ra
+      run: |
+        ENV_NAME="${{ inputs.python_version }}-test-${{ inputs.package_name }}"
+        echo "Running tox env: $ENV_NAME"
+        tox -e "$ENV_NAME" -- -ra

CHANGELOG.md

@@ -19,5 +19,10 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t
     ([#497](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/497))
 - Fix timeout handling for exceeded deadline in retry logic in OTLPAwsLogsExporter
   ([#501](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/501))
+- Fix: Unable to run ADOT Python unit tests with latest Tox release
+  ([#521](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/521))
 - Fix Gevent patch regression with correct import order
   ([#522](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/522))
+- Bump ADOT Python version to 0.13.0 and OTel dependencies to 1.37.0/0.58b0
+  ([#524](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/524))
+

aws-opentelemetry-distro/pyproject.toml

@@ -26,63 +26,63 @@ classifiers = [
 ]
 
 dependencies = [
-  "opentelemetry-api == 1.33.1",
-  "opentelemetry-sdk == 1.33.1",
-  "opentelemetry-exporter-otlp-proto-grpc == 1.33.1",
-  "opentelemetry-exporter-otlp-proto-http == 1.33.1",
-  "opentelemetry-propagator-b3 == 1.33.1",
-  "opentelemetry-propagator-jaeger == 1.33.1",
-  "opentelemetry-exporter-otlp-proto-common == 1.33.1",
+  "opentelemetry-api == 1.37.0",
+  "opentelemetry-sdk == 1.37.0",
+  "opentelemetry-exporter-otlp-proto-grpc == 1.37.0",
+  "opentelemetry-exporter-otlp-proto-http == 1.37.0",
+  "opentelemetry-propagator-b3 == 1.37.0",
+  "opentelemetry-propagator-jaeger == 1.37.0",
+  "opentelemetry-exporter-otlp-proto-common == 1.37.0",
   "opentelemetry-sdk-extension-aws == 2.0.2",
   "opentelemetry-propagator-aws-xray == 1.0.1",
-  "opentelemetry-distro == 0.54b1",
-  "opentelemetry-processor-baggage == 0.54b1",
-  "opentelemetry-propagator-ot-trace == 0.54b1",
-  "opentelemetry-instrumentation == 0.54b1",
-  "opentelemetry-instrumentation-aws-lambda == 0.54b1",
-  "opentelemetry-instrumentation-aio-pika == 0.54b1",
-  "opentelemetry-instrumentation-aiohttp-client == 0.54b1",
-  "opentelemetry-instrumentation-aiokafka == 0.54b1",
-  "opentelemetry-instrumentation-aiopg == 0.54b1",
-  "opentelemetry-instrumentation-asgi == 0.54b1",
-  "opentelemetry-instrumentation-asyncpg == 0.54b1",
-  "opentelemetry-instrumentation-boto == 0.54b1",
-  "opentelemetry-instrumentation-boto3sqs == 0.54b1",
-  "opentelemetry-instrumentation-botocore == 0.54b1",
-  "opentelemetry-instrumentation-celery == 0.54b1",
-  "opentelemetry-instrumentation-confluent-kafka == 0.54b1",
-  "opentelemetry-instrumentation-dbapi == 0.54b1",
-  "opentelemetry-instrumentation-django == 0.54b1",
-  "opentelemetry-instrumentation-elasticsearch == 0.54b1",
-  "opentelemetry-instrumentation-falcon == 0.54b1",
-  "opentelemetry-instrumentation-fastapi == 0.54b1",
-  "opentelemetry-instrumentation-flask == 0.54b1",
-  "opentelemetry-instrumentation-grpc == 0.54b1",
-  "opentelemetry-instrumentation-httpx == 0.54b1",
-  "opentelemetry-instrumentation-jinja2 == 0.54b1",
-  "opentelemetry-instrumentation-kafka-python == 0.54b1",
-  "opentelemetry-instrumentation-logging == 0.54b1",
-  "opentelemetry-instrumentation-mysql == 0.54b1",
-  "opentelemetry-instrumentation-mysqlclient == 0.54b1",
-  "opentelemetry-instrumentation-pika == 0.54b1",
-  "opentelemetry-instrumentation-psycopg2 == 0.54b1",
-  "opentelemetry-instrumentation-pymemcache == 0.54b1",
-  "opentelemetry-instrumentation-pymongo == 0.54b1",
-  "opentelemetry-instrumentation-pymysql == 0.54b1",
-  "opentelemetry-instrumentation-pyramid == 0.54b1",
-  "opentelemetry-instrumentation-redis == 0.54b1",
-  "opentelemetry-instrumentation-remoulade == 0.54b1",
-  "opentelemetry-instrumentation-requests == 0.54b1",
-  "opentelemetry-instrumentation-sqlalchemy == 0.54b1",
-  "opentelemetry-instrumentation-sqlite3 == 0.54b1",
-  "opentelemetry-instrumentation-starlette == 0.54b1",
-  "opentelemetry-instrumentation-system-metrics == 0.54b1",
-  "opentelemetry-instrumentation-tornado == 0.54b1",
-  "opentelemetry-instrumentation-tortoiseorm == 0.54b1",
-  "opentelemetry-instrumentation-urllib == 0.54b1",
-  "opentelemetry-instrumentation-urllib3 == 0.54b1",
-  "opentelemetry-instrumentation-wsgi == 0.54b1",
-  "opentelemetry-instrumentation-cassandra == 0.54b1",
+  "opentelemetry-distro == 0.58b0",
+  "opentelemetry-processor-baggage == 0.58b0",
+  "opentelemetry-propagator-ot-trace == 0.58b0",
+  "opentelemetry-instrumentation == 0.58b0",
+  "opentelemetry-instrumentation-aws-lambda == 0.58b0",
+  "opentelemetry-instrumentation-aio-pika == 0.58b0",
+  "opentelemetry-instrumentation-aiohttp-client == 0.58b0",
+  "opentelemetry-instrumentation-aiokafka == 0.58b0",
+  "opentelemetry-instrumentation-aiopg == 0.58b0",
+  "opentelemetry-instrumentation-asgi == 0.58b0",
+  "opentelemetry-instrumentation-asyncpg == 0.58b0",
+  "opentelemetry-instrumentation-boto == 0.58b0",
+  "opentelemetry-instrumentation-boto3sqs == 0.58b0",
+  "opentelemetry-instrumentation-botocore == 0.58b0",
+  "opentelemetry-instrumentation-celery == 0.58b0",
+  "opentelemetry-instrumentation-confluent-kafka == 0.58b0",
+  "opentelemetry-instrumentation-dbapi == 0.58b0",
+  "opentelemetry-instrumentation-django == 0.58b0",
+  "opentelemetry-instrumentation-elasticsearch == 0.58b0",
+  "opentelemetry-instrumentation-falcon == 0.58b0",
+  "opentelemetry-instrumentation-fastapi == 0.58b0",
+  "opentelemetry-instrumentation-flask == 0.58b0",
+  "opentelemetry-instrumentation-grpc == 0.58b0",
+  "opentelemetry-instrumentation-httpx == 0.58b0",
+  "opentelemetry-instrumentation-jinja2 == 0.58b0",
+  "opentelemetry-instrumentation-kafka-python == 0.58b0",
+  "opentelemetry-instrumentation-logging == 0.58b0",
+  "opentelemetry-instrumentation-mysql == 0.58b0",
+  "opentelemetry-instrumentation-mysqlclient == 0.58b0",
+  "opentelemetry-instrumentation-pika == 0.58b0",
+  "opentelemetry-instrumentation-psycopg2 == 0.58b0",
+  "opentelemetry-instrumentation-pymemcache == 0.58b0",
+  "opentelemetry-instrumentation-pymongo == 0.58b0",
+  "opentelemetry-instrumentation-pymysql == 0.58b0",
+  "opentelemetry-instrumentation-pyramid == 0.58b0",
+  "opentelemetry-instrumentation-redis == 0.58b0",
+  "opentelemetry-instrumentation-remoulade == 0.58b0",
+  "opentelemetry-instrumentation-requests == 0.58b0",
+  "opentelemetry-instrumentation-sqlalchemy == 0.58b0",
+  "opentelemetry-instrumentation-sqlite3 == 0.58b0",
+  "opentelemetry-instrumentation-starlette == 0.58b0",
+  "opentelemetry-instrumentation-system-metrics == 0.58b0",
+  "opentelemetry-instrumentation-tornado == 0.58b0",
+  "opentelemetry-instrumentation-tortoiseorm == 0.58b0",
+  "opentelemetry-instrumentation-urllib == 0.58b0",
+  "opentelemetry-instrumentation-urllib3 == 0.58b0",
+  "opentelemetry-instrumentation-wsgi == 0.58b0",
+  "opentelemetry-instrumentation-cassandra == 0.58b0",
 ]
 
 [project.optional-dependencies]

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_batch_unsampled_span_processor.py

@@ -12,6 +12,9 @@
 
 
 class BatchUnsampledSpanProcessor(BaseBatchSpanProcessor):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._spans_dropped = False
 
     # pylint: disable=no-self-use
     def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None:
@@ -22,19 +25,18 @@ def on_end(self, span: ReadableSpan) -> None:
         if span.context.trace_flags.sampled:
             return
 
-        if self.done:
+        if self._batch_processor._shutdown:
             logger.warning("Already shutdown, dropping span.")
             return
 
-        if len(self.queue) == self.max_queue_size:
+        if len(self._batch_processor._queue) == self._batch_processor._max_queue_size:
             # pylint: disable=access-member-before-definition
             if not self._spans_dropped:
                 logger.warning("Queue is full, likely spans will be dropped.")
                 # pylint: disable=attribute-defined-outside-init
                 self._spans_dropped = True
 
-        self.queue.appendleft(span)
+        self._batch_processor._queue.appendleft(span)
 
-        if len(self.queue) >= self.max_export_batch_size:
-            with self.condition:
-                self.condition.notify()
+        if len(self._batch_processor._queue) >= self._batch_processor._max_export_batch_size:
+            self._batch_processor._worker_awaken.set()

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -43,7 +43,7 @@
     _import_id_generator,
     _import_sampler,
     _OTelSDKConfigurator,
-    _patch_basic_config,
+    _overwrite_logging_config_fns,
 )
 from opentelemetry.sdk._events import EventLoggerProvider
 from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
@@ -230,11 +230,10 @@ def _init_logging(
     set_event_logger_provider(event_logger_provider)
 
     if setup_logging_handler:
-        _patch_basic_config()
-
         # Add OTel handler
         handler = LoggingHandler(level=logging.NOTSET, logger_provider=provider)
         logging.getLogger().addHandler(handler)
+        _overwrite_logging_config_fns(handler)
 
 
 def _init_tracing(

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/logs/_aws_cw_otlp_batch_log_record_processor.py

@@ -8,8 +8,8 @@
 from amazon.opentelemetry.distro.exporter.otlp.aws.logs.otlp_aws_logs_exporter import OTLPAwsLogExporter
 from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value
 from opentelemetry.sdk._logs import LogData
-from opentelemetry.sdk._logs._internal.export import BatchLogExportStrategy
 from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk._shared_internal import BatchExportStrategy
 from opentelemetry.util.types import AnyValue
 
 _logger = logging.getLogger(__name__)
@@ -89,7 +89,7 @@ def __init__(
 
         self._exporter = exporter
 
-    def _export(self, batch_strategy: BatchLogExportStrategy) -> None:
+    def _export(self, batch_strategy: BatchExportStrategy) -> None:
         """
         Explicitly overrides upstream _export method to add AWS CloudWatch size-based batching
         See:
@@ -102,20 +102,20 @@ def _export(self, batch_strategy: BatchLogExportStrategy) -> None:
         - Estimated data size of exported batches will typically be <= 1 MB except for the case below:
         - If the estimated data size of an exported batch is ever > 1 MB then the batch size is guaranteed to be 1
         """
-        with self._export_lock:
+        with self._batch_processor._export_lock:
             iteration = 0
             # We could see concurrent export calls from worker and force_flush. We call _should_export_batch
             # once the lock is obtained to see if we still need to make the requested export.
-            while self._should_export_batch(batch_strategy, iteration):
+            while self._batch_processor._should_export_batch(batch_strategy, iteration):
                 iteration += 1
                 token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
                 try:
-                    batch_length = min(self._max_export_batch_size, len(self._queue))
+                    batch_length = min(self._batch_processor._max_export_batch_size, len(self._batch_processor._queue))
                     batch_data_size = 0
                     batch = []
 
                     for _ in range(batch_length):
-                        log_data: LogData = self._queue.pop()
+                        log_data: LogData = self._batch_processor._queue.pop()
                         log_size = self._estimate_log_size(log_data)
 
                         if batch and (batch_data_size + log_size > self._MAX_LOG_REQUEST_BYTE_SIZE):
@@ -252,7 +252,7 @@ def _estimate_utf8_size(s: str):
     # https://github.com/open-telemetry/opentelemetry-python/issues/3193
     # https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/_shared_internal/__init__.py#L199
     def force_flush(self, timeout_millis: Optional[int] = None) -> bool:
-        if self._shutdown:
+        if self._batch_processor._shutdown:
             return False
-        self._export(BatchLogExportStrategy.EXPORT_AT_LEAST_ONE_BATCH)
+        self._export(BatchExportStrategy.EXPORT_AT_LEAST_ONE_BATCH)
         return True

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/exporter/otlp/aws/logs/otlp_aws_logs_exporter.py

@@ -18,6 +18,7 @@
 from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
 from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs
 from opentelemetry.exporter.otlp.proto.http import Compression
+from opentelemetry.exporter.otlp.proto.http._common import _is_retryable
 from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
 from opentelemetry.sdk._logs import LogData
 from opentelemetry.sdk._logs.export import LogExportResult
@@ -171,7 +172,7 @@ def _retryable(resp: Response) -> bool:
         """
         # See: https://opentelemetry.io/docs/specs/otlp/#otlphttp-throttling
 
-        return resp.status_code in (429, 503) or OTLPLogExporter._retryable(resp)
+        return resp.status_code in (429, 503) or _is_retryable(resp)
 
     def _get_retry_delay_sec(self, headers: CaseInsensitiveDict, retry_num: int) -> float:
         """

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/version.py

@@ -1,4 +1,4 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-__version__ = "0.12.1.dev0"
+__version__ = "0.13.0.dev0"