Stabilize integration tests; avoid using equality for comparison of units and compare unit names instead (#508)

* Avoid using equality for comparison of units and compare unit names instead

* Add check for None value for unit_excluded

* More deterministically determine if node functional after network restore

* Fix lint warnings + update outdated charm libs

Author: shayancanonical

lib/charms/tempo_k8s/v1/charm_tracing.py

@@ -224,7 +224,6 @@ def _remove_stale_otel_sdk_packages():
 
 _remove_stale_otel_sdk_packages()
 
-
 import functools
 import inspect
 import logging
@@ -271,7 +270,7 @@ def _remove_stale_otel_sdk_packages():
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
 
-LIBPATCH = 14
+LIBPATCH = 15
 
 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"]
 
@@ -281,7 +280,6 @@ def _remove_stale_otel_sdk_packages():
 # set this to 0 if you are debugging/developing this library source
 dev_logger.setLevel(logging.CRITICAL)
 
-
 _CharmType = Type[CharmBase]  # the type CharmBase and any subclass thereof
 _C = TypeVar("_C", bound=_CharmType)
 _T = TypeVar("_T", bound=type)
@@ -333,9 +331,22 @@ def _get_tracer() -> Optional[Tracer]:
     try:
         return tracer.get()
     except LookupError:
+        # fallback: this course-corrects for a user error where charm_tracing symbols are imported
+        # from different paths (typically charms.tempo_k8s... and lib.charms.tempo_k8s...)
         try:
             ctx: Context = copy_context()
             if context_tracer := _get_tracer_from_context(ctx):
+                logger.warning(
+                    "Tracer not found in `tracer` context var. "
+                    "Verify that you're importing all `charm_tracing` symbols from the same module path. \n"
+                    "For example, DO"
+                    ": `from charms.lib...charm_tracing import foo, bar`. \n"
+                    "DONT: \n"
+                    " \t - `from charms.lib...charm_tracing import foo` \n"
+                    " \t - `from lib...charm_tracing import bar` \n"
+                    "For more info: https://python-notes.curiousefficiency.org/en/latest/python"
+                    "_concepts/import_traps.html#the-double-import-trap"
+                )
                 return context_tracer.get()
             else:
                 return None

tests/integration/helpers.py

@@ -486,7 +486,7 @@ async def get_primary_unit_wrapper(ops_test: OpsTest, app_name: str, unit_exclud
     units = ops_test.model.applications[app_name].units
 
     for unit in units:
-        if unit == unit_excluded:
+        if unit_excluded and unit.name == unit_excluded.name:
             continue
         try:
             primary_unit = await get_primary_unit(ops_test, unit, app_name)

tests/integration/high_availability/test_self_healing.py

@@ -153,10 +153,11 @@ async def test_network_cut(ops_test: OpsTest, highly_available_cluster, continuo
     logger.info(f"Unit {primary_unit.name} it's on machine {primary_hostname} ✅")
 
     primary_unit_ip = await get_unit_ip(ops_test, primary_unit.name)
+    cluster_admin_password = await get_system_user_password(primary_unit, CLUSTER_ADMIN_USERNAME)
 
     config = {
         "username": CLUSTER_ADMIN_USERNAME,
-        "password": await get_system_user_password(primary_unit, CLUSTER_ADMIN_USERNAME),
+        "password": cluster_admin_password,
         "host": primary_unit_ip,
     }
 
@@ -193,10 +194,22 @@ async def test_network_cut(ops_test: OpsTest, highly_available_cluster, continuo
     # ensure continuous writes still incrementing for all units
     async with ops_test.fast_forward():
         # wait for the unit to be ready
-        logger.info(f"Waiting for {primary_unit.name} to enter maintenance")
-        await ops_test.model.block_until(
-            lambda: primary_unit.workload_status in ["maintenance"], timeout=30 * 60
-        )
+        for attempt in Retrying(stop=stop_after_attempt(60), wait=wait_fixed(10)):
+            with attempt:
+                new_unit_ip = await get_unit_ip(ops_test, primary_unit.name)
+                new_unit_config = {
+                    "username": CLUSTER_ADMIN_USERNAME,
+                    "password": cluster_admin_password,
+                    "host": new_unit_ip,
+                }
+
+                logger.debug(
+                    f"Waiting until connection possible after network restore on {new_unit_ip}"
+                )
+                assert is_connection_possible(
+                    new_unit_config
+                ), "❌ Connection is not possible after network restore"
+
         logger.info(f"Waiting for {primary_unit.name} to enter active")
         await ops_test.model.block_until(
             lambda: primary_unit.workload_status == "active", timeout=40 * 60