Merge pull request #21 from SmithChart/post-mortem

Add post-mortem information collection for failed tests

Author: SmithChart

conftest.py

@@ -5,6 +5,8 @@
 
 import pytest
 
+pytest_plugins = ["postmortem"]
+
 
 @pytest.fixture(scope="function")
 def barebox(strategy):

lxatacstrategy.py

@@ -204,3 +204,31 @@ def activate_optionals(self):
         if self.ethmux:
             self.target.activate(self.ethmux)
             self.ethmux.set(True)  # Connect upstream Ethernet to Lab network as default
+
+    def postmortem_info(self) -> dict[str, list[str]]:
+        pm_info: dict[str, list[str]] = {"status": [self.status.name]}
+
+        def get_info(shell, command):
+            pm_info[command] = shell.run_check(command)
+
+        if self.status == Status.barebox:
+            get_info(self.barebox, "version")
+            get_info(self.barebox, "mount")
+            get_info(self.barebox, "global")
+            get_info(self.barebox, "nv")
+            get_info(self.barebox, "dmesg")
+        elif self.status == Status.shell:
+            get_info(self.shell, "uname -a")
+            get_info(self.shell, "cat /etc/os-release")
+            get_info(self.shell, "cat /etc/buildinfo")
+            get_info(self.shell, "dmesg -l 5")
+            get_info(self.shell, "findmnt")
+            get_info(self.shell, "lsns")
+            get_info(self.shell, "ip -brief address")
+            get_info(self.shell, "ip -brief route")
+            get_info(self.shell, "ip -brief -6 route")
+            get_info(self.shell, "df --human-readable")
+            get_info(self.shell, "free -m")
+            get_info(self.shell, "systemctl list-units --failed")
+
+        return pm_info

postmortem.py

@@ -0,0 +1,49 @@
+import logging
+
+import pytest
+from pytest import CollectReport, StashKey
+
+_phase_report_key = StashKey[dict[str, CollectReport]]()
+_pm_logger = logging.getLogger("post-mortem")
+
+
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]):
+    """
+    Make the report for a step available in the item's stash, so that
+    fixtures can read the result in their teardown phase.
+    Heavily inspired by: https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
+    Implements the runtest_makereport-hook: https://docs.pytest.org/en/latest/reference/reference.html#std-hook-pytest_runtest_makereport
+
+    :param item: The item the report is generated for.
+    :param call: CallInfo for this phase
+    :return: The report re received from the previous hook.
+    """
+    rep = yield
+
+    # store test results for each phase of a call, which can
+    # be "setup", "call", "teardown"
+    item.stash.setdefault(_phase_report_key, {})[rep.when] = rep
+    return rep
+
+
+@pytest.fixture(autouse=True)
+def pm_system(request: pytest.FixtureRequest, strategy, record_property):
+    """
+    Retrieves post-mortem diagnosis information from the strategy and emits the information to the log with level
+    WARNING and also adds the information to the `junit.xml`.
+
+    The strategy must implement a strategy.postmotem_info().
+    It is up to the strategy to decide which information to collect depending on the DUTs status and the connections
+    available (e.g. serial or ssh).
+    """
+    yield
+
+    report = request.node.stash[_phase_report_key]
+    if "call" in report and report["call"].failed:
+        post_mortem_info: dict[str, list[str]] = strategy.postmortem_info()
+        record_property("postmortem", post_mortem_info)
+        for key, value in post_mortem_info.items():
+            _pm_logger.warning(f"POST-MORTEM INFO: {key}")
+            for line in value:
+                _pm_logger.warning(f"| {line}")

tests/test_tacd.py

@@ -34,7 +34,7 @@ def test_tacd_http_temperature(strategy, shell):
         (-0.01, 14, "v1/iobus/feedback/voltage"),
     ),
 )
-def test_tacd_http_adc(strategy, low, high, endpoint):
+def test_tacd_http_adc(strategy, shell, low, high, endpoint):
     """Test tacd ADC endpoints."""
     r = requests.get(f"http://{strategy.network.address}/{endpoint}")
     assert r.status_code == 200