testing: vmcore: Stop printing logs

The gitlab tests have reached their limit of log size (500K). It's very
difficult to look through the tests and see what failed.

Rectify this by logging test data to the JUnit XML format. Produces a
combined log for all vmcore tests. You can pretty easily browse the
output in less (though it is XML formatted). Gitlab will also understand
the logs!

In the process, I've also ensured that vmcore tests always run to
completion (something they didn't do before), and that the end of the
test log contains a brief summary of the vmcores which succeeded and
failed.

Signed-off-by: Stephen Brennan <[email protected]>

Author: brenns10

.gitignore

@@ -12,3 +12,4 @@ deploy_dev.sh
 drgntools-fio.dat
 testdata
 test.log
+vmcore.xml

.gitlab-ci.yml

@@ -6,3 +6,6 @@ vmtest:
     - tox -e runner -- python -m testing.vmcore test --core-directory /var/drgn-tools/vmcores
     - mkdir -p tmp/overlays tmp/info
     - tox -e runner -- python -m testing.heavyvm.runner --image-dir /var/drgn-tools/images --vm-info-dir tmp/info --overlay-dir tmp/overlays --tarball archive.tar.gz
+  artifacts:
+    reports:
+      junit: vmcore.xml

testing/requirements.txt

@@ -4,3 +4,4 @@ oci-cli
 paramiko
 rich
 types-paramiko
+junitparser==3.1.0

testing/vmcore.py

@@ -18,6 +18,7 @@
 from typing import Tuple
 
 import oci.config
+from junitparser import JUnitXml
 from oci.exceptions import ConfigFileNotFound
 from oci.object_storage import ObjectStorageClient
 from oci.object_storage import UploadManager
@@ -247,6 +248,11 @@ def should_run_vmcore(name: str) -> bool:
                 return True
         return False
 
+    failed = []
+    passed = []
+    xml = JUnitXml()
+    xml_run = Path("test.xml")
+
     for path in CORE_DIR.iterdir():
         core_name = path.name
         if not should_run_vmcore(core_name):
@@ -256,17 +262,37 @@ def should_run_vmcore(name: str) -> bool:
             f"Running tests on vmcore {core_name}",
             collapsed=True,
         ):
-            subprocess.run(
+            if xml_run.exists():
+                xml_run.unlink()
+            res = subprocess.run(
                 [
                     "tox",
                     "--",
                     "--vmcore",
                     core_name,
                     "--vmcore-dir",
                     str(CORE_DIR),
+                    "--junitxml=test.xml",
+                    "-o",
+                    "junit_logging=all",
                 ],
-                check=True,
             )
+            run_data = JUnitXml.fromfile(str(xml_run))
+            xml += run_data
+            if res.returncode != 0:
+                failed.append(core_name)
+            else:
+                passed.append(core_name)
+
+    xml_run.unlink()
+    xml.write("vmcore.xml")
+    print("Complete test logs: vmcore.xml")
+    print("Vmcore Test Summary -- Passed:")
+    print("\n".join(f"- {n}" for n in passed))
+    if failed:
+        print("Vmcore Test Summary -- FAILED:")
+        print("\n".join(f"- {n}" for n in failed))
+        sys.exit(1)
 
 
 def get_client() -> ObjectStorageClient:

tests/conftest.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2023, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import sys
 from fnmatch import fnmatch
 from pathlib import Path
 from typing import List
@@ -37,6 +38,16 @@ def prog_type() -> str:
         return "live"
 
 
+@pytest.fixture(scope="session", autouse=True)
+def log_global_env_facts(prog, record_testsuite_property):
+    if VMCORE:
+        record_testsuite_property("target", VMCORE_NAME)
+    else:
+        record_testsuite_property("target", "live")
+    release = prog["UTS_RELEASE"].string_().decode("utf-8")
+    record_testsuite_property("release", release)
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--vmcore",
@@ -104,6 +115,13 @@ def pytest_configure(config):
         for module in vmcore_dir.glob("*.ko.debug"):
             DEBUGINFO.append(module)
 
+    config.inicfg["junit_suite_name"] = "Python {}.{}.{} - {}".format(
+        sys.version_info.major,
+        sys.version_info.minor,
+        sys.version_info.micro,
+        f"vmcore {vmcore}" if vmcore else "live",
+    )
+
 
 def pytest_runtest_setup(item: pytest.Item):
     skip_live = False

tox.ini

@@ -10,7 +10,7 @@ deps =
     -rrequirements-dev.txt
     pytest-cov
 commands =
-    python -m pytest --cov=drgn_tools --cov=tests -rP {posargs}
+    python -m pytest --cov=drgn_tools --cov=tests --junitxml=test.xml -o junit_logging=all {posargs}
 passenv = DRGNTOOLS_*, GITLAB_CI, GITHUB_ACTIONS
 
 [testenv:docs]