Support live LTS tests on Azure Linux (#6825)

Co-authored-by: Amaury Chamayou <[email protected]>

.github/workflows/ci.yml

@@ -148,5 +148,5 @@ jobs:
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All other acceptably fast tests, which are now supported on Azure Linux.
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit|lts"
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|suite|unit"
         shell: bash

scripts/setup-dev.sh

@@ -5,4 +5,5 @@
 set -ex
 
 tdnf -y install  \
-    vim
+    vim  \
+    cpio # Used by LTS test to extract binaries from rpms

tests/infra/github.py

@@ -24,6 +24,7 @@
 TAG_RELEASE_CANDIDATE_SUFFIX = "-rc"
 MAIN_BRANCH_NAME = "main"
 DEBIAN_PACKAGE_EXTENSION = "_amd64.deb"
+RPM_PACKAGE_EXTENSION = "-1.x86_64.rpm"
 # This assumes that CCF is installed at `/opt/ccf`, which is true from 1.0.0
 INSTALL_DIRECTORY_PREFIX = "ccf_install_"
 INSTALL_DIRECTORY_SUB_PATH = "opt/ccf"
@@ -147,15 +148,23 @@ def get_major_version_from_branch_name(branch_name):
     )
 
 
-def get_debian_package_prefix_with_platform(tag_name, platform="sgx"):
+def get_package_prefix_with_platform(tag_name, platform="snp"):
     tag_components = tag_name.split("-")
-    tag_components[0] += f"_{platform}"
-    return "-".join(tag_components)
+    if len(tag_components) > 3:
+        assert False, f"Unexpected tag format: {tag_name} parsed into {tag_components}"
 
+    prefix = tag_components[0] + f"_{platform}" + f"-{tag_components[1]}"
+    if len(tag_components) == 3:
+        prefix += f"_{tag_components[2]}"
+    return prefix
 
-def get_debian_package_url_from_tag_name(tag_name, platform="sgx"):
+
+def get_package_url_from_tag_name(tag_name, platform="snp"):
+    # First release with RPM packages for Azure Linux
+    if get_version_from_tag_name(tag_name) >= Version("6.0.0.dev19"):
+        return f"{REMOTE_URL}/releases/download/{tag_name}/{get_package_prefix_with_platform(tag_name, platform)}{RPM_PACKAGE_EXTENSION}"
     if get_version_from_tag_name(tag_name) >= Version("3.0.0-rc0"):
-        return f'{REMOTE_URL}/releases/download/{tag_name}/{get_debian_package_prefix_with_platform(tag_name, platform).replace("-", "_")}{DEBIAN_PACKAGE_EXTENSION}'
+        return f'{REMOTE_URL}/releases/download/{tag_name}/{get_package_prefix_with_platform(tag_name, platform).replace("-", "_")}{DEBIAN_PACKAGE_EXTENSION}'
     else:
         return f'{REMOTE_URL}/releases/download/{tag_name}/{tag_name.replace("-", "_")}{DEBIAN_PACKAGE_EXTENSION}'
 
@@ -177,7 +186,7 @@ def __init__(self):
     def has_release_for_tag_name(self, tag_name):
         return (
             requests.head(
-                get_debian_package_url_from_tag_name(tag_name),
+                get_package_url_from_tag_name(tag_name),
                 allow_redirects=True,
                 timeout=30,
             ).status_code
@@ -291,7 +300,7 @@ def get_supported_lts_releases(self, branch):
             major_version += 1
         return releases
 
-    def install_release(self, tag, platform="sgx"):
+    def install_release(self, tag, platform="snp"):
         stripped_tag = strip_release_tag_name(tag)
         install_directory = f"{INSTALL_DIRECTORY_PREFIX}{stripped_tag}"
         if get_version_from_tag_name(tag) >= Version("3.0.0-rc1"):
@@ -304,7 +313,7 @@ def install_release(self, tag, platform="sgx"):
             install_path = os.path.abspath(
                 os.path.join(install_directory, INSTALL_DIRECTORY_SUB_PATH)
             )
-        debian_package_url = get_debian_package_url_from_tag_name(tag, platform)
+        package_url = get_package_url_from_tag_name(tag, platform)
         installed_file_path = os.path.join(install_path, INSTALL_SUCCESS_FILE)
 
         # Skip downloading release if it already exists
@@ -314,19 +323,30 @@ def install_release(self, tag, platform="sgx"):
             )
             return tag, install_path
 
-        debian_package_name = debian_package_url.split("/")[-1]
-        download_path = os.path.join(DOWNLOAD_FOLDER_NAME, debian_package_name)
-        LOG.info(f"Downloading {debian_package_url} to {download_path}...")
+        package_name = package_url.split("/")[-1]
+        download_path = os.path.join(DOWNLOAD_FOLDER_NAME, package_name)
+        LOG.info(f"Downloading {package_url} to {download_path}...")
         if not os.path.exists(DOWNLOAD_FOLDER_NAME):
             os.mkdir(DOWNLOAD_FOLDER_NAME)
 
         shutil.rmtree(download_path, ignore_errors=True)
-        urllib.request.urlretrieve(debian_package_url, download_path)
+        urllib.request.urlretrieve(package_url, download_path)
 
-        LOG.info("Unpacking debian package...")
         shutil.rmtree(install_directory, ignore_errors=True)
-        install_cmd = ["dpkg-deb", "-R", download_path, install_directory]
-        subprocess.run(install_cmd, check=True)
+
+        if download_path.endswith(DEBIAN_PACKAGE_EXTENSION):
+            LOG.info("Unpacking debian package...")
+            install_cmd = ["dpkg-deb", "-R", download_path, install_directory]
+            subprocess.run(install_cmd, check=True)
+        elif download_path.endswith(RPM_PACKAGE_EXTENSION):
+            LOG.info("Unpacking RPM package...")
+            os.makedirs(install_directory, exist_ok=True)
+            install_cmd = (
+                f"rpm2cpio {download_path} | cpio -idmv -D {install_directory}"
+            )
+            subprocess.run(install_cmd, shell=True, check=True)
+        else:
+            assert False, f"Unsupported package type: {download_path}"
 
         # Write new file to avoid having to download install again
         open(os.path.join(install_path, INSTALL_SUCCESS_FILE), "w+", encoding="utf-8")
@@ -403,16 +423,38 @@ def get_first_tag_for_next_release_branch(self, branch):
             return None
 
     def install_latest_lts_for_branch(
-        self, branch, this_release_branch_only, platform="sgx"
+        self, branch, this_release_branch_only, platform="snp"
     ):
         latest_tag = self.get_latest_released_tag_for_branch(
             branch, this_release_branch_only
         )
-        return (
-            self.install_release(latest_tag, platform) if latest_tag else (None, None)
-        )
 
-    def install_next_lts_for_branch(self, branch, platform="sgx"):
+        if not latest_tag:
+            return (None, None)
+
+        ccf_version = get_version_from_tag_name(latest_tag)
+        os_release = subprocess.check_output(
+            "cat /etc/os-release", universal_newlines=True, shell=True
+        ).lower()
+        if "ubuntu" not in os_release and ccf_version.major < 6:
+            # Target Azure Linux, which is only supported from 6.x onwards by forcing level-up to 6.0.0.
+            LOG.info(
+                f"Bump up the tag to 6.x: this_release_branch_only={this_release_branch_only}, branch={branch}, latest_tag={latest_tag}"
+            )
+            tags = sorted(
+                (
+                    tag
+                    for tag in self.tags
+                    if get_version_from_tag_name(tag) >= Version("6.0.0.dev0")
+                ),
+                key=get_version_from_tag_name,
+                reverse=True,
+            )
+            latest_tag = tags[0]
+
+        return self.install_release(latest_tag, platform)
+
+    def install_next_lts_for_branch(self, branch, platform="snp"):
         next_tag = self.get_first_tag_for_next_release_branch(branch)
         return self.install_release(next_tag, platform) if next_tag else (None, None)