Expand release-information with history of releases.json files

Ticket: ENT-13001

Author: sarakthon

src/cfengine_cli/masterfiles/analyze.py

@@ -91,3 +91,36 @@ def finalize_vcf(versions_dict, checksums_dict, files_dict):
     )
 
     return versions_dict, checksums_dict, files_dict
+
+
+def filter_unstable_releases(data):
+    # Filter the data to only include stable releases (not debug, alpha, or beta releases):
+    filtered_data = []
+
+    for release_data in data.get("releases", []):
+        if release_data.get("debug") is True:
+            continue
+        if release_data.get("alpha") is True:
+            continue
+        if release_data.get("beta") is True:
+            continue
+
+        filtered_data.append(release_data)
+
+    return filtered_data
+
+
+def sort_release_data(file_checksums_dict):
+    # Newest versions first, and files sorted alphabetically within each version
+    for v in file_checksums_dict.keys():
+        file_checksums_dict[v] = dict_sorted_by_key(file_checksums_dict[v])
+
+    sorted_dict = OrderedDict(
+        sorted(
+            file_checksums_dict.items(),
+            key=lambda p: version_as_comparable_list(p[0]),
+            reverse=True,
+        )
+    )
+
+    return sorted_dict

src/cfengine_cli/masterfiles/generate_release_information.py

@@ -1,10 +1,21 @@
-from cfengine_cli.masterfiles.download import download_all_versions
+from cfengine_cli.masterfiles.download import (
+    ENTERPRISE_RELEASES_URL,
+    download_all_versions,
+)
 from cfengine_cli.masterfiles.generate_vcf_download import generate_vcf_download
+from cfengine_cli.masterfiles.analyze import filter_unstable_releases, sort_release_data
 from cfengine_cli.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout
 from cfengine_cli.masterfiles.check_download_matches_git import (
     check_download_matches_git,
 )
-from cfbs.utils import immediate_subdirectories, version_is_at_least
+from cfbs.utils import (
+    get_json,
+    immediate_subdirectories,
+    version_is_at_least,
+    write_json,
+    CFBSNetworkError,
+    CFBSExitError,
+)
 
 DOWNLOAD_PATH = "downloaded_masterfiles"
 
@@ -33,6 +44,8 @@ def generate_release_information_impl(
     )
     generate_vcf_download(DOWNLOAD_PATH, downloaded_versions)
 
+    generate_release_history()
+
     if check:
         print(
             "Downloading releases of masterfiles from git (github.com) and generating "
@@ -52,3 +65,85 @@ def generate_release_information_impl(
             "(Run again with --check-against-git to download and compare with files "
             + "from git, and generate -git.json files)"
         )
+
+
+def generate_release_history():
+    print("Generating release history information...")
+
+    releases_data = download_releasedata()
+
+    stable_releases = filter_unstable_releases(releases_data)
+
+    file_checksums_dict = build_release_history(stable_releases)
+
+    sorted_releases = sort_release_data(file_checksums_dict)
+
+    write_json("./cfengine-enterprise/checksums.json", sorted_releases)
+
+
+def download_releasedata():
+    # Downloading releases.json:
+    try:
+        releases_data = get_json(ENTERPRISE_RELEASES_URL)
+
+    except CFBSNetworkError:
+        raise CFBSExitError(
+            "Downloading CFEngine release data failed - check your Wi-Fi / network settings."
+        )
+
+    return releases_data
+
+
+def build_release_history(filtered_releases):
+    release_history = {}
+
+    for release_data in filtered_releases:
+        if not release_data.get("version") or not release_data.get("URL"):
+            continue
+
+        subdata, version = download_release_version_data(release_data)
+        version_files = extract_version_files(subdata)
+
+        if version_files:
+            release_history[version] = version_files
+
+    return release_history
+
+
+def download_release_version_data(release_data):
+    # Downloads each versionnumber.json in releases.json
+    version = release_data.get("version")
+    url = release_data.get("URL")
+
+    try:
+        return get_json(url), version
+    except CFBSNetworkError:
+        raise CFBSExitError(
+            f"Downloading CFEngine release data for version {version} failed - check your Wi-Fi / network settings."
+        )
+
+
+def extract_version_files(subdata):
+    # Gets filenames and checksums for each file in the subdata of releases.json:
+    artifacts = subdata.get("artifacts", {})
+    version_files = {}
+
+    for asset_list in artifacts.values():
+        for asset_data in asset_list:
+            filename, checksum = extract_file_info(asset_data)
+
+            if filename and checksum:
+                version_files[filename] = checksum
+
+    return version_files
+
+
+def extract_file_info(asset_data):
+    url = asset_data.get("URL")
+    checksum = asset_data.get("SHA256")
+
+    if url and checksum:
+        filename = url.split("/")[-1]
+        return filename, checksum
+
+    return None, None