fix(ci): skip published cpan libs and fix promote-deb index read (backport master)

.github/actions/promote-to-stable-pulp/promote-deb.sh

@@ -68,11 +68,13 @@ suite_sha_file() {
 # resolve the served filename from the testing suite's Packages file by
 # sha256 (the published pool layout doesn't match the upload relative_path)
 download_testing_package() {
-  local sha256=$1 arch=$2 dest=$3 filename
-  filename=$(
-    content_curl -fsSL --retry 3 --retry-delay 5 "$PULP_CONTENT_URL/${LEGACY_TESTING_BASE_PATH:-$TESTING_DOMAIN/$BASE_PATH}/dists/$TESTING_SUITE/main/binary-$arch/Packages" |
-      awk -v sha="$sha256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } }'
-  )
+  local sha256=$1 arch=$2 dest=$3 filename packages_file
+  # read the index from a file: awk exits on the first match, which would
+  # SIGPIPE a piped curl on a large index (curl 23, fatal under pipefail)
+  packages_file=$(mktemp)
+  content_curl -fsSL --retry 3 --retry-delay 5 -o "$packages_file" "$PULP_CONTENT_URL/${LEGACY_TESTING_BASE_PATH:-$TESTING_DOMAIN/$BASE_PATH}/dists/$TESTING_SUITE/main/binary-$arch/Packages"
+  filename=$(awk -v sha="$sha256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } }' "$packages_file")
+  rm -f "$packages_file"
   if [[ -z "$filename" ]]; then
     echo "::error::Cannot locate the published file for sha256 $sha256 in $TESTING_SUITE ($arch)" >&2
     return 1

.github/actions/test-cpan-libs/action.yml

@@ -37,21 +37,26 @@ runs:
         curl -sSL $GPG_KEY_URL -o RPM-GPG-KEY-CES
         rpm --import RPM-GPG-KEY-CES
         # Add Centreon repository matching the current stability so that pre-existing packaged dependencies are available
-        {
-          echo "[centreon-plugins-${REPO_STABILITY}]"
-          echo "name=centreon plugins ${REPO_STABILITY} x86_64"
-          echo "baseurl=https://packages.centreon.com/rpm-plugins/${DISTRIB}/${REPO_STABILITY}/x86_64"
-          echo 'enabled=1'
-          echo 'gpgcheck=1'
-          echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES'
-          echo ''
-          echo "[centreon-plugins-${REPO_STABILITY}-noarch]"
-          echo "name=centreon plugins ${REPO_STABILITY} noarch"
-          echo "baseurl=https://packages.centreon.com/rpm-plugins/${DISTRIB}/${REPO_STABILITY}/noarch"
-          echo 'enabled=1'
-          echo 'gpgcheck=1'
-          echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES'
-        } > /etc/yum.repos.d/centreon-plugins-${REPO_STABILITY}.repo
+        # testing only holds the libs rebuilt for the release: unchanged dependencies come from stable
+        REPO_CHANNELS="$REPO_STABILITY"
+        [[ "$REPO_STABILITY" == "testing" ]] && REPO_CHANNELS="testing stable"
+        for CHANNEL in $REPO_CHANNELS; do
+          {
+            echo "[centreon-plugins-${CHANNEL}]"
+            echo "name=centreon plugins ${CHANNEL} x86_64"
+            echo "baseurl=https://packages.centreon.com/rpm-plugins/${DISTRIB}/${CHANNEL}/x86_64"
+            echo 'enabled=1'
+            echo 'gpgcheck=1'
+            echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES'
+            echo ''
+            echo "[centreon-plugins-${CHANNEL}-noarch]"
+            echo "name=centreon plugins ${CHANNEL} noarch"
+            echo "baseurl=https://packages.centreon.com/rpm-plugins/${DISTRIB}/${CHANNEL}/noarch"
+            echo 'enabled=1'
+            echo 'gpgcheck=1'
+            echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES'
+          } > /etc/yum.repos.d/centreon-plugins-${CHANNEL}.repo
+        done
       shell: bash
 
     - if: ${{ inputs.package_extension == 'deb' }}
@@ -70,12 +75,18 @@ runs:
           *) REPO_STABILITY="unstable" ;;
         esac
         # Add Centreon repository matching the current stability so that pre-existing packaged dependencies are available
-        if [[ "$DISTRIB" == "jammy" || "$DISTRIB" == "noble" ]]; then
-          repo="ubuntu-plugins-${REPO_STABILITY}"
-        else
-          repo="apt-plugins-${REPO_STABILITY}"
-        fi
-        echo "deb https://packages.centreon.com/$repo/ $DISTRIB main" | tee /etc/apt/sources.list.d/centreon-plugins.list
+        # testing only holds the libs rebuilt for the release: unchanged dependencies come from stable
+        REPO_CHANNELS="$REPO_STABILITY"
+        [[ "$REPO_STABILITY" == "testing" ]] && REPO_CHANNELS="testing stable"
+        : > /etc/apt/sources.list.d/centreon-plugins.list
+        for CHANNEL in $REPO_CHANNELS; do
+          if [[ "$DISTRIB" == "jammy" || "$DISTRIB" == "noble" ]]; then
+            repo="ubuntu-plugins-${CHANNEL}"
+          else
+            repo="apt-plugins-${CHANNEL}"
+          fi
+          echo "deb https://packages.centreon.com/$repo/ $DISTRIB main" | tee -a /etc/apt/sources.list.d/centreon-plugins.list
+        done
         # Avoid apt to clean packages cache directory
         rm -f /etc/apt/apt.conf.d/docker-clean
         apt-get update

.github/scripts/perl-cpan-libraries/cpan_matrix_lib.py

@@ -5,6 +5,7 @@
   generate-matrices.py     –  merges partial JSONs into final CI matrices
 """
 
+import gzip
 import json
 import re
 import shutil
@@ -161,8 +162,9 @@ def dist_to_deb_package(dist_name, fallback_module_name=""):
 
     Falls back to deriving from fallback_module_name (CPAN module name) when dist_name is empty.
     "ARGV::Struct" → "libargv-struct-perl",  "Libssh::Session" → "libssh-session-perl"
+    "Crypt-Blowfish_PP" → "libcrypt-blowfish-pp-perl"  (no '_' allowed in deb package names)
     """
-    name = (dist_name or fallback_module_name.replace("::", "-")).lower()
+    name = (dist_name or fallback_module_name.replace("::", "-")).lower().replace("_", "-")
     if not name:
         return ""
     return f"{name}-perl" if name.startswith("lib") else f"lib{name}-perl"
@@ -386,3 +388,105 @@ def get_centreon_deb_packages(base_url, distrib, stability, arch="amd64", family
             packages.update(_fetch_packages_index(base_url, repo, distrib, "all"))
         _deb_packages_cache[cache_key] = packages
     return _deb_packages_cache[cache_key]
+
+
+# ── Published (version, revision) lookups, read from the public repo metadata ──
+
+_RPM_REPO_NS = {"repo": "http://linux.duke.edu/metadata/repo"}
+_RPM_COMMON_NS = "{http://linux.duke.edu/metadata/common}"
+
+_published_cache: dict = {}
+
+
+def _fetch_bytes(base_url, path):
+    """GET {base_url}/{path} from an allowed host; return None on any error."""
+    url = f"{base_url}/{path}"
+    for candidate in (base_url, url):
+        parsed = urllib.parse.urlparse(candidate)
+        if parsed.scheme != "https" or parsed.hostname not in _ALLOWED_ARTIFACTORY_HOSTS:
+            print(f"  WARNING: {candidate} is not an allowed Centreon repository URL, skipping.", file=sys.stderr)
+            return None
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "generate-cpan-matrix/1.0"})
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return resp.read()
+    except Exception as exc:
+        print(f"  WARNING: could not fetch {url}: {exc}", file=sys.stderr)
+        return None
+
+
+def get_centreon_rpm_published(base_url, distrib, stability):
+    """Return {cpan_dist_name: {(version, revision), …}} from the RPM repo metadata.
+
+    Every published build is kept (noarch and x86_64), e.g. perl-JSON-Path 1.0.6-2.el9
+      → "JSON-Path": {("1.0.6", "2")}
+    """
+    # imported here: check-official-repos.py also imports this module, in containers without defusedxml
+    import defusedxml.ElementTree as ET
+
+    cache_key = ("rpm", base_url, distrib, stability)
+    if cache_key in _published_cache:
+        return _published_cache[cache_key]
+    result: dict = {}
+    for arch in ("noarch", "x86_64"):
+        repo_path = f"rpm-plugins/{distrib}/{stability}/{arch}"
+        repomd = _fetch_bytes(base_url, f"{repo_path}/repodata/repomd.xml")
+        if repomd is None:
+            continue
+        location = ET.fromstring(repomd).find("repo:data[@type='primary']/repo:location", _RPM_REPO_NS)
+        if location is None:
+            continue
+        primary = _fetch_bytes(base_url, f"{repo_path}/{location.get('href')}")
+        if primary is None:
+            continue
+        for pkg in ET.fromstring(gzip.decompress(primary)).iter(f"{_RPM_COMMON_NS}package"):
+            name = pkg.findtext(f"{_RPM_COMMON_NS}name", "")
+            version = pkg.find(f"{_RPM_COMMON_NS}version")
+            if not name.startswith("perl-") or version is None:
+                continue
+            revision = version.get("rel", "").split(".")[0]
+            result.setdefault(name[len("perl-"):], set()).add((version.get("ver", ""), revision))
+    _published_cache[cache_key] = result
+    return result
+
+
+def get_centreon_deb_published(base_url, distrib, stability, arch="amd64", family="debian"):
+    """Return {pkg_name: {(version, revision), …}} from the DEB Packages indexes ({arch} + all).
+
+    Every published build is kept, e.g. libssh-session-perl 1.1-2+deb12u1
+      → "libssh-session-perl": {("1.1", "2")}
+    """
+    repo = f"ubuntu-plugins-{stability}" if family == "ubuntu" else f"apt-plugins-{stability}"
+    cache_key = ("deb", base_url, repo, distrib, arch)
+    if cache_key in _published_cache:
+        return _published_cache[cache_key]
+    result: dict = {}
+    for index_arch in {arch, "all"}:
+        content = _fetch_bytes(base_url, f"artifactory/{repo}/dists/{distrib}/main/binary-{index_arch}/Packages")
+        if content is None:
+            continue
+        for stanza in content.decode("utf-8", errors="replace").split("\n\n"):
+            fields = dict(line.split(": ", 1) for line in stanza.splitlines() if ": " in line)
+            upstream = _DEB_VERSION_RE.match(re.sub(r"^\d+:", "", fields.get("Version", "").strip()))
+            if "Package" not in fields or not upstream:
+                continue
+            # "1.1-2+deb12u1" / "0.06-1-0ubuntu.24.04" → 2 / 1, legacy "0.06+deb12u1-1" → 1,
+            # no revision at all ("0.06") → None, which matches any revision
+            rest = upstream.string[upstream.end():]
+            lead, trail = re.match(r"\d+", rest), re.search(r"-(\d+)$", rest)
+            revision = lead.group(0) if lead else trail.group(1) if trail else None
+            result.setdefault(fields["Package"].strip(), set()).add((upstream.group(1), revision))
+    _published_cache[cache_key] = result
+    return result
+
+
+def published_matches(published, required_version, required_revision=None):
+    """True when one of the published (version, revision) pairs satisfies the requirement.
+
+    A None revision (published or required) is not compared.
+    """
+    return any(
+        versions_match(version, required_version)
+        and (revision is None or required_revision is None or revision == required_revision)
+        for version, revision in published
+    )