enh(ci): fixes after #389 review

.github/actions/test-packages/action.yml

@@ -8,19 +8,31 @@ inputs:
     description: "The distribution name"
     required: true
   cache_key:
-    description: "The cache key to restore packages"
-    required: true
+    description: "The cache key to restore the packages built in the same run"
+    required: false
+    default: ""
   test_type:
     description: "The type of test to run: dependency, library or stream-connector"
     required: true
   lib_cache_key:
     description: "The cache key of library packages built in the same run, installed before the tested packages instead of the ones from the repositories"
     required: false
     default: ""
+  published_packages:
+    description: "Space separated list of packages to download from the repositories and test, in addition to the ones restored from cache_key"
+    required: false
+    default: ""
 
 runs:
   using: "composite"
   steps:
+    - if: ${{ inputs.cache_key == '' && inputs.published_packages == '' }}
+      name: Check inputs
+      run: |
+        echo "::error::At least one of the cache_key or published_packages inputs must be set, otherwise no package is tested"
+        exit 1
+      shell: bash
+
     - if: ${{ inputs.package_extension == 'rpm' }}
       name: Install dependencies and configure Centreon repositories
       env:
@@ -244,13 +256,38 @@ runs:
         fi
       shell: bash
 
-    - name: Restore packages from cache
+    - if: ${{ inputs.cache_key != '' }}
+      name: Restore packages from cache
       uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: ./*.${{ inputs.package_extension }}
         key: ${{ inputs.cache_key }}
         fail-on-cache-miss: true
 
+    - if: ${{ inputs.published_packages != '' }}
+      name: Download published packages from the repositories
+      env:
+        PACKAGE_EXTENSION: ${{ inputs.package_extension }}
+        PUBLISHED_PACKAGES: ${{ inputs.published_packages }}
+      run: |
+        for package in $PUBLISHED_PACKAGES; do
+          echo "Downloading package: $package"
+          if [[ "$PACKAGE_EXTENSION" == "rpm" ]]; then
+            dnf download --destdir=. "$package" || echo "::warning::$package is not published for this distribution, it is not tested"
+          else
+            apt-get download "$package" || echo "::warning::$package is not published for this distribution, it is not tested"
+          fi
+        done
+        # name the deb packages like the ones built by nfpm, the test step extracts the package name from the file name
+        if [[ "$PACKAGE_EXTENSION" == "deb" ]]; then
+          for package_file in ./*.deb; do
+            [[ -f "$package_file" ]] || continue
+            target="./$(dpkg-deb -f "$package_file" Package)_$(dpkg-deb -f "$package_file" Version)_all.deb"
+            [[ "$package_file" == "$target" ]] || mv "$package_file" "$target"
+          done
+        fi
+      shell: bash
+
     - if: ${{ inputs.package_extension == 'rpm' }}
       name: Check packages installation / uninstallation
       env:

.github/workflows/stream-connectors-ci.yml

@@ -4,29 +4,28 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+# no paths filter: the detect-changes job decides what to run, and the ci-status job
+# always reports a result so that it can be used as a required check
 on:
   workflow_dispatch:
+    inputs:
+      build_lib:
+        description: "Build and test the library"
+        type: boolean
+        default: true
+      build_connectors:
+        description: "Build and test all the stream connectors (always done when build_lib is set)"
+        type: boolean
+        default: true
+      deliver_lib:
+        description: "Deliver the library (requires build_lib)"
+        type: boolean
+        default: false
   pull_request:
-    paths:
-      - '.github/workflows/stream-connectors-ci.yml'
-      - '.github/workflows/stream-connectors-lib.yml'
-      - 'packaging/connectors-lib/**'
-      - 'modules/centreon-stream-connectors-lib/**'
-      - '.github/workflows/stream-connectors.yml'
-      - 'centreon-certified/**'
-      - 'packaging/connectors/**'
   push:
     branches:
       - develop
       - master
-    paths:
-      - '.github/workflows/stream-connectors-ci.yml'
-      - '.github/workflows/stream-connectors-lib.yml'
-      - 'packaging/connectors-lib/**'
-      - 'modules/centreon-stream-connectors-lib/**'
-      - '.github/workflows/stream-connectors.yml'
-      - 'centreon-certified/**'
-      - 'packaging/connectors/**'
 
 jobs:
   dependency-scan:
@@ -36,26 +35,41 @@ jobs:
     needs: [dependency-scan]
     runs-on: ubuntu-24.04
     outputs:
-      # a manual run builds and tests both the library and the stream connectors
-      lib: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.lib }}
-      connectors: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.connectors }}
+      # a manual run builds, tests and delivers what has been asked through its inputs
+      lib: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_lib && 'true' || 'false') || steps.filter.outputs.lib }}
+      connectors: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_connectors && 'true' || 'false') || steps.filter.outputs.connectors }}
+      # CI changes test everything but do not deliver anything
+      ci: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.ci }}
+      # the library is only delivered when its code or packaging changed
+      deliver_lib: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_lib && inputs.deliver_lib && 'true' || 'false') || steps.filter.outputs.lib_code }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - if: ${{ github.event_name != 'workflow_dispatch' }}
-        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
         id: filter
         with:
           base: ${{ github.ref }}
           filters: |
-            lib:
-              - .github/workflows/stream-connectors-ci.yml
+            # this workflow only dispatches the jobs, its own changes do not trigger them
+            ci: &ci
               - .github/workflows/stream-connectors-lib.yml
-              - packaging/connectors-lib/**
+              - .github/workflows/stream-connectors-lib-delivery.yml
+              - .github/workflows/stream-connectors.yml
+              - .github/workflows/stream-connectors-delivery.yml
+              - .github/workflows/get-environment.yml
+              - .github/actions/**
+            lib_code: &lib_code
               - modules/centreon-stream-connectors-lib/**
+              - packaging/connectors-lib/**
+              # the library version is set in its workflow: a version bump must deliver the library
+              - .github/workflows/stream-connectors-lib.yml
+            lib:
+              - *ci
+              - *lib_code
+              - tests/packaging/**
             connectors:
-              - .github/workflows/stream-connectors-ci.yml
-              - .github/workflows/stream-connectors.yml
+              - *ci
               - centreon-certified/**
               - packaging/connectors/**
 
@@ -69,18 +83,61 @@ jobs:
       id-token: write
       pull-requests: read
 
+  # order of the jobs: package lib, test lib, package stream connectors, test stream connectors,
+  # deliver lib, deliver stream connectors. Nothing is delivered until everything has been tested,
+  # and the stream connectors are never delivered without the library they depend on.
+
   # when the library has been rebuilt in this run, the stream connectors are tested
   # against those freshly built packages instead of the ones from the repositories.
-  # A library change tests all the stream connectors, but only the changed ones are delivered
+  # - the changed stream connectors are built and tested (then delivered by stream-connectors-delivery)
+  # - a CI change builds and tests all the stream connectors, without delivering them
+  # - a library change tests the published packages of the stream connectors that are not built
   stream-connectors:
     needs: [detect-changes, stream-connectors-lib]
     if: ${{ !cancelled() && (needs.detect-changes.outputs.connectors == 'true' || needs.detect-changes.outputs.lib == 'true') && needs.stream-connectors-lib.result != 'failure' }}
     uses: ./.github/workflows/stream-connectors.yml
     with:
       use_local_lib: ${{ needs.stream-connectors-lib.result == 'success' }}
-      test_all_connectors: ${{ needs.detect-changes.outputs.lib == 'true' }}
+      build_all_connectors: ${{ needs.detect-changes.outputs.ci == 'true' }}
+      test_published_connectors: ${{ needs.detect-changes.outputs.lib == 'true' }}
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: read
+
+  # the library is delivered only once the stream connectors have also been tested against it
+  stream-connectors-lib-delivery:
+    needs: [detect-changes, stream-connectors-lib, stream-connectors]
+    if: ${{ !cancelled() && needs.detect-changes.outputs.deliver_lib == 'true' && needs.stream-connectors-lib.result == 'success' && needs.stream-connectors.result != 'failure' }}
+    uses: ./.github/workflows/stream-connectors-lib-delivery.yml
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: read
+
+  # the stream connectors are delivered only once the library they depend on has been delivered (or did not need to be)
+  stream-connectors-delivery:
+    needs: [stream-connectors, stream-connectors-lib-delivery]
+    if: ${{ !cancelled() && needs.stream-connectors.result == 'success' && needs.stream-connectors.outputs.delivered_connectors != '[]' && contains(fromJson('["success", "skipped"]'), needs.stream-connectors-lib-delivery.result) }}
+    uses: ./.github/workflows/stream-connectors-delivery.yml
+    with:
+      connectors: ${{ needs.stream-connectors.outputs.delivered_connectors }}
     secrets: inherit
     permissions:
       contents: read
       id-token: write
       pull-requests: read
+
+  # aggregated result of the workflow, meant to be used as the required check
+  ci-status:
+    needs: [dependency-scan, detect-changes, stream-connectors-lib, stream-connectors, stream-connectors-lib-delivery, stream-connectors-delivery]
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check jobs result
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          echo "At least one job failed or has been cancelled"
+          exit 1