Add triggers for stream connectors

.github/actions/test-packages/action.yml

@@ -13,6 +13,10 @@ inputs:
   test_type:
     description: "The type of test to run: dependency, library or stream-connector"
     required: true
+  lib_cache_key:
+    description: "The cache key of library packages built in the same run, installed before the tested packages instead of the ones from the repositories"
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -214,6 +218,32 @@ runs:
         apt-get update
       shell: bash
 
+    - if: ${{ inputs.lib_cache_key != '' }}
+      name: Restore library packages from cache
+      uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      with:
+        path: ./*.${{ inputs.package_extension }}
+        key: ${{ inputs.lib_cache_key }}
+        fail-on-cache-miss: true
+
+    # installed explicitly so they are kept by the autoremove run after each tested package
+    - if: ${{ inputs.lib_cache_key != '' }}
+      name: Install library packages
+      env:
+        PACKAGE_EXTENSION: ${{ inputs.package_extension }}
+      run: |
+        # move them aside so that they are not part of the tested packages
+        mkdir -p local-lib
+        mv ./*."$PACKAGE_EXTENSION" local-lib/
+        if [[ "$PACKAGE_EXTENSION" == "rpm" ]]; then
+          dnf install -y ./local-lib/*.rpm
+          rpm -q centreon-stream-connectors-lib
+        else
+          apt-get install -y ./local-lib/*.deb
+          dpkg -s centreon-stream-connectors-lib | grep -E '^(Package|Version):'
+        fi
+      shell: bash
+
     - name: Restore packages from cache
       uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:

.github/workflows/stream-connectors-ci.yml

@@ -0,0 +1,86 @@
+name: stream-connectors-ci
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/stream-connectors-ci.yml'
+      - '.github/workflows/stream-connectors-lib.yml'
+      - 'packaging/connectors-lib/**'
+      - 'modules/centreon-stream-connectors-lib/**'
+      - '.github/workflows/stream-connectors.yml'
+      - 'centreon-certified/**'
+      - 'packaging/connectors/**'
+  push:
+    branches:
+      - develop
+      - master
+    paths:
+      - '.github/workflows/stream-connectors-ci.yml'
+      - '.github/workflows/stream-connectors-lib.yml'
+      - 'packaging/connectors-lib/**'
+      - 'modules/centreon-stream-connectors-lib/**'
+      - '.github/workflows/stream-connectors.yml'
+      - 'centreon-certified/**'
+      - 'packaging/connectors/**'
+
+jobs:
+  dependency-scan:
+    uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main
+
+  detect-changes:
+    needs: [dependency-scan]
+    runs-on: ubuntu-24.04
+    outputs:
+      # a manual run builds and tests both the library and the stream connectors
+      lib: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.lib }}
+      connectors: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.connectors }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          base: ${{ github.ref }}
+          filters: |
+            lib:
+              - .github/workflows/stream-connectors-ci.yml
+              - .github/workflows/stream-connectors-lib.yml
+              - packaging/connectors-lib/**
+              - modules/centreon-stream-connectors-lib/**
+            connectors:
+              - .github/workflows/stream-connectors-ci.yml
+              - .github/workflows/stream-connectors.yml
+              - centreon-certified/**
+              - packaging/connectors/**
+
+  stream-connectors-lib:
+    needs: [detect-changes]
+    if: ${{ needs.detect-changes.outputs.lib == 'true' }}
+    uses: ./.github/workflows/stream-connectors-lib.yml
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: read
+
+  # when the library has been rebuilt in this run, the stream connectors are tested
+  # against those freshly built packages instead of the ones from the repositories.
+  # A library change tests all the stream connectors, but only the changed ones are delivered
+  stream-connectors:
+    needs: [detect-changes, stream-connectors-lib]
+    if: ${{ !cancelled() && (needs.detect-changes.outputs.connectors == 'true' || needs.detect-changes.outputs.lib == 'true') && needs.stream-connectors-lib.result != 'failure' }}
+    uses: ./.github/workflows/stream-connectors.yml
+    with:
+      use_local_lib: ${{ needs.stream-connectors-lib.result == 'success' }}
+      test_all_connectors: ${{ needs.detect-changes.outputs.lib == 'true' }}
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: read

.github/workflows/stream-connectors-lib.yml

@@ -1,31 +1,11 @@
 name: stream-connectors-lib
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
+# triggered by stream-connectors-ci.yml
 on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - '.github/workflows/stream-connectors-lib.yml'
-      - 'packaging/connectors-lib/**'
-      - 'modules/centreon-stream-connectors-lib/**'
-  push:
-    branches:
-      - develop
-      - master
-    paths:
-      - '.github/workflows/stream-connectors-lib.yml'
-      - 'packaging/connectors-lib/**'
-      - 'modules/centreon-stream-connectors-lib/**'
+  workflow_call:
 
 jobs:
-  dependency-scan:
-    uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main
-
   get-environment:
-    needs: [dependency-scan]
     uses: ./.github/workflows/get-environment.yml
 
   package:

.github/workflows/stream-connectors.yml

@@ -1,37 +1,29 @@
 name: stream-connectors
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
+# triggered by stream-connectors-ci.yml
 on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - '.github/workflows/stream-connectors.yml'
-      - 'centreon-certified/**'
-      - 'packaging/connectors/**'
-  push:
-    branches:
-      - develop
-      - master
-    paths:
-      - '.github/workflows/stream-connectors.yml'
-      - 'centreon-certified/**'
-      - 'packaging/connectors/**'
+  workflow_call:
+    inputs:
+      use_local_lib:
+        description: "Test the stream connectors against the library packages built earlier in the same run"
+        type: boolean
+        default: false
+      test_all_connectors:
+        description: "Build and test all the stream connectors (only the changed ones are delivered)"
+        type: boolean
+        default: false
 
 jobs:
-  dependency-scan:
-    uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main
-
   get-environment:
-    needs: [dependency-scan]
     uses: ./.github/workflows/get-environment.yml
 
   detect-changes:
     runs-on: ubuntu-24.04
     outputs:
+      # connectors to build and test
       connectors: ${{ steps.list-connectors.outputs.connectors }}
+      # connectors to deliver, only the ones that changed
+      delivered_connectors: ${{ steps.list-connectors.outputs.delivered_connectors }}
       packaging: ${{ steps.filter.outputs.packaging }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -53,11 +45,13 @@ jobs:
         env:
           CONNECTOR_FILES: ${{ steps.filter.outputs.connectors_files }}
           PACKAGING_CHANGED: ${{ steps.filter.outputs.packaging }}
+          TEST_ALL_CONNECTORS: ${{ inputs.test_all_connectors }}
         run: |
           folders=()
+          mapfile -t all_folders < <(find centreon-certified/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
           if [ "$PACKAGING_CHANGED" = "true" ]; then
             echo "Packaging files changed, adding all connectors"
-            mapfile -t folders < <(find centreon-certified/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
+            folders=("${all_folders[@]}")
           else
             for f in $CONNECTOR_FILES; do
               DIR_NAME="$(dirname "$f")"
@@ -67,6 +61,12 @@ jobs:
             done
           fi
           unique_folders=($(printf "%s\n" "${folders[@]}" | sort -u | tr '\n' ' '))
+          echo "delivered_connectors=$(jq --compact-output --null-input '$ARGS.positional' --args -- ${unique_folders[@]})" >> $GITHUB_OUTPUT
+
+          if [ "$TEST_ALL_CONNECTORS" = "true" ]; then
+            echo "Library changed, testing all connectors"
+            unique_folders=($(printf "%s\n" "${all_folders[@]}" | sort -u | tr '\n' ' '))
+          fi
           echo "connectors=$(jq --compact-output --null-input '$ARGS.positional' --args -- ${unique_folders[@]})" >> $GITHUB_OUTPUT
         shell: bash
 
@@ -214,6 +214,8 @@ jobs:
           package_extension: ${{ matrix.package_extension }}
           distrib: ${{ matrix.distrib }}
           cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.connector_path }}-${{ matrix.distrib }}
+          # must match the cache_key of the package job in stream-connectors-lib.yml
+          lib_cache_key: ${{ inputs.use_local_lib && format('{0}-{1}-{2}-{3}', github.sha, github.run_id, matrix.package_extension, matrix.distrib) || '' }}
           test_type: stream-connector
 
       - name: Upload error log
@@ -224,13 +226,13 @@ jobs:
           path: install_error_${{ matrix.distrib }}.log
 
   deliver-rpm:
-    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }}
+    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }}
     needs: [get-environment, detect-changes, package, test-packages]
     runs-on: ubuntu-24.04
     strategy:
       matrix:
         distrib: [el8, el9, el10]
-        connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }}
+        connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }}
     name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }}
 
     steps:
@@ -247,13 +249,13 @@ jobs:
           stability: ${{ needs.get-environment.outputs.stability }}
 
   deliver-deb:
-    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }}
+    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }}
     needs: [get-environment, detect-changes, package, test-packages]
     runs-on: ubuntu-24.04
     strategy:
       matrix:
         distrib: [bookworm, trixie]
-        connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }}
+        connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }}
     name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }}
 
     steps:
@@ -272,7 +274,7 @@ jobs:
   # pulp mirrors the artifactory delivery above;
   # stable is a promotion of the testing packages, keyed on the release/hotfix branch
   deliver-pulp:
-    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }}
+    if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }}
     needs: [get-environment, detect-changes, package, test-packages]
     runs-on: centreon-ubuntu-22.04
     permissions:
@@ -283,7 +285,7 @@ jobs:
       fail-fast: false
       matrix:
         distrib: [el8, el9, el10, bookworm, trixie]
-        connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }}
+        connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }}
         include:
           - distrib: el8
             package_extension: rpm