diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index d2f1643a..c974414a 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -8,8 +8,9 @@ inputs: description: "The distribution name" required: true cache_key: - description: "The cache key to restore packages" - required: true + description: "The cache key to restore the packages built in the same run" + required: false + default: "" test_type: description: "The type of test to run: dependency, library or stream-connector" required: true @@ -17,10 +18,21 @@ inputs: description: "The cache key of library packages built in the same run, installed before the tested packages instead of the ones from the repositories" required: false default: "" + published_packages: + description: "Space separated list of packages to download from the repositories and test, in addition to the ones restored from cache_key" + required: false + default: "" runs: using: "composite" steps: + - if: ${{ inputs.cache_key == '' && inputs.published_packages == '' }} + name: Check inputs + run: | + echo "::error::At least one of the cache_key or published_packages inputs must be set, otherwise no package is tested" + exit 1 + shell: bash + - if: ${{ inputs.package_extension == 'rpm' }} name: Install dependencies and configure Centreon repositories env: @@ -244,13 +256,38 @@ runs: fi shell: bash - - name: Restore packages from cache + - if: ${{ inputs.cache_key != '' }} + name: Restore packages from cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} fail-on-cache-miss: true + - if: ${{ inputs.published_packages != '' }} + name: Download published packages from the repositories + env: + PACKAGE_EXTENSION: ${{ inputs.package_extension }} + PUBLISHED_PACKAGES: ${{ inputs.published_packages }} + run: | + for package in $PUBLISHED_PACKAGES; do + echo "Downloading package: $package" + if [[ "$PACKAGE_EXTENSION" == "rpm" ]]; then + dnf download --destdir=. "$package" || echo "::warning::$package is not published for this distribution, it is not tested" + else + apt-get download "$package" || echo "::warning::$package is not published for this distribution, it is not tested" + fi + done + # name the deb packages like the ones built by nfpm, the test step extracts the package name from the file name + if [[ "$PACKAGE_EXTENSION" == "deb" ]]; then + for package_file in ./*.deb; do + [[ -f "$package_file" ]] || continue + target="./$(dpkg-deb -f "$package_file" Package)_$(dpkg-deb -f "$package_file" Version)_all.deb" + [[ "$package_file" == "$target" ]] || mv "$package_file" "$target" + done + fi + shell: bash + - if: ${{ inputs.package_extension == 'rpm' }} name: Check packages installation / uninstallation env: diff --git a/.github/workflows/stream-connectors-ci.yml b/.github/workflows/stream-connectors-ci.yml index 23198b47..ed445aef 100644 --- a/.github/workflows/stream-connectors-ci.yml +++ b/.github/workflows/stream-connectors-ci.yml @@ -4,29 +4,28 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +# no paths filter: the detect-changes job decides what to run, and the ci-status job +# always reports a result so that it can be used as a required check on: workflow_dispatch: + inputs: + build_lib: + description: "Build and test the library" + type: boolean + default: true + build_connectors: + description: "Build and test all the stream connectors (always done when build_lib is set)" + type: boolean + default: true + deliver_lib: + description: "Deliver the library (requires build_lib)" + type: boolean + default: false pull_request: - paths: - - '.github/workflows/stream-connectors-ci.yml' - - '.github/workflows/stream-connectors-lib.yml' - - 'packaging/connectors-lib/**' - - 'modules/centreon-stream-connectors-lib/**' - - '.github/workflows/stream-connectors.yml' - - 'centreon-certified/**' - - 'packaging/connectors/**' push: branches: - develop - master - paths: - - '.github/workflows/stream-connectors-ci.yml' - - '.github/workflows/stream-connectors-lib.yml' - - 'packaging/connectors-lib/**' - - 'modules/centreon-stream-connectors-lib/**' - - '.github/workflows/stream-connectors.yml' - - 'centreon-certified/**' - - 'packaging/connectors/**' jobs: dependency-scan: @@ -36,26 +35,41 @@ jobs: needs: [dependency-scan] runs-on: ubuntu-24.04 outputs: - # a manual run builds and tests both the library and the stream connectors - lib: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.lib }} - connectors: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.connectors }} + # a manual run builds, tests and delivers what has been asked through its inputs + lib: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_lib && 'true' || 'false') || steps.filter.outputs.lib }} + connectors: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_connectors && 'true' || 'false') || steps.filter.outputs.connectors }} + # CI changes test everything but do not deliver anything + ci: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.ci }} + # the library is only delivered when its code or packaging changed + deliver_lib: ${{ github.event_name == 'workflow_dispatch' && (inputs.build_lib && inputs.deliver_lib && 'true' || 'false') || steps.filter.outputs.lib_code }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: ${{ github.event_name != 'workflow_dispatch' }} - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter with: base: ${{ github.ref }} filters: | - lib: - - .github/workflows/stream-connectors-ci.yml + # this workflow only dispatches the jobs, its own changes do not trigger them + ci: &ci - .github/workflows/stream-connectors-lib.yml - - packaging/connectors-lib/** + - .github/workflows/stream-connectors-lib-delivery.yml + - .github/workflows/stream-connectors.yml + - .github/workflows/stream-connectors-delivery.yml + - .github/workflows/get-environment.yml + - .github/actions/** + lib_code: &lib_code - modules/centreon-stream-connectors-lib/** + - packaging/connectors-lib/** + # the library version is set in its workflow: a version bump must deliver the library + - .github/workflows/stream-connectors-lib.yml + lib: + - *ci + - *lib_code + - tests/packaging/** connectors: - - .github/workflows/stream-connectors-ci.yml - - .github/workflows/stream-connectors.yml + - *ci - centreon-certified/** - packaging/connectors/** @@ -69,18 +83,61 @@ jobs: id-token: write pull-requests: read + # order of the jobs: package lib, test lib, package stream connectors, test stream connectors, + # deliver lib, deliver stream connectors. Nothing is delivered until everything has been tested, + # and the stream connectors are never delivered without the library they depend on. + # when the library has been rebuilt in this run, the stream connectors are tested # against those freshly built packages instead of the ones from the repositories. - # A library change tests all the stream connectors, but only the changed ones are delivered + # - the changed stream connectors are built and tested (then delivered by stream-connectors-delivery) + # - a CI change builds and tests all the stream connectors, without delivering them + # - a library change tests the published packages of the stream connectors that are not built stream-connectors: needs: [detect-changes, stream-connectors-lib] if: ${{ !cancelled() && (needs.detect-changes.outputs.connectors == 'true' || needs.detect-changes.outputs.lib == 'true') && needs.stream-connectors-lib.result != 'failure' }} uses: ./.github/workflows/stream-connectors.yml with: use_local_lib: ${{ needs.stream-connectors-lib.result == 'success' }} - test_all_connectors: ${{ needs.detect-changes.outputs.lib == 'true' }} + build_all_connectors: ${{ needs.detect-changes.outputs.ci == 'true' }} + test_published_connectors: ${{ needs.detect-changes.outputs.lib == 'true' }} + secrets: inherit + permissions: + contents: read + id-token: write + pull-requests: read + + # the library is delivered only once the stream connectors have also been tested against it + stream-connectors-lib-delivery: + needs: [detect-changes, stream-connectors-lib, stream-connectors] + if: ${{ !cancelled() && needs.detect-changes.outputs.deliver_lib == 'true' && needs.stream-connectors-lib.result == 'success' && needs.stream-connectors.result != 'failure' }} + uses: ./.github/workflows/stream-connectors-lib-delivery.yml + secrets: inherit + permissions: + contents: read + id-token: write + pull-requests: read + + # the stream connectors are delivered only once the library they depend on has been delivered (or did not need to be) + stream-connectors-delivery: + needs: [stream-connectors, stream-connectors-lib-delivery] + if: ${{ !cancelled() && needs.stream-connectors.result == 'success' && needs.stream-connectors.outputs.delivered_connectors != '[]' && contains(fromJson('["success", "skipped"]'), needs.stream-connectors-lib-delivery.result) }} + uses: ./.github/workflows/stream-connectors-delivery.yml + with: + connectors: ${{ needs.stream-connectors.outputs.delivered_connectors }} secrets: inherit permissions: contents: read id-token: write pull-requests: read + + # aggregated result of the workflow, meant to be used as the required check + ci-status: + needs: [dependency-scan, detect-changes, stream-connectors-lib, stream-connectors, stream-connectors-lib-delivery, stream-connectors-delivery] + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + - name: Check jobs result + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + run: | + echo "At least one job failed or has been cancelled" + exit 1 diff --git a/.github/workflows/stream-connectors-delivery.yml b/.github/workflows/stream-connectors-delivery.yml new file mode 100644 index 00000000..fcd1a88f --- /dev/null +++ b/.github/workflows/stream-connectors-delivery.yml @@ -0,0 +1,170 @@ +name: stream-connectors-delivery + +# triggered by stream-connectors-ci.yml, once the stream connectors have been tested and the library has been delivered +# the packages are restored from the cache filled by the package job of stream-connectors.yml in the same run +on: + workflow_call: + inputs: + connectors: + description: "JSON list of the stream connectors to deliver" + type: string + required: true + +jobs: + get-environment: + uses: ./.github/workflows/get-environment.yml + + deliver-rpm: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && inputs.connectors != '[]' }} + needs: [get-environment] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [el8, el9, el10] + connector_path: ${{ fromJson(inputs.connectors) }} + name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Publish RPM packages + uses: ./.github/actions/rpm-delivery + with: + module_name: stream-connectors + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-${{ matrix.connector_path }}-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && inputs.connectors != '[]' }} + needs: [get-environment] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [bookworm, trixie] + connector_path: ${{ fromJson(inputs.connectors) }} + name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Publish DEB packages + uses: ./.github/actions/deb-delivery + with: + module_name: stream-connectors + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-${{ matrix.connector_path }}-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + # pulp mirrors the artifactory delivery above; + # stable is a promotion of the testing packages, keyed on the release/hotfix branch + deliver-pulp: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && inputs.connectors != '[]' }} + needs: [get-environment] + runs-on: centreon-ubuntu-26.04-small + permissions: + contents: read + id-token: write + pull-requests: read + strategy: + fail-fast: false + matrix: + distrib: [el8, el9, el10, bookworm, trixie] + connector_path: ${{ fromJson(inputs.connectors) }} + include: + - distrib: el8 + package_extension: rpm + - distrib: el9 + package_extension: rpm + - distrib: el10 + package_extension: rpm + - distrib: bookworm + package_extension: deb + - distrib: trixie + package_extension: deb + name: deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} on pulp + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Get release type + if: ${{ needs.get-environment.outputs.stability == 'stable' }} + id: get_release_type + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const commitSha = context.sha; + + const pulls = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + sort: 'updated', + direction: 'desc', + state: 'closed', + per_page: 100 + }); + + core.startGroup(`Checking pull request linked to commit ${commitSha}`); + const pr = pulls.data.find(p => { + console.log(`Checking pull request ${p.number}("${p.title}") with merge commit ${p.merge_commit_sha}`); + return p.merge_commit_sha === commitSha; + }); + core.endGroup(); + if (!pr) { + core.error(`No pull request found for merge commit ${commitSha}`); + core.setOutput('release_type', ''); + return; + } + + const prHeadRef = pr?.head?.ref || 'unknown'; + let releaseType = ''; + switch (true) { + case /^release.+/.test(prHeadRef): + releaseType = 'release'; + break; + case /^hotfix.+/.test(prHeadRef): + releaseType = 'hotfix'; + break; + default: + core.error(`No packages are promoted because push of branch ${prHeadRef} is not related to a hotfix/release pull request.`); + core.setOutput('release_type', ''); + return; + } + console.log(`Release type: ${releaseType}`); + core.setOutput('release_type', releaseType); + + - name: Deliver packages on Pulp + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} + uses: ./.github/actions/package-delivery-pulp + with: + module_name: stream-connectors + repository_name: plugins + distrib: ${{ matrix.distrib }} + version: "" + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.connector_path }}-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + release_type: ${{ needs.get-environment.outputs.release_type }} + is_cloud: "false" + pulp_url: ${{ vars.PULP_API_URL }} + pulp_content_url: ${{ vars.PULP_CONTENT_URL }} + audience: ${{ vars.PULP_OIDC_AUDIENCE }} + + - name: Promote packages on Pulp + if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} + uses: ./.github/actions/promote-to-stable-pulp + with: + module_name: stream-connectors + repository_name: plugins + distrib: ${{ matrix.distrib }} + major_version: "" + stability: stable + release_type: ${{ steps.get_release_type.outputs.release_type }} + is_cloud: "false" + pulp_url: ${{ vars.PULP_API_URL }} + pulp_content_url: ${{ vars.PULP_CONTENT_URL }} + audience: ${{ vars.PULP_OIDC_AUDIENCE }} diff --git a/.github/workflows/stream-connectors-lib-delivery.yml b/.github/workflows/stream-connectors-lib-delivery.yml new file mode 100644 index 00000000..6b490ad2 --- /dev/null +++ b/.github/workflows/stream-connectors-lib-delivery.yml @@ -0,0 +1,162 @@ +name: stream-connectors-lib-delivery + +# triggered by stream-connectors-ci.yml, once the library and the stream connectors have been tested +# the packages are restored from the cache filled by the package job of stream-connectors-lib.yml in the same run +on: + workflow_call: + +jobs: + get-environment: + uses: ./.github/workflows/get-environment.yml + + deliver-rpm: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [el8, el9, el10] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Publish RPM packages + uses: ./.github/actions/rpm-delivery + with: + module_name: stream-connectors-lib + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [bookworm, trixie] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Publish DEB packages + uses: ./.github/actions/deb-delivery + with: + module_name: stream-connectors-lib + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + # pulp mirrors the artifactory delivery above; + # stable is a promotion of the testing packages, keyed on the release/hotfix branch + deliver-pulp: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment] + runs-on: centreon-ubuntu-26.04-small + permissions: + contents: read + id-token: write + pull-requests: read + strategy: + fail-fast: false + matrix: + distrib: [el8, el9, el10, bookworm, trixie] + include: + - distrib: el8 + package_extension: rpm + - distrib: el9 + package_extension: rpm + - distrib: el10 + package_extension: rpm + - distrib: bookworm + package_extension: deb + - distrib: trixie + package_extension: deb + name: deliver ${{ matrix.distrib }} on pulp + + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Get release type + if: ${{ needs.get-environment.outputs.stability == 'stable' }} + id: get_release_type + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const commitSha = context.sha; + + const pulls = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + sort: 'updated', + direction: 'desc', + state: 'closed', + per_page: 100 + }); + + core.startGroup(`Checking pull request linked to commit ${commitSha}`); + const pr = pulls.data.find(p => { + console.log(`Checking pull request ${p.number}("${p.title}") with merge commit ${p.merge_commit_sha}`); + return p.merge_commit_sha === commitSha; + }); + core.endGroup(); + if (!pr) { + core.error(`No pull request found for merge commit ${commitSha}`); + core.setOutput('release_type', ''); + return; + } + + const prHeadRef = pr?.head?.ref || 'unknown'; + let releaseType = ''; + switch (true) { + case /^release.+/.test(prHeadRef): + releaseType = 'release'; + break; + case /^hotfix.+/.test(prHeadRef): + releaseType = 'hotfix'; + break; + default: + core.error(`No packages are promoted because push of branch ${prHeadRef} is not related to a hotfix/release pull request.`); + core.setOutput('release_type', ''); + return; + } + console.log(`Release type: ${releaseType}`); + core.setOutput('release_type', releaseType); + + - name: Deliver packages on Pulp + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} + uses: ./.github/actions/package-delivery-pulp + with: + module_name: stream-connectors-lib + repository_name: plugins + distrib: ${{ matrix.distrib }} + version: "" + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + release_type: ${{ needs.get-environment.outputs.release_type }} + is_cloud: "false" + pulp_url: ${{ vars.PULP_API_URL }} + pulp_content_url: ${{ vars.PULP_CONTENT_URL }} + audience: ${{ vars.PULP_OIDC_AUDIENCE }} + + - name: Promote packages on Pulp + if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} + uses: ./.github/actions/promote-to-stable-pulp + with: + module_name: stream-connectors-lib + repository_name: plugins + distrib: ${{ matrix.distrib }} + major_version: "" + stability: stable + release_type: ${{ steps.get_release_type.outputs.release_type }} + is_cloud: "false" + pulp_url: ${{ vars.PULP_API_URL }} + pulp_content_url: ${{ vars.PULP_CONTENT_URL }} + audience: ${{ vars.PULP_OIDC_AUDIENCE }} diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index f1ae0361..6ed4eb4d 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -1,6 +1,8 @@ name: stream-connectors-lib # triggered by stream-connectors-ci.yml +# only builds and tests the library, its delivery is done by stream-connectors-lib-delivery.yml +# once the stream connectors have also been tested against it on: workflow_call: @@ -51,7 +53,7 @@ jobs: with: nfpm_file_pattern: "packaging/connectors-lib/*.yaml" distrib: ${{ matrix.distrib }} - version: "3.8.1" # previous version:"3.8.0" + version: "3.8.2" # previous version:"3.8.1" release: "1" package_extension: ${{ matrix.package_extension }} arch: all @@ -105,155 +107,3 @@ jobs: with: name: install_error_log_${{ matrix.distrib }} path: install_error_${{ matrix.distrib }}.log - - deliver-rpm: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, package, test-packages] - runs-on: ubuntu-24.04 - strategy: - matrix: - distrib: [el8, el9, el10] - name: deliver ${{ matrix.distrib }} - - steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Publish RPM packages - uses: ./.github/actions/rpm-delivery - with: - module_name: stream-connectors-lib - distrib: ${{ matrix.distrib }} - artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - - deliver-deb: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, package, test-packages] - runs-on: ubuntu-24.04 - strategy: - matrix: - distrib: [bookworm, trixie] - name: deliver ${{ matrix.distrib }} - - steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Publish DEB packages - uses: ./.github/actions/deb-delivery - with: - module_name: stream-connectors-lib - distrib: ${{ matrix.distrib }} - artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - - # pulp mirrors the artifactory delivery above; - # stable is a promotion of the testing packages, keyed on the release/hotfix branch - deliver-pulp: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, package, test-packages] - runs-on: centreon-ubuntu-26.04-small - permissions: - contents: read - id-token: write - pull-requests: read - strategy: - fail-fast: false - matrix: - distrib: [el8, el9, el10, bookworm, trixie] - include: - - distrib: el8 - package_extension: rpm - - distrib: el9 - package_extension: rpm - - distrib: el10 - package_extension: rpm - - distrib: bookworm - package_extension: deb - - distrib: trixie - package_extension: deb - name: deliver ${{ matrix.distrib }} on pulp - - steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Get release type - if: ${{ needs.get-environment.outputs.stability == 'stable' }} - id: get_release_type - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const commitSha = context.sha; - - const pulls = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - sort: 'updated', - direction: 'desc', - state: 'closed', - per_page: 100 - }); - - core.startGroup(`Checking pull request linked to commit ${commitSha}`); - const pr = pulls.data.find(p => { - console.log(`Checking pull request ${p.number}("${p.title}") with merge commit ${p.merge_commit_sha}`); - return p.merge_commit_sha === commitSha; - }); - core.endGroup(); - if (!pr) { - core.error(`No pull request found for merge commit ${commitSha}`); - core.setOutput('release_type', ''); - return; - } - - const prHeadRef = pr?.head?.ref || 'unknown'; - let releaseType = ''; - switch (true) { - case /^release.+/.test(prHeadRef): - releaseType = 'release'; - break; - case /^hotfix.+/.test(prHeadRef): - releaseType = 'hotfix'; - break; - default: - core.error(`No packages are promoted because push of branch ${prHeadRef} is not related to a hotfix/release pull request.`); - core.setOutput('release_type', ''); - return; - } - console.log(`Release type: ${releaseType}`); - core.setOutput('release_type', releaseType); - - - name: Deliver packages on Pulp - if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} - uses: ./.github/actions/package-delivery-pulp - with: - module_name: stream-connectors-lib - repository_name: plugins - distrib: ${{ matrix.distrib }} - version: "" - cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} - is_cloud: "false" - pulp_url: ${{ vars.PULP_API_URL }} - pulp_content_url: ${{ vars.PULP_CONTENT_URL }} - audience: ${{ vars.PULP_OIDC_AUDIENCE }} - - - name: Promote packages on Pulp - if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} - uses: ./.github/actions/promote-to-stable-pulp - with: - module_name: stream-connectors-lib - repository_name: plugins - distrib: ${{ matrix.distrib }} - major_version: "" - stability: stable - release_type: ${{ steps.get_release_type.outputs.release_type }} - is_cloud: "false" - pulp_url: ${{ vars.PULP_API_URL }} - pulp_content_url: ${{ vars.PULP_CONTENT_URL }} - audience: ${{ vars.PULP_OIDC_AUDIENCE }} diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index bc7a715a..7f6d4e4e 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -1,15 +1,25 @@ name: stream-connectors # triggered by stream-connectors-ci.yml +# only builds and tests the stream connectors, their delivery is done by stream-connectors-delivery.yml +# once the library has been delivered on: workflow_call: + outputs: + delivered_connectors: + description: "JSON list of the stream connectors to deliver, only the ones that changed" + value: ${{ jobs.detect-changes.outputs.delivered_connectors }} inputs: use_local_lib: description: "Test the stream connectors against the library packages built earlier in the same run" type: boolean default: false - test_all_connectors: - description: "Build and test all the stream connectors (only the changed ones are delivered)" + build_all_connectors: + description: "Build and test all the stream connectors, used for CI changes (only the changed ones are delivered)" + type: boolean + default: false + test_published_connectors: + description: "Test the published packages of the stream connectors that are not built in this run, used for library changes" type: boolean default: false @@ -24,6 +34,8 @@ jobs: connectors: ${{ steps.list-connectors.outputs.connectors }} # connectors to deliver, only the ones that changed delivered_connectors: ${{ steps.list-connectors.outputs.delivered_connectors }} + # space separated packages of the connectors that are not built, tested from the repositories + published_packages: ${{ steps.list-connectors.outputs.published_packages }} packaging: ${{ steps.filter.outputs.packaging }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -36,6 +48,7 @@ jobs: filters: | connectors: - added|modified: centreon-certified/** + # the dependencies of the stream connectors packages are set in this workflow packaging: - modified: .github/workflows/stream-connectors.yml - added|modified: packaging/connectors/** @@ -45,7 +58,8 @@ jobs: env: CONNECTOR_FILES: ${{ steps.filter.outputs.connectors_files }} PACKAGING_CHANGED: ${{ steps.filter.outputs.packaging }} - TEST_ALL_CONNECTORS: ${{ inputs.test_all_connectors }} + BUILD_ALL_CONNECTORS: ${{ inputs.build_all_connectors }} + TEST_PUBLISHED_CONNECTORS: ${{ inputs.test_published_connectors }} run: | folders=() mapfile -t all_folders < <(find centreon-certified/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) @@ -63,11 +77,23 @@ jobs: unique_folders=($(printf "%s\n" "${folders[@]}" | sort -u | tr '\n' ' ')) echo "delivered_connectors=$(jq --compact-output --null-input '$ARGS.positional' --args -- ${unique_folders[@]})" >> $GITHUB_OUTPUT - if [ "$TEST_ALL_CONNECTORS" = "true" ]; then - echo "Library changed, testing all connectors" + if [ "$BUILD_ALL_CONNECTORS" = "true" ]; then + echo "CI changed, building and testing all connectors" unique_folders=($(printf "%s\n" "${all_folders[@]}" | sort -u | tr '\n' ' ')) fi echo "connectors=$(jq --compact-output --null-input '$ARGS.positional' --args -- ${unique_folders[@]})" >> $GITHUB_OUTPUT + + # the connectors that are not built in this run are tested with their published packages + published_packages=() + if [ "$TEST_PUBLISHED_CONNECTORS" = "true" ]; then + for folder in "${all_folders[@]}"; do + if [[ ! " ${unique_folders[*]} " =~ " ${folder} " ]]; then + published_packages+=("centreon-stream-connector-${folder}") + fi + done + echo "Library changed, testing published packages: ${published_packages[*]}" + fi + echo "published_packages=${published_packages[*]}" >> $GITHUB_OUTPUT shell: bash package: @@ -225,157 +251,50 @@ jobs: name: install_error_log_${{ matrix.connector_path }}_${{ matrix.distrib }} path: install_error_${{ matrix.distrib }}.log - deliver-rpm: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }} - needs: [get-environment, detect-changes, package, test-packages] - runs-on: ubuntu-24.04 - strategy: - matrix: - distrib: [el8, el9, el10] - connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }} - name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} - - steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Publish RPM packages - uses: ./.github/actions/rpm-delivery - with: - module_name: stream-connectors - distrib: ${{ matrix.distrib }} - artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-${{ matrix.connector_path }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - - deliver-deb: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }} - needs: [get-environment, detect-changes, package, test-packages] - runs-on: ubuntu-24.04 - strategy: - matrix: - distrib: [bookworm, trixie] - connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }} - name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} - - steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Publish DEB packages - uses: ./.github/actions/deb-delivery - with: - module_name: stream-connectors - distrib: ${{ matrix.distrib }} - artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-${{ matrix.connector_path }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - - # pulp mirrors the artifactory delivery above; - # stable is a promotion of the testing packages, keyed on the release/hotfix branch - deliver-pulp: - if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) && needs.detect-changes.outputs.delivered_connectors != '[]' }} - needs: [get-environment, detect-changes, package, test-packages] - runs-on: centreon-ubuntu-26.04-small - permissions: - contents: read - id-token: write - pull-requests: read + # the published packages of the connectors are tested against the library built in this run + test-published-packages: + if: ${{ needs.detect-changes.outputs.published_packages != '' }} + needs: [detect-changes] strategy: fail-fast: false matrix: distrib: [el8, el9, el10, bookworm, trixie] - connector_path: ${{ fromJson(needs.detect-changes.outputs.delivered_connectors) }} include: - distrib: el8 package_extension: rpm + image: almalinux:8 - distrib: el9 package_extension: rpm + image: almalinux:9 - distrib: el10 package_extension: rpm + image: almalinux:10 - distrib: bookworm package_extension: deb + image: debian:bookworm - distrib: trixie package_extension: deb - name: deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} on pulp - + image: debian:trixie + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + name: Test published ${{ matrix.distrib }} steps: - - name: Checkout sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Get release type - if: ${{ needs.get-environment.outputs.stability == 'stable' }} - id: get_release_type - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const commitSha = context.sha; - - const pulls = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - sort: 'updated', - direction: 'desc', - state: 'closed', - per_page: 100 - }); - - core.startGroup(`Checking pull request linked to commit ${commitSha}`); - const pr = pulls.data.find(p => { - console.log(`Checking pull request ${p.number}("${p.title}") with merge commit ${p.merge_commit_sha}`); - return p.merge_commit_sha === commitSha; - }); - core.endGroup(); - if (!pr) { - core.error(`No pull request found for merge commit ${commitSha}`); - core.setOutput('release_type', ''); - return; - } - - const prHeadRef = pr?.head?.ref || 'unknown'; - let releaseType = ''; - switch (true) { - case /^release.+/.test(prHeadRef): - releaseType = 'release'; - break; - case /^hotfix.+/.test(prHeadRef): - releaseType = 'hotfix'; - break; - default: - core.error(`No packages are promoted because push of branch ${prHeadRef} is not related to a hotfix/release pull request.`); - core.setOutput('release_type', ''); - return; - } - console.log(`Release type: ${releaseType}`); - core.setOutput('release_type', releaseType); + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Deliver packages on Pulp - if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-environment.outputs.stability) }} - uses: ./.github/actions/package-delivery-pulp + - name: Test published connectors + uses: ./.github/actions/test-packages with: - module_name: stream-connectors - repository_name: plugins + package_extension: ${{ matrix.package_extension }} distrib: ${{ matrix.distrib }} - version: "" - cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.connector_path }}-${{ matrix.distrib }} - stability: ${{ needs.get-environment.outputs.stability }} - release_type: ${{ needs.get-environment.outputs.release_type }} - is_cloud: "false" - pulp_url: ${{ vars.PULP_API_URL }} - pulp_content_url: ${{ vars.PULP_CONTENT_URL }} - audience: ${{ vars.PULP_OIDC_AUDIENCE }} + published_packages: ${{ needs.detect-changes.outputs.published_packages }} + # must match the cache_key of the package job in stream-connectors-lib.yml + lib_cache_key: ${{ inputs.use_local_lib && format('{0}-{1}-{2}-{3}', github.sha, github.run_id, matrix.package_extension, matrix.distrib) || '' }} + test_type: stream-connector - - name: Promote packages on Pulp - if: ${{ needs.get-environment.outputs.stability == 'stable' && steps.get_release_type.outputs.release_type != '' }} - uses: ./.github/actions/promote-to-stable-pulp + - name: Upload error log + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - module_name: stream-connectors - repository_name: plugins - distrib: ${{ matrix.distrib }} - major_version: "" - stability: stable - release_type: ${{ steps.get_release_type.outputs.release_type }} - is_cloud: "false" - pulp_url: ${{ vars.PULP_API_URL }} - pulp_content_url: ${{ vars.PULP_CONTENT_URL }} - audience: ${{ vars.PULP_OIDC_AUDIENCE }} + name: install_error_log_published_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log diff --git a/packaging/connectors/centreon-stream-connectors.yaml b/packaging/connectors/centreon-stream-connectors.yaml index 0e990b0d..07d86634 100644 --- a/packaging/connectors/centreon-stream-connectors.yaml +++ b/packaging/connectors/centreon-stream-connectors.yaml @@ -21,12 +21,12 @@ contents: overrides: rpm: depends: [ - centreon-stream-connectors-lib >= 3.8.0, + centreon-stream-connectors-lib >= 3.8.2, @RPM_DEPENDENCIES@ ] deb: depends: [ - "centreon-stream-connectors-lib (>= 3.8.0~)", + "centreon-stream-connectors-lib (>= 3.8.2~)", @DEB_DEPENDENCIES@ ] rpm: