From 3e7d32810c0f9907e50255370966dd7db38c850a Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 6 Jun 2025 12:34:53 +0100 Subject: [PATCH 1/4] CI: Add sbom generation to trivy image scans --- tools/scan-images.sh | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/scan-images.sh b/tools/scan-images.sh index f119851c35..6bacede068 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -17,8 +17,8 @@ fi # Clear any previous outputs rm -rf image-scan-output -# Make a fresh output directory -mkdir -p image-scan-output +# Make fresh output directories +mkdir -p image-scan-output image-sboms # Get built container images docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" > $1-scanned-container-images.txt @@ -40,6 +40,7 @@ for image in $images; do global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) touch .trivyignore + mkdir -p image-scan-output/$filename for vulnerability in $global_vulnerabilities; do echo $vulnerability >> .trivyignore done @@ -52,7 +53,7 @@ for image in $images; do --scanners vuln \ --format json \ --severity HIGH,CRITICAL \ - --output image-scan-output/${filename}.json \ + --output image-scan-output/${filename}/${filename}.json \ --ignore-unfixed \ --db-repository ghcr.io/aquasecurity/trivy-db:2 \ --db-repository public.ecr.aws/aquasecurity/trivy-db \ @@ -60,14 +61,14 @@ for image in $images; do --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ $image); then # Clean up the output file for any images with no vulnerabilities - rm -f image-scan-output/${filename}.json + rm -f image-scan-output/${filename}/${filename}.json # Add the image to the clean list echo "${image}" >> image-scan-output/clean-images.txt else # Write a header for the summary CSV - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}/${filename}.summary.csv # Write the summary CSV data jq -r '.Results[] @@ -88,9 +89,9 @@ for image in $images; do ] ) | .[] - | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv + | @csv' image-scan-output/${filename}/${filename}.json >> image-scan-output/${filename}/${filename}.summary.csv - if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then + if [ $(grep "CRITICAL" image-scan-output/${filename}/${filename}.summary.csv -c) -gt 0 ]; then # If the image contains critical vulnerabilities, add the image to critical list echo "${image}" >> image-scan-output/critical-images.txt else @@ -98,5 +99,13 @@ for image in $images; do echo "${image}" >> image-scan-output/dirty-images.txt fi fi - rm .trivyignore + trivy image \ + --quiet \ + --format spdx \ + --output image-scan-output/${filename}/${filename}-sbom.spdx \ + --db-repository ghcr.io/aquasecurity/trivy-db:2 \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ + --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ + $image done From 32cc1c45c5ca21517089d2180510a7a33156983f Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Sat, 7 Jun 2025 09:43:51 +0100 Subject: [PATCH 2/4] CI: Refactor container image scanning script --- .../stackhpc-container-image-build.yml | 2 +- tools/scan-images.sh | 231 +++++++++++------- 2 files changed, 141 insertions(+), 92 deletions(-) diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index c8a958d02f..5b6a766936 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -150,7 +150,7 @@ jobs: - name: Install Trivy run: | - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1 - name: Install yq run: | diff --git a/tools/scan-images.sh b/tools/scan-images.sh index 6bacede068..9737172834 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -1,111 +1,160 @@ #!/usr/bin/env bash set -eo pipefail -# Check correct usage -if [[ ! $2 ]]; then - echo "Usage: scan-images.sh " - exit 2 -fi - -set -u +# Global variables +scan_common_args=" \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --db-repository ghcr.io/aquasecurity/trivy-db:2 \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ + --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db " -# Check that trivy is installed -if ! trivy --version; then - echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1' -fi - -# Clear any previous outputs -rm -rf image-scan-output +# Print usage instructions and error with wrong inputs +usage() { + echo "Usage: scan-images.sh [--sbom]" + exit 2 +} -# Make fresh output directories -mkdir -p image-scan-output image-sboms +# Check dependencies are installed, print installation instructions otherwise +check_deps_installed() { + if ! trivy --version > /dev/null; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1' + exit 1 + fi + if ! yq --version > /dev/null; then + echo 'Please install yq: sudo dnf/apt install yq' + exit 1 + fi +} -# Get built container images -docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" > $1-scanned-container-images.txt +# Prepare output files +file_prep() { + rm -rf image-scan-output + mkdir -p image-scan-output + touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt +} -# Make a file of imagename:tag -images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) +# Gather image lists +get_images() { + docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" > $1-scanned-container-images.txt + grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d: +} -# Ensure output files exist -touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt +# Generate ignored vulnerabilities file +generate_trivy_ignore() { + local imagename=$1 + local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) -# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. -# If there are vulnerabilities detected, add it to dirty-images.txt and -# generate a csv summary -# If the image contains at least one critical vulnerabilities, add it to -# critical-images.txt -for image in $images; do - filename=$(basename $image | sed 's/:/\./g') - imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') - global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) - image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) touch .trivyignore - mkdir -p image-scan-output/$filename for vulnerability in $global_vulnerabilities; do echo $vulnerability >> .trivyignore done for vulnerability in $image_vulnerabilities; do echo $vulnerability >> .trivyignore done - if $(trivy image \ - --quiet \ - --exit-code 1 \ - --scanners vuln \ - --format json \ - --severity HIGH,CRITICAL \ - --output image-scan-output/${filename}/${filename}.json \ - --ignore-unfixed \ - --db-repository ghcr.io/aquasecurity/trivy-db:2 \ - --db-repository public.ecr.aws/aquasecurity/trivy-db \ - --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ - --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ - $image); then - # Clean up the output file for any images with no vulnerabilities - rm -f image-scan-output/${filename}/${filename}.json - - # Add the image to the clean list +} + +# Put results into CSV +generate_summary_csv() { + local imagename=$1 + local filename=$2 + + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv + + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv +} + +# Categorise images based on severity +categorise_image() { + local imagename=$1 + local filename=$2 + local image=$3 + + if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then + echo "${image}" >> image-scan-output/critical-images.txt + else + echo "${image}" >> image-scan-output/high-images.txt + fi +} + +# Scan images, generate SBOMs if requested +scan_image() { + local image=$1 + local filename=$(basename $image | sed 's/:/\./g') + local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') + + mkdir -p image-scan-output/$imagename + generate_trivy_ignore $imagename + + echo "Scanning $imagename" + + # If SBOM is required, generate that first, then generate scan results from it + if $generate_sbom; then + trivy image \ + --format spdx-json \ + --output image-scan-output/${imagename}/${filename}-sbom.json \ + $image + scan_command="trivy sbom $scan_common_args \ + --output image-scan-output/${imagename}/${filename}-scan.json \ + image-scan-output/${imagename}/${filename}-sbom.json" + else + scan_command="trivy image $scan_common_args \ + --output image-scan-output/${imagename}/${filename}-scan.json $image" + fi + echo "scan command" + echo "$scan_command" + # Run scan, against image or SBOM. If no results, delete files. + if $scan_command; then + rm -f image-scan-output/${imagename}/${filename}-scan.json echo "${image}" >> image-scan-output/clean-images.txt else + generate_summary_csv $imagename $filename + categorise_image $imagename $filename $image + fi +} - # Write a header for the summary CSV - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}/${filename}.summary.csv - - # Write the summary CSV data - jq -r '.Results[] - | select(.Vulnerabilities) - | .Vulnerabilities - # Ignore packages with "kernel" in the PkgName - | map(select(.PkgName | test("kernel") | not )) - | group_by(.VulnerabilityID) - | map( - [ - (map(.PkgName) | unique | join(";")), - (map(.PkgPath | select( . != null )) | join(";")), - .[0].PkgID, - .[0].VulnerabilityID, - .[0].FixedVersion, - .[0].PrimaryURL, - .[0].Severity - ] - ) - | .[] - | @csv' image-scan-output/${filename}/${filename}.json >> image-scan-output/${filename}/${filename}.summary.csv - - if [ $(grep "CRITICAL" image-scan-output/${filename}/${filename}.summary.csv -c) -gt 0 ]; then - # If the image contains critical vulnerabilities, add the image to critical list - echo "${image}" >> image-scan-output/critical-images.txt - else - # Otherwise, add the image to the dirty list - echo "${image}" >> image-scan-output/dirty-images.txt - fi +# Main function +main() { + if [[ ! $2 ]]; then + usage fi - trivy image \ - --quiet \ - --format spdx \ - --output image-scan-output/${filename}/${filename}-sbom.spdx \ - --db-repository ghcr.io/aquasecurity/trivy-db:2 \ - --db-repository public.ecr.aws/aquasecurity/trivy-db \ - --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ - --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ - $image -done + + generate_sbom=false + if [[ "$3" == "--sbom" ]]; then + generate_sbom=true + fi + + set -u + + check_deps_installed + file_prep + + images=$(get_images $1 $2) + for image in $images; do + scan_image $image + done +} + +main "$@" From 1f5a8e4b1dcf91b6b7a7207340b28d7a07e0bf95 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 9 Jun 2025 16:20:48 +0100 Subject: [PATCH 3/4] CI: Minor image scanning tweaks --- .../stackhpc-container-image-build.yml | 23 ++++++----- tools/scan-images.sh | 40 +++++++++++-------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 5b6a766936..b35dd20d41 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -33,7 +33,12 @@ on: type: boolean required: false default: true - push-dirty: + sbom: + description: Generate SBOM? + type: boolean + required: false + default: true + push-critical: description: Push scanned images that have critical vulnerabilities? type: boolean required: false @@ -254,14 +259,14 @@ jobs: run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi - name: Scan built container images - run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom' }} - name: Move image scan logs to output artifact run: mv image-scan-output image-build-logs/image-scan-output - - name: Fail if no images have passed scanning + - name: Fail if any images have critical vulnerabilities run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi - if: ${{ !inputs.push-dirty }} + if: ${{ !inputs.push-critical }} - name: Copy clean images to push-attempt-images list run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt @@ -271,13 +276,13 @@ jobs: # This should be reverted when it's decided to filter high level CVEs as well. - name: Append dirty images to push list run: | - cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt if: ${{ inputs.push }} - name: Append images with critical vulnerabilities to push list run: | cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt - if: ${{ inputs.push && inputs.push-dirty }} + if: ${{ inputs.push && inputs.push-critical }} - name: Push images run: | @@ -326,12 +331,12 @@ jobs: # This can be used again instead of "Fail when critical vulnerabilities are found" when it's # decided to fail the job on detecting high CVEs as well. # - name: Fail when images failed scanning - # run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi - # if: ${{ !inputs.push-dirty && !cancelled() }} + # run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi + # if: ${{ !inputs.push-critical && !cancelled() }} - name: Fail when critical vulnerabilities are found run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi - if: ${{ !inputs.push-dirty && !cancelled() }} + if: ${{ !inputs.push-critical && !cancelled() }} - name: Remove locally built images for this run if: always() && runner.arch == 'ARM64' diff --git a/tools/scan-images.sh b/tools/scan-images.sh index 9737172834..feba9bbc5f 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -35,7 +35,7 @@ check_deps_installed() { file_prep() { rm -rf image-scan-output mkdir -p image-scan-output - touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt + touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt } # Gather image lists @@ -99,7 +99,21 @@ categorise_image() { fi } -# Scan images, generate SBOMs if requested +# Generate SBOM, return correct scan command for SBOM +generate_sbom() { + local imagename=$1 + local filename=$2 + local image=$3 + trivy image \ + --format spdx-json \ + --output image-scan-output/${imagename}/${filename}-sbom.json \ + $image > /dev/null 2>&1 + echo "trivy sbom $scan_common_args \ + --output image-scan-output/${imagename}/${filename}-scan.json \ + image-scan-output/${imagename}/${filename}-sbom.json" +} + +# Scan images, generate SBOMs if requested scan_image() { local image=$1 local filename=$(basename $image | sed 's/:/\./g') @@ -108,25 +122,19 @@ scan_image() { mkdir -p image-scan-output/$imagename generate_trivy_ignore $imagename - echo "Scanning $imagename" - - # If SBOM is required, generate that first, then generate scan results from it + # If SBOM is required, generate it first and scan the results, otherwise we + # scan the image directly. if $generate_sbom; then - trivy image \ - --format spdx-json \ - --output image-scan-output/${imagename}/${filename}-sbom.json \ - $image - scan_command="trivy sbom $scan_common_args \ - --output image-scan-output/${imagename}/${filename}-scan.json \ - image-scan-output/${imagename}/${filename}-sbom.json" + echo "Generating SBOM for $imagename" + scan_command=$(generate_sbom $imagename $filename $image) else scan_command="trivy image $scan_common_args \ --output image-scan-output/${imagename}/${filename}-scan.json $image" fi - echo "scan command" - echo "$scan_command" - # Run scan, against image or SBOM. If no results, delete files. - if $scan_command; then + + # Run scan against image or SBOM, format output. If no results, delete files. + echo "Scanning $imagename for vulnerabilities" + if $scan_command > /dev/null 2>&1; then rm -f image-scan-output/${imagename}/${filename}-scan.json echo "${image}" >> image-scan-output/clean-images.txt else From 6b2d0983ba3f7e48d9798a866959a932982ff574 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 30 Jul 2025 09:25:44 +0100 Subject: [PATCH 4/4] Fix linter errors --- .../ansible/roles/pulp_auth_proxy/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml index 6cbe55e5e2..66550b8948 100644 --- a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml @@ -5,11 +5,11 @@ - name: Check if Docker bridge network exists community.docker.docker_host_info: networks: true - register: docker_host_info + register: pulp_auth_proxy_docker_host_info - name: Set a fact about the network mode ansible.builtin.set_fact: - pulp_auth_proxy_network_mode: "{{ 'host' if docker_host_info.networks | selectattr('Driver', 'equalto', 'bridge') | list | length == 0 else 'bridge' }}" + pulp_auth_proxy_network_mode: "{{ 'host' if pulp_auth_proxy_docker_host_info.networks | selectattr('Driver', 'equalto', 'bridge') | list | length == 0 else 'bridge' }}" - name: Assert that localhost is resolvable when using host networking ansible.builtin.assert: @@ -33,7 +33,7 @@ dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf" mode: "0600" become: true - register: pulp_proxy_conf + register: pulp_auth_proxy_conf - name: Ensure pulp_proxy container is running community.docker.docker_container: @@ -43,14 +43,14 @@ ports: - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80" restart_policy: "no" - restart: "{{ pulp_proxy_conf is changed }}" + restart: "{{ pulp_auth_proxy_conf is changed }}" volumes: - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro" - name: Wait for pulp_proxy container to become accessible ansible.builtin.uri: url: http://localhost/pulp/api/v3/status/ - register: uri_result - until: uri_result is success + register: pulp_auth_proxy_uri_result + until: pulp_auth_proxy_uri_result is success retries: 30 delay: 2