From 3a7498d86dbf8ebe6e9411daf2ab7ee57d4e3188 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 13 Feb 2025 14:22:21 +0100 Subject: [PATCH 01/27] Test is non CUDA builds are not add to accelorator path with jax --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml index 7ac4ba6cca..6408ec74c1 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -5,3 +5,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 + - jax-0.4.25-gfbf-2023a-CUDA-12.1.1.eb From b008485ebe7e92f1c274f19ceb48a4c5fb7db4c8 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 12:33:57 +0100 Subject: [PATCH 02/27] Create test-check_easystacks.yml --- .github/workflows/test-check_easystacks.yml | 96 +++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/test-check_easystacks.yml diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml new file mode 100644 index 0000000000..1b363e1bb2 --- /dev/null +++ b/.github/workflows/test-check_easystacks.yml @@ -0,0 +1,96 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for CUDA builds in changed easystacks +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 +jobs: + check_missing: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + filters: | + changed: + -'easystacks/**' + + - name: Test check_missing_installations.sh script + if: ${{ steps.changes.outputs.changed == 'true' }} + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), + # to prevent issues with checks in the Easybuild configuration that use this variable + export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} + module load EasyBuild + which eb + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + + # check for missing installations if a file in the easystacks directory is changed + echo + + # first check the CPU-only builds for this CPU target + #echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" + #for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do + # echo "check missing installations for ${easystack_file}..." + # ./check_missing_installations.sh ${easystack_file} + # ec=$? + # if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + #done + + # now check the accelerator builds for this CPU target + #accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") + #if [ -z ${accelerators} ]; then + # echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + #else + # for accel in ${accelerators}; do + # module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + # echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" + # for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do + # echo "check missing installations for ${easystack_file}..." + # ./check_missing_installations.sh ${easystack_file} + # ec=$? + # if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + # done + # module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + # done + #fi */ + + #if ./check_missing_installations.sh ${easystack_file}; then + # echo "did NOT capture missing package; test FAILED" + # exit 1 + #else + # echo "captured missing package; test PASSED" + exit 0 + fi From fb7ceaea24112c3a8b5431aa0d2eb1326e64ad92 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 12:40:02 +0100 Subject: [PATCH 03/27] add the filter for changed easystack files --- .github/workflows/test-check_easystacks.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 1b363e1bb2..5d83921c1c 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -38,10 +38,13 @@ jobs: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - filters: | - changed: - -'easystacks/**' - + - name: find changed easystacks + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + changed: + -'easystacks/**' - name: Test check_missing_installations.sh script if: ${{ steps.changes.outputs.changed == 'true' }} run: | From c4b39d36405f40c531ac7ced7d11585f1040ac8b Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:11:49 +0100 Subject: [PATCH 04/27] fix syntax of yml file --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 5d83921c1c..30939b2118 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -44,7 +44,7 @@ jobs: with: filters: | changed: - -'easystacks/**' + - 'easystacks/**' - name: Test check_missing_installations.sh script if: ${{ steps.changes.outputs.changed == 'true' }} run: | From 5b5434a4dd04dbc786f18075b77721a7a759012e Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:30:52 +0100 Subject: [PATCH 05/27] run check_missing_installation.sh in workflow --- .github/workflows/test-check_easystacks.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 30939b2118..a0792a9bf6 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -61,16 +61,13 @@ jobs: env | grep ^EESSI | sort # check for missing installations if a file in the easystacks directory is changed - echo - - # first check the CPU-only builds for this CPU target - #echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" - #for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do - # echo "check missing installations for ${easystack_file}..." - # ./check_missing_installations.sh ${easystack_file} - # ec=$? - # if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi - #done + echo "check for missing installation in changed easystack files" + for easystack_file in ${{ steps.changes.outputs.changed_files }}; do + echo "check missing installatios for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + echo "print the output of missing: ${eb_missing_out}" + done # now check the accelerator builds for this CPU target #accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") From b875d62dd3b2210094c568b506abc3201c4730ea Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:39:44 +0100 Subject: [PATCH 06/27] debugging checking missing installation step --- .github/workflows/test-check_easystacks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index a0792a9bf6..8fdc02b7f1 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -62,7 +62,9 @@ jobs: # check for missing installations if a file in the easystacks directory is changed echo "check for missing installation in changed easystack files" - for easystack_file in ${{ steps.changes.outputs.changed_files }}; do + echo ${{ steps.changes.outputs.changed_files }} + + for easystack_file in (${{ steps.changes.outputs.changed_files }}); do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 9a981a59560b944477e8fd6ef3d2d7f25d02dcfd Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:48:44 +0100 Subject: [PATCH 07/27] debugging getting changed file --- .github/workflows/test-check_easystacks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 8fdc02b7f1..49c094a7e7 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -62,9 +62,10 @@ jobs: # check for missing installations if a file in the easystacks directory is changed echo "check for missing installation in changed easystack files" + echo ${{ steps.changes.outputs }} echo ${{ steps.changes.outputs.changed_files }} - for easystack_file in (${{ steps.changes.outputs.changed_files }}); do + for easystack_file in ${{ steps.changes.outputs.changed_files }}; do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From e8acea1febb322804701814c754155dc21f05a2c Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 13:52:26 +0100 Subject: [PATCH 08/27] debugging getting changed file --- .github/workflows/test-check_easystacks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 49c094a7e7..04ef7f0c04 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -42,6 +42,7 @@ jobs: uses: dorny/paths-filter@v3 id: changes with: + list-files: shell filters: | changed: - 'easystacks/**' From 10d28378c5bcb1bfdc0e2ae652a0184b6f29ea2a Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 14:07:25 +0100 Subject: [PATCH 09/27] trying to get the list of missing installations --- .github/workflows/test-check_easystacks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 04ef7f0c04..fc42df2186 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -63,13 +63,12 @@ jobs: # check for missing installations if a file in the easystacks directory is changed echo "check for missing installation in changed easystack files" - echo ${{ steps.changes.outputs }} - echo ${{ steps.changes.outputs.changed_files }} - for easystack_file in ${{ steps.changes.outputs.changed_files }}; do + for easystack_file in (${{ steps.changes.outputs.changed_files }}); do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi echo "print the output of missing: ${eb_missing_out}" done From aedb5ebbf8e3aa5bf02586e3706015f810a2be8c Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 14:41:09 +0100 Subject: [PATCH 10/27] fix bash syntax --- .github/workflows/test-check_easystacks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index fc42df2186..b360c1b53e 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -63,8 +63,9 @@ jobs: # check for missing installations if a file in the easystacks directory is changed echo "check for missing installation in changed easystack files" + easystack_files=(${{ steps.changes.outputs.changed_files }}) - for easystack_file in (${{ steps.changes.outputs.changed_files }}); do + for easystack_file in easystack_files; do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 951ad4a1550e7573b805116b639e402ad487246e Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 15:00:39 +0100 Subject: [PATCH 11/27] debugging --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index b360c1b53e..5e0439e161 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -65,7 +65,7 @@ jobs: echo "check for missing installation in changed easystack files" easystack_files=(${{ steps.changes.outputs.changed_files }}) - for easystack_file in easystack_files; do + for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 561ee5e323f2e499c91ee460b9147b54ced0e442 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:00:45 +0100 Subject: [PATCH 12/27] debugging --- .github/workflows/test-check_easystacks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 5e0439e161..d5209d2fd7 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -69,6 +69,7 @@ jobs: echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? + echo "print what is in ${ec}" if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi echo "print the output of missing: ${eb_missing_out}" done From 0151275813e138c718ede58cabb7246fd6f19fff Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:12:26 +0100 Subject: [PATCH 13/27] debugging workflow --- .github/workflows/test-check_easystacks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index d5209d2fd7..a290824fa8 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -48,6 +48,7 @@ jobs: - 'easystacks/**' - name: Test check_missing_installations.sh script if: ${{ steps.changes.outputs.changed == 'true' }} + continue-on-error: true run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash From bff29c9d0465ab5b1ddbc3f1280184bb7ada5798 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:27:31 +0100 Subject: [PATCH 14/27] debugging --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index a290824fa8..dce36d73f4 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -47,8 +47,8 @@ jobs: changed: - 'easystacks/**' - name: Test check_missing_installations.sh script - if: ${{ steps.changes.outputs.changed == 'true' }} continue-on-error: true + if: ${{ steps.changes.outputs.changed == 'true' }} run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash From a860642b98ce9df51e0fa45cec8f49634ee437ff Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:13:17 +0100 Subject: [PATCH 15/27] Don't stop workflow due to failed missing installations --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index dce36d73f4..9c818c6b6e 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,7 +68,7 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} + ./check_missing_installations.sh ${easystack_file} || [ $? -ne 0 ] ec=$? echo "print what is in ${ec}" if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi From 9d3f2fede30bf4bf532cd13324741ce0cdd3cf95 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:28:10 +0100 Subject: [PATCH 16/27] get exit code of run check-missing-installations.sh --- .github/workflows/test-check_easystacks.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 9c818c6b6e..1a0e67b70d 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,9 +68,7 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} || [ $? -ne 0 ] - ec=$? - echo "print what is in ${ec}" + ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi echo "print the output of missing: ${eb_missing_out}" done From 555a3d15a8300169e4de8766f60aa3fc4a91bc10 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:23:21 +0100 Subject: [PATCH 17/27] add script for parsing missing installations output --- .../parse_missing-installations-output.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/parse_missing-installations-output.py diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py new file mode 100644 index 0000000000..3743675ffd --- /dev/null +++ b/.github/workflows/parse_missing-installations-output.py @@ -0,0 +1,16 @@ +import os +import re + +eb_missing_out = os.environ['eb_missing_out']) +missing = re.findall("\([A-Z_a-z0-9_\-_\.]*.eb\)", missing) +missing_cuda = [] +missing_cpu = [] +for ec in missing: + if re.search('CUDA', ec): + missing_cuda.append(ec) + else: + missing_cpu.append(ec) +if len(missing_cpu) != 0: + os.environ['missing_cpu'] = missing_cpu +if len(missing_gpu) !0: + os.environ['missing_cpu'] = missing_cpu From c7a7758c4f9302c2556a99610423a3f5fcd820c7 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:29:25 +0100 Subject: [PATCH 18/27] fix parse script --- .github/workflows/parse_missing-installations-output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index 3743675ffd..2a5afda447 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -2,7 +2,7 @@ import re eb_missing_out = os.environ['eb_missing_out']) -missing = re.findall("\([A-Z_a-z0-9_\-_\.]*.eb\)", missing) +missing = re.findall("\([A-Z_a-z0-9_\-_\.]*.eb\)", eb_missing_out) missing_cuda = [] missing_cpu = [] for ec in missing: From 754faa4b0a49e7a766292405e0a6812b753c256f Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:31:23 +0100 Subject: [PATCH 19/27] fix parse script --- .github/workflows/parse_missing-installations-output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index 2a5afda447..804ba449a4 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -12,5 +12,5 @@ missing_cpu.append(ec) if len(missing_cpu) != 0: os.environ['missing_cpu'] = missing_cpu -if len(missing_gpu) !0: +if len(missing_gpu) != 0: os.environ['missing_cpu'] = missing_cpu From 41df4465b881cea904834795014d52cfb4b6f269 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:48:27 +0100 Subject: [PATCH 20/27] fix parse script --- .github/workflows/parse_missing-installations-output.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index 804ba449a4..d4d930f332 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -1,7 +1,7 @@ import os import re -eb_missing_out = os.environ['eb_missing_out']) +eb_missing_out = os.environ['eb_missing_out'] missing = re.findall("\([A-Z_a-z0-9_\-_\.]*.eb\)", eb_missing_out) missing_cuda = [] missing_cpu = [] @@ -11,6 +11,6 @@ else: missing_cpu.append(ec) if len(missing_cpu) != 0: - os.environ['missing_cpu'] = missing_cpu -if len(missing_gpu) != 0: - os.environ['missing_cpu'] = missing_cpu + os.environ['MISSING_CPU'] = str(missing_cpu) +if len(missing_cuda) != 0: + os.environ['MISSING_CUDA'] = str(missing_cuda) From 6d1cd82730cd770a6c3f5a8044f08bb1bf8aada4 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Wed, 26 Feb 2025 16:43:45 +0100 Subject: [PATCH 21/27] add workflow to check the missing builds in the pr --- .../parse_missing-installations-output.py | 19 +++++++++++++------ .github/workflows/test-check_easystacks.yml | 6 ++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index d4d930f332..5b4379bbf5 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -1,8 +1,8 @@ import os import re -eb_missing_out = os.environ['eb_missing_out'] -missing = re.findall("\([A-Z_a-z0-9_\-_\.]*.eb\)", eb_missing_out) +missing = os.environ['missing'] +missing = missing.split('\n') missing_cuda = [] missing_cpu = [] for ec in missing: @@ -10,7 +10,14 @@ missing_cuda.append(ec) else: missing_cpu.append(ec) -if len(missing_cpu) != 0: - os.environ['MISSING_CPU'] = str(missing_cpu) -if len(missing_cuda) != 0: - os.environ['MISSING_CUDA'] = str(missing_cuda) +if len(missing_cpu) != 0 and len(missing_cuda) != 0: + print('Error: CPU dependencies for a GPU build must be submitted in a seperate pr') + print(f'Please open a seperate pr of the dependencies: {missing_cpu}') + exit(1) +elif len(missing_cuda) != 0: + # TODO: Make this set the accelorator label? + print(f'Have fun installing the following gpu builds: {missing_cuda}') +elif len(missing_cpu) != 0: + print(f'Have fun installing the following gpu builds: {missing_cpu}') +else: + print('no missing modules') diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 1a0e67b70d..50ba31b0a3 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -69,8 +69,10 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} || ec=$? - if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi - echo "print the output of missing: ${eb_missing_out}" + if [[ ${ec} -ne 0 ]]; then + echo "missing installations found for ${easystack_file}!" >&2 + python .github/workflows/parse_missing-installations-output.py + fi done # now check the accelerator builds for this CPU target From bf0e3fa82f31b00a3e6f587b2d90241e1f478c54 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:47:16 +0100 Subject: [PATCH 22/27] fix check easystack workflow --- .github/workflows/test-check_easystacks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 50ba31b0a3..ceca47ed18 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -71,6 +71,7 @@ jobs: ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 + export missing=$(env | grep .eb\) | tr -d \*) python .github/workflows/parse_missing-installations-output.py fi done From b4fff2ea1cb1e4c0074dad5984b807ae1c07f88e Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:50:19 +0100 Subject: [PATCH 23/27] fix workflow --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index ceca47ed18..065a4ace3a 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,7 +68,7 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} || ec=$? + export eb_missing_out=$(./check_missing_installations.sh ${easystack_file} || ec=$?) if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 export missing=$(env | grep .eb\) | tr -d \*) From bbac63b10d7c43576e0f212a728370fb01fd9f09 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:21:07 +0100 Subject: [PATCH 24/27] debugging workflow --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 065a4ace3a..19c44677e4 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,7 +68,7 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - export eb_missing_out=$(./check_missing_installations.sh ${easystack_file} || ec=$?) + export eb_missing_out=$(./check_missing_installations.sh ${easystack_file} >&1 || ec=$?) if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 export missing=$(env | grep .eb\) | tr -d \*) From e17f9da15ab50db69b41430482ddcf90d66f40ce Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:26:36 +0100 Subject: [PATCH 25/27] debugging workflow --- .github/workflows/test-check_easystacks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 19c44677e4..d4304e8558 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,7 +68,8 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - export eb_missing_out=$(./check_missing_installations.sh ${easystack_file} >&1 || ec=$?) + ./check_missing_installations.sh ${easystack_file} >&1 || ec=$? + export eb_missing_out=$(./check_missing_installations.sh ${easystack_file}) if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 export missing=$(env | grep .eb\) | tr -d \*) From 9e29265383a131f1d64652a651987023d7d34200 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Wed, 26 Feb 2025 17:40:18 +0100 Subject: [PATCH 26/27] clean-up workflow --- .github/workflows/parse_missing-installations-output.py | 2 +- .github/workflows/test-check_easystacks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index 5b4379bbf5..9ea3260550 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -11,8 +11,8 @@ else: missing_cpu.append(ec) if len(missing_cpu) != 0 and len(missing_cuda) != 0: - print('Error: CPU dependencies for a GPU build must be submitted in a seperate pr') print(f'Please open a seperate pr of the dependencies: {missing_cpu}') + os.write(2, b'Error: CPU dependencies for a GPU build must be submitted in a seperate pr') exit(1) elif len(missing_cuda) != 0: # TODO: Make this set the accelorator label? diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index d4304e8558..6c62cea9c6 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -68,7 +68,7 @@ jobs: for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} >&1 || ec=$? + ./check_missing_installations.sh ${easystack_file} || ec=$? export eb_missing_out=$(./check_missing_installations.sh ${easystack_file}) if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 From 8f15037dcd9ef380b9b736185c33fd5a11b9cb70 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Wed, 26 Feb 2025 17:44:22 +0100 Subject: [PATCH 27/27] remove continue on error --- .github/workflows/parse_missing-installations-output.py | 2 +- .github/workflows/test-check_easystacks.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py index 9ea3260550..9d399e1a23 100644 --- a/.github/workflows/parse_missing-installations-output.py +++ b/.github/workflows/parse_missing-installations-output.py @@ -12,7 +12,7 @@ missing_cpu.append(ec) if len(missing_cpu) != 0 and len(missing_cuda) != 0: print(f'Please open a seperate pr of the dependencies: {missing_cpu}') - os.write(2, b'Error: CPU dependencies for a GPU build must be submitted in a seperate pr') + os.write(2, b'Error: CPU dependencies for CUDA build must be build in a seperate pr') exit(1) elif len(missing_cuda) != 0: # TODO: Make this set the accelorator label? diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 6c62cea9c6..42aa87d706 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -47,7 +47,6 @@ jobs: changed: - 'easystacks/**' - name: Test check_missing_installations.sh script - continue-on-error: true if: ${{ steps.changes.outputs.changed == 'true' }} run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}