From 2cea59a79d159b700a02b9b296d763f0e11920af Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 13 Jun 2025 12:45:59 +0200 Subject: [PATCH 01/89] Update RSC with rocm 6.4.1, adding tuolumne and moving from poodle to dane --- .gitlab-ci.yml | 14 +++++---- .gitlab/custom-jobs-and-variables.yml | 24 ++++++++++----- .gitlab/jobs/{poodle.yml => dane.yml} | 16 +++++----- .gitlab/jobs/tuolumne.yml | 43 +++++++++++++++++++++++++++ .gitlab/subscribed-pipelines.yml | 30 ++++++++++++++----- tpl/RAJA | 2 +- 6 files changed, 100 insertions(+), 29 deletions(-) rename .gitlab/jobs/{poodle.yml => dane.yml} (75%) create mode 100644 .gitlab/jobs/tuolumne.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81d671448..12beab12e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2017-24, Lawrence Livermore National Security, LLC +# Copyright (c) 2017-25, Lawrence Livermore National Security, LLC # and RAJA Performance Suite project contributors. # See the RAJAPerf/LICENSE file for details. # @@ -29,14 +29,15 @@ # We define the following GitLab pipeline variables: variables: ##### LC GITLAB CONFIGURATION -# Use a LLNL service user to run CI. This prevents from running pipelines as an -# actual user. +# Use an LLNL service user to run CI. This prevents from running pipelines as +# an actual user. LLNL_SERVICE_USER: rajasa # Use the service user workspace. Solves permission issues, stores everything # at the same location whoever triggers a pipeline. # CUSTOM_CI_BUILDS_DIR: "/usr/workspace/rajasa/gitlab-runner" # Tells Gitlab to recursively update the submodules when cloning the project. GIT_SUBMODULE_STRATEGY: recursive + GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 ##### PROJECT VARIABLES # We build the projects in the CI clone directory (used in @@ -49,6 +50,8 @@ variables: GITHUB_PROJECT_NAME: "RAJAPerf" GITHUB_PROJECT_ORG: "LLNL" # Set the build-and-test command. +# Nested variables are allowed and useful to customize the job command. We +# prevent variable expansion so that you can define them at job level. JOB_CMD: value: "./scripts/gitlab/build_and_test.sh" expand: false @@ -72,7 +75,7 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'v2024.12.0' + ref: 'v2025.06.0' file: 'pipelines/${CI_MACHINE}.yml' - artifact: '${CI_MACHINE}-jobs.yml' job: 'generate-job-lists' @@ -81,11 +84,12 @@ stages: pipeline_variables: true include: + # Sets ID tokens for every job using `default:` - project: 'lc-templates/id_tokens' file: 'id_tokens.yml' # [Optional] checks preliminary to running the actual CI test #- project: 'radiuss/radiuss-shared-ci' - # ref: 'v2024.12.0' + # ref: 'v2025.06.0' # file: 'utilities/preliminary-ignore-draft-pr.yml' # pipelines subscribed by the project - local: '.gitlab/subscribed-pipelines.yml' diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 17875c955..1566c1c9c 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -24,15 +24,15 @@ variables: # Project specific deps for ruby PROJECT_RUBY_DEPS: "^blt@develop " -# Poodle +# Dane # Arguments for top level allocation - POODLE_SHARED_ALLOC: "--exclusive --time=45 --nodes=1" + DANE_SHARED_ALLOC: "--exclusive --time=45 --nodes=1" # Arguments for job level allocation - POODLE_JOB_ALLOC: "--nodes=1" -# Project specific variants for poodle - PROJECT_POODLE_VARIANTS: "~shared +openmp" -# Project specific deps for poodle - PROJECT_POODLE_DEPS: "^blt@develop " + DANE_JOB_ALLOC: "--nodes=1" +# Project specific variants for dane + PROJECT_DANE_VARIANTS: "~shared +openmp" +# Project specific deps for dane + PROJECT_DANE_DEPS: "^blt@develop " # Corona # Arguments for top level allocation @@ -54,6 +54,16 @@ variables: # Project specific deps for tioga PROJECT_TIOGA_DEPS: "^blt@develop " +# Tuolumne +# Arguments for top level allocation + TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=31m --nodes=1 -o per-resource.count=2" +# Arguments for job level allocation + TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" +# Project specific variants for tuolumne + PROJECT_TUOLUMNE_VARIANTS: "~shared +openmp" +# Project specific deps for tuolumne + PROJECT_TUOLUMNE_DEPS: "^blt@develop " + # Lassen and Butte use a different job scheduler (spectrum lsf) that does not # allow pre-allocation the same way slurm does. # Arguments for job level allocation diff --git a/.gitlab/jobs/poodle.yml b/.gitlab/jobs/dane.yml similarity index 75% rename from .gitlab/jobs/poodle.yml rename to .gitlab/jobs/dane.yml index aa42fe081..00f1e7f11 100644 --- a/.gitlab/jobs/poodle.yml +++ b/.gitlab/jobs/dane.yml @@ -7,7 +7,7 @@ ############################################################################## # Override reproducer section to define projet specific variables. -.poodle_reproducer_vars: +.dane_reproducer_vars: script: - !reference [.reproducer_vars, script] @@ -21,20 +21,20 @@ # custom variant oneapi_2023_2_1: variables: - SPEC: "${PROJECT_POODLE_VARIANTS} +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ${PROJECT_POODLE_DEPS}" - extends: .job_on_poodle + SPEC: "${PROJECT_DANE_VARIANTS} +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ${PROJECT_DANE_DEPS}" + extends: .job_on_dane # omptask variant clang_14_0_6: variables: - SPEC: "${PROJECT_POODLE_VARIANTS} +omptask %clang@=14.0.6 ${PROJECT_POODLE_DEPS}" - extends: .job_on_poodle + SPEC: "${PROJECT_DANE_VARIANTS} +omptask %clang@=14.0.6 ${PROJECT_DANE_DEPS}" + extends: .job_on_dane # omptask variant gcc_10_3_1: variables: - SPEC: "${PROJECT_POODLE_VARIANTS} +omptask %gcc@=10.3.1 ${PROJECT_POODLE_DEPS}" - extends: .job_on_poodle + SPEC: "${PROJECT_DANE_VARIANTS} +omptask %gcc@=10.3.1 ${PROJECT_DANE_DEPS}" + extends: .job_on_dane ############ # Extra jobs @@ -46,4 +46,4 @@ gcc_10_3_1: oneapi_2023_2_1_mpi: variables: SPEC: "~shared +openmp +mpi +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ^mvapich2 ^blt@develop" - extends: .job_on_poodle + extends: .job_on_dane diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml new file mode 100644 index 000000000..498aa976b --- /dev/null +++ b/.gitlab/jobs/tuolumne.yml @@ -0,0 +1,43 @@ +############################################################################### +# Copyright (c) 2017-25, Lawrence Livermore National Security, LLC +# and RAJA Performance Suite project contributors. +# See the RAJAPerf/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################# + +# Override reproducer section to define project specific variables. +.tuolumne_reproducer_vars: + script: + - !reference [.reproducer_vars, script] + +######################## +# Overridden shared jobs +######################## +# We duplicate the shared jobs description and add necessary changes for RAJA. +# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that +# the comparison with the original job is easier. + +# No overridden jobs so far. + +############ +# Extra jobs +############ +# We do not recommend using ${PROJECT__VARIANTS} and +# ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully +# describe the spec here. + +cce_17_0_1: + variables: + SPEC: "~shared +openmp %cce@=17.0.1 ^blt@develop" + extends: .job_on_tuolumne + +rocmcc_6_3_0_hip_openmp: + variables: + SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + extends: .job_on_tuolumne + +rocmcc_6_3_0_hip_openmp_mpi: + variables: + SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + extends: .job_on_tuolumne diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index 9a86c8a8b..ab91f85e5 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -42,17 +42,19 @@ generate-job-lists: LOCAL_JOBS_PATH: ".gitlab/jobs" script: - cat ${RADIUSS_JOBS_PATH}/ruby.yml ${LOCAL_JOBS_PATH}/ruby.yml > ruby-jobs.yml - - cat ${RADIUSS_JOBS_PATH}/poodle.yml ${LOCAL_JOBS_PATH}/poodle.yml > poodle-jobs.yml + - cat ${RADIUSS_JOBS_PATH}/dane.yml ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml - cat ${RADIUSS_JOBS_PATH}/lassen.yml ${LOCAL_JOBS_PATH}/lassen.yml > lassen-jobs.yml - cat ${RADIUSS_JOBS_PATH}/corona.yml ${LOCAL_JOBS_PATH}/corona.yml > corona-jobs.yml - cat ${RADIUSS_JOBS_PATH}/tioga.yml ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml + - cat ${RADIUSS_JOBS_PATH}/tuolumne.yml ${LOCAL_JOBS_PATH}/tuolumne.yml > tuolumne-jobs.yml artifacts: paths: - ruby-jobs.yml - - poodle-jobs.yml + - dane-jobs.yml - lassen-jobs.yml - corona-jobs.yml - tioga-jobs.yml + - tuolumne-jobs.yml # RUBY ruby-up-check: @@ -66,16 +68,16 @@ ruby-build-and-test: needs: [ruby-up-check, generate-job-lists] extends: [.build-and-test] -# POODLE -poodle-up-check: +# DANE +dane-up-check: variables: - CI_MACHINE: "poodle" + CI_MACHINE: "dane" extends: [.machine-check] -poodle-build-and-test: +dane-build-and-test: variables: - CI_MACHINE: "poodle" - needs: [poodle-up-check, generate-job-lists] + CI_MACHINE: "dane" + needs: [dane-up-check, generate-job-lists] extends: [.build-and-test] # CORONA @@ -102,6 +104,18 @@ tioga-build-and-test: needs: [tioga-up-check, generate-job-lists] extends: [.build-and-test] +# TUOLUMNE +tuolumne-up-check: + variables: + CI_MACHINE: "tuolumne" + extends: [.machine-check] + +tuolumne-build-and-test: + variables: + CI_MACHINE: "tuolumne" + needs: [tuolumne-up-check, generate-job-lists] + extends: [.build-and-test] + # LASSEN lassen-up-check: variables: diff --git a/tpl/RAJA b/tpl/RAJA index 6e36a9438..c0f6cbbe7 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 6e36a94380adbe88fed11a3213fc08461428ece0 +Subproject commit c0f6cbbe735fd16dcbe560cc0b9d6198c7c84578 From f82f31ee9832b708e196267423585c0e3d4e477a Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:15:35 +0200 Subject: [PATCH 02/89] Remove change in submodule options causing failure --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12beab12e..e9cb10ba9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,6 @@ variables: # CUSTOM_CI_BUILDS_DIR: "/usr/workspace/rajasa/gitlab-runner" # Tells Gitlab to recursively update the submodules when cloning the project. GIT_SUBMODULE_STRATEGY: recursive - GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 ##### PROJECT VARIABLES # We build the projects in the CI clone directory (used in From 94430b03bf69c7f6cd03855d66bd4f3cb96533d4 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 16 Jun 2025 11:02:47 +0200 Subject: [PATCH 03/89] CI script: Add core counts for new machines --- scripts/gitlab/build_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 283c88574..e7cbe00c7 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -211,7 +211,7 @@ then timed_message "Cleaning working directory" # Map CPU core allocations - declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32) + declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["dane"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) # If using Multi-project, set up the submodule if [[ -n ${raja_version} ]] From 017624ee1438efebe8e2ae572ef7c64882b96b9a Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 16 Jun 2025 12:18:20 +0200 Subject: [PATCH 04/89] Update RAJA --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index c0f6cbbe7..c8734ef9a 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit c0f6cbbe735fd16dcbe560cc0b9d6198c7c84578 +Subproject commit c8734ef9aeacff423998bdc2615119bdd41cc963 From a61a7fdb3213e4f12089e42652e3664324c31ccf Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 16 Jun 2025 17:15:29 +0200 Subject: [PATCH 05/89] Revert "Remove change in submodule options causing failure" This reverts commit f82f31ee9832b708e196267423585c0e3d4e477a. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e9cb10ba9..12beab12e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ variables: # CUSTOM_CI_BUILDS_DIR: "/usr/workspace/rajasa/gitlab-runner" # Tells Gitlab to recursively update the submodules when cloning the project. GIT_SUBMODULE_STRATEGY: recursive + GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 ##### PROJECT VARIABLES # We build the projects in the CI clone directory (used in From 71ff6394175828b84b421c9320219d515df4f27a Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 24 Jun 2025 16:35:38 +0200 Subject: [PATCH 06/89] RAJA: RSC: Update --- tpl/RAJA | 2 +- tpl/kokkos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tpl/RAJA b/tpl/RAJA index c8734ef9a..62c139241 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit c8734ef9aeacff423998bdc2615119bdd41cc963 +Subproject commit 62c1392419650850b7b264ead795d0877bd52b3e diff --git a/tpl/kokkos b/tpl/kokkos index 1a0c2ff6d..2834f94af 160000 --- a/tpl/kokkos +++ b/tpl/kokkos @@ -1 +1 @@ -Subproject commit 1a0c2ff6daf1068c65529ec04c2c046177847869 +Subproject commit 2834f94af9b01debf67c1aaa3f0eb0c903d72c8d From f30d4f527937efc6d2f06859a0d35a8ad37da4e0 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:37:14 +0200 Subject: [PATCH 07/89] RAJA: RSC: Add support for c++17 --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index 62c139241..ff269f8c2 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 62c1392419650850b7b264ead795d0877bd52b3e +Subproject commit ff269f8c228d9cf2c77d942cc9579c47156ebad4 From 0c80d33d41549d20c5cf7fffd404a00e106d3944 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:56:15 +0200 Subject: [PATCH 08/89] Update local CI jobs to rocm 6.4.1 and cce 19 --- .gitlab/jobs/tioga.yml | 12 ++++++------ .gitlab/jobs/tuolumne.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index baa0d96c9..4123c67e6 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -27,17 +27,17 @@ # ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully # describe the spec here. -cce_17_0_1: +cce_19_0_0: variables: - SPEC: "~shared +openmp %cce@=17.0.1 ^blt@develop" + SPEC: "~shared +openmp %cce@=19.0.0 ^blt@develop" extends: .job_on_tioga -rocmcc_6_3_0_hip_openmp: +rocmcc_6_4_1_hip_openmp: variables: - SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .job_on_tioga -rocmcc_6_3_0_hip_openmp_mpi: +rocmcc_6_4_1_hip_openmp_mpi: variables: - SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .job_on_tioga diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 498aa976b..54e365bf0 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -27,17 +27,17 @@ # ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully # describe the spec here. -cce_17_0_1: +cce_19_0_0: variables: - SPEC: "~shared +openmp %cce@=17.0.1 ^blt@develop" + SPEC: "~shared +openmp %cce@=19.0.0 ^blt@develop" extends: .job_on_tuolumne -rocmcc_6_3_0_hip_openmp: +rocmcc_6_4_1_hip_openmp: variables: - SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .job_on_tuolumne -rocmcc_6_3_0_hip_openmp_mpi: +rocmcc_6_4_1_hip_openmp_mpi: variables: - SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.3.0 ^hip@6.3.0 ^blt@develop" + SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .job_on_tuolumne From e33a4298d9f225a4079b2633f1f014e344832949 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 8 Jul 2025 14:50:43 +0200 Subject: [PATCH 09/89] Update RSC through RAJA, use ci reservation on dane --- .gitlab/custom-jobs-and-variables.yml | 4 ++-- tpl/RAJA | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 1566c1c9c..0f973e5da 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -26,9 +26,9 @@ variables: # Dane # Arguments for top level allocation - DANE_SHARED_ALLOC: "--exclusive --time=45 --nodes=1" + DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=45 --nodes=1" # Arguments for job level allocation - DANE_JOB_ALLOC: "--nodes=1" + DANE_JOB_ALLOC: "--reservation=ci --nodes=1" # Project specific variants for dane PROJECT_DANE_VARIANTS: "~shared +openmp" # Project specific deps for dane diff --git a/tpl/RAJA b/tpl/RAJA index ff269f8c2..33e07a221 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit ff269f8c228d9cf2c77d942cc9579c47156ebad4 +Subproject commit 33e07a2219fcc61c0e05ad7a4696c11449ab3a0a From ab0fcc45a615f90758fd32b8cb0608c2769143d4 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:17:40 +0200 Subject: [PATCH 10/89] Update CMakeLists.txt to allow Cache values of BLT_CXX_STD --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d2211207a..eec17937a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,10 +19,13 @@ option(ENABLE_KOKKOS "Include Kokkos implementations of the kernels in the RAJA if (ENABLE_KOKKOS OR RAJA_ENABLE_SYCL) set(CMAKE_CXX_STANDARD 17) - set(BLT_CXX_STD c++17) + set(BLT_CXX_STD c++17 CACHE STRING "C++ standard" FORCE) else() - set(CMAKE_CXX_STANDARD 14) - set(BLT_CXX_STD c++14) + # Only set if not already in cache + if(NOT DEFINED CACHE{BLT_CXX_STD}) + set(CMAKE_CXX_STANDARD 14) + set(BLT_CXX_STD c++14 CACHE STRING "C++ standard" FORCE) + endif() endif() include(blt/SetupBLT.cmake) From e4bbc77b3771d3432df521890044ef0f07247139 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 9 Jul 2025 16:22:09 +0200 Subject: [PATCH 11/89] Allow overlapping of MPI tests allocations in CI --- scripts/gitlab/build_and_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index e7cbe00c7..1dfd56d05 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -231,16 +231,16 @@ then rm -rf ${build_dir} 2>/dev/null mkdir -p ${build_dir} && cd ${build_dir} + timed_message "Building RAJAPerf" # We set the MPI tests command to allow overlapping. # Shared allocation: Allows build_and_test.sh to run within a sub-allocation (see CI config). # Use /dev/shm: Prevent MPI tests from running on a node where the build dir doesn't exist. cmake_options="" - if [[ "${truehostname}" == "ruby" || "${truehostname}" == "poodle" ]] + if [[ "${truehostname}" == "ruby" || "${truehostname}" == "poodle" || "${truehostname}" == "dane" ]] then cmake_options="-DBLT_MPI_COMMAND_APPEND:STRING=--overlap" fi - date $cmake_exe \ -C ${hostconfig_path} \ ${cmake_options} \ From f8f4f214d309902195a205b9f5d301d0350f94ff Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:17:00 +0200 Subject: [PATCH 12/89] Fix: gfx942 for tuolumne + variables to workaround bug --- .gitlab/jobs/tioga.yml | 6 ++++++ .gitlab/jobs/tuolumne.yml | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml index 4123c67e6..366704285 100644 --- a/.gitlab/jobs/tioga.yml +++ b/.gitlab/jobs/tioga.yml @@ -35,9 +35,15 @@ cce_19_0_0: rocmcc_6_4_1_hip_openmp: variables: SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + HSA_ENABLE_SDMA: "1" + HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE: "0" + GPU_FORCE_BLIT_COPY_SIZE: "0" extends: .job_on_tioga rocmcc_6_4_1_hip_openmp_mpi: variables: SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + HSA_ENABLE_SDMA: "1" + HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE: "0" + GPU_FORCE_BLIT_COPY_SIZE: "0" extends: .job_on_tioga diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 54e365bf0..fb6b05db5 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -34,10 +34,16 @@ cce_19_0_0: rocmcc_6_4_1_hip_openmp: variables: - SPEC: "~shared +rocm +openmp amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + SPEC: "~shared +rocm +openmp amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + HSA_ENABLE_SDMA: "1" + HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE: "0" + GPU_FORCE_BLIT_COPY_SIZE: "0" extends: .job_on_tuolumne rocmcc_6_4_1_hip_openmp_mpi: variables: - SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx90a %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + SPEC: "~shared +rocm +openmp +mpi amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + HSA_ENABLE_SDMA: "1" + HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE: "0" + GPU_FORCE_BLIT_COPY_SIZE: "0" extends: .job_on_tuolumne From 8a40f07c240f3269bfbe8d0d97949928beddec6d Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:22:26 +0200 Subject: [PATCH 13/89] RAJA Update: RSC Fix: gfx942 for tuolumne --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index 33e07a221..6e2234af9 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 33e07a2219fcc61c0e05ad7a4696c11449ab3a0a +Subproject commit 6e2234af9c140566fe3f2d252405693f63e0e116 From 38e5147c691e7e32bfa8eee7aaacc4e55d04a16c Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:28:25 +0200 Subject: [PATCH 14/89] RAJA: Point at RSC release tag --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index 6e2234af9..9ac115f9b 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 6e2234af9c140566fe3f2d252405693f63e0e116 +Subproject commit 9ac115f9ba4cd298cce9a513194f14ad7d2b0fc3 From 52fa51d4097017c78442a1eaf9a785dc2e8963af Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:32:50 +0200 Subject: [PATCH 15/89] Just adding the jobs to CI --- .gitlab/jobs/dane.yml | 7 +++++++ .gitlab/jobs/tuolumne.yml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/.gitlab/jobs/dane.yml b/.gitlab/jobs/dane.yml index 00f1e7f11..54af26b18 100644 --- a/.gitlab/jobs/dane.yml +++ b/.gitlab/jobs/dane.yml @@ -47,3 +47,10 @@ oneapi_2023_2_1_mpi: variables: SPEC: "~shared +openmp +mpi +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ^mvapich2 ^blt@develop" extends: .job_on_dane + +# TODO: Should I run with +omptask ? +clang_14_0_6_caliper: + variables: + CALI_CONFIG: "event-trace" + SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" + extends: .job_on_dane diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index fb6b05db5..2cd19f0c9 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -32,6 +32,12 @@ cce_19_0_0: SPEC: "~shared +openmp %cce@=19.0.0 ^blt@develop" extends: .job_on_tuolumne +rocmcc_6_4_1_hip_openmp_caliper: + variables: + CALI_CONFIG: "event-trace" + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + extends: .job_on_tuolumne + rocmcc_6_4_1_hip_openmp: variables: SPEC: "~shared +rocm +openmp amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" From f28bfcdf64c7734993ad9c2484a5feb832c5fc8c Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 16 Jul 2025 16:38:01 +0200 Subject: [PATCH 16/89] Create a new pipeline for performance jobs on any machine --- .gitlab-ci.yml | 18 ++++++++++++++++-- .gitlab/custom-jobs-and-variables.yml | 4 ++++ .gitlab/jobs/dane.yml | 7 ------- .gitlab/jobs/performances.yml | 20 ++++++++++++++++++++ .gitlab/jobs/tuolumne.yml | 6 ------ 5 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 .gitlab/jobs/performances.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12beab12e..15afeda8d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,6 +67,7 @@ variables: stages: - prerequisites - build-and-test + - performance-measurements # Template for jobs triggering a build-and-test sub-pipeline: .build-and-test: @@ -75,7 +76,7 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'v2025.06.0' + ref: 'woptim/caliper-ci' file: 'pipelines/${CI_MACHINE}.yml' - artifact: '${CI_MACHINE}-jobs.yml' job: 'generate-job-lists' @@ -83,13 +84,26 @@ stages: forward: pipeline_variables: true +performance-measurements: + stage: performance-measurements + trigger: + include: + - local: '.gitlab/custom-jobs-and-variables.yml' + - project: 'radiuss/radiuss-shared-ci' + ref: 'woptim/caliper-ci' + file: 'pipelines/performances.yml' + - local: '.gitlab/jobs/performances.yml' + strategy: depend + forward: + pipeline_variables: true + include: # Sets ID tokens for every job using `default:` - project: 'lc-templates/id_tokens' file: 'id_tokens.yml' # [Optional] checks preliminary to running the actual CI test #- project: 'radiuss/radiuss-shared-ci' - # ref: 'v2025.06.0' + # ref: 'woptim/caliper-ci' # file: 'utilities/preliminary-ignore-draft-pr.yml' # pipelines subscribed by the project - local: '.gitlab/subscribed-pipelines.yml' diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 0f973e5da..fc7a5f90f 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -19,6 +19,8 @@ variables: # Arguments for job level allocation # Note: We repeat the reservation, necessary when jobs are manually re-triggered. RUBY_JOB_ALLOC: "--reservation=ci --nodes=1" +# Arguments for performance job allocation (dedicated allocation with no overlapping). + RUBY_PERF_ALLOC: "--exclusive --reservation=ci --time=15 --nodes=1" # Project specific variants for ruby PROJECT_RUBY_VARIANTS: "~shared +openmp" # Project specific deps for ruby @@ -59,6 +61,8 @@ variables: TUOLUMNE_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=31m --nodes=1 -o per-resource.count=2" # Arguments for job level allocation TUOLUMNE_JOB_ALLOC: "--nodes=1 --begin-time=+5s" +# Arguments for performance job allocation (dedicated allocation with no overlapping). + TUOLUMNE_PERF_ALLOC: "--queue=pci --exclusive --time-limit=15m --nodes=1 --begin-time=+5s" # Project specific variants for tuolumne PROJECT_TUOLUMNE_VARIANTS: "~shared +openmp" # Project specific deps for tuolumne diff --git a/.gitlab/jobs/dane.yml b/.gitlab/jobs/dane.yml index 54af26b18..00f1e7f11 100644 --- a/.gitlab/jobs/dane.yml +++ b/.gitlab/jobs/dane.yml @@ -47,10 +47,3 @@ oneapi_2023_2_1_mpi: variables: SPEC: "~shared +openmp +mpi +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ^mvapich2 ^blt@develop" extends: .job_on_dane - -# TODO: Should I run with +omptask ? -clang_14_0_6_caliper: - variables: - CALI_CONFIG: "event-trace" - SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" - extends: .job_on_dane diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml new file mode 100644 index 000000000..52dfc3305 --- /dev/null +++ b/.gitlab/jobs/performances.yml @@ -0,0 +1,20 @@ +############################################################################## +# Copyright (c) 2025, Lawrence Livermore National Security, LLC +# and RAJA Performance Suite project contributors. +# See the RAJAPerf/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################## + +# TODO: Should I run with +omptask ? +clang_14_0_6_caliper: + variables: + CALI_CONFIG: "event-trace" + SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" + extends: .perf_on_dane + +rocmcc_6_4_1_hip_openmp_caliper: + variables: + CALI_CONFIG: "event-trace" + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + extends: .perf_on_tuolumne diff --git a/.gitlab/jobs/tuolumne.yml b/.gitlab/jobs/tuolumne.yml index 2cd19f0c9..fb6b05db5 100644 --- a/.gitlab/jobs/tuolumne.yml +++ b/.gitlab/jobs/tuolumne.yml @@ -32,12 +32,6 @@ cce_19_0_0: SPEC: "~shared +openmp %cce@=19.0.0 ^blt@develop" extends: .job_on_tuolumne -rocmcc_6_4_1_hip_openmp_caliper: - variables: - CALI_CONFIG: "event-trace" - SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" - extends: .job_on_tuolumne - rocmcc_6_4_1_hip_openmp: variables: SPEC: "~shared +rocm +openmp amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" From 65d75cb247e3899f89ddf6e8806f129d7124207e Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:09:11 +0200 Subject: [PATCH 17/89] Leverage .perf_job --- .gitlab/custom-jobs-and-variables.yml | 4 ++++ .gitlab/jobs/performances.yml | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index fc7a5f90f..7045fe650 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -89,6 +89,10 @@ variables: paths: - ./*.cmake +.perf_job: + variables: + CALI_CONFIG: "event-trace" + .reproducer_vars: script: - | diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 52dfc3305..c7ce28669 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -9,12 +9,10 @@ # TODO: Should I run with +omptask ? clang_14_0_6_caliper: variables: - CALI_CONFIG: "event-trace" SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" extends: .perf_on_dane rocmcc_6_4_1_hip_openmp_caliper: variables: - CALI_CONFIG: "event-trace" SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .perf_on_tuolumne From e0a429fb9a28e4c474a302731345a9712a3eda44 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 23 Jul 2025 15:56:25 +0200 Subject: [PATCH 18/89] Update RAJA --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index 9ac115f9b..ec35cd993 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 9ac115f9ba4cd298cce9a513194f14ad7d2b0fc3 +Subproject commit ec35cd9934ee736dd837ce939f37dd432d1e776e From 71f8820bbe1fb4f89f0fcd159863e069dfb3d659 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 23 Jul 2025 16:24:41 +0200 Subject: [PATCH 19/89] Copy performance data to artifact directory defined by shared CI config --- scripts/gitlab/build_and_test.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 1dfd56d05..b24458a61 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -288,6 +288,20 @@ then timed_message "RAJA Perf Suite tests completed" fi +# Export performance data +# This is CI specific, see RADIUSS Shared CI performance pipeline. +if [[ "${option}" != "--build-only" && -d "${PERF_ARTIFACT_DIR}" ]] +then + timed_message "Exporting performance data" + if [[ -d "${build_dir}/bin" ]] + then + cp ${build_dir}/bin/*.cali ${PERF_ARTIFACT_DIR} + cp ${build_dir}/bin/*.csv ${PERF_ARTIFACT_DIR} + else + echo "[Warning]: No bin directory found in build directory." + fi +fi + timed_message "Cleaning up" make clean From 635341aa77c892491aeb0c410874311f6003a638 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 23 Jul 2025 19:20:55 +0200 Subject: [PATCH 20/89] Update RAJA to develop with RSC 2025-06-0 --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index ec35cd993..178fe5960 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit ec35cd9934ee736dd837ce939f37dd432d1e776e +Subproject commit 178fe5960668ada99bf0d970ada544fd20bf1139 From 6ad3d28a5b2f92b67faacf2299e4f077ab3da200 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Wed, 23 Jul 2025 21:26:23 +0200 Subject: [PATCH 21/89] Allow for undefined PERF_ARTIFACT_DIR --- scripts/gitlab/build_and_test.sh | 10 +++++----- tpl/RAJA | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index b24458a61..b515fb052 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -29,6 +29,7 @@ use_dev_shm=${USE_DEV_SHM:-true} spack_debug=${SPACK_DEBUG:-false} debug_mode=${DEBUG_MODE:-false} push_to_registry=${PUSH_TO_REGISTRY:-true} +perf_artifact_dir=${PERF_ARTIFACT_DIR:-""} raja_version=${UPDATE_RAJA:-""} sys_type=${SYS_TYPE:-""} @@ -258,7 +259,6 @@ fi # Test if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then - if [[ ! -d ${build_dir} ]] then echo "[Error]: Build directory not found : ${build_dir}" && exit 1 @@ -290,15 +290,15 @@ fi # Export performance data # This is CI specific, see RADIUSS Shared CI performance pipeline. -if [[ "${option}" != "--build-only" && -d "${PERF_ARTIFACT_DIR}" ]] +if [[ "${option}" != "--build-only" && -d "${perf_artifact_dir}" ]] then timed_message "Exporting performance data" if [[ -d "${build_dir}/bin" ]] then - cp ${build_dir}/bin/*.cali ${PERF_ARTIFACT_DIR} - cp ${build_dir}/bin/*.csv ${PERF_ARTIFACT_DIR} + cp ${build_dir}/bin/*.cali ${perf_artifact_dir} + cp ${build_dir}/bin/*.csv ${perf_artifact_dir} else - echo "[Warning]: No bin directory found in build directory." + echo "[Warning]: No bin directory found in build directory." && exit 1 fi fi diff --git a/tpl/RAJA b/tpl/RAJA index 178fe5960..ec35cd993 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit 178fe5960668ada99bf0d970ada544fd20bf1139 +Subproject commit ec35cd9934ee736dd837ce939f37dd432d1e776e From d35135f42dead11bf81895b936291729618315a7 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 10:18:33 +0200 Subject: [PATCH 22/89] Update C++ requirement in azure pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0f9a16593..08d30331b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ jobs: vmImage: 'ubuntu-latest' variables: DOCKER_BUILDKIT: '1' - CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=14' + CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=17' steps: - checkout: self clean: boolean From cb83ab9571fc8cd2e9d383a80bfbe5fc6df5d905 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 10:18:33 +0200 Subject: [PATCH 23/89] Update C++ requirement in azure pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0f9a16593..08d30331b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ jobs: vmImage: 'ubuntu-latest' variables: DOCKER_BUILDKIT: '1' - CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=14' + CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=17' steps: - checkout: self clean: boolean From 2930fd92c16460c5ff61158ea33b9ba5e9609e35 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 12:03:35 +0200 Subject: [PATCH 24/89] At least get macos build to use c++17 std --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 08d30331b..0761cba14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -66,7 +66,7 @@ jobs: pool: vmImage: 'macOS-latest' variables: - CMAKE_EXTRA_FLAGS: '-DENABLE_OPENMP=Off' + CMAKE_EXTRA_FLAGS: '-DENABLE_OPENMP=Off -DCMAKE_CXX_STANDARD=17' steps: - checkout: self clean: boolean From 610fc35d46766cb18bccebdca4ccc011d0bb8f98 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 12:14:12 +0200 Subject: [PATCH 25/89] Use same logic as in RAJA for CXX STD definition in CMake --- CMakeLists.txt | 62 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eec17937a..4ff8858e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,20 +15,60 @@ else() cmake_minimum_required(VERSION 3.20) endif() -option(ENABLE_KOKKOS "Include Kokkos implementations of the kernels in the RAJA Perfsuite" Off) - -if (ENABLE_KOKKOS OR RAJA_ENABLE_SYCL) - set(CMAKE_CXX_STANDARD 17) - set(BLT_CXX_STD c++17 CACHE STRING "C++ standard" FORCE) -else() - # Only set if not already in cache - if(NOT DEFINED CACHE{BLT_CXX_STD}) - set(CMAKE_CXX_STANDARD 14) - set(BLT_CXX_STD c++14 CACHE STRING "C++ standard" FORCE) +# Detect C++ standard and add appropriate flag _before_ loading BLT +set(COMPILERS_KNOWN_TO_CMAKE33 AppleClang Clang GNU MSVC) + +include(CheckCXXCompilerFlag) +if(NOT DEFINED BLT_CXX_STD) + if("cxx_std_20" IN_LIST CMAKE_CXX_KNOWN_FEATURES) + set(BLT_CXX_STD c++20 CACHE STRING "Version of C++ standard") + message("Using C++ standard: ${BLT_CXX_STD}") + elseif("cxx_std_17" IN_LIST CMAKE_CXX_KNOWN_FEATURES) + set(BLT_CXX_STD c++17 CACHE STRING "Version of C++ standard") + message("Using C++ standard: ${BLT_CXX_STD}") + elseif("${CMAKE_CXX_COMPILER_ID}" IN_LIST COMPILERS_KNOWN_TO_CMAKE33) + set(BLT_CXX_STD c++17 CACHE STRING "Version of C++ standard") + message("Using C++ standard: ${BLT_CXX_STD}") + else() #cmake has no idea what to do, do it ourselves... + set(flag_var "c++17") + CHECK_CXX_COMPILER_FLAG("-std=${flag_var}" COMPILER_SUPPORTS_${flag_var}) + if(COMPILER_SUPPORTS_${flag_var}) + set(BLT_CXX_STD ${flag_var} CACHE STRING "Version of C++ standard") + message("Using C++ standard: ${BLT_CXX_STD}") + endif() + unset(flag_var) + endif() +else() #check BLT_CXX_STD is high enough by disallowing the only invalid option + if(("${BLT_CXX_STD}" STREQUAL "c++98") OR + ("${BLT_CXX_STD}" STREQUAL "c++11") OR + ("${BLT_CXX_STD}" STREQUAL "c++14")) + message(FATAL_ERROR "RAJA requires minimum C++ standard of c++17") endif() +endif(NOT DEFINED BLT_CXX_STD) + +set(CMAKE_CXX_EXTENSIONS OFF) + +if (NOT BLT_LOADED) + if (DEFINED BLT_SOURCE_DIR) + if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake) + message(FATAL_ERROR "Given BLT_SOURCE_DIR does not contain SetupBLT.cmake") + endif() + else () + set (BLT_SOURCE_DIR ${PROJECT_SOURCE_DIR}/blt CACHE PATH "") + + if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake) + message(FATAL_ERROR "\ + The BLT submodule is not present. \ + If in git repository run the following two commands:\n \ + git submodule init\n \ + git submodule update") + endif () + endif () + + include(${BLT_SOURCE_DIR}/SetupBLT.cmake) endif() -include(blt/SetupBLT.cmake) +option(ENABLE_KOKKOS "Include Kokkos implementations of the kernels in the RAJA Perfsuite" Off) # # Define RAJA PERFSUITE settings... From 7fdbfd9f2c0fd2469cb849be7dc54445a864966d Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 12:14:28 +0200 Subject: [PATCH 26/89] Revert "Update C++ requirement in azure pipelines" This reverts commit cb83ab9571fc8cd2e9d383a80bfbe5fc6df5d905. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0761cba14..7671eec0a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ jobs: vmImage: 'ubuntu-latest' variables: DOCKER_BUILDKIT: '1' - CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=17' + CMAKE_EXTRA_FLAGS: '-DENABLE_DEVELOPER_BENCHMARKS=On -DENABLE_DEVELOPER_DEFAULTS=On -DCMAKE_CXX_STANDARD=14' steps: - checkout: self clean: boolean From 6b377bf9aa80da2f424ff4a7083804ca834a8f97 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 12:14:40 +0200 Subject: [PATCH 27/89] Revert "At least get macos build to use c++17 std" This reverts commit 2930fd92c16460c5ff61158ea33b9ba5e9609e35. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7671eec0a..0f9a16593 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -66,7 +66,7 @@ jobs: pool: vmImage: 'macOS-latest' variables: - CMAKE_EXTRA_FLAGS: '-DENABLE_OPENMP=Off -DCMAKE_CXX_STANDARD=17' + CMAKE_EXTRA_FLAGS: '-DENABLE_OPENMP=Off' steps: - checkout: self clean: boolean From b077347dabf70a0f9ad150b214acecec98cfcad8 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 24 Jul 2025 14:48:38 +0200 Subject: [PATCH 28/89] Update RAJA to develop with RSC 2025-06-0 --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index ec35cd993..178fe5960 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit ec35cd9934ee736dd837ce939f37dd432d1e776e +Subproject commit 178fe5960668ada99bf0d970ada544fd20bf1139 From 10ee31bed31f08dc5e5e2514eb50d43eee987ecd Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:42:20 +0200 Subject: [PATCH 29/89] Update CI config for latest Shared CI changes --- .gitlab/custom-jobs-and-variables.yml | 2 +- .gitlab/jobs/performances.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 7045fe650..ee97e066d 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -89,7 +89,7 @@ variables: paths: - ./*.cmake -.perf_job: +.perf_custom: variables: CALI_CONFIG: "event-trace" diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index c7ce28669..0fa1c47df 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -16,3 +16,14 @@ rocmcc_6_4_1_hip_openmp_caliper: variables: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" extends: .perf_on_tuolumne + +process_results: + extends: .process_results + script: + - ls -l ${PERF_ARTIFACTS_DIR} + +report_results: + extends: .report_results + script: + - echo "No results to report, this is a demo pipeline." + From 2a076b903aa3ba2d7869e8468f1e6e1cc0ee0391 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:48:23 +0200 Subject: [PATCH 30/89] rename a template --- .gitlab/custom-jobs-and-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index ee97e066d..fd0bd974f 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -89,7 +89,7 @@ variables: paths: - ./*.cmake -.perf_custom: +.custom_perf: variables: CALI_CONFIG: "event-trace" From 8851b153fcd80232e6178ee4e3e11a2487da270a Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 29 Jul 2025 15:16:09 +0200 Subject: [PATCH 31/89] Fix: specify dependencies --- .gitlab/jobs/performances.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 0fa1c47df..a821864ec 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -21,9 +21,13 @@ process_results: extends: .process_results script: - ls -l ${PERF_ARTIFACTS_DIR} + dependencies: + - clang_14_0_6_caliper + - rocmcc_6_4_1_hip_openmp_caliper report_results: extends: .report_results script: - echo "No results to report, this is a demo pipeline." - + dependencies: + - process_results From fbb0a4407ee84a835f50f20184d712b59226ef63 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:10:38 +0200 Subject: [PATCH 32/89] Fix template job naming --- .gitlab/jobs/performances.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index a821864ec..38c8f172a 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -18,7 +18,7 @@ rocmcc_6_4_1_hip_openmp_caliper: extends: .perf_on_tuolumne process_results: - extends: .process_results + extends: .results_processing script: - ls -l ${PERF_ARTIFACTS_DIR} dependencies: @@ -26,7 +26,7 @@ process_results: - rocmcc_6_4_1_hip_openmp_caliper report_results: - extends: .report_results + extends: .results_reporting script: - echo "No results to report, this is a demo pipeline." dependencies: From e0de160599c16b55272bcd8c9f1f65936c3e5f48 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:58:44 +0200 Subject: [PATCH 33/89] Stop running CI on ruby --- .gitlab/custom-jobs-and-variables.yml | 15 +------- .gitlab/jobs/ruby.yml | 50 --------------------------- .gitlab/subscribed-pipelines.yml | 14 -------- scripts/gitlab/build_and_test.sh | 4 +-- 4 files changed, 3 insertions(+), 80 deletions(-) delete mode 100644 .gitlab/jobs/ruby.yml diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index fd0bd974f..a1e80ac2d 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -8,24 +8,11 @@ # We define the following GitLab pipeline variables: variables: -# On LLNL's ruby, this pipeline creates only one allocation shared among jobs +# On LLNL's machines, this pipeline creates only one allocation shared among jobs # in order to save time and resources. This allocation has to be uniquely named # so that we are sure to retrieve it and avoid collisions. ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} -# Ruby -# Arguments for top level allocation - RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --time=45 --nodes=1" -# Arguments for job level allocation -# Note: We repeat the reservation, necessary when jobs are manually re-triggered. - RUBY_JOB_ALLOC: "--reservation=ci --nodes=1" -# Arguments for performance job allocation (dedicated allocation with no overlapping). - RUBY_PERF_ALLOC: "--exclusive --reservation=ci --time=15 --nodes=1" -# Project specific variants for ruby - PROJECT_RUBY_VARIANTS: "~shared +openmp" -# Project specific deps for ruby - PROJECT_RUBY_DEPS: "^blt@develop " - # Dane # Arguments for top level allocation DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=45 --nodes=1" diff --git a/.gitlab/jobs/ruby.yml b/.gitlab/jobs/ruby.yml deleted file mode 100644 index d837b9f35..000000000 --- a/.gitlab/jobs/ruby.yml +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################### -# Copyright (c) 2017-25, Lawrence Livermore National Security, LLC -# and RAJA Performance Suite project contributors. -# See the RAJAPerf/LICENSE file for details. -# -# SPDX-License-Identifier: (BSD-3-Clause) -############################################################################## - -# Override reproducer section to define project specific variables. -.ruby_reproducer_vars: - script: - - !reference [.reproducer_vars, script] - -####################### -# Overridden shared jobs -######################## -# We duplicate the shared jobs description and add necessary changes for RAJA. -# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that -# the comparison with the original job is easier. - -# custom variant -oneapi_2023_2_1: - variables: - SPEC: "${PROJECT_RUBY_VARIANTS} +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ${PROJECT_RUBY_DEPS}" - extends: .job_on_ruby - -# omptask variant -clang_14_0_6: - variables: - SPEC: "${PROJECT_RUBY_VARIANTS} +omptask %clang@=14.0.6 ${PROJECT_RUBY_DEPS}" - extends: .job_on_ruby - -# omptask variant -gcc_10_3_1: - variables: - SPEC: "${PROJECT_RUBY_VARIANTS} +omptask %gcc@=10.3.1 ${PROJECT_RUBY_DEPS}" - RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1" - extends: .job_on_ruby - -############ -# Extra jobs -############ -# We do not recommend using ${PROJECT__VARIANTS} and -# ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully -# describe the spec here. - -oneapi_2023_2_1_mpi: - variables: - SPEC: "~shared +openmp +mpi +lowopttest cxxflags==-fp-model=precise %oneapi@=2023.2.1 ^mvapich2 ^blt@develop" - extends: .job_on_ruby diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index ab91f85e5..e609ac499 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -41,7 +41,6 @@ generate-job-lists: RADIUSS_JOBS_PATH: "tpl/RAJA/scripts/radiuss-spack-configs/gitlab/radiuss-jobs" LOCAL_JOBS_PATH: ".gitlab/jobs" script: - - cat ${RADIUSS_JOBS_PATH}/ruby.yml ${LOCAL_JOBS_PATH}/ruby.yml > ruby-jobs.yml - cat ${RADIUSS_JOBS_PATH}/dane.yml ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml - cat ${RADIUSS_JOBS_PATH}/lassen.yml ${LOCAL_JOBS_PATH}/lassen.yml > lassen-jobs.yml - cat ${RADIUSS_JOBS_PATH}/corona.yml ${LOCAL_JOBS_PATH}/corona.yml > corona-jobs.yml @@ -49,25 +48,12 @@ generate-job-lists: - cat ${RADIUSS_JOBS_PATH}/tuolumne.yml ${LOCAL_JOBS_PATH}/tuolumne.yml > tuolumne-jobs.yml artifacts: paths: - - ruby-jobs.yml - dane-jobs.yml - lassen-jobs.yml - corona-jobs.yml - tioga-jobs.yml - tuolumne-jobs.yml -# RUBY -ruby-up-check: - variables: - CI_MACHINE: "ruby" - extends: [.machine-check] - -ruby-build-and-test: - variables: - CI_MACHINE: "ruby" - needs: [ruby-up-check, generate-job-lists] - extends: [.build-and-test] - # DANE dane-up-check: variables: diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index b515fb052..34d1f2f92 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -212,7 +212,7 @@ then timed_message "Cleaning working directory" # Map CPU core allocations - declare -A core_counts=(["lassen"]=40 ["ruby"]=28 ["poodle"]=28 ["dane"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) + declare -A core_counts=(["lassen"]=40 ["poodle"]=28 ["dane"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) # If using Multi-project, set up the submodule if [[ -n ${raja_version} ]] @@ -237,7 +237,7 @@ then # Shared allocation: Allows build_and_test.sh to run within a sub-allocation (see CI config). # Use /dev/shm: Prevent MPI tests from running on a node where the build dir doesn't exist. cmake_options="" - if [[ "${truehostname}" == "ruby" || "${truehostname}" == "poodle" || "${truehostname}" == "dane" ]] + if [[ "${truehostname}" == "poodle" || "${truehostname}" == "dane" ]] then cmake_options="-DBLT_MPI_COMMAND_APPEND:STRING=--overlap" fi From 45ddb107fc77b43f3fd3a4fba598f3fa1de6c0a9 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:03:01 -0600 Subject: [PATCH 34/89] Add caliper to json conversion script for the performance CI --- scripts/gitlab/caliper_to_github_benchmark.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 scripts/gitlab/caliper_to_github_benchmark.py diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py new file mode 100755 index 000000000..073b6142f --- /dev/null +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Convert Caliper .cali output to GitHub benchmark action JSON format. + +Usage: python caliper_to_github_benchmark.py "pattern" output.json + python caliper_to_github_benchmark.py input.cali output.json + python caliper_to_github_benchmark.py "/path/to/*_*-*.cali" output.json +""" +import caliperreader as cr +import glob +import json +import sys +from pathlib import Path + +METRIC = 'avg#inclusive#sum#time.duration' + + +def caliper_to_benchmark_json(input_file): + """Convert one Caliper file to benchmark format. + Args: + input_file: Path to Caliper file + Returns: + List of benchmark dictionaries + """ + + filename = Path(input_file).stem + variant = filename.split('-')[0] + + reader = cr.CaliperReader() + reader.read(input_file) + + benchmarks = [] + + for record in reader.records: + path = record['path'] if 'path' in record else 'UNKNOWN' + time_s = record[METRIC] if METRIC in record else '0' + + # Gather leaf-level kernel paths like "RAJAPerf/Group/KernelName" + if isinstance(path, list) and len(path) >= 3: + kernel_name = path[-1] + benchmark = { + "name": f"{variant}_{kernel_name}", + "unit": "s", + "value": time_s + } + benchmarks.append(benchmark) + + return benchmarks + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit("Usage: caliper_to_github_benchmark.py \"input_file_glob_pattern\" output.json") + + pattern = sys.argv[1] + output_file = sys.argv[2] + + cali_files = glob.glob(pattern) + + if not cali_files: + sys.exit(f"Error: No .cali files found matching pattern: {pattern}") + + print(f"Found {len(cali_files)} Caliper files to process") + + all_benchmarks = [] + for cali_file in cali_files: + print(f"Processing {cali_file}") + benchmarks = caliper_to_benchmark_json(cali_file) + all_benchmarks.extend(benchmarks) + + with open(output_file, 'w') as f: + json.dump(all_benchmarks, f, indent=2) + + print(f"Converted {len(all_benchmarks)} benchmarks from {len(cali_files)} files to {output_file}") From 67b155f9cbd08bac1b05e2b96a4e341508803964 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:05:05 -0600 Subject: [PATCH 35/89] Update build script to run raja perf with specified kernels and variants --- scripts/gitlab/build_and_test.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 34d1f2f92..1b7b4e0eb 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -30,6 +30,9 @@ spack_debug=${SPACK_DEBUG:-false} debug_mode=${DEBUG_MODE:-false} push_to_registry=${PUSH_TO_REGISTRY:-true} perf_artifact_dir=${PERF_ARTIFACT_DIR:-""} +performance_tests=${PERFORMANCE_TESTS:-false} +kernels=${KERNELS:-""} +variants=${VARIANTS:-""} raja_version=${UPDATE_RAJA:-""} sys_type=${SYS_TYPE:-""} @@ -257,7 +260,7 @@ then fi # Test -if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} +if [[ "${option}" != "--build-only" && "${performance_tests}" != "true" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then if [[ ! -d ${build_dir} ]] then @@ -288,21 +291,22 @@ then timed_message "RAJA Perf Suite tests completed" fi -# Export performance data -# This is CI specific, see RADIUSS Shared CI performance pipeline. -if [[ "${option}" != "--build-only" && -d "${perf_artifact_dir}" ]] +# Performance tests +if [[ "${option}" != "--build-only" && "${performance_tests}" == "true" ]] then - timed_message "Exporting performance data" - if [[ -d "${build_dir}/bin" ]] - then - cp ${build_dir}/bin/*.cali ${perf_artifact_dir} - cp ${build_dir}/bin/*.csv ${perf_artifact_dir} - else - echo "[Warning]: No bin directory found in build directory." && exit 1 - fi + + cd ${project_dir} + mkdir -p ${perf_artifact_dir} + + timed_message "Performance tests for RAJA Perf Suite" + + ${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} + + timed_message "Performance tests for RAJA Perf Suite completed" fi timed_message "Cleaning up" +cd ${build_dir} make clean timed_message "Build and test completed" From c05b0c20268ffa482e88890784b29e5cddcedf51 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:06:00 -0600 Subject: [PATCH 36/89] Add dane performance allocation variable --- .gitlab/custom-jobs-and-variables.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index a1e80ac2d..0f1a68e2e 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -18,6 +18,8 @@ variables: DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=45 --nodes=1" # Arguments for job level allocation DANE_JOB_ALLOC: "--reservation=ci --nodes=1" + # Arguments for performance job allocation (dedicated allocation with no overlapping). + DANE_PERF_ALLOC: "--exclusive --reservation=ci --time=15 --nodes=1" # Project specific variants for dane PROJECT_DANE_VARIANTS: "~shared +openmp" # Project specific deps for dane From 1ce79acd89ba84e6237545554f0d308e73796bbe Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:09:03 -0600 Subject: [PATCH 37/89] Update gitlab performance job --- .gitlab/jobs/performances.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 38c8f172a..48fd16bca 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -6,28 +6,35 @@ # SPDX-License-Identifier: (BSD-3-Clause) ############################################################################## +variables: + PERF_ARTIFACT_DIR: "performance-results" + PERF_RESULTS_FILE: "*_*-*.cali" + PERF_PROCESSED_FILE: "processed_results.json" + PERF_PROCESSING_CMD: "${CI_PROJECT_DIR}/scripts/gitlab/caliper_to_github_benchmark.py" + PERFORMANCE_TESTS: "true" + # TODO: Should I run with +omptask ? clang_14_0_6_caliper: variables: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" + KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" + VARIANTS: "RAJA_Seq RAJA_OpenMP" extends: .perf_on_dane rocmcc_6_4_1_hip_openmp_caliper: variables: - SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop" + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" + KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" + VARIANTS: "RAJA_HIP" extends: .perf_on_tuolumne process_results: - extends: .results_processing - script: - - ls -l ${PERF_ARTIFACTS_DIR} + extends: .caliper_to_gh_benchmark dependencies: - clang_14_0_6_caliper - rocmcc_6_4_1_hip_openmp_caliper report_results: - extends: .results_reporting - script: - - echo "No results to report, this is a demo pipeline." + extends: .report_to_gh_benchmark dependencies: - process_results From 9a4f9022363c7566a32082e06576005d9b5239be Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:40:38 -0600 Subject: [PATCH 38/89] Add GitHub benchmark workflow --- .github/workflows/benchmark.yml | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..6163cff0b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,71 @@ +name: Process Benchmark + +on: + workflow_dispatch: + inputs: + benchmark_data: + description: 'Base64 encoded benchmark data' + required: true + benchmark_name: + description: 'Name for benchmark group on the dashboard' + required: false + default: 'RAJAPerf benchmarks' + +permissions: + checks: write + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + store-and-visualize: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Find associated PR + id: findpr + run: | + PR_NUMBER=$(gh pr list --head ${{ github.ref_name }} --json number -q '.[0].number') + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Decode benchmark data + run: echo "${{ github.event.inputs.benchmark_data }}" | base64 -d > benchmark.json + + - name: Store benchmark result + id: benchmark + continue-on-error: true + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ${{ github.event.inputs.benchmark_name }} + tool: 'customSmallerIsBetter' + output-file-path: benchmark.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true # TODO remove later, for testing + # auto-push: ${{ github.ref == 'refs/heads/main' }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + comment-always: true + fail-on-alert: true + alert-threshold: '120%' + max-items-in-chart: 100 + + - name: Create Check for PRs + if: steps.findpr.outputs.pr_number != '' + uses: LouisBrunner/checks-action@v1.6.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + name: Performance Benchmark + sha: ${{ github.sha }} + conclusion: ${{ steps.benchmark.outcome == 'success' && 'success' || 'failure' }} + output: | + {"summary": "${{ steps.benchmark.outcome == 'success' && 'Performance check passed' || 'Performance regression detected!' }}"} + + - name: Final status + if: steps.benchmark.outcome != 'success' + run: exit 1 From 8cbcd37a9fe258e018b771c2130e462d1efd18d3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 14:54:17 -0600 Subject: [PATCH 39/89] Remove concurrency section as we want the benchmarks to get updated whenever they are triggered --- .github/workflows/benchmark.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6163cff0b..7be60df4a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,10 +16,6 @@ permissions: contents: write pull-requests: write -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - jobs: store-and-visualize: runs-on: ubuntu-latest From 44f41d11caf00def9622c5ef921d161c9f4713aa Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 11 Sep 2025 16:19:11 -0600 Subject: [PATCH 40/89] Try out summary tool --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7be60df4a..6f078f5d1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,6 +47,7 @@ jobs: gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench comment-always: true + summary-always: true fail-on-alert: true alert-threshold: '120%' max-items-in-chart: 100 From 0f5c3f899faf395a308587a964eee37e78c568cf Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 15 Sep 2025 14:18:40 -0600 Subject: [PATCH 41/89] Set OMP_NUM_THREADS to 16 --- .gitlab/jobs/performances.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 48fd16bca..3e84f44f8 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -19,6 +19,7 @@ clang_14_0_6_caliper: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" VARIANTS: "RAJA_Seq RAJA_OpenMP" + OMP_NUM_THREADS: 16" extends: .perf_on_dane rocmcc_6_4_1_hip_openmp_caliper: From ddb33130815b230f55c7d47bf1d566354e0b9063 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 15 Sep 2025 14:30:34 -0600 Subject: [PATCH 42/89] Print link to summary of performance tests --- .github/workflows/benchmark.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6f078f5d1..66206d4a2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -61,7 +61,11 @@ jobs: sha: ${{ github.sha }} conclusion: ${{ steps.benchmark.outcome == 'success' && 'success' || 'failure' }} output: | - {"summary": "${{ steps.benchmark.outcome == 'success' && 'Performance check passed' || 'Performance regression detected!' }}"} + { + "title": "Performance Results", + "summary": "${{ steps.benchmark.outcome == 'success' && 'Performance check passed' || 'Performance regression detected!' }}", + "text_description": "See job summary for detailed benchmark results: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } - name: Final status if: steps.benchmark.outcome != 'success' From 859ef02430e9f12603c12612edddb86df9d669ab Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 15 Sep 2025 14:33:45 -0600 Subject: [PATCH 43/89] Turn off comment always --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 66206d4a2..be493a85b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: # auto-push: ${{ github.ref == 'refs/heads/main' }} gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench - comment-always: true + comment-always: false summary-always: true fail-on-alert: true alert-threshold: '120%' From 4576261397bf5f3fd29d8766fab3b8a05de988f3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 15 Sep 2025 14:39:37 -0600 Subject: [PATCH 44/89] Fix branch name --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index be493a85b..d57abcd72 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -43,7 +43,7 @@ jobs: output-file-path: benchmark.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # TODO remove later, for testing - # auto-push: ${{ github.ref == 'refs/heads/main' }} + # auto-push: ${{ github.ref == 'refs/heads/develop' }} gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench comment-always: false From 097801af31863913fb1cbf48ca12290d3907faf5 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 15 Sep 2025 14:41:51 -0600 Subject: [PATCH 45/89] Make sure each check on PR has a unique name --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d57abcd72..92251d5a1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -57,7 +57,7 @@ jobs: uses: LouisBrunner/checks-action@v1.6.1 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Performance Benchmark + name: Performance Benchmark (${{ github.event.inputs.benchmark_name }}) sha: ${{ github.sha }} conclusion: ${{ steps.benchmark.outcome == 'success' && 'success' || 'failure' }} output: | From ea48821cef5e7a06fac400db40f67ad3bcb79d71 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 16 Sep 2025 14:39:39 -0600 Subject: [PATCH 46/89] Turn on comment on alert --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 92251d5a1..38ab52e1a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,6 +47,7 @@ jobs: gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench comment-always: false + comment-on-alert: true summary-always: true fail-on-alert: true alert-threshold: '120%' From ae3a920538af3d70393da6dac557b8e370709ed7 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 16 Sep 2025 17:07:41 -0600 Subject: [PATCH 47/89] Improve GitHub benchmark workflow --- .github/workflows/benchmark.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 38ab52e1a..9efc59869 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,7 +25,10 @@ jobs: - name: Find associated PR id: findpr run: | + echo "Looking for PR with head branch: ${{ github.ref_name }}" + gh pr list --head ${{ github.ref_name }} --json number,title PR_NUMBER=$(gh pr list --head ${{ github.ref_name }} --json number -q '.[0].number') + echo "Found PR number: $PR_NUMBER" echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -55,7 +58,7 @@ jobs: - name: Create Check for PRs if: steps.findpr.outputs.pr_number != '' - uses: LouisBrunner/checks-action@v1.6.1 + uses: LouisBrunner/checks-action@v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} name: Performance Benchmark (${{ github.event.inputs.benchmark_name }}) From a0ed80077192d5e0624ae9eea6a22414a9912b19 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 18 Sep 2025 18:13:37 -0600 Subject: [PATCH 48/89] Print adiak metadata when running performance tests --- .gitlab/custom-jobs-and-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 0f1a68e2e..f1a538e46 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -80,7 +80,7 @@ variables: .custom_perf: variables: - CALI_CONFIG: "event-trace" + CALI_CONFIG: "event-trace,runtime-report,print.metadata" .reproducer_vars: script: From d99d67f713afa879acbce7e33f20f1fb901b24d0 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 22 Sep 2025 18:02:32 -0600 Subject: [PATCH 49/89] Remove todo --- .gitlab/jobs/performances.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 3e84f44f8..687d7622f 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -13,7 +13,6 @@ variables: PERF_PROCESSING_CMD: "${CI_PROJECT_DIR}/scripts/gitlab/caliper_to_github_benchmark.py" PERFORMANCE_TESTS: "true" -# TODO: Should I run with +omptask ? clang_14_0_6_caliper: variables: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" From c54fec1e005c6a2d1eb33bb36124e713a1ec9d95 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 22 Sep 2025 18:03:35 -0600 Subject: [PATCH 50/89] Fix formating of variable --- .gitlab/jobs/performances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 687d7622f..ae4bf758c 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -18,7 +18,7 @@ clang_14_0_6_caliper: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" VARIANTS: "RAJA_Seq RAJA_OpenMP" - OMP_NUM_THREADS: 16" + OMP_NUM_THREADS: "16" extends: .perf_on_dane rocmcc_6_4_1_hip_openmp_caliper: From 6db1cd3d2fab4c8654a15a78beb9aa35a3dc5996 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 12:26:39 -0600 Subject: [PATCH 51/89] Run all basic kernels --- .gitlab/jobs/performances.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index ae4bf758c..4818febb2 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -16,7 +16,7 @@ variables: clang_14_0_6_caliper: variables: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" - KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" + KERNELS: "Basic" VARIANTS: "RAJA_Seq RAJA_OpenMP" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -24,7 +24,7 @@ clang_14_0_6_caliper: rocmcc_6_4_1_hip_openmp_caliper: variables: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" - KERNELS: "Basic_PI_ATOMIC Basic_DAXPY" + KERNELS: "Basic" VARIANTS: "RAJA_HIP" extends: .perf_on_tuolumne From 0a64bf5685e542c257642f7405604710c71ba111 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 14:15:26 -0600 Subject: [PATCH 52/89] Leave default/new as part of the variant name --- scripts/gitlab/caliper_to_github_benchmark.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 073b6142f..b942c2c20 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -23,8 +23,7 @@ def caliper_to_benchmark_json(input_file): List of benchmark dictionaries """ - filename = Path(input_file).stem - variant = filename.split('-')[0] + variant = Path(input_file).stem reader = cr.CaliperReader() reader.read(input_file) From 159a4b00509755ba095577bf32db6c0ad64bb434 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 15:23:18 -0600 Subject: [PATCH 53/89] Allow adding extra raja perf options --- scripts/gitlab/build_and_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 1b7b4e0eb..6f37d19a5 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -33,6 +33,7 @@ perf_artifact_dir=${PERF_ARTIFACT_DIR:-""} performance_tests=${PERFORMANCE_TESTS:-false} kernels=${KERNELS:-""} variants=${VARIANTS:-""} +perf_run_opts=${PERF_RUN_OPTS:-""} raja_version=${UPDATE_RAJA:-""} sys_type=${SYS_TYPE:-""} @@ -300,7 +301,7 @@ then timed_message "Performance tests for RAJA Perf Suite" - ${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} + ${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} ${perf_run_opts} timed_message "Performance tests for RAJA Perf Suite completed" fi From 893c1065c1e970737521e6f6e6b25d6cc42ce3d8 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 12:30:58 -0600 Subject: [PATCH 54/89] Update performance jobs --- .gitlab/jobs/performances.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 4818febb2..2c7337440 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -13,7 +13,7 @@ variables: PERF_PROCESSING_CMD: "${CI_PROJECT_DIR}/scripts/gitlab/caliper_to_github_benchmark.py" PERFORMANCE_TESTS: "true" -clang_14_0_6_caliper: +clang_14_0_6_openmp_caliper: variables: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" KERNELS: "Basic" @@ -21,6 +21,14 @@ clang_14_0_6_caliper: OMP_NUM_THREADS: "16" extends: .perf_on_dane +gcc_10_3_1_openmp_caliper: + variables: + SPEC: "~shared +openmp +caliper %gcc@=10.3.1 ^blt@develop" + KERNELS: "Basic" + VARIANTS: "RAJA_Seq RAJA_OpenMP" + OMP_NUM_THREADS: "16" + extends: .perf_on_dane + rocmcc_6_4_1_hip_openmp_caliper: variables: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" @@ -31,7 +39,7 @@ rocmcc_6_4_1_hip_openmp_caliper: process_results: extends: .caliper_to_gh_benchmark dependencies: - - clang_14_0_6_caliper + - clang_14_0_6_openmp_caliper - rocmcc_6_4_1_hip_openmp_caliper report_results: From 233be29b7221bf2150d06485c8178b541edf1a5c Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 15:57:31 -0600 Subject: [PATCH 55/89] Add MPI job to performance tests --- .gitlab/jobs/performances.yml | 8 ++++++++ scripts/gitlab/build_and_test.sh | 11 ++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 2c7337440..2b9b1a10c 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -29,6 +29,14 @@ gcc_10_3_1_openmp_caliper: OMP_NUM_THREADS: "16" extends: .perf_on_dane +clang_14_0_6_mpi_caliper: + variables: + SPEC: "~shared +mpi +caliper %clang@=14.0.6 ^blt@develop" + KERNELS: "Basic" + VARIANTS: "RAJA_Seq" + PERF_MPI_EXEC: "srun --overlap -n 16" + extends: .perf_on_dane + rocmcc_6_4_1_hip_openmp_caliper: variables: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 6f37d19a5..c9a42657a 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -34,6 +34,7 @@ performance_tests=${PERFORMANCE_TESTS:-false} kernels=${KERNELS:-""} variants=${VARIANTS:-""} perf_run_opts=${PERF_RUN_OPTS:-""} +perf_mpi_exec=${PERF_MPI_EXEC:-""} raja_version=${UPDATE_RAJA:-""} sys_type=${SYS_TYPE:-""} @@ -301,7 +302,15 @@ then timed_message "Performance tests for RAJA Perf Suite" - ${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} ${perf_run_opts} + raja_perf_command="${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} ${perf_run_opts}" + + if [[ -n ${perf_mpi_exec} ]] + then + raja_perf_command="${perf_mpi_exec} ${raja_perf_command}" + fi + + echo "Running: ${raja_perf_command}" + ${raja_perf_command} timed_message "Performance tests for RAJA Perf Suite completed" fi From 2cc4c8ed59fea14c187277cb57fd64d91ca0c085 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 16:37:40 -0600 Subject: [PATCH 56/89] Increase number of repititions to get better statistics --- .gitlab/jobs/performances.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 2b9b1a10c..e63cbf50b 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -18,6 +18,7 @@ clang_14_0_6_openmp_caliper: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" KERNELS: "Basic" VARIANTS: "RAJA_Seq RAJA_OpenMP" + PERF_RUN_OPTS: "--repfact 10.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -26,6 +27,7 @@ gcc_10_3_1_openmp_caliper: SPEC: "~shared +openmp +caliper %gcc@=10.3.1 ^blt@develop" KERNELS: "Basic" VARIANTS: "RAJA_Seq RAJA_OpenMP" + PERF_RUN_OPTS: "--repfact 10.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -34,6 +36,7 @@ clang_14_0_6_mpi_caliper: SPEC: "~shared +mpi +caliper %clang@=14.0.6 ^blt@develop" KERNELS: "Basic" VARIANTS: "RAJA_Seq" + PERF_RUN_OPTS: "--repfact 10.0" PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane @@ -42,6 +45,7 @@ rocmcc_6_4_1_hip_openmp_caliper: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" KERNELS: "Basic" VARIANTS: "RAJA_HIP" + PERF_RUN_OPTS: "--repfact 10.0" extends: .perf_on_tuolumne process_results: From 52d0574c899b3d6b3534af9bdee01de76ebceac8 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 16:47:02 -0600 Subject: [PATCH 57/89] Fix process results dependencies --- .gitlab/jobs/performances.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index e63cbf50b..9c336ce39 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -52,6 +52,8 @@ process_results: extends: .caliper_to_gh_benchmark dependencies: - clang_14_0_6_openmp_caliper + - gcc_10_3_1_openmp_caliper + - clang_14_0_6_mpi_caliper - rocmcc_6_4_1_hip_openmp_caliper report_results: From 9304cf1e1ab35d7e05a7b286e8caca19ad5989e4 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 23 Sep 2025 19:08:01 -0600 Subject: [PATCH 58/89] Change the benchmark data format to be time per rep instead of total time --- scripts/gitlab/caliper_to_github_benchmark.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index b942c2c20..67bbb42b5 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -13,6 +13,7 @@ from pathlib import Path METRIC = 'avg#inclusive#sum#time.duration' +REPS_METRIC = 'any#any#max#Reps' def caliper_to_benchmark_json(input_file): @@ -32,15 +33,19 @@ def caliper_to_benchmark_json(input_file): for record in reader.records: path = record['path'] if 'path' in record else 'UNKNOWN' - time_s = record[METRIC] if METRIC in record else '0' + total_time_s = float(record[METRIC]) if METRIC in record else 0.0 + reps = int(record[REPS_METRIC]) if REPS_METRIC in record else 1 + + # Calculate average time per rep + avg_time_per_rep_s = total_time_s / reps if reps > 0 else total_time_s # Gather leaf-level kernel paths like "RAJAPerf/Group/KernelName" if isinstance(path, list) and len(path) >= 3: kernel_name = path[-1] benchmark = { "name": f"{variant}_{kernel_name}", - "unit": "s", - "value": time_s + "unit": "s/rep", + "value": avg_time_per_rep_s } benchmarks.append(benchmark) From 5e00c1bf8934d3737e542274a964f8b7adb9d6f6 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 09:30:39 -0600 Subject: [PATCH 59/89] Remove repfact 10 for GPU tests as they get slower the more was run before them --- .gitlab/jobs/performances.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 9c336ce39..353d8dd26 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -45,7 +45,6 @@ rocmcc_6_4_1_hip_openmp_caliper: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" KERNELS: "Basic" VARIANTS: "RAJA_HIP" - PERF_RUN_OPTS: "--repfact 10.0" extends: .perf_on_tuolumne process_results: From bc67220859a5d70a000c1df94b93c3fe8701dfda Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 10:05:46 -0600 Subject: [PATCH 60/89] Use ms instead of s for dashboard --- .gitlab/jobs/performances.yml | 1 + scripts/gitlab/caliper_to_github_benchmark.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 353d8dd26..ae3ec6640 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -45,6 +45,7 @@ rocmcc_6_4_1_hip_openmp_caliper: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" KERNELS: "Basic" VARIANTS: "RAJA_HIP" + MODULE_LIST: "rocm/6.4.1" extends: .perf_on_tuolumne process_results: diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 67bbb42b5..092279f0e 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -34,18 +34,19 @@ def caliper_to_benchmark_json(input_file): for record in reader.records: path = record['path'] if 'path' in record else 'UNKNOWN' total_time_s = float(record[METRIC]) if METRIC in record else 0.0 + total_time_ms = total_time_s * 1000 reps = int(record[REPS_METRIC]) if REPS_METRIC in record else 1 # Calculate average time per rep - avg_time_per_rep_s = total_time_s / reps if reps > 0 else total_time_s + avg_time_per_rep_ms = total_time_ms / reps if reps > 0 else total_time_ms # Gather leaf-level kernel paths like "RAJAPerf/Group/KernelName" if isinstance(path, list) and len(path) >= 3: kernel_name = path[-1] benchmark = { "name": f"{variant}_{kernel_name}", - "unit": "s/rep", - "value": avg_time_per_rep_s + "unit": "ms/rep", + "value": avg_time_per_rep_ms } benchmarks.append(benchmark) From b40b68ea95b449afdfadb2224949752ddaad1b14 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 13:18:05 -0600 Subject: [PATCH 61/89] Use pathlib instead of glob for pattern matching --- scripts/gitlab/caliper_to_github_benchmark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 092279f0e..83b47a0a1 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -7,7 +7,7 @@ python caliper_to_github_benchmark.py "/path/to/*_*-*.cali" output.json """ import caliperreader as cr -import glob +from pathlib import Path import json import sys from pathlib import Path @@ -24,10 +24,10 @@ def caliper_to_benchmark_json(input_file): List of benchmark dictionaries """ - variant = Path(input_file).stem + variant = input_file.stem reader = cr.CaliperReader() - reader.read(input_file) + reader.read(str(input_file)) benchmarks = [] @@ -60,7 +60,7 @@ def caliper_to_benchmark_json(input_file): pattern = sys.argv[1] output_file = sys.argv[2] - cali_files = glob.glob(pattern) + cali_files = list(Path('.').glob(pattern)) if not cali_files: sys.exit(f"Error: No .cali files found matching pattern: {pattern}") From 5026c1ef7bf2c5b1113602204fc7da8b7e193078 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 13:32:05 -0600 Subject: [PATCH 62/89] Simplify get from dictionary logic --- scripts/gitlab/caliper_to_github_benchmark.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 83b47a0a1..0d82b495d 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -33,9 +33,11 @@ def caliper_to_benchmark_json(input_file): for record in reader.records: path = record['path'] if 'path' in record else 'UNKNOWN' - total_time_s = float(record[METRIC]) if METRIC in record else 0.0 + + total_time_s = float(record.get(METRIC, '0')) + reps = int(record.get(REPS_METRIC, '1')) + total_time_ms = total_time_s * 1000 - reps = int(record[REPS_METRIC]) if REPS_METRIC in record else 1 # Calculate average time per rep avg_time_per_rep_ms = total_time_ms / reps if reps > 0 else total_time_ms From 068445217117f25b691483b69cad221f18c32fb3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 13:41:32 -0600 Subject: [PATCH 63/89] Print CLI usage upon error --- scripts/gitlab/caliper_to_github_benchmark.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 0d82b495d..3a69a81b1 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -65,6 +65,7 @@ def caliper_to_benchmark_json(input_file): cali_files = list(Path('.').glob(pattern)) if not cali_files: + print("Usage: caliper_to_github_benchmark.py \"input_file_glob_pattern\" output.json") sys.exit(f"Error: No .cali files found matching pattern: {pattern}") print(f"Found {len(cali_files)} Caliper files to process") From 8d94b3f11b9c0f73fa05f7cd1fd27e631d8daac7 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 13:43:24 -0600 Subject: [PATCH 64/89] Use commit hash instead of tag for github action --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9efc59869..66c69f3c1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -58,7 +58,7 @@ jobs: - name: Create Check for PRs if: steps.findpr.outputs.pr_number != '' - uses: LouisBrunner/checks-action@v2.0.0 + uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 with: token: ${{ secrets.GITHUB_TOKEN }} name: Performance Benchmark (${{ github.event.inputs.benchmark_name }}) From 8efe8d10f32fd1f294461365d415e9c376c356bd Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 14:10:25 -0600 Subject: [PATCH 65/89] Limit workflow permissions --- .github/workflows/benchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 66c69f3c1..f0f701103 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,8 +13,7 @@ on: permissions: checks: write - contents: write - pull-requests: write + contents: ${{ github.ref == 'refs/heads/develop' && 'write' || 'read' }} jobs: store-and-visualize: From ce46aa33dc5899878ada2094a28a9962e6e2c314 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 14:41:59 -0600 Subject: [PATCH 66/89] Make seperate job for develop and PRs so that permissions can be different --- .github/workflows/benchmark.yml | 42 +++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f0f701103..e68a2fb09 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,12 +11,12 @@ on: required: false default: 'RAJAPerf benchmarks' -permissions: - checks: write - contents: ${{ github.ref == 'refs/heads/develop' && 'write' || 'read' }} - jobs: - store-and-visualize: + check-performance: + if: github.ref != 'refs/heads/develop' + permissions: + checks: write + contents: write # TODO for test, later make it read runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -44,8 +44,7 @@ jobs: tool: 'customSmallerIsBetter' output-file-path: benchmark.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true # TODO remove later, for testing - # auto-push: ${{ github.ref == 'refs/heads/develop' }} + auto-push: true # TODO for testing, later make it false gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench comment-always: false @@ -73,3 +72,32 @@ jobs: - name: Final status if: steps.benchmark.outcome != 'success' run: exit 1 + + store-and-visualize-on-develop: + if: github.ref == 'refs/heads/develop' + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Decode benchmark data + run: echo "${{ github.event.inputs.benchmark_data }}" | base64 -d > benchmark.json + + - name: Store benchmark result + id: benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ${{ github.event.inputs.benchmark_name }} + tool: 'customSmallerIsBetter' + output-file-path: benchmark.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + comment-always: false + comment-on-alert: false + summary-always: true + fail-on-alert: true + alert-threshold: '120%' + max-items-in-chart: 100 From c2f0744b8df286b1ac5ffad94c6651c9d065b4ee Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 14:42:56 -0600 Subject: [PATCH 67/89] Update checkout action version --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e68a2fb09..1c22fb540 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,7 +19,7 @@ jobs: contents: write # TODO for test, later make it read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Find associated PR id: findpr @@ -79,7 +79,7 @@ jobs: contents: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Decode benchmark data run: echo "${{ github.event.inputs.benchmark_data }}" | base64 -d > benchmark.json From 8eadc4b7c6530bc8d5b4cce9db6bf2a4a1c841a6 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 14:50:47 -0600 Subject: [PATCH 68/89] Remove unneeded module --- .gitlab/jobs/performances.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index ae3ec6640..353d8dd26 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -45,7 +45,6 @@ rocmcc_6_4_1_hip_openmp_caliper: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" KERNELS: "Basic" VARIANTS: "RAJA_HIP" - MODULE_LIST: "rocm/6.4.1" extends: .perf_on_tuolumne process_results: From cc927b6b7f0039179af73927370d4f9d2d9e29f2 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 14:51:23 -0600 Subject: [PATCH 69/89] Make environment variable naming more consistent --- .gitlab/jobs/performances.yml | 18 +++++++++--------- scripts/gitlab/build_and_test.sh | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 353d8dd26..5d94e76fc 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -11,13 +11,13 @@ variables: PERF_RESULTS_FILE: "*_*-*.cali" PERF_PROCESSED_FILE: "processed_results.json" PERF_PROCESSING_CMD: "${CI_PROJECT_DIR}/scripts/gitlab/caliper_to_github_benchmark.py" - PERFORMANCE_TESTS: "true" + PERF_TESTS: "true" clang_14_0_6_openmp_caliper: variables: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" - KERNELS: "Basic" - VARIANTS: "RAJA_Seq RAJA_OpenMP" + PERF_KERNELS: "Basic" + PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" PERF_RUN_OPTS: "--repfact 10.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -25,8 +25,8 @@ clang_14_0_6_openmp_caliper: gcc_10_3_1_openmp_caliper: variables: SPEC: "~shared +openmp +caliper %gcc@=10.3.1 ^blt@develop" - KERNELS: "Basic" - VARIANTS: "RAJA_Seq RAJA_OpenMP" + PERF_KERNELS: "Basic" + PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" PERF_RUN_OPTS: "--repfact 10.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -34,8 +34,8 @@ gcc_10_3_1_openmp_caliper: clang_14_0_6_mpi_caliper: variables: SPEC: "~shared +mpi +caliper %clang@=14.0.6 ^blt@develop" - KERNELS: "Basic" - VARIANTS: "RAJA_Seq" + PERF_KERNELS: "Basic" + PERF_VARIANTS: "RAJA_Seq" PERF_RUN_OPTS: "--repfact 10.0" PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane @@ -43,8 +43,8 @@ clang_14_0_6_mpi_caliper: rocmcc_6_4_1_hip_openmp_caliper: variables: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" - KERNELS: "Basic" - VARIANTS: "RAJA_HIP" + PERF_KERNELS: "Basic" + PERF_VARIANTS: "RAJA_HIP" extends: .perf_on_tuolumne process_results: diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index c9a42657a..91d392353 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -30,9 +30,9 @@ spack_debug=${SPACK_DEBUG:-false} debug_mode=${DEBUG_MODE:-false} push_to_registry=${PUSH_TO_REGISTRY:-true} perf_artifact_dir=${PERF_ARTIFACT_DIR:-""} -performance_tests=${PERFORMANCE_TESTS:-false} -kernels=${KERNELS:-""} -variants=${VARIANTS:-""} +perf_tests=${PERF_TESTS:-false} +perf_kernels=${PERF_KERNELS:-""} +perf_variants=${PERF_VARIANTS:-""} perf_run_opts=${PERF_RUN_OPTS:-""} perf_mpi_exec=${PERF_MPI_EXEC:-""} @@ -262,7 +262,7 @@ then fi # Test -if [[ "${option}" != "--build-only" && "${performance_tests}" != "true" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} +if [[ "${option}" != "--build-only" && "${perf_tests}" != "true" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then if [[ ! -d ${build_dir} ]] then @@ -294,7 +294,7 @@ then fi # Performance tests -if [[ "${option}" != "--build-only" && "${performance_tests}" == "true" ]] +if [[ "${option}" != "--build-only" && "${perf_tests}" == "true" ]] then cd ${project_dir} @@ -302,7 +302,7 @@ then timed_message "Performance tests for RAJA Perf Suite" - raja_perf_command="${build_dir}/bin/raja-perf.exe --kernels ${kernels} --variants ${variants} --outdir ${perf_artifact_dir} ${perf_run_opts}" + raja_perf_command="${build_dir}/bin/raja-perf.exe --kernels ${perf_kernels} --variants ${perf_variants} --outdir ${perf_artifact_dir} ${perf_run_opts}" if [[ -n ${perf_mpi_exec} ]] then From 62d810ff4a4c5bcf52f37a4f55b3135ff5c2ec73 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 24 Sep 2025 16:33:34 -0600 Subject: [PATCH 70/89] Clarify permissions in workflow --- .github/workflows/benchmark.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1c22fb540..c586222f7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,8 +15,11 @@ jobs: check-performance: if: github.ref != 'refs/heads/develop' permissions: + # Needed to write PR check status checks: write - contents: write # TODO for test, later make it read + # Need write if you turn on comments + pull-requests: read + contents: read runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -35,7 +38,7 @@ jobs: - name: Decode benchmark data run: echo "${{ github.event.inputs.benchmark_data }}" | base64 -d > benchmark.json - - name: Store benchmark result + - name: Compare benchmark result id: benchmark continue-on-error: true uses: benchmark-action/github-action-benchmark@v1 @@ -44,11 +47,12 @@ jobs: tool: 'customSmallerIsBetter' output-file-path: benchmark.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true # TODO for testing, later make it false + auto-push: false + save-data-file: false gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench comment-always: false - comment-on-alert: true + comment-on-alert: false summary-always: true fail-on-alert: true alert-threshold: '120%' @@ -74,8 +78,9 @@ jobs: run: exit 1 store-and-visualize-on-develop: - if: github.ref == 'refs/heads/develop' + # if: github.ref == 'refs/heads/develop' # TODO uncomment before merging permissions: + # Needed to push to gh-pages branch contents: write runs-on: ubuntu-latest steps: From 86855312a227dcbc9e6ee88eb223f362f57279b8 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 29 Sep 2025 12:34:24 +0200 Subject: [PATCH 71/89] Update radiuss-shared-ci --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15afeda8d..cf89449d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,7 +76,7 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'woptim/caliper-ci' + ref: 'v2025.09.0' file: 'pipelines/${CI_MACHINE}.yml' - artifact: '${CI_MACHINE}-jobs.yml' job: 'generate-job-lists' @@ -90,7 +90,7 @@ performance-measurements: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'woptim/caliper-ci' + ref: 'v2025.09.0' file: 'pipelines/performances.yml' - local: '.gitlab/jobs/performances.yml' strategy: depend From b49ae10a2a6291d3e818f514867baf8605d06125 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 10 Nov 2025 11:12:16 +0100 Subject: [PATCH 72/89] Point at fixup in radiuss-shared-ci --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf89449d7..2564b2a30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,7 +76,7 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'v2025.09.0' + ref: 'woptim/workflow-token' file: 'pipelines/${CI_MACHINE}.yml' - artifact: '${CI_MACHINE}-jobs.yml' job: 'generate-job-lists' @@ -90,7 +90,7 @@ performance-measurements: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: 'v2025.09.0' + ref: 'woptim/workflow-token' file: 'pipelines/performances.yml' - local: '.gitlab/jobs/performances.yml' strategy: depend @@ -103,7 +103,7 @@ include: file: 'id_tokens.yml' # [Optional] checks preliminary to running the actual CI test #- project: 'radiuss/radiuss-shared-ci' - # ref: 'woptim/caliper-ci' + # ref: 'woptim/workflow-token' # file: 'utilities/preliminary-ignore-draft-pr.yml' # pipelines subscribed by the project - local: '.gitlab/subscribed-pipelines.yml' From 4d61d88081a8986289ba4a115ed044035cf36cd1 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 10 Nov 2025 14:08:51 +0100 Subject: [PATCH 73/89] TEMP: deactivate failing jobs --- .gitlab-ci.yml | 1 + .gitlab/jobs/performances.yml | 12 ++++++------ tpl/RAJA | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2564b2a30..e4faef0db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ # We define the following GitLab pipeline variables: variables: + ON_CORONA: "OFF" ##### LC GITLAB CONFIGURATION # Use an LLNL service user to run CI. This prevents from running pipelines as # an actual user. diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 5d94e76fc..32bda8e72 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -40,12 +40,12 @@ clang_14_0_6_mpi_caliper: PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane -rocmcc_6_4_1_hip_openmp_caliper: - variables: - SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" - PERF_KERNELS: "Basic" - PERF_VARIANTS: "RAJA_HIP" - extends: .perf_on_tuolumne +#rocmcc_6_4_1_hip_openmp_caliper: +# variables: +# SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" +# PERF_KERNELS: "Basic" +# PERF_VARIANTS: "RAJA_HIP" +# extends: .perf_on_tuolumne process_results: extends: .caliper_to_gh_benchmark diff --git a/tpl/RAJA b/tpl/RAJA index d5d53b759..ca756788d 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit d5d53b7594c0cc2eb1e2d9f65d9ab9bf2b3ccc7b +Subproject commit ca756788dbdd43fec2a3840389126ae94a905d5f From 5ab2a1acd60ab26c2f522c5d0a1c5cf06f40305e Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 10 Nov 2025 14:46:28 +0100 Subject: [PATCH 74/89] Update RAJA to develop for coherency with MATRIX support --- tpl/RAJA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/RAJA b/tpl/RAJA index d5d53b759..5e47f5827 160000 --- a/tpl/RAJA +++ b/tpl/RAJA @@ -1 +1 @@ -Subproject commit d5d53b7594c0cc2eb1e2d9f65d9ab9bf2b3ccc7b +Subproject commit 5e47f582790014fdbcb0060617d933c7e28243c0 From 779950be90ac1c32e6a1424f46fe1c35ac922335 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Mon, 10 Nov 2025 16:03:09 +0100 Subject: [PATCH 75/89] TEMP: fix related to temporary deactivation of job --- .gitlab/jobs/performances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 32bda8e72..e4ae40998 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -53,7 +53,7 @@ process_results: - clang_14_0_6_openmp_caliper - gcc_10_3_1_openmp_caliper - clang_14_0_6_mpi_caliper - - rocmcc_6_4_1_hip_openmp_caliper +# - rocmcc_6_4_1_hip_openmp_caliper report_results: extends: .report_to_gh_benchmark From 0b861c126d9668b318f7c342bf132349a6823440 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 20 Nov 2025 22:25:35 +0100 Subject: [PATCH 76/89] Temporary: trigger workflow on PR --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c586222f7..8402139a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,7 @@ name: Process Benchmark on: + pull_request: workflow_dispatch: inputs: benchmark_data: From abfa86886729d0dee81e83f58a991de540802b38 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 20 Nov 2025 22:27:01 +0100 Subject: [PATCH 77/89] Temporary: trigger workflow on push --- .github/workflows/benchmark.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8402139a4..14be69aaf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,16 +1,15 @@ name: Process Benchmark -on: - pull_request: - workflow_dispatch: - inputs: - benchmark_data: - description: 'Base64 encoded benchmark data' - required: true - benchmark_name: - description: 'Name for benchmark group on the dashboard' - required: false - default: 'RAJAPerf benchmarks' +on: push +# workflow_dispatch: +# inputs: +# benchmark_data: +# description: 'Base64 encoded benchmark data' +# required: true +# benchmark_name: +# description: 'Name for benchmark group on the dashboard' +# required: false +# default: 'RAJAPerf benchmarks' jobs: check-performance: From a4c46c8aa2c7fea92d5be2d6fd4972851c50f886 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:00:45 +0100 Subject: [PATCH 78/89] Revert "Temporary: trigger workflow on push" This reverts commit abfa86886729d0dee81e83f58a991de540802b38. --- .github/workflows/benchmark.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 14be69aaf..8402139a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,15 +1,16 @@ name: Process Benchmark -on: push -# workflow_dispatch: -# inputs: -# benchmark_data: -# description: 'Base64 encoded benchmark data' -# required: true -# benchmark_name: -# description: 'Name for benchmark group on the dashboard' -# required: false -# default: 'RAJAPerf benchmarks' +on: + pull_request: + workflow_dispatch: + inputs: + benchmark_data: + description: 'Base64 encoded benchmark data' + required: true + benchmark_name: + description: 'Name for benchmark group on the dashboard' + required: false + default: 'RAJAPerf benchmarks' jobs: check-performance: From d8f9cc96ffebb2127607c4cfeabf59d94bc275b9 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:00:54 +0100 Subject: [PATCH 79/89] Revert "Temporary: trigger workflow on PR" This reverts commit 0b861c126d9668b318f7c342bf132349a6823440. --- .github/workflows/benchmark.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8402139a4..c586222f7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,6 @@ name: Process Benchmark on: - pull_request: workflow_dispatch: inputs: benchmark_data: From b46ddd4c2d0bfb143788bdf21048597b5bb2a618 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:03:03 +0100 Subject: [PATCH 80/89] Revert "TEMP: fix related to temporary deactivation of job" This reverts commit 779950be90ac1c32e6a1424f46fe1c35ac922335. --- .gitlab/jobs/performances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index e4ae40998..32bda8e72 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -53,7 +53,7 @@ process_results: - clang_14_0_6_openmp_caliper - gcc_10_3_1_openmp_caliper - clang_14_0_6_mpi_caliper -# - rocmcc_6_4_1_hip_openmp_caliper + - rocmcc_6_4_1_hip_openmp_caliper report_results: extends: .report_to_gh_benchmark From 968c08f3ee04d1ae938ab417a65ebedf8e0dea16 Mon Sep 17 00:00:00 2001 From: "Adrien M. BERNEDE" <51493078+adrienbernede@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:03:21 +0100 Subject: [PATCH 81/89] Revert "TEMP: deactivate failing jobs" This reverts commit 4d61d88081a8986289ba4a115ed044035cf36cd1. --- .gitlab-ci.yml | 1 - .gitlab/jobs/performances.yml | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f13974292..dd525cc12 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,6 @@ # We define the following GitLab pipeline variables: variables: - ON_CORONA: "OFF" ##### LC GITLAB CONFIGURATION # Use an LLNL service user to run CI. This prevents from running pipelines as # an actual user. diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 32bda8e72..5d94e76fc 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -40,12 +40,12 @@ clang_14_0_6_mpi_caliper: PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane -#rocmcc_6_4_1_hip_openmp_caliper: -# variables: -# SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" -# PERF_KERNELS: "Basic" -# PERF_VARIANTS: "RAJA_HIP" -# extends: .perf_on_tuolumne +rocmcc_6_4_1_hip_openmp_caliper: + variables: + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" + PERF_KERNELS: "Basic" + PERF_VARIANTS: "RAJA_HIP" + extends: .perf_on_tuolumne process_results: extends: .caliper_to_gh_benchmark From 37ae75047c28f0532dc7e1fa4af2ae2786f904dc Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 30 Sep 2025 11:30:03 -0700 Subject: [PATCH 82/89] Reduce repfact --- .gitlab/jobs/performances.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 5d94e76fc..296e1f1fa 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -18,7 +18,7 @@ clang_14_0_6_openmp_caliper: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" - PERF_RUN_OPTS: "--repfact 10.0" + PERF_RUN_OPTS: "--repfact 5.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -27,7 +27,7 @@ gcc_10_3_1_openmp_caliper: SPEC: "~shared +openmp +caliper %gcc@=10.3.1 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" - PERF_RUN_OPTS: "--repfact 10.0" + PERF_RUN_OPTS: "--repfact 5.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -36,7 +36,7 @@ clang_14_0_6_mpi_caliper: SPEC: "~shared +mpi +caliper %clang@=14.0.6 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq" - PERF_RUN_OPTS: "--repfact 10.0" + PERF_RUN_OPTS: "--repfact 5.0" PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane From 36de3c1bfe3dce81bc6a5e7a2f7a19463c19ecb9 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 2 Oct 2025 17:13:41 -0700 Subject: [PATCH 83/89] Remove unused caliper config --- .gitlab/custom-jobs-and-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 16aed0a9f..e252b9830 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -81,7 +81,7 @@ variables: .custom_perf: variables: - CALI_CONFIG: "event-trace,runtime-report,print.metadata" + CALI_CONFIG: "print.metadata" .reproducer_vars: script: From 487cdc4f0ce1d1c7d4017678ff390e2679746fb8 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 3 Oct 2025 09:17:43 -0700 Subject: [PATCH 84/89] Divide timings by passes as well as reps --- scripts/gitlab/caliper_to_github_benchmark.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index 3a69a81b1..f038ba839 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -14,6 +14,7 @@ METRIC = 'avg#inclusive#sum#time.duration' REPS_METRIC = 'any#any#max#Reps' +PASSES_METRIC = 'sum#sum#rc.count' def caliper_to_benchmark_json(input_file): @@ -36,11 +37,13 @@ def caliper_to_benchmark_json(input_file): total_time_s = float(record.get(METRIC, '0')) reps = int(record.get(REPS_METRIC, '1')) + passes = int(record.get(PASSES_METRIC, '1')) total_time_ms = total_time_s * 1000 - # Calculate average time per rep - avg_time_per_rep_ms = total_time_ms / reps if reps > 0 else total_time_ms + # Calculate average time per rep (accounting for both reps and passes) + total_measurements = reps * passes + avg_time_per_rep_ms = total_time_ms / total_measurements if total_measurements > 0 else total_time_ms # Gather leaf-level kernel paths like "RAJAPerf/Group/KernelName" if isinstance(path, list) and len(path) >= 3: From 3775dc1ed6bd0e4b179f925ef284b76cc6b8ce32 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 3 Oct 2025 09:19:03 -0700 Subject: [PATCH 85/89] Add extra info to json --- scripts/gitlab/caliper_to_github_benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gitlab/caliper_to_github_benchmark.py b/scripts/gitlab/caliper_to_github_benchmark.py index f038ba839..eee115291 100755 --- a/scripts/gitlab/caliper_to_github_benchmark.py +++ b/scripts/gitlab/caliper_to_github_benchmark.py @@ -51,7 +51,8 @@ def caliper_to_benchmark_json(input_file): benchmark = { "name": f"{variant}_{kernel_name}", "unit": "ms/rep", - "value": avg_time_per_rep_ms + "value": avg_time_per_rep_ms, + "extra": f"reps: {reps}, passes: {passes}, total_time: {total_time_s:.3f}s" } benchmarks.append(benchmark) From 8141412a785efa3e62dfc67b5988760a0954ff9f Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 10 Nov 2025 09:34:57 -0800 Subject: [PATCH 86/89] Exclude variable kernels --- .gitlab/jobs/performances.yml | 40 ++++++++++++++++++++++++++++++++ scripts/gitlab/build_and_test.sh | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 296e1f1fa..92358787f 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -18,6 +18,13 @@ clang_14_0_6_openmp_caliper: SPEC: "~shared +openmp +caliper %clang@=14.0.6 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" + # Exclude unstable tests (~20% variability): + PERF_EXCLUDE_KERNELS: > + ARRAY_OF_PTRS + COPY8 + PI_ATOMIC + REDUCE3_INT + REDUCE_STRUCT PERF_RUN_OPTS: "--repfact 5.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -27,6 +34,20 @@ gcc_10_3_1_openmp_caliper: SPEC: "~shared +openmp +caliper %gcc@=10.3.1 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq RAJA_OpenMP" + # Exclude unstable tests (~20% variability): + PERF_EXCLUDE_KERNELS: > + ARRAY_OF_PTRS + COPY8 + DAXPY + EMPTY + INIT3 + INIT_VIEW1D + INIT_VIEW1D_OFFSET + MULADDSUB + PI_ATOMIC + PI_REDUCE + REDUCE3_INT + REDUCE_STRUCT PERF_RUN_OPTS: "--repfact 5.0" OMP_NUM_THREADS: "16" extends: .perf_on_dane @@ -36,6 +57,13 @@ clang_14_0_6_mpi_caliper: SPEC: "~shared +mpi +caliper %clang@=14.0.6 ^blt@develop" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_Seq" + # Exclude unstable tests (~20% variability): + PERF_EXCLUDE_KERNELS: > + ARRAY_OF_PTRS + COPY8 + DAXPY + DAXPY_ATOMIC + REDUCE_STRUCT PERF_RUN_OPTS: "--repfact 5.0" PERF_MPI_EXEC: "srun --overlap -n 16" extends: .perf_on_dane @@ -45,6 +73,18 @@ rocmcc_6_4_1_hip_openmp_caliper: SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_HIP" + # Exclude unstable tests (~20% variability): + PERF_EXCLUDE_KERNELS: > + ARRAY_OF_PTRS + COPY8 + IF_QUAD + INDEXLIST_3LOOP + MAT_MAT_SHARED + MULTI_REDUCE + PI_REDUCE + REDUCE3_INT + REDUCE_STRUCT + TRAP_INT extends: .perf_on_tuolumne process_results: diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index a03b3fb12..3c6adec14 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -32,6 +32,7 @@ push_to_registry=${PUSH_TO_REGISTRY:-true} perf_artifact_dir=${PERF_ARTIFACT_DIR:-""} perf_tests=${PERF_TESTS:-false} perf_kernels=${PERF_KERNELS:-""} +perf_exclude_kernels=${PERF_EXCLUDE_KERNELS:-""} perf_variants=${PERF_VARIANTS:-""} perf_run_opts=${PERF_RUN_OPTS:-""} perf_mpi_exec=${PERF_MPI_EXEC:-""} @@ -302,7 +303,7 @@ then timed_message "Performance tests for RAJA Perf Suite" - raja_perf_command="${build_dir}/bin/raja-perf.exe --kernels ${perf_kernels} --variants ${perf_variants} --outdir ${perf_artifact_dir} ${perf_run_opts}" + raja_perf_command="${build_dir}/bin/raja-perf.exe --kernels ${perf_kernels} --exclude-kernels ${perf_exclude_kernels} --variants ${perf_variants} --outdir ${perf_artifact_dir} ${perf_run_opts}" if [[ -n ${perf_mpi_exec} ]] then From 8ed142340fdcd6f852bfb19f8848194669992b98 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 4 Dec 2025 10:09:56 -0800 Subject: [PATCH 87/89] Fix build issue by disabling mpi in adiak --- .gitlab/jobs/performances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 92358787f..806e59161 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -70,7 +70,7 @@ clang_14_0_6_mpi_caliper: rocmcc_6_4_1_hip_openmp_caliper: variables: - SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared ^papi~shared" + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared~mpi ^papi~shared" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_HIP" # Exclude unstable tests (~20% variability): From e2c2882896061a1b3647065ab3a9238a87f90797 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 4 Dec 2025 16:11:12 -0800 Subject: [PATCH 88/89] Fix build error by disabling libunwind in caliper --- .gitlab/jobs/performances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 806e59161..172e32c44 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -70,7 +70,7 @@ clang_14_0_6_mpi_caliper: rocmcc_6_4_1_hip_openmp_caliper: variables: - SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared ^adiak~shared~mpi ^papi~shared" + SPEC: "~shared +rocm +openmp +caliper amdgpu_target=gfx942 %rocmcc@=6.4.1 ^hip@6.4.1 ^blt@develop ^caliper~shared~libunwind ^adiak~shared~mpi ^papi~shared" PERF_KERNELS: "Basic" PERF_VARIANTS: "RAJA_HIP" # Exclude unstable tests (~20% variability): From 084398178c48b6edb9dfd9e19e5c0343372c171d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 5 Dec 2025 14:15:06 -0800 Subject: [PATCH 89/89] Exclude another variable test --- .gitlab/jobs/performances.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/jobs/performances.yml b/.gitlab/jobs/performances.yml index 172e32c44..81a260bba 100644 --- a/.gitlab/jobs/performances.yml +++ b/.gitlab/jobs/performances.yml @@ -77,6 +77,7 @@ rocmcc_6_4_1_hip_openmp_caliper: PERF_EXCLUDE_KERNELS: > ARRAY_OF_PTRS COPY8 + EMPTY IF_QUAD INDEXLIST_3LOOP MAT_MAT_SHARED