Use components in CI

.gitlab-ci.yml

@@ -5,45 +5,14 @@
 # SPDX-License-Identifier: (MIT)
 ###############################################################################
 
-# DESCRIPTION:
 ###############################################################################
-# General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL).
-# This entire pipeline is LLNL-specific
-#
-# Important note: This file is a template provided by llnl/radiuss-shared-ci.
-# Remains to set variable values, change the reference to the radiuss-shared-ci
-# repo, opt-in and out optional features. The project can then extend it with
-# additional stages.
-#
-# In addition, each project should copy over and complete:
-# - .gitlab/custom-jobs-and-variables.yml
-# - .gitlab/subscribed-pipelines.yml
-#
-# The jobs should be specified in a file local to the project,
-# - .gitlab/jobs/${CI_MACHINE}.yml
-# or generated (see LLNL/Umpire for an example).
+# VARIABLES
 ###############################################################################
 
-# We define the following GitLab pipeline variables:
 variables:
-##### LC GITLAB CONFIGURATION
-# Use an LLNL service user to run CI. This prevents from running pipelines as
-# an actual user.
-  LLNL_SERVICE_USER: ""
-# Use the service user workspace. Solves permission issues, stores everything
-# at the same location whoever triggers a pipeline.
-#  CUSTOM_CI_BUILDS_DIR: ""
 # Tells Gitlab to recursively update the submodules when cloning the project.
   GIT_SUBMODULE_STRATEGY: recursive
-
-##### PROJECT VARIABLES
-# We build the projects in the CI clone directory.
-# Used in script/gitlab/build_and_test.sh script.
-# TODO: add a clean-up mechanism.
-  BUILD_ROOT: ${CI_PROJECT_DIR}
-
-##### SHARED_CI CONFIGURATION
+# SHARED_CI CONFIGURATION
 # Required information about GitHub repository
   GITHUB_PROJECT_NAME: "Quandary"
   GITHUB_PROJECT_ORG: "LLNL"
@@ -53,55 +22,126 @@ variables:
   JOB_CMD:
     value: ".ci-scripts/build_and_test.sh"
     expand: false
-# Override the pattern describing branches that will skip the "draft PR filter
-# test".  Add protected branches here. See default value in
-# preliminary-ignore-draft-pr.yml.
-#  ALWAYS_RUN_PATTERN: ""
 
-# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
-# corresponds to a test batch on a given machine.
+###############################################################################
+# MAIN PIPELINE STAGES
+###############################################################################
 
-# High level stages
 stages:
   - prerequisites
   - build-and-test
-  - perf-test
+  - performance-measurements
+
+###############################################################################
+# INCLUDES
+###############################################################################
 
-# Template for jobs triggering a build-and-test sub-pipeline:
-.build-and-test:
+include:
+  # Sets ID tokens for every job using `default:`
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+  # Base pipeline templates and utilities
+  - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/[email protected]
+    inputs:
+      github_project_name: $GITHUB_PROJECT_NAME
+      github_project_org: $GITHUB_PROJECT_ORG
+      github_token: $GITHUB_TOKEN
+
+  # Skips CI for branches that are not a PR
+  - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/[email protected]
+    inputs:
+      github_token: $GITHUB_TOKEN
+      github_project_name: $GITHUB_PROJECT_NAME
+      github_project_org: $GITHUB_PROJECT_ORG
+      always_run_pattern: "^develop$|^main$|^v[0-9.]*-RC$"
+
+  # Local custom variables (used for component inputs and forwarded to child pipelines)
+  - local: '.gitlab/custom-variables.yml'
+
+###############################################################################
+# MACHINE PIPELINES
+###############################################################################
+
+# One job to generate the job list for all the subpipelines
+generate-job-lists:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    RADIUSS_JOBS_PATH: ".ci-scripts/radiuss-spack-configs/gitlab/radiuss-jobs"
+    LOCAL_JOBS_PATH: ".gitlab/jobs"
+  script:
+    - cat ${RADIUSS_JOBS_PATH}/dane.yml ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/tioga.yml ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
+  artifacts:
+    paths:
+      - dane-jobs.yml
+      - tioga-jobs.yml
+
+# DANE
+dane-up-check:
+  extends: [.dane, .machine-check]
+
+dane-build-and-test:
   stage: build-and-test
+  extends: [.dane]
+  needs: [dane-up-check, generate-job-lists]
   trigger:
     include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.12.0'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      - artifact: '${CI_MACHINE}-jobs.yml'
+      - local: '.gitlab/custom-jobs.yml'
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/[email protected]
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $DANE_SHARED_ALLOC
+          job_alloc: $DANE_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - artifact: 'dane-jobs.yml'
         job: 'generate-job-lists'
     strategy: depend
     forward:
       pipeline_variables: true
 
-.performance-test:
-  stage: perf-test
+# TIOGA
+tioga-up-check:
+  extends: [.tioga, .machine-check]
+
+tioga-build-and-test:
+  stage: build-and-test
+  extends: [.tioga]
+  needs: [tioga-up-check, generate-job-lists]
   trigger:
     include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.12.0'
-        file: 'pipelines/performances.yml'
-      - local: '.gitlab/jobs/performances.yml'
+      - local: '.gitlab/custom-jobs.yml'
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/[email protected]
+        inputs:
+          job_cmd: $JOB_CMD
+          shared_alloc: $TIOGA_SHARED_ALLOC
+          job_alloc: $TIOGA_JOB_ALLOC
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+      - artifact: 'tioga-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
     forward:
       pipeline_variables: true
 
-include:
-  # Sets ID tokens for every job using `default:`
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-  # [Optional] checks preliminary to running the actual CI test
-  - project: 'radiuss/radiuss-shared-ci'
-    ref: 'v2025.12.0'
-    file: 'utilities/preliminary-ignore-draft-pr.yml'
-  # pipelines subscribed by the project
-  - local: '.gitlab/subscribed-pipelines.yml'
+# PERFORMANCE
+performance-test:
+  stage: performance-measurements
+  trigger:
+    include:
+      - local: '.gitlab/custom-jobs.yml'
+      - component: $CI_SERVER_FQDN/radiuss/radiuss-shared-ci/[email protected]
+        inputs:
+          job_cmd: $JOB_CMD
+          perf_processing_cmd: "${CI_PROJECT_DIR}/tests/performance/convert_benchmark_format.py"
+          github_project_name: $GITHUB_PROJECT_NAME
+          github_project_org: $GITHUB_PROJECT_ORG
+          dane_perf_alloc: $DANE_PERF_ALLOC
+      - local: '.gitlab/jobs/performances.yml'
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  variables:
+    PERFORMANCE_TESTS: "true"

.gitlab/custom-jobs.yml

@@ -0,0 +1,34 @@
+###############################################################################
+# Copyright (c) 2022-25, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# This file defines JOB TEMPLATES ONLY.
+# It is included in CHILD pipelines (via trigger: include:) to provide
+# templates that your jobs can extend.
+
+###############################################################################
+# JOB CUSTOMIZATION TEMPLATES
+###############################################################################
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+.custom_job:
+  artifacts:
+    reports:
+      junit: junit.xml
+
+.reproducer_vars:
+  script:
+    - |
+      echo -e "
+      # Required variables \n
+      export MODULE_LIST=\"${MODULE_LIST}\" \n
+      export SPEC=\"${SPEC//\"/\\\"}\" \n
+      # Allow to set job script for debugging (only this differs from CI) \n
+      export DEBUG_MODE=true \n
+      # Using the CI build cache is optional and requires a token. Set it like so: \n
+      # export REGISTRY_TOKEN=\"<your token here>\" \n"

.gitlab/custom-jobs-and-variables.yml → .gitlab/custom-variables.yml

@@ -35,30 +35,3 @@ variables:
 # Project specific deps for tioga
 # Force use of petsc >= 3.23 as 3.22 doesn't seem to work with rocm 6.4.1
   PROJECT_TIOGA_DEPS: "^[email protected]:"
-
-# Configuration shared by build and test jobs specific to this project.
-# Not all configuration can be shared. Here projects can fine tune the
-# CI behavior.
-.custom_job:
-  artifacts:
-    reports:
-      junit: junit.xml
-
-# Configuration shared by performance jobs specific to this project.
-# Not all configuration can be shared. Here projects can fine tune the
-# CI behavior.
-.custom_perf:
-  variables:
-    JOB_TEMPLATE_CANNOT_BE_EMPTY: "True"
-
-.reproducer_vars:
-  script:
-    - |
-      echo -e "
-      # Required variables \n
-      export MODULE_LIST=\"${MODULE_LIST}\" \n
-      export SPEC=\"${SPEC//\"/\\\"}\" \n
-      # Allow to set job script for debugging (only this differs from CI) \n
-      export DEBUG_MODE=true \n
-      # Using the CI build cache is optional and requires a token. Set it like so: \n
-      # export REGISTRY_TOKEN=\"<your token here>\" \n"

.gitlab/jobs/performances.yml

@@ -8,13 +8,10 @@
 clang_14_0_6_perf:
   extends: .perf_on_dane
   variables:
-    PERFORMANCE_TESTS: "true"
     SPEC: "${PROJECT_DANE_VARIANTS} %clang@=14.0.6 ${PROJECT_DANE_DEPS}"
 
 process_results:
   extends: .convert_to_gh_benchmark
-  variables:
-    PERF_PROCESSING_CMD: "${CI_PROJECT_DIR}/tests/performance/convert_benchmark_format.py"
   dependencies:
     - clang_14_0_6_perf