WIP: coupled dsl

README.md

@@ -4,67 +4,23 @@ Collection of utility scripts to build and run the EXCLAIM coupled atmosphere-oc
 
 ## Build
 
-### Step by step
-
-To build the coupled setup follow these steps:
-
-```bash
-# Start UENV
-uenv start --view=default icon/25.2:v3
-
-# Create a base directory
-mkdir -p coupled-setup
-
-# Clone this repository in the base directory
-cd coupled-setup && git clone git@github.com:C2SM/EXCLAIM_coupled_setup.git
-
-# Copy the build_tools into the base directory
-cp EXCLAIM_coupled_setup/build_utils/build_tools.sh .
-
-# Source the build_tools and initialize
-source build_tools.sh
-cao_init
-
-# Build CPU and GPU with Spack
-pushd icon-hybrid/build-cpu && cao_build cpu && popd
-pushd icon-hybrid/build-gpu && cao_build gpu && popd
-```
-
-Otherwise, if you want to build without Spack, but using our custom build scripts you can run the following instead:
-
-```bash
-# Build CPU and GPU without Spack
-pushd icon-hybrid/build-cpu && cao_build cpu_nospack && popd
-pushd icon-hybrid/build-gpu && cao_build gpu_nospack && popd
-```
-
-If you only want to rebuild (e.g. after you made some changes in the code), enter the build directories and run
-`cao_rebuild`.
-
-```bash
-# Make sure your UENV is active first
-uenv status
-
-# Rebuild CPU and GPU executables
-cd build-cpu && cao_rebuild && cd ..
-cd build-gpu && cao_rebuild && cd ..
-```
-
-> Note that to have the build utilities (e.g. `cao_init`, `cao_build`, `cao_rebuild`, ...) available you should source
-*build_tools.sh* in all new shells.
-
-### Full build script
-
-To directly do a clean build of the cpu and gpu executables, just execute
+To directly do a clean build of the cpu and gpu executables, just execute from the repository root
 ``` bash
-BUILD_TYPE="SPACK" sbatch ./build_utils/full_build.sh 
+sbatch ./build_utils/full_build.sh 
 ```
-from the repository root. You can also use `BUILD_TYPE="NOSPACK"`.
+By default:
+  - the script will compile the `cpu` and `gpu` targets. `--cpu-only` or `--gpu-only` can be used if needed.
+  - the gpu variant is `GPU_MODE="py-substitute"` (compiling with `icon4py` granules). It can be otherwise set with to `GPU_MODE="acc"` for pure fortran OpenACC implementation.
+  - the build type is set to `BUILD_TYPE="SPACK"`. A `BUILD_TYPE="NOSPACK"` option is also available but not with `GPU_MODE="py-substitute"`.
+
+`BUILD_TYPE` and `GPU_MODE` can be exported or set on the same line as the `sbatch` command.
 
-You can monitor the build with `tail -f full_build.o`. The build happens on `/dev/shm` on the login nodes. The icon clone will be retrieved at the end of the build as `icon-hybrid` containing the 2 `build-cpu` and `build-gpu` subdirectories. 
+You can monitor the build with `tail -f full_build.<jobid>.o`. The build happens on `/dev/shm` on the compute nodes. The icon repo will be retrieved at the end of the build as `icon-hybrid` containing the 2 `build-cpu` and `build-gpu-${GPU_MODE}` subdirectories. 
 
 ## Run
 
+### Basic runs
+
 To run the coupled setup follow these steps:
 
 ```bash
@@ -93,3 +49,7 @@ You can add these options at the end of the `sbatch` command in any order. All o
 ```bash
 jid=$(sbatch --parsable exp.EXCLAIM_COUPLED.run hybrid --profile)
 ```
+
+### Example script
+
+In order to keep a clean git history an example `launch_exp.sh` script is provided. It contains some of the possible settings (case, nodes, walltime, period, restart interval, etc ...). Copy it and modify at will.

build_utils/build_tools.sh

@@ -3,16 +3,24 @@
 
 CAO_BASEDIR="$(pwd)"
 
-CAO_ICON_REPO='git@gitlab.dkrz.de:icon/icon-nwp.git'
-CAO_ICON_BRANCH='master'
-CAO_ICON_COMMIT='bb4e1d8dc67545860e365841fde94be77d91e234'
+# nwp case
+#CAO_ICON_REPO='git@gitlab.dkrz.de:icon/icon-nwp.git'
+#CAO_ICON_BRANCH='master'
+#CAO_ICON_COMMIT='bb4e1d8dc67545860e365841fde94be77d91e234'
+
+# dsl case
+CAO_ICON_REPO='git@github.com:C2SM/icon-exclaim.git'
+CAO_ICON_BRANCH='icon-dsl'
+CAO_ICON_COMMIT='5c5b742a969af2bd491e26cd0a05a35838f121c4'
 
 CAO_BUILD_UTILS_DIR="${CAO_BASEDIR}/EXCLAIM_coupled_setup/build_utils/"
 CAO_ICON_DIR="${CAO_BASEDIR}/icon-hybrid"
 
+GPU_MODE="${GPU_MODE:-py-substitute}"
+
 CAO_BUILD_DIRS=(
   'build-cpu'
-  'build-gpu'
+  'build-gpu-py-substitute'
 )
 
 CAO_CONFIG_FILES=(
@@ -21,8 +29,8 @@ CAO_CONFIG_FILES=(
 )
 
 CAO_SPACK_YAML_FILES=(
-  'spack_cpu.yaml'
-  'spack_gpu.yaml'
+  'spack_cpu_double.yaml'
+  'spack_gpu_double.yaml'
 )
 
 CAO_CONFIG_NAMES=()
@@ -66,6 +74,8 @@ cao_init() {
     config_name=${config_name%%.*}
     cp ${CAO_BUILD_UTILS_DIR}/${spack_yaml_file} ${CAO_ICON_DIR}/config/cscs/spack/santis_${config_name}_double/spack.yaml
   done
+  cp ${CAO_BASEDIR}/EXCLAIM_coupled_setup/spack/santis_cpu_double/spack.yaml ${CAO_ICON_DIR}/config/cscs/spack/santis_cpu_double/spack.yaml
+  cp ${CAO_BASEDIR}/EXCLAIM_coupled_setup/spack/santis_gpu_double_py_substitute/spack.yaml ${CAO_ICON_DIR}/config/cscs/spack/santis_gpu_double_py_substitute/spack.yaml
 
   pushd ${CAO_ICON_DIR}
     for build_dir in "${CAO_BUILD_DIRS[@]}"; do
@@ -76,10 +86,18 @@ cao_init() {
 
 cao_build() {
   if [[ "$(pwd)" == "${CAO_ICON_DIR}" ]]; then
-    ./config/cscs/santis.${1}.nvhpc
+    if [[ "${1}" == "cpu" ]]; then
+      ./config/cscs/santis.${1}.nvhpc
+    else
+      ./config/cscs/santis.${1}.nvhpc.py.substitute
+    fi
     CONFIG_STATUS=$?
   else
-    ../config/cscs/santis.${1}.nvhpc
+    if [[ "${1}" == "cpu" ]]; then
+      ../config/cscs/santis.${1}.nvhpc
+    else
+      ../config/cscs/santis.${1}.nvhpc.py.substitute
+    fi
     CONFIG_STATUS=$?
   fi
 

build_utils/config_cscs/SPACK_TAG_SANTIS

@@ -0,0 +1 @@
+ghex_async-mpi

build_utils/config_cscs/spack/santis_cpu_double/spack.yaml

@@ -0,0 +1,21 @@
+# ICON
+#
+# ------------------------------------------
+# Copyright (C) 2004-2025, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
+# Contact information: icon-model.org
+# See AUTHORS.TXT for a list of authors
+# See LICENSES/ for license information
+# SPDX-License-Identifier: BSD-3-Clause
+# ------------------------------------------
+
+spack:
+  packages:
+    mpi:
+      require: [cray-mpich@8.1.30%nvhpc]
+  specs:
+    - hdf5%nvhpc +szip +hl +fortran +mpi 
+    - netcdf-c@4.9.2%nvhpc
+    - icon-exclaim @develop %nvhpc +ecrad ~aes +jsbach +ocean +coupling ~rte-rrtmgp
+  view: true
+  concretizer:
+    unify: true

build_utils/config_cscs/spack/santis_gpu_double/spack.yaml

@@ -0,0 +1,26 @@
+# ICON
+#
+# ------------------------------------------
+# Copyright (C) 2004-2025, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
+# Contact information: icon-model.org
+# See AUTHORS.TXT for a list of authors
+# See LICENSES/ for license information
+# SPDX-License-Identifier: BSD-3-Clause
+# ------------------------------------------
+
+spack:
+  packages:
+    all:
+      variants: +cuda cuda_arch=90
+    mpi:
+      require: [cray-mpich@8.1.30%nvhpc]
+  specs:
+    - hdf5%nvhpc +szip +hl +fortran +mpi 
+    - netcdf-c@4.9.2%nvhpc
+    - icon-exclaim @develop %nvhpc
+      ~aes +ecrad +jsbach +ocean +coupling ~rte-rrtmgp
+      gpu=nvidia-90 ~mpi-gpu +realloc-buf +cuda-graphs +pgi-inlib
+      fflags="-traceback"
+  view: true
+  concretizer:
+    unify: true

build_utils/config_cscs/spack/santis_gpu_double_py_substitute/spack.yaml

@@ -0,0 +1,27 @@
+# ICON
+#
+# ------------------------------------------
+# Copyright (C) 2004-2025, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
+# Contact information: icon-model.org
+# See AUTHORS.TXT for a list of authors
+# See LICENSES/ for license information
+# SPDX-License-Identifier: BSD-3-Clause
+# ------------------------------------------
+
+spack:
+  packages:
+    all:
+      variants: +cuda cuda_arch=90
+    mpi:
+      require: [cray-mpich@8.1.30%nvhpc]
+  specs:
+    - hdf5%nvhpc +szip +hl +fortran +mpi 
+    - netcdf-c@4.9.2%nvhpc
+    - icon4py@0.0.15%gcc
+    - icon-exclaim @develop %nvhpc
+      ~aes +ecrad +jsbach +ocean +coupling ~rte-rrtmgp
+      dsl=substitute gpu=nvidia-90 ~mpi-gpu +realloc-buf +cuda-graphs +pgi-inlib
+      fflags="-traceback"
+  view: true
+  concretizer:
+    unify: true

build_utils/config_cscs/spack_install

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# ICON
+#
+# ------------------------------------------
+# Copyright (C) 2004-2025, DWD, MPI-M, DKRZ, KIT, ETH, MeteoSwiss
+# Contact information: icon-model.org
+# See AUTHORS.TXT for a list of authors
+# See LICENSES/ for license information
+# SPDX-License-Identifier: BSD-3-Clause
+# ------------------------------------------
+set -eu
+set -o pipefail
+
+
+SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd)
+ICON_DIR=$(cd "${SCRIPT_DIR}/../.."; pwd)
+SPACK_VERSION=$1
+VARIANT=$2
+BUILD_DIR=$3
+ENV_TAG=$4
+BUILD_THREADS=$5
+
+# Unset positional parameters
+# so they are not forwarded to spack-c2sm/setup-env.sh
+set --
+set +x
+set -e
+
+# we allow the script to fail here, as the spack-c2sm repository might already exist in ICON_DIR
+if [ -e "$ICON_DIR/spack-c2sm" ]; then
+    echo "Info: spack-c2sm directory already exists"
+    echo "Info: In case of spack build issue or if the version is not $SPACK_VERSION, please remove folder $ICON_DIR/spack-c2sm"
+else
+    $ICON_DIR/config/cscs/spack/clean-spack-env.sh # remove spack env if there was any
+    git clone --depth 1 --recurse-submodules --shallow-submodules -b $SPACK_VERSION https://github.com/C2SM/spack-c2sm.git $ICON_DIR/spack-c2sm
+fi
+
+source $ICON_DIR/spack-c2sm/setup-env.sh $ENV_TAG
+
+# The spack spec, i.e. the icon configuration is defined in the enviroment file in the path below
+# and is set by the spack activate command
+SPACK_ICON_ENV_DIR=$ICON_DIR/config/cscs/spack/$VARIANT
+spack env activate -d $SPACK_ICON_ENV_DIR
+
+SPACK_PKG="icon-exclaim@develop"
+# if [[ "$VARIANT" == *py* ]]; then
+#     SPACK_PKG="icon-exclaim@develop"
+# else
+#     SPACK_PKG="icon@develop"
+# fi
+
+if [ "$ICON_DIR" != "$BUILD_DIR" ]; then
+  # out-of-source
+  spack develop --build-directory $BUILD_DIR --path $ICON_DIR $SPACK_PKG
+  # # Workaround for spach v0.22 need to concretize once to avoid error
+  # # ==> Error: build_directory
+  # if [ ! -d "$SPACK_ICON_ENV_DIR/.spack-env/repos" ]; then
+  #   set  +e
+  #   spack concretize > /dev/null 2>&1
+  #   set -e
+  # fi
+else
+  # in-source
+  spack develop --path $ICON_DIR $SPACK_PKG
+fi
+
+spack install -v  -j $BUILD_THREADS
+
+$SCRIPT_DIR/create_sh_env $VARIANT $BUILD_DIR