{2023.06}[2023a,2023b] rebuild CUDA/* modules to update module files

EESSI-determine-rebuilds.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Script to determine which parts of the EESSI software stack (version set through init/eessi_defaults)
+# have to be rebuilt
+
+# see example parsing of command line arguments at
+#   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
+#   https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -g | --generic         -  instructs script to build for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      DETECTION_PARAMETERS="--generic"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX)
+
+source $TOPDIR/scripts/utils.sh
+
+echo ">> Determining software subdirectory to use for current build host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/bash
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Configuring EasyBuild..."
+EB="eb"
+source $TOPDIR/configure_easybuild
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# if this script is run as root, use PR patch file to determine if software needs to be removed first
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
+if [ -z ${changed_easystacks_rebuilds} ]; then
+    echo "No software needs to be removed."
+else
+    for easystack_file in ${changed_easystacks_rebuilds}; do
+        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+        # load EasyBuild module (will be installed if it's not available yet)
+        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+        if [ -f ${easystack_file} ]; then
+            echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+            # we need to remove existing installation directories first,
+            # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+            #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+            #  * [F] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+            eb --dry-run-short --rebuild --easystack ${easystack_file}
+            rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[[FR]\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+            for app in ${rebuild_apps}; do
+                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                echo_yellow "Removing ${app_dir} and ${app_module}..."
+                find ${app_dir} -type d | sed -e 's/^/REMOVE_DIRECTORY /'
+                find ${app_dir} -type f | sed -e 's/^/REMOVE_FILE /'
+                echo "REMOVE_MODULE ${app_module}"
+            done
+        else
+            fatal_error "Easystack file ${easystack_file} not found!"
+        fi
+    done
+fi

EESSI-remove-software.sh

@@ -101,7 +101,7 @@ fi
 pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
-if [ $EUID -eq 0 ]; then
+if [ $EUID -ne 0 ]; then
     changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
     if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."
@@ -114,11 +114,15 @@ if [ $EUID -eq 0 ]; then
             source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
             if [ -f ${easystack_file} ]; then
-                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+                echo_green "Software rebuild(s) requested in ${easystack_file}, so"
+                echo_green "  determining which existing installation have to be removed (assuming contents"
+                echo_green "  have been made writable/deletable)..."
                 # we need to remove existing installation directories first,
                 # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
                 #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                eb --dry-run-short --rebuild --easystack ${easystack_file}
+                rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[[FR]\]" | grep -o "module: .*[^)]" | awk '{print $2}')
                 for app in ${rebuild_apps}; do
                     # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all:
                     app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://')
@@ -128,21 +132,25 @@ if [ $EUID -eq 0 ]; then
                     app_subdirs=$(find ${app_dir} -mindepth 1 -maxdepth 1 -type d)
                     app_module=${app_installprefix}/modules/all/${app}.lua
                     echo_yellow "Removing ${app_dir} and ${app_module}..."
-                    rm -rf ${app_dir}
-                    rm -rf ${app_module}
-                    # recreate the installation directory and do an ls on the first-level subdirectories to work around
-                    # permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556)
-                    echo_yellow "Recreating an empty ${app_dir}..."
-                    mkdir -p ${app_dir}
-                    # these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround,
-                    # so redirect to /dev/null and ignore the exit code
-                    ls ${app_subdirs} >& /dev/null || true
+                    rm -rdfv ${app_dir}
+                    rm -rdfv ${app_module}
+                    ### recreate the installation directory and do an ls on the first-level subdirectories to work around
+                    ### permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556)
+                    ##echo_yellow "Recreating an empty ${app_dir}..."
+                    ##mkdir -p ${app_dir}
+                    ##for app_subdir in ${app_subdirs}; do
+                    ##    mkdir -p ${app_subdir}
+                    ##done
+                    #### these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround,
+                    #### so redirect to /dev/null and ignore the exit code
+                    ###ls ${app_subdirs} >& /dev/null || true
                 done
             else
                 fatal_error "Easystack file ${easystack_file} not found!"
             fi
         done
     fi
 else
-    fatal_error "This script can only be run by root!"
+    # fatal_error "This script can only be run by root!"
+    fatal_error "This script must not be run by root!"
 fi

bot/build.sh

@@ -213,6 +213,49 @@ changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else
+    # determine which software packages (and modules) have to be removed
+    TARBALL_TMP_DETERMINE_STEP_DIR=${PREVIOUS_TMP_DIR}/determine_step
+    mkdir -p ${TARBALL_TMP_DETERMINE_STEP_DIR}
+
+    # prepare arguments to eessi_container.sh specific to determine step
+    declare -a DETERMINE_STEP_ARGS=()
+    DETERMINE_STEP_ARGS+=("--save" "${TARBALL_TMP_DETERMINE_STEP_DIR}")
+    DETERMINE_STEP_ARGS+=("--storage" "${STORAGE}")
+
+    # create tmp file for output of determine step
+    determine_outerr=$(mktemp determine.outerr.XXXX)
+
+    echo "Executing command to determine software to be removed:"
+    echo "${software_layer_dir}/eessi_container.sh ${COMMON_ARGS[@]} ${DETERMINE_STEP_ARGS[@]}"
+    echo "                     -- ${software_layer_dir}/EESSI-determine-rebuilds.sh \"${DETERMINE_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${determine_outerr}"
+    ${software_layer_dir}/eessi_container.sh "${COMMON_ARGS[@]}" "${DETERMINE_STEP_ARGS[@]}" \
+                         -- ${software_layer_dir}/EESSI-determine-rebuilds.sh "${DETERMINE_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${determine_outerr}
+
+    # process output file
+    #   for each line containing 'REMOVE_DIRECTORY some_path'
+    #     create a new directory ${STORAGE}/lower_dirs/some_path_stripped
+    #       where the prefix /cvmfs/repo_name is removed from some_path
+    #     set permission of the directory to u+rwx
+    #   for each line containing 'REMOVE_FILE some_file_path'
+    #     touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped
+    #       where the prefix /cvmfs/repo_name is removed from some_file_path
+    #     set permission of the file to u+rw
+
+    LOWER_DIRS="${STORAGE}/lower_dirs"
+    mkdir -p "${LOWER_DIRS}"
+
+    grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs
+    cat ${determine_outerr}.rm_dirs | while read remove_dir; do
+        mkdir -p ${STORAGE}/lower_dirs/${remove_dir}
+        chmod ugo+rwx ${STORAGE}/lower_dirs/${remove_dir}
+    done
+
+    grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files
+    cat ${determine_outerr}.rm_files | while read remove_file; do
+        touch ${STORAGE}/lower_dirs/${remove_file}
+        chmod ugo+rw ${STORAGE}/lower_dirs/${remove_file}
+    done
+
     # prepare directory to store tarball of tmp for removal and build steps
     TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
     mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}
@@ -224,7 +267,11 @@ else
 
     # add fakeroot option in order to be able to remove software, see:
     # https://github.com/EESSI/software-layer/issues/312
-    REMOVAL_STEP_ARGS+=("--fakeroot")
+    # REMOVAL_STEP_ARGS+=("--fakeroot")
+
+    if [[ ! -z ${LOWER_DIRS} ]]; then
+        REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
+    fi
 
     # create tmp file for output of removal step
     removal_outerr=$(mktemp remove.outerr.XXXX)
@@ -276,6 +323,10 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
+if [[ ! -z ${LOWER_DIRS} ]]; then
+    BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
+fi
+
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 

easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250215-eb-4.9.4-CUDA-update-module-files.yml

@@ -0,0 +1,14 @@
+# 2025.02.15
+# We need to update the module files for all CUDA installations to add
+# additional directories to LIBRARY_PATH.
+# See https://github.com/easybuilders/easybuild-easyblocks/pull/3516
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: CUDA
+        force: True
+        include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757
+  - CUDA-12.4.0.eb:
+      options:
+        accept-eula-for: CUDA
+        include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757

eessi_container.sh

@@ -89,6 +89,11 @@ display_help() {
   echo "  -n | --nvidia MODE      - configure the container to work with NVIDIA GPUs,"
   echo "                            MODE==install for a CUDA installation, MODE==run to"
   echo "                            attach a GPU, MODE==all for both [default: false]"
+  echo "  -o | --lower-dirs DIRS  - list of ':' separated directories that are used"
+  echo "                            in front of the default lower dir (CVMFS repo);"
+  echo "                            fuse-overlayfs will merge all lower directories;"
+  echo "                            the option can be used to make certain directories"
+  echo "                            in the CVMFS repo writable [default: none]"
   echo "  -r | --repository CFG   - configuration file or identifier defining the"
   echo "                            repository to use; can be given multiple times;"
   echo "                            CFG may include a suffix ',access={ro,rw}' to"
@@ -125,6 +130,7 @@ FAKEROOT=0
 VERBOSE=0
 STORAGE=
 LIST_REPOS=0
+LOWER_DIRS=
 MODE="shell"
 SETUP_NVIDIA=0
 REPOSITORIES=()
@@ -182,6 +188,10 @@ while [[ $# -gt 0 ]]; do
       NVIDIA_MODE="$2"
       shift 2
       ;;
+    -o|--lower-dirs)
+      LOWER_DIRS="$2"
+      shift 2
+      ;;
     -r|--repository)
       REPOSITORIES+=("$2")
       shift 2
@@ -771,6 +781,7 @@ do
             #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
             # the left-most directory given for the lowerdir argument is put on top,
             #   and with no upperdir=... the whole overlayfs is made available read-only
+            # TODO add LOWER_DIRS in between??? see rw access below
             EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
             EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
             export EESSI_READONLY_OVERLAY
@@ -798,6 +809,13 @@ do
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
         EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
+        # EESSI_WRITABLE_OVERLAY+=" -o debug" # for debug output
+        if [[ -z ${LOWER_DIRS} ]]; then
+            # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as
+            # separator while the lowerdir overlayfs option uses ':'
+            export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}"
+            lowerdirs=${LOWER_DIRS}:${lowerdirs}
+        fi
         EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
         EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
         EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"