diff --git a/bot/check-build.sh b/bot/check-build.sh index a3c3a3a..507d962 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -502,7 +502,9 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + reprod_entries=$(grep "${prefix}/reprod" ${tmpfile}) + reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod") other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) @@ -531,6 +533,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')" fi comment_artifacts_list="${comment_artifacts_list}" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)" + comment_artifacts_list="${comment_artifacts_list}
"
+ if [[ ! -z ${reprod_shortened} ]]; then
+ while IFS= read -r reprod ; do
+ comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__
' ${reprod})"
+ done <<< "${reprod_shortened}"
+ else
+ comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
+ fi
+ comment_artifacts_list="${comment_artifacts_list}
"
comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
comment_artifacts_list="${comment_artifacts_list}" if [[ ! -z ${other_shortened} ]]; then diff --git a/create_tarball.sh b/create_tarball.sh index 84956d1..d104e21 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -90,8 +90,13 @@ for subdir in ${sw_subdirs}; do for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list} + # if there is a directory for this installation in the stack's reprod directory, include that too + if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then + find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list} + fi done fi + done # add a bit debug output diff --git a/eb_hooks.py b/eb_hooks.py index e20ef2e..baa7958 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1,5 +1,6 @@ # Hooks to customize how EasyBuild installs software in EESSI # see https://docs.easybuild.io/en/latest/Hooks.html +import datetime import glob import os import re @@ -7,9 +8,10 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS +from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which +from easybuild.tools.config import build_option, install_path, update_build_option +from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -46,6 +48,9 @@ # Make sure a single environment variable name is used for this throughout the hooks EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" +STACK_REPROD_SUBDIR = 'reprod' + + def is_gcccore_1220_based(**kwargs): # ecname, ecversion, tcname, tcversion): """ @@ -516,6 +521,20 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] +def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs): + """ + Post easyblock hook that copies the easybuild subdirectory of every installed application + to a central and timestamped location in the root of the software stack, e.g.: + /path/to/stack/reprod/MyApp/1.2-foss-2025a/20250102T12:34:56Z + """ + + stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR) + now_utc_timestamp = datetime.datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S%Z') + app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg)) + app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild') + copy_dir(app_easybuild_dir, app_reprod_dir) + + # Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set # so that the modules can be succesfully loaded without printing the error (so that we can create a module # _with_ the warning for the current software being installed) @@ -1297,6 +1316,24 @@ def post_module_hook(self, *args, **kwargs): post_module_hook_zen4_gcccore1220(self, *args, **kwargs) +# The post_easyblock_hook was introduced in EasyBuild 5.1.1. +# Older versions would fail if the function is defined anyway, as EasyBuild performs some checks on function names in hooks files. +if EASYBUILD_VERSION >= '5.1.1': + def post_easyblock_hook(self, *args, **kwargs): + """Main post easyblock hook: trigger custom functions based on software name.""" + if self.name in POST_EASYBLOCK_HOOKS: + POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs) + + # Always trigger this one for EESSI CVMFS/site installations and version 2025.06 or newer, regardless of self.name + if os.getenv('EESSI_CVMFS_INSTALL') or os.getenv('EESSI_SITE_INSTALL'): + if os.getenv('EESSI_VERSION') and LooseVersion(os.getenv('EESSI_VERSION')) >= '2025.06': + post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs) + else: + self.log.debug("No CVMFS/site installation requested, not running post_easyblock_hook_copy_easybuild_subdir.") +else: + print_msg(f"Not enabling the post_easybuild_hook, as it requires EasyBuild 5.1.1 or newer.") + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -1365,6 +1402,8 @@ def post_module_hook(self, *args, **kwargs): POST_MODULE_HOOKS = {} +POST_EASYBLOCK_HOOKS = {} + # Define parallelism limit operations def divide_by_factor(parallel, factor): """Divide parallelism by given factor"""