Skip to content

Commit

Permalink
Regenerate modulefiles on update (fixes openhpc#1601)
Browse files Browse the repository at this point in the history
  • Loading branch information
opoplawski committed May 25, 2024
1 parent bb6ea36 commit 2a3f6ef
Show file tree
Hide file tree
Showing 4 changed files with 181 additions and 146 deletions.
1 change: 1 addition & 0 deletions components/OHPC_macros
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
%global OHPC_ADMIN %{OHPC_HOME}/admin
%global OHPC_PUB %{OHPC_HOME}/pub
%global OHPC_APPS %{OHPC_PUB}/apps
%global OHPC_BIN %{OHPC_PUB}/bin
%global OHPC_COMPILERS %{OHPC_PUB}/compiler
%global OHPC_LIBS %{OHPC_PUB}/libs
%global OHPC_MODULES %{OHPC_PUB}/modulefiles
Expand Down
5 changes: 3 additions & 2 deletions components/admin/ohpc-filesystem/SPECS/ohpc-filesystem.spec
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

Name: ohpc-filesystem
Version: %{version}
Release: %{?dist}.1
Release: %{?dist}.2
Summary: Common top-level OpenHPC directories

Group: ohpc/admin
Expand Down Expand Up @@ -48,7 +48,7 @@ builds.

%install
# The ohpc-filesystems owns all the common directories
mkdir -p $RPM_BUILD_ROOT/opt/ohpc/pub/{apps,doc,compiler,libs,moduledeps,modulefiles,mpi}
mkdir -p $RPM_BUILD_ROOT/opt/ohpc/pub/{apps,bin,doc,compiler,libs,moduledeps,modulefiles,mpi}
mkdir -p $RPM_BUILD_ROOT/opt/ohpc/admin/ohpc
mkdir -p $RPM_BUILD_ROOT/usr/lib/rpm/fileattrs

Expand Down Expand Up @@ -83,6 +83,7 @@ EOF
%dir /opt/ohpc/admin/
%dir /opt/ohpc/pub/
%dir /opt/ohpc/pub/apps/
%dir /opt/ohpc/pub/bin/
%dir /opt/ohpc/pub/doc/
%dir /opt/ohpc/pub/compiler/
%dir /opt/ohpc/pub/libs/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
#!/bin/bash
# Do not overwrite existing files
# Write the new file as rpmnew
testfile () {
if [ -e $1 ]; then
echo "$1.rpmnew"
else
echo "$1"
fi
}

# Remove .rpmnew files if they are identical
# otherwise record in manifest
modprocess () {
if [ "${1/.rpmnew/}" != "$1" ]; then
if cmp "${1/.rpmnew/}" "$1"; then
rm "$1"
return
fi
fi
md5sum ${modname} >> @@oneapi_manifest@@
}

rm -f @@oneapi_manifest@@

# Regenerate the oneAPI modules directory
echo "Generating new oneAPI modulefiles"
/opt/intel/oneapi/modulefiles-setup.sh --ignore-latest --force --output-dir=@@OHPC_MODULEDEPS@@/oneapi/ > /dev/null
if [ ! -d @@OHPC_MODULEDEPS@@/oneapi/compiler ]; then
echo "ERROR: Failed to create oneAPI module directory"
exit 1
fi

# Set system defaults for default OBS oneAPI modules
for tool in @@exact_deps@@; do
filename=@@OHPC_MODULEDEPS@@/oneapi/${tool%%/*}/.version
echo "#%Module1.0" > ${filename}
echo "set ModulesVersion \"${tool##*/}\"" >> ${filename}
done

# Create an OpenHPC module file for each version found in compilers
echo "Creating OpenHPC-style modulefiles for local oneAPI compiler installation(s)."
for compilers in @@OHPC_MODULEDEPS@@/oneapi/compiler/2*; do
ver=$(basename "$compilers")
# Skip 2023.2.0 due to errors that can hang installer
if [ "$ver" = "2023.2.0" ]; then
echo "--> Skipping version=2023.2.0 : environment module format error"
continue
fi
# For the default, also specify the MKL version
if [ "$ver" = "@@exact_intel_ver@@" ]; then
mklver="mkl/@@exact_mkl_ver@@"
else
mklver="mkl"
fi
echo "--> Installing modulefile for version=${ver}"
# Do not overwrite existing files
# Write the new file as rpmnew
modname=$(testfile @@OHPC_MODULES@@/intel/$ver)

cat << EOF > ${modname}
#%Module1.0#####################################################################
set version "$ver"
proc ModulesHelp { } {
global version
puts stderr "\nThis module loads the oneAPI compiler environment.\n"
puts stderr "\nSee the man pages for icc, icpc, and ifort for detailed information"
puts stderr "on available compiler options and command-line syntax."
puts stderr "\nVersion \$version\n"
}
module-whatis "Name: Intel(R) Compiler"
module-whatis "Version: \$version"
module-whatis "Category: compiler, runtime support"
module-whatis "Description: Intel(R) Compiler Family (C/C++/Fortran for x86_64)"
module-whatis "URL: http://software.intel.com/en-us/articles/intel-compilers/"
# update module path hierarchy
prepend-path MODULEPATH @@OHPC_MODULEDEPS@@/intel
prepend-path MODULEPATH @@OHPC_MODULEDEPS@@/oneapi
# Assume no PAC device; allow override on each node
if { ![info exists ::env(ACL_SKIP_BSP_CONF)] } {
setenv ACL_SKIP_BSP_CONF 1
}
module load "oclfpga"
module load "tbb"
module load "compiler-rt"
module load "compiler/\$version"
module load "$mklver"
family "compiler"
EOF

modprocess ${modname}

# Provide standalone module for use with GNU toolchain
modname=$(testfile @@OHPC_MODULEDEPS@@/gnu/mkl/$ver)

cat << EOF > ${modname}
#%Module1.0#####################################################################
set version "$ver"
proc ModulesHelp { } {
global version
puts stderr "\nConfigures oneAPI MKL environment\n"
puts stderr "\$version\n"
}
module-whatis "Name: Intel(R) Math Kernel Library"
module-whatis "Version: \$version"
module-whatis "Category: library, runtime support"
module-whatis "Description: Intel(R) Math Kernel Library for C/C++ and Fortran"
module-whatis "URL: https://software.intel.com/en-us/en-us/intel-mkl"
prepend-path MODULEPATH @@OHPC_MODULEDEPS@@/oneapi
module load "$mklver"
EOF

modprocess ${modname}
done

# Set default version to match that used to build OpenHPC packages
modname=$(testfile @@OHPC_MODULES@@/intel/.version)

cat << EOF > ${modname}
#%Module1.0#####################################################################
set ModulesVersion "@@exact_intel_ver_module@@"
EOF

modprocess ${modname}
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,15 @@
Summary: OpenHPC compatibility package for Intel(R) oneAPI HPC Toolkit
Name: %{pname}%{PROJ_DELIM}
Version: 2024.0
Release: %{?dist}.1
Release: %{?dist}.2
License: Apache-2.0
URL: https://github.com/openhpc/ohpc
Group: %{PROJ_NAME}/compiler-families
BuildArch: x86_64
AutoReq: no
Source1: mod_generator.sh
Source2: oneAPI.repo
Source3: ohpc-update-modules-intel


#!BuildIgnore: post-build-checks
Expand Down Expand Up @@ -70,13 +71,28 @@ install -D -m644 %{SOURCE2} -t %{buildroot}%{repodir}/
# Mod generator for PSXE support
install -D -m755 %{SOURCE1} %{buildroot}/%{OHPC_ADMIN}/compat/modulegen/mod_generator.sh

# Mod generator for oneAPI support
sed -e 's/@@oneapi_manifest@@/%{oneapi_manifes}/' \
-e 's/@@OHPC_MODULEDEPS@@/%{OHPC_MODULEDEPS}/' \
-e 's/@@OHPC_MODULES@@/%{OHPC_MODULES}/' \
-e 's/@@exact_deps@@/%{exact_deps}/' \
-e 's/@@exact_intel_ver@@/%{exact_intel_ver}/' \
-e 's/@@exact_intel_ver_module@@/%{exact_intel_ver_module}/' \
-e 's/@@exact_mkl_ver@@/%{exact_mkl_ver}/' %{SOURCE3} > ohpc-update-modules-intel
install -D -m755 ohpc-update-modules-intel %{buildroot}/%{OHPC_BIN}/ohpc-update-modules-intel

# Module directories
mkdir -p %{buildroot}/%{OHPC_MODULEDEPS}/oneapi
mkdir -p %{buildroot}/%{OHPC_MODULES}/intel
mkdir -p %{buildroot}/%{OHPC_MODULEDEPS}/gnu/mkl
mkdir -p %{buildroot}/%{OHPC_MODULEDEPS}/intel


# Regenerate module files when components are added or removed
%transfiletriggerin -- /opt/intel/oneapi
%{OHPC_BIN}/ohpc-update-modules-intel


%pre -p /bin/bash
if ! [ -f /opt/intel/oneapi/modulefiles-setup.sh ]; then
echo " "
Expand All @@ -87,157 +103,39 @@ fi


%post -p /bin/bash
# Do not overwrite existing files
# Write the new file as rpmnew
testfile () {
if [ -e $1 ]; then
echo "$1.rpmnew"
else
echo "$1"
fi
}

rm -f %{oneapi_manifest}
%{OHPC_BIN}/ohpc-update-modules-intel

# Regenerate the oneAPI modules directory
echo "Generating new oneAPI modulefiles"
/opt/intel/oneapi/modulefiles-setup.sh --ignore-latest --force --output-dir=%{OHPC_MODULEDEPS}/oneapi/ > /dev/null
if [ ! -d %{OHPC_MODULEDEPS}/oneapi/compiler ]; then
echo "ERROR: Failed to create oneAPI module directory"
exit 1
fi

# Set system defaults for default OBS oneAPI modules
for tool in %{exact_deps}; do
filename=%{OHPC_MODULEDEPS}/oneapi/${tool%%/*}/.version
echo "#%Module1.0" > ${filename}
echo "set ModulesVersion \"${tool##*/}\"" >> ${filename}
done

# Create an OpenHPC module file for each version found in compilers
echo "Creating OpenHPC-style modulefiles for local oneAPI compiler installation(s)."
for compilers in %{OHPC_MODULEDEPS}/oneapi/compiler/2*; do
ver=$(basename "$compilers")
# Skip 2023.2.0 due to errors that can hang installer
if [ "$ver" = "2023.2.0" ]; then
echo "--> Skipping version=2023.2.0 : environment module format error"
continue
fi
# For the default, also specify the MKL version
if [ "$ver" = "%{exact_intel_ver}" ]; then
mklver="mkl/%{exact_mkl_ver}"
%preun -p /bin/bash
if [ $1 -eq 0 ]; then
# Check current files against the manifest
# Remove files that match and backup files that don't
if [ -s %{oneapi_manifest} ]; then
echo "Removing module files"
while IFS= read -r line; do
f=${line%:*}
s=${line#*:}
if [ "${s: -2}" = "OK" ]; then
rm -f $f
elif [ -f $f ]; then
mv -T -f $f $f.rpmsave
fi
done <<< $(md5sum --check %{oneapi_manifest})
else
mklver="mkl"
# Don't touch any generated files if there's no manifest
# On upgrade, expect lots of modulefiles created as .rpmnew
echo "WARNING: Manifest not found. Previously generated"
echo " modulefiles will not be removed or moved."
fi
echo "--> Installing modulefile for version=${ver}"
# Do not overwrite existing files
# Write the new file as rpmnew
modname=$(testfile %{OHPC_MODULES}/intel/$ver)

cat << EOF > ${modname}
#%Module1.0#####################################################################

set version "$ver"

proc ModulesHelp { } {
global version
puts stderr "\nThis module loads the oneAPI compiler environment.\n"
puts stderr "\nSee the man pages for icc, icpc, and ifort for detailed information"
puts stderr "on available compiler options and command-line syntax."
puts stderr "\nVersion \$version\n"
}

module-whatis "Name: Intel(R) Compiler"
module-whatis "Version: \$version"
module-whatis "Category: compiler, runtime support"
module-whatis "Description: Intel(R) Compiler Family (C/C++/Fortran for x86_64)"
module-whatis "URL: http://software.intel.com/en-us/articles/intel-compilers/"

# update module path hierarchy
prepend-path MODULEPATH %{OHPC_MODULEDEPS}/intel
prepend-path MODULEPATH %{OHPC_MODULEDEPS}/oneapi

# Assume no PAC device; allow override on each node
if { ![info exists ::env(ACL_SKIP_BSP_CONF)] } {
setenv ACL_SKIP_BSP_CONF 1
}

module load "oclfpga"
module load "tbb"
module load "compiler-rt"
module load "compiler/\$version"
module load "$mklver"

family "compiler"
EOF

md5sum ${modname} >> %{oneapi_manifest}

# Provide standalone module for use with GNU toolchain
modname=$(testfile %{OHPC_MODULEDEPS}/gnu/mkl/$ver)

cat << EOF > ${modname}
#%Module1.0#####################################################################

set version "$ver"

proc ModulesHelp { } {
global version
puts stderr "\nConfigures oneAPI MKL environment\n"
puts stderr "\$version\n"
}

module-whatis "Name: Intel(R) Math Kernel Library"
module-whatis "Version: \$version"
module-whatis "Category: library, runtime support"
module-whatis "Description: Intel(R) Math Kernel Library for C/C++ and Fortran"
module-whatis "URL: https://software.intel.com/en-us/en-us/intel-mkl"

prepend-path MODULEPATH %{OHPC_MODULEDEPS}/oneapi
module load "$mklver"
EOF

md5sum ${modname} >> %{oneapi_manifest}
done

# Set default version to match that used to build OpenHPC packages
modname=$(testfile %{OHPC_MODULES}/intel/.version)

cat << EOF > ${modname}
#%Module1.0#####################################################################
set ModulesVersion "%{exact_intel_ver_module}"
EOF

md5sum ${modname} >> %{oneapi_manifest}


%preun -p /bin/bash
# Check current files against the manifest
# Remove files that match and backup files that don't
if [ -s %{oneapi_manifest} ]; then
echo "Removing module files"
while IFS= read -r line; do
f=${line%:*}
s=${line#*:}
if [ "${s: -2}" = "OK" ]; then
rm -f $f
elif [ -f $f ]; then
mv -T -f $f $f.rpmsave
fi
done <<< $(md5sum --check %{oneapi_manifest})
else
# Don't touch any generated files if there's no manifest
# On upgrade, expect lots of modulefiles created as .rpmnew
echo "WARNING: Manifest not found. Previously generated"
echo " modulefiles will not be removed or moved."
# Remove the generated oneAPI module links, remove directories if empty
echo "Removing oneAPI module directory"
find %{OHPC_MODULEDEPS}/oneapi -type l -delete || true
find %{OHPC_MODULEDEPS}/oneapi/* -empty -type d -delete || true
fi
# Remove the generated oneAPI module links, remove directories if empty
echo "Removing oneAPI module directory"
find %{OHPC_MODULEDEPS}/oneapi -type l -delete || true
find %{OHPC_MODULEDEPS}/oneapi/* -empty -type d -delete || true


%files
%{OHPC_BIN}/ohpc-update-modules-intel
%dir %{OHPC_MODULES}/intel
%dir %{OHPC_MODULEDEPS}/oneapi
%dir %{OHPC_MODULEDEPS}/intel
Expand Down

0 comments on commit 2a3f6ef

Please sign in to comment.