build(deps): bump mypy from 1.8.0 to 1.11.1 in /.github/workflows/requirements/style

.github/workflows/requirements/style/requirements.txt

@@ -2,6 +2,6 @@ black==24.8.0
 clingo==5.7.1
 flake8==7.1.1
 isort==5.13.2
-mypy==1.8.0
+mypy==1.11.1
 types-six==1.16.21.20240513
 vermin==1.6.0

lib/spack/docs/binary_caches.rst

@@ -191,6 +191,131 @@ Or you can directly edit the ``mirrors.yaml`` configuration file:
 
 See also :ref:`mirrors`.
 
+-----------------------------------------
+OCI / Docker V2 registries as build cache
+-----------------------------------------
+
+Spack can also use OCI or Docker V2 registries such as Dockerhub, Quay.io,
+Github Packages, GitLab Container Registry, JFrog Artifactory, and others
+as build caches. This is a convenient way to share binaries using public
+infrastructure, or to cache Spack built binaries in Github Actions and
+GitLab CI.
+
+To get started, configure an OCI mirror using ``oci://`` as the scheme,
+and optionally specify a username and password (or personal access token):
+
+.. code-block:: console
+
+    $ spack mirror add --oci-username username --oci-password password my_registry oci://example.com/my_image
+
+Spack follows the naming conventions of Docker, with Dockerhub as the default
+registry. To use Dockerhub, you can omit the registry domain:
+
+.. code-block:: console
+
+    $ spack mirror add --oci-username username --oci-password password my_registry oci://username/my_image
+
+From here, you can use the mirror as any other build cache:
+
+.. code-block:: console
+
+    $ spack buildcache push my_registry <specs...>  # push to the registry
+    $ spack install <specs...> # install from the registry
+
+A unique feature of buildcaches on top of OCI registries is that it's incredibly
+easy to generate get a runnable container image with the binaries installed. This
+is a great way to make applications available to users without requiring them to
+install Spack -- all you need is Docker, Podman or any other OCI-compatible container
+runtime.
+
+To produce container images, all you need to do is add the ``--base-image`` flag
+when pushing to the build cache:
+
+.. code-block:: console
+
+    $ spack buildcache push --base-image ubuntu:20.04 my_registry ninja
+    Pushed to example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack
+
+    $ docker run -it example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack
+    root@e4c2b6f6b3f4:/# ninja --version
+    1.11.1
+
+If ``--base-image`` is not specified, distroless images are produced. In practice,
+you won't be able to run these as containers, since they don't come with libc and
+other system dependencies. However, they are still compatible with tools like
+``skopeo``, ``podman``, and ``docker`` for pulling and pushing.
+
+.. note::
+    The docker ``overlayfs2`` storage driver is limited to 128 layers, above which a
+    ``max depth exceeded`` error may be produced when pulling the image. There
+    are `alternative drivers <https://docs.docker.com/storage/storagedriver/>`_.
+
+------------------------------------
+Using a buildcache in GitHub Actions
+------------------------------------
+
+GitHub Actions is a popular CI/CD platform for building and testing software,
+but each CI job has limited resources, making from source builds too slow for
+many applications. Spack build caches can be used to share binaries between CI
+runs, speeding up CI significantly.
+
+A typical workflow is to include a ``spack.yaml`` environment in your repository
+that specifies the packages to install:
+
+.. code-block:: yaml
+
+    spack:
+      specs: [pkg-x, pkg-y]
+    packages:
+      all:
+        require: target=x86_64_v2
+    mirrors:
+      github_packages: oci://ghcr.io/<user>/<repo>
+
+And a GitHub action that sets up Spack, installs packages from the build cache
+or from sources, and pushes newly built binaries to the build cache:
+
+.. code-block:: yaml
+
+    name: Install Spack packages
+
+    on: push
+
+    env:
+      SPACK_COLOR: always
+
+    jobs:
+      example:
+        runs-on: ubuntu-22.04
+        steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+
+        - name: Install Spack
+          run: |
+            git clone --depth=1 https://github.com/spack/spack.git
+            echo "$PWD/spack/bin/" >> "$GITHUB_PATH"
+
+        - name: Concretize
+          run: spack -e . concretize
+
+        - name: Install
+          run: spack -e . install --no-check-signature --fail-fast
+
+        - name: Push to buildcache
+          run: |
+            spack -e . mirror set --oci-username <user> --oci-password "${{ secrets.GITHUB_TOKEN }}" github_packages
+            spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index github_packages
+            if: always()
+
+The first time this action runs, it will build the packages from source and
+push them to the build cache. Subsequent runs will pull the binaries from the
+build cache. The concretizer will ensure that prebuilt binaries are favored
+over source builds.
+
+The build cache entries appear in the GitHub Packages section of your repository,
+and contain instructions for pulling and running them with ``docker`` or ``podman``.
+
 ----------
 Relocation
 ----------

lib/spack/spack/build_systems/oneapi.py

@@ -274,6 +274,35 @@ def libs(self):
         return find_libraries("*", self.component_prefix.sdk.lib64)
 
 
+class IntelOneApiLibraryPackageWithSdk(IntelOneApiPackage):
+    """Base class for Intel oneAPI library packages with SDK components.
+
+    Contains some convenient default implementations for libraries
+    that expose functionality in sdk subdirectories.
+    Implement the method directly in the package if something
+    different is needed.
+
+    """
+
+    @property
+    def include(self):
+        return join_path(self.component_prefix, "sdk", "include")
+
+    @property
+    def headers(self):
+        return find_headers("*", self.include, recursive=True)
+
+    @property
+    def lib(self):
+        lib_path = join_path(self.component_prefix, "sdk", "lib64")
+        lib_path = lib_path if isdir(lib_path) else dirname(lib_path)
+        return lib_path
+
+    @property
+    def libs(self):
+        return find_libraries("*", root=self.lib, shared=True, recursive=True)
+
+
 class IntelOneApiStaticLibraryList:
     """Provides ld_flags when static linking is needed
 

lib/spack/spack/build_systems/python.py

@@ -417,6 +417,51 @@ def libs(self) -> LibraryList:
         raise NoLibrariesError(msg.format(self.spec.name, platlib, purelib))
 
 
+def fixup_shebangs(path: str, old_interpreter: bytes, new_interpreter: bytes):
+    # Recurse into the install prefix and fixup shebangs
+    exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    dirs = [path]
+    hardlinks = set()
+
+    while dirs:
+        with os.scandir(dirs.pop()) as entries:
+            for entry in entries:
+                if entry.is_dir(follow_symlinks=False):
+                    dirs.append(entry.path)
+                    continue
+
+                # Only consider files, not symlinks
+                if not entry.is_file(follow_symlinks=False):
+                    continue
+
+                lstat = entry.stat(follow_symlinks=False)
+
+                # Skip over files that are not executable
+                if not (lstat.st_mode & exe):
+                    continue
+
+                # Don't modify hardlinks more than once
+                if lstat.st_nlink > 1:
+                    key = (lstat.st_ino, lstat.st_dev)
+                    if key in hardlinks:
+                        continue
+                    hardlinks.add(key)
+
+                # Finally replace shebangs if any.
+                with open(entry.path, "rb+") as f:
+                    contents = f.read(2)
+                    if contents != b"#!":
+                        continue
+                    contents += f.read()
+
+                    if old_interpreter not in contents:
+                        continue
+
+                    f.seek(0)
+                    f.write(contents.replace(old_interpreter, new_interpreter))
+                    f.truncate()
+
+
 @spack.builder.builder("python_pip")
 class PythonPipBuilder(BaseBuilder):
     phases = ("install",)
@@ -536,4 +581,28 @@ def install(self, pkg: PythonPackage, spec: Spec, prefix: Prefix) -> None:
         with fs.working_dir(self.build_directory):
             pip(*args)
 
+    @spack.builder.run_after("install")
+    def fixup_shebangs_pointing_to_build(self):
+        """When installing a package using an external python, we use a temporary virtual
+        environment which improves build isolation. The downside is that pip produces shebangs
+        that point to the temporary virtual environment. This method fixes them up to point to the
+        underlying Python."""
+        # No need to fixup shebangs if no build venv was used. (this post install function also
+        # runs when install was overridden in another package, so check existence of the venv path)
+        if not os.path.exists(self._build_venv_path):
+            return
+
+        # Use sys.executable, since that's what pip uses.
+        interpreter = (
+            lambda python: python("-c", "import sys; print(sys.executable)", output=str)
+            .strip()
+            .encode("utf-8")
+        )
+
+        fixup_shebangs(
+            path=self.spec.prefix,
+            old_interpreter=interpreter(self._build_venv_python),
+            new_interpreter=interpreter(self.spec["python"].command),
+        )
+
     spack.builder.run_after("install")(execute_install_time_tests)

lib/spack/spack/cmd/checksum.py

@@ -84,6 +84,12 @@ def setup_parser(subparser):
         "  `spack checksum [email protected]` autodetects versions 1.2.0 to 1.2.13 from the remote\n"
         "  `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\n"
     )
+    arguments.add_common_arguments(subparser, ["jobs"])
+    subparser.epilog = (
+        "examples:\n"
+        "  `spack checksum [email protected]` autodetects versions 1.2.0 to 1.2.13 from the remote\n"
+        "  `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\n"
+    )
 
 
 def checksum(parser, args):

lib/spack/spack/modules/common.py

@@ -514,6 +514,19 @@ def hidden(self):
 
         return hidden_as_implicit
 
+    @property
+    def hidden(self):
+        """Returns True if the module has been hidden, False otherwise."""
+
+        conf = self.module.configuration(self.name)
+
+        hidden_as_implicit = not self.explicit and conf.get("hide_implicits", False)
+
+        if hidden_as_implicit:
+            tty.debug(f"\tHIDDEN_AS_IMPLICIT : {self.spec.cshort_spec}")
+
+        return hidden_as_implicit
+
     @property
     def context(self):
         return self.conf.get("context", {})

lib/spack/spack/modules/lmod.py

@@ -284,6 +284,13 @@ def modulerc(self):
         """Returns the modulerc file associated with current module file"""
         return os.path.join(os.path.dirname(self.filename), f".modulerc.{self.extension}")
 
+    @property
+    def modulerc(self):
+        """Returns the modulerc file associated with current module file"""
+        return os.path.join(
+            os.path.dirname(self.filename), ".".join([".modulerc", self.extension])
+        )
+
     def token_to_path(self, name, value):
         """Transforms a hierarchy token into the corresponding path part.
 
@@ -481,6 +488,10 @@ class LmodModulefileWriter(BaseModuleFileWriter):
 
     hide_cmd_format = 'hide_version("%s")'
 
+    modulerc_header: list = []
+
+    hide_cmd_format = 'hide_version("%s")'
+
 
 class CoreCompilersNotFoundError(spack.error.SpackError, KeyError):
     """Error raised if the key 'core_compilers' has not been specified

lib/spack/spack/test/cmd/pkg.py

@@ -310,7 +310,7 @@ def test_pkg_grep(mock_packages, capfd):
     output, _ = capfd.readouterr()
     assert output.strip() == "\n".join(
         spack.repo.PATH.get_pkg_class(name).module.__file__
-        for name in ["splice-a", "splice-h", "splice-t", "splice-vh", "splice-z"]
+        for name in ["splice-a", "splice-b", "splice-h", "splice-t", "splice-vh", "splice-z"]
     )
 
     # ensure that this string isn't fouhnd

lib/spack/spack/test/concretize.py

@@ -29,6 +29,7 @@
 import spack.util.libc
 import spack.variant as vt
 from spack.concretize import find_spec
+from spack.main import SpackCommand
 from spack.spec import CompilerSpec, Spec
 from spack.version import Version, VersionList, ver
 
@@ -1589,7 +1590,7 @@ def test_non_default_provider_of_multiple_virtuals(self):
     @pytest.mark.regression("27237")
     @pytest.mark.parametrize(
         "spec_str,expect_installed",
-        [("mpich", True), ("mpich+debug", False), ("mpich~debug", True)],
+        [("mpich", True), ("mpich+debug", False), ("mpich~debug", True), ("mpich++debug", False)],
     )
     @pytest.mark.only_clingo("Use case not supported by the original concretizer")
     def test_concrete_specs_are_not_modified_on_reuse(

lib/spack/spack/user_environment.py

@@ -5,6 +5,10 @@
 import os
 import re
 import sys
+from contextlib import contextmanager
+from typing import Callable
+
+from llnl.util.lang import nullcontext
 
 import spack.build_environment
 import spack.config

lib/spack/spack/variant.py

@@ -255,19 +255,19 @@ def __init__(self, name, value, propagate=False):
         self.value = value
 
     @staticmethod
-    def from_node_dict(name, value):
+    def from_node_dict(name, value, propagate=False):
         """Reconstruct a variant from a node dict."""
         if isinstance(value, list):
             # read multi-value variants in and be faithful to the YAML
-            mvar = MultiValuedVariant(name, ())
+            mvar = MultiValuedVariant(name, (), propagate)
             mvar._value = tuple(value)
             mvar._original_value = mvar._value
             return mvar
 
         elif str(value).upper() == "TRUE" or str(value).upper() == "FALSE":
-            return BoolValuedVariant(name, value)
+            return BoolValuedVariant(name, value, propagate)
 
-        return SingleValuedVariant(name, value)
+        return SingleValuedVariant(name, value, propagate)
 
     def yaml_entry(self):
         """Returns a key, value tuple suitable to be an entry in a yaml dict.