build(deps): bump mypy from 1.6.1 to 1.8.0 in /.github/workflows/style

.github/workflows/style/requirements.txt

@@ -2,6 +2,6 @@ black==23.11.0
 clingo==5.6.2
 flake8==6.1.0
 isort==5.12.0
-mypy==1.6.1
+mypy==1.8.0
 types-six==1.16.21.9
 vermin==1.5.2

lib/spack/spack/build_systems/python.py

@@ -6,6 +6,7 @@
 import os
 import re
 import shutil
+import stat
 from typing import Optional
 
 import archspec
@@ -367,6 +368,51 @@ def libs(self):
         raise NoLibrariesError(msg.format(self.spec.name, root))
 
 
+def fixup_shebangs(path: str, old_interpreter: bytes, new_interpreter: bytes):
+    # Recurse into the install prefix and fixup shebangs
+    exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    dirs = [path]
+    hardlinks = set()
+
+    while dirs:
+        with os.scandir(dirs.pop()) as entries:
+            for entry in entries:
+                if entry.is_dir(follow_symlinks=False):
+                    dirs.append(entry.path)
+                    continue
+
+                # Only consider files, not symlinks
+                if not entry.is_file(follow_symlinks=False):
+                    continue
+
+                lstat = entry.stat(follow_symlinks=False)
+
+                # Skip over files that are not executable
+                if not (lstat.st_mode & exe):
+                    continue
+
+                # Don't modify hardlinks more than once
+                if lstat.st_nlink > 1:
+                    key = (lstat.st_ino, lstat.st_dev)
+                    if key in hardlinks:
+                        continue
+                    hardlinks.add(key)
+
+                # Finally replace shebangs if any.
+                with open(entry.path, "rb+") as f:
+                    contents = f.read(2)
+                    if contents != b"#!":
+                        continue
+                    contents += f.read()
+
+                    if old_interpreter not in contents:
+                        continue
+
+                    f.seek(0)
+                    f.write(contents.replace(old_interpreter, new_interpreter))
+                    f.truncate()
+
+
 @spack.builder.builder("python_pip")
 class PythonPipBuilder(BaseBuilder):
     phases = ("install",)
@@ -464,8 +510,11 @@ def global_options(self, spec, prefix):
         """
         return []
 
-    def install(self, pkg, spec, prefix):
-        """Install everything from build directory."""
+    @property
+    def _build_venv_path(self):
+        """Return the path to the virtual environment used for building when
+        python is external."""
+        return os.path.join(self.spec.package.stage.path, "build_env")
 
         args = PythonPipBuilder.std_args(pkg) + [f"--prefix={prefix}"]
 
@@ -492,4 +541,28 @@ def install(self, pkg, spec, prefix):
         with fs.working_dir(self.build_directory):
             pip(*args)
 
+    @spack.builder.run_after("install")
+    def fixup_shebangs_pointing_to_build(self):
+        """When installing a package using an external python, we use a temporary virtual
+        environment which improves build isolation. The downside is that pip produces shebangs
+        that point to the temporary virtual environment. This method fixes them up to point to the
+        underlying Python."""
+        # No need to fixup shebangs if no build venv was used. (this post install function also
+        # runs when install was overridden in another package, so check existence of the venv path)
+        if not os.path.exists(self._build_venv_path):
+            return
+
+        # Use sys.executable, since that's what pip uses.
+        interpreter = (
+            lambda python: python("-c", "import sys; print(sys.executable)", output=str)
+            .strip()
+            .encode("utf-8")
+        )
+
+        fixup_shebangs(
+            path=self.spec.prefix,
+            old_interpreter=interpreter(self._build_venv_python),
+            new_interpreter=interpreter(self.spec["python"].command),
+        )
+
     spack.builder.run_after("install")(execute_install_time_tests)

lib/spack/spack/modules/lmod.py

@@ -284,6 +284,13 @@ def modulerc(self):
         """Returns the modulerc file associated with current module file"""
         return os.path.join(os.path.dirname(self.filename), f".modulerc.{self.extension}")
 
+    @property
+    def modulerc(self):
+        """Returns the modulerc file associated with current module file"""
+        return os.path.join(
+            os.path.dirname(self.filename), ".".join([".modulerc", self.extension])
+        )
+
     def token_to_path(self, name, value):
         """Transforms a hierarchy token into the corresponding path part.
 
@@ -481,6 +488,10 @@ class LmodModulefileWriter(BaseModuleFileWriter):
 
     hide_cmd_format = 'hide_version("%s")'
 
+    modulerc_header: list = []
+
+    hide_cmd_format = 'hide_version("%s")'
+
 
 class CoreCompilersNotFoundError(spack.error.SpackError, KeyError):
     """Error raised if the key 'core_compilers' has not been specified

lib/spack/spack/solver/asp.py

@@ -8,7 +8,6 @@
 import enum
 import itertools
 import os
-import pathlib
 import pprint
 import re
 import types

lib/spack/spack/spec.py

@@ -180,6 +180,9 @@
 SPECFILE_FORMAT_VERSION = 4
 
 
+PropagateValue = collections.namedtuple("PropagateValue", ["value", "propagate"])
+
+
 class InstallStatus(enum.Enum):
     """Maps install statuses to symbols for display.
 
@@ -2042,6 +2045,10 @@ def to_node_dict(self, hash=ht.dag_hash):
 
         params = syaml.syaml_dict(sorted(v.yaml_entry() for _, v in self.variants.items()))
 
+        for k, v in params.items():
+            if self.variants[k].propagate:
+                params[k] = PropagateValue(params[k], self.variants[k].propagate)
+
         # Only need the string compiler flag for yaml file
         params.update(
             sorted(
@@ -5012,7 +5019,12 @@ def from_node_dict(cls, node):
                 for val in values:
                     spec.compiler_flags.add_flag(name, val, False)
             else:
-                spec.variants[name] = vt.MultiValuedVariant.from_node_dict(name, values)
+                if isinstance(values, PropagateValue):
+                    spec.variants[name] = vt.MultiValuedVariant.from_node_dict(
+                        name, values.value, values.propagate
+                    )
+                else:
+                    spec.variants[name] = vt.MultiValuedVariant.from_node_dict(name, values)
 
         spec.external_path = None
         spec.external_modules = None

lib/spack/spack/test/cmd/pkg.py

@@ -303,7 +303,7 @@ def test_pkg_grep(mock_packages, capfd):
     output, _ = capfd.readouterr()
     assert output.strip() == "\n".join(
         spack.repo.PATH.get_pkg_class(name).module.__file__
-        for name in ["splice-a", "splice-h", "splice-t", "splice-vh", "splice-z"]
+        for name in ["splice-a", "splice-b", "splice-h", "splice-t", "splice-vh", "splice-z"]
     )
 
     # ensure that this string isn't fouhnd

lib/spack/spack/test/concretize.py

@@ -25,6 +25,7 @@
 import spack.solver.asp
 import spack.variant as vt
 from spack.concretize import find_spec
+from spack.main import SpackCommand
 from spack.spec import CompilerSpec, Spec
 from spack.version import Version, ver
 
@@ -1468,7 +1469,7 @@ def test_non_default_provider_of_multiple_virtuals(self):
     @pytest.mark.regression("27237")
     @pytest.mark.parametrize(
         "spec_str,expect_installed",
-        [("mpich", True), ("mpich+debug", False), ("mpich~debug", True)],
+        [("mpich", True), ("mpich+debug", False), ("mpich~debug", True), ("mpich++debug", False)],
     )
     @pytest.mark.only_clingo("Use case not supported by the original concretizer")
     def test_concrete_specs_are_not_modified_on_reuse(

lib/spack/spack/test/spec_semantics.py

@@ -1183,6 +1183,84 @@ def test_spec_format_path_posix(spec_str, format_str, expected):
     _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PurePosixPath)
 
 
+@pytest.mark.parametrize(
+    "spec_str,format_str,expected",
+    [
+        ("[email protected]/bar", "{name}-{version}", str(pathlib.Path("zlib-git.foo_bar"))),
+        ("[email protected]/bar", "{name}-{version}-{/hash}", None),
+        ("[email protected]/bar", "{name}/{version}", str(pathlib.Path("zlib", "git.foo_bar"))),
+        (
+            "zlib@{0}=1.0%gcc".format("a" * 40),
+            "{name}/{version}/{compiler}",
+            str(pathlib.Path("zlib", "{0}_1.0".format("a" * 40), "gcc")),
+        ),
+        (
+            "[email protected]/bar=1.0%gcc",
+            "{name}/{version}/{compiler}",
+            str(pathlib.Path("zlib", "git.foo_bar_1.0", "gcc")),
+        ),
+    ],
+)
+def test_spec_format_path(spec_str, format_str, expected):
+    _check_spec_format_path(spec_str, format_str, expected)
+
+
+def _check_spec_format_path(spec_str, format_str, expected, path_ctor=None):
+    spec = Spec(spec_str)
+    if not expected:
+        with pytest.raises((spack.spec.SpecFormatPathError, spack.spec.SpecFormatStringError)):
+            spec.format_path(format_str, _path_ctor=path_ctor)
+    else:
+        formatted = spec.format_path(format_str, _path_ctor=path_ctor)
+        assert formatted == expected
+
+
+@pytest.mark.parametrize(
+    "spec_str,format_str,expected",
+    [
+        (
+            "[email protected]/bar",
+            r"C:\\installroot\{name}\{version}",
+            r"C:\installroot\zlib\git.foo_bar",
+        ),
+        (
+            "[email protected]/bar",
+            r"\\hostname\sharename\{name}\{version}",
+            r"\\hostname\sharename\zlib\git.foo_bar",
+        ),
+        # Windows doesn't attribute any significance to a leading
+        # "/" so it is discarded
+        ("[email protected]/bar", r"/installroot/{name}/{version}", r"installroot\zlib\git.foo_bar"),
+    ],
+)
+def test_spec_format_path_windows(spec_str, format_str, expected):
+    _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PureWindowsPath)
+
+
+@pytest.mark.parametrize(
+    "spec_str,format_str,expected",
+    [
+        ("[email protected]/bar", r"/installroot/{name}/{version}", "/installroot/zlib/git.foo_bar"),
+        ("[email protected]/bar", r"//installroot/{name}/{version}", "//installroot/zlib/git.foo_bar"),
+        # This is likely unintentional on Linux: Firstly, "\" is not a
+        # path separator for POSIX, so this is treated as a single path
+        # component (containing literal "\" characters); secondly,
+        # Spec.format treats "\" as an escape character, so is
+        # discarded (unless directly following another "\")
+        (
+            "[email protected]/bar",
+            r"C:\\installroot\package-{name}-{version}",
+            r"C__installrootpackage-zlib-git.foo_bar",
+        ),
+        # "\" is not a POSIX separator, and Spec.format treats "\{" as a literal
+        # "{", which means that the resulting format string is invalid
+        ("[email protected]/bar", r"package\{name}\{version}", None),
+    ],
+)
+def test_spec_format_path_posix(spec_str, format_str, expected):
+    _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PurePosixPath)
+
+
 @pytest.mark.regression("3887")
 @pytest.mark.parametrize("spec_str", ["py-extension2", "extension1", "perl-extension"])
 def test_is_extension_after_round_trip_to_dict(config, mock_packages, spec_str):

lib/spack/spack/variant.py

@@ -254,19 +254,19 @@ def __init__(self, name, value, propagate=False):
         self.value = value
 
     @staticmethod
-    def from_node_dict(name, value):
+    def from_node_dict(name, value, propagate=False):
         """Reconstruct a variant from a node dict."""
         if isinstance(value, list):
             # read multi-value variants in and be faithful to the YAML
-            mvar = MultiValuedVariant(name, ())
+            mvar = MultiValuedVariant(name, (), propagate)
             mvar._value = tuple(value)
             mvar._original_value = mvar._value
             return mvar
 
         elif str(value).upper() == "TRUE" or str(value).upper() == "FALSE":
-            return BoolValuedVariant(name, value)
+            return BoolValuedVariant(name, value, propagate)
 
-        return SingleValuedVariant(name, value)
+        return SingleValuedVariant(name, value, propagate)
 
     def yaml_entry(self):
         """Returns a key, value tuple suitable to be an entry in a yaml dict.

var/spack/repos/builtin.mock/packages/splice-b/package.py

@@ -0,0 +1,29 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class SpliceB(Package):
+    """Simple package with one optional dependency"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/splice-b-1.0.tar.gz"
+
+    version("1.0", md5="0123456789abcdef0123456789efghij")
+
+    variant("foo", default=True, description="nope")
+    variant("bar", default=False, description="nope")
+    variant("baz", default=False, description="nope")
+
+    depends_on("splice-a")
+
+    provides("something")
+    provides("somethingelse")
+
+    def install(self, spec, prefix):
+        with open(prefix.join("splice-b"), "w") as f:
+            f.write("splice-b: {0}".format(prefix))
+            f.write("splice-z: {0}".format(spec["splice-z"].prefix))

var/spack/repos/builtin/packages/clingo/setuptools.patch

@@ -0,0 +1,14 @@
+diff --git a/cmake/python-site.py b/cmake/python-site.py
+index 1e7fc8ce..95ef827f 100644
+--- a/cmake/python-site.py
++++ b/cmake/python-site.py
+@@ -1,4 +1,7 @@
+-from distutils.sysconfig import get_python_lib, get_config_vars
++try:
++    from setuptools.sysconfig import get_python_lib, get_config_vars
++except ImportError:
++    from distutils.sysconfig import get_python_lib, get_config_vars
+ import sys
+ if sys.argv[1] == "prefix":
+     print(get_python_lib(True, False, sys.argv[2] if len(sys.argv) > 2 else None))
+