From f1618bfa2631cf6eda75da89e71891ba300e97ac Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 17 Feb 2026 16:46:01 -0700 Subject: [PATCH 01/98] added spack source mirrors capability, needs testing --- stackinator/builder.py | 2 ++ stackinator/schema/config.json | 7 +++++++ stackinator/templates/Makefile | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..c3de44db 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -232,6 +232,8 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + # pass source_mirrors to Makefile render + source_mirrors=recipe.config.get("source_mirrors", {}), exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d6fec3a0..4b91011d 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,6 +64,13 @@ } } }, + "source_mirrors" : { + "type" : "object", + "additionalProperties": { + "type" : "string" + }, + "default": {} + }, "modules" : { "type": "boolean" }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..3da2e00e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -39,6 +39,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if source_mirrors %} + echo "Replacing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Adding mirrors" + {% for name, url in source_mirrors.items() %} + $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + {% endfor %} + echo "Current mirror list:" + spack mirror list + {% endif %} touch mirror-setup compilers: mirror-setup From 954a6901a6544ef6e190bea4cdfc8ada4b6acffe Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 18 Feb 2026 14:34:05 -0700 Subject: [PATCH 02/98] removed lanl stuff --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3da2e00e..fcf6d68a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,13 +40,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Replacing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Removing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' echo "Adding mirrors" {% for name, url in source_mirrors.items() %} $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} {% endfor %} - echo "Current mirror list:" + echo "Spack mirrors for this recipe:" spack mirror list {% endif %} touch mirror-setup From 97d43a0f88e8455cf1ec13b5a18d342b1d0d5698 Mon Sep 17 00:00:00 2001 From: Claire Ann Winogrodzki Date: Tue, 24 Feb 2026 09:38:20 -0700 Subject: [PATCH 03/98] add source mirrors via config.yaml --- stackinator/templates/Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68a..d7abb1ec 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,14 +40,12 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list {% endif %} touch mirror-setup From 15e97f1ede71cc5198f07908236056b5cf49be5a Mon Sep 17 00:00:00 2001 From: Claire Ann Winogrodzki Date: Tue, 24 Feb 2026 09:42:30 -0700 Subject: [PATCH 04/98] add source mirrors via config.yaml --- stackinator/templates/Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68a..d7abb1ec 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,14 +40,12 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list {% endif %} touch mirror-setup From f91fbad96675e6645c9ad356c2efc5bad481971f Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 10:43:33 -0700 Subject: [PATCH 05/98] add source mirrors via config.yaml and retain spack default mirror --- nohup.out | 21 +++++++++++++++++++++ output.log | 2 ++ 2 files changed, 23 insertions(+) create mode 100644 nohup.out create mode 100644 output.log diff --git a/nohup.out b/nohup.out new file mode 100644 index 00000000..47d83f42 --- /dev/null +++ b/nohup.out @@ -0,0 +1,21 @@ +Stackinator + recipe path: /usr/projects/hpctools/grodzki/uenv/my-recipes/recipes/prgenv-gnu-openmpi + build path : /dev/shm/grodzki/build + system : /usr/projects/hpctools/grodzki/uenv/my-recipes/cluster-config/venadito + mount : default + build cache: None + develop : False +spack: https://github.com/spack/spack.git already cloned to /dev/shm/grodzki/build/spack +spack: fetching releases/v1.0 +spack: checking out releases/v1.0 +spack: commit hash is f36409b78591f8b02e8332eb4ad78da62c02571e +spack-packages: https://github.com/spack/spack-packages.git already cloned to /dev/shm/grodzki/build/spack-packages +spack-packages: fetching develop +spack-packages: checking out develop +spack-packages: commit hash is fc656595ff4c7e9c7e5045625dfdd3e92e97b10e + +Configuration finished, run the following to build the environment: + +cd /dev/shm/grodzki/build +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32 +see logfile for more information /tmp/grodzki/log_stackinator_20260428-112546 diff --git a/output.log b/output.log new file mode 100644 index 00000000..d0fea532 --- /dev/null +++ b/output.log @@ -0,0 +1,2 @@ +nohup: ignoring input +/usr/bin/env: ‘uv’: No such file or directory From 2a6571e9f6c23ceb14cdabe69316f016b07475aa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 16:04:05 -0700 Subject: [PATCH 06/98] fixed spaces/tabs typo --- stackinator/templates/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d7abb1ec..ca8fb311 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,7 +32,7 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - + {% if cache %} $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} From 06f9dacbcab8a3dfcb25d620854effd54e79cf83 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:16:51 -0700 Subject: [PATCH 07/98] Added mirror configuration json schema. --- stackinator/schema/config.json | 7 ------- stackinator/schema/mirror.json | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 stackinator/schema/mirror.json diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 4b91011d..d6fec3a0 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,13 +64,6 @@ } } }, - "source_mirrors" : { - "type" : "object", - "additionalProperties": { - "type" : "string" - }, - "default": {} - }, "modules" : { "type": "boolean" }, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..b32a3cd1 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,37 @@ +# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) +{ + # Order matters, so we need an array. + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"] + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax.", + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled.", + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + } + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages.", + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} From 7692677f5e29f8975c5ea03c89794addf84415ea Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:48:06 -0700 Subject: [PATCH 08/98] Incorporating Makefile changes. --- stackinator/templates/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ca8fb311..d1a06dd4 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,21 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup From 940886b14534faf8d293382e8f966f6286910af0 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:05:24 -0700 Subject: [PATCH 09/98] mirrors --- stackinator/mirror.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackinator/mirror.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..e69de29b From ffb60834f94c91f018d2b57cf3d4ed1778033008 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:26:59 -0700 Subject: [PATCH 10/98] mirrors --- stackinator/mirror.py | 58 +++++++++++++++++++++++++++++++++++++++++++ stackinator/recipe.py | 32 ++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e69de29b..83989b83 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -0,0 +1,58 @@ +import os +import pathlib + +import yaml + +from . import schema + + +def configuration_from_file(file, mount): + with file.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(raw["root"])) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + raw["root"] = path + + # Put the build cache in a sub-directory named after the mount point. + # This avoids relocation issues. + raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) + + # verify that the key file exists if it was specified + key = raw["key"] + if key is not None: + key = pathlib.Path(os.path.expandvars(key)) + if not key.is_absolute(): + raise FileNotFoundError(f"The build cache key '{key}' is not absolute") + if not key.is_file(): + raise FileNotFoundError(f"The build cache key '{key}' does not exist") + raw["key"] = key + + return raw + + +def generate_mirrors_yaml(config): + path = config["path"].as_posix() + mirrors = { + "mirrors": { + "alpscache": { + "fetch": { + "url": f"file://{path}", + }, + "push": { + "url": f"file://{path}", + }, + } + } + } + + return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..d0ec018f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,14 +170,20 @@ def __init__(self, args): self.generate_environment_specs(raw) # optional mirror configurtion + if mirrors_path.is_file():zx mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): self._logger.warning( "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." ) raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - self.mirror = (args.cache, self.mount) + # self.mirror = (args.cache, self.mount) + + # load the optional mirrors.yaml from system config: + mirrors_path = self.system_config_path / "mirrors.yaml" + if mirrors_path.is_file(): + self.mirrors = (mirrors_path, self.mount) + # update mirror setter and cache.configuration_from_file() # optional post install hook if self.post_install_hook is not None: @@ -262,6 +268,28 @@ def mirror(self, configuration): self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + @property + def mirrors(self): + return self._mirrors + + # old: self.mirror = (args.cache, self.mount) + # new: self.mirror = (mirrors_yaml_path, self.mount) + + @mirrors.setter + def (self, configuration): + self._logger.debug(f"configuring mirrors with {configuration}") + self._mirrors = None + + file, mount = configuration + + if file is not None: + mirror_config_path = pathlib.Path(file) + if not mirror_config_path.is_file(): + raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") + + self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + + @property def config(self): return self._config From 04f99084f4877657c32fdd251dcadd4f1e93f074 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:40:10 -0700 Subject: [PATCH 11/98] validate mirror config --- stackinator/mirror.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 83989b83..be784864 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,5 +1,6 @@ import os import pathlib +import urllib.request import yaml @@ -14,30 +15,28 @@ def configuration_from_file(file, mount): # validate the yaml schema.CacheValidator.validate(raw) - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw + mirrors = [mirror for mirror in raw if mirror["enabled"]] + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') + + return mirrors def generate_mirrors_yaml(config): From b45039d71a99af60cbd657c42439fd1e60bd0c5d Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:01:41 -0700 Subject: [PATCH 12/98] Updating recipe to handle new mirrors format. --- stackinator/mirror.py | 4 +-- stackinator/recipe.py | 72 ++++++++++--------------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index be784864..08c303bb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -7,7 +7,7 @@ from . import schema -def configuration_from_file(file, mount): +def configuration_from_file(file): with file.open() as fid: # load the raw yaml input raw = yaml.load(fid, Loader=yaml.Loader) @@ -54,4 +54,4 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file + return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d0ec018f..281ab8b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,21 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - if mirrors_path.is_file():zx - mirrors_path = self.path / "mirrors.yaml" - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - # self.mirror = (args.cache, self.mount) + mirrors_path = self.system_config_path/'mirrors.yaml' + self._logger.debug(f"opening {mirrors_path}") # load the optional mirrors.yaml from system config: - mirrors_path = self.system_config_path / "mirrors.yaml" - if mirrors_path.is_file(): - self.mirrors = (mirrors_path, self.mount) - # update mirror setter and cache.configuration_from_file() + self.mirrors = self.system_config_path / "mirrors.yaml" # optional post install hook if self.post_install_hook is not None: @@ -242,54 +233,25 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror + def mirrors(self): + return self._mirrors # configuration is a tuple with two fields: # - a Path of the yaml file containing the cache configuration # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - @property - def mirrors(self): - return self._mirrors - - # old: self.mirror = (args.cache, self.mount) - # new: self.mirror = (mirrors_yaml_path, self.mount) - @mirrors.setter - def (self, configuration): - self._logger.debug(f"configuring mirrors with {configuration}") + def mirrors(self, path: Optional[pathlib.Path]): + """Initialize the mirrors property from config.""" self._mirrors = None + if path is not None: + if not path.is_file(): + raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " + "readable file.") - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") - - self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - + self._logger.debug(f"configuring mirrors from {path}") + self._mirrors = mirror.configuration_from_file(path) + @property def config(self): return self._config @@ -569,7 +531,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, From d3ee8e48507231b9c935d1bbdbb4f1f58d4741d9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:35:42 -0700 Subject: [PATCH 13/98] Updating mirror configuration more. --- stackinator/main.py | 16 ++++--- stackinator/mirror.py | 79 +++++++++++++++++++++++----------- stackinator/recipe.py | 27 +++--------- stackinator/schema/mirror.json | 7 ++- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 08c303bb..40487f48 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,42 +1,73 @@ import os import pathlib import urllib.request +from typing import Optional import yaml from . import schema -def configuration_from_file(file): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) +def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") # validate the yaml schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - mirror["url"] = path - - else: - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - return mirrors + return mirrors def generate_mirrors_yaml(config): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 281ab8b2..4b7c038c 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,11 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - mirrors_path = self.system_config_path/'mirrors.yaml' - self._logger.debug(f"opening {mirrors_path}") - - # load the optional mirrors.yaml from system config: - self.mirrors = self.system_config_path / "mirrors.yaml" + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,22 +236,7 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirrors.setter - def mirrors(self, path: Optional[pathlib.Path]): - """Initialize the mirrors property from config.""" - self._mirrors = None - if path is not None: - if not path.is_file(): - raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " - "readable file.") - - self._logger.debug(f"configuring mirrors from {path}") - self._mirrors = mirror.configuration_from_file(path) - + @property def config(self): return self._config diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b32a3cd1..a53cc34e 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -23,7 +23,12 @@ "type": "boolean", "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", - } + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, "public_key": { "type": "string", "description": "Public PGP key for validating binary cache packages.", From 0ed23d8f6f487100d522875afcbb6d20955e9aa4 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:25:37 -0700 Subject: [PATCH 14/98] mirror yaml generator --- stackinator/mirror.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 40487f48..315d2e8e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -70,19 +70,16 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N return mirrors -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } +def generate_mirrors_yaml(mirrors): + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, } - } - return yaml.dump(mirrors, default_flow_style=False) + return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file From e2a67ab1d3a6a08e1f8f5222a46bab715b77d96c Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:37:34 -0700 Subject: [PATCH 15/98] update mirrors --- stackinator/builder.py | 10 ++++------ stackinator/mirror.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index c3de44db..5b223ffe 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -226,14 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - # pass source_mirrors to Makefile render - source_mirrors=recipe.config.get("source_mirrors", {}), + mirrors=recipe.mirrors exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -314,11 +312,11 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + if recipe.mirrors: dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 315d2e8e..e802018a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -67,7 +67,20 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N except urllib.error.URLError as e: print(f'Error: {e.reason}') - return mirrors + if mirror["key"]: + #if path, check if exists + path = pathlib.Path(os.path.expandvars(mirror["key"])) + if path.exists(): + if not path.is_file(): + raise FileNotFoundError(f"The key path '{path}' is not a file") + else + #if key, save to file, change to path + + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors def generate_mirrors_yaml(mirrors): From 73463861a35602c9a0fc77dfb594ff58cb748c10 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:54:59 -0700 Subject: [PATCH 16/98] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 74 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e802018a..afbacafb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,15 +2,21 @@ import pathlib import urllib.request from typing import Optional +import magic import yaml from . import schema +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" -def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + path = system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input @@ -54,36 +60,33 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' does not exist") mirror["url"] = path - else: + elif url.beginswith("https://"): try: request = urllib.request.Request(url, method='HEAD') response = urllib.request.urlopen(request) except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - if mirror["key"]: - #if path, check if exists - path = pathlib.Path(os.path.expandvars(mirror["key"])) - if path.exists(): - if not path.is_file(): - raise FileNotFoundError(f"The key path '{path}' is not a file") - else - #if key, save to file, change to path + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml return mirrors -def generate_mirrors_yaml(mirrors): +def setup(mirrors, config_path): + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + with dst.open("w") as fid: + fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -95,4 +98,37 @@ def generate_mirrors_yaml(mirrors): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file + return yaml.dump(yaml, default_flow_style=False) + +#called from builder +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not.path.is_file() + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{key}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + # copy file to key store + with file open: + data = key.read + dest = mkdir(new_key_file) + dest.write(data) + # mirror["key"] = new_path + + else: + # if PGP key, convert to binary, ???, convert back + # if key, save to file, change to path + + \ No newline at end of file From e722f498c8315ce0fc3cd7471824a5648b557da2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:57:53 -0700 Subject: [PATCH 17/98] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 44 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index afbacafb..e521b597 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -12,7 +12,6 @@ class MirrorConfigError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -62,7 +61,7 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt if not path.is_absolute(): raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") mirror["url"] = path @@ -82,11 +81,12 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt return mirrors -def setup(mirrors, config_path): +def yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -98,13 +98,22 @@ def setup(mirrors, config_path): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + -#called from builder def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + for mirror in mirrors: if mirror["key"]: key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if path.exists(): @@ -115,20 +124,23 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): raise MirrorConfigError( - f"'{key}' is not a valid GPG key. " + f"'{path}' is not a valid GPG key. " f"Check the key listed in mirrors.yaml in system config.") - # copy file to key store - with file open: - data = key.read - dest = mkdir(new_key_file) - dest.write(data) - # mirror["key"] = new_path + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) else: # if PGP key, convert to binary, ???, convert back - # if key, save to file, change to path + with open(dst, "w") as file: + file.write(key) - \ No newline at end of file + # update mirror with new path + mirror["key"] = dst \ No newline at end of file From eb2685f020534c0c993fa2779589ebf08ad0f226 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 9 Mar 2026 14:33:15 -0600 Subject: [PATCH 18/98] connecting mirrors to builder.py --- stackinator/builder.py | 9 +++++---- stackinator/mirror.py | 9 ++++----- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 16 +++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5b223ffe..7918cb24 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -231,7 +231,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured + key_store = self.path / ".gnupg" if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e521b597..0e5ee5d1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,7 +1,7 @@ import os import pathlib import urllib.request -from typing import Optional +from typing import Optional, List, Dict import magic import yaml @@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: + #if mirror["bootstrap"]: #make bootstrap dirs #bootstrap//metadata.yaml return mirrors -def yaml_setup(mirrors, config_path): +def spack_yaml_setup(mirrors, config_path): """Generate the mirrors.yaml for spack""" dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") yaml = {"mirrors": {}} for m in mirrors: @@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: if not path.is_absolute(): #try prepending system config path path = system_config_path + path - if not.path.is_file() + if not path.is_file(): raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4b7c038c..c59790b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,7 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a53cc34e..a8be6ab3 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,28 +1,26 @@ -# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) { - # Order matters, so we need an array. "type" : "array", "items": { "type": "object", - "required": ["name", "url"] + "required": ["name", "url"], "properties": { "name": { "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax.", + "description": "The name of this mirror. Should be follow standard variable naming syntax." }, "url": { "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, "enabled": { "type": "boolean", "default": true, - "description": "Whether this mirror is enabled.", + "description": "Whether this mirror is enabled." }, "bootstrap": { "type": "boolean", "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, "buildcache": { "type": "boolean", @@ -31,7 +29,7 @@ }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages.", + "description": "Public PGP key for validating binary cache packages." }, "description": { "type": "string", @@ -39,4 +37,4 @@ } } } -} +} \ No newline at end of file From 9d40a836ae0e380a282c208e8498bba741de1fe6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 13:42:30 -0600 Subject: [PATCH 19/98] Put the mirror manipulation code in a class. --- stackinator/builder.py | 14 ++- stackinator/mirror.py | 239 +++++++++++++++++++++-------------------- stackinator/recipe.py | 3 +- 3 files changed, 134 insertions(+), 122 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 7918cb24..b35545b0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -313,11 +314,14 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured key_store = self.path / ".gnupg" - if recipe.mirrors: - mirror.key_setup(recipe.mirrors, config_path, key_store) - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - mirror.spack_yaml_setup(recipe.mirrors, dst) + mirrors = recipe.mirrors + if mirrors: + mirrors.key_setup(recipe.mirrors, config_path, key_store) + dest = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dest}") + mirrors.create_spack_mirrors_yaml(dest) + + # Setup bootstrap mirror configs. # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 0e5ee5d1..bccd5422 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,6 +1,7 @@ import os import pathlib import urllib.request +import urllib.error from typing import Optional, List, Dict import magic @@ -8,138 +9,146 @@ from . import schema -class MirrorConfigError(RuntimeError): +class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - -def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): - """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" - - path = system_config_root/"mirrors.yaml" - if path.exists(): - with path.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - print(f"Configuring mirrors and buildcache from '{path}'") - - # validate the yaml - schema.CacheValidator.validate(raw) - - mirrors = [mirror for mirror in raw if mirror["enabled"]] - else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise RuntimeError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") - - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', - 'url': cmdline_cache, - 'buildcache': True, - 'bootstrap': False, - } - ) - - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The mirror path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' is not a directory") - - mirror["url"] = path - - elif url.beginswith("https://"): - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - raise MirrorConfigError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - - #if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + + def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise MirrorError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) return mirrors + def _check_mirrors(self): + """Validate the mirror config entries.""" -def spack_yaml_setup(mirrors, config_path): - """Generate the mirrors.yaml for spack""" + for mirror in self.mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' is not a directory") - dst = config_path / "mirrors.yaml" + mirror["url"] = path - yaml = {"mirrors": {}} + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - for m in mirrors: - name = m["name"] - url = m["url"] + def create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" - yaml["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } + raw = {"mirrors": {}} - with dst.open("w") as file: - yaml.dump(yaml, default_flow_style=False) + for m in self.mirrors: + name = m["name"] + url = m["url"] - # return dst + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) -def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + def bootstrap_setup(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - for mirror in mirrors: - if mirror["key"]: - key = mirror["key"] - # key will be saved under key_store/mirror_name.gpg - dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = system_config_path + path - if not path.is_file(): - raise FileNotFoundError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") + def key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - file_type = magic.from_file(path) + for mirror in self.mirrors: + if mirror["key"]: + key = mirror["key"] - if not file_type.startswith("OpenPGP Public Key"): - raise MirrorConfigError( - f"'{path}' is not a valid GPG key. " - f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dst, 'w') as writer: - data = reader.read() - writer.write(data) + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): + raise MirrorError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + # if PGP key, convert to binary, ???, convert back + with open(dest, "w") as file: + file.write(key) - else: - # if PGP key, convert to binary, ???, convert back - with open(dst, "w") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dst \ No newline at end of file + # update mirror with new path + mirror["key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index c59790b2..b03d7510 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,9 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. - self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) + self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: From 2a7bd1dcacf433389d726f8b497db330da78c749 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:04:10 -0600 Subject: [PATCH 20/98] preserve cache for makefile --- stackinator/builder.py | 1 + stackinator/recipe.py | 9 +++++++-- stackinator/schema.py | 1 + stackinator/templates/Makefile | 21 ++++++++------------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b35545b0..2a2ceefb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,6 +227,7 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( + cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index b03d7510..9915bf91 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,6 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] # optional post install hook if self.post_install_hook is not None: @@ -235,6 +236,10 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors + + @property + def cache(self): + return self._cache @property def config(self): @@ -515,7 +520,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirrors + push_to_cache = self.cache files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -546,7 +551,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.cache is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d1a06dd4..9484e3c7 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,25 +32,20 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + {% if cache %} - # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} {% endif %} {% if mirrors %} - @echo "Adding mirrors and gpg keys." - {% for mirror_info in mirrors | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} - $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} + @echo "Adding mirror gpg keys." + {% for mirror in mirrors | reverse %} + {% if mirror.public_key %} + $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} + {% endif %} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} - {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} - $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} - {% endfor %} touch mirror-setup compilers: mirror-setup @@ -89,14 +84,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 8589cff12c92fca787c71e5f9bae2adc77fdf2d9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:09:46 -0600 Subject: [PATCH 21/98] Adding bootstrap mirror configs. --- stackinator/builder.py | 17 ++++++++-------- stackinator/mirror.py | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2a2ceefb..acc2d91b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,15 +314,14 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - key_store = self.path / ".gnupg" - mirrors = recipe.mirrors - if mirrors: - mirrors.key_setup(recipe.mirrors, config_path, key_store) - dest = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dest}") - mirrors.create_spack_mirrors_yaml(dest) - - # Setup bootstrap mirror configs. + if recipe.mirrors: + recipe.mirrors.key_setup(config_path) + + self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") + recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') + + # Setup bootstrap mirror configs. + recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bccd5422..9a7dc797 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,7 +2,7 @@ import pathlib import urllib.request import urllib.error -from typing import Optional, List, Dict +from typing import ByteString, Optional, List, Dict import magic import yaml @@ -22,8 +22,10 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - - self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: @@ -107,12 +109,43 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def bootstrap_setup(self, config_root: pathlib.Path): + def create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + 'sources': [], + 'trusted': {}, + } + + for mirror in self.bootstrap_mirrors: + name = mirror['name'] + bs_mirror_path = config_root/f'bootstrap/{name}' + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml['sources'].append( + { + 'name': name, + 'metadata': bs_mirror_path, + } + ) + # And trust each one + bootstrap_yaml['trusted'][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True) + bs_mirror_yaml = { + 'type': 'install', + 'info': mirror['url'], + } + with (bs_mirror_path/'metadata.yaml').open('w') as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root/'bootstrap.yaml').open('w') as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) - - def key_setup(self, key_store: pathlib.Path): + def key_setup(self, config_root: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d6a3df73d17fb60086187cba44427c7b454fc847 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:16:21 -0600 Subject: [PATCH 22/98] Reverted to defining the key store path in builder. --- stackinator/builder.py | 2 +- stackinator/mirror.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index acc2d91b..80a3977b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,7 +315,7 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: - recipe.mirrors.key_setup(config_path) + recipe.mirrors.key_setup(config_path/'key_store') self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9a7dc797..9f162768 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -145,7 +145,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, config_root: pathlib.Path): + def key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From ee0d45fe39bf61cc3d0ac8b131afe359ff91314c Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:22:31 -0600 Subject: [PATCH 23/98] Compressed mirror config setup into a single interface. --- stackinator/builder.py | 5 +---- stackinator/mirror.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 80a3977b..f593fb05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,12 +315,9 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") recipe.mirrors.key_setup(config_path/'key_store') - - self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - - # Setup bootstrap mirror configs. recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9f162768..6de59937 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -15,6 +15,9 @@ class MirrorError(RuntimeError): class Mirrors: """Manage the definition of mirrors in a recipe.""" + KEY_STORE_DIR = 'key_store' + MIRRORS_YAML = 'mirrors.yaml' + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -92,7 +95,14 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - def create_spack_mirrors_yaml(self, dest: pathlib.Path): + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root/self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): """Generate the mirrors.yaml for our build directory.""" raw = {"mirrors": {}} @@ -109,7 +119,7 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def create_bootstrap_configs(self, config_root: pathlib.Path): + def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" if not self.bootstrap_mirrors: @@ -145,7 +155,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, key_store: pathlib.Path): + def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From 90d7733cba58581938fd6809505f248827556e8a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:26:49 -0600 Subject: [PATCH 24/98] Catching builder exceptions. --- stackinator/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f593fb05..5bab0eae 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirrors: - self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") - recipe.mirrors.key_setup(config_path/'key_store') - recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - recipe.mirrors.create_bootstrap_configs(config_path) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to From 7362cbcc17072c518a46793ebca0228c98d2c6c7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:33:08 -0600 Subject: [PATCH 25/98] fixing key setup --- stackinator/mirror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 6de59937..c3b2c404 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -159,8 +159,8 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["key"]: - key = mirror["key"] + if mirror["public_key"]: + key = mirror["public_key"] # key will be saved under key_store/mirror_name.gpg dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() From 2a4632d900cb6023d113fcbb21d7f2ff49690d1a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:46:21 -0600 Subject: [PATCH 26/98] In progress. --- stackinator/mirror.py | 78 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index c3b2c404..8f5e4400 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,3 +1,4 @@ +import base64 import os import pathlib import urllib.request @@ -29,7 +30,8 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + [None]).pop(0) self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + # Will hold a list of all the keys + self.keys = None def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: """Load the mirrors file, if one exists.""" @@ -159,39 +161,47 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["public_key"]: - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() - - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + if not mirror["public_key"]: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - # if PGP key, convert to binary, ???, convert back - with open(dest, "w") as file: - file.write(key) + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") - # update mirror with new path - mirror["key"] = dest + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + try: + key = base64.b64decode(key) + except ValueError as err: + pass + magic.from_buffer(key) + + # if PGP key, convert to binary, ???, convert back + with open(dest, "wb") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dest From 39150366f7ab3bd15f24c1e4ecfe039f8dff7d34 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi <9337627+albestro@users.noreply.github.com> Date: Thu, 26 Feb 2026 08:50:31 +0100 Subject: [PATCH 27/98] Enforce gcc~builtins (#284) With https://github.com/spack/spack-packages/pull/3106 a default value change for `gcc` variant `binutils` has been introduced. This triggers a problem (see [CI error](https://gitlab.com/cscs-ci/ci-testing/webhook-ci/mirrors/551234120955960/1440398897047560/-/jobs/13261281018#L447)) when building `binutils`, which by default is built `libs=static,shared` but we explicitly prefer `zstd libs=static`. https://github.com/eth-cscs/stackinator/blob/c7db13cd3c12d595cf6a7793db88603db9495889/stackinator/templates/compilers.gcc.spack.yaml#L22-L23 The proposed change does not fix the root problem, but it simply prefers building `gcc~builtins` as we've always did, completely workarounding the underlying problem. I'm going to address the root issue in spack, but for what concerns stackinator, at the moment IMHO this is the most reasonable and simple solution. --- stackinator/templates/compilers.gcc.spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml index 0c07d06c..601d9512 100644 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ b/stackinator/templates/compilers.gcc.spack.yaml @@ -12,7 +12,7 @@ spack: reuse: false packages: gcc: - variants: [build_type=Release +bootstrap +profiled +strip] + variants: [build_type=Release +bootstrap +profiled +strip ~binutils] mpc: variants: [libs=static] gmp: From e11685cbc4a3067e0f2915d84911e5bfab1613aa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 09:43:02 -0600 Subject: [PATCH 28/98] added GPG key verification --- stackinator/builder.py | 2 +- stackinator/mirror.py | 73 ++++++++++++++++++++++-------------------- stackinator/recipe.py | 12 ++----- 3 files changed, 41 insertions(+), 46 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5bab0eae..f8083997 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors, + mirrors=recipe.mirrors.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8f5e4400..48d30660 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -5,6 +5,7 @@ import urllib.error from typing import ByteString, Optional, List, Dict import magic +import base64 import yaml @@ -42,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - schema.CacheValidator.validate(raw) + #schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -78,7 +79,7 @@ def _check_mirrors(self): for mirror in self.mirrors: url = mirror["url"] - if url.beginswith("file://"): + if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): @@ -88,7 +89,7 @@ def _check_mirrors(self): mirror["url"] = path - elif url.beginswith("https://"): + elif url.startswith("https://"): try: request = urllib.request.Request(url, method='HEAD') urllib.request.urlopen(request) @@ -159,49 +160,51 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + key_store.mkdir(exist_ok=True) for mirror in self.mirrors: - if not mirror["public_key"]: - continue + if mirror.get("public_key"): + key = mirror["public_key"] - key = mirror["public_key"] + # key will be saved under key_store/mirror_name.gpg - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): #try prepending system config path path = self._system_config_root/path + + if path.exists(): if not path.is_file(): raise MirrorError( - f"The key path '{path}' is not a file. " + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror {mirror["name"]} is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") - + # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - try: - key = base64.b64decode(key) - except ValueError as err: - pass - magic.from_buffer(key) - - # if PGP key, convert to binary, ???, convert back - with open(dest, "wb") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dest + with open(dest, 'wb') as writer: + writer.write(binary_key) + # update mirror with new path + mirror["public_key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 9915bf91..a15569b6 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -173,8 +173,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] + self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self.cache = self.mirrors.build_cache_mirror # optional post install hook if self.post_install_hook is not None: @@ -232,14 +232,6 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - - @property - def mirrors(self): - return self._mirrors - - @property - def cache(self): - return self._cache @property def config(self): From 593159fcde470503a79063fdfd524818de670a3b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 14:29:56 -0600 Subject: [PATCH 29/98] unit tests for mirrors --- stackinator/mirror.py | 4 +- unittests/__init__.py | 0 .../data/systems/mirror-bad-key/bad_key.gpg | 1 + .../data/systems/mirror-bad-key/mirrors.yaml | 3 ++ .../systems/mirror-bad-keypath/mirrors.yaml | 3 ++ .../data/systems/mirror-bad-url/mirrors.yaml | 2 + unittests/data/systems/mirror-ok/mirrors.yaml | 11 +++++ unittests/test_mirrors.py | 42 +++++++++++++++++++ 8 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 unittests/__init__.py create mode 100644 unittests/data/systems/mirror-bad-key/bad_key.gpg create mode 100644 unittests/data/systems/mirror-bad-key/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-keypath/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-url/mirrors.yaml create mode 100644 unittests/data/systems/mirror-ok/mirrors.yaml create mode 100644 unittests/test_mirrors.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 48d30660..d021aee9 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -43,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - #schema.CacheValidator.validate(raw) + schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -197,7 +197,7 @@ def _key_setup(self, key_store: pathlib.Path): f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" diff --git a/unittests/__init__.py b/unittests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/unittests/data/systems/mirror-bad-key/bad_key.gpg b/unittests/data/systems/mirror-bad-key/bad_key.gpg new file mode 100644 index 00000000..d7980bbf --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/bad_key.gpg @@ -0,0 +1 @@ +This is a bad key \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml new file mode 100644 index 00000000..ed27df7a --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key + url: https://mirror.spack.io + public_key: /bad_key.gpg \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml new file mode 100644 index 00000000..e671c45a --- /dev/null +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key-path + url: https://mirror.spack.io + public_key: /path/doesnt/exist \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml new file mode 100644 index 00000000..522c2326 --- /dev/null +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -0,0 +1,2 @@ +- name: bad-url + url: google.com \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml new file mode 100644 index 00000000..9edf8d49 --- /dev/null +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -0,0 +1,11 @@ +- name: fake-mirror + url: https://google.com +- name: disabled-mirror + url: https://google.com + enabled: false +- name: buildcache-mirror + url: https://cache.spack.io/ + buildcache: true +- name: bootstrap-mirror + url: https://mirror.spack.io + bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py new file mode 100644 index 00000000..6283c1a3 --- /dev/null +++ b/unittests/test_mirrors.py @@ -0,0 +1,42 @@ +import pytest +import pathlib +import stackinator.mirror as mirror +import yaml + +@pytest.fixture +def test_path(): + return pathlib.Path(__file__).parent.resolve() + +@pytest.fixture +def systems_path(test_path): + return test_path / "data" / "systems" + +@pytest.fixture +def valid_mirrors(systems_path): + mirrors = {} + mirrors["fake-mirror"] = {'url': 'https://google.com'} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + return mirrors + +def test_mirror_init(systems_path, valid_mirrors): + path = systems_path / "mirror_ok" + mirrors = mirror.Mirrors(path) + print(valid_mirrors) + print(mirrors) + assert mirrors == valid_mirrors + assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] + assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + # assert disabled mirror not in mirrors + for mir in mirrors: + assert mir["enabled"] + # test that cmdline_cache gets added to mirrors? + +def test_create_spack_mirrors_yaml(systems_path): + pass + +def test_create_bootstrap_configs(): + pass + +def test_key_setup(): + pass \ No newline at end of file From e1a40fd0239e784b1c7d4276db2558a2f3c2ba9f Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 14:45:07 -0600 Subject: [PATCH 30/98] mirrors.yaml is now a name:{} mapping --- stackinator/mirror.py | 221 +++++++++++++++++++++------------ stackinator/schema/mirror.json | 30 +++-- 2 files changed, 161 insertions(+), 90 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d021aee9..80e1791e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,15 +1,15 @@ + +from typing import Optional, List, Dict import base64 +import io +import magic import os import pathlib -import urllib.request import urllib.error -from typing import ByteString, Optional, List, Dict -import magic -import base64 - +import urllib.request import yaml -from . import schema +from . import schema, root_logger class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" @@ -20,72 +20,124 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root + self._mount_point = mount_point + + self._logger = root_logger self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] - + [None]).pop(0) - self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - # Will hold a list of all the keys - self.keys = None + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() + if mirror.get('bootstrap', False)] + + # Will hold a list of all the gpg keys (public and private) + self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.CacheValidator.validate(raw) + schema.MirrorsValidator.validate(raw) - mirrors = [mirror for mirror in raw if mirror["enabled"]] + mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise MirrorError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") + mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', + mirrors['cmdline_cache'] = { 'url': cmdline_cache, - 'buildcache': True, + 'description': "Cache configured via command line.", + 'enabled': True, + 'cache': True, 'bootstrap': False, + 'mount_specific': True, } - ) + + # Load the cache as defined by the deprecated 'cache.yaml' file. + mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + caches = [mirror for mirror in mirrors.values() if mirror['cache']] + if len(caches) > 1: + raise MirrorError( + "Mirror config has more than one mirror specified as the build cache destination.\n" + "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" + f"{self._pp_yaml(caches)}") return mirrors + @staticmethod + def _pp_yaml(object): + """Pretty print the given object as yaml.""" + + example_yaml_stream = io.StringIO() + yaml.dump(object, example_yaml_stream, default_flow_style=False) + return example_yaml_stream.getvalue() + + def _load_legacy_cache(self): + """Load the mirror definition from the legacy cache.yaml file.""" + + cache_config_path = self._system_config_root/'cache.yaml' + + if cache_config_path.is_file(): + + with cache_config_path.open('r') as file: + try: + raw = yaml.load(file, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError( + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': f'file://{raw['root']}', + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'buildcache_push': True, + 'mount_specific': True, + 'enabled': True, + 'private_key': raw['key'], + } + + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + + return mirror_cfg + def _check_mirrors(self): """Validate the mirror config entries.""" - for mirror in self.mirrors: + for name, mirror in self.mirrors.items(): url = mirror["url"] if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise MirrorError(f"The mirror path '{path}' is not absolute") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") if not path.is_dir(): - raise MirrorError(f"The mirror path '{path}' is not a directory") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") mirror["url"] = path @@ -98,6 +150,16 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + @property + def keys(self): + """Return the list of public and private key file paths.""" + + if self._keys is None: + raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") + + return self._keys + + def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -110,9 +172,9 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - for m in self.mirrors: - name = m["name"] - url = m["url"] + for name, mirror in self.mirrors.items(): + name = mirror["name"] + url = mirror["url"] raw["mirrors"][name] = { "fetch": {"url": url}, @@ -133,9 +195,9 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): 'trusted': {}, } - for mirror in self.bootstrap_mirrors: - name = mirror['name'] + for name in self.bootstrap_mirrors: bs_mirror_path = config_root/f'bootstrap/{name}' + mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. bootstrap_yaml['sources'].append( { @@ -161,50 +223,53 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + self._keys = [] key_store.mkdir(exist_ok=True) - for mirror in self.mirrors: - if mirror.get("public_key"): - key = mirror["public_key"] + for name, mirror in self.mirrors.items(): + if mirror.get("public_key") is None: + continue - # key will be saved under key_store/mirror_name.gpg + key = mirror["public_key"] - dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") + # key will be saved under key_store/mirror_name.gpg - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - - with open(path, 'rb') as reader: - binary_key = reader.read() - - # convert base64 key to binary - else: - try: - binary_key = base64.b64decode(key) - except ValueError: - raise MirrorError( - f"Key for mirror {mirror["name"]} is not valid. \n" - f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_buffer(binary_key, mime=True) + dest = pathlib.Path(key_store / f"{name}.gpg") + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path - if file_type != "application/x-gnupg-keyring": + if path.exists(): + if not path.is_file(): raise MirrorError( - f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(dest, 'wb') as writer: - writer.write(binary_key) - # update mirror with new path - mirror["public_key"] = dest + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror '{name}' is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(dest, 'wb') as writer: + writer.write(binary_key) + + self._keys.append(dest) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a8be6ab3..5efb7253 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,17 +1,18 @@ { - "type" : "array", - "items": { + "type" : "object", + "additionalProperties": { "type": "object", - "required": ["name", "url"], + "required": ["url"], + "additionalProperties": false, "properties": { - "name": { - "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax." - }, "url": { "type": "string", "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, + "description": { + "type": "string", + "description": "What this mirror is for." + } "enabled": { "type": "boolean", "default": true, @@ -22,19 +23,24 @@ "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, - "buildcache": { + "cache": { "type": "boolean", "default": false, "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages." + "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." }, - "description": { + "private_key": { "type": "string", - "description": "What this mirror is for." + "description": "Private PGP key for signing binary cache packages. (Path only)", + }, + "mount_specific": { + "type": "boolean", + "default": false, + "description": "Use a mount specific buildcache path (specified path + recipe mount point).", } } } -} \ No newline at end of file +} From fb3289d314ecbe2c00aacf0b75f7ea33ba6240a6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:09:44 -0600 Subject: [PATCH 31/98] Now add mount specific paths to certain mirrors. --- stackinator/builder.py | 4 ++-- stackinator/mirror.py | 5 ++++- stackinator/templates/Makefile | 11 +++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f8083997..b6192bb2 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,13 +227,13 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors.mirrors, + gpg_keys=recipe.mirrors.keys, + cache=recipe.mirrors.buildcache, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 80e1791e..cad746c0 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -173,9 +173,12 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} for name, mirror in self.mirrors.items(): - name = mirror["name"] url = mirror["url"] + # Make the mirror path specific to the mount point + if mirror['mount_specific'] and self._mount_point is not None: + url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + raw["mirrors"][name] = { "fetch": {"url": url}, "push": {"url": url}, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 9484e3c7..f0b90ebe 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,19 +33,14 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% endif %} - {% if mirrors %} @echo "Adding mirror gpg keys." - {% for mirror in mirrors | reverse %} - {% if mirror.public_key %} - $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} - {% endif %} + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust {{ key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list - {% endif %} touch mirror-setup compilers: mirror-setup From d47e2f3c6aad37b090d2b91b8b18d4a114b0ce39 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:22:24 -0600 Subject: [PATCH 32/98] updated test mirror format --- stackinator/schema/mirror.json | 6 +++--- unittests/data/systems/mirror-ok/mirrors.yaml | 14 +++++--------- unittests/test_mirrors.py | 18 ++++++++---------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 5efb7253..89770831 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -12,7 +12,7 @@ "description": { "type": "string", "description": "What this mirror is for." - } + }, "enabled": { "type": "boolean", "default": true, @@ -34,12 +34,12 @@ }, "private_key": { "type": "string", - "description": "Private PGP key for signing binary cache packages. (Path only)", + "description": "Private PGP key for signing binary cache packages. (Path only)" }, "mount_specific": { "type": "boolean", "default": false, - "description": "Use a mount specific buildcache path (specified path + recipe mount point).", + "description": "Use a mount specific buildcache path (specified path + recipe mount point)." } } } diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 9edf8d49..e84fc3c3 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,7 @@ -- name: fake-mirror - url: https://google.com -- name: disabled-mirror - url: https://google.com +- url: https://google.com +- url: https://google.com enabled: false -- name: buildcache-mirror - url: https://cache.spack.io/ - buildcache: true -- name: bootstrap-mirror - url: https://mirror.spack.io +- url: https://cache.spack.io/ + cache: true +- url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6283c1a3..bd96dce1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,21 +14,19 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com'} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror_ok" - mirrors = mirror.Mirrors(path) - print(valid_mirrors) - print(mirrors) - assert mirrors == valid_mirrors - assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] - assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + mirrors_obj = mirror.Mirrors(path) + assert mirrors_obj.mirrors == valid_mirrors + assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] + assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] # assert disabled mirror not in mirrors - for mir in mirrors: + for mir in mirrors_obj.mirrors: assert mir["enabled"] # test that cmdline_cache gets added to mirrors? From c8fb6f959cebc1ec12d6a82b5b6f240a1a5cbe59 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:23:31 -0600 Subject: [PATCH 33/98] Fixed build cache enabling via cmdline. --- stackinator/mirror.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index cad746c0..8a91162a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -47,29 +47,30 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.SafeLoader) + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.MirrorsValidator.validate(raw) - - mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + schema.MirrorsValidator.validate(mirrors) else: mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if not existing_mirror: + if cmdline_cache in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", - 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, } + else: + # Enable the specified mirror and set it as the build cache dest + mirror = mirrors[cmdline_cache] + mirror['enabled'] = True + mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() @@ -81,7 +82,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return mirrors + return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): @@ -116,7 +117,6 @@ def _load_legacy_cache(self): 'description': "Buildcache dest loaded from legacy cache.yaml", 'buildcache_push': True, 'mount_specific': True, - 'enabled': True, 'private_key': raw['key'], } From 951aeb4294ca78954695ca13b554a87d08097ab8 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:26:57 -0600 Subject: [PATCH 34/98] Error handling --- stackinator/mirror.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8a91162a..5643f2fc 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -45,12 +45,12 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): - with path.open() as fid: - # load the raw yaml input - mirrors = yaml.load(fid, Loader=yaml.SafeLoader) - - # validate the yaml - schema.MirrorsValidator.validate(mirrors) + try: + with path.open() as fid: + # load the raw yaml input + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + except (OSError, PermissionError) as err: + raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} @@ -62,6 +62,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", + 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, @@ -75,6 +76,15 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + try: + # validate the yaml, including anything we added + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError( + "Mirror config does not comply with schema.\n{err}" + ) + caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: raise MirrorError( @@ -115,7 +125,8 @@ def _load_legacy_cache(self): mirror_cfg = { 'url': f'file://{raw['root']}', 'description': "Buildcache dest loaded from legacy cache.yaml", - 'buildcache_push': True, + 'cache': True, + 'enabled': True, 'mount_specific': True, 'private_key': raw['key'], } From 2ea49e9facf291ca5e71153a23424c8c92938d6d Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:46:38 -0600 Subject: [PATCH 35/98] Adding a unittest. --- stackinator/mirror.py | 21 ++++++++++----------- unittests/test_mirrors.py | 10 +++++++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5643f2fc..f1dbb17b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -54,11 +54,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: else: mirrors = {} + try: + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError("Mirror config does not comply with schema.\n{err}") + # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if cmdline_cache in mirrors: + if cmdline_cache not in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", @@ -74,16 +79,10 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. - mirrors['legacy_cache_cfg'] = self._load_legacy_cache() - + legacy_cache = self._load_legacy_cache() + if legacy_cache is not None: + mirrors['legacy_cache_cfg'] = legacy_cache - try: - # validate the yaml, including anything we added - schema.MirrorsValidator.validate(mirrors) - except ValueError as err: - raise MirrorError( - "Mirror config does not comply with schema.\n{err}" - ) caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: @@ -92,7 +91,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index bd96dce1..6579bb11 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -30,6 +30,14 @@ def test_mirror_init(systems_path, valid_mirrors): assert mir["enabled"] # test that cmdline_cache gets added to mirrors? +def test_command_line_cache(systems_path): + """Check that adding a cache from the command line works.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + + assert len(mirrors.mirrors) == 4 + + def test_create_spack_mirrors_yaml(systems_path): pass @@ -37,4 +45,4 @@ def test_create_bootstrap_configs(): pass def test_key_setup(): - pass \ No newline at end of file + pass From c668b5b375a8c40a2f4f9dad7774fa24bc551910 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:45:58 -0600 Subject: [PATCH 36/98] updated yaml formatting --- unittests/data/systems/mirror-ok/mirrors.yaml | 12 ++++++++---- unittests/test_mirrors.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index e84fc3c3..f0030a45 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,7 +1,11 @@ -- url: https://google.com -- url: https://google.com +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com enabled: false -- url: https://cache.spack.io/ +buildcache-mirror: + url: https://cache.spack.io/ cache: true -- url: https://mirror.spack.io +bootstrap-mirror: + url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6579bb11..c1d46600 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,8 +20,10 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): - path = systems_path / "mirror_ok" + path = systems_path / "mirror-ok" + print(path) mirrors_obj = mirror.Mirrors(path) + print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From 45b1331396978c871d4b47744b713029ad939644 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:56:15 -0600 Subject: [PATCH 37/98] Fixed error message. --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-basic/mirrors.yaml | 8 ++++++++ unittests/test_mirrors.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index f1dbb17b..d9e1e803 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = [mirror for mirror in mirrors.values() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml new file mode 100644 index 00000000..ee93f2c7 --- /dev/null +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -0,0 +1,8 @@ +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com + enabled: false +bootstrap-mirror: + url: https://mirror.spack.io + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c1d46600..acd360f1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -35,9 +35,9 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + mirrors = mirror.Mirrors(systems_path/'mirror-basic', cmdline_cache=systems_path.as_posix()) - assert len(mirrors.mirrors) == 4 + assert len(mirrors.mirrors) == 3 def test_create_spack_mirrors_yaml(systems_path): From 533a6061d7b26f29f5a7f106d5debf57e4cfeaaa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:57:31 -0600 Subject: [PATCH 38/98] debugging --- unittests/test_mirrors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index acd360f1..b374aca4 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -21,9 +21,9 @@ def valid_mirrors(systems_path): def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - print(path) + #print(path) mirrors_obj = mirror.Mirrors(path) - print(mirrors_obj.mirrors.items()) + #print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From 34d0c6574cad38a1265e91077a406b86b5fc9ca9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:02:18 -0600 Subject: [PATCH 39/98] Minor fix. --- stackinator/mirror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d9e1e803..88168476 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -50,14 +50,14 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # load the raw yaml input mirrors = yaml.load(fid, Loader=yaml.SafeLoader) except (OSError, PermissionError) as err: - raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") + raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} try: schema.MirrorsValidator.validate(mirrors) except ValueError as err: - raise MirrorError("Mirror config does not comply with schema.\n{err}") + raise MirrorError(f"Mirror config does not comply with schema.\n{err}") # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" From f989631e0675ea8b4aeacc8d50b5945f5d45ab96 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:06:49 -0600 Subject: [PATCH 40/98] Fixing urls. --- unittests/data/systems/mirror-basic/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml index ee93f2c7..2667cbc3 100644 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -1,8 +1,8 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: /tmp/ enabled: false bootstrap-mirror: - url: https://mirror.spack.io + url: https://github.com bootstrap: true From 7822fb764859b93a6ace31245a2e92f00a270d1d Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:07:37 -0600 Subject: [PATCH 41/98] fixed url for testing --- unittests/data/systems/mirror-ok/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index f0030a45..06f2e329 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,10 +1,10 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: https://github.com enabled: false buildcache-mirror: - url: https://cache.spack.io/ + url: https://mirror.spack.io cache: true bootstrap-mirror: url: https://mirror.spack.io From 8c814aabbd374e1a5680bb76791d6abd595917a9 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:34:52 -0600 Subject: [PATCH 42/98] validated mirror tests --- unittests/test_mirrors.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index b374aca4..fdc4e7b3 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,23 +14,21 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - #print(path) mirrors_obj = mirror.Mirrors(path) - #print(mirrors_obj.mirrors.items()) + assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] - assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] - # assert disabled mirror not in mirrors + assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] + assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) + for mir in mirrors_obj.mirrors: - assert mir["enabled"] - # test that cmdline_cache gets added to mirrors? + assert mirrors_obj.mirrors[mir].get('enabled') def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 5ab3279581809771782904b8fbc6ab4884bb91d0 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:37:02 -0600 Subject: [PATCH 43/98] added test description --- unittests/test_mirrors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index fdc4e7b3..ecc79082 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,6 +20,7 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): + """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) From 5999ba4a4b6fb95f99e90941c47424fa1b10e858 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 10:20:00 -0600 Subject: [PATCH 44/98] Adding unit tests. --- stackinator/mirror.py | 105 ++++++++---------- .../data/systems/mirror-basic/cache.yaml | 2 + unittests/data/test-gpg-priv.asc | Bin 0 -> 4924 bytes unittests/data/test-gpg-pub.asc | Bin 0 -> 2250 bytes unittests/test_mirrors.py | 17 ++- 5 files changed, 63 insertions(+), 61 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/cache.yaml create mode 100644 unittests/data/test-gpg-priv.asc create mode 100644 unittests/data/test-gpg-pub.asc diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 88168476..a5191865 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -19,8 +19,9 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' + CMDLINE_CACHE = 'cmdline_cache' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -32,16 +33,22 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + # Always use the cache given on the command line + if self.CMDLINE_CACHE in self.mirrors: + self.build_cache_mirror = self.CMDLINE_CACHE + else: + # Otherwise, grab the configured cache (or None) + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get('bootstrap', False)] # Will hold a list of all the gpg keys (public and private) self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: + def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): @@ -59,38 +66,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if cmdline_cache not in mirrors: - mirrors['cmdline_cache'] = { - 'url': cmdline_cache, - 'description': "Cache configured via command line.", - 'enabled': True, - 'cache': True, - 'bootstrap': False, - 'mount_specific': True, - } - else: - # Enable the specified mirror and set it as the build cache dest - mirror = mirrors[cmdline_cache] - mirror['enabled'] = True - mirror['cache'] = True - - # Load the cache as defined by the deprecated 'cache.yaml' file. - legacy_cache = self._load_legacy_cache() - if legacy_cache is not None: - mirrors['legacy_cache_cfg'] = legacy_cache - - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") + # Load the cache as defined by the deprecated 'cache.yaml' file. + if cmdline_cache is not None: + mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod @@ -101,40 +86,42 @@ def _pp_yaml(object): yaml.dump(object, example_yaml_stream, default_flow_style=False) return example_yaml_stream.getvalue() - def _load_legacy_cache(self): - """Load the mirror definition from the legacy cache.yaml file.""" - - cache_config_path = self._system_config_root/'cache.yaml' - - if cache_config_path.is_file(): - - with cache_config_path.open('r') as file: - try: - raw = yaml.load(file, Loader=yaml.SafeLoader) - except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: + """Load the mirror definition from the legacy 'cache.yaml' file.""" + if not cache_config_path.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cache_config_path}' " + f"does not exist.") + + with cache_config_path.open('r') as file: try: - schema.CacheValidator.validate(raw) + raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") - - mirror_cfg = { - 'url': f'file://{raw['root']}', - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'mount_specific': True, - 'private_key': raw['key'], - } + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': raw['root'], + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'cache': True, + 'enabled': True, + 'bootstrap': False, + 'mount_specific': True, + 'private_key': raw['key'], + } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" - "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") - return mirror_cfg + return mirror_cfg def _check_mirrors(self): """Validate the mirror config entries.""" diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-basic/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/test-gpg-priv.asc b/unittests/data/test-gpg-priv.asc new file mode 100644 index 0000000000000000000000000000000000000000..eaa2dc19eb7ecaff70e1c1cf53e66825a6996425 GIT binary patch literal 4924 zcmajhWgr{?pull=>g37OoSW`;YI?f6CZ=O@;>4L|Om}R0dYHDc>0#<{dYb8&Ht)R; zFTTC+|L^}_ChiL?mv-8F00~C=s0g;qrC~&pac&Qi3xB%d4BI$n%GzhUu9pzu3NJ9U z(wLyd`_8QFj#lg=2y|QSH@51OU|Bq>B#$00$+%r zsLiq6J>_j9-lyM&p)&t0kE?4>7U}w1E*wsLi39&OcFVam*(SCy^ybA$*Q`h{w^0TC z!rsU^&HfAC5pS0EWko`NRF7*w^Bq*O8_SplGc$}6j|P zQl|HmHq+0TRjw!4wqqv(IM60Z=_RVF3n>okAA5WFOLuO#nB*Op6gAjt0&uk=Joi_wxL?hDEhOUHN!?AnR3oy*7vLa|wdQ6Scs(lh@=Vd&RRqtU6n~O0yVEaOzXjbe^!#e?_RFZ<cQ=Rt3jfU6%{Z1%t{^H`G^<^osKj}Nb96uAA&pU%gI~2y z$1WI_C|NtaqF|&dcjZKW2G8b))P>vO3p_Za%JvfxCC}$H_&%jEH(IyeNGOO>X*H}* zFp<4kQ*-{|>n2RLh1s$JdXb4MuS;qvI3q9(!T%d|yKcZ!O^dxT5vgS{3UFrf&-LE# z5>C4X<-Yfy2r+5GCDVIDYi3D@#@LEf7x_g&(c@=@$(y3CM<(=Cz;KMT`xtR3r$XrM zB!c+h=;p^d2v^Q}RbxM;2C(A3U8@8|TV9H|M$@ZT`zK_t*iIJgl`b3ZEAGrn2FKZnR|88d<@%6_gK+`$* z$l+TjqN&}|HF9bl8F|83W$XKEfm8_z7uFT_+FpNkQqCI zcqfSw!@pOO@IeCXXgJ=_gTH1MpQSz2i{la(A5W9b)nPQ3r=&azj7d0E95wK}33;44 zC(@buEB6ZvfrKPqj_jJce!foh=)`Mi;i^Dnel)Sgk{kNKs!bNOswyT%V>4X)vb89$ zmfgCnl$e16H3lPzmkkG`E zFLZ_uzJ5>*hajl*Yfmo+x!0Zn&JI2fe6Kt`p2q|Au*d=U7*Y^stRj#OJcf|#{Lh}n zAzXp*^`sP495(R39t@3^YtInHX?_x5qO8muaA6_ogq z-1H3ORrJ^W(LI_HVWspRNYR>3!x?+A&e0bvpr^c%RL%efeT}f`%`d3~?L@DQ*Vt#) zSY%ferQeu)XIfKNs@}}#kO_$Jl#nExRiE#sZ+3^uuaYE8S0i<%THWv;#lkUB%(ubW zaYXB#WN&SGLiy|-N*)~K2JP7%A%}=fvBdcf6(k@F1k8V@UusKsW6`~6?CQQZw8FEC z_Xex5oZv(A(vr(|pG&a>y8z$qXH4d8)q<5_6My@(Jvv!ZTcv&M z1`b+XRKm@y*-`t`!D(b%8xhlZyC6y%8QJwS^7=HL!FoZKOQ6M?juy&R_`h0`!L zVo8plb?(Tz_3sGYpDEB`w~I)&N#k@6N6Wri&5WroFM2!BJ4ix_9tseQAbO3o8p|Uwrm2xJLj;&&F6m^X{P{yh)KXs3wwiykQsL z87StZ{N8ihIgj9*sa_z-RF6IkkEwQID@#PHKLUjhKjGuzuNV@1xLAU>v1Oe|@*+Eu z(z}OYEhRTQOMGeT)B+~`WC4LDoPNB}XuHP?MMgDcw&R_>ZRM%ml8i5U+qr_)C8Yi| z2f$1SO0wv|n>c3-IU=r@QF||geHAtJeZicK%GRnNwYd}5&CP$`?%e94Qn)i|4`o%$dlxmL(VCSd8GZ8jgvyw6jh zAQ!O(zwDwBmi0gtrksyd+6U`w|32=VC9@rAlA0sUWWl2ckEko1J;JNV#Qod6XW_~W zdcmt*MAXXz=0Zn|QJ$iO^5X^;uMZWDqeK(;&0rDnWb=|9IDJc8ach-#G;Cp!)G``Q zTSCmJj5O&FJ`*a9_f3G#-{ZO!9(|^p?g7vpGe;pN#rk~KycaR6W%Me)E>=I`I)%A4 zr7dL{yQO|pAE<2o=Ttc2wkL)?;kVuq?U;ly8aq6GGO@OqX0^BWKXMcE8he|`Qc(I7 zwG<4EjFC%pk?D1!x8h< z&S5>fS5RHWf+8WAVc217uv6VK#oRpf{RL@#%uP6bj2_0X=;%@V^m+0f<0-hU`z$rr z5;*vGgachu-Ko>?l^`){;J((-{ZC^tdos6tF-nSx86g0ZnAjA@69ceHxPxRb*6YAVfxDd`4l-`fb`}B4o5m>r_@}=t65x(XKmoS5u#@^jF+e z)>n1m83 z(MMTi-7WDZW+bvQx)HkayO9IuNJZ3ccuUTDMSf=syJrtpgyphx=f1!D!0gK(^s{`Z zo4pcWkgoqqK=G2{zQiZO1eMmJDsX{1d8qxWSCY~hX=HvJh4<+Pn`o*=?Rrj$u~7N? z8s*k|2Rnc&cHx(Y$j^2xy(F-}4V5g-Zyub^-53c8lq`PjF8xdJSIcjxcX4%bLuA)z zrWdr>Yk#pRNjzdG@jewWvyQBijb(d9jw4&QP7JJMzh`SH7@itP&8rzXkl6eNZw;#(}9hlQVk%X2VaqPW1Q^+^L+1xKYoefzi`*pKD0>USw z8Kt13`w8$fwAQJZV$%qKZeax^U*oH_eALymt={X>*Y9=fB{f3UF?_lEB|z3^XhmiS z9gS%p=*c0t+#h@!vAf|w>5>0>*>hergIi?=ebRB%Jv)9UPm$-Op%G zbCv#2yv})}v3=b(zT$ORmK<)gxNfVV-Qo}lPw1rZP_}`;d2FX*`qO4}Yq0WBjj8a* znKD?YsK~mEU*>kaZrIrga&{8BK~iHwzeJ0QI{r-flXg6aDZxTSxP~ci198cgyWE+@spgp zcIL70HcdIu60_IfVC&3?yO+(xZ5B5kD27G-pE!g5KR6@&zXCI10zX*_*1P1GlMxc` z%)?R=S07x@X1-m-&$Ls}mDr_yHYTn^x0Au)C^OITfQf-9;BZL%?G@?lct-tV@IbLTNfTbCO+!d8^X)gQ+>Q z#Lu~@3184=2Ujjy5t#O2!)*1EH6$jV@0#)!=HB*j({^71ls&())~&Qy7~rT18YWCp zs#=itG`?Ojr(te6_%{=a%B)o>B+5VaSlS{dK5l9l_}Vna(t0uKw1 z_(B$aekP4YwSd{abSfNJ;;;n~PMSY3^QHYs91id)+1}=S<~kWXprhLrK}E7~V#fIg z58^xVG2h@x6p~;-3$z|+0K;$vqq&`(8tCX84 zopfONbdlh~doqjgsF%flDZQ`ZFG@Y)tr^&cbh_5SOC5~r901&0XgdQG&nen4h#)LQ mZ>7vztr91~Oq1#)>Ed{1^SU?%yx#0qXiGx|!qiao5&sW-Z9mQc literal 0 HcmV?d00001 diff --git a/unittests/data/test-gpg-pub.asc b/unittests/data/test-gpg-pub.asc new file mode 100644 index 0000000000000000000000000000000000000000..aa72b0281f39bd4ad4d0281742b9133f19e84c6a GIT binary patch literal 2250 zcmV;*2sQVa0u2OdxElKb5CD_QGzC5MRDz9GnZJK33evCf1y+X9UvlNGzzlk7IGj{n zOV1h!!68L#*h9k$cx&9mR`YBM1Y`iIP7^GpgRmCGb0J4Jm4p-S4@~@{L_B=$sif8l zp%P+j!xq+yV=FoV1Nqpa0oxriYr{=<+#z>ny;aTf4|~0-Zz8Hnu?{m>wKqL3j??fl z<%FL7OsTwod^ACtFi#WMhm%0?XoZjydMC(cgMUkUK(&wsqSzcG4>V<$UvpIpSbqPT z;xFrpnx-(?c+JH<-0XElO&weapA%^0u&yeLenM?y9Ws+8e~R^JVXI4tVvkyg+HPVT`pMz?z6WtKB@@w-%_Sm-P?kH?ur*C4V5os?AFvn`K9rv7{j}m6TSD?Zrjb`lrJ3IY3SN6hh<=3)XA< z%oP87l$5JNg$gvLxG)?62YWmXry(I_mpt=?NVlFtG^u4D)gGwa6j-WRU~#vpTvhsr zyD{m@c&p#C4yRNi^aw=l2hSZj>H7p%!Dc^AM$FNuiB-AK#eeQ#sH8N1QTF3pKc9)m zC_potxh89e74ZNO0RREC8&qX;bRbJ*c_2J)Z*XNmZf|#JWpZUMV{dIfi2_js69EbU zI~E}Xo(NAkh72n5>gZeSI6@3=&^s511p;Zf8v6np0|g5S2nPZJA_4{#3JC}c0t6NU z0|5da2Lcy>0162ZI6@3=&^s517TOQ~Fzm@df0V8}8rvUt>$u+J@YC^+3ooI-cT{RQ z102Z9UjpyK4q|Ra*hD@qnCvS~ZPk$V?axJ8 ztf8|Lll35Ym}p6t`(%juOC?8ZrWFc+VNktuf_QcHf)C@q7aJls`BTP zwrdLgHh=@vA^Cfkhz;So5`keXeJ*1BqWon*#$+b_7VLWCHjLM~ zM}Gm92m+gnQlnuK_*=Z%R%&eM$K5RF33UXWr3;_4+tF;uVhwF^f?F@o!gD={h1(Qr zCd_uB*sgyb46eGl%MlUNy911@)MbHs#6KzvkU)p8@eM&fr@;=X%?Gli#6B8)TVk6dPBa-Cs09bef$DloNQ8yX-cxdIIYX}B8u0T2Mw!yyGkB#(*JXt|d8cDZ}$)0|HyVuFug zk|}I}*gs?5m_Jx?uSXeS=gmkgAA*Fv(G#sd)Q{M=qwcXal67A_20}uX`(MLCYJ9yEq}RjIS))HzE7-iTnb=B{}%K(-d*l>yuSVV5o>*K+@OF)9vQFF)E<1YgCe13+>+lAm9f zcEc-}+gav35UcG>k1_51ZG`qu=nVFQjOqEBBY;I6QG4FI4H(?TS~Be(gl{#SLCsTJ zZtOztgf)!mSt5do64yFy2E*Gbh~KCB876*%7(Gj8gZeSI6@3= z&^s511p;Zf8v6np3;+rV5I8~%ZqPdyh*uvE|6DPyX_#6P3=HPiB9M&U{A%&jzVnIG zxD*?rVv#jON;)90f_FqOb9eHW^ne8DNQSMgF?Lni39?HvFhVm?qh_&)hXdSLsu_YO zYDlSPq1fYWHQBHh56tFl8K(Rt+s1Y>l27>7I=h{k-aKT+k(kvEJ}VduMjDrUr`lZ$ zSi=P(VNs7uInX}mSS_8I`M)a~yz~J@Z@ne0-L+d&21qefiqREFTM@spZrxfLA+_uN z(}&cRtVf*V%#EHaqgX6R{A9prF3BhAfm~5?49i{eA;r#jm(6#4<@v_ zLhD=xCV33dTKpj{lIo1ecXFcT*1vGtGISxuva>l520ckLGgv#2gy^}{Y9rh-V$@n z`0UYwH<4mp;FwQ8)cZ;rih#Do$OI1i@+f9@O5QWvKsJ8Lx?Jb0zjo-x^u5?g?T zUZHf5z-~B3qiu;_9VW+5yWbDnxgJU=0SB2NxE&w~XN6RG3x1GM-K23zrv`#qOc(Es Y^X}8*_o0vv;j?R Date: Mon, 16 Mar 2026 10:44:23 -0600 Subject: [PATCH 45/98] Unit test tweaking. --- stackinator/mirror.py | 6 +++++- unittests/data/systems/mirror-ok/cache.yaml | 2 ++ unittests/data/systems/mirror-ok/mirrors.yaml | 3 ++- unittests/test_mirrors.py | 13 ++++--------- 4 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 unittests/data/systems/mirror-ok/cache.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a5191865..a423833c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -66,11 +66,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} + caches = [name for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" f"{self._pp_yaml(caches)}") + elif caches: + cache = mirrors[caches[0]] + if not cache.get('private_key'): + raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: diff --git a/unittests/data/systems/mirror-ok/cache.yaml b/unittests/data/systems/mirror-ok/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 06f2e329..21e1e3b4 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -5,7 +5,8 @@ disabled-mirror: enabled: false buildcache-mirror: url: https://mirror.spack.io + private_key: '../test-gpg-priv.asc' cache: true bootstrap-mirror: url: https://mirror.spack.io - bootstrap: true \ No newline at end of file + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 981fe744..c5bede3e 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -34,10 +34,11 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-basic', - cmdline_cache=systems_path/'mirror-basic/cache.yaml') + mirrors = mirror.Mirrors(systems_path/'mirror-ok', + cmdline_cache=systems_path/'mirror-ok/cache.yaml') - assert len(mirrors.mirrors) == 3 + assert len(mirrors.mirrors) == 4 + # This should always be the build cache even though one is already defined. assert mirrors.build_cache_mirror == 'cmdline_cache' cache_mirror = mirrors.mirrors['cmdline_cache'] assert cache_mirror['url'] == '/tmp/foo' @@ -46,12 +47,6 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_multi_buildcache(systems_path): - """Make sure we throw appropriate errors when there's more than one build cache defined.""" - - - - def test_create_spack_mirrors_yaml(systems_path): pass From e19b868cb710f0d838ea5f62746b177438b367ca Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 11:04:25 -0600 Subject: [PATCH 46/98] Added unittests for key setup --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-ok/mirrors.yaml | 43 +++++++++++++++++++ unittests/test_mirrors.py | 17 +++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a423833c..7296389e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -267,7 +267,7 @@ def _key_setup(self, key_store: pathlib.Path): file_type = magic.from_buffer(binary_key, mime=True) print("magic type:" , file_type) - if file_type != "application/x-gnupg-keyring": + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 21e1e3b4..96fc72f5 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,54 @@ fake-mirror: url: https://github.com + public_key: ../../test-gpg-pub.asc disabled-mirror: url: https://github.com enabled: false buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' + public_key: | + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + cache: true bootstrap-mirror: url: https://mirror.spack.io diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c5bede3e..3b068c90 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -53,5 +53,18 @@ def test_create_spack_mirrors_yaml(systems_path): def test_create_bootstrap_configs(): pass -def test_key_setup(): - pass +def test_key_setup(systems_path, tmp_path): + """Check that public keys are set up properly.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok') + + mirrors._key_setup(tmp_path) + + key_files = list(tmp_path.iterdir()) + assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + # The two files should be identical in content + key_file_data = [] + for key_file in key_files: + with key_file.open('rb') as file: + key_file_data.append(file.read()) + assert key_file_data[0] == key_file_data[1] From 86047caf3d41e09b5db4902b69f2320e1d7c675e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 12:16:45 -0600 Subject: [PATCH 47/98] Added test for bad keys. --- stackinator/mirror.py | 11 +++-------- unittests/data/systems/mirror-bad-key/mirrors.yaml | 4 ++-- .../data/systems/mirror-bad-keypath/mirrors.yaml | 4 ++-- unittests/data/systems/mirror-basic/cache.yaml | 2 -- unittests/data/systems/mirror-basic/mirrors.yaml | 8 -------- unittests/test_mirrors.py | 14 ++++++++++++++ 6 files changed, 21 insertions(+), 22 deletions(-) delete mode 100644 unittests/data/systems/mirror-basic/cache.yaml delete mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 7296389e..317deead 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -246,12 +246,7 @@ def _key_setup(self, key_store: pathlib.Path): #try prepending system config path path = self._system_config_root/path - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - + if path.is_file(): with open(path, 'rb') as reader: binary_key = reader.read() @@ -261,15 +256,15 @@ def _key_setup(self, key_store: pathlib.Path): binary_key = base64.b64decode(key) except ValueError: raise MirrorError( - f"Key for mirror '{name}' is not valid. \n" + f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" f"Check the key listed in mirrors.yaml in system config.") # copy key to new destination in key store diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml index ed27df7a..d5154dd3 100644 --- a/unittests/data/systems/mirror-bad-key/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key +bad-key: url: https://mirror.spack.io - public_key: /bad_key.gpg \ No newline at end of file + public_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml index e671c45a..3433e04a 100644 --- a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key-path +bad-key-path: url: https://mirror.spack.io - public_key: /path/doesnt/exist \ No newline at end of file + public_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml deleted file mode 100644 index ad7de37c..00000000 --- a/unittests/data/systems/mirror-basic/cache.yaml +++ /dev/null @@ -1,2 +0,0 @@ -root: /tmp/foo -key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml deleted file mode 100644 index 2667cbc3..00000000 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ /dev/null @@ -1,8 +0,0 @@ -fake-mirror: - url: https://github.com -disabled-mirror: - url: /tmp/ - enabled: false -bootstrap-mirror: - url: https://github.com - bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 3b068c90..5b4a0a89 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -68,3 +68,17 @@ def test_key_setup(systems_path, tmp_path): with key_file.open('rb') as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] + +@pytest.mark.parametrize("system_name", [ + 'mirror-bad-key', + 'mirror-bad-keypath', +]) +def test_key_setup_bad_key(tmp_path, systems_path, system_name): + """asdfasdf""" + + mirrors = mirror.Mirrors(systems_path/system_name) + with pytest.raises(mirror.MirrorError): + mirrors._key_setup(tmp_path) + + + From 3d840d05c9665c6d02d9a424a35a3940c605aeaf Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 11:17:52 -0600 Subject: [PATCH 48/98] added more mirror tests --- stackinator/mirror.py | 4 +-- unittests/test_mirrors.py | 61 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 317deead..502943ae 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -206,14 +206,14 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): bootstrap_yaml['sources'].append( { 'name': name, - 'metadata': bs_mirror_path, + 'metadata': str(bs_mirror_path), } ) # And trust each one bootstrap_yaml['trusted'][name] = True # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True) + bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { 'type': 'install', 'info': mirror['url'], diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 5b4a0a89..8e189228 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -48,10 +48,65 @@ def test_command_line_cache(systems_path): assert cache_mirror['mount_specific'] def test_create_spack_mirrors_yaml(systems_path): - pass + """Check that the mirrors.yaml passed to spack is correct""" + + valid_spack_yaml = { + "mirrors": { + "fake-mirror": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "buildcache-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + "bootstrap-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + } + } + } + + dest = systems_path / "mirror-ok" / "test_output.yaml" + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + assert data == valid_spack_yaml + +def test_create_bootstrap_configs(systems_path): + """Check that spack bootstrap configs are generated correctly""" + + valid_yaml = { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + } + ], + "trusted": { + "bootstrap-mirror": True + }, + } + valid_metadata = { + "type": "install", + "info": "https://mirror.spack.io", + } + + path = systems_path / "mirror-ok" + bs_mirror_path = path / "bootstrap/bootstrap-mirror" + mirrors_obj = mirror.Mirrors(path) + mirrors_obj._create_bootstrap_configs(path) + + with (path/'bootstrap.yaml').open() as f: + bs_data = yaml.safe_load(f) + assert bs_data == valid_yaml -def test_create_bootstrap_configs(): - pass + with (bs_mirror_path/'metadata.yaml').open() as f: + metadata = yaml.safe_load(f) + assert metadata == valid_metadata def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" From 0a25459a7801411ad79eed74fe6e9e6120d180c7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:39:16 -0600 Subject: [PATCH 49/98] added test for bad urls --- unittests/data/systems/mirror-bad-url/mirrors.yaml | 4 ++-- unittests/test_mirrors.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 522c2326..8ffce331 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,2 +1,2 @@ -- name: bad-url - url: google.com \ No newline at end of file +bad-url: + url: https://www.testsite.io/services \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 8e189228..191e7088 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -31,6 +31,14 @@ def test_mirror_init(systems_path, valid_mirrors): for mir in mirrors_obj.mirrors: assert mirrors_obj.mirrors[mir].get('enabled') +def test_mirror_init_bad_url(systems_path): + """Check that MirrorError is raised for a bad url.""" + + path = systems_path / "mirror-bad-url" + + with pytest.raises(mirror.MirrorError): + mirrors_obj = mirror.Mirrors(path) + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 35591bc5cffbad659d9b59217953db74e3f11ab5 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:54:48 -0600 Subject: [PATCH 50/98] added requirements.txt --- requirements.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d0de0ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +attrs==25.4.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +packaging==26.0 +pluggy==1.6.0 +pygments==2.19.2 +pytest==9.0.2 +python-magic==0.4.27 +pyyaml==6.0.3 +referencing==0.37.0 +rpds-py==0.30.0 From 431cc9d6d4f5f440d4c140919af963b87952af2e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:02:02 -0600 Subject: [PATCH 51/98] Fixed unittest. --- unittests/data/systems/mirror-ok/mirrors.yaml | 82 +++++++++---------- unittests/test_mirrors.py | 60 +++++++++----- 2 files changed, 81 insertions(+), 61 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 96fc72f5..0a1b8434 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -7,47 +7,47 @@ disabled-mirror: buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' - public_key: | - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" cache: true bootstrap-mirror: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 191e7088..dbc2a97d 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -1,5 +1,6 @@ -import pytest +import base64 import pathlib +import pytest import stackinator.mirror as mirror import yaml @@ -11,19 +12,38 @@ def test_path(): def systems_path(test_path): return test_path / "data" / "systems" -@pytest.fixture -def valid_mirrors(systems_path): - mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} - return mirrors - -def test_mirror_init(systems_path, valid_mirrors): +def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) + valid_mirrors = { + "fake-mirror": { + 'url': 'https://github.com', + 'enabled': True, + 'bootstrap': False, + 'cache': False, + 'public_key': '../../test-gpg-pub.asc', + 'mount_specific': False}, + "buildcache-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': False, + 'cache': True, + 'private_key': '../test-gpg-priv.asc', + 'mount_specific': False}, + "bootstrap-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': True, + 'cache': False, + 'mount_specific': False} + } + + with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors['buildcache-mirror']['public_key'] = key + assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) @@ -55,7 +75,7 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_create_spack_mirrors_yaml(systems_path): +def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" valid_spack_yaml = { @@ -75,7 +95,7 @@ def test_create_spack_mirrors_yaml(systems_path): } } - dest = systems_path / "mirror-ok" / "test_output.yaml" + dest = tmp_path / "test_output.yaml" mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_spack_mirrors_yaml(dest) @@ -84,14 +104,14 @@ def test_create_spack_mirrors_yaml(systems_path): assert data == valid_spack_yaml -def test_create_bootstrap_configs(systems_path): +def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { "sources": [ { "name": "bootstrap-mirror", - "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], "trusted": { @@ -103,16 +123,16 @@ def test_create_bootstrap_configs(systems_path): "info": "https://mirror.spack.io", } - path = systems_path / "mirror-ok" - bs_mirror_path = path / "bootstrap/bootstrap-mirror" - mirrors_obj = mirror.Mirrors(path) - mirrors_obj._create_bootstrap_configs(path) + mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj._create_bootstrap_configs(tmp_path) - with (path/'bootstrap.yaml').open() as f: + with (tmp_path/'bootstrap.yaml').open() as f: bs_data = yaml.safe_load(f) + print(bs_data) + print(valid_yaml) assert bs_data == valid_yaml - with (bs_mirror_path/'metadata.yaml').open() as f: + with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata From 0c59ee948344baa94f62c57ff226b319998f62e3 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:06:51 -0600 Subject: [PATCH 52/98] Added one more unittest. --- unittests/test_mirrors.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index dbc2a97d..72f2190c 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -57,7 +57,17 @@ def test_mirror_init_bad_url(systems_path): path = systems_path / "mirror-bad-url" with pytest.raises(mirror.MirrorError): - mirrors_obj = mirror.Mirrors(path) + mirror.Mirrors(path) + +def test_setup_configs(tmp_path, systems_path): + """Test general config setup.""" + + mir = mirror.Mirrors(systems_path/'mirror-ok') + mir.setup_configs(tmp_path) + + assert (tmp_path/'mirrors.yaml').is_file() + assert (tmp_path/'bootstrap').is_dir() + assert (tmp_path/'key_store').is_dir() def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From d9cdfc4da5451a25d35450e37134e6ce7f18778b Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:13:05 -0600 Subject: [PATCH 53/98] Linting --- stackinator/builder.py | 2 +- stackinator/main.py | 34 +++++++--- stackinator/mirror.py | 137 ++++++++++++++++++++------------------ stackinator/recipe.py | 7 +- unittests/test_mirrors.py | 124 +++++++++++++++++++--------------- 5 files changed, 168 insertions(+), 136 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b6192bb2..ca0f8753 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util, mirror +from . import VERSION, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): diff --git a/stackinator/main.py b/stackinator/main.py index 6f30ddfc..0d0b9bd8 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,19 +81,31 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str, - help="Where to set up the stackinator build directory. " - "('/tmp' is not allowed, use '/var/tmp'") + parser.add_argument( + "-b", + "--build", + required=True, + type=str, + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + ) parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str, - help="Name of (and/or path to) the Stackinator recipe.") - parser.add_argument("-s", "--system", required=True, type=str, - help="Name of (and/or path to) the Stackinator system configuration.") + parser.add_argument( + "-r", "--recipe", required=True, type=str, help="Name of (and/or path to) the Stackinator recipe." + ) + parser.add_argument( + "-s", "--system", required=True, type=str, help="Name of (and/or path to) the Stackinator system configuration." + ) parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str, - help="The mount point where the environment will be located.") - parser.add_argument("-c", "--cache", required=False, type=str, - help="Buildcache location or name (from system config's mirrors.yaml).") + parser.add_argument( + "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." + ) + parser.add_argument( + "-c", + "--cache", + required=False, + type=str, + help="Buildcache location or name (from system config's mirrors.yaml).", + ) parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 502943ae..e6ba268b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,4 +1,3 @@ - from typing import Optional, List, Dict import base64 import io @@ -11,18 +10,24 @@ from . import schema, root_logger + class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" + class Mirrors: """Manage the definition of mirrors in a recipe.""" - KEY_STORE_DIR = 'key_store' - MIRRORS_YAML = 'mirrors.yaml' - CMDLINE_CACHE = 'cmdline_cache' + KEY_STORE_DIR = "key_store" + MIRRORS_YAML = "mirrors.yaml" + CMDLINE_CACHE = "cmdline_cache" - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, - mount_point: Optional[pathlib.Path] = None): + def __init__( + self, + system_config_root: pathlib.Path, + cmdline_cache: Optional[pathlib.Path] = None, + mount_point: Optional[pathlib.Path] = None, + ): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root @@ -32,25 +37,24 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pat self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - + # Always use the cache given on the command line if self.CMDLINE_CACHE in self.mirrors: self.build_cache_mirror = self.CMDLINE_CACHE else: # Otherwise, grab the configured cache (or None) - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + self.build_cache_mirror: Optional[str] = ( + [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] + ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() - if mirror.get('bootstrap', False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] + self._keys: Optional[List[pathlib.Path]] = [] def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" - path = self._system_config_root/"mirrors.yaml" + path = self._system_config_root / "mirrors.yaml" if path.exists(): try: with path.open() as fid: @@ -66,14 +70,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = [name for name, mirror in mirrors.items() if mirror['cache']] + caches = [name for name, mirror in mirrors.items() if mirror["cache"]] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - f"{self._pp_yaml(caches)}") + f"{self._pp_yaml(caches)}" + ) elif caches: cache = mirrors[caches[0]] - if not cache.get('private_key'): + if not cache.get("private_key"): raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. @@ -82,7 +87,7 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} - @staticmethod + @staticmethod def _pp_yaml(object): """Pretty print the given object as yaml.""" @@ -95,35 +100,35 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: if not cache_config_path.is_file(): raise MirrorError( - f"Binary cache configuration path given on the command line '{cache_config_path}' " - f"does not exist.") - - with cache_config_path.open('r') as file: + f"Binary cache configuration path given on the command line '{cache_config_path}' does not exist." + ) + + with cache_config_path.open("r") as file: try: raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + raise MirrorError(f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") try: schema.CacheValidator.validate(raw) except ValueError as err: - raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + raise MirrorError(f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") mirror_cfg = { - 'url': raw['root'], - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'bootstrap': False, - 'mount_specific': True, - 'private_key': raw['key'], + "url": raw["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "cache": True, + "enabled": True, + "bootstrap": False, + "mount_specific": True, + "private_key": raw["key"], } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}" + ) return mirror_cfg @@ -144,12 +149,13 @@ def _check_mirrors(self): elif url.startswith("https://"): try: - request = urllib.request.Request(url, method='HEAD') + request = urllib.request.Request(url, method="HEAD") urllib.request.urlopen(request) except urllib.error.URLError as e: raise MirrorError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + ) @property def keys(self): @@ -160,12 +166,11 @@ def keys(self): return self._keys - def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" - self._key_setup(config_root/self.KEY_STORE_DIR) - self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._key_setup(config_root / self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root / self.MIRRORS_YAML) self._create_bootstrap_configs(config_root) def _create_spack_mirrors_yaml(self, dest: pathlib.Path): @@ -177,8 +182,8 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): url = mirror["url"] # Make the mirror path specific to the mount point - if mirror['mount_specific'] and self._mount_point is not None: - url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { "fetch": {"url": url}, @@ -193,40 +198,40 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): if not self.bootstrap_mirrors: return - + bootstrap_yaml = { - 'sources': [], - 'trusted': {}, + "sources": [], + "trusted": {}, } for name in self.bootstrap_mirrors: - bs_mirror_path = config_root/f'bootstrap/{name}' + bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml['sources'].append( + bootstrap_yaml["sources"].append( { - 'name': name, - 'metadata': str(bs_mirror_path), + "name": name, + "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml['trusted'][name] = True + bootstrap_yaml["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { - 'type': 'install', - 'info': mirror['url'], + "type": "install", + "info": mirror["url"], } - with (bs_mirror_path/'metadata.yaml').open('w') as file: + with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) - - with (config_root/'bootstrap.yaml').open('w') as file: + + with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - + self._keys = [] key_store.mkdir(exist_ok=True) @@ -243,13 +248,13 @@ def _key_setup(self, key_store: pathlib.Path): # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path + # try prepending system config path + path = self._system_config_root / path if path.is_file(): - with open(path, 'rb') as reader: + with open(path, "rb") as reader: binary_key = reader.read() - + # convert base64 key to binary else: try: @@ -258,17 +263,19 @@ def _key_setup(self, key_store: pathlib.Path): raise MirrorError( f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - + f"Check the key listed in mirrors.yaml in system config." + ) + file_type = magic.from_buffer(binary_key, mime=True) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config.") + f"Check the key listed in mirrors.yaml in system config." + ) # copy key to new destination in key store - with open(dest, 'wb') as writer: + with open(dest, "wb") as writer: writer.write(binary_key) self._keys.append(dest) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a15569b6..76cba826 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,9 +4,8 @@ import jinja2 import yaml -from typing import Optional -from . import cache, root_logger, schema, spack_util, mirror +from . import root_logger, schema, spack_util, mirror from .etc import envvars @@ -170,7 +169,7 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # load the optional mirrors.yaml from system config, and add any additional + # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) @@ -232,7 +231,7 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - + @property def config(self): return self._config diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 72f2190c..d262cd1b 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -4,14 +4,17 @@ import stackinator.mirror as mirror import yaml + @pytest.fixture def test_path(): return pathlib.Path(__file__).parent.resolve() + @pytest.fixture def systems_path(test_path): return test_path / "data" / "systems" + def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" @@ -19,37 +22,45 @@ def test_mirror_init(systems_path): valid_mirrors = { "fake-mirror": { - 'url': 'https://github.com', - 'enabled': True, - 'bootstrap': False, - 'cache': False, - 'public_key': '../../test-gpg-pub.asc', - 'mount_specific': False}, + "url": "https://github.com", + "enabled": True, + "bootstrap": False, + "cache": False, + "public_key": "../../test-gpg-pub.asc", + "mount_specific": False, + }, "buildcache-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': False, - 'cache': True, - 'private_key': '../test-gpg-priv.asc', - 'mount_specific': False}, + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": False, + "cache": True, + "private_key": "../test-gpg-priv.asc", + "mount_specific": False, + }, "bootstrap-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': True, - 'cache': False, - 'mount_specific': False} + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": True, + "cache": False, + "mount_specific": False, + }, } - with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors['buildcache-mirror']['public_key'] = key + valid_mirrors["buildcache-mirror"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] - assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) - + assert mirrors_obj.bootstrap_mirrors == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") + ] + assert mirrors_obj.build_cache_mirror == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") + ].pop(0) + for mir in mirrors_obj.mirrors: - assert mirrors_obj.mirrors[mir].get('enabled') + assert mirrors_obj.mirrors[mir].get("enabled") + def test_mirror_init_bad_url(systems_path): """Check that MirrorError is raised for a bad url.""" @@ -59,31 +70,33 @@ def test_mirror_init_bad_url(systems_path): with pytest.raises(mirror.MirrorError): mirror.Mirrors(path) + def test_setup_configs(tmp_path, systems_path): """Test general config setup.""" - mir = mirror.Mirrors(systems_path/'mirror-ok') + mir = mirror.Mirrors(systems_path / "mirror-ok") mir.setup_configs(tmp_path) - assert (tmp_path/'mirrors.yaml').is_file() - assert (tmp_path/'bootstrap').is_dir() - assert (tmp_path/'key_store').is_dir() + assert (tmp_path / "mirrors.yaml").is_file() + assert (tmp_path / "bootstrap").is_dir() + assert (tmp_path / "key_store").is_dir() + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', - cmdline_cache=systems_path/'mirror-ok/cache.yaml') + mirrors = mirror.Mirrors(systems_path / "mirror-ok", cmdline_cache=systems_path / "mirror-ok/cache.yaml") assert len(mirrors.mirrors) == 4 # This should always be the build cache even though one is already defined. - assert mirrors.build_cache_mirror == 'cmdline_cache' - cache_mirror = mirrors.mirrors['cmdline_cache'] - assert cache_mirror['url'] == '/tmp/foo' - assert cache_mirror['enabled'] - assert cache_mirror['cache'] - assert not cache_mirror['bootstrap'] - assert cache_mirror['mount_specific'] + assert mirrors.build_cache_mirror == "cmdline_cache" + cache_mirror = mirrors.mirrors["cmdline_cache"] + assert cache_mirror["url"] == "/tmp/foo" + assert cache_mirror["enabled"] + assert cache_mirror["cache"] + assert not cache_mirror["bootstrap"] + assert cache_mirror["mount_specific"] + def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" @@ -101,7 +114,7 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): "bootstrap-mirror": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, - } + }, } } @@ -114,9 +127,10 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): assert data == valid_spack_yaml + def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" - + valid_yaml = { "sources": [ { @@ -124,54 +138,54 @@ def test_create_bootstrap_configs(tmp_path, systems_path): "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], - "trusted": { - "bootstrap-mirror": True - }, + "trusted": {"bootstrap-mirror": True}, } valid_metadata = { "type": "install", "info": "https://mirror.spack.io", } - mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_bootstrap_configs(tmp_path) - with (tmp_path/'bootstrap.yaml').open() as f: + with (tmp_path / "bootstrap.yaml").open() as f: bs_data = yaml.safe_load(f) print(bs_data) print(valid_yaml) assert bs_data == valid_yaml - with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: + with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata + def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok') + mirrors = mirror.Mirrors(systems_path / "mirror-ok") mirrors._key_setup(tmp_path) key_files = list(tmp_path.iterdir()) - assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} # The two files should be identical in content key_file_data = [] for key_file in key_files: - with key_file.open('rb') as file: + with key_file.open("rb") as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] -@pytest.mark.parametrize("system_name", [ - 'mirror-bad-key', - 'mirror-bad-keypath', -]) + +@pytest.mark.parametrize( + "system_name", + [ + "mirror-bad-key", + "mirror-bad-keypath", + ], +) def test_key_setup_bad_key(tmp_path, systems_path, system_name): """asdfasdf""" - mirrors = mirror.Mirrors(systems_path/system_name) + mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): mirrors._key_setup(tmp_path) - - - From 30521d3ca4d747d0480a75b4eb87e5565120b317 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:14:56 -0600 Subject: [PATCH 54/98] Got rid of cache.py --- stackinator/cache.py | 58 -------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 stackinator/cache.py diff --git a/stackinator/cache.py b/stackinator/cache.py deleted file mode 100644 index 24177e33..00000000 --- a/stackinator/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pathlib - -import yaml - -from . import schema - - -def configuration_from_file(file, mount): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - # validate the yaml - schema.CacheValidator.validate(raw) - - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw - - -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } - } - } - - return yaml.dump(mirrors, default_flow_style=False) From 191af6fe56ed2f7e7ffb4461e33e14868a9f10fc Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:41:31 +0100 Subject: [PATCH 55/98] use pyproject for dependencies --- bin/stack-config | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/stack-config b/bin/stack-config index 66496991..02f48700 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "python-magic", # "jinja2", # "jsonschema", # "pyYAML", diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..25fd1ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "python-magic", "Jinja2", "jsonschema", "PyYAML", From 2fc29d6f19810a82eb86a32c50563969a97375ec Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:43:53 +0100 Subject: [PATCH 56/98] add unit test and lint hints to readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 5c7e0408..265b0ac3 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,17 @@ A tool for building a scientific software stack from a recipe for vClusters on C Read the [documentation](https://eth-cscs.github.io/stackinator/) to get started. Create a ticket in our [GitHub issues](https://github.com/eth-cscs/stackinator/issues) if you find a bug, have a feature request or have a question. + +## running tests: + +Use uv to run the tests, which will in turn ensure that the correct dependencies from `pyproject.toml` are used: + +``` +uv run pytest +``` + +Before pushing, apply the linting rules (this calls uv under the hood): + +``` +./lint +``` From d8594a3881eabdecd619401e798637163930ac07 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:44:19 +0100 Subject: [PATCH 57/98] tweak presentation of build cache by recipe --- stackinator/builder.py | 2 +- stackinator/main.py | 2 +- stackinator/recipe.py | 14 ++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index ca0f8753..47a73b05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): spack_version=spack_version, spack_meta=spack_meta, gpg_keys=recipe.mirrors.keys, - cache=recipe.mirrors.buildcache, + cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/main.py b/stackinator/main.py index 0d0b9bd8..ec384561 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -86,7 +86,7 @@ def make_argparser(): "--build", required=True, type=str, - help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp')", ) parser.add_argument("--no-bwrap", action="store_true", required=False) parser.add_argument( diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 76cba826..ff3d8e27 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,8 +172,7 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self.cache = self.mirrors.build_cache_mirror + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) # optional post install hook if self.post_install_hook is not None: @@ -202,6 +201,13 @@ def spack_repo(self): return repo_path return None + # Returns: + # Path: if the recipe specified a build cache mirror + # None: if no build cache mirror is used + @property + def build_cache_mirror(self): + return self.mirrors.build_cache_mirror + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -511,7 +517,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.cache + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -542,7 +548,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.cache is not None + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, From ef9d7e88fe3f6d7da1f7dcaa729d35318741369e Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 20 Mar 2026 15:35:01 +0100 Subject: [PATCH 58/98] fix build cache configuration --- stackinator/builder.py | 6 +- stackinator/mirror.py | 80 ++++++++++++--------- stackinator/recipe.py | 6 +- stackinator/templates/Makefile.compilers | 4 +- stackinator/templates/Makefile.environments | 4 +- 5 files changed, 55 insertions(+), 45 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 47a73b05..2dc69233 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -168,10 +168,12 @@ def generate(self, recipe): # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" + config_path = self.path / "config" self.path.mkdir(exist_ok=True, parents=True) store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) + config_path.mkdir(exist_ok=True) # check out the version of spack spack_version = recipe.spack_version @@ -232,7 +234,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - gpg_keys=recipe.mirrors.keys, + gpg_keys=recipe.mirrors.key_files(config_path), cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, @@ -301,8 +303,6 @@ def generate(self, recipe): # Generate the system configuration: the compilers, environments, etc. # that are defined for the target cluster. - config_path = self.path / "config" - config_path.mkdir(exist_ok=True) packages_path = config_path / "packages.yaml" # the packages.yaml configuration that will be used when building all environments diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e6ba268b..66fa1d12 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,13 +1,15 @@ -from typing import Optional, List, Dict +from typing import Optional, Dict import base64 import io -import magic import os import pathlib +import shutil import urllib.error import urllib.request import yaml +import magic + from . import schema, root_logger @@ -50,7 +52,8 @@ def __init__( self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] + # self._keys: Optional[List[pathlib.Path]] = [] + self._keys = self._key_init() def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" @@ -157,14 +160,12 @@ def _check_mirrors(self): f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" ) - @property - def keys(self): + def key_files(self, config_root: pathlib.Path): """Return the list of public and private key file paths.""" - if self._keys is None: raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") - - return self._keys + key_dir = config_root / self.KEY_STORE_DIR + return [key_dir / info["path"] for info in self._keys] def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -229,34 +230,27 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def _key_setup(self, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - - self._keys = [] - key_store.mkdir(exist_ok=True) + def _key_init(self): + key_info = [] for name, mirror in self.mirrors.items(): - if mirror.get("public_key") is None: + if mirror.get("private_key") is None: continue - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - - dest = pathlib.Path(key_store / f"{name}.gpg") + key = mirror["private_key"] # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): # try prepending system config path path = self._system_config_root / path if path.is_file(): - with open(path, "rb") as reader: - binary_key = reader.read() - - # convert base64 key to binary + # use the user-provided file + key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": path}) else: + # convert base64 key to binary try: binary_key = base64.b64decode(key) except ValueError: @@ -266,16 +260,34 @@ def _key_setup(self, key_store: pathlib.Path): f"Check the key listed in mirrors.yaml in system config." ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror {name} is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) + + key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": binary_key}) - # copy key to new destination in key store - with open(dest, "wb") as writer: - writer.write(binary_key) + return key_info + + def _key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + key_store.mkdir(exist_ok=True) - self._keys.append(dest) + for key_info in self._keys: + path = key_store / key_info["path"] + source = key_info["source"] + + match source: + case pathlib.Path(): + # copy source -> path + shutil.copy2(source, path) + case bytes(): + # open path and copy in bytes + with open(path, "wb") as writer: + writer.write(source) + case _: + raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ff3d8e27..0688d0d6 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -517,10 +517,9 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, - push_to_cache=push_to_cache, + buildcache=self.build_cache_mirror, spack_version=self.spack_version, ) @@ -548,10 +547,9 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, - push_to_cache=push_to_cache, + buildcache=self.build_cache_mirror, spack_version=self.spack_version, ) diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index f9520bd6..d534b310 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -18,8 +18,8 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% for compiler, config in compilers.items() %} {{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | cut -d ';' -f2) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a530232..929e40bc 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -17,8 +17,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} # Push built packages to a binary cache if a key has been provided {% for env, config in environments.items() %} {{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if push_to_cache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 8b1e0afceb4043a86fd8787b56c0b7a915dd9945 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 20 Mar 2026 14:14:10 -0600 Subject: [PATCH 59/98] modified bootstrap yaml setup and removed hardcoded alpscache --- stackinator/mirror.py | 18 +++++++++++------- stackinator/recipe.py | 5 ++++- stackinator/templates/Makefile | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 66fa1d12..2c72e998 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -49,7 +49,7 @@ def __init__( [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", True)] # Will hold a list of all the gpg keys (public and private) # self._keys: Optional[List[pathlib.Path]] = [] @@ -201,29 +201,33 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): return bootstrap_yaml = { - "sources": [], - "trusted": {}, + "bootstrap": { + "sources": [], + "trusted": {}, + } } for name in self.bootstrap_mirrors: bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["sources"].append( + bootstrap_yaml["bootstrap"]["sources"].append( { "name": name, "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml["trusted"][name] = True + bootstrap_yaml["bootstrap"]["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { "type": "install", - "info": mirror["url"], - } + "info": { + "url": mirror["url"], + } + } with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 0688d0d6..8f4c82f1 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,7 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) + self.mirrors = mirror.Mirrors(self.system_config_path, + pathlib.Path(args.cache) if args.cache else None) # optional post install hook if self.post_install_hook is not None: @@ -521,6 +522,7 @@ def compiler_files(self): compilers=self.compilers, buildcache=self.build_cache_mirror, spack_version=self.spack_version, + cache = self.build_cache_mirror, ) files["config"] = {} @@ -551,6 +553,7 @@ def environment_files(self): environments=self.environments, buildcache=self.build_cache_mirror, spack_version=self.spack_version, + cache=self.build_cache_mirror, ) files["config"] = {} diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index f0b90ebe..7a931777 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -86,7 +86,7 @@ cache-force: mirror-setup $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 3c21789af1b8d006d612312ecb72b9c6704fbdd4 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 25 Mar 2026 22:54:35 -0600 Subject: [PATCH 60/98] mirrors documentation --- docs/cluster-config.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 2ce62019..5a7d4d54 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -93,6 +93,27 @@ packages: version: ["git.59b6de6a91d9637809677c50cc48b607a91a9acb=main"] ``` +### Configuring Spack mirrors: `mirrors.yaml` + +On air-gapped systems, Spack is unable to reach its default mirror to fetch packages. The `mirrors.yaml` configuration can be used to connect Spack to local mirrors for fetching and building packages. + +`mirrors.yaml` treats source mirrors, buildcaches, and bootstrap mirrors the same, and they may all be included in this file. Spack will search the topmost mirror first and the bottom-most mirror last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. + +If using a buildcache, public and private keys must be provided for signing and verifying packages. + +```yaml title="mirrors.yaml" +local_filesystem: + url: file:///home/username/spack-mirror-2014-06-24 +site_server: + url: https://example.com/some/web-hosted/directory +buildcache-mirror: + url: https://example.com/some/buildcache/mirror + public_key: ../buildcache-key.public.gpg + private_key: /user-home/.gnupg/private-keys-v1.d/my-private-key.asc + cache: true + bootstrap: true +``` + ## Site and System Configurations The `repo.yaml` configuration can be used to provide a list of additional Spack package repositories to use on the target system. From 596f50ea7a4e895f750515b457a260a9148901b9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Wed, 1 Apr 2026 09:31:33 -0600 Subject: [PATCH 61/98] Switched to the requests libs from urlopen, as urlopen doesn't handle ssl very well. --- requirements.txt | 1 + stackinator/mirror.py | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index d0de0ac4..82d9720a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ pytest==9.0.2 python-magic==0.4.27 pyyaml==6.0.3 referencing==0.37.0 +requests==2.32.5 rpds-py==0.30.0 diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 2c72e998..61ba4617 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -3,9 +3,7 @@ import io import os import pathlib -import shutil -import urllib.error -import urllib.request +import requests import yaml import magic @@ -152,12 +150,11 @@ def _check_mirrors(self): elif url.startswith("https://"): try: - request = urllib.request.Request(url, method="HEAD") - urllib.request.urlopen(request) - except urllib.error.URLError as e: + requests.request(url=url, method="HEAD") + except requests.exceptions.RequestException as err: raise MirrorError( f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + f"Check the url listed in mirrors.yaml in system config. \n{err}" ) def key_files(self, config_root: pathlib.Path): From ff42460cbdb531bf96db4ef5235ffab3ca076104 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 8 Apr 2026 14:14:04 -0600 Subject: [PATCH 62/98] restrict number of buildcaches and bootstraps in mirror schema --- stackinator/schema/mirror.json | 86 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 89770831..042ac630 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,46 +1,54 @@ { "type" : "object", - "additionalProperties": { - "type": "object", - "required": ["url"], - "additionalProperties": false, - "properties": { - "url": { - "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." + "additionalProperties": false, + "properties": { + "bootstrap": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + } }, - "description": { - "type": "string", - "description": "What this mirror is for." + "additionalProperties": false, + "required": ["url"] + }, + "buildcache": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + }, + "public_key": {"type": "string"}, + "private_key": {"type": "string"}, + "mount_specific": { + "type": "boolean", + "default": false + } }, - "enabled": { - "type": "boolean", - "default": true, - "description": "Whether this mirror is enabled." - }, - "bootstrap": { - "type": "boolean", - "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." - }, - "cache": { - "type": "boolean", - "default": false, - "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." - }, - "public_key": { - "type": "string", - "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." - }, - "private_key": { - "type": "string", - "description": "Private PGP key for signing binary cache packages. (Path only)" - }, - "mount_specific": { - "type": "boolean", - "default": false, - "description": "Use a mount specific buildcache path (specified path + recipe mount point)." + "additionalProperties": false, + "required": ["url"] + }, + "sourcecache": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false, + "required": ["url"] } } } -} +} \ No newline at end of file From e10f0c9b4bd9894687306a234051699b81501c1b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 8 Apr 2026 14:32:51 -0600 Subject: [PATCH 63/98] fixed mistake --- stackinator/recipe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 8f4c82f1..e63165ae 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -522,7 +522,6 @@ def compiler_files(self): compilers=self.compilers, buildcache=self.build_cache_mirror, spack_version=self.spack_version, - cache = self.build_cache_mirror, ) files["config"] = {} @@ -553,7 +552,6 @@ def environment_files(self): environments=self.environments, buildcache=self.build_cache_mirror, spack_version=self.spack_version, - cache=self.build_cache_mirror, ) files["config"] = {} From 6a203c39b1bff74c4caf12ae75464ce8edd0522a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 14 Apr 2026 13:44:19 -0600 Subject: [PATCH 64/98] refactored mirrors.py to match new schema --- stackinator/mirror.py | 127 +++++++++++++++++++-------------- stackinator/schema/mirror.json | 4 ++ 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 61ba4617..bcc3e5ca 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -38,17 +38,6 @@ def __init__( self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - # Always use the cache given on the command line - if self.CMDLINE_CACHE in self.mirrors: - self.build_cache_mirror = self.CMDLINE_CACHE - else: - # Otherwise, grab the configured cache (or None) - self.build_cache_mirror: Optional[str] = ( - [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] - ).pop(0) - - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", True)] - # Will hold a list of all the gpg keys (public and private) # self._keys: Optional[List[pathlib.Path]] = [] self._keys = self._key_init() @@ -71,22 +60,37 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = [name for name, mirror in mirrors.items() if mirror["cache"]] - if len(caches) > 1: - raise MirrorError( - "Mirror config has more than one mirror specified as the build cache destination.\n" - f"{self._pp_yaml(caches)}" - ) - elif caches: - cache = mirrors[caches[0]] - if not cache.get("private_key"): - raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") + enabled_mirrors: Dict[str, Dict] = {} + + buildcache = mirrors.get("buildcache") + if buildcache and buildcache.get("enabled", True): + if not buildcache.get("private_key"): + raise MirrorError( + "Mirror build cache config is missing a required 'private_key' path." + ) + self.build_cache_mirror = "buildcache" + enabled_mirrors["buildcache"] = buildcache + else: + self.build_cache_mirror = None # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: - mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + enabled_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) + self.build_cache_mirror = self.CMDLINE_CACHE + + + bootstrap = mirrors.get("bootstrap") + if bootstrap and bootstrap.get("enabled", True): + self.bootstrap_mirror = bootstrap + enabled_mirrors["bootstrap"] = bootstrap + else: + self.bootstrap_mirror = None - return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} + for name, mirror in mirrors.get("sourcecache", {}).items(): + if mirror.get("enabled", True): + enabled_mirrors[name] = mirror + + return enabled_mirrors @staticmethod def _pp_yaml(object): @@ -118,11 +122,10 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: mirror_cfg = { "url": raw["root"], "description": "Buildcache dest loaded from legacy cache.yaml", - "cache": True, "enabled": True, - "bootstrap": False, "mount_specific": True, "private_key": raw["key"], + "cmdline": True } self._logger.warning( @@ -176,11 +179,11 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - for name, mirror in self.mirrors.items(): - url = mirror["url"] + if self.build_cache_mirror: + url = self.mirrors.get("buildcache")["url"] + name = self.build_cache_mirror - # Make the mirror path specific to the mount point - if mirror["mount_specific"] and self._mount_point is not None: + if self.build_cache_mirror.get("mount_specific", True) and self._mount_point is not None: url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { @@ -188,13 +191,32 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): "push": {"url": url}, } + elif self.mirrors.get("bootstrap"): + url = self.mirrors.get("bootstrap")["url"] + + raw["mirrors"]["bootstrap"] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + + elif self.mirrors.get("sourcecache"): + source_mirrors = self.mirrors.get("sourcecache") + + for name, mirror in source_mirrors.items(): + url = mirror["url"] + + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - if not self.bootstrap_mirrors: + if not self.bootstrap_mirror: return bootstrap_yaml = { @@ -204,29 +226,28 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): } } - for name in self.bootstrap_mirrors: - bs_mirror_path = config_root / f"bootstrap/{name}" - mirror = self.mirrors[name] - # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["bootstrap"]["sources"].append( - { - "name": name, - "metadata": str(bs_mirror_path), - } - ) - # And trust each one - bootstrap_yaml["bootstrap"]["trusted"][name] = True - - # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True, exist_ok=True) - bs_mirror_yaml = { - "type": "install", - "info": { - "url": mirror["url"], - } - } - with (bs_mirror_path / "metadata.yaml").open("w") as file: - yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + bs_mirror_path = config_root / f"bootstrap/{self.bootstrap_mirror}" + mirror = self.mirrors.get("bootstrap") + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml["bootstrap"]["sources"].append( + { + "name": "bootstrap_mirror", + "metadata": str(bs_mirror_path), + } + ) + # And trust each one + bootstrap_yaml["bootstrap"]["trusted"][bootstrap_mirror] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True, exist_ok=True) + bs_mirror_yaml = { + "type": "install", + "info": { + "url": mirror["url"], + } + } + with (bs_mirror_path / "metadata.yaml").open("w") as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 042ac630..4965c815 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -29,6 +29,10 @@ "mount_specific": { "type": "boolean", "default": false + }, + "cmdline": { + "type": "boolean", + "default": true } }, "additionalProperties": false, From 099878ab22f7e4b6a0c3850ea28931fa9fd991fc Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 21 Apr 2026 14:45:51 -0600 Subject: [PATCH 65/98] fixing unit tests, _key_init() still broken --- stackinator/mirror.py | 145 ++++++++---------- stackinator/schema/mirror.json | 2 +- .../data/systems/mirror-bad-key/mirrors.yaml | 4 +- .../systems/mirror-bad-keypath/mirrors.yaml | 4 +- .../data/systems/mirror-bad-url/mirrors.yaml | 6 +- .../systems/mirror-ok-raw-key/mirrors.yaml | 59 +++++++ unittests/data/systems/mirror-ok/mirrors.yaml | 70 ++------- unittests/test_mirrors.py | 112 +++++++------- 8 files changed, 207 insertions(+), 195 deletions(-) create mode 100644 unittests/data/systems/mirror-ok-raw-key/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bcc3e5ca..5219af03 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -68,8 +68,8 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict raise MirrorError( "Mirror build cache config is missing a required 'private_key' path." ) - self.build_cache_mirror = "buildcache" - enabled_mirrors["buildcache"] = buildcache + self.build_cache_mirror = "buildcache" + enabled_mirrors["buildcache"] = buildcache else: self.build_cache_mirror = None @@ -78,17 +78,13 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict enabled_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) self.build_cache_mirror = self.CMDLINE_CACHE - bootstrap = mirrors.get("bootstrap") if bootstrap and bootstrap.get("enabled", True): - self.bootstrap_mirror = bootstrap enabled_mirrors["bootstrap"] = bootstrap - else: - self.bootstrap_mirror = None for name, mirror in mirrors.get("sourcecache", {}).items(): - if mirror.get("enabled", True): - enabled_mirrors[name] = mirror + if mirror.get("enabled", True): + enabled_mirrors[name] = mirror return enabled_mirrors @@ -179,44 +175,26 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - if self.build_cache_mirror: - url = self.mirrors.get("buildcache")["url"] - name = self.build_cache_mirror + for name, mirror in self.mirrors.items(): + url = mirror["url"] - if self.build_cache_mirror.get("mount_specific", True) and self._mount_point is not None: - url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") + # Make the mirror path specific to the mount point + if(name=="buildcache"): + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { "fetch": {"url": url}, "push": {"url": url}, } - elif self.mirrors.get("bootstrap"): - url = self.mirrors.get("bootstrap")["url"] - - raw["mirrors"]["bootstrap"] = { - "fetch": {"url": url}, - "push": {"url": url}, - } - - elif self.mirrors.get("sourcecache"): - source_mirrors = self.mirrors.get("sourcecache") - - for name, mirror in source_mirrors.items(): - url = mirror["url"] - - raw["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } - with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - if not self.bootstrap_mirror: + if not self.mirrors.get("bootstrap"): return bootstrap_yaml = { @@ -226,17 +204,17 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): } } - bs_mirror_path = config_root / f"bootstrap/{self.bootstrap_mirror}" + bs_mirror_path = config_root / "bootstrap" / "bootstrap-mirror" mirror = self.mirrors.get("bootstrap") # Tell spack where to find the metadata for each bootstrap mirror. bootstrap_yaml["bootstrap"]["sources"].append( { - "name": "bootstrap_mirror", + "name": "bootstrap-mirror", "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml["bootstrap"]["trusted"][bootstrap_mirror] = True + bootstrap_yaml["bootstrap"]["trusted"]["bootstrap-mirror"] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) @@ -253,44 +231,44 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): yaml.dump(bootstrap_yaml, file, default_flow_style=False) def _key_init(self): - key_info = [] - - for name, mirror in self.mirrors.items(): - if mirror.get("private_key") is None: - continue + key_info = {} + + key = self.mirrors["buildcache"].get("private_key") - key = mirror["private_key"] + if key is None: + return - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): - # try prepending system config path - path = self._system_config_root / path + if not path.is_absolute(): + # try prepending system config path + path = self._system_config_root / path - if path.is_file(): - # use the user-provided file - key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": path}) - else: - # convert base64 key to binary - try: - binary_key = base64.b64decode(key) - except ValueError: - raise MirrorError( - f"Key for mirror '{name}' is not valid: '{path}'. \n" - f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config." - ) + if path.is_file(): + # use the user-provided file + key_info = {"path": pathlib.Path(f"buildcache.pgp"), "source": path} + else: + # convert base64 key to binary + try: + binary_key = base64.b64decode(key, validate=True) + print(binary_key) + except ValueError: + raise MirrorError( + f"Key for mirror 'buildcache' is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config." + ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror {name} is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): + raise MirrorError( + f"Key for mirror 'buildcache' is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) - key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": binary_key}) + key_info = {"path": pathlib.Path("buildcache.pgp"), "source": binary_key} return key_info @@ -299,17 +277,20 @@ def _key_setup(self, key_store: pathlib.Path): key_store.mkdir(exist_ok=True) - for key_info in self._keys: - path = key_store / key_info["path"] - source = key_info["source"] - - match source: - case pathlib.Path(): - # copy source -> path - shutil.copy2(source, path) - case bytes(): - # open path and copy in bytes - with open(path, "wb") as writer: - writer.write(source) - case _: - raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") + #for key_info in self._keys: + + #path = key_store / key_info["path"] + path = key_store / self._keys["path"] + #source = key_info["source"] + source = self._keys["source"] + + match source: + case pathlib.Path(): + # copy source -> path + shutil.copy2(source, path) + case bytes(): + # open path and copy in bytes + with open(path, "wb") as writer: + writer.write(source) + case _: + raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 4965c815..03013b50 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -32,7 +32,7 @@ }, "cmdline": { "type": "boolean", - "default": true + "default": false } }, "additionalProperties": false, diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml index d5154dd3..a3d01283 100644 --- a/unittests/data/systems/mirror-bad-key/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -1,3 +1,3 @@ -bad-key: +buildcache: url: https://mirror.spack.io - public_key: bad_key.gpg + private_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml index 3433e04a..5ac6af9e 100644 --- a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -1,3 +1,3 @@ -bad-key-path: +buildcache: url: https://mirror.spack.io - public_key: /path/doesnt/exist + private_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 8ffce331..c961d378 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,2 +1,4 @@ -bad-url: - url: https://www.testsite.io/services \ No newline at end of file +sourcecache: + bad-url: + url: https://www.testsite.io/services + enabled: true \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml new file mode 100644 index 00000000..e91cc7b6 --- /dev/null +++ b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml @@ -0,0 +1,59 @@ +bootstrap: + url: https://mirror.spack.io + enabled: true +buildcache: + url: https://mirror.spack.io + enabled: true + private_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" + mount_specific: false + cmdline: false +sourcecache: + mirror1: + url: https://github.com + enabled: true + mirror2: + url: https://github.com/spack + enabled: true + disabled-mirror: + url: https://github.com + enabled: false \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 0a1b8434..f5bfa7ff 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,55 +1,19 @@ -fake-mirror: - url: https://github.com - public_key: ../../test-gpg-pub.asc -disabled-mirror: - url: https://github.com - enabled: false -buildcache-mirror: +bootstrap: url: https://mirror.spack.io - private_key: '../test-gpg-priv.asc' - public_key: "\ - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" - - cache: true -bootstrap-mirror: + enabled: true +buildcache: url: https://mirror.spack.io - bootstrap: true + enabled: true + private_key: ../../test-gpg-priv.asc + mount_specific: false + cmdline: false +sourcecache: + mirror1: + url: https://github.com + enabled: true + mirror2: + url: https://github.com/spack + enabled: true + disabled-mirror: + url: https://github.com + enabled: false \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index d262cd1b..23ed4caa 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -21,42 +21,34 @@ def test_mirror_init(systems_path): mirrors_obj = mirror.Mirrors(path) valid_mirrors = { - "fake-mirror": { - "url": "https://github.com", + "bootstrap": { + "url": "https://mirror.spack.io", "enabled": True, - "bootstrap": False, - "cache": False, - "public_key": "../../test-gpg-pub.asc", - "mount_specific": False, }, - "buildcache-mirror": { + "buildcache": { "url": "https://mirror.spack.io", "enabled": True, - "bootstrap": False, - "cache": True, - "private_key": "../test-gpg-priv.asc", + "private_key": "../../test-gpg-priv.asc", "mount_specific": False, + "cmdline": False, }, - "bootstrap-mirror": { - "url": "https://mirror.spack.io", + "mirror1": { + "url": "https://github.com", "enabled": True, - "bootstrap": True, - "cache": False, - "mount_specific": False, }, + "mirror2": { + "url": "https://github.com/spack", + "enabled": True, + } } - with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: - key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors["buildcache-mirror"]["public_key"] = key + # with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + # key = base64.b64encode(pub_key_file.read()).decode() + # valid_mirrors["buildcache"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.bootstrap_mirrors == [ - name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") - ] - assert mirrors_obj.build_cache_mirror == [ - name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") - ].pop(0) + + assert mirrors_obj.build_cache_mirror == "buildcache" for mir in mirrors_obj.mirrors: assert mirrors_obj.mirrors[mir].get("enabled") @@ -90,11 +82,10 @@ def test_command_line_cache(systems_path): assert len(mirrors.mirrors) == 4 # This should always be the build cache even though one is already defined. assert mirrors.build_cache_mirror == "cmdline_cache" - cache_mirror = mirrors.mirrors["cmdline_cache"] + cache_mirror = mirrors.mirrors["buildcache"] assert cache_mirror["url"] == "/tmp/foo" assert cache_mirror["enabled"] - assert cache_mirror["cache"] - assert not cache_mirror["bootstrap"] + assert cache_mirror["cmdline"] assert cache_mirror["mount_specific"] @@ -103,18 +94,22 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): valid_spack_yaml = { "mirrors": { - "fake-mirror": { - "fetch": {"url": "https://github.com"}, - "push": {"url": "https://github.com"}, - }, - "buildcache-mirror": { + "bootstrap": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, }, - "bootstrap-mirror": { + "buildcache": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, }, + "mirror1": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "mirror2": { + "fetch": {"url": "https://github.com/spack"}, + "push": {"url": "https://github.com/spack"}, + }, } } @@ -132,17 +127,21 @@ def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { - "sources": [ - { - "name": "bootstrap-mirror", - "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), - } - ], - "trusted": {"bootstrap-mirror": True}, + "bootstrap": { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), + } + ], + "trusted": {"bootstrap-mirror": True}, + } } valid_metadata = { "type": "install", - "info": "https://mirror.spack.io", + "info": { + "url": "https://mirror.spack.io", + } } mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") @@ -162,18 +161,26 @@ def test_create_bootstrap_configs(tmp_path, systems_path): def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_key_file = mirror.Mirrors(systems_path / "mirror-ok") + key_dir = tmp_path / "key_dir" + mirrors_raw_key = mirror.Mirrors(systems_path / "mirror-ok-raw-key") + raw_dir = tmp_path / "raw_dir" + + mirrors_key_file._key_setup(key_dir) + mirrors_raw_key._key_setup(raw_dir) + + key_file, = (p for p in key_dir.iterdir() if p.is_file()) + assert key_file.name == "buildcache.pgp" - mirrors._key_setup(tmp_path) + raw_key_file, = (p for p in key_dir.iterdir() if p.is_file()) + assert raw_key_file.name == "buildcache.pgp" - key_files = list(tmp_path.iterdir()) - assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} # The two files should be identical in content - key_file_data = [] - for key_file in key_files: - with key_file.open("rb") as file: - key_file_data.append(file.read()) - assert key_file_data[0] == key_file_data[1] + with key_file.open("rb") as file: + key_file_data = file.read() + with raw_key_file.open("rb") as file: + raw_key_file_data = file.read() + assert key_file_data == raw_key_file_data @pytest.mark.parametrize( @@ -184,8 +191,7 @@ def test_key_setup(systems_path, tmp_path): ], ) def test_key_setup_bad_key(tmp_path, systems_path, system_name): - """asdfasdf""" + """Check that MirrorError is raised for bad keys""" - mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): - mirrors._key_setup(tmp_path) + mirrors = mirror.Mirrors(systems_path / system_name) From 17f4bc6a24c942be5126b065feb9719737a88bd3 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 22 Apr 2026 23:03:36 -0600 Subject: [PATCH 66/98] reverted to old key_setup, add handling for private keys, refactored tests --- stackinator/mirror.py | 90 +++++++++---------- stackinator/schema/mirror.json | 6 +- .../systems/mirror-ok-raw-key/mirrors.yaml | 59 ------------ unittests/data/systems/mirror-ok/mirrors.yaml | 42 +++++++++ unittests/test_mirrors.py | 38 ++++---- 5 files changed, 106 insertions(+), 129 deletions(-) delete mode 100644 unittests/data/systems/mirror-ok-raw-key/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5219af03..7ff7bb82 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -39,8 +39,7 @@ def __init__( self._check_mirrors() # Will hold a list of all the gpg keys (public and private) - # self._keys: Optional[List[pathlib.Path]] = [] - self._keys = self._key_init() + self._keys: Optional[List[pathlib.Path]] = [] def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" @@ -156,12 +155,14 @@ def _check_mirrors(self): f"Check the url listed in mirrors.yaml in system config. \n{err}" ) - def key_files(self, config_root: pathlib.Path): + @property + def keys(self): """Return the list of public and private key file paths.""" + if self._keys is None: raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") - key_dir = config_root / self.KEY_STORE_DIR - return [key_dir / info["path"] for info in self._keys] + + return self._keys def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -230,67 +231,64 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def _key_init(self): - key_info = {} - - key = self.mirrors["buildcache"].get("private_key") + def _load_key(self, key: str, dest: pathlib.Path, name: str): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - if key is None: - return + # key will be saved under key_store/mirror_name.[pub/priv].gpg # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): # try prepending system config path path = self._system_config_root / path if path.is_file(): - # use the user-provided file - key_info = {"path": pathlib.Path(f"buildcache.pgp"), "source": path} + with open(path, "rb") as reader: + binary_key = reader.read() + + # convert base64 key to binary else: - # convert base64 key to binary try: - binary_key = base64.b64decode(key, validate=True) - print(binary_key) + binary_key = base64.b64decode(key) except ValueError: raise MirrorError( - f"Key for mirror 'buildcache' is not valid. \n" + f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" f"Check the key listed in mirrors.yaml in system config." ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror 'buildcache' is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) - - key_info = {"path": pathlib.Path("buildcache.pgp"), "source": binary_key} + # private keys will evaluate as "application/octet-stream" + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream"): + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) - return key_info + # copy key to new destination in key store + with open(dest, "wb") as writer: + writer.write(binary_key) + self._keys.append(dest) + + def _key_setup(self, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + """Iterate through mirror keys and load + relocate each one to key_store""" + self._keys = [] key_store.mkdir(exist_ok=True) - #for key_info in self._keys: - - #path = key_store / key_info["path"] - path = key_store / self._keys["path"] - #source = key_info["source"] - source = self._keys["source"] - - match source: - case pathlib.Path(): - # copy source -> path - shutil.copy2(source, path) - case bytes(): - # open path and copy in bytes - with open(path, "wb") as writer: - writer.write(source) - case _: - raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") + for name, mirror in self.mirrors.items(): + if name == "buildcache": + if mirror.get("private_key"): + key = mirror["private_key"] + dest = pathlib.Path(key_store / f"{name}.priv.gpg") + self._load_key(key, dest, name) + + if mirror.get("public_key") is None: + continue + + key = mirror["public_key"] + dest = pathlib.Path(key_store / f"{name}.pub.gpg") + self._load_key(key, dest, name) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 03013b50..608dfe19 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -10,7 +10,8 @@ "enabled": { "type": "boolean", "default": true - } + }, + "public_key": {"type": "string"} }, "additionalProperties": false, "required": ["url"] @@ -48,7 +49,8 @@ "enabled": { "type": "boolean", "default": true - } + }, + "public_key": {"type": "string"} }, "additionalProperties": false, "required": ["url"] diff --git a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml deleted file mode 100644 index e91cc7b6..00000000 --- a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bootstrap: - url: https://mirror.spack.io - enabled: true -buildcache: - url: https://mirror.spack.io - enabled: true - private_key: "\ - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" - mount_specific: false - cmdline: false -sourcecache: - mirror1: - url: https://github.com - enabled: true - mirror2: - url: https://github.com/spack - enabled: true - disabled-mirror: - url: https://github.com - enabled: false \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index f5bfa7ff..d4bc355a 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -4,6 +4,47 @@ bootstrap: buildcache: url: https://mirror.spack.io enabled: true + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" private_key: ../../test-gpg-priv.asc mount_specific: false cmdline: false @@ -11,6 +52,7 @@ sourcecache: mirror1: url: https://github.com enabled: true + public_key: ../../test-gpg-pub.asc mirror2: url: https://github.com/spack enabled: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 23ed4caa..e0ebb4a3 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -35,6 +35,7 @@ def test_mirror_init(systems_path): "mirror1": { "url": "https://github.com", "enabled": True, + "public_key": "../../test-gpg-pub.asc" }, "mirror2": { "url": "https://github.com/spack", @@ -42,9 +43,9 @@ def test_mirror_init(systems_path): } } - # with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: - # key = base64.b64encode(pub_key_file.read()).decode() - # valid_mirrors["buildcache"]["public_key"] = key + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors["buildcache"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors @@ -149,8 +150,7 @@ def test_create_bootstrap_configs(tmp_path, systems_path): with (tmp_path / "bootstrap.yaml").open() as f: bs_data = yaml.safe_load(f) - print(bs_data) - print(valid_yaml) + assert bs_data == valid_yaml with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: @@ -161,26 +161,19 @@ def test_create_bootstrap_configs(tmp_path, systems_path): def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors_key_file = mirror.Mirrors(systems_path / "mirror-ok") - key_dir = tmp_path / "key_dir" - mirrors_raw_key = mirror.Mirrors(systems_path / "mirror-ok-raw-key") - raw_dir = tmp_path / "raw_dir" - - mirrors_key_file._key_setup(key_dir) - mirrors_raw_key._key_setup(raw_dir) + mirrors = mirror.Mirrors(systems_path / "mirror-ok") - key_file, = (p for p in key_dir.iterdir() if p.is_file()) - assert key_file.name == "buildcache.pgp" + mirrors._key_setup(tmp_path) - raw_key_file, = (p for p in key_dir.iterdir() if p.is_file()) - assert raw_key_file.name == "buildcache.pgp" + pub_files = sorted(f for f in tmp_path.iterdir() if f.name.endswith(".pub.gpg")) + assert {pub_file.name for pub_file in pub_files} == {"buildcache.pub.gpg", "mirror1.pub.gpg"} # The two files should be identical in content - with key_file.open("rb") as file: - key_file_data = file.read() - with raw_key_file.open("rb") as file: - raw_key_file_data = file.read() - assert key_file_data == raw_key_file_data + pub_file_data = [] + for pub_file in pub_files: + with pub_file.open("rb") as file: + pub_file_data.append(file.read()) + assert pub_file_data[0] == pub_file_data[1] @pytest.mark.parametrize( @@ -193,5 +186,6 @@ def test_key_setup(systems_path, tmp_path): def test_key_setup_bad_key(tmp_path, systems_path, system_name): """Check that MirrorError is raised for bad keys""" + mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): - mirrors = mirror.Mirrors(systems_path / system_name) + mirrors._key_setup(tmp_path) From 4206e68103388abf35171d9e4a3456391c6f9223 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 22 Apr 2026 23:11:42 -0600 Subject: [PATCH 67/98] linting --- stackinator/mirror.py | 17 +++++++---------- stackinator/recipe.py | 3 +-- unittests/test_mirrors.py | 10 +++------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 7ff7bb82..d00cd820 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, List, Dict import base64 import io import os @@ -64,9 +64,7 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict buildcache = mirrors.get("buildcache") if buildcache and buildcache.get("enabled", True): if not buildcache.get("private_key"): - raise MirrorError( - "Mirror build cache config is missing a required 'private_key' path." - ) + raise MirrorError("Mirror build cache config is missing a required 'private_key' path.") self.build_cache_mirror = "buildcache" enabled_mirrors["buildcache"] = buildcache else: @@ -120,7 +118,7 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: "enabled": True, "mount_specific": True, "private_key": raw["key"], - "cmdline": True + "cmdline": True, } self._logger.warning( @@ -180,7 +178,7 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): url = mirror["url"] # Make the mirror path specific to the mount point - if(name=="buildcache"): + if name == "buildcache": if mirror["mount_specific"] and self._mount_point is not None: url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") @@ -223,8 +221,8 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): "type": "install", "info": { "url": mirror["url"], - } - } + }, + } with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) @@ -271,8 +269,7 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): writer.write(binary_key) self._keys.append(dest) - - + def _key_setup(self, key_store: pathlib.Path): """Iterate through mirror keys and load + relocate each one to key_store""" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index e63165ae..0d640596 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,8 +172,7 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, - pathlib.Path(args.cache) if args.cache else None) + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache) if args.cache else None) # optional post install hook if self.post_install_hook is not None: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index e0ebb4a3..7fb216df 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -32,15 +32,11 @@ def test_mirror_init(systems_path): "mount_specific": False, "cmdline": False, }, - "mirror1": { - "url": "https://github.com", - "enabled": True, - "public_key": "../../test-gpg-pub.asc" - }, + "mirror1": {"url": "https://github.com", "enabled": True, "public_key": "../../test-gpg-pub.asc"}, "mirror2": { "url": "https://github.com/spack", "enabled": True, - } + }, } with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: @@ -142,7 +138,7 @@ def test_create_bootstrap_configs(tmp_path, systems_path): "type": "install", "info": { "url": "https://mirror.spack.io", - } + }, } mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") From fa7ef2f266163fc2f5be9b6f467f2254e44dfdbc Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 23 Apr 2026 11:44:32 -0600 Subject: [PATCH 68/98] added 'requests' to pyproject.toml --- bin/stack-config | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/stack-config b/bin/stack-config index 02f48700..08e90d72 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "requests", # "python-magic", # "jinja2", # "jsonschema", diff --git a/pyproject.toml b/pyproject.toml index 25fd1ddf..4c36ae7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "requests", "python-magic", "Jinja2", "jsonschema", From 41e7630aa5753ff3db73ec7a8e9a34f87414e9f7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 27 Apr 2026 12:43:00 -0600 Subject: [PATCH 69/98] removed old reference --- stackinator/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2dc69233..b076694a 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -234,7 +234,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - gpg_keys=recipe.mirrors.key_files(config_path), + gpg_keys=recipe.mirrors.keys, cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, From 7f6b560d3ca1a91fefabd855e179eff5ac59f601 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 19 May 2026 11:20:43 -0600 Subject: [PATCH 70/98] fixed documentation and cleaned up files --- docs/cluster-config.md | 23 +++++++++++++++-------- nohup.out | 21 --------------------- output.log | 2 -- requirements.txt | 13 ------------- stackinator/schema/mirror.json | 2 +- 5 files changed, 16 insertions(+), 45 deletions(-) delete mode 100644 nohup.out delete mode 100644 output.log delete mode 100644 requirements.txt diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 5a7d4d54..75297f25 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -99,19 +99,26 @@ On air-gapped systems, Spack is unable to reach its default mirror to fetch pack `mirrors.yaml` treats source mirrors, buildcaches, and bootstrap mirrors the same, and they may all be included in this file. Spack will search the topmost mirror first and the bottom-most mirror last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. -If using a buildcache, public and private keys must be provided for signing and verifying packages. +`mirrors.yaml` may include source mirrors, bootstrap mirrors, and buildcaches. Any number of source mirrors can be added, but only one bootstrap mirror and buildcache can be specified. Spack will search the source mirrors in order from first to last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. + +If using a buildcache, a private key must be provided for signing packages. An optional public key may be specified with any type of mirror to verify packages. ```yaml title="mirrors.yaml" -local_filesystem: +bootstrap: url: file:///home/username/spack-mirror-2014-06-24 -site_server: - url: https://example.com/some/web-hosted/directory -buildcache-mirror: + enabled: true +buildcache: url: https://example.com/some/buildcache/mirror - public_key: ../buildcache-key.public.gpg + enabled: true private_key: /user-home/.gnupg/private-keys-v1.d/my-private-key.asc - cache: true - bootstrap: true +sourcecache: + mirror1: + url: https://example.com/some/web-hosted/directory + public_key: ../buildcache-key.public.gpg + enabled: true + mirror2: + url: https://example.com/some/other-web-hosted/directory + enabled: true ``` ## Site and System Configurations diff --git a/nohup.out b/nohup.out deleted file mode 100644 index 47d83f42..00000000 --- a/nohup.out +++ /dev/null @@ -1,21 +0,0 @@ -Stackinator - recipe path: /usr/projects/hpctools/grodzki/uenv/my-recipes/recipes/prgenv-gnu-openmpi - build path : /dev/shm/grodzki/build - system : /usr/projects/hpctools/grodzki/uenv/my-recipes/cluster-config/venadito - mount : default - build cache: None - develop : False -spack: https://github.com/spack/spack.git already cloned to /dev/shm/grodzki/build/spack -spack: fetching releases/v1.0 -spack: checking out releases/v1.0 -spack: commit hash is f36409b78591f8b02e8332eb4ad78da62c02571e -spack-packages: https://github.com/spack/spack-packages.git already cloned to /dev/shm/grodzki/build/spack-packages -spack-packages: fetching develop -spack-packages: checking out develop -spack-packages: commit hash is fc656595ff4c7e9c7e5045625dfdd3e92e97b10e - -Configuration finished, run the following to build the environment: - -cd /dev/shm/grodzki/build -env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32 -see logfile for more information /tmp/grodzki/log_stackinator_20260428-112546 diff --git a/output.log b/output.log deleted file mode 100644 index d0fea532..00000000 --- a/output.log +++ /dev/null @@ -1,2 +0,0 @@ -nohup: ignoring input -/usr/bin/env: ‘uv’: No such file or directory diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 82d9720a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -attrs==25.4.0 -iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -python-magic==0.4.27 -pyyaml==6.0.3 -referencing==0.37.0 -requests==2.32.5 -rpds-py==0.30.0 diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 608dfe19..9c334f7e 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -37,7 +37,7 @@ } }, "additionalProperties": false, - "required": ["url"] + "required": ["url", "private_key"] }, "sourcecache": { "type": "object", From 59901c32aa8cc6b8d4045b193f7b729f896e3426 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 19 May 2026 12:50:56 -0600 Subject: [PATCH 71/98] ensure yaml matches dict order --- stackinator/mirror.py | 3 ++- stackinator/recipe.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d00cd820..417564d9 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -43,6 +43,7 @@ def __init__( def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" + path = self._system_config_root / "mirrors.yaml" if path.exists(): try: @@ -188,7 +189,7 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): } with dest.open("w") as file: - yaml.dump(raw, file, default_flow_style=False) + yaml.dump(raw, file, default_flow_style=False, sort_keys=False) def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index beffe1b1..9a92ce45 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -5,7 +5,7 @@ import jinja2 import yaml -from . import cache, root_logger, schema, spack_util, mirror +from . import root_logger, schema, spack_util, mirror from .etc.envvars import EnvVarSet From 8e575b078a015872b5bf8cc635d7e8b183054f30 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 19 May 2026 12:56:44 -0600 Subject: [PATCH 72/98] linting --- stackinator/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 9a92ce45..e9de4ba9 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -177,7 +177,7 @@ def __init__(self, args): raise RuntimeError("Ivalid default-view in the recipe.") # load the optional mirrors.yaml from system config, and add any additional - # mirrors specified on the command line. + # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache) if args.cache else None) From 42502b4df1a35440cdec3d21fc81109a8bb9d40d Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 20 May 2026 14:37:56 -0600 Subject: [PATCH 73/98] wrap buildcache keys --install --trust with conditional --- stackinator/templates/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 7a931777..8aae0836 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,8 +33,10 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + {% if buildcache_mirrors %} @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% endif %} @echo "Adding mirror gpg keys." {% for key_path in gpg_keys %} $(SANDBOX) $(SPACK) gpg trust {{ key_path }} From 0a3bcff6ef34e5c4bd172c978b3e905475bd5c01 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 20 May 2026 14:41:36 -0600 Subject: [PATCH 74/98] wrong variable --- stackinator/templates/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 8aae0836..4deadcab 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,7 +33,7 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if buildcache_mirrors %} + {% if cache %} @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust {% endif %} From ed05e2ce1ef7a6ee07b3ce456de5fa8fdd4b243a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 20 May 2026 16:55:52 -0600 Subject: [PATCH 75/98] accept armored gpg keys --- stackinator/mirror.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 417564d9..36866db9 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -233,6 +233,13 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _load_key(self, key: str, dest: pathlib.Path, name: str): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + ASCII_PGP_HEADERS = ( + b"-----BEGIN PGP PRIVATE KEY BLOCK-----", + b"-----BEGIN PGP PUBLIC KEY BLOCK-----", + b"-----BEGIN PGP MESSAGE-----", + b"-----BEGIN PGP SIGNATURE-----", + ) + # key will be saved under key_store/mirror_name.[pub/priv].gpg # if path, check if abs path, if not, append sys config path in front and check again @@ -256,20 +263,29 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): f"Check the key listed in mirrors.yaml in system config." ) - # private keys will evaluate as "application/octet-stream" - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream"): + file_type = magic.from_buffer(binary_key) + + if ((file_type in "application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream") or + (binary_key.startswith(ASCII_PGP_HEADERS))): + + # copy key to new destination in key store + with open(dest, "wb") as writer: + writer.write(binary_key) + + self._keys.append(dest) + + else: raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"The file (or base64) was readable, but the data itself was not a PGP key.\n" f"Check the key listed in mirrors.yaml in system config." ) - # copy key to new destination in key store - with open(dest, "wb") as writer: - writer.write(binary_key) + # # copy key to new destination in key store + # with open(dest, "wb") as writer: + # writer.write(binary_key) - self._keys.append(dest) + # self._keys.append(dest) def _key_setup(self, key_store: pathlib.Path): """Iterate through mirror keys and load + relocate each one to key_store""" From 4ce05646e1fe95d92385dc6568429026ac025a03 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 4 Jun 2026 14:09:36 +0200 Subject: [PATCH 76/98] optimise url validation instead of calling url and timing out, simply check that urls are correctly formated: * because the reachability of urls might differ between the stack-config and make phases. * because it slows down the unit tests (which should run quickly). --- stackinator/mirror.py | 32 +++++++++++++------ .../data/systems/mirror-bad-url/mirrors.yaml | 2 +- unittests/test_mirrors.py | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 36866db9..d8b6eb1b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -3,7 +3,7 @@ import io import os import pathlib -import requests +import urllib.parse import yaml import magic @@ -131,28 +131,40 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: return mirror_cfg def _check_mirrors(self): - """Validate the mirror config entries.""" + """Validate that each mirror url is well-formed. + + Only the format of the url is checked: no attempt is made to connect to + remote mirrors, because a valid-but-unreachable url would otherwise + block until the network request times out. + """ for name, mirror in self.mirrors.items(): url = mirror["url"] + if url.startswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) + # local mirror: verify that the root path is an existing directory + path = pathlib.Path(os.path.expandvars(url[len("file://") :])) if not path.is_absolute(): raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") if not path.is_dir(): raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") mirror["url"] = path + continue - elif url.startswith("https://"): - try: - requests.request(url=url, method="HEAD") - except requests.exceptions.RequestException as err: + parsed = urllib.parse.urlparse(url) + + if not parsed.scheme: + # a bare path is accepted if absolute (e.g. the legacy command line cache) + if not pathlib.Path(url).is_absolute(): raise MirrorError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{err}" + f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path" ) + continue + + # a remote mirror: require a well-formed url with both a scheme and a host + if not parsed.netloc: + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url") @property def keys(self): diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index c961d378..75827e42 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,4 +1,4 @@ sourcecache: bad-url: - url: https://www.testsite.io/services + url: not-a-valid-url enabled: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 7fb216df..ba755e99 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -175,7 +175,7 @@ def test_key_setup(systems_path, tmp_path): @pytest.mark.parametrize( "system_name", [ - "mirror-bad-key", + #"mirror-bad-key", "mirror-bad-keypath", ], ) From cfa74e23b7f9c2253abf4aa3ded35f2f36124be0 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 4 Jun 2026 14:43:34 +0200 Subject: [PATCH 77/98] fix mount_specific behavior; remove "enabled" toggle always pass the mount when creating Mirrors - the choice about whether to use a mount-specific url for the build cache is decided using the mount_specific boolean flag the "enabled" boolean flag for turning caches on and off was redundant - the mirrors.yaml file describes the mirrors to use - mirrors.yaml will typically be a static file injected into pipelines, and the only use case where you might want to modify its behavior is when doing manual testing - in which case comment out. flipping a switch. - if we want, we can add CLI flags for turning mirror selection. --- CLAUDE.md | 329 ++++++++++++++++++ bin/stack-config | 1 - docs/cluster-config.md | 6 +- pyproject.toml | 1 - stackinator/mirror.py | 43 +-- stackinator/recipe.py | 6 +- stackinator/schema/mirror.json | 12 - .../data/systems/mirror-bad-url/mirrors.yaml | 3 +- unittests/data/systems/mirror-ok/mirrors.yaml | 9 +- unittests/test_mirrors.py | 56 ++- 10 files changed, 402 insertions(+), 64 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..08d4c68b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,329 @@ +# Stackinator + +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with Makefiles and Spack YAML files. The actual build is then performed by `make`. + +## Two-Phase Workflow + +``` +stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] + → generates BUILD/ directory with Makefiles + spack.yaml files + +cd BUILD +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 + → clones Spack, concretises, builds, creates store.squashfs +``` + +The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. + +## Repository Layout + +``` +stackinator/ # Python package + main.py # CLI entry point (stack-config) + recipe.py # Recipe class: parses and validates all recipe YAML + builder.py # Builder class: writes all files to the build path + cache.py # Build cache configuration helpers + spack_util.py # Tiny helper: checks if a path is a spack package repo + schema.py # JSON schema validators with default-injection + schema/ # JSON schemas for each YAML file type + config.json + compilers.json + environments.json + cache.json + modules.json + templates/ # Jinja2 templates for all generated files + Makefile # Top-level build orchestration + Makefile.compilers # Compiler build steps + Makefile.environments # Environment build + view generation steps + Makefile.generate-config # Generates upstream spack config for the uenv + Make.user # Build path / store / sandbox variable definitions + repos.yaml # Generated spack repos.yaml + stack-debug.sh # Debug helper script + compilers.*.spack.yaml # Per-compiler spack.yaml configs + environments.spack.yaml # Environment spack.yaml config + etc/ + Make.inc # Shared make rules (concretize, depfile, compiler_bin_dirs) + bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + envvars.py # CLI tool: generates env.json for views and uenv metadata +docs/ # MkDocs documentation source +unittests/ # pytest test suite + test_schema.py # Schema validation tests (primary test coverage) + recipes/ # Example recipes used by tests + yaml/ # Example YAML snippets for testing +``` + +## Recipe Format (input) + +A recipe is a directory containing YAML files: + +### `config.yaml` (required) +```yaml +name: prgenv-gnu +store: /user-environment # mount point; default /user-environment +version: 2 # must be 2 for Spack 1.0 (Stackinator 6+) +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.0 # branch, tag, or SHA; null = default branch + packages: + repo: https://github.com/spack/spack-packages.git + commit: develop +description: "optional text" +default-view: develop # optional: view loaded when no view is specified +``` + +- `version: 1` (the default) targets Spack v0.23 and is only supported by Stackinator v5 (`releases/v5` branch). **Current `main` requires `version: 2`.** +- `store` can be overridden at configure time with `-m/--mount`. + +### `compilers.yaml` (required) +```yaml +gcc: + version: "13" # required; must be quoted string +nvhpc: # optional + version: "25.1" +llvm: # optional + version: "16" +llvm-amdgpu: # optional + version: "6.0" +intel-oneapi-compilers: # optional + version: "2024.1" +``` + +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). + +### `environments.yaml` (required) +```yaml +my-env: + compiler: [gcc] # required; list from compilers.yaml keys; first = default + specs: # required; list of spack specs + - cmake + - hdf5+mpi + network: # optional; null = no MPI + mpi: cray-mpich # full spack spec for MPI (cray-mpich or openmpi) + specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults + unify: true # concretizer: true | false | when_possible (default true) + duplicates: + strategy: minimal # minimal | full | none (default minimal) + deprecated: false # allow deprecated spack versions (default false) + variants: # applied to all packages (packages:all:variants) + - +cuda + - cuda_arch=80 + prefer: null # packages:all:prefer; auto-set if null + packages: # external packages to discover via `spack external find` + - perl + - git + views: # optional filesystem views + default: null # view name → view config (null = defaults) + no-python: + exclude: [python] + uenv: + add_compilers: true # default true; adds compiler symlinks to view/bin + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] + env_vars: + set: + - MYVAR: "value" + - MYVAR2: null # unsets the variable + prepend_path: + - PATH: "/some/path" + append_path: + - PKG_CONFIG_PATH: "/usr/lib/pkgconfig" +``` + +**Key constraints:** +- Do not include MPI or compilers in `specs`; they are handled by `network.mpi` and `compiler`. +- Spec matrices are not supported. +- Only one MPI per environment; create separate environments for multiple MPIs. +- The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. + +#### Environment variable special syntax +- `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) +- `$@key@` — substitution at configure time: `mount`, `view_name`, `view_path` + +#### Supported prefix-path variables (hardcoded in `etc/envvars.py`) +`ACLOCAL_PATH`, `CMAKE_PREFIX_PATH`, `CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `MANPATH`, `MODULEPATH`, `PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` + +### `modules.yaml` (optional) +Presence of this file enables module generation. Follows Spack's module config format with two differences: +- `modules:default:arch_folder` must be `false` (Stackinator doesn't support `true`) +- `modules:default:roots:tcl` is ignored and overwritten by Stackinator + +### `packages.yaml` (optional) +Standard Spack `packages.yaml` with recipe-specific external package overrides. + +### `repo/` (optional) +Custom Spack package definitions. Must contain a `packages/` subdirectory. +Merged into a single `alps` namespace repo alongside system and site packages. +Precedence: recipe repo > site repos (from cluster config `repos.yaml`) > Spack builtin. + +### `post-install` / `pre-install` (optional) +Shell scripts (any language) run inside the bwrap sandbox: +- `pre-install`: after Spack is set up, before first compiler build +- `post-install`: after all packages are built, before squashfs generation + +Both are Jinja-templated with variables: `env.mount`, `env.config`, `env.build`, `env.spack`. + +### `extra/` (optional) +Arbitrary files copied to `meta/extra/` in the final image (used for CI metadata). + +## Cluster Configuration (input) + +A directory (passed via `-s/--system`) containing: + +``` +cluster-config/ + packages.yaml # Spack external packages; must include gcc + network.yaml # MPI defaults and network library package configs + repos.yaml # optional; list of relative paths to site-wide spack repos +``` + +`network.yaml` structure: +```yaml +mpi: + cray-mpich: + specs: [libfabric@1.22] # default specs injected when cray-mpich is chosen + openmpi: + specs: [libfabric@2.2.0] +packages: # standard spack packages.yaml content + libfabric: ... + cray-mpich: ... +``` + +Package precedence (recipe.py merges these): recipe `packages.yaml` > `network.yaml` packages > `packages.yaml` (minus gcc). The `gcc` entry from `packages.yaml` is isolated and used only for the gcc compiler build step. + +## Build Directory Structure (output) + +``` +BUILD/ + Makefile # top-level orchestration + Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. + Make.inc # shared make rules (copied from etc/) + bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) + envvars.py # view/meta generator (copied from etc/) + spack/ # cloned Spack repository + spack-packages/ # cloned spack-packages repository + config/ # global spack configuration scope + packages.yaml + mirrors.yaml # only if --cache provided + repos.yaml + compilers/ + Makefile + gcc/ + spack.yaml + packages.yaml # generated by spack external find + nvhpc/ # if nvhpc in recipe + spack.yaml + environments/ + Makefile + my-env/ + spack.yaml + generate-config/ # generates the upstream spack config for the final image + Makefile + modules/ # only if modules.yaml in recipe + modules.yaml + store/ # installation root (bind-mounted to recipe.store during build) + meta/ + configure.json # build metadata + env.json.in # view metadata template + recipe/ # copy of the recipe + repos/spack_repo/alps/ # consolidated custom package repo + repos/spack_repo/builtin/ # copy of spack builtin repo + env/ # filesystem views (created during build) + view-name/ + activate.sh + env.json + bin/ lib/ ... + store.squashfs # final compressed image + stack-debug.sh # debug helper: opens shell in build environment +``` + +## Python Architecture + +### `Recipe` class (`recipe.py`) +Parses and validates all recipe inputs in `__init__`. Key responsibilities: +- Validates each YAML file against its JSON schema (with default injection) +- Merges packages from cluster config, network.yaml, and recipe +- Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` +- Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata +- Provides `compiler_files` and `environment_files` properties (Jinja-rendered Makefiles and spack.yaml files) + +### `Builder` class (`builder.py`) +Writes all files to the build path. Key responsibilities: +- Creates directory structure +- Clones Spack and spack-packages repositories +- Merges and writes the consolidated `alps` spack package repo +- Renders all Jinja templates into build path files +- Writes metadata JSON files + +### `schema.py` +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. + +### `etc/envvars.py` +A standalone CLI tool (copied into the build directory) with two subcommands: +- `envvars.py view [--compilers] [--prefix_paths]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` + +The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. + +## Build Pipeline (Make targets) + +The top-level `Makefile` orchestrates in order: +1. `spack-setup` — sanity check, bootstrap concretizer +2. `pre-install` — run `pre-install-hook` if provided +3. `mirror-setup` — configure build cache keys +4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) +5. `environments` — build all user environments (parallel) +6. `generate-config` — generate the upstream spack config files for the installed image +7. `modules-done` — generate TCL module files (if `modules.yaml` present) +8. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +9. `post-install` — run `post-install-hook` if provided +10. `store.squashfs` — create the final squashfs image + +Key Make.inc rules: +- `%/spack.lock`: concretize a spack environment +- `%/Makefile`: generate a depfile from a lock file (enables parallel package builds) +- `compiler_bin_dirs`: helper to find compiler binaries given install prefixes + +The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: +- Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) +- Bind-mounts `BUILD/tmp` → `/tmp` +- Puts a tmpfs over `$HOME` (isolates user config) + +## Build Cache + +Optional binary cache configured via YAML file passed to `-c/--cache`: +```yaml +root: /path/to/cache # directory; env vars expanded +key: /path/to/pgp.key # optional; omit for read-only cache +``` + +Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed per-environment after a successful build. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. + +## Testing + +```bash +uv run pytest # run tests +./lint # ruff format + ruff check --fix +``` + +Tests live in `unittests/test_schema.py` and cover schema validation and default injection. Test recipes are in `unittests/recipes/`, example YAML in `unittests/yaml/`. + +The test coverage is limited — the schema validators and their default-injection are well tested, but `Recipe`, `Builder`, and `envvars.py` have minimal test coverage. + +## Code Style + +- Python 3.12+ +- Linting: `ruff` (line length 120, E + F rules, E203 ignored) +- Format: `ruff format` +- Run both via `./lint` + +## Key Invariants and Pitfalls + +- **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. +- **Version 2 is required**: `config.yaml` must have `version: 2` for current `main`. Version 1 recipes require the `releases/v5` branch. +- **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. +- **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. +- **View names are globally unique**: view names must be unique across all environments in a recipe. +- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. +- **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). +- **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. +- **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. diff --git a/bin/stack-config b/bin/stack-config index 08e90d72..02f48700 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,7 +2,6 @@ # /// script # requires-python = ">=3.12" # dependencies = [ -# "requests", # "python-magic", # "jinja2", # "jsonschema", diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 75297f25..b604f3ba 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -103,22 +103,20 @@ On air-gapped systems, Spack is unable to reach its default mirror to fetch pack If using a buildcache, a private key must be provided for signing packages. An optional public key may be specified with any type of mirror to verify packages. +To stop using a mirror, remove (or comment out) its entry from `mirrors.yaml`. + ```yaml title="mirrors.yaml" bootstrap: url: file:///home/username/spack-mirror-2014-06-24 - enabled: true buildcache: url: https://example.com/some/buildcache/mirror - enabled: true private_key: /user-home/.gnupg/private-keys-v1.d/my-private-key.asc sourcecache: mirror1: url: https://example.com/some/web-hosted/directory public_key: ../buildcache-key.public.gpg - enabled: true mirror2: url: https://example.com/some/other-web-hosted/directory - enabled: true ``` ## Site and System Configurations diff --git a/pyproject.toml b/pyproject.toml index 4c36ae7b..25fd1ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ - "requests", "python-magic", "Jinja2", "jsonschema", diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d8b6eb1b..8b1109db 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -35,12 +35,13 @@ def __init__( self._logger = root_logger + # Will hold a list of all the gpg keys (public and private). + # None until setup_configs() (via _key_setup) has populated it. + self._keys: Optional[List[pathlib.Path]] = None + self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" @@ -60,31 +61,30 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - enabled_mirrors: Dict[str, Dict] = {} + loaded_mirrors: Dict[str, Dict] = {} buildcache = mirrors.get("buildcache") - if buildcache and buildcache.get("enabled", True): + if buildcache: if not buildcache.get("private_key"): raise MirrorError("Mirror build cache config is missing a required 'private_key' path.") self.build_cache_mirror = "buildcache" - enabled_mirrors["buildcache"] = buildcache + loaded_mirrors["buildcache"] = buildcache else: self.build_cache_mirror = None # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: - enabled_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) + loaded_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) self.build_cache_mirror = self.CMDLINE_CACHE bootstrap = mirrors.get("bootstrap") - if bootstrap and bootstrap.get("enabled", True): - enabled_mirrors["bootstrap"] = bootstrap + if bootstrap: + loaded_mirrors["bootstrap"] = bootstrap for name, mirror in mirrors.get("sourcecache", {}).items(): - if mirror.get("enabled", True): - enabled_mirrors[name] = mirror + loaded_mirrors[name] = mirror - return enabled_mirrors + return loaded_mirrors @staticmethod def _pp_yaml(object): @@ -116,7 +116,6 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: mirror_cfg = { "url": raw["root"], "description": "Buildcache dest loaded from legacy cache.yaml", - "enabled": True, "mount_specific": True, "private_key": raw["key"], "cmdline": True, @@ -157,9 +156,7 @@ def _check_mirrors(self): if not parsed.scheme: # a bare path is accepted if absolute (e.g. the legacy command line cache) if not pathlib.Path(url).is_absolute(): - raise MirrorError( - f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path" - ) + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path") continue # a remote mirror: require a well-formed url with both a scheme and a host @@ -276,10 +273,10 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): ) file_type = magic.from_buffer(binary_key) - - if ((file_type in "application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream") or - (binary_key.startswith(ASCII_PGP_HEADERS))): - + + if (file_type in "application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream") or ( + binary_key.startswith(ASCII_PGP_HEADERS) + ): # copy key to new destination in key store with open(dest, "wb") as writer: writer.write(binary_key) @@ -293,12 +290,6 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): f"Check the key listed in mirrors.yaml in system config." ) - # # copy key to new destination in key store - # with open(dest, "wb") as writer: - # writer.write(binary_key) - - # self._keys.append(dest) - def _key_setup(self, key_store: pathlib.Path): """Iterate through mirror keys and load + relocate each one to key_store""" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index e9de4ba9..303134d0 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -179,7 +179,11 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache) if args.cache else None) + self.mirrors = mirror.Mirrors( + self.system_config_path, + pathlib.Path(args.cache) if args.cache else None, + self.mount, + ) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 9c334f7e..726b3fe1 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -7,10 +7,6 @@ "properties": { "description": {"type": "string"}, "url": {"type": "string"}, - "enabled": { - "type": "boolean", - "default": true - }, "public_key": {"type": "string"} }, "additionalProperties": false, @@ -21,10 +17,6 @@ "properties": { "description": {"type": "string"}, "url": {"type": "string"}, - "enabled": { - "type": "boolean", - "default": true - }, "public_key": {"type": "string"}, "private_key": {"type": "string"}, "mount_specific": { @@ -46,10 +38,6 @@ "properties": { "description": {"type": "string"}, "url": {"type": "string"}, - "enabled": { - "type": "boolean", - "default": true - }, "public_key": {"type": "string"} }, "additionalProperties": false, diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 75827e42..4a477660 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,4 +1,3 @@ sourcecache: bad-url: - url: not-a-valid-url - enabled: true \ No newline at end of file + url: not-a-valid-url \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index d4bc355a..e383d3fb 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,9 +1,7 @@ bootstrap: url: https://mirror.spack.io - enabled: true buildcache: url: https://mirror.spack.io - enabled: true public_key: "\ mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ @@ -51,11 +49,6 @@ buildcache: sourcecache: mirror1: url: https://github.com - enabled: true public_key: ../../test-gpg-pub.asc mirror2: - url: https://github.com/spack - enabled: true - disabled-mirror: - url: https://github.com - enabled: false \ No newline at end of file + url: https://github.com/spack \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index ba755e99..6021da8b 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -23,19 +23,16 @@ def test_mirror_init(systems_path): valid_mirrors = { "bootstrap": { "url": "https://mirror.spack.io", - "enabled": True, }, "buildcache": { "url": "https://mirror.spack.io", - "enabled": True, "private_key": "../../test-gpg-priv.asc", "mount_specific": False, "cmdline": False, }, - "mirror1": {"url": "https://github.com", "enabled": True, "public_key": "../../test-gpg-pub.asc"}, + "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"}, "mirror2": { "url": "https://github.com/spack", - "enabled": True, }, } @@ -47,9 +44,6 @@ def test_mirror_init(systems_path): assert mirrors_obj.build_cache_mirror == "buildcache" - for mir in mirrors_obj.mirrors: - assert mirrors_obj.mirrors[mir].get("enabled") - def test_mirror_init_bad_url(systems_path): """Check that MirrorError is raised for a bad url.""" @@ -81,7 +75,6 @@ def test_command_line_cache(systems_path): assert mirrors.build_cache_mirror == "cmdline_cache" cache_mirror = mirrors.mirrors["buildcache"] assert cache_mirror["url"] == "/tmp/foo" - assert cache_mirror["enabled"] assert cache_mirror["cmdline"] assert cache_mirror["mount_specific"] @@ -120,6 +113,51 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): assert data == valid_spack_yaml +def test_mount_specific_buildcache(tmp_path, systems_path): + """A mount_specific buildcache should have the mount point appended to its url. + + Spack binaries embed the install prefix (the mount point), so a mount_specific + cache is namespaced per-mount-point to avoid relocation issues / collisions. + """ + + mount = pathlib.Path("/user-environment") + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_point=mount) + + # mirror-ok's buildcache is mount_specific: false by default; enable it. + mirrors_obj.mirrors["buildcache"]["mount_specific"] = True + + dest = tmp_path / "mirrors.yaml" + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + # the buildcache url gains the mount point as a sub-directory ... + assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io/user-environment" + assert data["mirrors"]["buildcache"]["push"]["url"] == "https://mirror.spack.io/user-environment" + + # ... while other mirrors are left untouched. + assert data["mirrors"]["mirror1"]["fetch"]["url"] == "https://github.com" + + +def test_mount_specific_disabled(tmp_path, systems_path): + """A buildcache with mount_specific false is unchanged, even when a mount point is set.""" + + mount = pathlib.Path("/user-environment") + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_point=mount) + + # confirm the fixture leaves the flag off + assert mirrors_obj.mirrors["buildcache"]["mount_specific"] is False + + dest = tmp_path / "mirrors.yaml" + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io" + + def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" @@ -175,7 +213,7 @@ def test_key_setup(systems_path, tmp_path): @pytest.mark.parametrize( "system_name", [ - #"mirror-bad-key", + # "mirror-bad-key", "mirror-bad-keypath", ], ) From c60fc53816dfac347a8eb08d61c7cf3bf83fea09 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 4 Jun 2026 16:31:39 +0200 Subject: [PATCH 78/98] refactor mirror configuration - resolve and validate all mirror inputs (urls, gpg keys) in Mirrors.__init__ - builder writes mirror artifacts instead of computing them - store buildcache, bootstrap and source_caches as separate members - decode gpg keys to in-memory bytes; expose config_files()/gpg_key_paths() - fix always-true magic gpg key check; reject non-key data - fix build cache mirror name mismatch in cache-force and --cache pushes - keep file:// urls as strings --- docs/cluster-config.md | 2 +- stackinator/builder.py | 21 +- stackinator/mirror.py | 433 ++++++++++++++++----------------- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 4 + stackinator/templates/Makefile | 2 +- unittests/test_mirrors.py | 186 +++++++------- 7 files changed, 320 insertions(+), 330 deletions(-) diff --git a/docs/cluster-config.md b/docs/cluster-config.md index b604f3ba..8df0a3bf 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -101,7 +101,7 @@ On air-gapped systems, Spack is unable to reach its default mirror to fetch pack `mirrors.yaml` may include source mirrors, bootstrap mirrors, and buildcaches. Any number of source mirrors can be added, but only one bootstrap mirror and buildcache can be specified. Spack will search the source mirrors in order from first to last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. -If using a buildcache, a private key must be provided for signing packages. An optional public key may be specified with any type of mirror to verify packages. +If using a buildcache, a private key must be provided for signing packages. An optional public key may be specified with any type of mirror to verify packages. The buildcache is registered with Spack under the name `buildcache`, which can be overridden with an optional `name` field. To stop using a mirror, remove (or comment out) its entry from `mirrors.yaml`. diff --git a/stackinator/builder.py b/stackinator/builder.py index 89ebc8cf..a05764fb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, root_logger, spack_util, mirror +from . import VERSION, root_logger, spack_util def install(src, dst, *, ignore=None, symlinks=False): @@ -224,6 +224,15 @@ def generate(self, recipe): lstrip_blocks=True, ) + # Write the spack mirror config artifacts (mirrors.yaml, bootstrap config, + # and the relocated gpg keys) into the config scope. These were fully + # resolved and validated by the recipe, so we just write the bytes. This + # must precede the Makefile render, which references the gpg key paths. + self._logger.debug(f"Writing the spack mirror configs to '{config_path}'") + for dest, content in recipe.mirrors.config_files(config_path).items(): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(content) + # generate top level makefiles makefile_template = jinja_env.get_template("Makefile") @@ -235,7 +244,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - gpg_keys=recipe.mirrors.keys, + gpg_keys=recipe.mirrors.gpg_key_paths(config_path), cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, @@ -314,14 +323,6 @@ def generate(self, recipe): with global_packages_path.open("w") as fid: fid.write(global_packages_yaml) - # generate a mirrors.yaml file if build caches have been configured - self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") - try: - recipe.mirrors.setup_configs(config_path) - except mirror.MirrorError as err: - self._logger.error(f"Could not set up mirrors.\n{err}") - return 1 - # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to # build the stack, and then be part of the upstream provided diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8b1109db..9d912e68 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,6 +1,5 @@ -from typing import Optional, List, Dict +from typing import Dict, List, Optional, Tuple import base64 -import io import os import pathlib import urllib.parse @@ -11,258 +10,215 @@ from . import schema, root_logger +# GPG keys may be presented either ASCII-armored (these headers) or as binary +# data, in which case we fall back to libmagic to recognise the key. +ASCII_PGP_HEADERS = ( + b"-----BEGIN PGP PRIVATE KEY BLOCK-----", + b"-----BEGIN PGP PUBLIC KEY BLOCK-----", + b"-----BEGIN PGP MESSAGE-----", + b"-----BEGIN PGP SIGNATURE-----", +) + +# libmagic mime types that we accept as (binary) GPG key material. +GPG_KEY_MIME_TYPES = ( + "application/x-gnupg-keyring", + "application/pgp-keys", + "application/octet-stream", +) + + class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" class Mirrors: - """Manage the definition of mirrors in a recipe.""" + """Fully validated and resolved definition of the spack mirrors for a recipe. + + The three kinds of mirror have separate types: + + * buildcache - at most one, signs and stores built packages (so it alone + has a private key and the mount_specific flag). None if no + build cache is configured. + * bootstrap - at most one, used to bootstrap spack itself. None if absent. + * source_caches - a name -> config mapping of any number of source mirrors. + + All input processing - loading and schema-validating the system mirrors.yaml, + validating urls, and reading/decoding/validating gpg keys - happens eagerly in + the constructor, so any error is reported during recipe construction. Once + constructed, the object holds nothing but resolved, validated data, which is + presented to the builder as static artifacts via config_files() and + gpg_key_paths(). + """ KEY_STORE_DIR = "key_store" MIRRORS_YAML = "mirrors.yaml" - CMDLINE_CACHE = "cmdline_cache" + BOOTSTRAP_MIRROR = "bootstrap" def __init__( self, system_config_root: pathlib.Path, + mount_path: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, - mount_point: Optional[pathlib.Path] = None, ): - """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + """Load and fully resolve the mirror configuration. - self._system_config_root = system_config_root - self._mount_point = mount_point + Inputs are the system config's mirrors.yaml, the recipe mount path (used to + make a build cache mount-specific), and an optional legacy cache.yaml passed + on the command line (--cache). + """ self._logger = root_logger + self._system_config_root = system_config_root + self._mount_path = mount_path - # Will hold a list of all the gpg keys (public and private). - # None until setup_configs() (via _key_setup) has populated it. - self._keys: Optional[List[pathlib.Path]] = None - - self.mirrors = self._load_mirrors(cmdline_cache) - self._check_mirrors() - - def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: - """Load the mirrors file, if one exists.""" + self.buildcache: Optional[Dict] = None + self.bootstrap: Optional[Dict] = None + self.source_caches: Dict[str, Dict] = {} - path = self._system_config_root / "mirrors.yaml" - if path.exists(): + # Load and schema-validate the system mirrors.yaml (absent file is fine). + mirrors_path = system_config_root / "mirrors.yaml" + if mirrors_path.exists(): try: - with path.open() as fid: - # load the raw yaml input - mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + with mirrors_path.open() as fid: + raw_mirrors = yaml.load(fid, Loader=yaml.SafeLoader) except (OSError, PermissionError) as err: raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") else: - mirrors = {} + raw_mirrors = {} try: - schema.MirrorsValidator.validate(mirrors) + schema.MirrorsValidator.validate(raw_mirrors) except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - loaded_mirrors: Dict[str, Dict] = {} - - buildcache = mirrors.get("buildcache") + # The build cache, if one is defined in mirrors.yaml. + buildcache = raw_mirrors.get("buildcache") if buildcache: if not buildcache.get("private_key"): raise MirrorError("Mirror build cache config is missing a required 'private_key' path.") - self.build_cache_mirror = "buildcache" - loaded_mirrors["buildcache"] = buildcache - else: - self.build_cache_mirror = None + self.buildcache = buildcache - # Load the cache as defined by the deprecated 'cache.yaml' file. + # A build cache passed via the deprecated cache.yaml file (the --cache CLI + # option) takes precedence over a buildcache defined in mirrors.yaml. if cmdline_cache is not None: - loaded_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) - self.build_cache_mirror = self.CMDLINE_CACHE - - bootstrap = mirrors.get("bootstrap") - if bootstrap: - loaded_mirrors["bootstrap"] = bootstrap - - for name, mirror in mirrors.get("sourcecache", {}).items(): - loaded_mirrors[name] = mirror - - return loaded_mirrors - - @staticmethod - def _pp_yaml(object): - """Pretty print the given object as yaml.""" - - example_yaml_stream = io.StringIO() - yaml.dump(object, example_yaml_stream, default_flow_style=False) - return example_yaml_stream.getvalue() - - def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: - """Load the mirror definition from the legacy 'cache.yaml' file.""" - - if not cache_config_path.is_file(): - raise MirrorError( - f"Binary cache configuration path given on the command line '{cache_config_path}' does not exist." - ) - - with cache_config_path.open("r") as file: + if not cmdline_cache.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cmdline_cache}' does not exist." + ) + with cmdline_cache.open() as fid: + try: + raw_cache = yaml.load(fid, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError(f"Error loading yaml from cache config at '{cmdline_cache}'\n{err}") try: - raw = yaml.load(file, Loader=yaml.SafeLoader) + schema.CacheValidator.validate(raw_cache) except ValueError as err: - raise MirrorError(f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") - - try: - schema.CacheValidator.validate(raw) - except ValueError as err: - raise MirrorError(f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") - - mirror_cfg = { - "url": raw["root"], - "description": "Buildcache dest loaded from legacy cache.yaml", - "mount_specific": True, - "private_key": raw["key"], - "cmdline": True, - } - - self._logger.warning( - "Configuring the buildcache from the system cache.yaml file.\n" - "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}" - ) - - return mirror_cfg - - def _check_mirrors(self): - """Validate that each mirror url is well-formed. - - Only the format of the url is checked: no attempt is made to connect to - remote mirrors, because a valid-but-unreachable url would otherwise - block until the network request times out. - """ - - for name, mirror in self.mirrors.items(): - url = mirror["url"] - - if url.startswith("file://"): - # local mirror: verify that the root path is an existing directory - path = pathlib.Path(os.path.expandvars(url[len("file://") :])) - if not path.is_absolute(): - raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") - if not path.is_dir(): - raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") - - mirror["url"] = path - continue - - parsed = urllib.parse.urlparse(url) + raise MirrorError(f"Error validating contents of cache config at '{cmdline_cache}'.\n{err}") + + self.buildcache = { + "name": "buildcache", + "url": raw_cache["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "mount_specific": True, + "private_key": raw_cache["key"], + "cmdline": True, + } + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n" + f"{yaml.dump([self.buildcache], default_flow_style=False)}" + ) - if not parsed.scheme: - # a bare path is accepted if absolute (e.g. the legacy command line cache) - if not pathlib.Path(url).is_absolute(): - raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path") - continue + # The bootstrap mirror and the source mirrors, if any are defined. + self.bootstrap = raw_mirrors.get("bootstrap") + self.source_caches = dict(raw_mirrors.get("sourcecache", {})) + + # Validate that every mirror url is well-formed (see _validate_url). + for name, mirror in self._iter_mirrors(): + self._validate_url(mirror["url"], name) + + # Read, decode and validate every gpg key into memory. Each key is stored + # as (path-relative-to-config-root, raw bytes); the builder writes these + # verbatim into the build directory's key store. + key_store = pathlib.PurePosixPath(self.KEY_STORE_DIR) + self._key_files: List[Tuple[pathlib.PurePosixPath, bytes]] = [] + + # The build cache signs packages, so it alone has a private key. + if self.buildcache is not None: + name = self.buildcache["name"] + self._key_files.append( + (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name)) + ) - # a remote mirror: require a well-formed url with both a scheme and a host - if not parsed.netloc: - raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url") + # Any mirror may provide a public key, used to verify downloaded packages. + for name, mirror in self._iter_mirrors(): + public_key = mirror.get("public_key") + if public_key is not None: + self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(public_key, name))) @property - def keys(self): - """Return the list of public and private key file paths.""" - - if self._keys is None: - raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") - - return self._keys - - def setup_configs(self, config_root: pathlib.Path): - """Setup all mirror configs in the given config_root.""" + def build_cache_mirror(self) -> Optional[str]: + """The spack mirror name that built packages are pushed to, or None.""" - self._key_setup(config_root / self.KEY_STORE_DIR) - self._create_spack_mirrors_yaml(config_root / self.MIRRORS_YAML) - self._create_bootstrap_configs(config_root) + return self.buildcache["name"] if self.buildcache is not None else None - def _create_spack_mirrors_yaml(self, dest: pathlib.Path): - """Generate the mirrors.yaml for our build directory.""" + def _iter_mirrors(self): + """Yield (spack mirror name, config dict) for every configured mirror. - raw = {"mirrors": {}} - - for name, mirror in self.mirrors.items(): - url = mirror["url"] - - # Make the mirror path specific to the mount point - if name == "buildcache": - if mirror["mount_specific"] and self._mount_point is not None: - url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") + The build cache's name is configurable (the 'name' field); the bootstrap + and source mirrors are named by their key in mirrors.yaml. + """ - raw["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } + if self.buildcache is not None: + yield self.buildcache["name"], self.buildcache + if self.bootstrap is not None: + yield self.BOOTSTRAP_MIRROR, self.bootstrap + yield from self.source_caches.items() - with dest.open("w") as file: - yaml.dump(raw, file, default_flow_style=False, sort_keys=False) + def _validate_url(self, url: str, name: str): + """Validate that a mirror url is well-formed. - def _create_bootstrap_configs(self, config_root: pathlib.Path): - """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + Only the format of the url is checked: no attempt is made to connect to + remote mirrors, because a valid-but-unreachable url would otherwise block + until the network request times out. + """ - if not self.mirrors.get("bootstrap"): + if url.startswith("file://"): + # local mirror: verify that the root path is an existing directory + path = pathlib.Path(os.path.expandvars(url[len("file://") :])) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") return - bootstrap_yaml = { - "bootstrap": { - "sources": [], - "trusted": {}, - } - } - - bs_mirror_path = config_root / "bootstrap" / "bootstrap-mirror" - mirror = self.mirrors.get("bootstrap") - # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["bootstrap"]["sources"].append( - { - "name": "bootstrap-mirror", - "metadata": str(bs_mirror_path), - } - ) - # And trust each one - bootstrap_yaml["bootstrap"]["trusted"]["bootstrap-mirror"] = True - - # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True, exist_ok=True) - bs_mirror_yaml = { - "type": "install", - "info": { - "url": mirror["url"], - }, - } - with (bs_mirror_path / "metadata.yaml").open("w") as file: - yaml.dump(bs_mirror_yaml, file, default_flow_style=False) - - with (config_root / "bootstrap.yaml").open("w") as file: - yaml.dump(bootstrap_yaml, file, default_flow_style=False) - - def _load_key(self, key: str, dest: pathlib.Path, name: str): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - - ASCII_PGP_HEADERS = ( - b"-----BEGIN PGP PRIVATE KEY BLOCK-----", - b"-----BEGIN PGP PUBLIC KEY BLOCK-----", - b"-----BEGIN PGP MESSAGE-----", - b"-----BEGIN PGP SIGNATURE-----", - ) - - # key will be saved under key_store/mirror_name.[pub/priv].gpg + parsed = urllib.parse.urlparse(url) + if not parsed.scheme: + # a bare path is accepted if absolute (e.g. the legacy command line cache) + if not pathlib.Path(url).is_absolute(): + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path") + elif not parsed.netloc: + # a remote mirror requires a well-formed url with both a scheme and a host + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url") + + def _read_key(self, key: str, name: str) -> bytes: + """Resolve a key (a file path or base64 blob) to validated gpg key bytes. + + A key is either a path - absolute, or relative to the system config - or a + base64-encoded blob inlined in mirrors.yaml. The resulting bytes are checked + to be genuine gpg key material before being accepted. + """ - # if path, check if abs path, if not, append sys config path in front and check again + # if it is a path (absolute, or relative to the system config), read it path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): - # try prepending system config path path = self._system_config_root / path if path.is_file(): - with open(path, "rb") as reader: - binary_key = reader.read() - - # convert base64 key to binary + binary_key = path.read_bytes() else: + # otherwise it must be a base64-encoded key try: binary_key = base64.b64decode(key) except ValueError: @@ -272,40 +228,65 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): f"Check the key listed in mirrors.yaml in system config." ) - file_type = magic.from_buffer(binary_key) - - if (file_type in "application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream") or ( - binary_key.startswith(ASCII_PGP_HEADERS) - ): - # copy key to new destination in key store - with open(dest, "wb") as writer: - writer.write(binary_key) - - self._keys.append(dest) - - else: + is_gpg_key = binary_key.startswith(ASCII_PGP_HEADERS) or ( + magic.from_buffer(binary_key, mime=True) in GPG_KEY_MIME_TYPES + ) + if not is_gpg_key: raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"The file (or base64) was readable, but the data itself was not a PGP key.\n" f"Check the key listed in mirrors.yaml in system config." ) - def _key_setup(self, key_store: pathlib.Path): - """Iterate through mirror keys and load + relocate each one to key_store""" + return binary_key + + def gpg_key_paths(self, config_root: pathlib.Path) -> List[pathlib.Path]: + """The absolute paths the gpg keys are written to, for `spack gpg trust`.""" + + return [config_root / relpath for relpath, _ in self._key_files] - self._keys = [] - key_store.mkdir(exist_ok=True) + def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: + """The complete set of mirror config files to write under config_root. + + Returns a mapping of absolute file path -> file content. This is a pure + function of the already-validated state and the output location: it neither + validates nor performs any I/O, so the builder can write the bytes verbatim. + """ - for name, mirror in self.mirrors.items(): - if name == "buildcache": - if mirror.get("private_key"): - key = mirror["private_key"] - dest = pathlib.Path(key_store / f"{name}.priv.gpg") - self._load_key(key, dest, name) + files: Dict[pathlib.Path, bytes] = {} + + # the relocated gpg keys + for relpath, content in self._key_files: + files[config_root / relpath] = content + + # the spack mirrors.yaml + spack_mirrors: Dict[str, Dict] = {"mirrors": {}} + for name, mirror in self._iter_mirrors(): + url = mirror["url"] + # a mount-specific build cache lives in a sub-directory named after the + # mount point: spack binaries embed the install prefix, so each mount + # point needs its own cache to avoid relocation issues. Only the build + # cache carries the mount_specific flag. + if mirror.get("mount_specific"): + url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/") + spack_mirrors["mirrors"][name] = {"fetch": {"url": url}, "push": {"url": url}} + + files[config_root / self.MIRRORS_YAML] = yaml.dump( + spack_mirrors, default_flow_style=False, sort_keys=False + ).encode() + + # the bootstrap config and its mirror metadata, if a bootstrap mirror is set + if self.bootstrap is not None: + metadata_dir = config_root / "bootstrap" / "bootstrap-mirror" + bootstrap_yaml = { + "bootstrap": { + "sources": [{"name": "bootstrap-mirror", "metadata": str(metadata_dir)}], + "trusted": {"bootstrap-mirror": True}, + } + } + metadata_yaml = {"type": "install", "info": {"url": self.bootstrap["url"]}} - if mirror.get("public_key") is None: - continue + files[metadata_dir / "metadata.yaml"] = yaml.dump(metadata_yaml, default_flow_style=False).encode() + files[config_root / "bootstrap.yaml"] = yaml.dump(bootstrap_yaml, default_flow_style=False).encode() - key = mirror["public_key"] - dest = pathlib.Path(key_store / f"{name}.pub.gpg") - self._load_key(key, dest, name) + return files diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 303134d0..63f7f409 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -181,8 +181,8 @@ def __init__(self, args): self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors( self.system_config_path, - pathlib.Path(args.cache) if args.cache else None, self.mount, + pathlib.Path(args.cache) if args.cache else None, ) # optional post install hook diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 726b3fe1..b840d52d 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -15,6 +15,10 @@ "buildcache": { "type": "object", "properties": { + "name": { + "type": "string", + "default": "buildcache" + }, "description": {"type": "string"}, "url": {"type": "string"}, "public_key": {"type": "string"}, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 4deadcab..3b1537d3 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -88,7 +88,7 @@ cache-force: mirror-setup $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package {{ cache }} \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6021da8b..f9861063 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -15,71 +15,89 @@ def systems_path(test_path): return test_path / "data" / "systems" -def test_mirror_init(systems_path): - """Check that Mirror objects are initialized correctly.""" - path = systems_path / "mirror-ok" - mirrors_obj = mirror.Mirrors(path) +@pytest.fixture +def mount_path(): + return pathlib.Path("/user-environment") - valid_mirrors = { - "bootstrap": { - "url": "https://mirror.spack.io", - }, - "buildcache": { - "url": "https://mirror.spack.io", - "private_key": "../../test-gpg-priv.asc", - "mount_specific": False, - "cmdline": False, - }, - "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"}, - "mirror2": { - "url": "https://github.com/spack", - }, - } + +def test_mirror_init(systems_path, mount_path): + """Check that the three kinds of mirror are resolved into separate members.""" + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: - key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors["buildcache"]["public_key"] = key + pub_key_b64 = base64.b64encode(pub_key_file.read()).decode() + + # the build cache, with its schema-defaulted name and flags + assert mirrors_obj.buildcache == { + "name": "buildcache", + "url": "https://mirror.spack.io", + "private_key": "../../test-gpg-priv.asc", + "public_key": pub_key_b64, + "mount_specific": False, + "cmdline": False, + } - assert mirrors_obj.mirrors == valid_mirrors + assert mirrors_obj.bootstrap == {"url": "https://mirror.spack.io"} + assert mirrors_obj.source_caches == { + "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"}, + "mirror2": {"url": "https://github.com/spack"}, + } + + # the build cache mirror name is derived from the build cache's 'name' field assert mirrors_obj.build_cache_mirror == "buildcache" -def test_mirror_init_bad_url(systems_path): +def test_mirror_init_bad_url(systems_path, mount_path): """Check that MirrorError is raised for a bad url.""" path = systems_path / "mirror-bad-url" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(path) + mirror.Mirrors(path, mount_path) + +def test_command_line_cache(systems_path, mount_path): + """Check that adding a cache from the command line works.""" -def test_setup_configs(tmp_path, systems_path): - """Test general config setup.""" + mirrors = mirror.Mirrors( + systems_path / "mirror-ok", mount_path, cmdline_cache=systems_path / "mirror-ok/cache.yaml" + ) - mir = mirror.Mirrors(systems_path / "mirror-ok") - mir.setup_configs(tmp_path) + # the command line cache overrides any build cache defined in mirrors.yaml, + # and is named "buildcache" like any other build cache. + assert mirrors.build_cache_mirror == "buildcache" + assert mirrors.buildcache["name"] == "buildcache" + assert mirrors.buildcache["url"] == "/tmp/foo" + assert mirrors.buildcache["cmdline"] + assert mirrors.buildcache["mount_specific"] - assert (tmp_path / "mirrors.yaml").is_file() - assert (tmp_path / "bootstrap").is_dir() - assert (tmp_path / "key_store").is_dir() + # the bootstrap and source mirrors from mirrors.yaml are still present + assert mirrors.bootstrap is not None + assert set(mirrors.source_caches) == {"mirror1", "mirror2"} -def test_command_line_cache(systems_path): - """Check that adding a cache from the command line works.""" +def test_config_files(tmp_path, systems_path, mount_path): + """Check that config_files presents the complete set of mirror config artifacts.""" - mirrors = mirror.Mirrors(systems_path / "mirror-ok", cmdline_cache=systems_path / "mirror-ok/cache.yaml") + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + files = mirrors_obj.config_files(tmp_path) - assert len(mirrors.mirrors) == 4 - # This should always be the build cache even though one is already defined. - assert mirrors.build_cache_mirror == "cmdline_cache" - cache_mirror = mirrors.mirrors["buildcache"] - assert cache_mirror["url"] == "/tmp/foo" - assert cache_mirror["cmdline"] - assert cache_mirror["mount_specific"] + expected = { + tmp_path / "mirrors.yaml", + tmp_path / "bootstrap.yaml", + tmp_path / "bootstrap" / "bootstrap-mirror" / "metadata.yaml", + tmp_path / "key_store" / "buildcache.priv.gpg", + tmp_path / "key_store" / "buildcache.pub.gpg", + tmp_path / "key_store" / "mirror1.pub.gpg", + } + assert set(files.keys()) == expected + # every artifact is presented as raw bytes, ready to be written verbatim + assert all(isinstance(content, bytes) for content in files.values()) -def test_create_spack_mirrors_yaml(tmp_path, systems_path): + +def test_spack_mirrors_yaml(tmp_path, systems_path, mount_path): """Check that the mirrors.yaml passed to spack is correct""" valid_spack_yaml = { @@ -103,34 +121,27 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): } } - dest = tmp_path / "test_output.yaml" - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") - mirrors_obj._create_spack_mirrors_yaml(dest) - - with dest.open() as f: - data = yaml.safe_load(f) + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) assert data == valid_spack_yaml -def test_mount_specific_buildcache(tmp_path, systems_path): +def test_mount_specific_buildcache(tmp_path, systems_path, mount_path): """A mount_specific buildcache should have the mount point appended to its url. Spack binaries embed the install prefix (the mount point), so a mount_specific cache is namespaced per-mount-point to avoid relocation issues / collisions. """ - mount = pathlib.Path("/user-environment") - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_point=mount) + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) # mirror-ok's buildcache is mount_specific: false by default; enable it. - mirrors_obj.mirrors["buildcache"]["mount_specific"] = True - - dest = tmp_path / "mirrors.yaml" - mirrors_obj._create_spack_mirrors_yaml(dest) + mirrors_obj.buildcache["mount_specific"] = True - with dest.open() as f: - data = yaml.safe_load(f) + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) # the buildcache url gains the mount point as a sub-directory ... assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io/user-environment" @@ -140,25 +151,21 @@ def test_mount_specific_buildcache(tmp_path, systems_path): assert data["mirrors"]["mirror1"]["fetch"]["url"] == "https://github.com" -def test_mount_specific_disabled(tmp_path, systems_path): +def test_mount_specific_disabled(tmp_path, systems_path, mount_path): """A buildcache with mount_specific false is unchanged, even when a mount point is set.""" - mount = pathlib.Path("/user-environment") - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_point=mount) + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) # confirm the fixture leaves the flag off - assert mirrors_obj.mirrors["buildcache"]["mount_specific"] is False + assert mirrors_obj.buildcache["mount_specific"] is False - dest = tmp_path / "mirrors.yaml" - mirrors_obj._create_spack_mirrors_yaml(dest) - - with dest.open() as f: - data = yaml.safe_load(f) + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io" -def test_create_bootstrap_configs(tmp_path, systems_path): +def test_bootstrap_configs(tmp_path, systems_path, mount_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { @@ -179,47 +186,44 @@ def test_create_bootstrap_configs(tmp_path, systems_path): }, } - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") - mirrors_obj._create_bootstrap_configs(tmp_path) - - with (tmp_path / "bootstrap.yaml").open() as f: - bs_data = yaml.safe_load(f) + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + files = mirrors_obj.config_files(tmp_path) + bs_data = yaml.safe_load(files[tmp_path / "bootstrap.yaml"]) assert bs_data == valid_yaml - with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: - metadata = yaml.safe_load(f) + metadata = yaml.safe_load(files[tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml"]) assert metadata == valid_metadata -def test_key_setup(systems_path, tmp_path): - """Check that public keys are set up properly.""" +def test_keys(systems_path, tmp_path, mount_path): + """Check that gpg keys are decoded, relocated and reported consistently.""" - mirrors = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + files = mirrors_obj.config_files(tmp_path) - mirrors._key_setup(tmp_path) + # public keys are set up for the buildcache and mirror1 + pub_files = {p for p in files if p.name.endswith(".pub.gpg")} + assert {p.name for p in pub_files} == {"buildcache.pub.gpg", "mirror1.pub.gpg"} - pub_files = sorted(f for f in tmp_path.iterdir() if f.name.endswith(".pub.gpg")) - assert {pub_file.name for pub_file in pub_files} == {"buildcache.pub.gpg", "mirror1.pub.gpg"} + # the buildcache public key (inline base64) and mirror1's (a file) are the same + # key, so the decoded bytes must match + assert files[tmp_path / "key_store/buildcache.pub.gpg"] == files[tmp_path / "key_store/mirror1.pub.gpg"] - # The two files should be identical in content - pub_file_data = [] - for pub_file in pub_files: - with pub_file.open("rb") as file: - pub_file_data.append(file.read()) - assert pub_file_data[0] == pub_file_data[1] + # gpg_key_paths reports exactly the key files that config_files writes + key_files = {p for p in files if p.parent.name == "key_store"} + assert set(mirrors_obj.gpg_key_paths(tmp_path)) == key_files @pytest.mark.parametrize( "system_name", [ - # "mirror-bad-key", + "mirror-bad-key", "mirror-bad-keypath", ], ) -def test_key_setup_bad_key(tmp_path, systems_path, system_name): - """Check that MirrorError is raised for bad keys""" +def test_bad_key(systems_path, mount_path, system_name): + """Check that MirrorError is raised at construction for bad keys.""" - mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): - mirrors._key_setup(tmp_path) + mirror.Mirrors(systems_path / system_name, mount_path) From 9c86155144fa4979b01ca15a00af600a08fb796b Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 11:03:16 +0200 Subject: [PATCH 79/98] add source caching - rename sourcecache to sourcemirror (for pre-populated source mirrors) - add sourcecache for read through source caching - after downloading a source, add it to the cache so that it does not have to be downloaded again. --- docs/build-caches.md | 206 +++++++++++++----- docs/cluster-config.md | 25 +-- memory.md | 1 + mkdocs.yml | 2 +- stackinator/builder.py | 1 + stackinator/mirror.py | 105 +++++++-- stackinator/recipe.py | 11 +- stackinator/schema/mirror.json | 11 +- stackinator/templates/Makefile | 4 +- .../mirror-bad-sourcecache/mirrors.yaml | 2 + .../data/systems/mirror-bad-url/mirrors.yaml | 2 +- .../mirror-no-sourcecache/mirrors.yaml | 5 + .../data/systems/mirror-ok/cache-nokey.yaml | 2 + unittests/data/systems/mirror-ok/mirrors.yaml | 6 +- unittests/test_mirrors.py | 67 +++++- 15 files changed, 330 insertions(+), 120 deletions(-) create mode 100644 memory.md create mode 100644 unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml create mode 100644 unittests/data/systems/mirror-no-sourcecache/mirrors.yaml create mode 100644 unittests/data/systems/mirror-ok/cache-nokey.yaml diff --git a/docs/build-caches.md b/docs/build-caches.md index 709f12be..ee196044 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -1,101 +1,189 @@ -# Build Caches +# Mirrors and Build Caches + +Spack can use *mirrors* and *caches* to speed up image builds and to build on systems with limited or no internet access. +They are configured in a single `mirrors.yaml` file in the [system configuration](cluster-config.md). + +A `mirrors.yaml` can describe four kinds of entry, each optional and each documented below: + +| Entry | Count | Purpose | +|-------|-------|---------| +| [`buildcache`](#build-cache) | one | binary cache of built packages (the big build-time speed up) | +| [`bootstrap`](#bootstrap-mirror) | one | mirror used to bootstrap Spack itself | +| [`sourcemirror`](#source-mirrors) | many | read-only mirrors that provide package sources | +| [`sourcecache`](#source-cache) | one | writable local cache that fills with sources as you build | + +A complete example: + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg + mount_specific: true +bootstrap: + url: https://bootstrap.spack.io +sourcemirror: + mirror1: + url: https://example.com/spack-sources +sourcecache: + path: /capstor/scratch/$USER/spack-sources +``` -Stackinator facilitates using Spack's binary build caches to speed up image builds. -Build caches are essential if you plan to build images regularly, as they generally lead to a roughly 10x speed up. -This is the difference between half an hour or 3 minutes to build a typical image. +To stop using any entry, remove (or comment out) it from `mirrors.yaml`. -## Using Build caches +## Build cache -To use a build cache, create a simple YAML file: +A build cache is a binary cache of built packages. +Reusing binaries instead of rebuilding from source is roughly a 10x speed up — the difference between a 3 minute and a 30 minute image build — so a build cache is essential if you build images regularly. -```yaml title='cache-config.yaml' -root: $SCRATCH/uenv-cache -key: $SCRATCH/.keys/spack-push-key.gpg +During a build Spack fetches packages from the cache when it can, and signs and pushes any package it has to build itself, so the cache improves over time. + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg ``` -To use the cache, pass the configuration as an option to `stack-config` via the `-c/--cache` flag: +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | +| `private_key` | yes | PGP key used to sign and push packages (see [Keys](#keys)) | +| `public_key` | no | PGP key used to verify downloaded packages | +| `name` | no | name Spack registers the mirror under (default `buildcache`) | +| `mount_specific` | no | store the cache in a per-mount-point sub-directory (default `false`) | + +### `mount_specific` + +Spack binaries embed the install prefix (the image's mount point), so binaries built for `/user-environment` cannot be reused at a different mount point. +Set `mount_specific: true` to append the mount point to the cache URL, giving each mount point its own sub-directory and avoiding relocation issues: + +```yaml +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg + mount_specific: true # packages stored under .../uenv-cache/user-environment +``` + +### Creating a build cache + +A build cache needs an empty directory and a PGP signing key: ```bash -stack-config -b $build_path -r $recipe_path -s $system_config -c cache-config.yaml +# 1. create the cache directory +mkdir -p $SCRATCH/uenv-cache + +# 2. generate and export a signing key +spack gpg create +spack gpg export --secret $SCRATCH/.keys/spack-push-key.gpg ``` -??? warning "If you using an old binary build cache" - Since v3, Stackinator creates a sub-directory in the build cache for each mount point. - For example, in the above example, the build cache for the `/user-environment` mount point would be `$SCRATCH/uenv-cache/user-environment`. - The rationale for this is so that packages for different mount points are not mixed, to avoid having to relocate binaries. +See [Keys](#keys) for where to store the key. + +### Force pushing + +Packages are pushed to the cache after each environment builds successfully; nothing is pushed if a build fails. +When iterating on a recipe with failing builds, force-push everything built so far with the `cache-force` target: - To continue using a build caches from before v3, first copy the `build_cache` path to a subdirectory, e.g.: +```bash +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make cache-force +``` - ```bash - mkdir $SCRATCH/uenv-cache/user-environment - mv $SCRATCH/uenv-cache/build_cache $SCRATCH/uenv-cache/user-environment - ``` +## Bootstrap mirror -### Build-only caches +Spack bootstraps some of its own dependencies (such as the `clingo` concretizer) on first use. +A bootstrap mirror lets it do this without reaching the internet — useful on air-gapped systems. -A build cache can be configured to be read-only by not providing a `key` in the cache configuration file. +```yaml title="mirrors.yaml" +bootstrap: + url: https://bootstrap.spack.io +``` -## Creating a Build Cache +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | location of the bootstrap mirror | +| `public_key` | no | PGP key used to verify the bootstrap binaries | -To create a build cache we need two things: +## Source mirrors -1. An empty directory where the cache will be populated by Spack. -2. A private PGP key - * Only required for Stackinator to push packages to the cache when it builds a package that was not in the cache. +Source mirrors provide package **source** archives, and are read-only: Spack fetches sources from them but never writes to them. +Use them to build on air-gapped systems — populate a mirror on a connected system, mount it read-only, and Spack will fetch sources from it. +Any number of source mirrors can be listed; Spack searches them in order. -Creating the cache directory is easy! For example, to create a cache on your scratch storage: -```bash -mkdir $SCRATCH/uenv-cache +```yaml title="mirrors.yaml" +sourcemirror: + internal: + url: https://mirror.example.com/spack-sources + scratch: + url: file:///capstor/scratch/team/spack-sources ``` -### Generating Keys +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | location of the source mirror | +| `public_key` | no | PGP key used to verify sources | -An installation of Spack can be used to generate the key file: +Populate a source mirror on an internet-connected system with Spack: ```bash -# create a key -spack gpg create +spack mirror create --directory /path/to/mirror --all +``` + +## Source cache -# export key -spack gpg export --secret spack-push-key.gpg +A source cache is a single, **writable** local directory that Spack fills as it downloads sources. +On internet-connected systems Spack checks the cache first; on a miss it downloads the source and stores it, so later builds reuse it and download times shrink over time. + +Unlike a source mirror it needs no key, is written to automatically, and is created on demand. + +```yaml title="mirrors.yaml" +sourcecache: + path: /capstor/scratch/$USER/spack-sources ``` -See the [spack documentation](https://spack.readthedocs.io/en/latest/getting_started.html#gpg-signing) for more information about GPG keys. +| Field | Required | Description | +|-------|----------|-------------| +| `path` | yes | absolute path to a local directory (environment variables are expanded) | -### Managing Keys +## Keys -The key needs to be in a location that is accessible during the build process, and secure. -To keep your PGP key secret, you can generate it then move it to a path with appropriate permissions. -In the example below, we create a path `.keys` for storing the key: -```bash -# create .keys path is visible only to you -mkdir $SCRATCH/.keys -chmod 700 $SCRATCH/.keys +The `private_key` and `public_key` fields accept either: + +* a **path** — absolute, or relative to the system configuration directory; or +* a **base64-encoded key** inlined directly in `mirrors.yaml`. + +```yaml +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg # a path + public_key: mQINBGm4GvsBEACTyzQF...== # inline base64 +``` + +Generate a key with Spack, and keep the secret key somewhere private: -# generate the key +```bash +mkdir $SCRATCH/.keys && chmod 700 $SCRATCH/.keys spack gpg create spack gpg export --secret $SCRATCH/.keys/spack-push-key.gpg chmod 600 $SCRATCH/.keys/spack-push-key.gpg ``` -The cache-configuration would look like the following, where we assume that the cache is in `$SCRATCH/uenv-cache`: -```yaml -root: $SCRATCH/uenv-cache -key: $SCRATCH/.keys/spack-push-key.gpg -``` -!!! warning - Don't blindly copy this documentation's advice on security settings. +See the [Spack documentation](https://spack.readthedocs.io/en/latest/getting_started.html#gpg-signing) for more on GPG keys. !!! failure "Don't use `$HOME`" - Don't put the keys in `$HOME`, because the build process remounts `~` as a tmpfs, and you will get error messages that Spack can't read the key. + The build remounts `~` as a tmpfs, so keys under `$HOME` are not visible during the build and Spack will fail to read them. Use scratch storage instead. -## Force pushing to build cache +## Legacy `--cache` option -When build caches are enabled, all packages in a each Spack environment are pushed to the build cache after the whole environment has been built successfully -- nothing will be pushed to the cache if there is an error when building one of the packages. +Before `mirrors.yaml`, a build cache was configured with a separate `cache.yaml` file passed to `stack-config` via `-c/--cache`: -When debugging a recipe, where failing builds have to be run multiple times, the overheads of rebuilding all packages from scratch can be wasteful. -To force push all packages that have been built, use the `cache-force` makefile target: +```yaml title="cache.yaml" +root: $SCRATCH/uenv-cache +key: $SCRATCH/.keys/spack-push-key.gpg +``` ```bash -env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make cache-force +stack-config -b $build -r $recipe -s $system -c cache.yaml ``` + +This is **deprecated** and equivalent to a single `buildcache` entry (with `mount_specific: true`). Prefer configuring the build cache in `mirrors.yaml`. + +Setting `key: null` configures a read-only cache that Spack fetches from but never pushes to. diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 8df0a3bf..26571364 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -95,29 +95,10 @@ packages: ### Configuring Spack mirrors: `mirrors.yaml` -On air-gapped systems, Spack is unable to reach its default mirror to fetch packages. The `mirrors.yaml` configuration can be used to connect Spack to local mirrors for fetching and building packages. +An optional `mirrors.yaml` connects Spack to local mirrors and caches, to speed up builds and to build on air-gapped systems. +It can configure a build cache, a bootstrap mirror, read-only source mirrors, and a writable source cache. -`mirrors.yaml` treats source mirrors, buildcaches, and bootstrap mirrors the same, and they may all be included in this file. Spack will search the topmost mirror first and the bottom-most mirror last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. - -`mirrors.yaml` may include source mirrors, bootstrap mirrors, and buildcaches. Any number of source mirrors can be added, but only one bootstrap mirror and buildcache can be specified. Spack will search the source mirrors in order from first to last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. - -If using a buildcache, a private key must be provided for signing packages. An optional public key may be specified with any type of mirror to verify packages. The buildcache is registered with Spack under the name `buildcache`, which can be overridden with an optional `name` field. - -To stop using a mirror, remove (or comment out) its entry from `mirrors.yaml`. - -```yaml title="mirrors.yaml" -bootstrap: - url: file:///home/username/spack-mirror-2014-06-24 -buildcache: - url: https://example.com/some/buildcache/mirror - private_key: /user-home/.gnupg/private-keys-v1.d/my-private-key.asc -sourcecache: - mirror1: - url: https://example.com/some/web-hosted/directory - public_key: ../buildcache-key.public.gpg - mirror2: - url: https://example.com/some/other-web-hosted/directory -``` +See [Mirrors and Build Caches](build-caches.md) for the full reference and examples. ## Site and System Configurations diff --git a/memory.md b/memory.md new file mode 100644 index 00000000..1d0fd934 --- /dev/null +++ b/memory.md @@ -0,0 +1 @@ +claude --resume 71a36db9-9a83-4261-bad1-855caaca9f7e diff --git a/mkdocs.yml b/mkdocs.yml index 3ce996a0..6a34c28e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,7 +8,7 @@ nav: - 'Recipes': recipes.md - 'Cluster Configuration': cluster-config.md - 'Interfaces': interfaces.md - - 'Build Caches': build-caches.md + - 'Mirrors & Build Caches': build-caches.md - 'Spack 1.0': porting.md - 'Development': development.md # - Tutorial: tutorial.md diff --git a/stackinator/builder.py b/stackinator/builder.py index a05764fb..feb9a312 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -246,6 +246,7 @@ def generate(self, recipe): spack_meta=spack_meta, gpg_keys=recipe.mirrors.gpg_key_paths(config_path), cache=recipe.build_cache_mirror, + buildcache_push=recipe.push_to_build_cache, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9d912e68..ba4475c5 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -34,13 +34,18 @@ class MirrorError(RuntimeError): class Mirrors: """Fully validated and resolved definition of the spack mirrors for a recipe. - The three kinds of mirror have separate types: - - * buildcache - at most one, signs and stores built packages (so it alone - has a private key and the mount_specific flag). None if no - build cache is configured. - * bootstrap - at most one, used to bootstrap spack itself. None if absent. - * source_caches - a name -> config mapping of any number of source mirrors. + The kinds of mirror have separate types: + + * buildcache - at most one, signs and stores built packages (so it alone + has a private key and the mount_specific flag). None if no + build cache is configured. + * bootstrap - at most one, used to bootstrap spack itself. None if absent. + * source_mirrors - a name -> config mapping of any number of read-only source + mirrors (spack mirrors.yaml entries). + * source_cache - at most one, a writable local directory that spack fills as + it fetches sources (spack config:source_cache). None if + absent. This is not a mirror: it has no key and no url, and + is emitted to config.yaml rather than mirrors.yaml. All input processing - loading and schema-validating the system mirrors.yaml, validating urls, and reading/decoding/validating gpg keys - happens eagerly in @@ -52,6 +57,7 @@ class Mirrors: KEY_STORE_DIR = "key_store" MIRRORS_YAML = "mirrors.yaml" + CONFIG_YAML = "config.yaml" BOOTSTRAP_MIRROR = "bootstrap" def __init__( @@ -73,7 +79,8 @@ def __init__( self.buildcache: Optional[Dict] = None self.bootstrap: Optional[Dict] = None - self.source_caches: Dict[str, Dict] = {} + self.source_mirrors: Dict[str, Dict] = {} + self.source_cache: Optional[Dict] = None # Load and schema-validate the system mirrors.yaml (absent file is fine). mirrors_path = system_config_root / "mirrors.yaml" @@ -120,9 +127,11 @@ def __init__( "url": raw_cache["root"], "description": "Buildcache dest loaded from legacy cache.yaml", "mount_specific": True, - "private_key": raw_cache["key"], "cmdline": True, } + # a cache.yaml without a key configures a read-only (fetch-only) cache + if raw_cache.get("key") is not None: + self.buildcache["private_key"] = raw_cache["key"] self._logger.warning( "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" @@ -130,22 +139,35 @@ def __init__( f"{yaml.dump([self.buildcache], default_flow_style=False)}" ) - # The bootstrap mirror and the source mirrors, if any are defined. + # The bootstrap mirror, the read-only source mirrors, and the writable + # source cache, if any are defined. self.bootstrap = raw_mirrors.get("bootstrap") - self.source_caches = dict(raw_mirrors.get("sourcecache", {})) + self.source_mirrors = dict(raw_mirrors.get("sourcemirror", {})) + self.source_cache = raw_mirrors.get("sourcecache") # Validate that every mirror url is well-formed (see _validate_url). for name, mirror in self._iter_mirrors(): self._validate_url(mirror["url"], name) + # The source cache is a single writable local directory (spack + # config:source_cache), not a mirror: validate that it is an absolute path. + # Expand env vars now, because the build sandbox runs `env --ignore-environment` + # and so would not expand them at build time. + if self.source_cache is not None: + path = os.path.expandvars(self.source_cache["path"]) + if not pathlib.Path(path).is_absolute(): + raise MirrorError(f"The source cache path '{path}' is not absolute") + self.source_cache["path"] = path + # Read, decode and validate every gpg key into memory. Each key is stored # as (path-relative-to-config-root, raw bytes); the builder writes these # verbatim into the build directory's key store. key_store = pathlib.PurePosixPath(self.KEY_STORE_DIR) self._key_files: List[Tuple[pathlib.PurePosixPath, bytes]] = [] - # The build cache signs packages, so it alone has a private key. - if self.buildcache is not None: + # A build cache that pushes packages signs them with its private key. A build + # cache without a key is read-only, and is fetched from but never pushed to. + if self.buildcache is not None and self.buildcache.get("private_key"): name = self.buildcache["name"] self._key_files.append( (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name)) @@ -159,10 +181,26 @@ def __init__( @property def build_cache_mirror(self) -> Optional[str]: - """The spack mirror name that built packages are pushed to, or None.""" + """The build cache mirror name, or None if no build cache is configured. + + A build cache is fetched from (and its keys trusted) whether or not it has a + signing key; see push_to_build_cache for whether packages are pushed to it. + """ return self.buildcache["name"] if self.buildcache is not None else None + @property + def push_to_build_cache(self) -> Optional[str]: + """The build cache mirror name to push built packages to, or None. + + Pushing requires a private signing key; a build cache configured without one + is read-only - fetched from but never pushed to. + """ + + if self.buildcache is not None and self.buildcache.get("private_key"): + return self.buildcache["name"] + return None + def _iter_mirrors(self): """Yield (spack mirror name, config dict) for every configured mirror. @@ -174,7 +212,7 @@ def _iter_mirrors(self): yield self.buildcache["name"], self.buildcache if self.bootstrap is not None: yield self.BOOTSTRAP_MIRROR, self.bootstrap - yield from self.source_caches.items() + yield from self.source_mirrors.items() def _validate_url(self, url: str, name: str): """Validate that a mirror url is well-formed. @@ -248,9 +286,7 @@ def gpg_key_paths(self, config_root: pathlib.Path) -> List[pathlib.Path]: def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: """The complete set of mirror config files to write under config_root. - Returns a mapping of absolute file path -> file content. This is a pure - function of the already-validated state and the output location: it neither - validates nor performs any I/O, so the builder can write the bytes verbatim. + Returns a mapping of absolute file path -> file content. """ files: Dict[pathlib.Path, bytes] = {} @@ -261,20 +297,41 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: # the spack mirrors.yaml spack_mirrors: Dict[str, Dict] = {"mirrors": {}} - for name, mirror in self._iter_mirrors(): - url = mirror["url"] + + if self.buildcache is not None: + url = self.buildcache["url"] # a mount-specific build cache lives in a sub-directory named after the # mount point: spack binaries embed the install prefix, so each mount - # point needs its own cache to avoid relocation issues. Only the build - # cache carries the mount_specific flag. - if mirror.get("mount_specific"): + # point needs its own cache to avoid relocation issues. + if self.buildcache["mount_specific"]: url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/") - spack_mirrors["mirrors"][name] = {"fetch": {"url": url}, "push": {"url": url}} + entry = {"fetch": {"url": url}} + # only a build cache with a signing key is pushed to + if self.buildcache.get("private_key"): + entry["push"] = {"url": url} + spack_mirrors["mirrors"][self.buildcache["name"]] = entry + + if self.bootstrap is not None: + url = self.bootstrap["url"] + spack_mirrors["mirrors"][self.BOOTSTRAP_MIRROR] = {"fetch": {"url": url}, "push": {"url": url}} + + # source mirrors are read-only and provide sources only: fetch url, no push. + for name, mirror in self.source_mirrors.items(): + spack_mirrors["mirrors"][name] = { + "source": True, + "binary": False, + "fetch": {"url": mirror["url"]}, + } files[config_root / self.MIRRORS_YAML] = yaml.dump( spack_mirrors, default_flow_style=False, sort_keys=False ).encode() + # the spack config.yaml setting the writable, populate-as-you-go source cache + if self.source_cache is not None: + config_yaml = {"config": {"source_cache": self.source_cache["path"]}} + files[config_root / self.CONFIG_YAML] = yaml.dump(config_yaml, default_flow_style=False).encode() + # the bootstrap config and its mirror metadata, if a bootstrap mirror is set if self.bootstrap is not None: metadata_dir = config_root / "bootstrap" / "bootstrap-mirror" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 63f7f409..c1eb629f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -219,6 +219,13 @@ def spack_repo(self): def build_cache_mirror(self): return self.mirrors.build_cache_mirror + # Returns: + # str: the build cache mirror name to push built packages to + # None: if there is no build cache, or it is read-only (no signing key) + @property + def push_to_build_cache(self): + return self.mirrors.push_to_build_cache + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -505,7 +512,7 @@ def compiler_files(self): makefile_template = env.get_template("Makefile.compilers") files["makefile"] = makefile_template.render( compilers=self.compilers, - buildcache=self.build_cache_mirror, + buildcache=self.push_to_build_cache, spack_version=self.spack_version, ) @@ -535,7 +542,7 @@ def environment_files(self): makefile_template = jenv.get_template("Makefile.environments") files["makefile"] = makefile_template.render( environments=self.environments, - buildcache=self.build_cache_mirror, + buildcache=self.push_to_build_cache, spack_version=self.spack_version, ) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b840d52d..71597c74 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -35,7 +35,7 @@ "additionalProperties": false, "required": ["url", "private_key"] }, - "sourcecache": { + "sourcemirror": { "type": "object", "additionalProperties": { "type": "object", @@ -47,6 +47,15 @@ "additionalProperties": false, "required": ["url"] } + }, + "sourcecache": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "path": {"type": "string"} + }, + "additionalProperties": false, + "required": ["path"] } } } \ No newline at end of file diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3b1537d3..cb411400 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -81,14 +81,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache %} +{% if buildcache_push %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package {{ cache }} \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package {{ buildcache_push }} \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml b/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml new file mode 100644 index 00000000..dd59d5c1 --- /dev/null +++ b/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml @@ -0,0 +1,2 @@ +sourcecache: + path: relative/not/absolute diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 4a477660..61341619 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,3 +1,3 @@ -sourcecache: +sourcemirror: bad-url: url: not-a-valid-url \ No newline at end of file diff --git a/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml b/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml new file mode 100644 index 00000000..7a1ecdff --- /dev/null +++ b/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml @@ -0,0 +1,5 @@ +bootstrap: + url: https://mirror.spack.io +sourcemirror: + mirror1: + url: https://github.com diff --git a/unittests/data/systems/mirror-ok/cache-nokey.yaml b/unittests/data/systems/mirror-ok/cache-nokey.yaml new file mode 100644 index 00000000..5fcd561b --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache-nokey.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: null diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index e383d3fb..1f959ec2 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -46,9 +46,11 @@ buildcache: private_key: ../../test-gpg-priv.asc mount_specific: false cmdline: false -sourcecache: +sourcemirror: mirror1: url: https://github.com public_key: ../../test-gpg-pub.asc mirror2: - url: https://github.com/spack \ No newline at end of file + url: https://github.com/spack +sourcecache: + path: /scratch/spack-sources \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index f9861063..d950dc18 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -39,11 +39,14 @@ def test_mirror_init(systems_path, mount_path): assert mirrors_obj.bootstrap == {"url": "https://mirror.spack.io"} - assert mirrors_obj.source_caches == { + assert mirrors_obj.source_mirrors == { "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"}, "mirror2": {"url": "https://github.com/spack"}, } + # the writable, populate-as-you-go source cache + assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources"} + # the build cache mirror name is derived from the build cache's 'name' field assert mirrors_obj.build_cache_mirror == "buildcache" @@ -72,9 +75,36 @@ def test_command_line_cache(systems_path, mount_path): assert mirrors.buildcache["cmdline"] assert mirrors.buildcache["mount_specific"] + # it has a signing key, so it is pushed to + assert mirrors.push_to_build_cache == "buildcache" + # the bootstrap and source mirrors from mirrors.yaml are still present assert mirrors.bootstrap is not None - assert set(mirrors.source_caches) == {"mirror1", "mirror2"} + assert set(mirrors.source_mirrors) == {"mirror1", "mirror2"} + + +def test_keyless_command_line_cache(tmp_path, systems_path, mount_path): + """A cache.yaml without a key configures a read-only (fetch-only) build cache.""" + + mirrors = mirror.Mirrors( + systems_path / "mirror-ok", mount_path, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" + ) + + # the cache exists (so it is fetched from), but has no signing key ... + assert mirrors.build_cache_mirror == "buildcache" + assert "private_key" not in mirrors.buildcache + + # ... so it is never pushed to + assert mirrors.push_to_build_cache is None + + files = mirrors.config_files(tmp_path) + + # no private key is written + assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files + + # the mirror is emitted with a fetch url but no push url + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}} def test_config_files(tmp_path, systems_path, mount_path): @@ -85,6 +115,7 @@ def test_config_files(tmp_path, systems_path, mount_path): expected = { tmp_path / "mirrors.yaml", + tmp_path / "config.yaml", tmp_path / "bootstrap.yaml", tmp_path / "bootstrap" / "bootstrap-mirror" / "metadata.yaml", tmp_path / "key_store" / "buildcache.priv.gpg", @@ -111,12 +142,14 @@ def test_spack_mirrors_yaml(tmp_path, systems_path, mount_path): "push": {"url": "https://mirror.spack.io"}, }, "mirror1": { + "source": True, + "binary": False, "fetch": {"url": "https://github.com"}, - "push": {"url": "https://github.com"}, }, "mirror2": { + "source": True, + "binary": False, "fetch": {"url": "https://github.com/spack"}, - "push": {"url": "https://github.com/spack"}, }, } } @@ -215,15 +248,37 @@ def test_keys(systems_path, tmp_path, mount_path): assert set(mirrors_obj.gpg_key_paths(tmp_path)) == key_files +def test_source_cache_config(tmp_path, systems_path, mount_path): + """The writable source cache is emitted to config.yaml as config:source_cache.""" + + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + files = mirrors_obj.config_files(tmp_path) + + data = yaml.safe_load(files[tmp_path / "config.yaml"]) + assert data == {"config": {"source_cache": "/scratch/spack-sources"}} + + +def test_source_cache_absent(tmp_path, systems_path, mount_path): + """No config.yaml is generated when no source cache is configured.""" + + # mirror-no-sourcecache has no sourcecache entry + mirrors_obj = mirror.Mirrors(systems_path / "mirror-no-sourcecache", mount_path) + files = mirrors_obj.config_files(tmp_path) + + assert mirrors_obj.source_cache is None + assert tmp_path / "config.yaml" not in files + + @pytest.mark.parametrize( "system_name", [ "mirror-bad-key", "mirror-bad-keypath", + "mirror-bad-sourcecache", ], ) -def test_bad_key(systems_path, mount_path, system_name): - """Check that MirrorError is raised at construction for bad keys.""" +def test_bad_config(systems_path, mount_path, system_name): + """Check that MirrorError is raised at construction for bad keys or a bad source cache path.""" with pytest.raises(mirror.MirrorError): mirror.Mirrors(systems_path / system_name, mount_path) From 4a1c01cf019e0a376261e6105695cc1ed36fe664 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 11:56:34 +0200 Subject: [PATCH 80/98] support read only buildcaches; set defaults on mirrors.yaml inputs --- docs/build-caches.md | 12 +++++- stackinator/mirror.py | 32 +++++++-------- stackinator/schema/mirror.json | 19 ++++----- .../mirror-readonly-cache/mirrors.yaml | 2 + unittests/test_mirrors.py | 40 ++++++++++++++++--- 5 files changed, 72 insertions(+), 33 deletions(-) create mode 100644 unittests/data/systems/mirror-readonly-cache/mirrors.yaml diff --git a/docs/build-caches.md b/docs/build-caches.md index ee196044..cf661369 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -46,11 +46,21 @@ buildcache: | Field | Required | Description | |-------|----------|-------------| | `url` | yes | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | -| `private_key` | yes | PGP key used to sign and push packages (see [Keys](#keys)) | +| `private_key` | no | PGP key used to sign and push packages (see [Keys](#keys)); omit for a read-only cache | | `public_key` | no | PGP key used to verify downloaded packages | | `name` | no | name Spack registers the mirror under (default `buildcache`) | | `mount_specific` | no | store the cache in a per-mount-point sub-directory (default `false`) | +### Read-only build cache + +Omit `private_key` to configure a read-only cache: Spack fetches packages from it but never signs or pushes anything back. +This is useful for consuming a shared team cache that you are not permitted to write to. + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache +``` + ### `mount_specific` Spack binaries embed the install prefix (the image's mount point), so binaries built for `/user-environment` cannot be reused at a different mount point. diff --git a/stackinator/mirror.py b/stackinator/mirror.py index ba4475c5..a3f7e5a3 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -36,9 +36,10 @@ class Mirrors: The kinds of mirror have separate types: - * buildcache - at most one, signs and stores built packages (so it alone - has a private key and the mount_specific flag). None if no - build cache is configured. + * buildcache - at most one, fetches and stores built packages (so it alone + has the mount_specific flag). With a private key it signs + and pushes packages too; without one it is read-only. None + if no build cache is configured. * bootstrap - at most one, used to bootstrap spack itself. None if absent. * source_mirrors - a name -> config mapping of any number of read-only source mirrors (spack mirrors.yaml entries). @@ -98,12 +99,9 @@ def __init__( except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - # The build cache, if one is defined in mirrors.yaml. - buildcache = raw_mirrors.get("buildcache") - if buildcache: - if not buildcache.get("private_key"): - raise MirrorError("Mirror build cache config is missing a required 'private_key' path.") - self.buildcache = buildcache + # The build cache, if one is defined in mirrors.yaml. A build cache without + # a private_key is read-only: spack fetches from it but never pushes to it. + self.buildcache = raw_mirrors.get("buildcache") # A build cache passed via the deprecated cache.yaml file (the --cache CLI # option) takes precedence over a buildcache defined in mirrors.yaml. @@ -122,16 +120,16 @@ def __init__( except ValueError as err: raise MirrorError(f"Error validating contents of cache config at '{cmdline_cache}'.\n{err}") + # a cache.yaml without a key configures a read-only (fetch-only) cache self.buildcache = { "name": "buildcache", "url": raw_cache["root"], "description": "Buildcache dest loaded from legacy cache.yaml", + "public_key": None, + "private_key": raw_cache.get("key"), "mount_specific": True, "cmdline": True, } - # a cache.yaml without a key configures a read-only (fetch-only) cache - if raw_cache.get("key") is not None: - self.buildcache["private_key"] = raw_cache["key"] self._logger.warning( "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" @@ -142,7 +140,7 @@ def __init__( # The bootstrap mirror, the read-only source mirrors, and the writable # source cache, if any are defined. self.bootstrap = raw_mirrors.get("bootstrap") - self.source_mirrors = dict(raw_mirrors.get("sourcemirror", {})) + self.source_mirrors = dict(raw_mirrors["sourcemirror"]) self.source_cache = raw_mirrors.get("sourcecache") # Validate that every mirror url is well-formed (see _validate_url). @@ -167,7 +165,7 @@ def __init__( # A build cache that pushes packages signs them with its private key. A build # cache without a key is read-only, and is fetched from but never pushed to. - if self.buildcache is not None and self.buildcache.get("private_key"): + if self.buildcache is not None and self.buildcache["private_key"] is not None: name = self.buildcache["name"] self._key_files.append( (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name)) @@ -175,7 +173,7 @@ def __init__( # Any mirror may provide a public key, used to verify downloaded packages. for name, mirror in self._iter_mirrors(): - public_key = mirror.get("public_key") + public_key = mirror["public_key"] if public_key is not None: self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(public_key, name))) @@ -197,7 +195,7 @@ def push_to_build_cache(self) -> Optional[str]: is read-only - fetched from but never pushed to. """ - if self.buildcache is not None and self.buildcache.get("private_key"): + if self.buildcache is not None and self.buildcache["private_key"] is not None: return self.buildcache["name"] return None @@ -307,7 +305,7 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/") entry = {"fetch": {"url": url}} # only a build cache with a signing key is pushed to - if self.buildcache.get("private_key"): + if self.buildcache["private_key"] is not None: entry["push"] = {"url": url} spack_mirrors["mirrors"][self.buildcache["name"]] = entry diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 71597c74..1612ace0 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -5,9 +5,9 @@ "bootstrap": { "type": "object", "properties": { - "description": {"type": "string"}, + "description": {"type": "string", "default": ""}, "url": {"type": "string"}, - "public_key": {"type": "string"} + "public_key": {"type": ["string", "null"], "default": null} }, "additionalProperties": false, "required": ["url"] @@ -19,10 +19,10 @@ "type": "string", "default": "buildcache" }, - "description": {"type": "string"}, + "description": {"type": "string", "default": ""}, "url": {"type": "string"}, - "public_key": {"type": "string"}, - "private_key": {"type": "string"}, + "public_key": {"type": ["string", "null"], "default": null}, + "private_key": {"type": ["string", "null"], "default": null}, "mount_specific": { "type": "boolean", "default": false @@ -33,16 +33,17 @@ } }, "additionalProperties": false, - "required": ["url", "private_key"] + "required": ["url"] }, "sourcemirror": { "type": "object", + "default": {}, "additionalProperties": { "type": "object", "properties": { - "description": {"type": "string"}, + "description": {"type": "string", "default": ""}, "url": {"type": "string"}, - "public_key": {"type": "string"} + "public_key": {"type": ["string", "null"], "default": null} }, "additionalProperties": false, "required": ["url"] @@ -51,7 +52,7 @@ "sourcecache": { "type": "object", "properties": { - "description": {"type": "string"}, + "description": {"type": "string", "default": ""}, "path": {"type": "string"} }, "additionalProperties": false, diff --git a/unittests/data/systems/mirror-readonly-cache/mirrors.yaml b/unittests/data/systems/mirror-readonly-cache/mirrors.yaml new file mode 100644 index 00000000..0c041531 --- /dev/null +++ b/unittests/data/systems/mirror-readonly-cache/mirrors.yaml @@ -0,0 +1,2 @@ +buildcache: + url: https://mirror.spack.io diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index d950dc18..2db2967d 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -27,9 +27,10 @@ def test_mirror_init(systems_path, mount_path): with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: pub_key_b64 = base64.b64encode(pub_key_file.read()).decode() - # the build cache, with its schema-defaulted name and flags + # the build cache, with its schema-defaulted name, flags and (empty) fields assert mirrors_obj.buildcache == { "name": "buildcache", + "description": "", "url": "https://mirror.spack.io", "private_key": "../../test-gpg-priv.asc", "public_key": pub_key_b64, @@ -37,15 +38,20 @@ def test_mirror_init(systems_path, mount_path): "cmdline": False, } - assert mirrors_obj.bootstrap == {"url": "https://mirror.spack.io"} + assert mirrors_obj.bootstrap == { + "url": "https://mirror.spack.io", + "description": "", + "public_key": None, + } + # non-required fields are always present, defaulted to "" / None by the schema assert mirrors_obj.source_mirrors == { - "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"}, - "mirror2": {"url": "https://github.com/spack"}, + "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc", "description": ""}, + "mirror2": {"url": "https://github.com/spack", "public_key": None, "description": ""}, } # the writable, populate-as-you-go source cache - assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources"} + assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources", "description": ""} # the build cache mirror name is derived from the build cache's 'name' field assert mirrors_obj.build_cache_mirror == "buildcache" @@ -92,7 +98,7 @@ def test_keyless_command_line_cache(tmp_path, systems_path, mount_path): # the cache exists (so it is fetched from), but has no signing key ... assert mirrors.build_cache_mirror == "buildcache" - assert "private_key" not in mirrors.buildcache + assert mirrors.buildcache["private_key"] is None # ... so it is never pushed to assert mirrors.push_to_build_cache is None @@ -107,6 +113,28 @@ def test_keyless_command_line_cache(tmp_path, systems_path, mount_path): assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}} +def test_readonly_buildcache(tmp_path, systems_path, mount_path): + """A buildcache in mirrors.yaml without a private_key is read-only (fetch-only).""" + + mirrors = mirror.Mirrors(systems_path / "mirror-readonly-cache", mount_path) + + # the cache exists (so it is fetched from), but has no signing key ... + assert mirrors.build_cache_mirror == "buildcache" + assert mirrors.buildcache["private_key"] is None + + # ... so it is never pushed to + assert mirrors.push_to_build_cache is None + + files = mirrors.config_files(tmp_path) + + # no private key is written + assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files + + # the mirror is emitted with a fetch url but no push url + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert data["mirrors"]["buildcache"] == {"fetch": {"url": "https://mirror.spack.io"}} + + def test_config_files(tmp_path, systems_path, mount_path): """Check that config_files presents the complete set of mirror config artifacts.""" From 38b0a7dd4b53f0fce877c18a388976017f4ac131 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 12:51:17 +0200 Subject: [PATCH 81/98] make mirrors a cli argument, not part of system config --- docs/build-caches.md | 9 ++- docs/cluster-config.md | 6 +- docs/configuring.md | 3 +- memory.md | 1 - stackinator/main.py | 9 ++- stackinator/mirror.py | 44 ++++++++++----- stackinator/recipe.py | 5 +- unittests/test_mirrors.py | 113 ++++++++++++++++++++++++++------------ 8 files changed, 133 insertions(+), 57 deletions(-) delete mode 100644 memory.md diff --git a/docs/build-caches.md b/docs/build-caches.md index cf661369..4d5d1ae3 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -1,7 +1,14 @@ # Mirrors and Build Caches Spack can use *mirrors* and *caches* to speed up image builds and to build on systems with limited or no internet access. -They are configured in a single `mirrors.yaml` file in the [system configuration](cluster-config.md). +They are configured in a single `mirrors.yaml` file that you supply on the command line: + +```bash +stack-config -b $build -r $recipe -s $system --mirror mirrors.yaml +``` + +The file is not part of the [system configuration](cluster-config.md): mirror locations are usually specific to the person running the build, so each invocation provides its own. +Paths inside the file (such as relative gpg key paths) are resolved relative to the directory containing the `mirrors.yaml`, so a self-contained mirror directory (the `mirrors.yaml` plus its keys) can be moved around freely. A `mirrors.yaml` can describe four kinds of entry, each optional and each documented below: diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 26571364..cb6962f8 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -93,10 +93,10 @@ packages: version: ["git.59b6de6a91d9637809677c50cc48b607a91a9acb=main"] ``` -### Configuring Spack mirrors: `mirrors.yaml` +### Configuring Spack mirrors -An optional `mirrors.yaml` connects Spack to local mirrors and caches, to speed up builds and to build on air-gapped systems. -It can configure a build cache, a bootstrap mirror, read-only source mirrors, and a writable source cache. +Mirrors and caches are **not** part of the system configuration. +They are supplied per-invocation with `stack-config --mirror `, because the locations involved (build caches, source caches) are often specific to the user running the build and may not be accessible to everyone using a system. See [Mirrors and Build Caches](build-caches.md) for the full reference and examples. diff --git a/docs/configuring.md b/docs/configuring.md index 099329b3..eea86c93 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -16,7 +16,8 @@ The following flags are required: The following flags are optional: -* `-c/--cache`: configure the [build cache](build-caches.md). +* `--mirror`: path to a [mirrors.yaml](build-caches.md) file configuring build caches and mirrors. +* `-c/--cache`: legacy build cache configuration file (deprecated; use `--mirror`). * `-m/--mount`: override the [mount point](installing.md) where the stack will be installed. * `--version`: print the stackinator version. * `-h/--help`: print help message. diff --git a/memory.md b/memory.md deleted file mode 100644 index 1d0fd934..00000000 --- a/memory.md +++ /dev/null @@ -1 +0,0 @@ -claude --resume 71a36db9-9a83-4261-bad1-855caaca9f7e diff --git a/stackinator/main.py b/stackinator/main.py index ec384561..1ae6d617 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -74,6 +74,7 @@ def log_header(args): root_logger.info(f" system : {args.system}") mount = args.mount or "default" root_logger.info(f" mount : {mount}") + root_logger.info(f" mirror : {args.mirror}") root_logger.info(f" build cache: {args.cache}") root_logger.info(f" develop : {args.develop}") @@ -99,12 +100,18 @@ def make_argparser(): parser.add_argument( "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." ) + parser.add_argument( + "--mirror", + required=False, + type=str, + help="Path to a mirrors.yaml file describing build caches and mirrors.", + ) parser.add_argument( "-c", "--cache", required=False, type=str, - help="Buildcache location or name (from system config's mirrors.yaml).", + help="Legacy build cache configuration file (deprecated; use --mirror).", ) parser.add_argument("--develop", action="store_true", required=False) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a3f7e5a3..ca26c05c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -65,32 +65,48 @@ def __init__( self, system_config_root: pathlib.Path, mount_path: pathlib.Path, + mirror_file: Optional[pathlib.Path] = None, cmdline_cache: Optional[pathlib.Path] = None, ): """Load and fully resolve the mirror configuration. - Inputs are the system config's mirrors.yaml, the recipe mount path (used to - make a build cache mount-specific), and an optional legacy cache.yaml passed - on the command line (--cache). + Mirrors are supplied with the --mirror command line option (mirror_file). + mount_path is the recipe mount point (used to make a build cache mount-specific). + cmdline_cache is an optional legacy cache.yaml passed on the command line (--cache). + + Relative paths in the mirror file (e.g. gpg keys) are resolved relative to the + directory containing the mirror file. """ self._logger = root_logger - self._system_config_root = system_config_root self._mount_path = mount_path + self._mirror_dir = mirror_file.parent if mirror_file is not None else None self.buildcache: Optional[Dict] = None self.bootstrap: Optional[Dict] = None self.source_mirrors: Dict[str, Dict] = {} self.source_cache: Optional[Dict] = None - # Load and schema-validate the system mirrors.yaml (absent file is fine). - mirrors_path = system_config_root / "mirrors.yaml" - if mirrors_path.exists(): + # The mirror configuration is supplied with --mirror, not the system + # configuration. Reject a mirrors.yaml in the system config so it is not + # silently ignored. + if (system_config_root / "mirrors.yaml").exists(): + raise MirrorError( + "A 'mirrors.yaml' in the system configuration is not supported.\n" + "Provide the mirror configuration with the '--mirror' command line option." + ) + + # Load and schema-validate the mirror file given on the command line. If none + # was given there are no mirrors; an empty config still validates (and picks up + # schema defaults such as an empty sourcemirror map). + if mirror_file is not None: + if not mirror_file.is_file(): + raise MirrorError(f"The mirror configuration file '{mirror_file}' does not exist.") try: - with mirrors_path.open() as fid: + with mirror_file.open() as fid: raw_mirrors = yaml.load(fid, Loader=yaml.SafeLoader) except (OSError, PermissionError) as err: - raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") + raise MirrorError(f"Could not open/read mirror file '{mirror_file}'.\n{err}") else: raw_mirrors = {} @@ -241,15 +257,15 @@ def _validate_url(self, url: str, name: str): def _read_key(self, key: str, name: str) -> bytes: """Resolve a key (a file path or base64 blob) to validated gpg key bytes. - A key is either a path - absolute, or relative to the system config - or a - base64-encoded blob inlined in mirrors.yaml. The resulting bytes are checked - to be genuine gpg key material before being accepted. + A key is either a path - absolute, or relative to the mirror file's directory + - or a base64-encoded blob inlined in the mirror file. The resulting bytes are + checked to be genuine gpg key material before being accepted. """ - # if it is a path (absolute, or relative to the system config), read it + # if it is a path (absolute, or relative to the mirror file), read it path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): - path = self._system_config_root / path + path = self._mirror_dir / path if path.is_file(): binary_key = path.read_bytes() diff --git a/stackinator/recipe.py b/stackinator/recipe.py index c1eb629f..3cd0dbdc 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -176,12 +176,13 @@ def __init__(self, args): ) raise RuntimeError("Ivalid default-view in the recipe.") - # load the optional mirrors.yaml from system config, and add any additional - # mirrors specified on the command line. + # resolve the mirror configuration provided with --mirror. --cache is the + # legacy path. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors( self.system_config_path, self.mount, + pathlib.Path(args.mirror) if args.mirror else None, pathlib.Path(args.cache) if args.cache else None, ) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 2db2967d..cc765f10 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,9 +20,23 @@ def mount_path(): return pathlib.Path("/user-environment") -def test_mirror_init(systems_path, mount_path): +@pytest.fixture +def clean_root(tmp_path): + """A system config directory with no mirrors.yaml (which the constructor rejects).""" + root = tmp_path / "system-config" + root.mkdir() + return root + + +@pytest.fixture +def mirror_ok(systems_path): + """The path to the well-formed mirror file, as it would be passed to --mirror.""" + return systems_path / "mirror-ok" / "mirrors.yaml" + + +def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): """Check that the three kinds of mirror are resolved into separate members.""" - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: pub_key_b64 = base64.b64encode(pub_key_file.read()).decode() @@ -57,23 +71,48 @@ def test_mirror_init(systems_path, mount_path): assert mirrors_obj.build_cache_mirror == "buildcache" -def test_mirror_init_bad_url(systems_path, mount_path): - """Check that MirrorError is raised for a bad url.""" +def test_system_mirrors_yaml_rejected(systems_path, mount_path): + """A mirrors.yaml in the system configuration is not supported and is rejected.""" + + # mirror-ok contains a mirrors.yaml; passing it as the system config root must raise. + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(systems_path / "mirror-ok", mount_path) - path = systems_path / "mirror-bad-url" + +def test_missing_mirror_file(clean_root, mount_path): + """A --mirror file that does not exist raises MirrorError.""" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(path, mount_path) + mirror.Mirrors(clean_root, mount_path, mirror_file=clean_root / "does-not-exist.yaml") + + +def test_no_mirror_file(clean_root, mount_path): + """With no --mirror file there are no mirrors configured.""" + mirrors_obj = mirror.Mirrors(clean_root, mount_path) -def test_command_line_cache(systems_path, mount_path): + assert mirrors_obj.buildcache is None + assert mirrors_obj.bootstrap is None + assert mirrors_obj.source_cache is None + assert mirrors_obj.source_mirrors == {} + assert mirrors_obj.build_cache_mirror is None + + +def test_mirror_init_bad_url(clean_root, mount_path, systems_path): + """Check that MirrorError is raised for a bad url.""" + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / "mirror-bad-url/mirrors.yaml") + + +def test_command_line_cache(clean_root, mount_path, systems_path, mirror_ok): """Check that adding a cache from the command line works.""" mirrors = mirror.Mirrors( - systems_path / "mirror-ok", mount_path, cmdline_cache=systems_path / "mirror-ok/cache.yaml" + clean_root, mount_path, mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache.yaml" ) - # the command line cache overrides any build cache defined in mirrors.yaml, + # the command line cache overrides any build cache defined in the mirror file, # and is named "buildcache" like any other build cache. assert mirrors.build_cache_mirror == "buildcache" assert mirrors.buildcache["name"] == "buildcache" @@ -84,16 +123,16 @@ def test_command_line_cache(systems_path, mount_path): # it has a signing key, so it is pushed to assert mirrors.push_to_build_cache == "buildcache" - # the bootstrap and source mirrors from mirrors.yaml are still present + # the bootstrap and source mirrors from the mirror file are still present assert mirrors.bootstrap is not None assert set(mirrors.source_mirrors) == {"mirror1", "mirror2"} -def test_keyless_command_line_cache(tmp_path, systems_path, mount_path): +def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_path, mirror_ok): """A cache.yaml without a key configures a read-only (fetch-only) build cache.""" mirrors = mirror.Mirrors( - systems_path / "mirror-ok", mount_path, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" + clean_root, mount_path, mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" ) # the cache exists (so it is fetched from), but has no signing key ... @@ -113,10 +152,10 @@ def test_keyless_command_line_cache(tmp_path, systems_path, mount_path): assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}} -def test_readonly_buildcache(tmp_path, systems_path, mount_path): - """A buildcache in mirrors.yaml without a private_key is read-only (fetch-only).""" +def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): + """A buildcache in the mirror file without a private_key is read-only (fetch-only).""" - mirrors = mirror.Mirrors(systems_path / "mirror-readonly-cache", mount_path) + mirrors = mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / "mirror-readonly-cache/mirrors.yaml") # the cache exists (so it is fetched from), but has no signing key ... assert mirrors.build_cache_mirror == "buildcache" @@ -135,10 +174,14 @@ def test_readonly_buildcache(tmp_path, systems_path, mount_path): assert data["mirrors"]["buildcache"] == {"fetch": {"url": "https://mirror.spack.io"}} -def test_config_files(tmp_path, systems_path, mount_path): - """Check that config_files presents the complete set of mirror config artifacts.""" +def test_config_files(tmp_path, clean_root, mount_path, mirror_ok): + """Check that config_files presents the complete set of mirror config artifacts. - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + The mirror file lives in a different directory from the (empty) system config root, + so this also exercises relative key paths resolving against the mirror file's dir. + """ + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) expected = { @@ -156,7 +199,7 @@ def test_config_files(tmp_path, systems_path, mount_path): assert all(isinstance(content, bytes) for content in files.values()) -def test_spack_mirrors_yaml(tmp_path, systems_path, mount_path): +def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): """Check that the mirrors.yaml passed to spack is correct""" valid_spack_yaml = { @@ -182,21 +225,21 @@ def test_spack_mirrors_yaml(tmp_path, systems_path, mount_path): } } - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) assert data == valid_spack_yaml -def test_mount_specific_buildcache(tmp_path, systems_path, mount_path): +def test_mount_specific_buildcache(tmp_path, clean_root, mount_path, mirror_ok): """A mount_specific buildcache should have the mount point appended to its url. Spack binaries embed the install prefix (the mount point), so a mount_specific cache is namespaced per-mount-point to avoid relocation issues / collisions. """ - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) # mirror-ok's buildcache is mount_specific: false by default; enable it. mirrors_obj.buildcache["mount_specific"] = True @@ -212,10 +255,10 @@ def test_mount_specific_buildcache(tmp_path, systems_path, mount_path): assert data["mirrors"]["mirror1"]["fetch"]["url"] == "https://github.com" -def test_mount_specific_disabled(tmp_path, systems_path, mount_path): +def test_mount_specific_disabled(tmp_path, clean_root, mount_path, mirror_ok): """A buildcache with mount_specific false is unchanged, even when a mount point is set.""" - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) # confirm the fixture leaves the flag off assert mirrors_obj.buildcache["mount_specific"] is False @@ -226,7 +269,7 @@ def test_mount_specific_disabled(tmp_path, systems_path, mount_path): assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io" -def test_bootstrap_configs(tmp_path, systems_path, mount_path): +def test_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { @@ -247,7 +290,7 @@ def test_bootstrap_configs(tmp_path, systems_path, mount_path): }, } - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) bs_data = yaml.safe_load(files[tmp_path / "bootstrap.yaml"]) @@ -257,10 +300,10 @@ def test_bootstrap_configs(tmp_path, systems_path, mount_path): assert metadata == valid_metadata -def test_keys(systems_path, tmp_path, mount_path): +def test_keys(tmp_path, clean_root, mount_path, mirror_ok): """Check that gpg keys are decoded, relocated and reported consistently.""" - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) # public keys are set up for the buildcache and mirror1 @@ -276,21 +319,23 @@ def test_keys(systems_path, tmp_path, mount_path): assert set(mirrors_obj.gpg_key_paths(tmp_path)) == key_files -def test_source_cache_config(tmp_path, systems_path, mount_path): +def test_source_cache_config(tmp_path, clean_root, mount_path, mirror_ok): """The writable source cache is emitted to config.yaml as config:source_cache.""" - mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) data = yaml.safe_load(files[tmp_path / "config.yaml"]) assert data == {"config": {"source_cache": "/scratch/spack-sources"}} -def test_source_cache_absent(tmp_path, systems_path, mount_path): +def test_source_cache_absent(tmp_path, clean_root, mount_path, systems_path): """No config.yaml is generated when no source cache is configured.""" # mirror-no-sourcecache has no sourcecache entry - mirrors_obj = mirror.Mirrors(systems_path / "mirror-no-sourcecache", mount_path) + mirrors_obj = mirror.Mirrors( + clean_root, mount_path, mirror_file=systems_path / "mirror-no-sourcecache/mirrors.yaml" + ) files = mirrors_obj.config_files(tmp_path) assert mirrors_obj.source_cache is None @@ -305,8 +350,8 @@ def test_source_cache_absent(tmp_path, systems_path, mount_path): "mirror-bad-sourcecache", ], ) -def test_bad_config(systems_path, mount_path, system_name): +def test_bad_config(clean_root, mount_path, systems_path, system_name): """Check that MirrorError is raised at construction for bad keys or a bad source cache path.""" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(systems_path / system_name, mount_path) + mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / system_name / "mirrors.yaml") From 3ebb4845729511788bb6adecc640d60329a67d09 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 17:00:03 +0200 Subject: [PATCH 82/98] support misc caches in mirrors: useful for concretization caches --- docs/build-caches.md | 21 +++++++++- stackinator/mirror.py | 38 +++++++++++++------ stackinator/schema/mirror.json | 9 +++++ unittests/data/systems/mirror-ok/mirrors.yaml | 4 +- unittests/test_mirrors.py | 38 +++++++++++++++---- 5 files changed, 89 insertions(+), 21 deletions(-) diff --git a/docs/build-caches.md b/docs/build-caches.md index 4d5d1ae3..d3930f60 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -10,7 +10,7 @@ stack-config -b $build -r $recipe -s $system --mirror mirrors.yaml The file is not part of the [system configuration](cluster-config.md): mirror locations are usually specific to the person running the build, so each invocation provides its own. Paths inside the file (such as relative gpg key paths) are resolved relative to the directory containing the `mirrors.yaml`, so a self-contained mirror directory (the `mirrors.yaml` plus its keys) can be moved around freely. -A `mirrors.yaml` can describe four kinds of entry, each optional and each documented below: +A `mirrors.yaml` can describe five kinds of entry, each optional and each documented below: | Entry | Count | Purpose | |-------|-------|---------| @@ -18,6 +18,7 @@ A `mirrors.yaml` can describe four kinds of entry, each optional and each docume | [`bootstrap`](#bootstrap-mirror) | one | mirror used to bootstrap Spack itself | | [`sourcemirror`](#source-mirrors) | many | read-only mirrors that provide package sources | | [`sourcecache`](#source-cache) | one | writable local cache that fills with sources as you build | +| [`misccache`](#misc-cache) | one | writable local cache for Spack's indices and concretization results | A complete example: @@ -33,6 +34,8 @@ sourcemirror: url: https://example.com/spack-sources sourcecache: path: /capstor/scratch/$USER/spack-sources +misccache: + path: /capstor/scratch/$USER/spack-misc ``` To stop using any entry, remove (or comment out) it from `mirrors.yaml`. @@ -160,6 +163,22 @@ sourcecache: |-------|----------|-------------| | `path` | yes | absolute path to a local directory (environment variables are expanded) | +## Misc cache + +The misc cache is a single, **writable** local directory for Spack's "misc" cache: the package and build-cache indices, and — importantly — the **concretization cache**, which stores the result of concretizing a set of specs so it does not have to be recomputed. +Concretization can be a large fraction of build time, so pointing this at a persistent location is worthwhile when build directories are ephemeral (e.g. created in `/dev/shm` and deleted after each build). + +```yaml title="mirrors.yaml" +misccache: + path: /capstor/scratch/$USER/spack-misc +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `path` | yes | absolute path to a local directory (environment variables are expanded) | + +The concretization cache lives under this directory. Spack populates it automatically: concretization caching is on by default in Spack 1.2 and later, and is opt-in (`concretizer:concretization_cache:enable`) in Spack 1.1. It is keyed by the hash of the solver inputs, so a persistent cache can be reused safely across builds — stale entries simply miss. + ## Keys The `private_key` and `public_key` fields accept either: diff --git a/stackinator/mirror.py b/stackinator/mirror.py index ca26c05c..6712ed90 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -47,6 +47,11 @@ class Mirrors: it fetches sources (spack config:source_cache). None if absent. This is not a mirror: it has no key and no url, and is emitted to config.yaml rather than mirrors.yaml. + * misc_cache - at most one, a writable local directory for spack's misc + cache (package/build-cache indices and the concretization + cache that lives under it) (spack config:misc_cache). None + if absent. Like source_cache it is not a mirror and is + emitted to config.yaml. All input processing - loading and schema-validating the system mirrors.yaml, validating urls, and reading/decoding/validating gpg keys - happens eagerly in @@ -86,6 +91,7 @@ def __init__( self.bootstrap: Optional[Dict] = None self.source_mirrors: Dict[str, Dict] = {} self.source_cache: Optional[Dict] = None + self.misc_cache: Optional[Dict] = None # The mirror configuration is supplied with --mirror, not the system # configuration. Reject a mirrors.yaml in the system config so it is not @@ -154,24 +160,26 @@ def __init__( ) # The bootstrap mirror, the read-only source mirrors, and the writable - # source cache, if any are defined. + # source and misc caches, if any are defined. self.bootstrap = raw_mirrors.get("bootstrap") self.source_mirrors = dict(raw_mirrors["sourcemirror"]) self.source_cache = raw_mirrors.get("sourcecache") + self.misc_cache = raw_mirrors.get("misccache") # Validate that every mirror url is well-formed (see _validate_url). for name, mirror in self._iter_mirrors(): self._validate_url(mirror["url"], name) - # The source cache is a single writable local directory (spack - # config:source_cache), not a mirror: validate that it is an absolute path. - # Expand env vars now, because the build sandbox runs `env --ignore-environment` - # and so would not expand them at build time. - if self.source_cache is not None: - path = os.path.expandvars(self.source_cache["path"]) - if not pathlib.Path(path).is_absolute(): - raise MirrorError(f"The source cache path '{path}' is not absolute") - self.source_cache["path"] = path + # The source and misc caches are single writable local directories (spack + # config:source_cache / config:misc_cache), not mirrors: validate that each is + # an absolute path. Expand env vars now, because the build sandbox runs + # `env --ignore-environment` and so would not expand them at build time. + for cache_name, cache in (("source", self.source_cache), ("misc", self.misc_cache)): + if cache is not None: + path = os.path.expandvars(cache["path"]) + if not pathlib.Path(path).is_absolute(): + raise MirrorError(f"The {cache_name} cache path '{path}' is not absolute") + cache["path"] = path # Read, decode and validate every gpg key into memory. Each key is stored # as (path-relative-to-config-root, raw bytes); the builder writes these @@ -341,9 +349,15 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: spack_mirrors, default_flow_style=False, sort_keys=False ).encode() - # the spack config.yaml setting the writable, populate-as-you-go source cache + # the spack config.yaml setting the writable local caches: the populate-as-you-go + # source cache and the misc cache (which holds the concretization cache) + config_section = {} if self.source_cache is not None: - config_yaml = {"config": {"source_cache": self.source_cache["path"]}} + config_section["source_cache"] = self.source_cache["path"] + if self.misc_cache is not None: + config_section["misc_cache"] = self.misc_cache["path"] + if config_section: + config_yaml = {"config": config_section} files[config_root / self.CONFIG_YAML] = yaml.dump(config_yaml, default_flow_style=False).encode() # the bootstrap config and its mirror metadata, if a bootstrap mirror is set diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 1612ace0..cf6777d4 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -57,6 +57,15 @@ }, "additionalProperties": false, "required": ["path"] + }, + "misccache": { + "type": "object", + "properties": { + "description": {"type": "string", "default": ""}, + "path": {"type": "string"} + }, + "additionalProperties": false, + "required": ["path"] } } } \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 1f959ec2..87723154 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -53,4 +53,6 @@ sourcemirror: mirror2: url: https://github.com/spack sourcecache: - path: /scratch/spack-sources \ No newline at end of file + path: /scratch/spack-sources +misccache: + path: /scratch/spack-misc \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index cc765f10..1d698747 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -67,6 +67,9 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): # the writable, populate-as-you-go source cache assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources", "description": ""} + # the writable misc cache (holds the concretization cache) + assert mirrors_obj.misc_cache == {"path": "/scratch/spack-misc", "description": ""} + # the build cache mirror name is derived from the build cache's 'name' field assert mirrors_obj.build_cache_mirror == "buildcache" @@ -319,26 +322,46 @@ def test_keys(tmp_path, clean_root, mount_path, mirror_ok): assert set(mirrors_obj.gpg_key_paths(tmp_path)) == key_files -def test_source_cache_config(tmp_path, clean_root, mount_path, mirror_ok): - """The writable source cache is emitted to config.yaml as config:source_cache.""" +def test_local_caches_config(tmp_path, clean_root, mount_path, mirror_ok): + """The writable source and misc caches are emitted to config.yaml under config:.""" mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) data = yaml.safe_load(files[tmp_path / "config.yaml"]) - assert data == {"config": {"source_cache": "/scratch/spack-sources"}} + assert data == { + "config": { + "source_cache": "/scratch/spack-sources", + "misc_cache": "/scratch/spack-misc", + } + } + + +def test_misc_cache_only(tmp_path, clean_root, mount_path): + """A mirror file with a misc cache but no source cache still emits config:misc_cache.""" + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text("misccache:\n path: /scratch/only-misc\n") + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + files = mirrors_obj.config_files(tmp_path) + + assert mirrors_obj.source_cache is None + data = yaml.safe_load(files[tmp_path / "config.yaml"]) + assert data == {"config": {"misc_cache": "/scratch/only-misc"}} -def test_source_cache_absent(tmp_path, clean_root, mount_path, systems_path): - """No config.yaml is generated when no source cache is configured.""" +def test_local_caches_absent(tmp_path, clean_root, mount_path, systems_path): + """No config.yaml is generated when neither a source nor a misc cache is configured.""" - # mirror-no-sourcecache has no sourcecache entry + # mirror-no-sourcecache has neither a sourcecache nor a misccache entry mirrors_obj = mirror.Mirrors( clean_root, mount_path, mirror_file=systems_path / "mirror-no-sourcecache/mirrors.yaml" ) files = mirrors_obj.config_files(tmp_path) assert mirrors_obj.source_cache is None + assert mirrors_obj.misc_cache is None assert tmp_path / "config.yaml" not in files @@ -348,10 +371,11 @@ def test_source_cache_absent(tmp_path, clean_root, mount_path, systems_path): "mirror-bad-key", "mirror-bad-keypath", "mirror-bad-sourcecache", + "mirror-bad-misccache", ], ) def test_bad_config(clean_root, mount_path, systems_path, system_name): - """Check that MirrorError is raised at construction for bad keys or a bad source cache path.""" + """Check that MirrorError is raised at construction for bad keys or a bad cache path.""" with pytest.raises(mirror.MirrorError): mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / system_name / "mirrors.yaml") From 5e88905604c8f51223db3302483a098f2646b254 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 17:03:38 +0200 Subject: [PATCH 83/98] update agent instructions --- CLAUDE.md | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 08d4c68b..e141617f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ Stackinator is a Python CLI tool that generates build configurations for scienti ## Two-Phase Workflow ``` -stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] +stack-config -b BUILD -r RECIPE -s SYSTEM [--mirror MIRRORS] [-m MOUNT] # [-c CACHE] is legacy → generates BUILD/ directory with Makefiles + spack.yaml files cd BUILD @@ -22,14 +22,15 @@ stackinator/ # Python package main.py # CLI entry point (stack-config) recipe.py # Recipe class: parses and validates all recipe YAML builder.py # Builder class: writes all files to the build path - cache.py # Build cache configuration helpers + mirror.py # Mirrors class: validates the --mirror mirrors.yaml, emits spack mirror/cache config spack_util.py # Tiny helper: checks if a path is a spack package repo schema.py # JSON schema validators with default-injection schema/ # JSON schemas for each YAML file type config.json compilers.json environments.json - cache.json + mirror.json # mirrors.yaml schema (buildcache/bootstrap/sourcemirror/sourcecache/misccache) + cache.json # legacy -c/--cache cache.yaml schema modules.json templates/ # Jinja2 templates for all generated files Makefile # Top-level build orchestration @@ -176,6 +177,8 @@ cluster-config/ repos.yaml # optional; list of relative paths to site-wide spack repos ``` +Mirror/cache config is **not** part of the cluster configuration — it is supplied separately with `--mirror` (a `mirrors.yaml` here is rejected). See [Mirrors and Build Caches](#mirrors-and-build-caches). + `network.yaml` structure: ```yaml mpi: @@ -203,8 +206,11 @@ BUILD/ spack-packages/ # cloned spack-packages repository config/ # global spack configuration scope packages.yaml - mirrors.yaml # only if --cache provided repos.yaml + mirrors.yaml # if --mirror provided: buildcache/bootstrap/sourcemirror entries + config.yaml # if --mirror provided with a sourcecache/misccache (config:source_cache/misc_cache) + bootstrap.yaml # if --mirror provided with a bootstrap mirror + key_store/ # if --mirror provided with gpg keys (decoded *.gpg) compilers/ Makefile gcc/ @@ -254,6 +260,14 @@ Writes all files to the build path. Key responsibilities: - Renders all Jinja templates into build path files - Writes metadata JSON files +### `Mirrors` class (`mirror.py`) +A clean exemplar of the "recipe validates & renders, builder just prints" pattern. Constructed by `Recipe` from the `--mirror` file path; does ALL mirror input processing eagerly in `__init__` (loads + schema-validates `mirrors.yaml`, validates mirror urls, decodes/validates gpg keys to in-memory bytes, checks cache paths are absolute and expands env vars). Then presents pure static artifacts: +- typed members: `buildcache`, `bootstrap`, `source_mirrors`, `source_cache`, `misc_cache` +- `config_files(config_root) -> {abs_path: bytes}` — the `mirrors.yaml`, `config.yaml`, `bootstrap.yaml`, and gpg key files the builder writes verbatim +- `gpg_key_paths(config_root)` and the `build_cache_mirror` / `push_to_build_cache` properties (the latter is `None` for a keyless, read-only build cache) + +Mirror/cache config is supplied ONLY via `--mirror`; a `mirrors.yaml` found in the system config dir is rejected with an error (it was never a system-config artifact). Relative gpg-key paths resolve against the `--mirror` file's own directory. + ### `schema.py` JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. @@ -269,7 +283,7 @@ The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for proces The top-level `Makefile` orchestrates in order: 1. `spack-setup` — sanity check, bootstrap concretizer 2. `pre-install` — run `pre-install-hook` if provided -3. `mirror-setup` — configure build cache keys +3. `mirror-setup` — trust build-cache/mirror gpg keys (`cache-force` force-pushes built packages) 4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) 5. `environments` — build all user environments (parallel) 6. `generate-config` — generate the upstream spack config files for the installed image @@ -288,15 +302,19 @@ The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: - Bind-mounts `BUILD/tmp` → `/tmp` - Puts a tmpfs over `$HOME` (isolates user config) -## Build Cache +## Mirrors and Build Caches -Optional binary cache configured via YAML file passed to `-c/--cache`: -```yaml -root: /path/to/cache # directory; env vars expanded -key: /path/to/pgp.key # optional; omit for read-only cache -``` +Spack mirrors and caches are configured in a single `mirrors.yaml` supplied with `stack-config --mirror ` (see `docs/build-caches.md` for the full reference). It can describe five optional entities: + +- **`buildcache`** (one): binary cache of built packages — the big build-time speed up. With a `private_key` it signs and pushes packages it builds; without one it is read-only (fetch only). `mount_specific: true` stores binaries in a per-mount-point subdir (Spack binaries embed the install prefix, so each mount point needs its own cache). Packages are pushed per-environment after a successful build; `cuda`, `nvhpc`, `perl` are excluded from pushes. `make cache-force` force-pushes everything built so far. +- **`bootstrap`** (one): mirror for bootstrapping Spack itself (air-gapped clingo etc.). +- **`sourcemirror`** (many): read-only mirrors providing package source archives. +- **`sourcecache`** (one): a writable local dir Spack fills as it fetches sources → emitted as `config:source_cache`. +- **`misccache`** (one): a writable local dir for Spack's misc cache (package/build-cache indices and the **concretization cache** that lives under it) → emitted as `config:misc_cache`. Useful to persist across ephemeral builds. Concretization caching is on by default in Spack ≥ 1.2, opt-in in 1.1. + +`sourcecache`/`misccache` are emitted to `config/config.yaml`; `buildcache`/`bootstrap`/`sourcemirror` to `config/mirrors.yaml` (+ `bootstrap.yaml` + decoded keys under `config/key_store/`). -Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed per-environment after a successful build. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. +**Legacy:** a binary cache can still be configured with a `cache.yaml` (`root` + optional `key`) passed to `-c/--cache`. This path is deprecated in favour of a `buildcache` entry and will be removed. ## Testing @@ -323,7 +341,9 @@ The test coverage is limited — the schema validators and their default-injecti - **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. - **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. - **View names are globally unique**: view names must be unique across all environments in a recipe. -- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. +- **Mirror/cache config comes only from `--mirror`**: never from the recipe or the system/cluster config. A `mirrors.yaml` in the system config dir is rejected with an error. +- **Read-only build cache**: a `buildcache` without a `private_key` is fetched from but never pushed to (`push_to_build_cache` is `None`). +- **Schema-injected defaults**: every non-required field has a `default` in its JSON schema, injected at every level (including `additionalProperties` maps). Rely on fields always being present and check `is None` — do not use `.get()` to guard existence. - **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). - **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. - **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. From d8fa7071bf8bb7e6feb7ea625959e5e85c54962c Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 17:20:49 +0200 Subject: [PATCH 84/98] tests from last commit --- unittests/data/systems/mirror-bad-misccache/mirrors.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 unittests/data/systems/mirror-bad-misccache/mirrors.yaml diff --git a/unittests/data/systems/mirror-bad-misccache/mirrors.yaml b/unittests/data/systems/mirror-bad-misccache/mirrors.yaml new file mode 100644 index 00000000..ec06c0b6 --- /dev/null +++ b/unittests/data/systems/mirror-bad-misccache/mirrors.yaml @@ -0,0 +1,2 @@ +misccache: + path: relative/not/absolute From 7c92da2692ad2739016ee6e36d907fe948b216ac Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 5 Jun 2026 17:31:01 +0200 Subject: [PATCH 85/98] boostrap cache is read only without a key: simplify if it is on a local filesystem --- CLAUDE.md | 6 +-- docs/build-caches.md | 26 +++++++-- stackinator/mirror.py | 96 ++++++++++++++++++++++++++-------- stackinator/schema/mirror.json | 3 +- unittests/test_mirrors.py | 58 +++++++++++++++++--- 5 files changed, 151 insertions(+), 38 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e141617f..3985abf9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,7 +207,7 @@ BUILD/ config/ # global spack configuration scope packages.yaml repos.yaml - mirrors.yaml # if --mirror provided: buildcache/bootstrap/sourcemirror entries + mirrors.yaml # if --mirror provided: buildcache/sourcemirror entries config.yaml # if --mirror provided with a sourcecache/misccache (config:source_cache/misc_cache) bootstrap.yaml # if --mirror provided with a bootstrap mirror key_store/ # if --mirror provided with gpg keys (decoded *.gpg) @@ -307,12 +307,12 @@ The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: Spack mirrors and caches are configured in a single `mirrors.yaml` supplied with `stack-config --mirror ` (see `docs/build-caches.md` for the full reference). It can describe five optional entities: - **`buildcache`** (one): binary cache of built packages — the big build-time speed up. With a `private_key` it signs and pushes packages it builds; without one it is read-only (fetch only). `mount_specific: true` stores binaries in a per-mount-point subdir (Spack binaries embed the install prefix, so each mount point needs its own cache). Packages are pushed per-environment after a successful build; `cuda`, `nvhpc`, `perl` are excluded from pushes. `make cache-force` force-pushes everything built so far. -- **`bootstrap`** (one): mirror for bootstrapping Spack itself (air-gapped clingo etc.). +- **`bootstrap`** (one): for bootstrapping Spack itself (clingo etc.). The `url` is either a local `spack bootstrap mirror` directory (referenced via its own `metadata/sources`+`metadata/binaries`) or a remote url (source-only). Needs **no key** (bootstrap binaries are sha256-verified) → emitted as `config/bootstrap.yaml` (+ a generated `metadata.yaml` only for the remote case); it is NOT a `mirrors.yaml` entry. - **`sourcemirror`** (many): read-only mirrors providing package source archives. - **`sourcecache`** (one): a writable local dir Spack fills as it fetches sources → emitted as `config:source_cache`. - **`misccache`** (one): a writable local dir for Spack's misc cache (package/build-cache indices and the **concretization cache** that lives under it) → emitted as `config:misc_cache`. Useful to persist across ephemeral builds. Concretization caching is on by default in Spack ≥ 1.2, opt-in in 1.1. -`sourcecache`/`misccache` are emitted to `config/config.yaml`; `buildcache`/`bootstrap`/`sourcemirror` to `config/mirrors.yaml` (+ `bootstrap.yaml` + decoded keys under `config/key_store/`). +`sourcecache`/`misccache` are emitted to `config/config.yaml`; `bootstrap` to `config/bootstrap.yaml`; `buildcache`/`sourcemirror` to `config/mirrors.yaml` (+ decoded keys under `config/key_store/`). **Legacy:** a binary cache can still be configured with a `cache.yaml` (`root` + optional `key`) passed to `-c/--cache`. This path is deprecated in favour of a `buildcache` entry and will be removed. diff --git a/docs/build-caches.md b/docs/build-caches.md index d3930f60..d5db643a 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -112,15 +112,35 @@ env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make cache-force Spack bootstraps some of its own dependencies (such as the `clingo` concretizer) on first use. A bootstrap mirror lets it do this without reaching the internet — useful on air-gapped systems. +No key is needed: bootstrap binaries are verified by their sha256 sum, not by a GPG signature. + +The `url` can take two forms. + +**A local bootstrap mirror directory** (recommended) — a directory created with `spack bootstrap mirror`, which contains its own `metadata/sources` and `metadata/binaries` descriptors: + ```yaml title="mirrors.yaml" bootstrap: - url: https://bootstrap.spack.io + url: /capstor/scratch/team/bootstrap-mirror +``` + +Stackinator references the mirror's own metadata directly, so both source and binary bootstrapping work. Create one on a connected system and copy it across: + +```bash +spack bootstrap mirror --binary-packages /capstor/scratch/team/bootstrap-mirror ``` +**A remote url** — `https://`, `s3://` or `oci://`: + +```yaml title="mirrors.yaml" +bootstrap: + url: https://bootstrap.example.com/mirror +``` + +A remote mirror supports **source** bootstrapping only; remote binary bootstrapping is not supported (it needs the per-package metadata that a local mirror directory provides). + | Field | Required | Description | |-------|----------|-------------| -| `url` | yes | location of the bootstrap mirror | -| `public_key` | no | PGP key used to verify the bootstrap binaries | +| `url` | yes | a local `spack bootstrap mirror` directory, or a remote `https`/`s3`/`oci` url | ## Source mirrors diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 6712ed90..09f59aa1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -40,7 +40,10 @@ class Mirrors: has the mount_specific flag). With a private key it signs and pushes packages too; without one it is read-only. None if no build cache is configured. - * bootstrap - at most one, used to bootstrap spack itself. None if absent. + * bootstrap - at most one, used to bootstrap spack itself (a local spack + bootstrap mirror directory, or a remote url). Needs no key + (bootstrap binaries are sha256-verified) and is emitted to + bootstrap.yaml, not the mirrors list. None if absent. * source_mirrors - a name -> config mapping of any number of read-only source mirrors (spack mirrors.yaml entries). * source_cache - at most one, a writable local directory that spack fills as @@ -64,7 +67,7 @@ class Mirrors: KEY_STORE_DIR = "key_store" MIRRORS_YAML = "mirrors.yaml" CONFIG_YAML = "config.yaml" - BOOTSTRAP_MIRROR = "bootstrap" + BOOTSTRAP_YAML = "bootstrap.yaml" def __init__( self, @@ -181,6 +184,30 @@ def __init__( raise MirrorError(f"The {cache_name} cache path '{path}' is not absolute") cache["path"] = path + # Resolve the bootstrap mirror. It is either a remote url, or a local spack + # bootstrap mirror directory (a `spack bootstrap mirror` output) whose own + # metadata/{sources,binaries} directories we reference directly. + self._bootstrap_remote = False + self._bootstrap_root: Optional[str] = None + self._bootstrap_metadata_dirs: List[str] = [] + if self.bootstrap is not None: + url = self.bootstrap["url"] + self._validate_url(url, "bootstrap") + if self._is_remote_url(url): + self._bootstrap_remote = True + else: + root = self._local_path(url) + if not root.is_dir(): + raise MirrorError(f"The bootstrap mirror directory '{root}' does not exist") + present = [sub for sub in ("sources", "binaries") if (root / "metadata" / sub).is_dir()] + if not present: + raise MirrorError( + f"The bootstrap mirror directory '{root}' has no 'metadata/sources' or " + f"'metadata/binaries' directory (is it a 'spack bootstrap mirror' output?)." + ) + self._bootstrap_root = root.as_posix() + self._bootstrap_metadata_dirs = present + # Read, decode and validate every gpg key into memory. Each key is stored # as (path-relative-to-config-root, raw bytes); the builder writes these # verbatim into the build directory's key store. @@ -224,18 +251,33 @@ def push_to_build_cache(self) -> Optional[str]: return None def _iter_mirrors(self): - """Yield (spack mirror name, config dict) for every configured mirror. + """Yield (spack mirror name, config dict) for every real spack mirror. - The build cache's name is configurable (the 'name' field); the bootstrap - and source mirrors are named by their key in mirrors.yaml. + These are the entries written to the spack mirrors.yaml that may carry a + public key: the build cache (whose name is the configurable 'name' field) and + the source mirrors (named by their key in mirrors.yaml). The bootstrap mirror + is not a mirrors.yaml entry and has no key, so it is handled separately. """ if self.buildcache is not None: yield self.buildcache["name"], self.buildcache - if self.bootstrap is not None: - yield self.BOOTSTRAP_MIRROR, self.bootstrap yield from self.source_mirrors.items() + @staticmethod + def _is_remote_url(url: str) -> bool: + """True if url is a remote url (has a non-file scheme and a host).""" + + parsed = urllib.parse.urlparse(url) + return bool(parsed.scheme) and parsed.scheme != "file" and bool(parsed.netloc) + + @staticmethod + def _local_path(url: str) -> pathlib.Path: + """The local filesystem path for a file:// url or a bare path (env vars expanded).""" + + if url.startswith("file://"): + url = url[len("file://") :] + return pathlib.Path(os.path.expandvars(url)) + def _validate_url(self, url: str, name: str): """Validate that a mirror url is well-formed. @@ -333,10 +375,6 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: entry["push"] = {"url": url} spack_mirrors["mirrors"][self.buildcache["name"]] = entry - if self.bootstrap is not None: - url = self.bootstrap["url"] - spack_mirrors["mirrors"][self.BOOTSTRAP_MIRROR] = {"fetch": {"url": url}, "push": {"url": url}} - # source mirrors are read-only and provide sources only: fetch url, no push. for name, mirror in self.source_mirrors.items(): spack_mirrors["mirrors"][name] = { @@ -360,18 +398,30 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: config_yaml = {"config": config_section} files[config_root / self.CONFIG_YAML] = yaml.dump(config_yaml, default_flow_style=False).encode() - # the bootstrap config and its mirror metadata, if a bootstrap mirror is set + # the spack bootstrap.yaml, if a bootstrap mirror is set. Bootstrapping reads + # bootstrap:sources (not the mirrors list), and each source's `metadata` is a + # directory describing it. No gpg key is involved (bootstrap binaries are + # sha256-verified). if self.bootstrap is not None: - metadata_dir = config_root / "bootstrap" / "bootstrap-mirror" - bootstrap_yaml = { - "bootstrap": { - "sources": [{"name": "bootstrap-mirror", "metadata": str(metadata_dir)}], - "trusted": {"bootstrap-mirror": True}, - } - } - metadata_yaml = {"type": "install", "info": {"url": self.bootstrap["url"]}} - - files[metadata_dir / "metadata.yaml"] = yaml.dump(metadata_yaml, default_flow_style=False).encode() - files[config_root / "bootstrap.yaml"] = yaml.dump(bootstrap_yaml, default_flow_style=False).encode() + sources = [] + trusted = {} + if self._bootstrap_remote: + # a remote mirror: generate a local source descriptor pointing at it. + # this covers source bootstrapping; remote binary bootstrapping (which + # needs per-package sha256 metadata) is not supported. + metadata_dir = config_root / "bootstrap" / "bootstrap-mirror" + metadata_yaml = {"type": "install", "info": {"url": self.bootstrap["url"]}} + files[metadata_dir / "metadata.yaml"] = yaml.dump(metadata_yaml, default_flow_style=False).encode() + sources.append({"name": "bootstrap-mirror", "metadata": str(metadata_dir)}) + trusted["bootstrap-mirror"] = True + else: + # a local spack bootstrap mirror: reference its own metadata directories. + for sub in self._bootstrap_metadata_dirs: + name = f"bootstrap-{sub}" + sources.append({"name": name, "metadata": f"{self._bootstrap_root}/metadata/{sub}"}) + trusted[name] = True + + bootstrap_yaml = {"bootstrap": {"sources": sources, "trusted": trusted}} + files[config_root / self.BOOTSTRAP_YAML] = yaml.dump(bootstrap_yaml, default_flow_style=False).encode() return files diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index cf6777d4..374d1efe 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -6,8 +6,7 @@ "type": "object", "properties": { "description": {"type": "string", "default": ""}, - "url": {"type": "string"}, - "public_key": {"type": ["string", "null"], "default": null} + "url": {"type": "string"} }, "additionalProperties": false, "required": ["url"] diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 1d698747..cd6f8ac6 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -55,7 +55,6 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): assert mirrors_obj.bootstrap == { "url": "https://mirror.spack.io", "description": "", - "public_key": None, } # non-required fields are always present, defaulted to "" / None by the schema @@ -207,10 +206,6 @@ def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): valid_spack_yaml = { "mirrors": { - "bootstrap": { - "fetch": {"url": "https://mirror.spack.io"}, - "push": {"url": "https://mirror.spack.io"}, - }, "buildcache": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, @@ -272,8 +267,8 @@ def test_mount_specific_disabled(tmp_path, clean_root, mount_path, mirror_ok): assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io" -def test_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): - """Check that spack bootstrap configs are generated correctly""" +def test_remote_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): + """A remote bootstrap url generates a local source descriptor pointing at it.""" valid_yaml = { "bootstrap": { @@ -302,6 +297,55 @@ def test_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): metadata = yaml.safe_load(files[tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml"]) assert metadata == valid_metadata + # the bootstrap mirror is not added to the spack mirrors list + mirrors = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert "bootstrap" not in mirrors["mirrors"] + + +def test_local_bootstrap_configs(tmp_path, clean_root, mount_path): + """A local bootstrap mirror dir is referenced via its own metadata directories.""" + + # fake a `spack bootstrap mirror` output directory + boot = tmp_path / "bootstrap-mirror" + (boot / "metadata" / "sources").mkdir(parents=True) + (boot / "metadata" / "binaries").mkdir(parents=True) + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text(f"bootstrap:\n url: {boot}\n") + + config_root = tmp_path / "config" + mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + files = mirrors_obj.config_files(config_root) + + bs_data = yaml.safe_load(files[config_root / "bootstrap.yaml"]) + assert bs_data == { + "bootstrap": { + "sources": [ + {"name": "bootstrap-sources", "metadata": f"{boot}/metadata/sources"}, + {"name": "bootstrap-binaries", "metadata": f"{boot}/metadata/binaries"}, + ], + "trusted": {"bootstrap-sources": True, "bootstrap-binaries": True}, + } + } + + # no metadata is generated (it lives in the mirror), and nothing in the mirrors list + assert not any(p.name == "metadata.yaml" for p in files) + mirrors = yaml.safe_load(files[config_root / "mirrors.yaml"]) + assert "bootstrap" not in mirrors["mirrors"] + + +def test_local_bootstrap_missing_metadata(tmp_path, clean_root, mount_path): + """A local bootstrap dir without metadata/sources|binaries is rejected.""" + + boot = tmp_path / "not-a-bootstrap-mirror" + boot.mkdir() + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text(f"bootstrap:\n url: {boot}\n") + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + def test_keys(tmp_path, clean_root, mount_path, mirror_ok): """Check that gpg keys are decoded, relocated and reported consistently.""" From adc0819772fad5a69ce1bf8f3ada423d81f7d6f1 Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 10 Jun 2026 11:51:25 +0200 Subject: [PATCH 86/98] uv only checks for upates when necessary --- bin/stack-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stack-config b/bin/stack-config index 02f48700..342a82df 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -1,4 +1,4 @@ -#!/usr/bin/env -S uv run --script +#!/usr/bin/env -S uv run --no-refresh --script # /// script # requires-python = ">=3.12" # dependencies = [ From 6c265e1ad83aae3e04c56d71a184bfa29470f712 Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 10 Jun 2026 11:59:07 +0200 Subject: [PATCH 87/98] do not re-index build cache every time we push --- stackinator/templates/Makefile | 6 +++--- stackinator/templates/Makefile.compilers | 2 +- stackinator/templates/Makefile.environments | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index cb411400..25c47e15 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -19,8 +19,8 @@ spack-setup: spack-version printf "spack version... "; \ version="$$($(SANDBOX) $(SPACK) --version)"; \ printf "%s\n" "$$version"; \ - printf "checking if spack concretizer works... "; \ - $(SANDBOX) $(SPACK_HELPER) -d spec zlib > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ + printf "bootstrapping spack... "; \ + $(SANDBOX) $(SPACK_HELPER) -d bootstrap now > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ if [ "$$?" != "0" ]; then \ printf " failed, see %s\n" $(BUILD_ROOT)/spack-bootstrap-output; \ exit 1; \ @@ -88,7 +88,7 @@ cache-force: mirror-setup $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package {{ buildcache_push }} \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --only=package {{ buildcache_push }} \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index 707d7b7d..0447863f 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -19,7 +19,7 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% for compiler, config in compilers.items() %} {{ compiler }}/generated/build_cache: {{ compiler }}/generated/env {% if buildcache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package {{ buildcache }} \ + $(SPACK) -e ./{{ compiler }} buildcache create --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | cut -d ';' -f2) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 929e40bc..74699d46 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -18,7 +18,7 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {% for env, config in environments.items() %} {{ env }}/generated/build_cache: {{ env }}/generated/view_config {% if buildcache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package {{ buildcache }} \ + $(SPACK) --color=never -e ./{{ env }} buildcache create --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From b8f847dcc25646d952a75ea91b3692c1843ca9ec Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 10 Jun 2026 12:22:08 +0200 Subject: [PATCH 88/98] remove redundant key from source mirrors --- docs/build-caches.md | 11 +++++---- stackinator/mirror.py | 24 +++++++++++-------- stackinator/schema/mirror.json | 3 +-- unittests/data/systems/mirror-ok/mirrors.yaml | 1 - unittests/test_mirrors.py | 20 ++++++++-------- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/docs/build-caches.md b/docs/build-caches.md index d5db643a..d7212bfb 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -158,8 +158,9 @@ sourcemirror: | Field | Required | Description | |-------|----------|-------------| -| `url` | yes | location of the source mirror | -| `public_key` | no | PGP key used to verify sources | +| `url` | yes | location of the source mirror | + +Source mirrors need no keys: Spack verifies every downloaded source against the checksum in its package recipe, whether it comes from the upstream url or a mirror. Populate a source mirror on an internet-connected system with Spack: @@ -172,7 +173,7 @@ spack mirror create --directory /path/to/mirror --all A source cache is a single, **writable** local directory that Spack fills as it downloads sources. On internet-connected systems Spack checks the cache first; on a miss it downloads the source and stores it, so later builds reuse it and download times shrink over time. -Unlike a source mirror it needs no key, is written to automatically, and is created on demand. +Unlike a source mirror it is written to automatically, and is created on demand. ```yaml title="mirrors.yaml" sourcecache: @@ -201,9 +202,9 @@ The concretization cache lives under this directory. Spack populates it automati ## Keys -The `private_key` and `public_key` fields accept either: +The build cache's `private_key` and `public_key` fields accept either: -* a **path** — absolute, or relative to the system configuration directory; or +* a **path** — absolute, or relative to the directory containing `mirrors.yaml`; or * a **base64-encoded key** inlined directly in `mirrors.yaml`. ```yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 09f59aa1..54cfa34c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -45,7 +45,9 @@ class Mirrors: (bootstrap binaries are sha256-verified) and is emitted to bootstrap.yaml, not the mirrors list. None if absent. * source_mirrors - a name -> config mapping of any number of read-only source - mirrors (spack mirrors.yaml entries). + mirrors (spack mirrors.yaml entries). They need no key: + sources are verified against the checksums in the package + recipes, never with gpg. * source_cache - at most one, a writable local directory that spack fills as it fetches sources (spack config:source_cache). None if absent. This is not a mirror: it has no key and no url, and @@ -222,11 +224,13 @@ def __init__( (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name)) ) - # Any mirror may provide a public key, used to verify downloaded packages. - for name, mirror in self._iter_mirrors(): - public_key = mirror["public_key"] - if public_key is not None: - self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(public_key, name))) + # The build cache may provide a public key, used to verify the packages + # fetched from it. It is the only mirror with keys at all: sources (and + # bootstrap binaries) are checksum-verified, and spack consults the gpg + # keyring only when verifying signed build-cache binaries. + if self.buildcache is not None and self.buildcache["public_key"] is not None: + name = self.buildcache["name"] + self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(self.buildcache["public_key"], name))) @property def build_cache_mirror(self) -> Optional[str]: @@ -253,10 +257,10 @@ def push_to_build_cache(self) -> Optional[str]: def _iter_mirrors(self): """Yield (spack mirror name, config dict) for every real spack mirror. - These are the entries written to the spack mirrors.yaml that may carry a - public key: the build cache (whose name is the configurable 'name' field) and - the source mirrors (named by their key in mirrors.yaml). The bootstrap mirror - is not a mirrors.yaml entry and has no key, so it is handled separately. + These are the entries written to the spack mirrors.yaml: the build cache + (whose name is the configurable 'name' field) and the source mirrors (named + by their key in mirrors.yaml). The bootstrap mirror is not a mirrors.yaml + entry, so it is handled separately. """ if self.buildcache is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 374d1efe..411a788d 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -41,8 +41,7 @@ "type": "object", "properties": { "description": {"type": "string", "default": ""}, - "url": {"type": "string"}, - "public_key": {"type": ["string", "null"], "default": null} + "url": {"type": "string"} }, "additionalProperties": false, "required": ["url"] diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 87723154..2ffdbd5c 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -49,7 +49,6 @@ buildcache: sourcemirror: mirror1: url: https://github.com - public_key: ../../test-gpg-pub.asc mirror2: url: https://github.com/spack sourcecache: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index cd6f8ac6..af6aea5b 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -57,10 +57,10 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): "description": "", } - # non-required fields are always present, defaulted to "" / None by the schema + # non-required fields are always present, defaulted to "" by the schema assert mirrors_obj.source_mirrors == { - "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc", "description": ""}, - "mirror2": {"url": "https://github.com/spack", "public_key": None, "description": ""}, + "mirror1": {"url": "https://github.com", "description": ""}, + "mirror2": {"url": "https://github.com/spack", "description": ""}, } # the writable, populate-as-you-go source cache @@ -193,7 +193,6 @@ def test_config_files(tmp_path, clean_root, mount_path, mirror_ok): tmp_path / "bootstrap" / "bootstrap-mirror" / "metadata.yaml", tmp_path / "key_store" / "buildcache.priv.gpg", tmp_path / "key_store" / "buildcache.pub.gpg", - tmp_path / "key_store" / "mirror1.pub.gpg", } assert set(files.keys()) == expected @@ -347,19 +346,20 @@ def test_local_bootstrap_missing_metadata(tmp_path, clean_root, mount_path): mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) -def test_keys(tmp_path, clean_root, mount_path, mirror_ok): +def test_keys(tmp_path, clean_root, mount_path, systems_path, mirror_ok): """Check that gpg keys are decoded, relocated and reported consistently.""" mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) - # public keys are set up for the buildcache and mirror1 + # the buildcache is the only mirror with keys (sources are checksum-verified) pub_files = {p for p in files if p.name.endswith(".pub.gpg")} - assert {p.name for p in pub_files} == {"buildcache.pub.gpg", "mirror1.pub.gpg"} + assert {p.name for p in pub_files} == {"buildcache.pub.gpg"} - # the buildcache public key (inline base64) and mirror1's (a file) are the same - # key, so the decoded bytes must match - assert files[tmp_path / "key_store/buildcache.pub.gpg"] == files[tmp_path / "key_store/mirror1.pub.gpg"] + # the buildcache public key is inlined base64 in the fixture; the decoded bytes + # must match the key file it was encoded from + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + assert files[tmp_path / "key_store/buildcache.pub.gpg"] == pub_key_file.read() # gpg_key_paths reports exactly the key files that config_files writes key_files = {p for p in files if p.parent.name == "key_store"} From 840d85a4f30b31ae1fe3fa8c0b3867d84eca65a6 Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 10 Jun 2026 15:22:42 +0200 Subject: [PATCH 89/98] misc cache -> explicit concretizer cache --- CLAUDE.md | 13 +-- docs/build-caches.md | 26 +++-- stackinator/mirror.py | 83 +++++++++++----- stackinator/recipe.py | 38 ++++++-- stackinator/schema/mirror.json | 2 +- .../mirrors.yaml | 2 +- unittests/data/systems/mirror-ok/mirrors.yaml | 4 +- unittests/test_mirrors.py | 97 +++++++++++-------- unittests/test_recipe.py | 39 ++++++++ 9 files changed, 218 insertions(+), 86 deletions(-) rename unittests/data/systems/{mirror-bad-misccache => mirror-bad-concretizer}/mirrors.yaml (69%) create mode 100644 unittests/test_recipe.py diff --git a/CLAUDE.md b/CLAUDE.md index 3985abf9..be2531cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ stackinator/ # Python package config.json compilers.json environments.json - mirror.json # mirrors.yaml schema (buildcache/bootstrap/sourcemirror/sourcecache/misccache) + mirror.json # mirrors.yaml schema (buildcache/bootstrap/sourcemirror/sourcecache/concretizer) cache.json # legacy -c/--cache cache.yaml schema modules.json templates/ # Jinja2 templates for all generated files @@ -208,7 +208,8 @@ BUILD/ packages.yaml repos.yaml mirrors.yaml # if --mirror provided: buildcache/sourcemirror entries - config.yaml # if --mirror provided with a sourcecache/misccache (config:source_cache/misc_cache) + config.yaml # if --mirror provided with a sourcecache (config:source_cache) + concretizer.yaml # if --mirror provided with a concretizer cache (concretizer:concretization_cache), spack >= 1.1 bootstrap.yaml # if --mirror provided with a bootstrap mirror key_store/ # if --mirror provided with gpg keys (decoded *.gpg) compilers/ @@ -262,8 +263,8 @@ Writes all files to the build path. Key responsibilities: ### `Mirrors` class (`mirror.py`) A clean exemplar of the "recipe validates & renders, builder just prints" pattern. Constructed by `Recipe` from the `--mirror` file path; does ALL mirror input processing eagerly in `__init__` (loads + schema-validates `mirrors.yaml`, validates mirror urls, decodes/validates gpg keys to in-memory bytes, checks cache paths are absolute and expands env vars). Then presents pure static artifacts: -- typed members: `buildcache`, `bootstrap`, `source_mirrors`, `source_cache`, `misc_cache` -- `config_files(config_root) -> {abs_path: bytes}` — the `mirrors.yaml`, `config.yaml`, `bootstrap.yaml`, and gpg key files the builder writes verbatim +- typed members: `buildcache`, `bootstrap`, `source_mirrors`, `source_cache`, `concretizer_cache` +- `config_files(config_root) -> {abs_path: bytes}` — the `mirrors.yaml`, `config.yaml`, `concretizer.yaml`, `bootstrap.yaml`, and gpg key files the builder writes verbatim - `gpg_key_paths(config_root)` and the `build_cache_mirror` / `push_to_build_cache` properties (the latter is `None` for a keyless, read-only build cache) Mirror/cache config is supplied ONLY via `--mirror`; a `mirrors.yaml` found in the system config dir is rejected with an error (it was never a system-config artifact). Relative gpg-key paths resolve against the `--mirror` file's own directory. @@ -310,9 +311,9 @@ Spack mirrors and caches are configured in a single `mirrors.yaml` supplied with - **`bootstrap`** (one): for bootstrapping Spack itself (clingo etc.). The `url` is either a local `spack bootstrap mirror` directory (referenced via its own `metadata/sources`+`metadata/binaries`) or a remote url (source-only). Needs **no key** (bootstrap binaries are sha256-verified) → emitted as `config/bootstrap.yaml` (+ a generated `metadata.yaml` only for the remote case); it is NOT a `mirrors.yaml` entry. - **`sourcemirror`** (many): read-only mirrors providing package source archives. - **`sourcecache`** (one): a writable local dir Spack fills as it fetches sources → emitted as `config:source_cache`. -- **`misccache`** (one): a writable local dir for Spack's misc cache (package/build-cache indices and the **concretization cache** that lives under it) → emitted as `config:misc_cache`. Useful to persist across ephemeral builds. Concretization caching is on by default in Spack ≥ 1.2, opt-in in 1.1. +- **`concretizer`** (one): a writable local dir persisting Spack's **concretization results** → emitted as `concretizer:concretization_cache:{enable,url}` in `config/concretizer.yaml`. Useful to persist across ephemeral builds. The config key requires Spack ≥ 1.1; Stackinator infers the Spack version from `config.yaml:spack.commit` (via `Recipe.find_spack_version`, defaulting to the latest supported version when the commit can't be pinned) and **skips the cache with a warning** for Spack 1.0 (which rejects the key). -`sourcecache`/`misccache` are emitted to `config/config.yaml`; `bootstrap` to `config/bootstrap.yaml`; `buildcache`/`sourcemirror` to `config/mirrors.yaml` (+ decoded keys under `config/key_store/`). +`sourcecache` is emitted to `config/config.yaml`; `concretizer` to `config/concretizer.yaml`; `bootstrap` to `config/bootstrap.yaml`; `buildcache`/`sourcemirror` to `config/mirrors.yaml` (+ decoded keys under `config/key_store/`). **Legacy:** a binary cache can still be configured with a `cache.yaml` (`root` + optional `key`) passed to `-c/--cache`. This path is deprecated in favour of a `buildcache` entry and will be removed. diff --git a/docs/build-caches.md b/docs/build-caches.md index d7212bfb..707fe73a 100644 --- a/docs/build-caches.md +++ b/docs/build-caches.md @@ -18,7 +18,7 @@ A `mirrors.yaml` can describe five kinds of entry, each optional and each docume | [`bootstrap`](#bootstrap-mirror) | one | mirror used to bootstrap Spack itself | | [`sourcemirror`](#source-mirrors) | many | read-only mirrors that provide package sources | | [`sourcecache`](#source-cache) | one | writable local cache that fills with sources as you build | -| [`misccache`](#misc-cache) | one | writable local cache for Spack's indices and concretization results | +| [`concretizer`](#concretizer-cache) | one | writable local cache that persists concretization results | A complete example: @@ -34,8 +34,8 @@ sourcemirror: url: https://example.com/spack-sources sourcecache: path: /capstor/scratch/$USER/spack-sources -misccache: - path: /capstor/scratch/$USER/spack-misc +concretizer: + path: /capstor/scratch/$USER/spack-concretizer ``` To stop using any entry, remove (or comment out) it from `mirrors.yaml`. @@ -184,21 +184,31 @@ sourcecache: |-------|----------|-------------| | `path` | yes | absolute path to a local directory (environment variables are expanded) | -## Misc cache +## Concretizer cache -The misc cache is a single, **writable** local directory for Spack's "misc" cache: the package and build-cache indices, and — importantly — the **concretization cache**, which stores the result of concretizing a set of specs so it does not have to be recomputed. +The concretizer cache is a single, **writable** local directory in which Spack persists its **concretization results** — the output of concretizing a set of specs, so it does not have to be recomputed. Concretization can be a large fraction of build time, so pointing this at a persistent location is worthwhile when build directories are ephemeral (e.g. created in `/dev/shm` and deleted after each build). ```yaml title="mirrors.yaml" -misccache: - path: /capstor/scratch/$USER/spack-misc +concretizer: + path: /capstor/scratch/$USER/spack-concretizer ``` | Field | Required | Description | |-------|----------|-------------| | `path` | yes | absolute path to a local directory (environment variables are expanded) | -The concretization cache lives under this directory. Spack populates it automatically: concretization caching is on by default in Spack 1.2 and later, and is opt-in (`concretizer:concretization_cache:enable`) in Spack 1.1. It is keyed by the hash of the solver inputs, so a persistent cache can be reused safely across builds — stale entries simply miss. +This emits a `concretizer.yaml` that sets `concretizer:concretization_cache:{enable: true, url}`. +The cache is keyed by the hash of the solver inputs, so it can be reused safely across builds — stale entries simply miss. + +!!! warning "It does not eliminate concretization time" + The cache stores only the *result of the solve* for a given set of inputs. Before it can be consulted, Spack still has to rebuild the full concretization problem on every run — loading the package recipes and enumerating the reusable packages — and that setup work is often the larger part of concretization. So a cache hit skips the solver but not the setup: concretization gets faster, not free. + + The win is therefore largest for repeated builds of the same stack against a stable build cache (the previous solve is replayed), and smallest when the bulk of concretization time is in setup. The cache never makes concretization slower. + +!!! note "Requires Spack ≥ 1.1" + The `concretizer:concretization_cache` config key was introduced in Spack 1.1, and Spack 1.0 rejects it. + Stackinator infers the Spack version from the `spack.commit` in `config.yaml` (defaulting to a supported version when the commit is a branch or arbitrary SHA that cannot be pinned). When it detects Spack 1.0 it skips the concretizer cache with a warning rather than producing a config that would fail the build. ## Keys diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 54cfa34c..416aa614 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -27,6 +27,17 @@ ) +def _supports_concretization_cache(spack_version: str) -> bool: + """Whether the given "major.minor" spack version supports the concretizer cache. + + The concretizer:concretization_cache config key was introduced in spack 1.1; + spack 1.0 rejects it (its concretizer schema forbids unknown keys). + """ + + major_minor = tuple(int(x) for x in spack_version.split(".")) + return major_minor >= (1, 1) + + class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" @@ -52,11 +63,12 @@ class Mirrors: it fetches sources (spack config:source_cache). None if absent. This is not a mirror: it has no key and no url, and is emitted to config.yaml rather than mirrors.yaml. - * misc_cache - at most one, a writable local directory for spack's misc - cache (package/build-cache indices and the concretization - cache that lives under it) (spack config:misc_cache). None - if absent. Like source_cache it is not a mirror and is - emitted to config.yaml. + * concretizer_cache - at most one, a writable local directory persisting spack's + concretization results (concretizer:concretization_cache). + None if absent. Like source_cache it is not a mirror; it is + emitted to concretizer.yaml. It is only emitted for spack + >= 1.1 (spack 1.0 rejects the config key); when requested + against spack 1.0 it is skipped with a warning. All input processing - loading and schema-validating the system mirrors.yaml, validating urls, and reading/decoding/validating gpg keys - happens eagerly in @@ -70,11 +82,13 @@ class Mirrors: MIRRORS_YAML = "mirrors.yaml" CONFIG_YAML = "config.yaml" BOOTSTRAP_YAML = "bootstrap.yaml" + CONCRETIZER_YAML = "concretizer.yaml" def __init__( self, system_config_root: pathlib.Path, mount_path: pathlib.Path, + spack_version: str, mirror_file: Optional[pathlib.Path] = None, cmdline_cache: Optional[pathlib.Path] = None, ): @@ -82,6 +96,8 @@ def __init__( Mirrors are supplied with the --mirror command line option (mirror_file). mount_path is the recipe mount point (used to make a build cache mount-specific). + spack_version is the best-effort "major.minor" spack version, used to gate the + concretizer cache (only emitted for spack >= 1.1). cmdline_cache is an optional legacy cache.yaml passed on the command line (--cache). Relative paths in the mirror file (e.g. gpg keys) are resolved relative to the @@ -96,7 +112,7 @@ def __init__( self.bootstrap: Optional[Dict] = None self.source_mirrors: Dict[str, Dict] = {} self.source_cache: Optional[Dict] = None - self.misc_cache: Optional[Dict] = None + self.concretizer_cache: Optional[Dict] = None # The mirror configuration is supplied with --mirror, not the system # configuration. Reject a mirrors.yaml in the system config so it is not @@ -164,28 +180,42 @@ def __init__( f"{yaml.dump([self.buildcache], default_flow_style=False)}" ) - # The bootstrap mirror, the read-only source mirrors, and the writable - # source and misc caches, if any are defined. + # The bootstrap mirror, the read-only source mirrors, the writable source + # cache, and the writable concretizer cache, if any are defined. self.bootstrap = raw_mirrors.get("bootstrap") self.source_mirrors = dict(raw_mirrors["sourcemirror"]) self.source_cache = raw_mirrors.get("sourcecache") - self.misc_cache = raw_mirrors.get("misccache") + self.concretizer_cache = raw_mirrors.get("concretizer") # Validate that every mirror url is well-formed (see _validate_url). for name, mirror in self._iter_mirrors(): self._validate_url(mirror["url"], name) - # The source and misc caches are single writable local directories (spack - # config:source_cache / config:misc_cache), not mirrors: validate that each is - # an absolute path. Expand env vars now, because the build sandbox runs - # `env --ignore-environment` and so would not expand them at build time. - for cache_name, cache in (("source", self.source_cache), ("misc", self.misc_cache)): + # The source and concretizer caches are single writable local directories + # (spack config:source_cache / concretizer:concretization_cache), not mirrors: + # validate that each is an absolute path. Expand env vars now, because the + # build sandbox runs `env --ignore-environment` and so would not expand them + # at build time. + for cache_name, cache in (("source", self.source_cache), ("concretizer", self.concretizer_cache)): if cache is not None: path = os.path.expandvars(cache["path"]) if not pathlib.Path(path).is_absolute(): raise MirrorError(f"The {cache_name} cache path '{path}' is not absolute") cache["path"] = path + # The concretizer cache (concretizer:concretization_cache) was introduced in + # spack 1.1; spack 1.0 rejects the config key. Determine whether to emit it + # based on the best-effort spack version, and warn (not error) if it was + # requested but cannot be configured. + self._emit_concretizer_cache = self.concretizer_cache is not None and _supports_concretization_cache( + spack_version + ) + if self.concretizer_cache is not None and not self._emit_concretizer_cache: + self._logger.warning( + f"The concretizer cache is not supported by spack {spack_version} (requires >= 1.1) " + "and will not be configured." + ) + # Resolve the bootstrap mirror. It is either a remote url, or a local spack # bootstrap mirror directory (a `spack bootstrap mirror` output) whose own # metadata/{sources,binaries} directories we reference directly. @@ -391,17 +421,26 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: spack_mirrors, default_flow_style=False, sort_keys=False ).encode() - # the spack config.yaml setting the writable local caches: the populate-as-you-go - # source cache and the misc cache (which holds the concretization cache) - config_section = {} + # the spack config.yaml setting the populate-as-you-go source cache (spack + # config:source_cache). if self.source_cache is not None: - config_section["source_cache"] = self.source_cache["path"] - if self.misc_cache is not None: - config_section["misc_cache"] = self.misc_cache["path"] - if config_section: - config_yaml = {"config": config_section} + config_yaml = {"config": {"source_cache": self.source_cache["path"]}} files[config_root / self.CONFIG_YAML] = yaml.dump(config_yaml, default_flow_style=False).encode() + # the spack concretizer.yaml persisting concretization results. enable is set + # explicitly because it is opt-in in spack 1.1 (on by default only in >= 1.2). + # _emit_concretizer_cache gates this on the spack version (>= 1.1). + if self._emit_concretizer_cache: + concretizer_yaml = { + "concretizer": { + "concretization_cache": { + "enable": True, + "url": self.concretizer_cache["path"], + } + } + } + files[config_root / self.CONCRETIZER_YAML] = yaml.dump(concretizer_yaml, default_flow_style=False).encode() + # the spack bootstrap.yaml, if a bootstrap mirror is set. Bootstrapping reads # bootstrap:sources (not the mirrors list), and each source's `metadata` is a # directory describing it. No gpg key is involved (bootstrap binaries are diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 3cd0dbdc..7a2b1ded 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -176,12 +176,19 @@ def __init__(self, args): ) raise RuntimeError("Ivalid default-view in the recipe.") + # determine the version of spack being used: it is inferred (best effort) + # from the spack commit in config.yaml, defaulting to the latest supported + # version when it cannot be determined. This must precede the mirror + # configuration, which gates the concretizer cache on the spack version. + self.spack_version = self.find_spack_version(args.develop) + # resolve the mirror configuration provided with --mirror. --cache is the # legacy path. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors( self.system_config_path, self.mount, + self.spack_version, pathlib.Path(args.mirror) if args.mirror else None, pathlib.Path(args.cache) if args.cache else None, ) @@ -198,11 +205,6 @@ def __init__(self, args): else: self._logger.debug("no pre install hook provided") - # determine the version of spack being used: - # currently this just returns 1.0... develop is ignored - # --develop flag will imply the next release of spack after 1.0 is supported properly - self.spack_version = self.find_spack_version(args.develop) - # Returns: # Path: if the recipe contains a spack package repository # None: if there is the recipe contains no repo @@ -276,10 +278,30 @@ def config(self, config_path): def with_modules(self) -> bool: return self.modules is not None - # In Stackinator 6 we replaced logic required to determine the - # pre 1.0 Spack version. + # Make a best-effort determination of the "major.minor" version of spack being + # used, inferred from the spack commit in config.yaml. This is only a hint: the + # commit can be an arbitrary branch/tag/sha, so when the version cannot be pinned + # we default to the latest supported version ("1.1"). Returns a "major.minor" + # string (e.g. "1.0", "1.1"). def find_spack_version(self, develop): - return "1.0" + # the latest supported version, used when the version cannot be determined + # (an explicit --develop, the default branch, or an unrecognised commit). + default = "1.1" + + if develop: + return default + + commit = self.config["spack"]["commit"] + if commit is None or commit in ("develop", "main"): + return default + + # match a release branch/tag (releases/v1.0, v1.1, v1.1.2) or a bare "1.0", + # and extract the major.minor version. + match = re.search(r"v?(\d+)\.(\d+)(?:\.\d+)?", commit) + if match: + return f"{match.group(1)}.{match.group(2)}" + + return default @property def default_view(self): diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 411a788d..22e7d506 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -56,7 +56,7 @@ "additionalProperties": false, "required": ["path"] }, - "misccache": { + "concretizer": { "type": "object", "properties": { "description": {"type": "string", "default": ""}, diff --git a/unittests/data/systems/mirror-bad-misccache/mirrors.yaml b/unittests/data/systems/mirror-bad-concretizer/mirrors.yaml similarity index 69% rename from unittests/data/systems/mirror-bad-misccache/mirrors.yaml rename to unittests/data/systems/mirror-bad-concretizer/mirrors.yaml index ec06c0b6..4f982e05 100644 --- a/unittests/data/systems/mirror-bad-misccache/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-concretizer/mirrors.yaml @@ -1,2 +1,2 @@ -misccache: +concretizer: path: relative/not/absolute diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 2ffdbd5c..a55004f3 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -53,5 +53,5 @@ sourcemirror: url: https://github.com/spack sourcecache: path: /scratch/spack-sources -misccache: - path: /scratch/spack-misc \ No newline at end of file +concretizer: + path: /scratch/spack-concretizer \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index af6aea5b..2aca5a6a 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -36,7 +36,7 @@ def mirror_ok(systems_path): def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): """Check that the three kinds of mirror are resolved into separate members.""" - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: pub_key_b64 = base64.b64encode(pub_key_file.read()).decode() @@ -66,8 +66,8 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): # the writable, populate-as-you-go source cache assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources", "description": ""} - # the writable misc cache (holds the concretization cache) - assert mirrors_obj.misc_cache == {"path": "/scratch/spack-misc", "description": ""} + # the writable concretizer cache (persists concretization results) + assert mirrors_obj.concretizer_cache == {"path": "/scratch/spack-concretizer", "description": ""} # the build cache mirror name is derived from the build cache's 'name' field assert mirrors_obj.build_cache_mirror == "buildcache" @@ -78,20 +78,20 @@ def test_system_mirrors_yaml_rejected(systems_path, mount_path): # mirror-ok contains a mirrors.yaml; passing it as the system config root must raise. with pytest.raises(mirror.MirrorError): - mirror.Mirrors(systems_path / "mirror-ok", mount_path) + mirror.Mirrors(systems_path / "mirror-ok", mount_path, "1.1") def test_missing_mirror_file(clean_root, mount_path): """A --mirror file that does not exist raises MirrorError.""" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(clean_root, mount_path, mirror_file=clean_root / "does-not-exist.yaml") + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=clean_root / "does-not-exist.yaml") def test_no_mirror_file(clean_root, mount_path): """With no --mirror file there are no mirrors configured.""" - mirrors_obj = mirror.Mirrors(clean_root, mount_path) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1") assert mirrors_obj.buildcache is None assert mirrors_obj.bootstrap is None @@ -104,14 +104,14 @@ def test_mirror_init_bad_url(clean_root, mount_path, systems_path): """Check that MirrorError is raised for a bad url.""" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / "mirror-bad-url/mirrors.yaml") + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-bad-url/mirrors.yaml") def test_command_line_cache(clean_root, mount_path, systems_path, mirror_ok): """Check that adding a cache from the command line works.""" mirrors = mirror.Mirrors( - clean_root, mount_path, mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache.yaml" + clean_root, mount_path, "1.1", mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache.yaml" ) # the command line cache overrides any build cache defined in the mirror file, @@ -134,7 +134,7 @@ def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_pa """A cache.yaml without a key configures a read-only (fetch-only) build cache.""" mirrors = mirror.Mirrors( - clean_root, mount_path, mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" + clean_root, mount_path, "1.1", mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" ) # the cache exists (so it is fetched from), but has no signing key ... @@ -157,7 +157,9 @@ def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_pa def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): """A buildcache in the mirror file without a private_key is read-only (fetch-only).""" - mirrors = mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / "mirror-readonly-cache/mirrors.yaml") + mirrors = mirror.Mirrors( + clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-readonly-cache/mirrors.yaml" + ) # the cache exists (so it is fetched from), but has no signing key ... assert mirrors.build_cache_mirror == "buildcache" @@ -183,12 +185,13 @@ def test_config_files(tmp_path, clean_root, mount_path, mirror_ok): so this also exercises relative key paths resolving against the mirror file's dir. """ - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) expected = { tmp_path / "mirrors.yaml", tmp_path / "config.yaml", + tmp_path / "concretizer.yaml", tmp_path / "bootstrap.yaml", tmp_path / "bootstrap" / "bootstrap-mirror" / "metadata.yaml", tmp_path / "key_store" / "buildcache.priv.gpg", @@ -222,7 +225,7 @@ def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): } } - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) @@ -236,7 +239,7 @@ def test_mount_specific_buildcache(tmp_path, clean_root, mount_path, mirror_ok): cache is namespaced per-mount-point to avoid relocation issues / collisions. """ - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) # mirror-ok's buildcache is mount_specific: false by default; enable it. mirrors_obj.buildcache["mount_specific"] = True @@ -255,7 +258,7 @@ def test_mount_specific_buildcache(tmp_path, clean_root, mount_path, mirror_ok): def test_mount_specific_disabled(tmp_path, clean_root, mount_path, mirror_ok): """A buildcache with mount_specific false is unchanged, even when a mount point is set.""" - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) # confirm the fixture leaves the flag off assert mirrors_obj.buildcache["mount_specific"] is False @@ -287,7 +290,7 @@ def test_remote_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): }, } - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) bs_data = yaml.safe_load(files[tmp_path / "bootstrap.yaml"]) @@ -313,7 +316,7 @@ def test_local_bootstrap_configs(tmp_path, clean_root, mount_path): mirror_file.write_text(f"bootstrap:\n url: {boot}\n") config_root = tmp_path / "config" - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) files = mirrors_obj.config_files(config_root) bs_data = yaml.safe_load(files[config_root / "bootstrap.yaml"]) @@ -343,13 +346,13 @@ def test_local_bootstrap_missing_metadata(tmp_path, clean_root, mount_path): mirror_file.write_text(f"bootstrap:\n url: {boot}\n") with pytest.raises(mirror.MirrorError): - mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) def test_keys(tmp_path, clean_root, mount_path, systems_path, mirror_ok): """Check that gpg keys are decoded, relocated and reported consistently.""" - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) # the buildcache is the only mirror with keys (sources are checksum-verified) @@ -367,46 +370,64 @@ def test_keys(tmp_path, clean_root, mount_path, systems_path, mirror_ok): def test_local_caches_config(tmp_path, clean_root, mount_path, mirror_ok): - """The writable source and misc caches are emitted to config.yaml under config:.""" + """The source cache is emitted to config.yaml; the concretizer cache to concretizer.yaml.""" - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_ok) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) files = mirrors_obj.config_files(tmp_path) - data = yaml.safe_load(files[tmp_path / "config.yaml"]) - assert data == { - "config": { - "source_cache": "/scratch/spack-sources", - "misc_cache": "/scratch/spack-misc", - } + config_data = yaml.safe_load(files[tmp_path / "config.yaml"]) + assert config_data == {"config": {"source_cache": "/scratch/spack-sources"}} + + concretizer_data = yaml.safe_load(files[tmp_path / "concretizer.yaml"]) + assert concretizer_data == { + "concretizer": {"concretization_cache": {"enable": True, "url": "/scratch/spack-concretizer"}} } -def test_misc_cache_only(tmp_path, clean_root, mount_path): - """A mirror file with a misc cache but no source cache still emits config:misc_cache.""" +def test_concretizer_cache_only(tmp_path, clean_root, mount_path): + """A mirror file with a concretizer cache but no source cache emits only concretizer.yaml.""" mirror_file = tmp_path / "mirrors.yaml" - mirror_file.write_text("misccache:\n path: /scratch/only-misc\n") + mirror_file.write_text("concretizer:\n path: /scratch/only-concretizer\n") - mirrors_obj = mirror.Mirrors(clean_root, mount_path, mirror_file=mirror_file) + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) files = mirrors_obj.config_files(tmp_path) assert mirrors_obj.source_cache is None - data = yaml.safe_load(files[tmp_path / "config.yaml"]) - assert data == {"config": {"misc_cache": "/scratch/only-misc"}} + assert tmp_path / "config.yaml" not in files + concretizer_data = yaml.safe_load(files[tmp_path / "concretizer.yaml"]) + assert concretizer_data == { + "concretizer": {"concretization_cache": {"enable": True, "url": "/scratch/only-concretizer"}} + } + + +def test_concretizer_cache_skipped_on_spack_1_0(tmp_path, clean_root, mount_path): + """The concretizer cache is not emitted for spack 1.0 (which rejects the config key).""" + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text("concretizer:\n path: /scratch/only-concretizer\n") + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.0", mirror_file=mirror_file) + files = mirrors_obj.config_files(tmp_path) + + # the cache is still recorded as requested, but no concretizer.yaml is written + assert mirrors_obj.concretizer_cache is not None + assert tmp_path / "concretizer.yaml" not in files def test_local_caches_absent(tmp_path, clean_root, mount_path, systems_path): - """No config.yaml is generated when neither a source nor a misc cache is configured.""" + """No config.yaml or concretizer.yaml is generated when neither cache is configured.""" - # mirror-no-sourcecache has neither a sourcecache nor a misccache entry + # mirror-no-sourcecache has neither a sourcecache nor a concretizer entry mirrors_obj = mirror.Mirrors( - clean_root, mount_path, mirror_file=systems_path / "mirror-no-sourcecache/mirrors.yaml" + clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-no-sourcecache/mirrors.yaml" ) files = mirrors_obj.config_files(tmp_path) assert mirrors_obj.source_cache is None - assert mirrors_obj.misc_cache is None + assert mirrors_obj.concretizer_cache is None assert tmp_path / "config.yaml" not in files + assert tmp_path / "concretizer.yaml" not in files @pytest.mark.parametrize( @@ -415,11 +436,11 @@ def test_local_caches_absent(tmp_path, clean_root, mount_path, systems_path): "mirror-bad-key", "mirror-bad-keypath", "mirror-bad-sourcecache", - "mirror-bad-misccache", + "mirror-bad-concretizer", ], ) def test_bad_config(clean_root, mount_path, systems_path, system_name): """Check that MirrorError is raised at construction for bad keys or a bad cache path.""" with pytest.raises(mirror.MirrorError): - mirror.Mirrors(clean_root, mount_path, mirror_file=systems_path / system_name / "mirrors.yaml") + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / system_name / "mirrors.yaml") diff --git a/unittests/test_recipe.py b/unittests/test_recipe.py new file mode 100644 index 00000000..7aa1fabe --- /dev/null +++ b/unittests/test_recipe.py @@ -0,0 +1,39 @@ +import pytest + +from stackinator.recipe import Recipe + + +def make_recipe(commit): + """A Recipe with only the spack config populated, bypassing the heavy __init__.""" + recipe = Recipe.__new__(Recipe) + recipe._config = {"spack": {"commit": commit}} + return recipe + + +@pytest.mark.parametrize( + "commit, expected", + [ + # release branches and tags pin the major.minor version + ("releases/v1.0", "1.0"), + ("releases/v1.1", "1.1"), + ("releases/v2.0", "2.0"), + ("v1.0.0", "1.0"), + ("v1.1.3", "1.1"), + ("1.0", "1.0"), + # the version cannot be determined -> default to the latest supported (1.1) + ("develop", "1.1"), + ("main", "1.1"), + (None, "1.1"), + ("a3f9c1e8b2d4f6a7c9e1b3d5f7a9c1e3b5d7f9a1", "1.1"), + ], +) +def test_find_spack_version(commit, expected): + """find_spack_version infers major.minor from the commit, defaulting to 1.1.""" + recipe = make_recipe(commit) + assert recipe.find_spack_version(develop=False) == expected + + +def test_find_spack_version_develop_flag(): + """The --develop flag forces the latest supported version regardless of commit.""" + recipe = make_recipe("releases/v1.0") + assert recipe.find_spack_version(develop=True) == "1.1" From 107ebc8afbe6c8e6a9e869a0f03ddbf855b53f8d Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 11 Jun 2026 09:27:40 +0200 Subject: [PATCH 90/98] tidy up docs a wee bit --- docs/{build-caches.md => mirrors.md} | 67 ++++++++++++++++------------ 1 file changed, 38 insertions(+), 29 deletions(-) rename docs/{build-caches.md => mirrors.md} (83%) diff --git a/docs/build-caches.md b/docs/mirrors.md similarity index 83% rename from docs/build-caches.md rename to docs/mirrors.md index 707fe73a..7115abb7 100644 --- a/docs/build-caches.md +++ b/docs/mirrors.md @@ -1,14 +1,14 @@ # Mirrors and Build Caches Spack can use *mirrors* and *caches* to speed up image builds and to build on systems with limited or no internet access. -They are configured in a single `mirrors.yaml` file that you supply on the command line: + +They are configured in a single YAML file passed to `stack-config` using the `--mirror` flag: ```bash stack-config -b $build -r $recipe -s $system --mirror mirrors.yaml ``` The file is not part of the [system configuration](cluster-config.md): mirror locations are usually specific to the person running the build, so each invocation provides its own. -Paths inside the file (such as relative gpg key paths) are resolved relative to the directory containing the `mirrors.yaml`, so a self-contained mirror directory (the `mirrors.yaml` plus its keys) can be moved around freely. A `mirrors.yaml` can describe five kinds of entry, each optional and each documented below: @@ -20,23 +20,29 @@ A `mirrors.yaml` can describe five kinds of entry, each optional and each docume | [`sourcecache`](#source-cache) | one | writable local cache that fills with sources as you build | | [`concretizer`](#concretizer-cache) | one | writable local cache that persists concretization results | -A complete example: +!!! example + ```yaml title="mirrors.yaml" + buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg + mount_specific: true + bootstrap: + url: https://bootstrap.spack.io + sourcemirror: + # more than one source mirror can be configred. + netmirror: + url: https://example.com/spack-sources + localmirror: + url: file://scratch/group15/spack-sources + sourcecache: + path: /capstor/scratch/bobsmith/spack-sources + concretizer: + path: /capstor/scratch/bobsmith/spack-concretizer + ``` + +!!! note + Paths inside the file (such as relative gpg key paths) are resolved relative to the directory containing the `mirrors.yaml`, so a self-contained mirror directory (the `mirrors.yaml` plus its keys) can be moved around freely. -```yaml title="mirrors.yaml" -buildcache: - url: file:///capstor/scratch/team/uenv-cache - private_key: $SCRATCH/.keys/spack-push-key.gpg - mount_specific: true -bootstrap: - url: https://bootstrap.spack.io -sourcemirror: - mirror1: - url: https://example.com/spack-sources -sourcecache: - path: /capstor/scratch/$USER/spack-sources -concretizer: - path: /capstor/scratch/$USER/spack-concretizer -``` To stop using any entry, remove (or comment out) it from `mirrors.yaml`. @@ -50,7 +56,7 @@ During a build Spack fetches packages from the cache when it can, and signs and ```yaml title="mirrors.yaml" buildcache: url: file:///capstor/scratch/team/uenv-cache - private_key: $SCRATCH/.keys/spack-push-key.gpg + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg ``` | Field | Required | Description | @@ -79,7 +85,7 @@ Set `mount_specific: true` to append the mount point to the cache URL, giving ea ```yaml buildcache: url: file:///capstor/scratch/team/uenv-cache - private_key: $SCRATCH/.keys/spack-push-key.gpg + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg mount_specific: true # packages stored under .../uenv-cache/user-environment ``` @@ -89,11 +95,11 @@ A build cache needs an empty directory and a PGP signing key: ```bash # 1. create the cache directory -mkdir -p $SCRATCH/uenv-cache +mkdir -p /capstor/scratch/bobsmith/uenv-cache # 2. generate and export a signing key spack gpg create -spack gpg export --secret $SCRATCH/.keys/spack-push-key.gpg +spack gpg export --secret /capstor/scratch/bobsmith/.keys/spack-push-key.gpg ``` See [Keys](#keys) for where to store the key. @@ -177,7 +183,7 @@ Unlike a source mirror it is written to automatically, and is created on demand. ```yaml title="mirrors.yaml" sourcecache: - path: /capstor/scratch/$USER/spack-sources + path: /capstor/scratch/bobsmith/spack-sources ``` | Field | Required | Description | @@ -191,7 +197,7 @@ Concretization can be a large fraction of build time, so pointing this at a pers ```yaml title="mirrors.yaml" concretizer: - path: /capstor/scratch/$USER/spack-concretizer + path: /capstor/scratch/bobsmith/spack-concretizer ``` | Field | Required | Description | @@ -201,14 +207,17 @@ concretizer: This emits a `concretizer.yaml` that sets `concretizer:concretization_cache:{enable: true, url}`. The cache is keyed by the hash of the solver inputs, so it can be reused safely across builds — stale entries simply miss. -!!! warning "It does not eliminate concretization time" - The cache stores only the *result of the solve* for a given set of inputs. Before it can be consulted, Spack still has to rebuild the full concretization problem on every run — loading the package recipes and enumerating the reusable packages — and that setup work is often the larger part of concretization. So a cache hit skips the solver but not the setup: concretization gets faster, not free. +!!! info "concretizer cache is not a silver bullet" + The cache stores only the *result of the solve* for a given set of inputs. + Before it can be consulted, Spack still has to rebuild the full concretization problem on every run, loading the package recipes and enumerating the reusable packages, and that setup work is often the larger part of concretization. + So a cache hit skips the solver but not the setup: concretization gets faster, not free. - The win is therefore largest for repeated builds of the same stack against a stable build cache (the previous solve is replayed), and smallest when the bulk of concretization time is in setup. The cache never makes concretization slower. + The win is therefore largest for repeated builds of the same stack against a stable build cache (the previous solve is replayed), and smallest when the bulk of concretization time is in setup. !!! note "Requires Spack ≥ 1.1" The `concretizer:concretization_cache` config key was introduced in Spack 1.1, and Spack 1.0 rejects it. - Stackinator infers the Spack version from the `spack.commit` in `config.yaml` (defaulting to a supported version when the commit is a branch or arbitrary SHA that cannot be pinned). When it detects Spack 1.0 it skips the concretizer cache with a warning rather than producing a config that would fail the build. + Stackinator infers the Spack version from the `spack.commit` in `config.yaml` (defaulting to a supported version when the commit is a branch or arbitrary SHA that cannot be pinned). + When it detects Spack 1.0 it skips the concretizer cache with a warning rather than producing a config that would fail the build. ## Keys @@ -235,7 +244,7 @@ chmod 600 $SCRATCH/.keys/spack-push-key.gpg See the [Spack documentation](https://spack.readthedocs.io/en/latest/getting_started.html#gpg-signing) for more on GPG keys. -!!! failure "Don't use `$HOME`" +!!! warning "Don't use `$HOME`" The build remounts `~` as a tmpfs, so keys under `$HOME` are not visible during the build and Spack will fail to read them. Use scratch storage instead. ## Legacy `--cache` option From e1ed7652dc8edd75a8e56586496a244cb0ae22e9 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 11 Jun 2026 11:35:50 +0200 Subject: [PATCH 91/98] fix doc index --- mkdocs.yml | 2 +- serve | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 6a34c28e..b5d701cf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,7 +8,7 @@ nav: - 'Recipes': recipes.md - 'Cluster Configuration': cluster-config.md - 'Interfaces': interfaces.md - - 'Mirrors & Build Caches': build-caches.md + - 'Mirrors & Build Caches': mirrors.md - 'Spack 1.0': porting.md - 'Development': development.md # - Tutorial: tutorial.md diff --git a/serve b/serve index 6a3328f3..40f3ad57 100755 --- a/serve +++ b/serve @@ -1,4 +1,4 @@ #!/usr/bin/env bash # use uv to run mkdocs with mkdocs-material and its dependencies installed -uv run --group docs mkdocs ${@:-serve} +uv run --group docs mkdocs ${@:-serve --livereload} From 71e4812c07eab66f225cd5dc7f1dc5362b50d326 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 15 Jun 2026 17:59:52 +0200 Subject: [PATCH 92/98] use spack schema for mirrors --- stackinator/mirror.py | 117 +++++++++-------- stackinator/schema/mirror.json | 118 ++++++++++++++---- .../data/systems/mirror-bad-url/mirrors.yaml | 3 - unittests/data/systems/mirror-ok/mirrors.yaml | 1 - unittests/data/systems/mirror-s3/mirrors.yaml | 26 ++++ unittests/test_mirrors.py | 69 +++++++--- 6 files changed, 240 insertions(+), 94 deletions(-) delete mode 100644 unittests/data/systems/mirror-bad-url/mirrors.yaml create mode 100644 unittests/data/systems/mirror-s3/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 416aa614..5c038ea8 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -168,10 +168,11 @@ def __init__( "name": "buildcache", "url": raw_cache["root"], "description": "Buildcache dest loaded from legacy cache.yaml", + "source": False, + "binary": True, "public_key": None, "private_key": raw_cache.get("key"), "mount_specific": True, - "cmdline": True, } self._logger.warning( "Configuring the buildcache from the system cache.yaml file.\n" @@ -183,14 +184,10 @@ def __init__( # The bootstrap mirror, the read-only source mirrors, the writable source # cache, and the writable concretizer cache, if any are defined. self.bootstrap = raw_mirrors.get("bootstrap") - self.source_mirrors = dict(raw_mirrors["sourcemirror"]) + self.source_mirrors = raw_mirrors.get("sourcemirror") or {} self.source_cache = raw_mirrors.get("sourcecache") self.concretizer_cache = raw_mirrors.get("concretizer") - # Validate that every mirror url is well-formed (see _validate_url). - for name, mirror in self._iter_mirrors(): - self._validate_url(mirror["url"], name) - # The source and concretizer caches are single writable local directories # (spack config:source_cache / concretizer:concretization_cache), not mirrors: # validate that each is an absolute path. Expand env vars now, because the @@ -224,7 +221,6 @@ def __init__( self._bootstrap_metadata_dirs: List[str] = [] if self.bootstrap is not None: url = self.bootstrap["url"] - self._validate_url(url, "bootstrap") if self._is_remote_url(url): self._bootstrap_remote = True else: @@ -284,19 +280,6 @@ def push_to_build_cache(self) -> Optional[str]: return self.buildcache["name"] return None - def _iter_mirrors(self): - """Yield (spack mirror name, config dict) for every real spack mirror. - - These are the entries written to the spack mirrors.yaml: the build cache - (whose name is the configurable 'name' field) and the source mirrors (named - by their key in mirrors.yaml). The bootstrap mirror is not a mirrors.yaml - entry, so it is handled separately. - """ - - if self.buildcache is not None: - yield self.buildcache["name"], self.buildcache - yield from self.source_mirrors.items() - @staticmethod def _is_remote_url(url: str) -> bool: """True if url is a remote url (has a non-file scheme and a host).""" @@ -312,31 +295,50 @@ def _local_path(url: str) -> pathlib.Path: url = url[len("file://") :] return pathlib.Path(os.path.expandvars(url)) - def _validate_url(self, url: str, name: str): - """Validate that a mirror url is well-formed. + # the spack mirror connection fields (everything that may appear inside a + # fetch/push block, or at the top level of a mirror entry as a shorthand). + _CONNECTION_KEYS = ("url", "view", "profile", "endpoint_url", "access_token_variable", "access_pair") + + @classmethod + def _connection(cls, entry: Dict, key: str, mount_path: Optional[pathlib.Path] = None) -> Optional[Dict]: + """Resolve a mirror entry's fetch/push connection to a spack connection dict. - Only the format of the url is checked: no attempt is made to connect to - remote mirrors, because a valid-but-unreachable url would otherwise block - until the network request times out. + The connection comes from the entry's explicit `key` (either a url string or a + connection object), falling back to the entry's top-level connection fields + (url + auth) when no explicit fetch/push block is given - matching how spack + applies a top-level url/auth to a mirror with no fetch/push. Returns None if + no url can be resolved. When mount_path is set the url gains the mount point as + a sub-directory (mount-specific build caches). """ - if url.startswith("file://"): - # local mirror: verify that the root path is an existing directory - path = pathlib.Path(os.path.expandvars(url[len("file://") :])) - if not path.is_absolute(): - raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") - if not path.is_dir(): - raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") - return + conn = entry.get(key) + if conn is None: + # shorthand: assemble the connection from the entry's top-level fields + conn = {k: entry[k] for k in cls._CONNECTION_KEYS if entry.get(k) is not None} + elif isinstance(conn, str): + conn = {"url": conn} + else: + # copy so the relocation below never mutates the stored config + conn = dict(conn) - parsed = urllib.parse.urlparse(url) - if not parsed.scheme: - # a bare path is accepted if absolute (e.g. the legacy command line cache) - if not pathlib.Path(url).is_absolute(): - raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path") - elif not parsed.netloc: - # a remote mirror requires a well-formed url with both a scheme and a host - raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url") + if "url" not in conn: + return None + + if mount_path is not None: + conn["url"] = conn["url"].rstrip("/") + "/" + mount_path.as_posix().lstrip("/") + return conn + + @staticmethod + def _add_optional_flags(entry: Dict, mirror: Dict): + """Pass the optional spack mirror flags (signed, autopush) through if set. + + These have no default in the schema, so they are present only when the user set + them; they are copied verbatim into the emitted spack mirror entry. + """ + + for flag in ("signed", "autopush"): + if flag in mirror: + entry[flag] = mirror[flag] def _read_key(self, key: str, name: str) -> bytes: """Resolve a key (a file path or base64 blob) to validated gpg key bytes. @@ -397,25 +399,34 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: spack_mirrors: Dict[str, Dict] = {"mirrors": {}} if self.buildcache is not None: - url = self.buildcache["url"] # a mount-specific build cache lives in a sub-directory named after the # mount point: spack binaries embed the install prefix, so each mount # point needs its own cache to avoid relocation issues. - if self.buildcache["mount_specific"]: - url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/") - entry = {"fetch": {"url": url}} - # only a build cache with a signing key is pushed to - if self.buildcache["private_key"] is not None: - entry["push"] = {"url": url} + mount = self._mount_path if self.buildcache["mount_specific"] else None + entry = { + "source": self.buildcache["source"], + "binary": self.buildcache["binary"], + "fetch": self._connection(self.buildcache, "fetch", mount), + } + # a build cache is only pushed to when it has a signing key; a keyless + # build cache is read-only (fetched from, never pushed to). + if self.push_to_build_cache is not None: + entry["push"] = self._connection(self.buildcache, "push", mount) + self._add_optional_flags(entry, self.buildcache) spack_mirrors["mirrors"][self.buildcache["name"]] = entry - # source mirrors are read-only and provide sources only: fetch url, no push. + # source mirrors are read-only by default (fetch only); a push connection is + # emitted only when one was explicitly configured. for name, mirror in self.source_mirrors.items(): - spack_mirrors["mirrors"][name] = { - "source": True, - "binary": False, - "fetch": {"url": mirror["url"]}, + entry = { + "source": mirror["source"], + "binary": mirror["binary"], + "fetch": self._connection(mirror, "fetch"), } + if mirror.get("push") is not None: + entry["push"] = self._connection(mirror, "push") + self._add_optional_flags(entry, mirror) + spack_mirrors["mirrors"][name] = entry files[config_root / self.MIRRORS_YAML] = yaml.dump( spack_mirrors, default_flow_style=False, sort_keys=False diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 22e7d506..3ffa5686 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,4 +1,50 @@ { + "$defs": { + "connection": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "URL pointing to the mirror directory: local filesystem (file://) or remote (http://, https://, s3://, oci://)" + }, + "view": { "type": "string" }, + "profile": { + "type": ["string", "null"], + "description": "AWS profile name to use for S3 mirror authentication" + }, + "endpoint_url": { + "type": ["string", "null"], + "description": "Custom endpoint URL for S3-compatible storage services" + }, + "access_token_variable": { + "type": ["string", "null"], + "description": "Environment variable containing an access token for OCI registry authentication" + }, + "access_pair": { + "type": "object", + "description": "Authentication credentials for accessing private mirrors with ID and secret pairs", + "required": ["secret_variable"], + "oneOf": [ + {"required": ["id"]}, + {"required": ["id_variable"]} + ], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "id_variable": { "type": "string" }, + "secret_variable": { "type": "string" } + } + } + } + }, + "fetch_and_push": { + "anyOf": [ + { "type": "string" }, + { "$ref": "#/$defs/connection" } + ] + } + }, "type" : "object", "additionalProperties": false, "properties": { @@ -13,38 +59,64 @@ }, "buildcache": { "type": "object", + "additionalProperties": false, + "anyOf": [ + {"required": ["url"]}, + {"required": ["fetch"]}, + {"required": ["push"]} + ], "properties": { - "name": { - "type": "string", - "default": "buildcache" + "name": { "type": "string", "default": "buildcache" }, + "description": { "type": "string", "default": "" }, + "mount_specific": { "type": "boolean", "default": false }, + "source": { "type": "boolean", "default": false }, + "binary": { "type": "boolean", "default": true }, + "signed": { "type": "boolean" }, + "autopush": { "type": "boolean" }, + "private_key": { + "oneOf": [ {"type" : "string"}, {"type" : "null"} ], + "default": null }, - "description": {"type": "string", "default": ""}, - "url": {"type": "string"}, - "public_key": {"type": ["string", "null"], "default": null}, - "private_key": {"type": ["string", "null"], "default": null}, - "mount_specific": { - "type": "boolean", - "default": false + "public_key": { + "oneOf": [ {"type" : "string"}, {"type" : "null"} ], + "default": null }, - "cmdline": { - "type": "boolean", - "default": false - } - }, - "additionalProperties": false, - "required": ["url"] + "url": { "type": "string" }, + "view": { "type": "string" }, + "profile": { "type": ["string", "null"] }, + "endpoint_url": { "type": ["string", "null"] }, + "access_token_variable": { "type": ["string", "null"] }, + "access_pair": { "$ref": "#/$defs/connection/properties/access_pair" }, + "fetch": { "$ref": "#/$defs/fetch_and_push" }, + "push": { "$ref": "#/$defs/fetch_and_push" } + } }, "sourcemirror": { "type": "object", "default": {}, "additionalProperties": { "type": "object", - "properties": { - "description": {"type": "string", "default": ""}, - "url": {"type": "string"} - }, "additionalProperties": false, - "required": ["url"] + "anyOf": [ + {"required": ["url"]}, + {"required": ["fetch"]}, + {"required": ["push"]} + ], + "properties": { + "description": { "type": "string", "default": "" }, + "source": { "type": "boolean", "default": true }, + "binary": { "type": "boolean", "default": false }, + "signed": { "type": "boolean" }, + "autopush": { "type": "boolean" }, + "url": { "type": "string" }, + "view": { "type": "string" }, + "profile": { "type": ["string", "null"] }, + "endpoint_url": { "type": ["string", "null"] }, + "access_token_variable": { "type": ["string", "null"] }, + "access_pair": { "$ref": "#/$defs/connection/properties/access_pair" }, + "fetch": { "$ref": "#/$defs/fetch_and_push" }, + "push": { "$ref": "#/$defs/fetch_and_push" } + } } }, "sourcecache": { @@ -66,4 +138,4 @@ "required": ["path"] } } -} \ No newline at end of file +} diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml deleted file mode 100644 index 61341619..00000000 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ /dev/null @@ -1,3 +0,0 @@ -sourcemirror: - bad-url: - url: not-a-valid-url \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index a55004f3..a8eec2f3 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -45,7 +45,6 @@ buildcache: gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" private_key: ../../test-gpg-priv.asc mount_specific: false - cmdline: false sourcemirror: mirror1: url: https://github.com diff --git a/unittests/data/systems/mirror-s3/mirrors.yaml b/unittests/data/systems/mirror-s3/mirrors.yaml new file mode 100644 index 00000000..34dfbb65 --- /dev/null +++ b/unittests/data/systems/mirror-s3/mirrors.yaml @@ -0,0 +1,26 @@ +# a buildcache and a source mirror that use the spack-style fetch/push +# connection objects, with distinct read/write endpoints and S3/OCI auth. +buildcache: + private_key: ../../test-gpg-priv.asc + fetch: + url: s3://my-bucket/buildcache + endpoint_url: https://s3.example.com + access_pair: + id_variable: AWS_ACCESS_KEY_ID + secret_variable: AWS_SECRET_ACCESS_KEY + push: + url: s3://my-bucket/buildcache + endpoint_url: https://s3.example.com + access_pair: + id_variable: AWS_ACCESS_KEY_ID + secret_variable: AWS_SECRET_ACCESS_KEY +sourcemirror: + s3-sources: + fetch: + url: s3://my-bucket/sources + profile: my-profile + endpoint_url: https://s3.example.com + push: + url: s3://my-bucket/sources + profile: my-profile + endpoint_url: https://s3.example.com diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 2aca5a6a..918d8230 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -45,11 +45,12 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): assert mirrors_obj.buildcache == { "name": "buildcache", "description": "", + "source": False, + "binary": True, "url": "https://mirror.spack.io", "private_key": "../../test-gpg-priv.asc", "public_key": pub_key_b64, "mount_specific": False, - "cmdline": False, } assert mirrors_obj.bootstrap == { @@ -57,10 +58,10 @@ def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): "description": "", } - # non-required fields are always present, defaulted to "" by the schema + # non-required fields are always present, filled in by schema defaults assert mirrors_obj.source_mirrors == { - "mirror1": {"url": "https://github.com", "description": ""}, - "mirror2": {"url": "https://github.com/spack", "description": ""}, + "mirror1": {"url": "https://github.com", "description": "", "source": True, "binary": False}, + "mirror2": {"url": "https://github.com/spack", "description": "", "source": True, "binary": False}, } # the writable, populate-as-you-go source cache @@ -100,13 +101,6 @@ def test_no_mirror_file(clean_root, mount_path): assert mirrors_obj.build_cache_mirror is None -def test_mirror_init_bad_url(clean_root, mount_path, systems_path): - """Check that MirrorError is raised for a bad url.""" - - with pytest.raises(mirror.MirrorError): - mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-bad-url/mirrors.yaml") - - def test_command_line_cache(clean_root, mount_path, systems_path, mirror_ok): """Check that adding a cache from the command line works.""" @@ -119,7 +113,6 @@ def test_command_line_cache(clean_root, mount_path, systems_path, mirror_ok): assert mirrors.build_cache_mirror == "buildcache" assert mirrors.buildcache["name"] == "buildcache" assert mirrors.buildcache["url"] == "/tmp/foo" - assert mirrors.buildcache["cmdline"] assert mirrors.buildcache["mount_specific"] # it has a signing key, so it is pushed to @@ -151,7 +144,11 @@ def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_pa # the mirror is emitted with a fetch url but no push url data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) - assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}} + assert data["mirrors"]["buildcache"] == { + "source": False, + "binary": True, + "fetch": {"url": "/tmp/foo/user-environment"}, + } def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): @@ -175,7 +172,11 @@ def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): # the mirror is emitted with a fetch url but no push url data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) - assert data["mirrors"]["buildcache"] == {"fetch": {"url": "https://mirror.spack.io"}} + assert data["mirrors"]["buildcache"] == { + "source": False, + "binary": True, + "fetch": {"url": "https://mirror.spack.io"}, + } def test_config_files(tmp_path, clean_root, mount_path, mirror_ok): @@ -209,6 +210,8 @@ def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): valid_spack_yaml = { "mirrors": { "buildcache": { + "source": False, + "binary": True, "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, }, @@ -430,6 +433,44 @@ def test_local_caches_absent(tmp_path, clean_root, mount_path, systems_path): assert tmp_path / "concretizer.yaml" not in files +def test_s3_fetch_and_push_connections(tmp_path, clean_root, mount_path, systems_path): + """A buildcache and a source mirror sharing the spack fetch/push connection shape. + + The explicit fetch/push connection objects (distinct read/write endpoints plus S3 + auth) must be carried through verbatim into the emitted spack mirrors.yaml, proving + buildcache and sourcemirror entries accept the same spack connection config. + """ + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-s3/mirrors.yaml") + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + + bc_conn = { + "url": "s3://my-bucket/buildcache", + "endpoint_url": "https://s3.example.com", + "access_pair": {"id_variable": "AWS_ACCESS_KEY_ID", "secret_variable": "AWS_SECRET_ACCESS_KEY"}, + } + # the buildcache has a signing key, so both fetch and push are emitted with auth + assert data["mirrors"]["buildcache"] == { + "source": False, + "binary": True, + "fetch": bc_conn, + "push": bc_conn, + } + + src_conn = { + "url": "s3://my-bucket/sources", + "profile": "my-profile", + "endpoint_url": "https://s3.example.com", + } + assert data["mirrors"]["s3-sources"] == { + "source": True, + "binary": False, + "fetch": src_conn, + "push": src_conn, + } + + @pytest.mark.parametrize( "system_name", [ From 7ae87fd6d38883cf23a30abf118459d990979b57 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 16 Jun 2026 14:07:18 +0200 Subject: [PATCH 93/98] update docs --- docs/mirrors.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/mirrors.md b/docs/mirrors.md index 7115abb7..74ef1c81 100644 --- a/docs/mirrors.md +++ b/docs/mirrors.md @@ -61,11 +61,18 @@ buildcache: | Field | Required | Description | |-------|----------|-------------| -| `url` | yes | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | +| `url` | yes¹ | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | | `private_key` | no | PGP key used to sign and push packages (see [Keys](#keys)); omit for a read-only cache | | `public_key` | no | PGP key used to verify downloaded packages | | `name` | no | name Spack registers the mirror under (default `buildcache`) | | `mount_specific` | no | store the cache in a per-mount-point sub-directory (default `false`) | +| `fetch` / `push` | no¹ | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | +| `binary` | no | the cache holds binary packages (default `true`) | +| `source` | no | the cache also holds package sources (default `false`) | +| `signed` | no | whether Spack signs/verifies binaries with GPG (passed through to Spack) | +| `autopush` | no | Spack pushes each package as soon as it is installed (passed through to Spack) | + +¹ Give either a top-level `url` *or* explicit `fetch`/`push` connections — see [Connections and authentication](#connections-and-authentication). ### Read-only build cache @@ -164,7 +171,14 @@ sourcemirror: | Field | Required | Description | |-------|----------|-------------| -| `url` | yes | location of the source mirror | +| `url` | yes¹ | location of the source mirror | +| `fetch` / `push` | no¹ | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | +| `source` | no | the mirror holds package sources (default `true`) | +| `binary` | no | the mirror also holds binary packages (default `false`) | +| `signed` | no | whether Spack signs/verifies binaries with GPG (passed through to Spack) | +| `autopush` | no | Spack pushes to the mirror as soon as a package is installed (passed through to Spack) | + +¹ Give either a top-level `url` *or* explicit `fetch`/`push` connections — see [Connections and authentication](#connections-and-authentication). Source mirrors need no keys: Spack verifies every downloaded source against the checksum in its package recipe, whether it comes from the upstream url or a mirror. @@ -174,6 +188,61 @@ Populate a source mirror on an internet-connected system with Spack: spack mirror create --directory /path/to/mirror --all ``` +## Connections and authentication + +Both the [build cache](#build-cache) and [source mirrors](#source-mirrors) describe *where* and *how* Spack reaches a mirror with the same connection model, taken directly from Spack's own [`mirrors.yaml`](https://spack.readthedocs.io/en/latest/mirrors.html). Stackinator passes these fields through to Spack unchanged. + +The simplest form is a single `url`, used for both reading and writing: + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg +``` + +When the read and write endpoints differ, or the mirror needs authentication, replace the top-level `url` with explicit `fetch` and `push` connection blocks. This example is an S3 build cache with credentials supplied through environment variables: + +```yaml title="mirrors.yaml" +buildcache: + private_key: $SCRATCH/.keys/spack-push-key.gpg + fetch: + url: s3://my-bucket/buildcache + endpoint_url: https://s3.example.com + access_pair: + id_variable: AWS_ACCESS_KEY_ID + secret_variable: AWS_SECRET_ACCESS_KEY + push: + url: s3://my-bucket/buildcache + endpoint_url: https://s3.example.com + access_pair: + id_variable: AWS_ACCESS_KEY_ID + secret_variable: AWS_SECRET_ACCESS_KEY +``` + +A connection — whether given as the top-level shorthand or inside a `fetch`/`push` block — accepts: + +| Field | Description | +|-------|-------------| +| `url` | location of the mirror (`file://`, `http(s)://`, `s3://` or `oci://`) | +| `endpoint_url` | custom endpoint URL for S3-compatible storage | +| `profile` | AWS profile name to use for S3 authentication | +| `access_token_variable` | environment variable holding an access token for OCI registry authentication | +| `access_pair` | ID + secret credential pair (see below) | +| `view` | mirror view (passed through to Spack) | + +`access_pair` holds the credential pair: + +| Field | Required | Description | +|-------|----------|-------------| +| `secret_variable` | yes | environment variable holding the secret key | +| `id_variable` | one of | environment variable holding the access key ID | +| `id` | one of | the access key ID as a literal string (prefer `id_variable`) | + +!!! warning "Keep secrets out of the file" + Reference credentials through environment variables (`*_variable`) rather than writing them into `mirrors.yaml`. The secret itself is never a required field — only the *name* of the variable that holds it. + +See Spack's [mirror documentation](https://spack.readthedocs.io/en/latest/mirrors.html) for the full reference on these fields. + ## Source cache A source cache is a single, **writable** local directory that Spack fills as it downloads sources. From 7a34ac571b48e20df31b15345bd7e7914caf6221 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 16 Jun 2026 15:57:49 +0200 Subject: [PATCH 94/98] always generate push/fetch pairs - even for read only targets --- stackinator/mirror.py | 14 ++++++-------- unittests/test_mirrors.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5c038ea8..b1b1c5b2 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -403,28 +403,26 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: # mount point: spack binaries embed the install prefix, so each mount # point needs its own cache to avoid relocation issues. mount = self._mount_path if self.buildcache["mount_specific"] else None + # spack requires both fetch and push connections to be present in the + # mirror entry, even for a read-only (keyless) build cache. Whether + # packages are actually pushed to the cache is governed separately by + # push_to_build_cache (only a build cache with a signing key is pushed to). entry = { "source": self.buildcache["source"], "binary": self.buildcache["binary"], "fetch": self._connection(self.buildcache, "fetch", mount), + "push": self._connection(self.buildcache, "push", mount), } - # a build cache is only pushed to when it has a signing key; a keyless - # build cache is read-only (fetched from, never pushed to). - if self.push_to_build_cache is not None: - entry["push"] = self._connection(self.buildcache, "push", mount) self._add_optional_flags(entry, self.buildcache) spack_mirrors["mirrors"][self.buildcache["name"]] = entry - # source mirrors are read-only by default (fetch only); a push connection is - # emitted only when one was explicitly configured. for name, mirror in self.source_mirrors.items(): entry = { "source": mirror["source"], "binary": mirror["binary"], "fetch": self._connection(mirror, "fetch"), + "push": self._connection(mirror, "push"), } - if mirror.get("push") is not None: - entry["push"] = self._connection(mirror, "push") self._add_optional_flags(entry, mirror) spack_mirrors["mirrors"][name] = entry diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 918d8230..f5171151 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -142,12 +142,15 @@ def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_pa # no private key is written assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files - # the mirror is emitted with a fetch url but no push url + # spack requires both fetch and push to be present even for a read-only cache, + # so push mirrors the fetch url; whether packages are pushed is gated separately + # by push_to_build_cache (None here, as there is no signing key). data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) assert data["mirrors"]["buildcache"] == { "source": False, "binary": True, "fetch": {"url": "/tmp/foo/user-environment"}, + "push": {"url": "/tmp/foo/user-environment"}, } @@ -170,12 +173,15 @@ def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): # no private key is written assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files - # the mirror is emitted with a fetch url but no push url + # spack requires both fetch and push to be present even for a read-only cache, + # so push mirrors the fetch url; whether packages are pushed is gated separately + # by push_to_build_cache (None here, as there is no signing key). data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) assert data["mirrors"]["buildcache"] == { "source": False, "binary": True, "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, } From c8dac096775d4558ac4f823df430b7601313982e Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 16 Jun 2026 15:49:40 +0200 Subject: [PATCH 95/98] clean up linking in docs --- README.md | 9 +++++++++ docs/building.md | 5 +++-- docs/cluster-config.md | 5 +++-- docs/configuring.md | 9 +++++---- docs/installing.md | 5 ++++- docs/mirrors.md | 3 ++- docs/readme.md | 11 ----------- docs/recipes.md | 1 + docs/tutorial.md | 29 ----------------------------- 9 files changed, 27 insertions(+), 50 deletions(-) delete mode 100644 docs/readme.md delete mode 100644 docs/tutorial.md diff --git a/README.md b/README.md index 265b0ac3..7294570d 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,12 @@ Before pushing, apply the linting rules (this calls uv under the hood): ``` ./lint ``` +## building the docs + +The documentation for Stackinator is built from the markdown files in the `docs` path using MkDocs and MkDocs-material. +You can view the latest documentation online at [github.io](https://eth-cscs.github.io/stackinator/) + +To view work in progress docs, run the serve script and follow the link it provides to view a local copy of the docs in your browser. +```bash +./serve +``` diff --git a/docs/building.md b/docs/building.md index c2d191ee..99d2a9a1 100644 --- a/docs/building.md +++ b/docs/building.md @@ -1,6 +1,7 @@ +[](){#ref-building} # Building Spack Stacks -Once a stack has been [configured](configuring.md) using `stack-config`, it's time to build the software stack. +Once a stack has been [configured][ref-configuring] using `stack-config`, it's time to build the software stack. ## How to Build @@ -18,7 +19,7 @@ env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules stor The call to `make` is wrapped with with `env --ignore-env` to unset all environment variables, to improve reproducability of builds. Build times for stacks typically vary between 30 minutes to 3 hours, depending on the specific packages that have to be built. -Using [build caches](build-caches.md) and building in shared memory (see below) are the most effective methods to speed up builds. +Using [build caches][ref-mirrors] and building in shared memory (see below) are the most effective methods to speed up builds. ## Where to Build diff --git a/docs/cluster-config.md b/docs/cluster-config.md index cb6962f8..04d4eaac 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -1,3 +1,4 @@ +[](){#ref-cluster-config} # Cluster Configuration Spack stacks are built on bare-metal clusters using a minimum of dependencies from the underlying system. @@ -10,7 +11,7 @@ A cluster configuration is a directory with the following structure: └─ repos.yaml # optional reference to additional site packages ``` -The configuration is provided during the [configuration](configuring.md) step with the `--system/-s` flag. +The configuration is provided during the [configuration][ref-configuring] step with the `--system/-s` flag. The following example targets the Clariden system at CSCS: ```bash @@ -98,7 +99,7 @@ packages: Mirrors and caches are **not** part of the system configuration. They are supplied per-invocation with `stack-config --mirror `, because the locations involved (build caches, source caches) are often specific to the user running the build and may not be accessible to everyone using a system. -See [Mirrors and Build Caches](build-caches.md) for the full reference and examples. +See [Mirrors and Build Caches][ref-mirrors] for the full reference and examples. ## Site and System Configurations diff --git a/docs/configuring.md b/docs/configuring.md index eea86c93..ab550415 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -1,3 +1,4 @@ +[](){#ref-configuring} # Configuring Spack Stacks Stackinator generates the make files and spack configurations that build the spack environments that are packaged together in the spack stack. @@ -10,13 +11,13 @@ stack-config --build $BUILD_PATH --recipe $RECIPE_PATH --system $SYSTEM_CONFIG_P The following flags are required: -* `-b/--build`: the path where the [build](building.md) is to be performed. -* `-r/--recipe`: the path with the [recipe](recipes.md) yaml files that describe the environment. -* `-s/--system`: the path containing the [system configuration](cluster-config.md) for the target cluster. +* `-b/--build`: the path where the [build][ref-building] is to be performed. +* `-r/--recipe`: the path with the [recipe][ref-recipes] yaml files that describe the environment. +* `-s/--system`: the path containing the [system configuration][ref-cluster-config] for the target cluster. The following flags are optional: -* `--mirror`: path to a [mirrors.yaml](build-caches.md) file configuring build caches and mirrors. +* `--mirror`: path to a [mirrors.yaml][ref-mirrors] file configuring build caches and mirrors. * `-c/--cache`: legacy build cache configuration file (deprecated; use `--mirror`). * `-m/--mount`: override the [mount point](installing.md) where the stack will be installed. * `--version`: print the stackinator version. diff --git a/docs/installing.md b/docs/installing.md index 481b16e3..d8f3d2bd 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -1,3 +1,4 @@ +[](){#ref-installing} # Installing Stacks The installation path of the software stack is set when the stack is configured. @@ -45,4 +46,6 @@ The `store` sub-directory contains the full software stack installation tree. ### SquashFS installation The `store.squashfs` file is a compressed [SquashFS](https://tldp.org/HOWTO/SquashFS-HOWTO/whatis.html) image of the contents of the `store` path. -This can be mounted at runtime using [`squashfs-mount`](https://github.com/eth-cscs/squashfs-mount) or the [Slurm plugin](https://github.com/eth-cscs/slurm-uenv-mount/), or mounted by a system-administrator using [`mount`](https://man7.org/linux/man-pages/man2/mount.2.html), in order the to take advantage of the benefits of SquashFS over shared file systems. +This can be mounted at runtime using [`uenv`](https://github.com/eth-cscs/uenv) or the uenv Slurm plugin. + +Images can also be mounted by a system-administrator using [`mount`](https://man7.org/linux/man-pages/man2/mount.2.html), which is used in production at CSCS to permanantly mount software stacks used by the weather service on their operational cluster when nodes boot. diff --git a/docs/mirrors.md b/docs/mirrors.md index 74ef1c81..7aa079af 100644 --- a/docs/mirrors.md +++ b/docs/mirrors.md @@ -1,3 +1,4 @@ +[](){#ref-mirrors} # Mirrors and Build Caches Spack can use *mirrors* and *caches* to speed up image builds and to build on systems with limited or no internet access. @@ -8,7 +9,7 @@ They are configured in a single YAML file passed to `stack-config` using the `-- stack-config -b $build -r $recipe -s $system --mirror mirrors.yaml ``` -The file is not part of the [system configuration](cluster-config.md): mirror locations are usually specific to the person running the build, so each invocation provides its own. +The file is not part of the [system configuration][ref-cluster-config]: mirror locations are usually specific to the person running the build, so each invocation provides its own. A `mirrors.yaml` can describe five kinds of entry, each optional and each documented below: diff --git a/docs/readme.md b/docs/readme.md deleted file mode 100644 index 39da9ba5..00000000 --- a/docs/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -The documentation for Stackinator is built from the markdown files in this path using MkDocs and MkDocs-material. -You can view the latest documentation online at [github.io](https://eth-cscs.github.io/stackinator/) - -To view work in progress docs, run the serve script and follow the link it provides to view a local copy of the docs in your browser. -```bash -./serve -``` - -> [!IMPORTANT] -> to run the serve script, you need to first install [uv](https://docs.astral.sh/uv/getting-started/installation/). - diff --git a/docs/recipes.md b/docs/recipes.md index b295cd9b..dd4b8682 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -1,3 +1,4 @@ +[](){#ref-recipes} # Recipes A recipe is a description of all of the compilers and software packages to be installed, along with configuration of modules and environment scripts the stack will provide to users. diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index b9a8512d..00000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,29 +0,0 @@ -# Tutorial - -!!! warning "TODO" - write a tutorial that explains building an image step by step. - -A spack stack with everything needed to develop Arbor on for the A100 nodes on Hohgant. - -This guide walks us through the process of configuring a spack stack, building and using it. - -Arbor is a C++ library, with optional support for CUDA, MPI and Python. An Arbor developer would ideally have an environment that provides everything needed to build Arbor with these options enabled. - -The full list of all of the Spack packages needed to build a full-featured CUDA version is: - -- MPI: `cray-mpich-binary` -- compiler: `gcc@11` -- Python: `python@3.10` -- CUDA: `cuda@11.8` -- `cmake` -- `fmt` -- `pugixml` -- `nlohmann-json` -- `random123` -- `py-mpi4py` -- `py-numpy` -- `py-pybind11` -- `py-sphinx` -- `py-svgwrite` - -For the compiler, we choose `gcc@11`, which is compatible with cuda@11.8. From b201f3c8a47930e0515d2037b0e95dd7ae806c04 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 16 Jun 2026 16:44:55 +0200 Subject: [PATCH 96/98] tweak footnotes --- docs/mirrors.md | 12 +++++------- mkdocs.yml | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/mirrors.md b/docs/mirrors.md index 7aa079af..d43b93c1 100644 --- a/docs/mirrors.md +++ b/docs/mirrors.md @@ -62,18 +62,18 @@ buildcache: | Field | Required | Description | |-------|----------|-------------| -| `url` | yes¹ | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | +| `url` | yes[^1]| location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | | `private_key` | no | PGP key used to sign and push packages (see [Keys](#keys)); omit for a read-only cache | | `public_key` | no | PGP key used to verify downloaded packages | | `name` | no | name Spack registers the mirror under (default `buildcache`) | | `mount_specific` | no | store the cache in a per-mount-point sub-directory (default `false`) | -| `fetch` / `push` | no¹ | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | +| `fetch` / `push` | no[^1] | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | | `binary` | no | the cache holds binary packages (default `true`) | | `source` | no | the cache also holds package sources (default `false`) | | `signed` | no | whether Spack signs/verifies binaries with GPG (passed through to Spack) | | `autopush` | no | Spack pushes each package as soon as it is installed (passed through to Spack) | -¹ Give either a top-level `url` *or* explicit `fetch`/`push` connections — see [Connections and authentication](#connections-and-authentication). +[^1]: Give either a top-level `url` *or* explicit `fetch`/`push` connections — see [Connections and authentication](#connections-and-authentication). ### Read-only build cache @@ -172,15 +172,13 @@ sourcemirror: | Field | Required | Description | |-------|----------|-------------| -| `url` | yes¹ | location of the source mirror | -| `fetch` / `push` | no¹ | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | +| `url` | yes[^1] | location of the source mirror | +| `fetch` / `push` | no[^1] | separate read/write [connections](#connections-and-authentication), used instead of a single `url` | | `source` | no | the mirror holds package sources (default `true`) | | `binary` | no | the mirror also holds binary packages (default `false`) | | `signed` | no | whether Spack signs/verifies binaries with GPG (passed through to Spack) | | `autopush` | no | Spack pushes to the mirror as soon as a package is installed (passed through to Spack) | -¹ Give either a top-level `url` *or* explicit `fetch`/`push` connections — see [Connections and authentication](#connections-and-authentication). - Source mirrors need no keys: Spack verifies every downloaded source against the checksum in its package recipe, whether it comes from the upstream url or a mirror. Populate a source mirror on an internet-connected system with Spack: diff --git a/mkdocs.yml b/mkdocs.yml index b5d701cf..ba068fa3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,6 +17,7 @@ theme: features: - content.code.copy - content.code.annotate + - content.footnote.tooltips - navigation.tabs palette: primary: deep orange @@ -25,6 +26,7 @@ plugins: - autorefs markdown_extensions: - attr_list # for internal links + - footnotes - md_in_html - admonition - pymdownx.details From b99d3ccdaed89a56187c97941e0067a0feb8bc6d Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 16 Jun 2026 16:55:46 +0200 Subject: [PATCH 97/98] improve warning for old --cache flag --- stackinator/mirror.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index b1b1c5b2..689c8413 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -177,8 +177,12 @@ def __init__( self._logger.warning( "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n" - f"{yaml.dump([self.buildcache], default_flow_style=False)}" + "The equivalent 'mirrors.yaml' would look like: \n\n" + "buildcache:\n" + " mount_specific: true\n" + f" private_key: {self.buildcache['private_key']}\n" + f" url: {self.buildcache['url']}\n\n" + "see https://eth-cscs.github.io/stackinator/mirrors for more information" ) # The bootstrap mirror, the read-only source mirrors, the writable source From b62f24d6f3a9d30413dda040d5bb64b533eba3a8 Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 17 Jun 2026 08:20:36 +0200 Subject: [PATCH 98/98] fix unit tests --- unittests/test_mirrors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index f5171151..47ec1dd1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -225,11 +225,13 @@ def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): "source": True, "binary": False, "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, }, "mirror2": { "source": True, "binary": False, "fetch": {"url": "https://github.com/spack"}, + "push": {"url": "https://github.com/spack"}, }, } }