ScaFFold Benchmark and spack-pip Package Manager

checkout-versions.yaml

@@ -3,6 +3,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 versions:
-  ramble: 6b4ecac  # develop on 5/06/2025 (newer than 0.6.0 release)
+  ramble: 9f321a6 # develop on 10/13/2025 (newer than 0.6.0 release)
   spack: a85ec51  # Dec. 1 2025 v1.1.0
-  spack-packages: d13c881  # Dec. 3 2025
+  spack-packages: 7a3a7ad # Jan. 5 2026

experiments/py-scaffold/experiment.py

@@ -0,0 +1,97 @@
+# Copyright 2024 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant, maintainers
+from benchpark.experiment import Experiment
+from benchpark.cuda import CudaExperiment
+from benchpark.rocm import ROCmExperiment
+from benchpark.scaling import ScalingMode, Scaling
+from benchpark.caliper import Caliper
+
+
+class PyScaffold(
+    Experiment,
+    CudaExperiment,
+    ROCmExperiment,
+    Scaling(ScalingMode.Strong, ScalingMode.Weak),
+    Caliper,
+):
+
+    maintainers("michaelmckinsey1")
+
+    variant(
+        "workload",
+        default="sweep",
+        values=("sweep",),
+    )
+
+    variant(
+        "scaffold_path",
+        default=" ",
+        description="Path to local repository of ScaFFold (i.e. git clone), since it is private.",
+    )
+
+    variant("version", default="main", values=("main", "sharedmem"), description="app version")
+
+    def compute_applications_section(self):
+        self.add_experiment_variable(
+            "package_path", self.spec.variants["scaffold_path"][0], False
+        )
+
+        if self.spec.satisfies("+strong"):
+            n_gpus = 4
+            problem_scale = 7
+        elif self.spec.satisfies("+weak"):
+            n_gpus = 1
+            problem_scale = 5
+        else:
+            n_gpus = 1
+            problem_scale = 7
+
+        self.add_experiment_variable("n_gpus", n_gpus, True)
+        self.add_experiment_variable("problem_scale", problem_scale, True)
+
+        self.register_scaling_config(
+            {
+                ScalingMode.Strong: {
+                    "n_gpus": lambda var, itr, dim, scaling_factor: var.val(dim)
+                    * scaling_factor,
+                    "problem_scale": lambda var, itr, dim, scaling_factor: var.val(dim),
+                },
+                ScalingMode.Weak: {
+                    "n_gpus": lambda var, itr, dim, scaling_factor: var.val(dim)
+                    * 2**3,
+                    "problem_scale": lambda var, itr, dim, scaling_factor: var.val(dim)
+                    + 1,
+                }
+            }
+        )
+
+        self.set_required_variables(
+            n_resources="{n_gpus}",
+            process_problem_size="({problem_scale}-4)/({n_gpus}/({problem_scale}-4)**3)",
+            total_problem_size="{problem_scale}",
+        )
+
+    def compute_package_section(self):
+        # Spec that will be written into requirements.txt for pip install
+        if self.spec.satisfies("+rocm"):
+            model = "rocm"
+        elif self.spec.satisfies("+cuda"):
+            model = "cuda"
+        self.add_package_spec(
+            self.name,
+            [f"py-scaffold@{self.spec.variants['version'][0]}"],
+            package_manager="spack",
+        )
+        self.add_package_spec(
+            self.name,
+            [
+                # extra index for torch wheel and pypi index for packages that won't be found on WCI
+                f"--extra-index-url https://download.pytorch.org/whl/\n--extra-index-url https://pypi.org/simple\n{self.spec.variants['scaffold_path'][0]}[{model}]",
+            ],
+            package_manager="pip",
+        )

lib/benchpark/caliper.py

@@ -28,6 +28,16 @@ class Caliper:
         description="caliper mode",
     )
 
+    variant(
+        "cali_version",
+        default="master",
+        values=(
+            "master",
+            "2.13.1",
+        ),
+        description="version",
+    )
+
     class Helper(ExperimentHelper):
         def compute_modifiers_section(self):
             modifier_list = []
@@ -44,7 +54,7 @@ def compute_modifiers_section(self):
 
         def compute_package_section(self):
             # set package versions
-            caliper_version = "master"
+            caliper_version = self.spec.variants["cali_version"][0]
 
             # get system config options
             # TODO: Get compiler/mpi/package handles directly from system.py
@@ -60,12 +70,12 @@ def compute_package_section(self):
 
             if not self.spec.satisfies("caliper=none"):
                 package_specs["caliper"] = {
-                    "pkg_spec": f"caliper@{caliper_version}+adiak+mpi~libunwind~libdw",
+                    "spack_pkg_spec": f"caliper@{caliper_version}+adiak+mpi~libunwind~libdw",
                 }
                 if any("topdown" in var for var in self.spec.variants["caliper"]):
                     papi_support = True  # check if target system supports papi
                     if papi_support:
-                        package_specs["caliper"]["pkg_spec"] += "+papi"
+                        package_specs["caliper"]["spack_pkg_spec"] += "+papi"
                     else:
                         raise NotImplementedError(
                             "Target system does not support the papi interface"
@@ -76,7 +86,7 @@ def compute_package_section(self):
                     )  # check if target system supports cuda
                     if cuda_support:
                         package_specs["caliper"][
-                            "pkg_spec"
+                            "spack_pkg_spec"
                         ] += "~papi+cuda cuda_arch={}".format(system_specs["cuda_arch"])
                     else:
                         raise NotImplementedError(
@@ -88,7 +98,7 @@ def compute_package_section(self):
                     )  # check if target system supports rocm
                     if rocm_support:
                         package_specs["caliper"][
-                            "pkg_spec"
+                            "spack_pkg_spec"
                         ] += "~papi+rocm amdgpu_target={}".format(
                             system_specs["rocm_arch"]
                         )
@@ -99,7 +109,7 @@ def compute_package_section(self):
                 elif self.spec.satisfies("caliper=time") or self.spec.satisfies(
                     "caliper=mpi"
                 ):
-                    package_specs["caliper"]["pkg_spec"] += "~papi"
+                    package_specs["caliper"]["spack_pkg_spec"] += "~papi"
 
             return {
                 "packages": {k: v for k, v in package_specs.items() if v},

lib/benchpark/cmd/setup.py

@@ -204,7 +204,7 @@ def include_fn(fname):
     )
 
     pkg_str = ""
-    if pkg_manager == "spack":
+    if "spack" in pkg_manager:
         spack_build_stage = experiments_root / "builds"
         spack_user_cache_path = experiments_root / "spack-cache"
         spack, first_time_spack = per_workspace_setup.spack_first_time_setup()

lib/benchpark/experiment.py

@@ -129,14 +129,14 @@ def compute_package_section(self):
 
             if not self.spec.satisfies("affinity=none"):
                 package_specs["affinity"] = {
-                    "pkg_spec": f"affinity@{affinity_version}+mpi",
+                    "spack_pkg_spec": f"affinity@{affinity_version}+mpi",
                     "compiler": system_specs["compiler"],
                 }
                 if self.spec.satisfies("+cuda"):
-                    package_specs["affinity"]["pkg_spec"] += "+cuda"
+                    package_specs["affinity"]["spack_pkg_spec"] += "+cuda"
                 elif self.spec.satisfies("+rocm"):
                     package_specs["affinity"][
-                        "pkg_spec"
+                        "spack_pkg_spec"
                     ] += "+rocm amdgpu_target={rocm_arch}"
 
             return {
@@ -201,7 +201,7 @@ class Experiment(ExperimentSystemBase, ExecMode, Affinity, Hwloc):
     variant(
         "package_manager",
         default="spack",
-        values=("spack", "environment-modules", "user-managed"),
+        values=("spack", "environment-modules", "user-managed", "pip", "spack-pip"),
         description="package manager to use",
     )
 
@@ -360,7 +360,7 @@ def compute_config_section(self):
             "system": system_dict,
             "spec": str(self.spec),
         }
-        if self.spec.variants["package_manager"][0] == "spack":
+        if "spack" in self.spec.variants["package_manager"][0]:
             default_config["spack_flags"] = {
                 "install": "--add --keep-stage",
                 "concretize": "-U -f",
@@ -485,11 +485,16 @@ def compute_applications_section_wrapper(self):
             }
         }
 
-    def add_package_spec(self, package_name, spec=None):
+    def add_package_spec(self, package_name, spec=None, package_manager="spack"):
         if spec:
-            self.package_specs[package_name] = {
-                "pkg_spec": spec[0],
-            }
+            if package_name not in self.package_specs:
+                self.package_specs[package_name] = {
+                    f"{package_manager}_pkg_spec": spec[0],
+                }
+            else:
+                self.package_specs[package_name][f"{package_manager}_pkg_spec"] = spec[
+                    0
+                ]
         else:
             self.package_specs[package_name] = {}
 
@@ -517,14 +522,14 @@ def compute_package_section_wrapper(self):
                 f"Package section must be defined for application package {self.name}"
             )
 
-        if pkg_manager == "spack":
+        if "spack" in pkg_manager:
             spack_variants = list(
                 filter(
                     lambda v: v is not None,
                     (cls.get_spack_variants() for cls in self.helpers),
                 )
             )
-            self.package_specs[self.name]["pkg_spec"] += " ".join(
+            self.package_specs[self.name]["spack_pkg_spec"] += " ".join(
                 spack_variants
             ).strip()
 

lib/benchpark/test/caliper.py

@@ -39,6 +39,7 @@ def test_experiment_compute_variables_section_caliper(monkeypatch):
             "n_threads_per_proc": "{n_threads_per_proc}",
             "benchpark_spec": ["~cuda+mpi~openmp~rocm"],
             "append_path": "'",
+            "cali_version": "master",
             "caliper": "time",
             "exec_mode": "test",
             "package_manager": "spack",
@@ -119,6 +120,7 @@ def test_caliper_modifier(monkeypatch):
         "benchpark_spec": "['~cuda+mpi~openmp~rocm']",
         "affinity": "none",
         "append_path": "'",
+        "cali_version": "master",
         "caliper": "time",
         "exec_mode": "test",
         "hwloc": "none",

modifiers/caliper/modifier.py

@@ -87,7 +87,7 @@ def modify_experiment(self, app):
 
     add_mode(
         mode_name="rocm",
-        mode_option="profile.hip,rocm.gputime",
+        mode_option="rocm.gputime",
         description="Profile HIP API functions, time spent on GPU",
     )
 
@@ -154,6 +154,6 @@ def _build_metadata(self, workspace, app_inst):
         with open(cali_metadata_file, "w") as f:
             f.write(json.dumps(cali_metadata))
 
-    software_spec("caliper", pkg_spec="caliper")
-
-    required_package("caliper")
+        with when("package_manager_family=spack"):
+            software_spec("caliper", pkg_spec="caliper", package_manager="spack")
+            required_package("caliper")