Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
71 commits
Select commit Hold shift + click to select a range
40daec6
Init files
michaelmckinsey1 Mar 20, 2025
d46582f
In progress
michaelmckinsey1 Mar 20, 2025
a9e618d
Testing
michaelmckinsey1 Mar 20, 2025
7259757
Using pip instead of spack
michaelmckinsey1 Apr 2, 2025
97e46ed
Working simple implementation
michaelmckinsey1 Apr 2, 2025
2eeab4a
black
michaelmckinsey1 Apr 2, 2025
183e19b
Remove package file
michaelmckinsey1 Apr 2, 2025
7e75829
Change argument
michaelmckinsey1 Apr 2, 2025
3c11279
Run interactive since already in flux allocation
michaelmckinsey1 Apr 3, 2025
b02b9a3
Raise experiment timeout to match benchmark default timeout
michaelmckinsey1 Apr 3, 2025
f072d10
pip install during workspace setup
michaelmckinsey1 Apr 3, 2025
d346a0d
Cleanup
michaelmckinsey1 Apr 3, 2025
ad8382f
Black
michaelmckinsey1 Apr 3, 2025
fd4a694
Add path for modules during execution
michaelmckinsey1 Apr 7, 2025
eb7d4f8
Add option to specify different config
michaelmckinsey1 Apr 10, 2025
39408b6
Remove config step
michaelmckinsey1 Apr 21, 2025
e02a481
Add --parent argument to run correctly
michaelmckinsey1 Apr 22, 2025
54263f1
Add strong scaling
michaelmckinsey1 May 15, 2025
e95c5db
Update experiment
michaelmckinsey1 May 15, 2025
3a9ca63
Merge branch 'develop' into scaffold
michaelmckinsey1 May 29, 2025
685f157
Fix
michaelmckinsey1 May 29, 2025
ff8dd20
Merge branch 'develop' into scaffold
michaelmckinsey1 Jun 13, 2025
8283b76
Merge branch 'develop' into scaffold
michaelmckinsey1 Jun 27, 2025
d70fa80
Dynamically add path
michaelmckinsey1 Jun 27, 2025
f54cb1e
Remove module load
michaelmckinsey1 Jun 27, 2025
bb0fff3
Set wheel version based on progmodel
michaelmckinsey1 Jun 27, 2025
19df6a3
Try inferring wheel
michaelmckinsey1 Jun 27, 2025
0eeacb2
Merge remote-tracking branch 'origin/develop' into scaffold
michaelmckinsey1 Jul 31, 2025
03f88d7
Changes to get caliper working
michaelmckinsey1 Jul 31, 2025
616572c
Merge remote-tracking branch 'origin/develop' into scaffold
michaelmckinsey1 Aug 14, 2025
7b40511
Merge remote-tracking branch 'origin/develop' into scaffold
michaelmckinsey1 Aug 23, 2025
3a916ab
Merge remote-tracking branch 'origin/develop' into scaffold
michaelmckinsey1 Aug 26, 2025
0675a16
Update
michaelmckinsey1 Sep 4, 2025
d687be7
Merge branch 'develop' into scaffold
michaelmckinsey1 Sep 4, 2025
0b48651
Update experiment
michaelmckinsey1 Sep 4, 2025
cc460fc
WIP spack package
michaelmckinsey1 Sep 5, 2025
0e40c2c
Working without caliper
michaelmckinsey1 Sep 25, 2025
0b3a523
undo changes
michaelmckinsey1 Sep 25, 2025
f835621
Caliper and adiak working
michaelmckinsey1 Sep 26, 2025
8e9648b
Update
michaelmckinsey1 Sep 26, 2025
6ab2ad0
spack-pip
michaelmckinsey1 Sep 30, 2025
4e11254
Working with spack-pip but without rocm
michaelmckinsey1 Sep 30, 2025
ee8d6a3
Update
michaelmckinsey1 Oct 7, 2025
5ac6294
mgr specific-specs
michaelmckinsey1 Oct 7, 2025
ae7625d
Update
michaelmckinsey1 Oct 10, 2025
3e04ebe
Fix paths
michaelmckinsey1 Oct 10, 2025
f176445
Update ramble version to include spack-pip
michaelmckinsey1 Oct 13, 2025
42be94d
Working on tuolumne (without rocm profiling - it freezes)
michaelmckinsey1 Oct 14, 2025
7793796
fix pybinddir
michaelmckinsey1 Oct 14, 2025
3ffde3a
rccl module
michaelmckinsey1 Oct 14, 2025
6c67449
Merge remote-tracking branch 'origin/develop' into scaffold-spack
michaelmckinsey1 Oct 14, 2025
7c217f6
Fix for spack1 packaging
michaelmckinsey1 Oct 14, 2025
fea0db4
spack1.0 update
michaelmckinsey1 Oct 14, 2025
c60d1f1
Update spack packages to get updated pybind and add git external
michaelmckinsey1 Oct 14, 2025
fe8ac39
Fix test and licenses
michaelmckinsey1 Oct 14, 2025
dfa6dad
lint
michaelmckinsey1 Oct 14, 2025
07b8545
cleanup
michaelmckinsey1 Oct 14, 2025
b5d431b
Fix dependencies
michaelmckinsey1 Oct 15, 2025
6390da7
Bump ramble version to use new var
michaelmckinsey1 Oct 15, 2025
a22ac71
Add ramble req
michaelmckinsey1 Oct 15, 2025
109e74f
lint
michaelmckinsey1 Oct 15, 2025
c6fec15
pin version because open3d dependency
michaelmckinsey1 Oct 15, 2025
067458d
upstream adiak
michaelmckinsey1 Nov 6, 2025
cd9afb5
Update package.py
michaelmckinsey1 Dec 3, 2025
ef05c9e
Merge remote-tracking branch 'origin/develop' into scaffold-spack
michaelmckinsey1 Dec 4, 2025
8d7ec5d
Fixes
michaelmckinsey1 Dec 5, 2025
e1b3f3b
Weak scaling and modules
michaelmckinsey1 Dec 11, 2025
b242fc3
Fix
michaelmckinsey1 Dec 11, 2025
c95f6b4
expr
michaelmckinsey1 Dec 12, 2025
0acd3a6
Merge remote-tracking branch 'origin/develop' into scaffold-spack
michaelmckinsey1 Jan 5, 2026
7bf7ae6
Upstream adiak package
michaelmckinsey1 Jan 5, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions checkout-versions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
#
# SPDX-License-Identifier: Apache-2.0
versions:
ramble: 6b4ecac # develop on 5/06/2025 (newer than 0.6.0 release)
ramble: 9f321a6 # develop on 10/13/2025 (newer than 0.6.0 release)
spack: a85ec51 # Dec. 1 2025 v1.1.0
spack-packages: d13c881 # Dec. 3 2025
spack-packages: 7a3a7ad # Jan. 5 2026
97 changes: 97 additions & 0 deletions experiments/py-scaffold/experiment.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Copyright 2024 Lawrence Livermore National Security, LLC and other
# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: Apache-2.0

from benchpark.error import BenchparkError
from benchpark.directives import variant, maintainers
from benchpark.experiment import Experiment
from benchpark.cuda import CudaExperiment
from benchpark.rocm import ROCmExperiment
from benchpark.scaling import ScalingMode, Scaling
from benchpark.caliper import Caliper


class PyScaffold(
Experiment,
CudaExperiment,
ROCmExperiment,
Scaling(ScalingMode.Strong, ScalingMode.Weak),
Caliper,
):

maintainers("michaelmckinsey1")

variant(
"workload",
default="sweep",
values=("sweep",),
)

variant(
"scaffold_path",
default=" ",
description="Path to local repository of ScaFFold (i.e. git clone), since it is private.",
)

variant("version", default="main", values=("main", "sharedmem"), description="app version")

def compute_applications_section(self):
self.add_experiment_variable(
"package_path", self.spec.variants["scaffold_path"][0], False
)

if self.spec.satisfies("+strong"):
n_gpus = 4
problem_scale = 7
elif self.spec.satisfies("+weak"):
n_gpus = 1
problem_scale = 5
else:
n_gpus = 1
problem_scale = 7

self.add_experiment_variable("n_gpus", n_gpus, True)
self.add_experiment_variable("problem_scale", problem_scale, True)

self.register_scaling_config(
{
ScalingMode.Strong: {
"n_gpus": lambda var, itr, dim, scaling_factor: var.val(dim)
* scaling_factor,
"problem_scale": lambda var, itr, dim, scaling_factor: var.val(dim),
},
ScalingMode.Weak: {
"n_gpus": lambda var, itr, dim, scaling_factor: var.val(dim)
* 2**3,
"problem_scale": lambda var, itr, dim, scaling_factor: var.val(dim)
+ 1,
}
}
)

self.set_required_variables(
n_resources="{n_gpus}",
process_problem_size="({problem_scale}-4)/({n_gpus}/({problem_scale}-4)**3)",
total_problem_size="{problem_scale}",
)

def compute_package_section(self):
# Spec that will be written into requirements.txt for pip install
if self.spec.satisfies("+rocm"):
model = "rocm"
elif self.spec.satisfies("+cuda"):
model = "cuda"
self.add_package_spec(
self.name,
[f"py-scaffold@{self.spec.variants['version'][0]}"],
package_manager="spack",
)
self.add_package_spec(
self.name,
[
# extra index for torch wheel and pypi index for packages that won't be found on WCI
f"--extra-index-url https://download.pytorch.org/whl/\n--extra-index-url https://pypi.org/simple\n{self.spec.variants['scaffold_path'][0]}[{model}]",
],
package_manager="pip",
)
22 changes: 16 additions & 6 deletions lib/benchpark/caliper.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ class Caliper:
description="caliper mode",
)

variant(
"cali_version",
default="master",
values=(
"master",
"2.13.1",
),
description="version",
)

class Helper(ExperimentHelper):
def compute_modifiers_section(self):
modifier_list = []
Expand All @@ -44,7 +54,7 @@ def compute_modifiers_section(self):

def compute_package_section(self):
# set package versions
caliper_version = "master"
caliper_version = self.spec.variants["cali_version"][0]

# get system config options
# TODO: Get compiler/mpi/package handles directly from system.py
Expand All @@ -60,12 +70,12 @@ def compute_package_section(self):

if not self.spec.satisfies("caliper=none"):
package_specs["caliper"] = {
"pkg_spec": f"caliper@{caliper_version}+adiak+mpi~libunwind~libdw",
"spack_pkg_spec": f"caliper@{caliper_version}+adiak+mpi~libunwind~libdw",
}
if any("topdown" in var for var in self.spec.variants["caliper"]):
papi_support = True # check if target system supports papi
if papi_support:
package_specs["caliper"]["pkg_spec"] += "+papi"
package_specs["caliper"]["spack_pkg_spec"] += "+papi"
else:
raise NotImplementedError(
"Target system does not support the papi interface"
Expand All @@ -76,7 +86,7 @@ def compute_package_section(self):
) # check if target system supports cuda
if cuda_support:
package_specs["caliper"][
"pkg_spec"
"spack_pkg_spec"
] += "~papi+cuda cuda_arch={}".format(system_specs["cuda_arch"])
else:
raise NotImplementedError(
Expand All @@ -88,7 +98,7 @@ def compute_package_section(self):
) # check if target system supports rocm
if rocm_support:
package_specs["caliper"][
"pkg_spec"
"spack_pkg_spec"
] += "~papi+rocm amdgpu_target={}".format(
system_specs["rocm_arch"]
)
Expand All @@ -99,7 +109,7 @@ def compute_package_section(self):
elif self.spec.satisfies("caliper=time") or self.spec.satisfies(
"caliper=mpi"
):
package_specs["caliper"]["pkg_spec"] += "~papi"
package_specs["caliper"]["spack_pkg_spec"] += "~papi"

return {
"packages": {k: v for k, v in package_specs.items() if v},
Expand Down
2 changes: 1 addition & 1 deletion lib/benchpark/cmd/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def include_fn(fname):
)

pkg_str = ""
if pkg_manager == "spack":
if "spack" in pkg_manager:
spack_build_stage = experiments_root / "builds"
spack_user_cache_path = experiments_root / "spack-cache"
spack, first_time_spack = per_workspace_setup.spack_first_time_setup()
Expand Down
27 changes: 16 additions & 11 deletions lib/benchpark/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,14 @@ def compute_package_section(self):

if not self.spec.satisfies("affinity=none"):
package_specs["affinity"] = {
"pkg_spec": f"affinity@{affinity_version}+mpi",
"spack_pkg_spec": f"affinity@{affinity_version}+mpi",
"compiler": system_specs["compiler"],
}
if self.spec.satisfies("+cuda"):
package_specs["affinity"]["pkg_spec"] += "+cuda"
package_specs["affinity"]["spack_pkg_spec"] += "+cuda"
elif self.spec.satisfies("+rocm"):
package_specs["affinity"][
"pkg_spec"
"spack_pkg_spec"
] += "+rocm amdgpu_target={rocm_arch}"

return {
Expand Down Expand Up @@ -201,7 +201,7 @@ class Experiment(ExperimentSystemBase, ExecMode, Affinity, Hwloc):
variant(
"package_manager",
default="spack",
values=("spack", "environment-modules", "user-managed"),
values=("spack", "environment-modules", "user-managed", "pip", "spack-pip"),
description="package manager to use",
)

Expand Down Expand Up @@ -360,7 +360,7 @@ def compute_config_section(self):
"system": system_dict,
"spec": str(self.spec),
}
if self.spec.variants["package_manager"][0] == "spack":
if "spack" in self.spec.variants["package_manager"][0]:
default_config["spack_flags"] = {
"install": "--add --keep-stage",
"concretize": "-U -f",
Expand Down Expand Up @@ -485,11 +485,16 @@ def compute_applications_section_wrapper(self):
}
}

def add_package_spec(self, package_name, spec=None):
def add_package_spec(self, package_name, spec=None, package_manager="spack"):
if spec:
self.package_specs[package_name] = {
"pkg_spec": spec[0],
}
if package_name not in self.package_specs:
self.package_specs[package_name] = {
f"{package_manager}_pkg_spec": spec[0],
}
else:
self.package_specs[package_name][f"{package_manager}_pkg_spec"] = spec[
0
]
else:
self.package_specs[package_name] = {}

Expand Down Expand Up @@ -517,14 +522,14 @@ def compute_package_section_wrapper(self):
f"Package section must be defined for application package {self.name}"
)

if pkg_manager == "spack":
if "spack" in pkg_manager:
spack_variants = list(
filter(
lambda v: v is not None,
(cls.get_spack_variants() for cls in self.helpers),
)
)
self.package_specs[self.name]["pkg_spec"] += " ".join(
self.package_specs[self.name]["spack_pkg_spec"] += " ".join(
spack_variants
).strip()

Expand Down
2 changes: 2 additions & 0 deletions lib/benchpark/test/caliper.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def test_experiment_compute_variables_section_caliper(monkeypatch):
"n_threads_per_proc": "{n_threads_per_proc}",
"benchpark_spec": ["~cuda+mpi~openmp~rocm"],
"append_path": "'",
"cali_version": "master",
"caliper": "time",
"exec_mode": "test",
"package_manager": "spack",
Expand Down Expand Up @@ -119,6 +120,7 @@ def test_caliper_modifier(monkeypatch):
"benchpark_spec": "['~cuda+mpi~openmp~rocm']",
"affinity": "none",
"append_path": "'",
"cali_version": "master",
"caliper": "time",
"exec_mode": "test",
"hwloc": "none",
Expand Down
8 changes: 4 additions & 4 deletions modifiers/caliper/modifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def modify_experiment(self, app):

add_mode(
mode_name="rocm",
mode_option="profile.hip,rocm.gputime",
mode_option="rocm.gputime",
description="Profile HIP API functions, time spent on GPU",
)

Expand Down Expand Up @@ -154,6 +154,6 @@ def _build_metadata(self, workspace, app_inst):
with open(cali_metadata_file, "w") as f:
f.write(json.dumps(cali_metadata))

software_spec("caliper", pkg_spec="caliper")

required_package("caliper")
with when("package_manager_family=spack"):
software_spec("caliper", pkg_spec="caliper", package_manager="spack")
required_package("caliper")
Loading
Loading