diff --git a/.github/workflows/sycl-ur-perf-benchmarking.yml b/.github/workflows/sycl-ur-perf-benchmarking.yml index 0cc4894bcb0f3..055f44b3b58fb 100644 --- a/.github/workflows/sycl-ur-perf-benchmarking.yml +++ b/.github/workflows/sycl-ur-perf-benchmarking.yml @@ -135,6 +135,7 @@ jobs: build_sycl_dispatch: name: '[Dispatch] Build SYCL' needs: [ sanitize_inputs_dispatch ] + if: false uses: ./.github/workflows/sycl-linux-build.yml with: build_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }} @@ -224,7 +225,7 @@ jobs: # Benchmark framework builds and runs on PRs path: build_pr: name: '[PR] Build SYCL' - if: github.event_name == 'pull_request' + if: false && github.event_name == 'pull_request' uses: ./.github/workflows/sycl-linux-build.yml with: build_ref: ${{ github.sha }} @@ -237,6 +238,39 @@ jobs: changes: '[]' toolchain_artifact: sycl_linux_default + unittest_pr: + name: '[PR] Benchmarks Unit Tests' + if: github.event_name == 'pull_request' + # needs: [build_pr] + runs-on: PVC_PERF + container: + image: 'ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest' + env: + CMPLR_ROOT: '/opt/sycl/' + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install python libraries and dependencies + run: | + pip install --user --break-system-packages -r ./devops/scripts/benchmarks/requirements.txt + - name: Checkout compute benchmarks code + uses: actions/checkout@v3 + with: + repository: intel/compute-benchmarks + ref: master + path: compute-benchmarks + - name: Configure Compute Benchmarks + run: | + cmake -B compute-benchmarks/build -S compute-benchmarks -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DNULL_L0=ON -DBUILD_SYCL=ON -DBUILD_UR=OFF -DBUILD_OMP=OFF -DBUILD_HELLO_WORLD=OFF + - name: Build Compute Benchmarks + run: | + cmake --build compute-benchmarks/build -- -j$(nproc) + - name: Run tests + run: | + LLVM_BENCHMARKS_UNIT_TESTING=1 COMPUTE_BENCHMARKS_BUILD_PATH=$(pwd)/compute-benchmarks/build python3 ./devops/scripts/benchmarks/tests/test_integration.py + # TODO: When we have stable BMG runner(s), consider moving this job to that runner. test_benchmark_framework: name: '[PR] Benchmark suite testing' diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index f59f2e9b9bf83..d8ff6ecb7d76f 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -269,6 +269,48 @@ def benchmarks(self) -> list[Benchmark]: ) ) + record_and_replay_params = product([0, 1], [0, 1]) + for emulate, instantiate in record_and_replay_params: + + def createRrBench(**kwargs): + return RecordAndReplay( + self, + RUNTIMES.LEVEL_ZERO, + PROFILERS.TIMER, + mRec=1, + mInst=instantiate, + mDest=0, + emulate=emulate, + **kwargs, + ) + + benches += [ + createRrBench( + nForksInLvl=2, + nLvls=4, + nCmdSetsInLvl=10, + nInstantiations=10, + nAppendKern=10, + nAppendCopy=1, + ), + createRrBench( + nForksInLvl=1, + nLvls=1, + nCmdSetsInLvl=10, + nInstantiations=10, + nAppendKern=10, + nAppendCopy=10, + ), + createRrBench( + nForksInLvl=1, + nLvls=4, + nCmdSetsInLvl=1, + nInstantiations=0, + nAppendKern=1, + nAppendCopy=0, + ), + ] + # Add UR-specific benchmarks benches += [ # TODO: multithread_benchmark_ur fails with segfault @@ -647,6 +689,43 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]: ] +class RecordAndReplay(ComputeBenchmark): + def __init__(self, bench, runtime: RUNTIMES, profiler_type, **kwargs): + self.rr_params = kwargs + self.iterations_regular = 1000 + self.iterations_trace = 10 + super().__init__( + bench, + f"record_and_replay_benchmark_{runtime.value}", + "RecordGraph", + runtime, + profiler_type, + ) + + def name(self): + ret = [] + for k, v in self.rr_params.items(): + if k[0] == "n": # numeric parameter + ret.append(f"{k[1:]} {v}") + elif k[0] == "m": + if v != 0: # measure parameter + ret.append(f"{k[1:]}") + else: # boolean parameter + if v != 0: + ret.append(k) + ret.sort() + return f"L0 {self.test} " + ", ".join(ret) + + def display_name(self) -> str: + return self.name() + + def get_tags(self): + return ["L0"] + + def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]: + return [f"--{k}={v}" for k, v in self.rr_params.items()] + + class QueueInOrderMemcpy(ComputeBenchmark): def __init__(self, bench, isCopyOnly, source, destination, size, profiler_type): self.isCopyOnly = isCopyOnly diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py index 7d2ffd6706b44..1761b0049224b 100644 --- a/devops/scripts/benchmarks/git_project.py +++ b/devops/scripts/benchmarks/git_project.py @@ -3,6 +3,7 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +import os from pathlib import Path import shutil @@ -10,7 +11,6 @@ from utils.utils import run from options import options - class GitProject: def __init__( self, @@ -167,6 +167,11 @@ def _setup_repo(self) -> bool: Returns: bool: True if the repository was cloned or updated, False if it was already up-to-date. """ + if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1": + log.debug( + f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)." + ) + return False if not self.src_dir.exists(): self._git_clone() return True diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index de372d7279cc2..5981175886c3b 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -137,10 +137,13 @@ def process_results( stddev_threshold_override if stddev_threshold_override is not None else options.stddev_threshold - ) * mean_value + ) + threshold_scaled = threshold * mean_value - if stddev > threshold: - log.warning(f"stddev {stddev} above the threshold {threshold} for {label}") + if stddev > threshold_scaled: + log.warning( + f"stddev {stddev} above the threshold {threshold_scaled} ({threshold} times {mean_value}) for {label}" + ) valid_results = False rlist.sort(key=lambda res: res.value) @@ -228,6 +231,9 @@ def main(directory, additional_env_vars, compare_names, filter): benchmark for benchmark in s.benchmarks() if benchmark.enabled() ] if filter: + log.debug( + f"Filtering {len(suite_benchmarks)} benchmarks in {s.name()} suite for {filter.pattern}" + ) suite_benchmarks = [ benchmark for benchmark in suite_benchmarks @@ -713,6 +719,7 @@ def validate_and_parse_env_args(env_args): options.dry_run = args.dry_run options.umf = args.umf options.iterations_stddev = args.iterations_stddev + options.stddev_threshold = args.stddev_threshold options.build_igc = args.build_igc options.current_run_name = args.relative_perf options.cudnn_directory = args.cudnn_directory diff --git a/devops/scripts/benchmarks/tests/test_integration.py b/devops/scripts/benchmarks/tests/test_integration.py new file mode 100644 index 0000000000000..678e15d367ddd --- /dev/null +++ b/devops/scripts/benchmarks/tests/test_integration.py @@ -0,0 +1,147 @@ +# Copyright (C) 2025 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import os +import shutil +import unittest +import tempfile +import subprocess +import json +from collections import namedtuple + +# oneapi has to be installed and sourced for sycl benchmarks tests + +DataJson = namedtuple("DataJson", ["runs", "metadata", "tags", "names"]) +DataJsonRun = namedtuple("DataJsonRun", ["name", "results"]) +DataJsonResult = namedtuple( + "DataJsonResult", ["name", "label", "suite", "value", "unit"] +) + +class App: + def __init__(self): + self.OUTPUT_DIR = None + self.RESULTS_DIR = None + self.WORKDIR_DIR = None + + def prepare_dirs(self): + self.OUTPUT_DIR = tempfile.mkdtemp() + self.RESULTS_DIR = tempfile.mkdtemp() + self.WORKDIR_DIR = tempfile.mkdtemp() + + # when UT does not want to build compute-benchmarks from scratch, it can provide prebuilt path + cb_targetpath = os.environ.get("COMPUTE_BENCHMARKS_BUILD_PATH") + if cb_targetpath and os.path.isdir(cb_targetpath): + cb_build_dir = os.path.join(self.WORKDIR_DIR, "compute-benchmarks-build") + os.symlink(cb_targetpath, cb_build_dir) + with open( + os.path.join(self.WORKDIR_DIR, "BENCH_WORKDIR_VERSION"), "w" + ) as f: + f.write("2.0") # TODO: take from main.INTERNAL_WORKDIR_VERSION + + def remove_dirs(self): + for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]: + if d is not None: + shutil.rmtree(d, ignore_errors=True) + + def run_main(self, *args): + + # TODO: not yet tested: "--detect-version", "sycl,compute_runtime" + + return subprocess.run( + [ + "./devops/scripts/benchmarks/main.py", + self.WORKDIR_DIR, + "--sycl", + os.environ.get("CMPLR_ROOT"), + "--save", + "testfile", + "--output-html", + "remote", + "--results-dir", + self.RESULTS_DIR, + "--output-dir", + self.OUTPUT_DIR, + "--preset", + "Minimal", + "--timestamp-override", + "20240102_030405", + "--stddev-threshold", + "999999999.9", + "--exit-on-failure", + *args, + ] + ) + + def get_output(self): + with open(os.path.join(self.OUTPUT_DIR, "data.json")) as f: + out = json.load(f) + return DataJson( + runs=[ + DataJsonRun( + name=run["name"], + results=[ + DataJsonResult( + name=r["name"], + label=r["label"], + suite=r["suite"], + value=r["value"], + unit=r["unit"], + ) + for r in run["results"] + ], + ) + for run in out["benchmarkRuns"] + ], + metadata=out["benchmarkMetadata"], + tags=out["benchmarkTags"], + names=out["defaultCompareNames"], + ) + + +# add "--verbose" for debug logs + + +class TestE2E(unittest.TestCase): + def setUp(self): + # Load test data + self.app = App() + self.app.remove_dirs() + self.app.prepare_dirs() + + # clean directory with input, output + + def tearDown(self): + self.app.remove_dirs() + + def test_record_and_replay(self): + caseName = "L0 RecordGraph AppendCopy 1, AppendKern 10, CmdSetsInLvl 10, ForksInLvl 2, Instantiations 10, Lvls 4, Rec" + run_result = self.app.run_main("--filter", caseName + "$") + self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly") + + out = self.app.get_output() + + self.assertIn(caseName, [r.name for r in out.runs[0].results]) + + metadata = out.metadata[caseName] + self.assertEqual(metadata["type"], "benchmark") + self.assertEqual(set(metadata["tags"]), {"L0"}) + + def test_submit_kernel(self): + caseName = "SubmitKernel out of order with measure completion KernelExecTime=20" + run_result = self.app.run_main("--filter", caseName + "$") + self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly") + + out = self.app.get_output() + + testName = "api_overhead_benchmark_l0 " + caseName + self.assertIn(testName, [r.name for r in out.runs[0].results]) + + metadata = out.metadata[testName] + self.assertEqual(metadata["type"], "benchmark") + self.assertEqual(set(metadata["tags"]), {"L0", "latency", "micro", "submit"}) + + +if __name__ == "__main__": + unittest.main()