[Benchmarks] add Record and Replay benchmarks

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -135,6 +135,7 @@ jobs:
   build_sycl_dispatch:
     name: '[Dispatch] Build SYCL'
     needs: [ sanitize_inputs_dispatch ]
+    if: false
     uses: ./.github/workflows/sycl-linux-build.yml
     with:
       build_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }}
@@ -224,7 +225,7 @@ jobs:
   # Benchmark framework builds and runs on PRs path:
   build_pr:
     name: '[PR] Build SYCL'
-    if: github.event_name == 'pull_request'
+    if: false && github.event_name == 'pull_request'
     uses: ./.github/workflows/sycl-linux-build.yml
     with:
       build_ref: ${{ github.sha }}
@@ -237,6 +238,39 @@ jobs:
       changes: '[]'
       toolchain_artifact: sycl_linux_default
 
+  unittest_pr:
+    name: '[PR] Benchmarks Unit Tests'
+    if: github.event_name == 'pull_request'
+    # needs: [build_pr]
+    runs-on: PVC_PERF
+    container:
+      image: 'ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest'
+    env:
+      CMPLR_ROOT: '/opt/sycl/'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Install python libraries and dependencies
+        run: |
+          pip install --user --break-system-packages -r ./devops/scripts/benchmarks/requirements.txt
+      - name: Checkout compute benchmarks code
+        uses: actions/checkout@v3
+        with:
+          repository: intel/compute-benchmarks
+          ref: master
+          path: compute-benchmarks
+      - name: Configure Compute Benchmarks
+        run: |
+          cmake -B compute-benchmarks/build -S compute-benchmarks -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+          -DNULL_L0=ON -DBUILD_SYCL=ON -DBUILD_UR=OFF -DBUILD_OMP=OFF -DBUILD_HELLO_WORLD=OFF
+      - name: Build Compute Benchmarks
+        run: |
+          cmake --build compute-benchmarks/build -- -j$(nproc)
+      - name: Run tests
+        run: |
+          LLVM_BENCHMARKS_UNIT_TESTING=1 COMPUTE_BENCHMARKS_BUILD_PATH=$(pwd)/compute-benchmarks/build python3 ./devops/scripts/benchmarks/tests/test_integration.py
+
   # TODO: When we have stable BMG runner(s), consider moving this job to that runner.
   test_benchmark_framework:
     name: '[PR] Benchmark suite testing'

devops/scripts/benchmarks/benches/compute.py

@@ -269,6 +269,48 @@ def benchmarks(self) -> list[Benchmark]:
                 )
             )
 
+        record_and_replay_params = product([0, 1], [0, 1])
+        for emulate, instantiate in record_and_replay_params:
+
+            def createRrBench(**kwargs):
+                return RecordAndReplay(
+                    self,
+                    RUNTIMES.LEVEL_ZERO,
+                    PROFILERS.TIMER,
+                    mRec=1,
+                    mInst=instantiate,
+                    mDest=0,
+                    emulate=emulate,
+                    **kwargs,
+                )
+
+            benches += [
+                createRrBench(
+                    nForksInLvl=2,
+                    nLvls=4,
+                    nCmdSetsInLvl=10,
+                    nInstantiations=10,
+                    nAppendKern=10,
+                    nAppendCopy=1,
+                ),
+                createRrBench(
+                    nForksInLvl=1,
+                    nLvls=1,
+                    nCmdSetsInLvl=10,
+                    nInstantiations=10,
+                    nAppendKern=10,
+                    nAppendCopy=10,
+                ),
+                createRrBench(
+                    nForksInLvl=1,
+                    nLvls=4,
+                    nCmdSetsInLvl=1,
+                    nInstantiations=0,
+                    nAppendKern=1,
+                    nAppendCopy=0,
+                ),
+            ]
+
         # Add UR-specific benchmarks
         benches += [
             # TODO: multithread_benchmark_ur fails with segfault
@@ -647,6 +689,43 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
         ]
 
 
+class RecordAndReplay(ComputeBenchmark):
+    def __init__(self, bench, runtime: RUNTIMES, profiler_type, **kwargs):
+        self.rr_params = kwargs
+        self.iterations_regular = 1000
+        self.iterations_trace = 10
+        super().__init__(
+            bench,
+            f"record_and_replay_benchmark_{runtime.value}",
+            "RecordGraph",
+            runtime,
+            profiler_type,
+        )
+
+    def name(self):
+        ret = []
+        for k, v in self.rr_params.items():
+            if k[0] == "n":  # numeric parameter
+                ret.append(f"{k[1:]} {v}")
+            elif k[0] == "m":
+                if v != 0:  # measure parameter
+                    ret.append(f"{k[1:]}")
+            else:  # boolean parameter
+                if v != 0:
+                    ret.append(k)
+        ret.sort()
+        return f"L0 {self.test} " + ", ".join(ret)
+
+    def display_name(self) -> str:
+        return self.name()
+
+    def get_tags(self):
+        return ["L0"]
+
+    def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
+        return [f"--{k}={v}" for k, v in self.rr_params.items()]
+
+
 class QueueInOrderMemcpy(ComputeBenchmark):
     def __init__(self, bench, isCopyOnly, source, destination, size, profiler_type):
         self.isCopyOnly = isCopyOnly

devops/scripts/benchmarks/git_project.py

@@ -3,14 +3,14 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import os
 from pathlib import Path
 import shutil
 
 from utils.logger import log
 from utils.utils import run
 from options import options
 
-
 class GitProject:
     def __init__(
         self,
@@ -167,6 +167,11 @@ def _setup_repo(self) -> bool:
         Returns:
             bool: True if the repository was cloned or updated, False if it was already up-to-date.
         """
+        if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1":
+            log.debug(
+                f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)."
+            )
+            return False
         if not self.src_dir.exists():
             self._git_clone()
             return True

devops/scripts/benchmarks/main.py

@@ -137,10 +137,13 @@ def process_results(
             stddev_threshold_override
             if stddev_threshold_override is not None
             else options.stddev_threshold
-        ) * mean_value
+        )
+        threshold_scaled = threshold * mean_value
 
-        if stddev > threshold:
-            log.warning(f"stddev {stddev} above the threshold {threshold} for {label}")
+        if stddev > threshold_scaled:
+            log.warning(
+                f"stddev {stddev} above the threshold {threshold_scaled} ({threshold} times {mean_value}) for {label}"
+            )
             valid_results = False
 
         rlist.sort(key=lambda res: res.value)
@@ -228,6 +231,9 @@ def main(directory, additional_env_vars, compare_names, filter):
             benchmark for benchmark in s.benchmarks() if benchmark.enabled()
         ]
         if filter:
+            log.debug(
+                f"Filtering {len(suite_benchmarks)} benchmarks in {s.name()} suite for {filter.pattern}"
+            )
             suite_benchmarks = [
                 benchmark
                 for benchmark in suite_benchmarks
@@ -713,6 +719,7 @@ def validate_and_parse_env_args(env_args):
     options.dry_run = args.dry_run
     options.umf = args.umf
     options.iterations_stddev = args.iterations_stddev
+    options.stddev_threshold = args.stddev_threshold
     options.build_igc = args.build_igc
     options.current_run_name = args.relative_perf
     options.cudnn_directory = args.cudnn_directory

devops/scripts/benchmarks/tests/test_integration.py

@@ -0,0 +1,147 @@
+# Copyright (C) 2025 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+import os
+import shutil
+import unittest
+import tempfile
+import subprocess
+import json
+from collections import namedtuple
+
+# oneapi has to be installed and sourced for sycl benchmarks tests
+
+DataJson = namedtuple("DataJson", ["runs", "metadata", "tags", "names"])
+DataJsonRun = namedtuple("DataJsonRun", ["name", "results"])
+DataJsonResult = namedtuple(
+    "DataJsonResult", ["name", "label", "suite", "value", "unit"]
+)
+
+class App:
+    def __init__(self):
+        self.OUTPUT_DIR = None
+        self.RESULTS_DIR = None
+        self.WORKDIR_DIR = None
+
+    def prepare_dirs(self):
+        self.OUTPUT_DIR = tempfile.mkdtemp()
+        self.RESULTS_DIR = tempfile.mkdtemp()
+        self.WORKDIR_DIR = tempfile.mkdtemp()
+
+        # when UT does not want to build compute-benchmarks from scratch, it can provide prebuilt path
+        cb_targetpath = os.environ.get("COMPUTE_BENCHMARKS_BUILD_PATH")
+        if cb_targetpath and os.path.isdir(cb_targetpath):
+            cb_build_dir = os.path.join(self.WORKDIR_DIR, "compute-benchmarks-build")
+            os.symlink(cb_targetpath, cb_build_dir)
+            with open(
+                os.path.join(self.WORKDIR_DIR, "BENCH_WORKDIR_VERSION"), "w"
+            ) as f:
+                f.write("2.0")  # TODO: take from main.INTERNAL_WORKDIR_VERSION
+
+    def remove_dirs(self):
+        for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]:
+            if d is not None:
+                shutil.rmtree(d, ignore_errors=True)
+
+    def run_main(self, *args):
+
+        # TODO: not yet tested: "--detect-version", "sycl,compute_runtime"
+
+        return subprocess.run(
+            [
+                "./devops/scripts/benchmarks/main.py",
+                self.WORKDIR_DIR,
+                "--sycl",
+                os.environ.get("CMPLR_ROOT"),
+                "--save",
+                "testfile",
+                "--output-html",
+                "remote",
+                "--results-dir",
+                self.RESULTS_DIR,
+                "--output-dir",
+                self.OUTPUT_DIR,
+                "--preset",
+                "Minimal",
+                "--timestamp-override",
+                "20240102_030405",
+                "--stddev-threshold",
+                "999999999.9",
+                "--exit-on-failure",
+                *args,
+            ]
+        )
+
+    def get_output(self):
+        with open(os.path.join(self.OUTPUT_DIR, "data.json")) as f:
+            out = json.load(f)
+            return DataJson(
+                runs=[
+                    DataJsonRun(
+                        name=run["name"],
+                        results=[
+                            DataJsonResult(
+                                name=r["name"],
+                                label=r["label"],
+                                suite=r["suite"],
+                                value=r["value"],
+                                unit=r["unit"],
+                            )
+                            for r in run["results"]
+                        ],
+                    )
+                    for run in out["benchmarkRuns"]
+                ],
+                metadata=out["benchmarkMetadata"],
+                tags=out["benchmarkTags"],
+                names=out["defaultCompareNames"],
+            )
+
+
+# add "--verbose" for debug logs
+
+
+class TestE2E(unittest.TestCase):
+    def setUp(self):
+        # Load test data
+        self.app = App()
+        self.app.remove_dirs()
+        self.app.prepare_dirs()
+
+        # clean directory with input, output
+
+    def tearDown(self):
+        self.app.remove_dirs()
+
+    def test_record_and_replay(self):
+        caseName = "L0 RecordGraph AppendCopy 1, AppendKern 10, CmdSetsInLvl 10, ForksInLvl 2, Instantiations 10, Lvls 4, Rec"
+        run_result = self.app.run_main("--filter", caseName + "$")
+        self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly")
+
+        out = self.app.get_output()
+
+        self.assertIn(caseName, [r.name for r in out.runs[0].results])
+
+        metadata = out.metadata[caseName]
+        self.assertEqual(metadata["type"], "benchmark")
+        self.assertEqual(set(metadata["tags"]), {"L0"})
+
+    def test_submit_kernel(self):
+        caseName = "SubmitKernel out of order with measure completion KernelExecTime=20"
+        run_result = self.app.run_main("--filter", caseName + "$")
+        self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly")
+
+        out = self.app.get_output()
+
+        testName = "api_overhead_benchmark_l0 " + caseName
+        self.assertIn(testName, [r.name for r in out.runs[0].results])
+
+        metadata = out.metadata[testName]
+        self.assertEqual(metadata["type"], "benchmark")
+        self.assertEqual(set(metadata["tags"]), {"L0", "latency", "micro", "submit"})
+
+
+if __name__ == "__main__":
+    unittest.main()