[Benchmarks] add unittest, add rr benchmarks

Author: lslusarczyk

devops/scripts/benchmarks/benches/compute.py

@@ -269,6 +269,56 @@ def benchmarks(self) -> list[Benchmark]:
                 )
             )
 
+        record_and_reply_params = product(list(PROFILERS), [0, 1], [0, 1])
+        for profiler_type, emulate, instantiate in record_and_reply_params:
+            benches += [
+                RecordAndReply(
+                    self,
+                    RUNTIMES.LEVEL_ZERO,
+                    profiler_type,
+                    nForksInLvl=2,
+                    nLvls=4,
+                    nCmdSetsInLvl=10,
+                    nInstantiations=10,
+                    nAppendKern=10,
+                    nAppendCopy=1,
+                    mRec=1,
+                    mInst=instantiate,
+                    mDest=0,
+                    emulate=emulate,
+                ),
+                RecordAndReply(
+                    self,
+                    RUNTIMES.LEVEL_ZERO,
+                    profiler_type,
+                    nForksInLvl=1,
+                    nLvls=1,
+                    nCmdSetsInLvl=10,
+                    nInstantiations=10,
+                    nAppendKern=10,
+                    nAppendCopy=10,
+                    mRec=1,
+                    mInst=instantiate,
+                    mDest=0,
+                    emulate=emulate,
+                ),
+                RecordAndReply(
+                    self,
+                    RUNTIMES.LEVEL_ZERO,
+                    profiler_type,
+                    nForksInLvl=1,
+                    nLvls=4,
+                    nCmdSetsInLvl=1,
+                    nInstantiations=0,
+                    nAppendKern=1,
+                    nAppendCopy=0,
+                    mRec=1,
+                    mInst=instantiate,
+                    mDest=0,
+                    emulate=emulate,
+                ),
+            ]
+
         # Add UR-specific benchmarks
         benches += [
             # TODO: multithread_benchmark_ur fails with segfault
@@ -646,6 +696,45 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
             f"--profilerType={self.profiler_type.value}",
         ]
 
+class RecordAndReply(ComputeBenchmark):
+    def __init__(self, bench, runtime: RUNTIMES, profiler_type, **kwargs):
+        self.rr_params = kwargs
+        self.iterations_regular = 1000
+        self.iterations_trace = 10
+        super().__init__(
+                bench,
+                f"record_and_replay_benchmark_{runtime.value}",
+                "RecordGraph",
+                runtime,
+                profiler_type,
+            )
+
+    def name(self):
+        ret = []
+        for k, v in self.rr_params.items():
+            if k[0] == 'n': #numeric parameter
+                ret.append(f"{k[1:]} {v}")
+            elif k[0] == 'm':
+                if v != 0: # measure parameter
+                    ret.append(f"{k[1:]}")
+            else: # boolean parameter
+                if v != 0:
+                    ret.append(k)
+        ret.sort()
+        return f"{self.bench_name} {self.test} " + ", ".join(ret)
+
+    def display_name(self) -> str:
+        return self.name()
+
+    def description(self) -> str:
+        return f"{self.runtime.value} Graphs record and reply"
+
+    def get_tags(self):
+        return ["L0", "micro"]
+
+    def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
+        return [f"--{k}={v}" for k, v in self.rr_params.items()]
+
 
 class QueueInOrderMemcpy(ComputeBenchmark):
     def __init__(self, bench, isCopyOnly, source, destination, size, profiler_type):

devops/scripts/benchmarks/git_project.py

@@ -2,7 +2,7 @@
 # Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
+import os
 from pathlib import Path
 import shutil
 
@@ -167,6 +167,11 @@ def _setup_repo(self) -> bool:
         Returns:
             bool: True if the repository was cloned or updated, False if it was already up-to-date.
         """
+        if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1":
+            log.debug(
+                f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)."
+            )
+            return False
         if not self.src_dir.exists():
             self._git_clone()
             return True

devops/scripts/benchmarks/main.py

@@ -228,6 +228,7 @@ def main(directory, additional_env_vars, compare_names, filter):
             benchmark for benchmark in s.benchmarks() if benchmark.enabled()
         ]
         if filter:
+            log.debug(f"Filtering {len(suite_benchmarks)} benchmarks in {s.name()} suite for {filter.pattern}")
             suite_benchmarks = [
                 benchmark
                 for benchmark in suite_benchmarks

devops/scripts/benchmarks/tests/test_integration.py

@@ -0,0 +1,110 @@
+import os
+import shutil
+import unittest
+import logging
+
+import subprocess
+import json
+from collections import namedtuple
+
+# class TestBasic(unittest.TestCase):
+#     def setUp(self):
+#         # Load test data
+#         self.app = App(database="fixtures/test_basic.json")
+#
+#         def test_customer_count(self):
+#             self.assertEqual(len(self.app.customers), 100)
+#
+#         def test_existence_of_customer(self):
+#             customer = self.app.get_customer(id=10)
+#             self.assertEqual(customer.name, "Org XYZ")
+#             self.assertEqual(customer.address, "10 Red Road, Reading")
+            
+
+# https://realpython.com/python-testing/
+
+DataJson = namedtuple("DataJson", ["runs", "metadata", "tags", "names"])
+DataJsonRun = namedtuple("DataJsonRun", ["name", "results"])
+DataJsonResult = namedtuple("DataJsonResult", ["name", "label", "suite", "value", "unit"])
+
+class App:
+    def __init__(self):
+        self.TMP_DIR = os.path.dirname(__file__)
+        self.OUTPUT_DIR = os.path.join(self.TMP_DIR, "tmp-output")
+        self.RESULTS_DIR = os.path.join(self.TMP_DIR, "tmp-results")
+        self.WORKDIR_DIR = os.path.join(self.TMP_DIR, "tmp-workdir")
+
+
+
+    def prepare_dirs(self):
+        for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]:
+            os.makedirs(d)
+
+        # when UT does not want to build compute-benchmarks from scratch, it can provide prebuilt path
+        cb_targetpath=os.environ.get("COMPUTE_BENCHMARKS_BUILD_PATH")
+        if cb_targetpath and os.path.isdir(cb_targetpath):
+            cb_build_dir = os.path.join(self.WORKDIR_DIR, "compute-benchmarks-build")
+            os.symlink(cb_targetpath, cb_build_dir)
+            with open(os.path.join(self.WORKDIR_DIR, "BENCH_WORKDIR_VERSION"), "w") as f:
+                f.write("2.0") # TODO: take from main.INTERNAL_WORKDIR_VERSION
+
+
+    def remove_dirs(self):
+        for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]:
+            if os.path.exists(d):
+                shutil.rmtree(d)
+    def run_main(self, *args):
+
+        # TODO: not yet tested: "--detect-version", "sycl,compute_runtime"
+
+        return subprocess.run(["./devops/scripts/benchmarks/main.py", self.WORKDIR_DIR,
+        "--sycl", os.environ.get("ONEAPI_ROOT"),
+        "--ur", os.environ.get("CMPLR_ROOT"),
+        "--adapter", "opencl",
+        "--save", "testplik",
+        "--output-html", "remote",
+        "--results-dir", self.RESULTS_DIR,
+        "--output-dir", self.OUTPUT_DIR,
+        "--preset", "Minimal",
+        "--timestamp-override", "20240102_030405",
+        "--exit-on-failure",
+        *args])
+
+    def get_output(self):
+        output_file = os.path.join(self.OUTPUT_DIR, "data.json")
+        with open(output_file) as f:
+            out = json.load(f)
+            return DataJson(
+                runs=[DataJsonRun(name=run["name"], results=[DataJsonResult(name=r["name"], label=r["label"], suite=r["suite"], value=r["value"], unit=r["unit"]) for r in run["results"]]) for run in out["benchmarkRuns"]],
+                metadata=out["benchmarkMetadata"],
+                tags=out["benchmarkTags"],
+                names=out["defaultCompareNames"],
+            )
+
+# add "--verbose" for debug logs
+
+class TestE2E(unittest.TestCase):
+    def setUp(self):
+        # Load test data
+        self.app = App()
+        self.app.remove_dirs()
+        self.app.prepare_dirs()
+
+        run_result = self.app.run_main("--filter", "RecordGraph")
+        self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly")
+        # clean directory with input, output
+        out = self.app.get_output()
+        self.assertIn("record_and_replay_benchmark_l0 RecordGraph AppendCopy 1, AppendKern 10, CmdSetsInLvl 10, ForksInLvl 2, Instantiations 10, Lvls 4, Rec", [r.name for r in out.runs[0].results])
+
+
+    def tearDown(self):
+        pass
+        # self.app.remove_dirs()
+                        
+    def test_record_and_reply(self):
+        pass
+
+
+if __name__ == "__main__":
+    unittest.main()
+