[Benchmarks] support CPU counter measurements

for api_overhead_benchmark_ur

Author: igchor

scripts/benchmarks/benches/compute.py

@@ -19,7 +19,7 @@ def setup(self):
         if options.sycl is None:
             return
 
-        repo_path = git_clone(self.directory, "compute-benchmarks-repo", "https://github.com/intel/compute-benchmarks.git", "c80ddec9f0b4905bcbeb0f264f710093dc70340d")
+        repo_path = git_clone(self.directory, "compute-benchmarks-repo", "https://github.com/intel/compute-benchmarks.git", "XXX")
         build_path = create_build_path(self.directory, 'compute-benchmarks-build')
 
         configure_command = [
@@ -71,17 +71,20 @@ def benchmarks(self) -> list[Benchmark]:
 
         if options.ur is not None:
             benches += [
-                SubmitKernelUR(self, 0),
-                SubmitKernelUR(self, 1),
+                SubmitKernelURWithTime(self, 0),
+                SubmitKernelURWithTime(self, 1),
+                SubmitKernelURWithCPUCounter(self, 0),
+                SubmitKernelURWithCPUCounter(self, 1),
             ]
 
         return benches
 
 class ComputeBenchmark(Benchmark):
-    def __init__(self, bench, name, test):
+    def __init__(self, bench, name, test, unit="time [us]"):
         self.bench = bench
         self.bench_name = name
         self.test = test
+        self.unitType = unit
         super().__init__(bench.directory)
 
     def bin_args(self) -> list[str]:
@@ -91,7 +94,7 @@ def extra_env_vars(self) -> dict:
         return {}
 
     def unit(self):
-        return "μs"
+        return {"time [us]": "μs", "hw instructions [count]": "CPU instructions count"}[self.unitType]
 
     def setup(self):
         self.benchmark_bin = os.path.join(self.bench.directory, 'compute-benchmarks-build', 'bin', self.bench_name)
@@ -108,30 +111,41 @@ def run(self, env_vars) -> list[Result]:
         env_vars.update(self.extra_env_vars())
 
         result = self.run_bench(command, env_vars)
-        (label, mean) = self.parse_output(result)
-        return [ Result(label=self.name(), value=mean, command=command, env=env_vars, stdout=result) ]
+        parsed_results = self.parse_output(result)
+        ret = []
+        for label, mean, unit in parsed_results:
+            if self.unitType in unit:
+                extra_label = " CPU count" if self.unitType == "hw instructions [count]" else ""
+                ret.append(Result(label=label + extra_label, value=mean, command=command, env=env_vars, stdout=result))
+        return ret
 
     def parse_output(self, output):
         csv_file = io.StringIO(output)
         reader = csv.reader(csv_file)
         next(reader, None)
-        data_row = next(reader, None)
-        if data_row is None:
+        results = []
+        while True:
+            data_row = next(reader, None)
+            if data_row is None:
+                break
+            try:
+                label = data_row[0]
+                mean = float(data_row[1])
+                unit = data_row[7]
+                results.append((label, mean, unit))
+            except (ValueError, IndexError) as e:
+                raise ValueError(f"Error parsing output: {e}")
+        if len(results) == 0:
             raise ValueError("Benchmark output does not contain data.")
-        try:
-            label = data_row[0]
-            mean = float(data_row[1])
-            return (label, mean)
-        except (ValueError, IndexError) as e:
-            raise ValueError(f"Error parsing output: {e}")
+        return results
 
     def teardown(self):
         return
 
 class SubmitKernelSYCL(ComputeBenchmark):
     def __init__(self, bench, ioq):
         self.ioq = ioq
-        super().__init__(bench, "api_overhead_benchmark_sycl", "SubmitKernel")
+        super().__init__(bench, "api_overhead_benchmark_sycl", "SubmitKernel", "time [us]")
 
     def name(self):
         order = "in order" if self.ioq else "out of order"
@@ -148,14 +162,35 @@ def bin_args(self) -> list[str]:
             "--KernelExecTime=1"
         ]
 
-class SubmitKernelUR(ComputeBenchmark):
+class SubmitKernelURWithTime(ComputeBenchmark):
     def __init__(self, bench, ioq):
         self.ioq = ioq
-        super().__init__(bench, "api_overhead_benchmark_ur", "SubmitKernel")
+        super().__init__(bench, "api_overhead_benchmark_ur", "SubmitKernel", "time [us]")
 
     def name(self):
         order = "in order" if self.ioq else "out of order"
-        return f"api_overhead_benchmark_ur SubmitKernel {order}"
+        return f"api_overhead_benchmark_ur SubmitKernel {order} time"
+
+    def bin_args(self) -> list[str]:
+        return [
+            f"--Ioq={self.ioq}",
+            "--DiscardEvents=0",
+            "--MeasureCompletion=0",
+            "--iterations=100000",
+            "--Profiling=0",
+            "--NumKernels=10",
+            "--KernelExecTime=1"
+        ]
+
+
+class SubmitKernelURWithCPUCounter(ComputeBenchmark):
+    def __init__(self, bench, ioq):
+        self.ioq = ioq
+        super().__init__(bench, "api_overhead_benchmark_ur", "SubmitKernel", "hw instructions [count]")
+
+    def name(self):
+        order = "in order" if self.ioq else "out of order"
+        return f"api_overhead_benchmark_ur SubmitKernel {order} CPU count"
 
     def bin_args(self) -> list[str]:
         return [
@@ -175,7 +210,7 @@ def __init__(self, bench, ioq, isCopyOnly, source, destination, size):
         self.source = source
         self.destination = destination
         self.size = size
-        super().__init__(bench, "api_overhead_benchmark_sycl", "ExecImmediateCopyQueue")
+        super().__init__(bench, "api_overhead_benchmark_sycl", "ExecImmediateCopyQueue", "time [us]")
 
     def name(self):
         order = "in order" if self.ioq else "out of order"
@@ -198,7 +233,7 @@ def __init__(self, bench, isCopyOnly, source, destination, size):
         self.source = source
         self.destination = destination
         self.size = size
-        super().__init__(bench, "memory_benchmark_sycl", "QueueInOrderMemcpy")
+        super().__init__(bench, "memory_benchmark_sycl", "QueueInOrderMemcpy", "time [us]")
 
     def name(self):
         return f"memory_benchmark_sycl QueueInOrderMemcpy from {self.source} to {self.destination}, size {self.size}"
@@ -218,7 +253,7 @@ def __init__(self, bench, source, destination, size):
         self.source = source
         self.destination = destination
         self.size = size
-        super().__init__(bench, "memory_benchmark_sycl", "QueueMemcpy")
+        super().__init__(bench, "memory_benchmark_sycl", "QueueMemcpy", "time [us]")
 
     def name(self):
         return f"memory_benchmark_sycl QueueMemcpy from {self.source} to {self.destination}, size {self.size}"
@@ -236,7 +271,7 @@ def __init__(self, bench, type, size, placement):
         self.type = type
         self.size = size
         self.placement = placement
-        super().__init__(bench, "memory_benchmark_sycl", "StreamMemory")
+        super().__init__(bench, "memory_benchmark_sycl", "StreamMemory", "time [us]")
 
     def name(self):
         return f"memory_benchmark_sycl StreamMemory, placement {self.placement}, type {self.type}, size {self.size}"
@@ -253,7 +288,7 @@ def bin_args(self) -> list[str]:
 
 class VectorSum(ComputeBenchmark):
     def __init__(self, bench):
-        super().__init__(bench, "miscellaneous_benchmark_sycl", "VectorSum")
+        super().__init__(bench, "miscellaneous_benchmark_sycl", "VectorSum", "time [us]")
 
     def name(self):
         return f"miscellaneous_benchmark_sycl VectorSum"
@@ -274,7 +309,7 @@ def __init__(self, bench, numOpsPerThread, numThreads, allocSize, iterations, sr
         self.iterations = iterations
         self.srcUSM = srcUSM
         self.dstUSM = dstUSM
-        super().__init__(bench, "multithread_benchmark_ur", "MemcpyExecute")
+        super().__init__(bench, "multithread_benchmark_ur", "MemcpyExecute", "time [us]")
 
     def name(self):
         return f"multithread_benchmark_ur MemcpyExecute opsPerThread:{self.numOpsPerThread}, numThreads:{self.numThreads}, allocSize:{self.allocSize} srcUSM:{self.srcUSM} dstUSM:{self.dstUSM}"