feat: add profiler_device speedup metric for CUDA benchmarks

Add PyTorch Profiler device time measurement and speedup calculation
for CUDA benchmarks. This provides a more accurate kernel-level
performance comparison between eager and compiled models.

- Add --profiler-device-time CLI argument
- Measure profiler_device via torch.profiler with CUDA activity
- Print [Speedup][profiler_device] in benchmark results
- Propagate flag through multi-model test runners

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: fangfangssj

graph_net_bench/test_compiler_util.py

@@ -205,6 +205,7 @@ def print_times_and_speedup(args, eager_stats, compiled_stats):
 
     e2e_speedup = 0
     gpu_speedup = 0
+    profiler_device_speedup = 0
 
     eager_e2e_time_ms = eager_stats.get("e2e", {}).get("mean", 0)
     compiled_e2e_time_ms = compiled_stats.get("e2e", {}).get("mean", 0)
@@ -219,12 +220,29 @@ def print_times_and_speedup(args, eager_stats, compiled_stats):
         if eager_gpu_time_ms > 0 and compiled_gpu_time_ms > 0:
             gpu_speedup = eager_gpu_time_ms / compiled_gpu_time_ms
 
+        eager_profiler_device_ms = eager_stats.get("profiler_device", {}).get("mean", 0)
+        compiled_profiler_device_ms = compiled_stats.get("profiler_device", {}).get(
+            "mean", 0
+        )
+
+        if eager_profiler_device_ms > 0 and compiled_profiler_device_ms > 0:
+            profiler_device_speedup = (
+                eager_profiler_device_ms / compiled_profiler_device_ms
+            )
+
     if e2e_speedup > 0:
         print_with_log_prompt("[Speedup][e2e]:", f"{e2e_speedup:.5f}", args.log_prompt)
 
     if is_gpu_device(args.device) and gpu_speedup > 0:
         print_with_log_prompt("[Speedup][gpu]:", f"{gpu_speedup:.5f}", args.log_prompt)
 
+    if is_gpu_device(args.device) and profiler_device_speedup > 0:
+        print_with_log_prompt(
+            "[Speedup][profiler_device]:",
+            f"{profiler_device_speedup:.5f}",
+            args.log_prompt,
+        )
+
 
 def check_type_match(eager_dtypes, compiled_dtypes):
     if len(eager_dtypes) != len(compiled_dtypes):

graph_net_bench/torch/test_compiler.py

@@ -206,6 +206,28 @@ def measure_performance(model_call, args, compiler):
             e2e_times.append(duration_box.value)
         stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times)
 
+    # Profiler device_time measurement (only when enabled and on CUDA)
+    if args.profiler_device_time and "cuda" in args.device:
+        profiler_device_times = []
+        for i in range(args.trials):
+            with torch.profiler.profile(
+                activities=[torch.profiler.ProfilerActivity.CUDA],
+            ) as prof:
+                model_call()
+            compiler.synchronize()
+            device_time_ms = (
+                sum(evt.self_device_time_total for evt in prof.key_averages()) / 1000.0
+            )
+            profiler_device_times.append(device_time_ms)
+            print(
+                f"Trial {i + 1}: profiler_device={device_time_ms:.5f} ms",
+                file=sys.stderr,
+                flush=True,
+            )
+        stats["profiler_device"] = test_compiler_util.get_timing_stats(
+            profiler_device_times
+        )
+
     return outs, stats
 
 
@@ -423,6 +445,7 @@ def test_multi_models(args):
                     f"--trials {args.trials}",
                     f"--log-prompt {args.log_prompt}",
                     f"--config {args.config}",
+                    "--profiler-device-time" if args.profiler_device_time else "",
                 ]
             )
             try:
@@ -472,6 +495,7 @@ def test_multi_models_with_prefix(args):
                 f"--trials {args.trials}",
                 f"--log-prompt {args.log_prompt}",
                 f"--config {args.config}",
+                "--profiler-device-time" if args.profiler_device_time else "",
             ]
         )
         try:
@@ -558,5 +582,11 @@ def main(args):
         default=None,
         help="base64 encode configuration json.",
     )
+    parser.add_argument(
+        "--profiler-device-time",
+        action="store_true",
+        default=False,
+        help="Enable PyTorch Profiler device time measurement (CUDA only)",
+    )
     args = parser.parse_args()
     main(args=args)