diff --git a/graph_net/benchmark_result.py b/graph_net/benchmark_result.py deleted file mode 100644 index a7ffbf9ab8..0000000000 --- a/graph_net/benchmark_result.py +++ /dev/null @@ -1,103 +0,0 @@ -import os -import sys -import json -import re - - -class BenchmarkResult: - def __init__(self, args, framework, hardware, compile_framework_version): - self.configuration = { - "model_name": self.get_model_name(args), - "subgraph_tag": self.get_subgraph_tag(args), - "model_path": args.model_path, - "device": args.device, - "hardware": hardware, - "framework": framework, - "compiler": args.compiler, - "compile_framework_version": compile_framework_version, - "warmup": args.warmup, - "trials": args.trials, - } - self.model_info = { - "num_ops": -1, - "input_dtypes": None, - "param_dtypes": None, - } - self.correctness = {} - self.performance = { - "eager": None, - "compiled": None, - "speedup": {}, - } - - self.device = args.device - - self.eager_e2e_time_ms = -1 - self.compiled_e2e_time_ms = -1 - self.e2e_speedup = -1 - - self.eager_gpu_time_ms = -1 - self.compiled_gpu_time_ms = -1 - self.gpu_speedup = -1 - - def get_model_name(self, args): - model_name = None - with open(os.path.join(args.model_path, "graph_net.json"), "r") as f: - data = json.load(f) - model_name = data.get("model_name", None) - - if model_name is not None: - fields = args.model_path.split(os.sep) - pattern = rf"^subgraph(_\d+)?$" - model_name = fields[-2] if re.match(pattern, fields[-1]) else fields[-1] - return model_name - - def get_subgraph_tag(self, args): - fields = args.model_path.split(os.sep) - pattern = rf"^subgraph(_\d+)?$" - return fields[-1] if re.match(pattern, fields[-1]) else "" - - def update_model_info(self, num_ops, input_dtypes, param_dtypes): - self.model_info["num_ops"] = num_ops - self.model_info["input_dtypes"] = input_dtypes - self.model_info["param_dtypes"] = param_dtypes - - def update_corrrectness(self, key, cmp_ret): - self.correctness[key] = cmp_ret - - def update_performance(self, eager_stats, compiled_stats): - self.performance["eager"] = eager_stats - self.performance["compiled"] = compiled_stats - - self.eager_e2e_time_ms = eager_stats.get("e2e", {}).get("mean", -1) - self.compiled_e2e_time_ms = compiled_stats.get("e2e", {}).get("mean", -1) - if self.eager_e2e_time_ms > 0 and self.compiled_e2e_time_ms > 0: - self.e2e_speedup = self.eager_e2e_time_ms / self.compiled_e2e_time_ms - self.performance["speedup"]["e2e"] = float(f"{self.e2e_speedup:.6g}") - - if "cuda" in self.device: - self.eager_gpu_time_ms = eager_stats.get("gpu", {}).get("mean", -1) - self.compiled_gpu_time_ms = compiled_stats.get("gpu", {}).get("mean", -1) - if self.eager_gpu_time_ms > 0 and self.compiled_gpu_time_ms > 0: - self.gpu_speedup = self.eager_gpu_time_ms / self.compiled_gpu_time_ms - self.performance["speedup"]["gpu"] = float(f"{self.gpu_speedup:.6g}") - - def write_to_json(self, output_dir): - assert output_dir is not None - os.makedirs(output_dir, exist_ok=True) - result_data = { - "configuration": self.configuration, - "model_info": self.model_info, - "correctness": self.correctness, - "performance": self.performance, - } - model_name = self.configuration["model_name"] - subgraph_tag = self.configuration["subgraph_tag"] - compiler_name = self.configuration["compiler"] - file_path = os.path.join( - output_dir, f"{model_name}_{subgraph_tag}_{compiler_name}.json" - ) - with open(file_path, "w") as f: - json.dump(result_data, f, indent=4) - print(f"Result saved to {file_path}", file=sys.stderr) - print(result_data) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 527509a585..51da7f5408 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -7,12 +7,39 @@ from dataclasses import dataclass from contextlib import contextmanager import time +import math import numpy as np import random import platform +import traceback from graph_net.paddle import utils -from graph_net.benchmark_result import BenchmarkResult +from graph_net import path_utils +from graph_net import test_compiler_util + + +def set_seed(random_seed): + paddle.seed(random_seed) + random.seed(random_seed) + np.random.seed(random_seed) + + +def get_hardward_name(args): + if args.device == "cuda": + hardware = paddle.device.cuda.get_device_name(0) + elif args.device == "cpu": + hardware = platform.processor() + else: + hardware = "unknown" + return hardware + + +def get_compile_framework_version(args): + if args.compiler == "cinn": + compile_framework_version = paddle.__version__ + else: + compile_framework_version = "unknown" + return compile_framework_version def load_class_from_file(file_path: str, class_name: str): @@ -49,21 +76,9 @@ def get_input_dict(args): params = inputs_params["weight_info"] inputs = inputs_params["input_info"] - param_dtypes = set() - for name, info in params.items(): - dtype = str(info["info"]["dtype"]) - if dtype not in param_dtypes: - param_dtypes.add(dtype) - - input_dtypes = set() - for name, info in inputs.items(): - dtype = str(info["info"]["dtype"]) - if dtype not in input_dtypes: - input_dtypes.add(dtype) - params.update(inputs) state_dict = {k: utils.replay_tensor(v) for k, v in params.items()} - return state_dict, list(input_dtypes), list(param_dtypes) + return state_dict def get_input_spec(args): @@ -86,253 +101,200 @@ def get_compiled_model(args, model): full_graph=True, ) compiled_model.eval() + program = compiled_model.forward.concrete_program.main_program return compiled_model -def regular_item(item): - assert isinstance(item, paddle.Tensor) - if item.dtype not in [paddle.float32, paddle.float64]: - item = item.astype("float32") - return item - - -def count_number_of_ops(args, model, eager_mode): - if eager_mode: - static_model = paddle.jit.to_static( - model, - input_spec=get_input_spec(args), - full_graph=True, - backend=None, - ) - static_model.eval() - program = static_model.forward.concrete_program.main_program - else: - program = model.forward.concrete_program.main_program - print(program) - - num_ops = 0 - for block in program.blocks: - for op in block.ops: - if op.name() != "pd_op.data" and not op.name().startswith("builtin."): - num_ops += 1 - print(f"Totally {num_ops} ops.") - print("") - return num_ops - - -@dataclass -class DurationBox: - value: int - - -@contextmanager -def naive_timer(duration_box, synchronizer_func): - synchronizer_func() - start = time.time() - yield - synchronizer_func() - end = time.time() - duration_box.value = (end - start) * 1000 # Store in milliseconds - - -def get_timing_stats(elapsed_times): - stats = { - "mean": float(f"{np.mean(elapsed_times):.6g}"), - "std": float(f"{np.std(elapsed_times):.6g}"), - "min": float(f"{np.min(elapsed_times):.6g}"), - "max": float(f"{np.max(elapsed_times):.6g}"), - } - return stats +def get_static_model(args, model): + static_model = paddle.jit.to_static( + model, + input_spec=get_input_spec(args), + full_graph=True, + backend=None, + ) + static_model.eval() + program = static_model.forward.concrete_program.main_program + return static_model -def measure_performance(model_call, args, synchronizer_func): +def measure_performance(model_call, args, synchronizer_func, profile=False): + runtime_seed = 1024 stats = {} + paddle.seed(runtime_seed) + outs = model_call() + # Warmup runs for _ in range(args.warmup): - outs = model_call() + model_call() synchronizer_func() + hardware_name = get_hardward_name(args) + print( + f"[Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}", + file=sys.stderr, + flush=True, + ) + if "cuda" in args.device: """ Acknowledgement: We evaluate the performance on both end-to-end and GPU-only timings, With reference to methods only based on CUDA events from KernelBench in https://github.com/ScalingIntelligence/KernelBench """ - hardware_name = paddle.device.cuda.get_device_name(0) - print( - f"{args.log_prompt} [Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}" - ) e2e_times = [] gpu_times = [] + if profile: + paddle.base.core.nvprof_start() for i in range(args.trials): # End-to-end timing (naive_timer) - duration_box = DurationBox(-1) - with naive_timer(duration_box, synchronizer_func): + duration_box = test_compiler_util.DurationBox(-1) + with test_compiler_util.naive_timer(duration_box, synchronizer_func): # GPU-only timing (CUDA Events) start_event = paddle.device.Event(enable_timing=True) end_event = paddle.device.Event(enable_timing=True) start_event.record() - outs = model_call() + model_call() end_event.record() gpu_time_ms = start_event.elapsed_time(end_event) e2e_times.append(duration_box.value) gpu_times.append(gpu_time_ms) print( - f"Trial {i + 1}: e2e={duration_box.value:.4f} ms, gpu={gpu_time_ms:.5g} ms" + f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms", + file=sys.stderr, + flush=True, ) + if profile: + paddle.base.core.nvprof_stop() - stats["e2e"] = get_timing_stats(e2e_times) - stats["gpu"] = get_timing_stats(gpu_times) + stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) + stats["gpu"] = test_compiler_util.get_timing_stats(gpu_times) else: # CPU or other devices - hardware_name = platform.processor() - print( - f"[Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}" - ) - e2e_times = [] for i in range(args.trials): - duration_box = DurationBox(-1) - with naive_timer(duration_box, compiler.synchronize): - outs = model_call() + duration_box = test_compiler_util.DurationBox(-1) + with test_compiler_util.naive_timer(duration_box, synchronizer_func): + model_call() print(f"Trial {i + 1}: e2e={duration_box.value:.4f} ms") e2e_times.append(duration_box.value) - stats["e2e"] = get_timing_stats(e2e_times) + stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) return outs, stats -def init_benchmark_result(args): - if args.device == "cuda": - hardware = paddle.device.cuda.get_device_name(0) - elif args.device == "cpu": - hardware = platform.processor() - else: - hardware = "unknown" +def check_outputs(args, expected_out, compiled_out): + if isinstance(expected_out, paddle.Tensor): + expected_out = [expected_out] + if isinstance(compiled_out, paddle.Tensor): + compiled_out = [compiled_out] - if args.compiler == "cinn": - compile_framework_version = paddle.__version__ - else: - compile_framework_version = "unknown" + eager_dtypes = [None] * len(expected_out) + for i, tensor in enumerate(expected_out): + eager_dtypes[i] = ( + str(tensor.dtype).replace("paddle.", "") if tensor is not None else "none" + ) - result_data = BenchmarkResult( - args=args, - framework="PaddlePaddle", - hardware=hardware, - compile_framework_version=compile_framework_version, + compiled_dtypes = [None] * len(compiled_out) + for i, tensor in enumerate(compiled_out): + compiled_dtypes[i] = ( + str(tensor.dtype).replace("paddle.", "") if tensor is not None else "none" + ) + + type_match = test_compiler_util.check_output_datatype( + args, eager_dtypes, compiled_dtypes ) - return result_data + + def transfer_to_float(origin_outputs): + outputs = [] + for item in origin_outputs: + if ( + item is not None + and isinstance(item, paddle.Tensor) + and item.dtype not in [paddle.float32, paddle.float64] + ): + item = item.astype("float32") + outputs.append(item) + return outputs + + if type_match: + test_compiler_util.check_equal( + args, + expected_out, + compiled_out, + cmp_equal_func=get_cmp_equal, + ) + + expected_out_fp32 = transfer_to_float(expected_out) + compiled_out_fp32 = transfer_to_float(compiled_out) + test_compiler_util.check_allclose( + args, + expected_out_fp32, + compiled_out_fp32, + cmp_all_close_func=get_cmp_all_close, + cmp_max_diff_func=get_cmp_max_diff, + cmp_mean_diff_func=get_cmp_mean_diff, + cmp_max_relative_diff_func=get_cmp_max_relative_diff, + cmp_mean_relative_diff_func=get_cmp_mean_relative_diff, + ) def test_single_model(args): synchronizer_func = get_synchronizer_func(args) - input_dict, input_dtypes, param_dtypes = get_input_dict(args) + input_dict = get_input_dict(args) model = get_model(args) model.eval() - # Collect model information - num_eager_ops = count_number_of_ops(args, model, eager_mode=True) - - # Initialize benchmark result - result_data = init_benchmark_result(args) - result_data.update_model_info(num_eager_ops, input_dtypes, param_dtypes) + test_compiler_util.print_basic_config( + args, get_hardward_name(args), get_compile_framework_version(args) + ) # Run on eager mode - expected_out, eager_time_stats = measure_performance( - lambda: model(**input_dict), args, synchronizer_func - ) + eager_success = False + try: + print("Run model in eager mode.") + static_model = get_static_model(args, model) + expected_out, eager_time_stats = measure_performance( + lambda: static_model(**input_dict), args, synchronizer_func, profile=False + ) + eager_success = True + except Exception as e: + print(f"Run model in eager mode failed: {str(e)}\n{traceback.format_exc()}") # Run on compiling mode - compiled_model = get_compiled_model(args, model) - compiled_out, compiled_time_stats = measure_performance( - lambda: compiled_model(**input_dict), args, synchronizer_func - ) - - if isinstance(expected_out, paddle.Tensor): - expected_out = [expected_out] - compiled_out = [compiled_out] - if isinstance(expected_out, list) or isinstance(expected_out, tuple): - output_dtypes = [] - for a, b in zip(expected_out, compiled_out): - if (a is None and b is not None) or (a is not None and b is None): - raise ValueError("Both expected_out and compiled_out must be not None.") - if a is not None and b is not None: - assert ( - a.dtype == b.dtype - ), f"expected_out's dtype ({a.dtype}) is not the same as compiled_out's dtype {b.dtype}." - output_dtypes.append(str(a.dtype)) - result_data.update_corrrectness("num_outpus", len(output_dtypes)) - result_data.update_corrrectness("output_dtyps", output_dtypes) - - # Remove all None in outputs - expected_out = [x for x in expected_out if x is not None] - compiled_out = [x for x in compiled_out if x is not None] - expected_out = [ - regular_item(item) - for item in expected_out - if item is not None and np.array(item).size != 0 - ] - compiled_out = [ - regular_item(item) - for item in compiled_out - if item is not None and np.array(item).size != 0 - ] - else: - raise ValueError("Illegal return value.") - - def print_cmp(key, func, **kwargs): - cmp_ret = func(expected_out, compiled_out, **kwargs) - result_data.update_corrrectness(key, cmp_ret) - print( - f"{args.log_prompt} {key} model_path:{args.model_path} {cmp_ret}", - file=sys.stderr, + compiled_success = False + try: + print("Run model in compiled mode.") + compiled_model = get_compiled_model(args, model) + compiled_out, compiled_time_stats = measure_performance( + lambda: compiled_model(**input_dict), args, synchronizer_func, profile=False ) + compiled_success = True + except Exception as e: + print(f"Run model in compiled mode failed: {str(e)}\n{traceback.format_exc()}") - print_cmp("cmp.equal", get_cmp_equal) - print_cmp("cmp.all_close_atol8_rtol8", get_cmp_all_close, atol=1e-8, rtol=1e-8) - print_cmp("cmp.all_close_atol8_rtol5", get_cmp_all_close, atol=1e-8, rtol=1e-5) - print_cmp("cmp.all_close_atol5_rtol5", get_cmp_all_close, atol=1e-5, rtol=1e-5) - print_cmp("cmp.all_close_atol3_rtol2", get_cmp_all_close, atol=1e-3, rtol=1e-2) - print_cmp("cmp.all_close_atol2_rtol1", get_cmp_all_close, atol=1e-2, rtol=1e-1) - print_cmp("cmp.max_diff", get_cmp_max_diff) - print_cmp("cmp.mean_diff", get_cmp_mean_diff) - print_cmp("cmp.diff_count_atol8_rtol8", get_cmp_diff_count, atol=1e-8, rtol=1e-8) - print_cmp("cmp.diff_count_atol8_rtol5", get_cmp_diff_count, atol=1e-8, rtol=1e-5) - print_cmp("cmp.diff_count_atol5_rtol5", get_cmp_diff_count, atol=1e-5, rtol=1e-5) - print_cmp("cmp.diff_count_atol3_rtol2", get_cmp_diff_count, atol=1e-3, rtol=1e-2) - print_cmp("cmp.diff_count_atol2_rtol1", get_cmp_diff_count, atol=1e-2, rtol=1e-1) - - print( - f"{args.log_prompt} information model_path:{args.model_path} {num_eager_ops} ops, param_dtypes:{param_dtypes}, input_dtypes:{input_dtypes}", - file=sys.stderr, - ) - - result_data.update_performance(eager_time_stats, compiled_time_stats) - duration_log = ( - f"{args.log_prompt} [Duration] " - f"eager_e2e:{result_data.eager_e2e_time_ms:.4f} ms compiled_e2e:{result_data.compiled_e2e_time_ms:.4f} ms" - ) - speedup_log = ( - f"{args.log_prompt} [Speedup] " f"e2e_speedup:{result_data.e2e_speedup:.4f}" - ) - - if "cuda" in args.device: - duration_log += f" eager_gpu:{result_data.eager_gpu_time_ms:.4f} ms compiled_gpu:{result_data.compiled_gpu_time_ms:.4f} ms" - speedup_log += f" gpu_speedup:{result_data.gpu_speedup:.4f}" - - print(duration_log, file=sys.stderr) - print(speedup_log, file=sys.stderr) + test_compiler_util.print_running_status(args, eager_success, compiled_success) + if eager_success and compiled_success: + check_outputs(args, expected_out, compiled_out) - if args.output_dir: - result_data.write_to_json(args.output_dir) + test_compiler_util.print_times_and_speedup( + args, eager_time_stats, compiled_time_stats + ) def get_cmp_equal(expected_out, compiled_out): + def convert(x): + if x.dtype in [paddle.float16, paddle.bfloat16]: + return x.astype("float32") + elif x.dtype in [paddle.uint8, paddle.int8, paddle.int16]: + return x.astype("int32") + return x + return " ".join( - str(int(paddle.equal_all(a, b))) for a, b in zip(expected_out, compiled_out) + str(int(paddle.equal_all(convert(a), convert(b)))) + for a, b in zip(expected_out, compiled_out) ) @@ -357,63 +319,78 @@ def get_cmp_mean_diff(expected_out, compiled_out): ) -def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): +def get_cmp_max_relative_diff(expected_out, compiled_out): + epsilon = 1e-8 return " ".join( - str(paddle.sum(~paddle.isclose(a, b, atol=atol, rtol=rtol)).item()) + str(paddle.max(paddle.abs(a - b) / (paddle.abs(a) + epsilon)).item()) for a, b in zip(expected_out, compiled_out) ) -def test_multi_models(args): - for model_path in get_recursively_model_path(args.model_path): - cmd = "".join( - [ - sys.executable, - "-m graph_net.paddle.test_compiler", - f"--model-path {model_path}", - f"--compiler {args.compiler}", - f"--device {args.device}", - f"--warmup {args.warmup}", - f"--trials {args.trials}", - f"--log-prompt {args.log_prompt}", - f"--output-dir {args.output_dir}", - ] - ) - cmd_ret = os.system(cmd) - assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" - +def get_cmp_mean_relative_diff(expected_out, compiled_out): + epsilon = 1e-8 + return " ".join( + str(paddle.mean(paddle.abs(a - b) / (paddle.abs(a) + epsilon)).item()) + for a, b in zip(expected_out, compiled_out) + ) -def get_recursively_model_path(root_dir): - for sub_dir in get_immediate_subdirectory_paths(root_dir): - if is_single_model_dir(sub_dir): - yield sub_dir - else: - yield from get_recursively_model_path(sub_dir) +def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): + return " ".join( + str(paddle.sum(~paddle.isclose(a, b, atol=atol, rtol=rtol)).item()) + for a, b in zip(expected_out, compiled_out) + ) -def get_immediate_subdirectory_paths(parent_dir): - return [ - sub_dir - for name in os.listdir(parent_dir) - for sub_dir in [os.path.join(parent_dir, name)] - if os.path.isdir(sub_dir) - ] +def test_multi_models(args): + verified_samples = None + if args.verified_samples_list_path is not None: + assert os.path.isfile(args.verified_samples_list_path) + graphnet_root = path_utils.get_graphnet_root() + print(f"graphnet_root: {graphnet_root}") + verified_samples = [] + with open(args.verified_samples_list_path, "r") as f: + for line in f.readlines(): + verified_samples.append(os.path.join(graphnet_root, line.strip())) + + sample_idx = 0 + failed_samples = [] + for model_path in path_utils.get_recursively_model_path(args.model_path): + if verified_samples is None or os.path.abspath(model_path) in verified_samples: + print(f"[{sample_idx}] test_compiler, model_path: {model_path}") + cmd = " ".join( + [ + sys.executable, + "-m graph_net.paddle.test_compiler", + f"--model-path {model_path}", + f"--compiler {args.compiler}", + f"--device {args.device}", + f"--warmup {args.warmup}", + f"--trials {args.trials}", + f"--log-prompt {args.log_prompt}", + ] + ) + cmd_ret = os.system(cmd) + # assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" + if cmd_ret != 0: + failed_samples.append(model_path) + sample_idx += 1 -def is_single_model_dir(model_dir): - return os.path.isfile(f"{model_dir}/graph_net.json") + print( + f"Totally {sample_idx} verified samples, failed {len(failed_samples)} samples." + ) + for model_path in failed_samples: + print(f"- {model_path}") def main(args): assert os.path.isdir(args.model_path) assert args.compiler == "cinn" - random_seed = 123 - paddle.seed(random_seed) - random.seed(random_seed) - np.random.seed(random_seed) + initalize_seed = 123 + set_seed(random_seed=initalize_seed) - if is_single_model_dir(args.model_path): + if path_utils.is_single_model_dir(args.model_path): test_single_model(args) else: test_multi_models(args) @@ -455,11 +432,11 @@ def main(args): help="Log prompt for performance log filtering.", ) parser.add_argument( - "--output-dir", + "--verified-samples-list-path", type=str, required=False, default=None, - help="Directory to save the structured JSON result file.", + help="Path to model file(s), each subdirectory containing graph_net.json will be regarded as a model", ) args = parser.parse_args() main(args=args) diff --git a/graph_net/paddle/utils.py b/graph_net/paddle/utils.py index c9b73bc2f5..4dbe8b3392 100644 --- a/graph_net/paddle/utils.py +++ b/graph_net/paddle/utils.py @@ -122,7 +122,7 @@ def load_converted_list_from_text(file_path): return [*weight_info, *input_info] -def ConvertToValidNumber(data_type, value): +def convert_to_valid_number(data_type, value): if value is not None and data_type in [ paddle.float32, paddle.float16, @@ -160,10 +160,10 @@ def convert_meta_classes_to_tensors(file_path): "shape": attrs.get("shape", []), "dtype": data_type, "device": attrs.get("device", "gpu"), - "mean": ConvertToValidNumber(data_type, attrs.get("mean", None)), - "std": ConvertToValidNumber(data_type, attrs.get("std", None)), - "min_val": ConvertToValidNumber(data_type, attrs.get("min_val", 0)), - "max_val": ConvertToValidNumber(data_type, attrs.get("max_val", 2)), + "mean": convert_to_valid_number(data_type, attrs.get("mean", None)), + "std": convert_to_valid_number(data_type, attrs.get("std", None)), + "min_val": convert_to_valid_number(data_type, attrs.get("min_val", 0)), + "max_val": convert_to_valid_number(data_type, attrs.get("max_val", 2)), }, "data": data_value, "name": attrs.get("name"), diff --git a/graph_net/path_utils.py b/graph_net/path_utils.py new file mode 100644 index 0000000000..8672109e69 --- /dev/null +++ b/graph_net/path_utils.py @@ -0,0 +1,27 @@ +import os +import graph_net + + +def get_graphnet_root(): + return os.path.dirname(os.path.dirname(graph_net.__file__)) + + +def is_single_model_dir(model_dir): + return os.path.isfile(f"{model_dir}/graph_net.json") + + +def get_recursively_model_path(root_dir): + for sub_dir in get_immediate_subdirectory_paths(root_dir): + if is_single_model_dir(sub_dir): + yield sub_dir + else: + yield from get_recursively_model_path(sub_dir) + + +def get_immediate_subdirectory_paths(parent_dir): + return [ + sub_dir + for name in os.listdir(parent_dir) + for sub_dir in [os.path.join(parent_dir, name)] + if os.path.isdir(sub_dir) + ] diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index a8520b6b2b..2b4d083b2d 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -1,4 +1,162 @@ import os +import re +import sys +import json +import time +import numpy as np +from dataclasses import dataclass +from contextlib import contextmanager + + +@dataclass +class DurationBox: + value: int + + +@contextmanager +def naive_timer(duration_box, synchronizer_func): + synchronizer_func() + start = time.time() + yield + synchronizer_func() + end = time.time() + duration_box.value = (end - start) * 1000 # Store in milliseconds + + +def get_timing_stats(elapsed_times): + stats = { + "mean": float(f"{np.mean(elapsed_times):.6g}"), + "std": float(f"{np.std(elapsed_times):.6g}"), + "min": float(f"{np.min(elapsed_times):.6g}"), + "max": float(f"{np.max(elapsed_times):.6g}"), + } + return stats + + +def get_model_name(model_path): + model_name = None + with open(os.path.join(model_path, "graph_net.json"), "r") as f: + data = json.load(f) + model_name = data.get("model_name", None) + + if model_name is not None: + fields = model_path.split(os.sep) + pattern = rf"^subgraph(_\d+)?$" + model_name = fields[-2] if re.match(pattern, fields[-1]) else fields[-1] + return model_name + + +def get_subgraph_tag(model_path): + fields = model_path.split(os.sep) + pattern = rf"^subgraph(_\d+)?$" + return fields[-1] if re.match(pattern, fields[-1]) else "" + + +def print_with_log_prompt(key, value, log_prompt): + print(f"{log_prompt} {key} {value}", file=sys.stderr, flush=True) + + +def print_basic_config(args, hardware_name, compile_framework_version): + model_path = os.path.normpath(args.model_path) + print_with_log_prompt("[Processing]", model_path, args.log_prompt) + + model_name = get_model_name(model_path) + print_with_log_prompt("[Config] model:", model_name, args.log_prompt) + + print_with_log_prompt("[Config] device:", args.device, args.log_prompt) + print_with_log_prompt("[Config] hardware:", hardware_name, args.log_prompt) + print_with_log_prompt("[Config] compiler:", args.compiler, args.log_prompt) + print_with_log_prompt("[Config] warmup:", args.warmup, args.log_prompt) + print_with_log_prompt("[Config] trials:", args.trials, args.log_prompt) + print_with_log_prompt( + "[Config] compile_framework_version:", + compile_framework_version, + args.log_prompt, + ) + + +def print_running_status(args, eager_success, compiled_success): + def convert_to_str(b): + return "success" if b else "failed" + + print_with_log_prompt( + "[Result][status]", + f"eager:{convert_to_str(eager_success)} compiled:{convert_to_str(compiled_success)}", + args.log_prompt, + ) + + +def print_times_and_speedup(args, eager_stats, compiled_stats): + print_with_log_prompt( + "[Performance][eager]:", json.dumps(eager_stats), args.log_prompt + ) + print_with_log_prompt( + "[Performance][compiled]:", json.dumps(compiled_stats), args.log_prompt + ) + + e2e_speedup = 0 + gpu_speedup = 0 + + eager_e2e_time_ms = eager_stats.get("e2e", {}).get("mean", 0) + compiled_e2e_time_ms = compiled_stats.get("e2e", {}).get("mean", 0) + + if eager_e2e_time_ms > 0 and compiled_e2e_time_ms > 0: + e2e_speedup = eager_e2e_time_ms / compiled_e2e_time_ms + + if "cuda" in args.device: + eager_gpu_time_ms = eager_stats.get("gpu", {}).get("mean", 0) + compiled_gpu_time_ms = compiled_stats.get("gpu", {}).get("mean", 0) + + if eager_gpu_time_ms > 0 and compiled_gpu_time_ms > 0: + gpu_speedup = eager_gpu_time_ms / compiled_gpu_time_ms + + if e2e_speedup > 0: + print_with_log_prompt("[Speedup][e2e]:", f"{e2e_speedup:.5f}", args.log_prompt) + + if "cuda" in args.device and gpu_speedup > 0: + print_with_log_prompt("[Speedup][gpu]:", f"{gpu_speedup:.5f}", args.log_prompt) + + +def check_type_match(eager_dtypes, compiled_dtypes): + if len(eager_dtypes) != len(compiled_dtypes): + type_match = False + else: + type_match = all( + eager == compiled for eager, compiled in zip(eager_dtypes, compiled_dtypes) + ) + return type_match + + +def check_output_datatype(args, eager_dtypes, compiled_dtypes): + print_with_log_prompt("[Datatype][eager]:", " ".join(eager_dtypes), args.log_prompt) + print_with_log_prompt( + "[Datatype][compiled]:", " ".join(compiled_dtypes), args.log_prompt + ) + + # datatype check + type_match = check_type_match(eager_dtypes, compiled_dtypes) + print_with_log_prompt( + "[DataType]", + f"eager:{eager_dtypes} compiled:{compiled_dtypes} match:{type_match}", + args.log_prompt, + ) + return type_match + + +def print_and_store_cmp(key, cmp_func, args, expected_out, compiled_out, **kwargs): + cmp_ret = cmp_func(expected_out, compiled_out, **kwargs) + print_with_log_prompt(f"[Correctness]{key}:", cmp_ret, args.log_prompt) + return cmp_ret + + +def check_equal(args, expected_out, compiled_out, cmp_equal_func): + print_and_store_cmp( + key="[equal]", + cmp_func=cmp_equal_func, + args=args, + expected_out=expected_out, + compiled_out=compiled_out, + ) def tolerance_generator(t): @@ -37,3 +195,30 @@ def generate_allclose_configs(cmp_all_close_func): (f"[all_close_atol_{atol:.2E}_rtol_{rtol:.2E}]", cmp_all_close_func, pair) ) return cmp_configs + + +def check_allclose( + args, + expected_out, + compiled_out, + cmp_all_close_func, + cmp_max_diff_func, + cmp_mean_diff_func, + cmp_max_relative_diff_func, + cmp_mean_relative_diff_func, +): + cmp_configs = generate_allclose_configs(cmp_all_close_func) + cmp_configs.append(("[max_diff]", cmp_max_diff_func, {})) + cmp_configs.append(("[mean_diff]", cmp_mean_diff_func, {})) + cmp_configs.append(("[max_relative_diff]", cmp_max_relative_diff_func, {})) + cmp_configs.append(("[mean_relative_diff]", cmp_mean_relative_diff_func, {})) + + for key, func, kwargs in cmp_configs: + print_and_store_cmp( + key=key, + cmp_func=func, + args=args, + expected_out=expected_out, + compiled_out=compiled_out, + **kwargs, + ) diff --git a/graph_net/torch/test_compiler.py b/graph_net/torch/test_compiler.py index 5922991c52..63ba5974fa 100644 --- a/graph_net/torch/test_compiler.py +++ b/graph_net/torch/test_compiler.py @@ -142,7 +142,7 @@ def measure_performance(model_call, args, compiler): e2e_times.append(duration_box.value) gpu_times.append(gpu_time_ms) print( - f"Trial {i + 1}: e2e={duration_box.value:.2f} ms, gpu={gpu_time_ms:.3g} ms", + f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms", file=sys.stderr, flush=True, ) @@ -164,7 +164,7 @@ def measure_performance(model_call, args, compiler): with naive_timer(duration_box, compiler.synchronize): model_call() print( - f"Trial {i + 1}: e2e={duration_box.value:.2f} ms", + f"Trial {i + 1}: e2e={duration_box.value:.5f} ms", file=sys.stderr, flush=True, )