From 05de56533b4b1cc51f4f4f6fa06df83e6eff7e6a Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Sat, 6 Sep 2025 18:53:41 +0800 Subject: [PATCH 01/16] Optimize the check codes of test_compiler. --- .../paddle/check_redundant_incrementally.py | 6 +- graph_net/paddle/test_compiler.py | 173 ++++++++++-------- graph_net/torch/test_compiler.py | 6 +- 3 files changed, 107 insertions(+), 78 deletions(-) diff --git a/graph_net/paddle/check_redundant_incrementally.py b/graph_net/paddle/check_redundant_incrementally.py index 256cb34a06..0ea2de4ccd 100644 --- a/graph_net/paddle/check_redundant_incrementally.py +++ b/graph_net/paddle/check_redundant_incrementally.py @@ -43,8 +43,10 @@ def is_single_model_dir(model_dir): def main(args): - assert os.path.isdir(args.model_path) - assert os.path.isdir(args.graph_net_samples_path) + assert os.path.isdir(args.model_path), f"model_path: {args.model_path}" + assert os.path.isdir( + args.graph_net_samples_path + ), f"graph_net_samples_path: {args.graph_net_samples_path}" current_model_graph_hash_pathes = set( graph_hash_path for model_path in get_recursively_model_pathes(args.model_path) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 527509a585..f76f96cd5b 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -10,6 +10,7 @@ import numpy as np import random import platform +import traceback from graph_net.paddle import utils from graph_net.benchmark_result import BenchmarkResult @@ -89,13 +90,6 @@ def get_compiled_model(args, model): return compiled_model -def regular_item(item): - assert isinstance(item, paddle.Tensor) - if item.dtype not in [paddle.float32, paddle.float64]: - item = item.astype("float32") - return item - - def count_number_of_ops(args, model, eager_mode): if eager_mode: static_model = paddle.jit.to_static( @@ -227,70 +221,64 @@ def init_benchmark_result(args): return result_data -def test_single_model(args): - synchronizer_func = get_synchronizer_func(args) - input_dict, input_dtypes, param_dtypes = get_input_dict(args) - model = get_model(args) - model.eval() - - # Collect model information - num_eager_ops = count_number_of_ops(args, model, eager_mode=True) - - # Initialize benchmark result - result_data = init_benchmark_result(args) - result_data.update_model_info(num_eager_ops, input_dtypes, param_dtypes) - - # Run on eager mode - expected_out, eager_time_stats = measure_performance( - lambda: model(**input_dict), args, synchronizer_func - ) - - # Run on compiling mode - compiled_model = get_compiled_model(args, model) - compiled_out, compiled_time_stats = measure_performance( - lambda: compiled_model(**input_dict), args, synchronizer_func - ) - +def check_outputs(args, expected_out, compiled_out, result_data): if isinstance(expected_out, paddle.Tensor): expected_out = [expected_out] + if isinstance(compiled_out, paddle.Tensor): compiled_out = [compiled_out] - if isinstance(expected_out, list) or isinstance(expected_out, tuple): - output_dtypes = [] - for a, b in zip(expected_out, compiled_out): - if (a is None and b is not None) or (a is not None and b is None): - raise ValueError("Both expected_out and compiled_out must be not None.") - if a is not None and b is not None: - assert ( - a.dtype == b.dtype - ), f"expected_out's dtype ({a.dtype}) is not the same as compiled_out's dtype {b.dtype}." - output_dtypes.append(str(a.dtype)) - result_data.update_corrrectness("num_outpus", len(output_dtypes)) - result_data.update_corrrectness("output_dtyps", output_dtypes) - - # Remove all None in outputs - expected_out = [x for x in expected_out if x is not None] - compiled_out = [x for x in compiled_out if x is not None] - expected_out = [ - regular_item(item) - for item in expected_out - if item is not None and np.array(item).size != 0 - ] - compiled_out = [ - regular_item(item) - for item in compiled_out - if item is not None and np.array(item).size != 0 - ] - else: - raise ValueError("Illegal return value.") + + eager_output_dtypes = [None] * len(expected_out) + for i, tensor in enumerate(expected_out): + if tensor is not None: + eager_output_dtypes[i] = str(tensor.dtype) + result_data.update_corrrectness("num_eager_outputs", len(expected_out)) + result_data.update_corrrectness("eager_output_dtypes", eager_output_dtypes) + + compiled_output_dtypes = [None] * len(compiled_out) + for i, tensor in enumerate(compiled_out): + if tensor is not None: + compiled_output_dtypes[i] = str(tensor.dtype) + result_data.update_corrrectness("num_compiled_outputs", len(compiled_out)) + result_data.update_corrrectness("compiled_output_dtypes", compiled_output_dtypes) + + is_output_consistent = len(expected_out) == len(compiled_out) + for a, b in zip(expected_out, compiled_out): + if (a is None and b is not None) or (a is not None and b is None): + is_output_consistent = False + if a is not None and b is not None and a.dtype != b.dtype: + is_output_consistent = False + result_data.update_corrrectness("output_consistent", is_output_consistent) + + def regular_outputs(origin_outputs): + outputs = [] + for item in origin_outputs: + if ( + item is not None + and isinstance(item, paddle.Tensor) + and item.dtype not in [paddle.float32, paddle.float64] + ): + item = item.astype("float32") + outputs.append(item) + return outputs + + expected_out = regular_outputs(expected_out) + compiled_out = regular_outputs(compiled_out) def print_cmp(key, func, **kwargs): - cmp_ret = func(expected_out, compiled_out, **kwargs) + try: + cmp_ret = func(expected_out, compiled_out, **kwargs) + except Exception as e: + cmp_ret = f"{key} failed: {str(e)}\n{traceback.format_exc()}" result_data.update_corrrectness(key, cmp_ret) print( f"{args.log_prompt} {key} model_path:{args.model_path} {cmp_ret}", file=sys.stderr, ) + print( + f"{args.log_prompt} output_dtypes model_path:{args.model_path} eager:{eager_output_dtypes} compiled:{compiled_output_dtypes}", + file=sys.stderr, + ) print_cmp("cmp.equal", get_cmp_equal) print_cmp("cmp.all_close_atol8_rtol8", get_cmp_all_close, atol=1e-8, rtol=1e-8) print_cmp("cmp.all_close_atol8_rtol5", get_cmp_all_close, atol=1e-8, rtol=1e-5) @@ -305,26 +293,65 @@ def print_cmp(key, func, **kwargs): print_cmp("cmp.diff_count_atol3_rtol2", get_cmp_diff_count, atol=1e-3, rtol=1e-2) print_cmp("cmp.diff_count_atol2_rtol1", get_cmp_diff_count, atol=1e-2, rtol=1e-1) + +def test_single_model(args): + synchronizer_func = get_synchronizer_func(args) + input_dict, input_dtypes, param_dtypes = get_input_dict(args) + model = get_model(args) + model.eval() + + # Collect model information + num_eager_ops = count_number_of_ops(args, model, eager_mode=True) + + # Initialize benchmark result + result_data = init_benchmark_result(args) + result_data.update_model_info(num_eager_ops, input_dtypes, param_dtypes) + + # Run on eager mode + running_eager_success = False + try: + print("Run model in eager mode.") + expected_out, eager_time_stats = measure_performance( + lambda: model(**input_dict), args, synchronizer_func + ) + running_eager_success = True + except Exception as e: + print(f"Run model in eager mode failed: {str(e)}\n{traceback.format_exc()}") + + # Run on compiling mode + running_compiled_success = False + try: + print("Run model in compiled mode.") + compiled_model = get_compiled_model(args, model) + compiled_out, compiled_time_stats = measure_performance( + lambda: compiled_model(**input_dict), args, synchronizer_func + ) + running_compiled_success = True + except Exception as e: + print(f"Run model in compiled mode failed: {str(e)}\n{traceback.format_exc()}") + print( f"{args.log_prompt} information model_path:{args.model_path} {num_eager_ops} ops, param_dtypes:{param_dtypes}, input_dtypes:{input_dtypes}", file=sys.stderr, ) + if running_eager_success and running_compiled_success: + check_outputs(args, expected_out, compiled_out, result_data) - result_data.update_performance(eager_time_stats, compiled_time_stats) - duration_log = ( - f"{args.log_prompt} [Duration] " - f"eager_e2e:{result_data.eager_e2e_time_ms:.4f} ms compiled_e2e:{result_data.compiled_e2e_time_ms:.4f} ms" - ) - speedup_log = ( - f"{args.log_prompt} [Speedup] " f"e2e_speedup:{result_data.e2e_speedup:.4f}" - ) + result_data.update_performance(eager_time_stats, compiled_time_stats) + duration_log = ( + f"{args.log_prompt} [Duration] " + f"eager_e2e:{result_data.eager_e2e_time_ms:.4f} ms compiled_e2e:{result_data.compiled_e2e_time_ms:.4f} ms" + ) + speedup_log = ( + f"{args.log_prompt} [Speedup] " f"e2e_speedup:{result_data.e2e_speedup:.4f}" + ) - if "cuda" in args.device: - duration_log += f" eager_gpu:{result_data.eager_gpu_time_ms:.4f} ms compiled_gpu:{result_data.compiled_gpu_time_ms:.4f} ms" - speedup_log += f" gpu_speedup:{result_data.gpu_speedup:.4f}" + if "cuda" in args.device: + duration_log += f" eager_gpu:{result_data.eager_gpu_time_ms:.4f} ms compiled_gpu:{result_data.compiled_gpu_time_ms:.4f} ms" + speedup_log += f" gpu_speedup:{result_data.gpu_speedup:.4f}" - print(duration_log, file=sys.stderr) - print(speedup_log, file=sys.stderr) + print(duration_log, file=sys.stderr) + print(speedup_log, file=sys.stderr) if args.output_dir: result_data.write_to_json(args.output_dir) diff --git a/graph_net/torch/test_compiler.py b/graph_net/torch/test_compiler.py index 4c7f2318af..e7d12ba6cb 100644 --- a/graph_net/torch/test_compiler.py +++ b/graph_net/torch/test_compiler.py @@ -140,7 +140,7 @@ def measure_performance(model_call, args, compiler): e2e_times.append(duration_box.value) gpu_times.append(gpu_time_ms) print( - f"Trial {i + 1}: e2e={duration_box.value:.2f} ms, gpu={gpu_time_ms:.3g} ms" + f"Trial {i + 1}: e2e={duration_box.value:.4f} ms, gpu={gpu_time_ms:.5g} ms" ) stats["e2e"] = get_timing_stats(e2e_times) @@ -157,7 +157,7 @@ def measure_performance(model_call, args, compiler): duration_box = DurationBox(-1) with naive_timer(duration_box, compiler.synchronize): model_call() - print(f"Trial {i + 1}: e2e={duration_box.value:.2f} ms") + print(f"Trial {i + 1}: e2e={duration_box.value:.4f} ms") e2e_times.append(duration_box.value) stats["e2e"] = get_timing_stats(e2e_times) @@ -220,7 +220,7 @@ def test_single_model(args): "compile_framework_version" ] = f"BladeDISC {compiler.version}" else: - result_data["configuration"]["compiler_version"] = "unknown" + result_data["configuration"]["compile_framework_version"] = "unknown" eager_model_call = lambda: model(**input_dict) compiled_model_call = lambda: compiled_model(**input_dict) From 6d9b156d97bae69641ef1d015c3239fdb4caed03 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Fri, 19 Sep 2025 15:29:31 +0800 Subject: [PATCH 02/16] Implement collecting stats for paddle. --- graph_net/paddle/collect_stats.py | 342 ++++++++++++++++++++++++++++++ 1 file changed, 342 insertions(+) create mode 100644 graph_net/paddle/collect_stats.py diff --git a/graph_net/paddle/collect_stats.py b/graph_net/paddle/collect_stats.py new file mode 100644 index 0000000000..2894de4edd --- /dev/null +++ b/graph_net/paddle/collect_stats.py @@ -0,0 +1,342 @@ +import argparse +import os +import sys +import ast +import math +import importlib +import inspect +import subprocess +from datetime import datetime +from typing import Type +from dataclasses import dataclass, field +from collections import defaultdict + +import paddle +from graph_net.paddle import utils + + +def is_single_model_dir(model_dir): + return os.path.isfile(f"{model_dir}/graph_net.json") + + +def load_class_from_file(file_path: str, class_name: str) -> Type[paddle.nn.Layer]: + spec = importlib.util.spec_from_file_location("unnamed", file_path) + unnamed = importlib.util.module_from_spec(spec) + spec.loader.exec_module(unnamed) + model_class = getattr(unnamed, class_name, None) + return model_class + + +def get_argument_name_and_types(model_class, func_name): + argument_name2types = {} + for name, func in inspect.getmembers(model_class, predicate=inspect.isfunction): + if name == func_name: + for arg_name, arg in inspect.signature(func).parameters.items(): + if arg_name != "self": + argument_name2types[arg_name] = ( + None if arg.annotation is inspect._empty else arg.annotation + ) + return argument_name2types + + +def get_number_of_returns(file_path, class_name, func_name): + source = None + with open(f"{file_path}", "r") as f: + source = f.read() + + tree = ast.parse(source) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == class_name: + for f in node.body: + if isinstance(f, ast.FunctionDef) and f.name == func_name: + for stmt in ast.walk(f): + if isinstance(stmt, ast.Return): + if stmt.value is None: + return 0 + elif isinstance(stmt.value, ast.Tuple): + return len(stmt.value.elts) + else: + return 1 + return 0 + + +def read_graph_source_and_tag(model_path): + try: + with open(os.path.join(model_path, "graph_net.json"), "r") as f: + data = json.load(f) + return data["source"], data["heuristic_tag"] + except Exception: + if "PaddleX" in model_path: + return "PaddleX", "computer_vision" + elif "PaddleNLP" in model_path: + return "PaddleNLP", "nlp" + elif "PaddleScience" in model_path: + return "PaddleScience", "scientific_computing" + else: + return "unknown", "unknown" + + +def get_input_spec(model_path): + inputs_params_list = utils.load_converted_list_from_text(f"{model_path}") + input_spec = [None] * len(inputs_params_list) + for i, v in enumerate(inputs_params_list): + dtype = v["info"]["dtype"] + shape = v["info"]["shape"] + input_spec[i] = paddle.static.InputSpec(shape, dtype) + return input_spec + + +@dataclass +class OpStat: + op_name: str + op_dtypes: dict[str, int] = field(default_factory=dict) + count: int = 0 + + def update(self, other): + if isinstance(other, OpStat) and self.op_name == other.op_name: + self.count += other.count + for name, count in other.op_dtypes.items(): + self.op_dtypes[name] = self.op_dtypes.get(name, 0) + count + + +class ProgramAnalyzer: + def __init__(self): + self.op_stats = {} + self.input_dict = {} + self.num_ops = 0 + self.num_ops_misses_dtypes = 0 + self.is_complete = True + + def update_op_stats(self, op_name, op_dtype): + if op_name is not None: + dtype_str = str(op_dtype).replace("paddle.", "") + if self.op_stats.get(op_name, None) is None: + self.op_stats[op_name] = OpStat(op_name, {dtype_str: 1}, 1) + else: + self.op_stats[op_name].op_dtypes[dtype_str] = ( + self.op_stats[op_name].op_dtypes.get(dtype_str, 0) + 1 + ) + self.op_stats[op_name].count += 1 + + def __call__(self, program): + assert isinstance(program, paddle.base.libpaddle.pir.Program) + + self.op_stats = {} + self.num_ops_misses_dtypes = 0 + self.num_ops = 0 + for block in program.blocks: + for op in block.ops: + op_name = None + op_dtype = None + if op.name() == "pd_op.data": + op_attrs = op.attrs() + op_dtype = op_attrs["dtype"] + self.input_dict[op_attrs["name"]] = { + "dtype": str(op_dtype).replace("paddle.", ""), + "shape": op_attrs["shape"], + } + elif not op.name().startswith("builtin."): + self.num_ops += 1 + op_name = op.name().replace("pd_op.", "") + if len(op.results()) > 0: + op_dtype = op.results()[0].dtype + + if op_name is not None: + self.update_op_stats(op_name, op_dtype) + elif op_dtype is None: + self.num_ops_misses_dtypes += 1 + + if self.num_ops_misses_dtypes > 0: + self.is_complete = False + + def summary(self): + print( + f"Totally {self.num_ops} operators, and {self.num_ops_misses_dtypes} operators failed to inference dtypes." + ) + + +def collect_op_stats(model, model_path): + assert isinstance(model, paddle.nn.Layer), f"{type(model)=}" + try: + static_model = paddle.jit.to_static( + model, + input_spec=get_input_spec(model_path), + full_graph=True, + backend=None, + ) + static_model.eval() + program = static_model.forward.concrete_program.main_program + + program_analyzer = ProgramAnalyzer() + program_analyzer(program) + program_analyzer.summary() + return program_analyzer + except Exception: + print("Failed with to_static") + return None + + +def collect_model_stats(model_path, log_prompt): + file_path = os.path.join(model_path, "model.py") + model_class = load_class_from_file(file_path, "GraphModule") + model = model_class() + num_outputs = get_number_of_returns(file_path, "GraphModule", "forward") + + model_size = 0 + input_dtypes = {} + param_dtypes = {} + ops_count_dict = {} + op_dtypes = {} + + program_analyzer = collect_op_stats(model, model_path) + if program_analyzer is not None: + for op_name, stat in sorted(program_analyzer.op_stats.items()): + ops_count_dict[op_name] = stat.count + for dtype_str, num in stat.op_dtypes.items(): + if dtype_str is not None and dtype_str != "None": + op_dtypes[dtype_str] = op_dtypes.get(dtype_str, 0) + num + + inputs_params = utils.load_converted_from_text(f"{model_path}") + params = inputs_params["weight_info"] + inputs = inputs_params["input_info"] + + for name, value in program_analyzer.input_dict.items(): + dtype_str = value["dtype"] + if name in params.keys(): + param_numel = math.prod(value["shape"]) + model_size += param_numel + param_dtypes[dtype_str] = param_dtypes.get(dtype_str, 0) + 1 + elif name in inputs.keys(): + input_dtypes[dtype_str] = input_dtypes.get(dtype_str, 0) + 1 + + model_size_in_billion = model_size / 1e9 + num_params = sum(param_dtypes.values()) + num_inputs = sum(input_dtypes.values()) + num_ops = sum(ops_count_dict.values()) + source, heuristic_tag = read_graph_source_and_tag(model_path) + method = "to_static" + is_complete = ( + program_analyzer.is_complete if program_analyzer is not None else False + ) + + def dict_to_string(d): + kv_list = [f"{k}:{v}" for k, v in d.items()] + return " ".join(kv_list) + + def print_with_log_prompt(key, value): + print( + f"{log_prompt} [ModelStats.{key}] model_path:{model_path} {value}", + flush=True, + ) + + print_with_log_prompt("num_inputs", num_inputs) + print_with_log_prompt("num_params", num_params) + print_with_log_prompt("num_outputs", num_outputs) + print_with_log_prompt("num_ops", num_ops) + print_with_log_prompt("model_size", f"{model_size_in_billion}B") + print_with_log_prompt("input_dtypes", dict_to_string(input_dtypes)) + print_with_log_prompt("param_dtypes", dict_to_string(param_dtypes)) + print_with_log_prompt("op_dtypes", dict_to_string(op_dtypes)) + print_with_log_prompt("ops", dict_to_string(ops_count_dict)) + print_with_log_prompt("source", source) + print_with_log_prompt("heuristic_tag", heuristic_tag) + print_with_log_prompt("method", method) + print_with_log_prompt("is_complete", is_complete) + + +def main(args): + if args.model_path is not None: + assert os.path.isdir(args.model_path) + assert is_single_model_dir(args.model_path) + timestamp_sec = datetime.now().timestamp() + dt = datetime.fromtimestamp(timestamp_sec) + formatted_dt = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + print(f"[{formatted_dt}] Collect information for {args.model_path}") + collect_model_stats(args.model_path, args.log_prompt) + else: + graph_net_samples_path = ( + (graph_net.paddle.samples_util.get_default_samples_directory()) + if args.graph_net_samples_path is None + else args.graph_net_samples_path + ) + + previous_failed_model_pathes = [] + if args.previous_collect_result_path is not None: + with open(args.previous_collect_result_path, "r") as f: + for line in f.readlines(): + if "[ModelStats]" in line: + fields = line.strip().split() + model_path = fields[2].split(":")[-1] + is_complete = fields[-1].split(":")[-1] + if is_complete == "False": + previous_failed_model_pathes.append(model_path) + + i = 0 + for root, dirs, files in os.walk(graph_net_samples_path): + if is_single_model_dir(root) and ( + args.previous_collect_result_path is None + or root in previous_failed_model_pathes + ): + print(f"[{i}] Collect information for {root}") + cmd = [ + "python", + "-m", + "graph_net.torch.collect_stats", + f"--device={args.device}", + f"--model-path={root}", + f"--log-prompt={args.log_prompt}", + ] + result = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=600, + ) + print(result.stdout) + if result.returncode != 0: + print(result.stderr) + i += 1 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Collect stats for computation graph samples. return 0 if success" + ) + parser.add_argument( + "--device", + type=str, + required=False, + default="cuda", + help="Device for testing the compiler (e.g., 'cpu' or 'cuda')", + ) + parser.add_argument( + "--model-path", + type=str, + required=False, + default=None, + help="Computation graph sample directory. e.g '../../paddle_samples/PaddleX/ResNet18'", + ) + parser.add_argument( + "--graph-net-samples-path", + type=str, + required=False, + default=None, + help="GraphNet samples directory. e.g '../../paddle_samples'", + ) + parser.add_argument( + "--previous-collect-result-path", + type=str, + required=False, + default=None, + help="Previous collect result path, use to recollect the failed cases", + ) + parser.add_argument( + "--log-prompt", + type=str, + required=False, + default="graph-net-collect-stats-log", + help="Log prompt for stats log filtering.", + ) + args = parser.parse_args() + main(args=args) From 39b1806d0c028e45f5394d99be346a342ec6295d Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Fri, 19 Sep 2025 15:30:37 +0800 Subject: [PATCH 03/16] Refine some codes and fix support for VectorType. --- graph_net/paddle/collect_stats.py | 53 +++++++++++++++++++++++++------ graph_net/paddle/utils.py | 10 +++--- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/graph_net/paddle/collect_stats.py b/graph_net/paddle/collect_stats.py index 2894de4edd..c9e12f0b05 100644 --- a/graph_net/paddle/collect_stats.py +++ b/graph_net/paddle/collect_stats.py @@ -1,5 +1,6 @@ import argparse import os +import re import sys import ast import math @@ -118,6 +119,24 @@ def update_op_stats(self, op_name, op_dtype): ) self.op_stats[op_name].count += 1 + def parse_pir_value_dtypes(self, type_str): + short_form2dtype = { + "f32": "float32", + "f16": "float16", + "bf16": "bfloat16", + "i64": "int64", + } + # type_str: "vec[tensor<1x18x13x9xf32>,tensor<1x9x13x9xf32>]" + matches = re.findall(r"tensor<([^>]+)>", type_str) + dtype_strs = [] + for s in matches: + parts = s.split("x") + assert len(parts) > 0 + + dtype = parts[-1].lower() + dtype_strs.append(short_form2dtype[dtype]) + return dtype_strs + def __call__(self, program): assert isinstance(program, paddle.base.libpaddle.pir.Program) @@ -129,22 +148,38 @@ def __call__(self, program): op_name = None op_dtype = None if op.name() == "pd_op.data": + op_name = "data" op_attrs = op.attrs() op_dtype = op_attrs["dtype"] self.input_dict[op_attrs["name"]] = { "dtype": str(op_dtype).replace("paddle.", ""), "shape": op_attrs["shape"], } - elif not op.name().startswith("builtin."): + elif op.name().startswith("pd_op."): self.num_ops += 1 op_name = op.name().replace("pd_op.", "") - if len(op.results()) > 0: - op_dtype = op.results()[0].dtype - - if op_name is not None: - self.update_op_stats(op_name, op_dtype) - elif op_dtype is None: - self.num_ops_misses_dtypes += 1 + try: + if len(op.results()) > 0: + out = op.results()[0] + if out.is_dense_tensor_type(): + op_dtype = out.dtype + else: + # for paddle.base.libpaddle.pir.VectorType, but cannot be accurately determined + if op_name in ["split", "split_with_num", "meshgrid"]: + op_dtype = self.parse_pir_value_dtypes( + str(out.type()) + )[0] + else: + assert False, f"Unsupport op: {op}" + except Exception: + if self.num_ops_misses_dtypes == 0: + print(f"dtype inference failed for {op_name}") + if op_dtype is not None: + self.update_op_stats(op_name, op_dtype) + else: + self.num_ops_misses_dtypes += 1 + elif not op.name().startswith("builtin."): + assert False, f"Unrecognized op: {op}" if self.num_ops_misses_dtypes > 0: self.is_complete = False @@ -281,7 +316,7 @@ def main(args): cmd = [ "python", "-m", - "graph_net.torch.collect_stats", + "graph_net.paddle.collect_stats", f"--device={args.device}", f"--model-path={root}", f"--log-prompt={args.log_prompt}", diff --git a/graph_net/paddle/utils.py b/graph_net/paddle/utils.py index c9b73bc2f5..4dbe8b3392 100644 --- a/graph_net/paddle/utils.py +++ b/graph_net/paddle/utils.py @@ -122,7 +122,7 @@ def load_converted_list_from_text(file_path): return [*weight_info, *input_info] -def ConvertToValidNumber(data_type, value): +def convert_to_valid_number(data_type, value): if value is not None and data_type in [ paddle.float32, paddle.float16, @@ -160,10 +160,10 @@ def convert_meta_classes_to_tensors(file_path): "shape": attrs.get("shape", []), "dtype": data_type, "device": attrs.get("device", "gpu"), - "mean": ConvertToValidNumber(data_type, attrs.get("mean", None)), - "std": ConvertToValidNumber(data_type, attrs.get("std", None)), - "min_val": ConvertToValidNumber(data_type, attrs.get("min_val", 0)), - "max_val": ConvertToValidNumber(data_type, attrs.get("max_val", 2)), + "mean": convert_to_valid_number(data_type, attrs.get("mean", None)), + "std": convert_to_valid_number(data_type, attrs.get("std", None)), + "min_val": convert_to_valid_number(data_type, attrs.get("min_val", 0)), + "max_val": convert_to_valid_number(data_type, attrs.get("max_val", 2)), }, "data": data_value, "name": attrs.get("name"), From 6589e7a1d3ecb89a90487eb3552ae2269bbba1f9 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Sun, 21 Sep 2025 10:38:41 +0800 Subject: [PATCH 04/16] Clear codes. --- graph_net/paddle/test_compiler.py | 67 +++---------------------------- 1 file changed, 5 insertions(+), 62 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index f76f96cd5b..89a72c6705 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -50,21 +50,9 @@ def get_input_dict(args): params = inputs_params["weight_info"] inputs = inputs_params["input_info"] - param_dtypes = set() - for name, info in params.items(): - dtype = str(info["info"]["dtype"]) - if dtype not in param_dtypes: - param_dtypes.add(dtype) - - input_dtypes = set() - for name, info in inputs.items(): - dtype = str(info["info"]["dtype"]) - if dtype not in input_dtypes: - input_dtypes.add(dtype) - params.update(inputs) state_dict = {k: utils.replay_tensor(v) for k, v in params.items()} - return state_dict, list(input_dtypes), list(param_dtypes) + return state_dict def get_input_spec(args): @@ -90,30 +78,6 @@ def get_compiled_model(args, model): return compiled_model -def count_number_of_ops(args, model, eager_mode): - if eager_mode: - static_model = paddle.jit.to_static( - model, - input_spec=get_input_spec(args), - full_graph=True, - backend=None, - ) - static_model.eval() - program = static_model.forward.concrete_program.main_program - else: - program = model.forward.concrete_program.main_program - print(program) - - num_ops = 0 - for block in program.blocks: - for op in block.ops: - if op.name() != "pd_op.data" and not op.name().startswith("builtin."): - num_ops += 1 - print(f"Totally {num_ops} ops.") - print("") - return num_ops - - @dataclass class DurationBox: value: int @@ -176,7 +140,7 @@ def measure_performance(model_call, args, synchronizer_func): e2e_times.append(duration_box.value) gpu_times.append(gpu_time_ms) print( - f"Trial {i + 1}: e2e={duration_box.value:.4f} ms, gpu={gpu_time_ms:.5g} ms" + f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms" ) stats["e2e"] = get_timing_stats(e2e_times) @@ -221,7 +185,7 @@ def init_benchmark_result(args): return result_data -def check_outputs(args, expected_out, compiled_out, result_data): +def check_outputs(args, expected_out, compiled_out): if isinstance(expected_out, paddle.Tensor): expected_out = [expected_out] if isinstance(compiled_out, paddle.Tensor): @@ -231,15 +195,11 @@ def check_outputs(args, expected_out, compiled_out, result_data): for i, tensor in enumerate(expected_out): if tensor is not None: eager_output_dtypes[i] = str(tensor.dtype) - result_data.update_corrrectness("num_eager_outputs", len(expected_out)) - result_data.update_corrrectness("eager_output_dtypes", eager_output_dtypes) compiled_output_dtypes = [None] * len(compiled_out) for i, tensor in enumerate(compiled_out): if tensor is not None: compiled_output_dtypes[i] = str(tensor.dtype) - result_data.update_corrrectness("num_compiled_outputs", len(compiled_out)) - result_data.update_corrrectness("compiled_output_dtypes", compiled_output_dtypes) is_output_consistent = len(expected_out) == len(compiled_out) for a, b in zip(expected_out, compiled_out): @@ -247,7 +207,6 @@ def check_outputs(args, expected_out, compiled_out, result_data): is_output_consistent = False if a is not None and b is not None and a.dtype != b.dtype: is_output_consistent = False - result_data.update_corrrectness("output_consistent", is_output_consistent) def regular_outputs(origin_outputs): outputs = [] @@ -269,7 +228,6 @@ def print_cmp(key, func, **kwargs): cmp_ret = func(expected_out, compiled_out, **kwargs) except Exception as e: cmp_ret = f"{key} failed: {str(e)}\n{traceback.format_exc()}" - result_data.update_corrrectness(key, cmp_ret) print( f"{args.log_prompt} {key} model_path:{args.model_path} {cmp_ret}", file=sys.stderr, @@ -296,17 +254,10 @@ def print_cmp(key, func, **kwargs): def test_single_model(args): synchronizer_func = get_synchronizer_func(args) - input_dict, input_dtypes, param_dtypes = get_input_dict(args) + input_dict = get_input_dict(args) model = get_model(args) model.eval() - # Collect model information - num_eager_ops = count_number_of_ops(args, model, eager_mode=True) - - # Initialize benchmark result - result_data = init_benchmark_result(args) - result_data.update_model_info(num_eager_ops, input_dtypes, param_dtypes) - # Run on eager mode running_eager_success = False try: @@ -330,14 +281,9 @@ def test_single_model(args): except Exception as e: print(f"Run model in compiled mode failed: {str(e)}\n{traceback.format_exc()}") - print( - f"{args.log_prompt} information model_path:{args.model_path} {num_eager_ops} ops, param_dtypes:{param_dtypes}, input_dtypes:{input_dtypes}", - file=sys.stderr, - ) if running_eager_success and running_compiled_success: - check_outputs(args, expected_out, compiled_out, result_data) + check_outputs(args, expected_out, compiled_out) - result_data.update_performance(eager_time_stats, compiled_time_stats) duration_log = ( f"{args.log_prompt} [Duration] " f"eager_e2e:{result_data.eager_e2e_time_ms:.4f} ms compiled_e2e:{result_data.compiled_e2e_time_ms:.4f} ms" @@ -353,9 +299,6 @@ def test_single_model(args): print(duration_log, file=sys.stderr) print(speedup_log, file=sys.stderr) - if args.output_dir: - result_data.write_to_json(args.output_dir) - def get_cmp_equal(expected_out, compiled_out): return " ".join( From 946824477953ecbee0edac22d0c35c24ad175ac4 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Mon, 22 Sep 2025 16:31:55 +0800 Subject: [PATCH 05/16] Fix test_compiler of paddle and remove collecl_stats. --- graph_net/paddle/collect_stats.py | 377 ------------------------------ graph_net/paddle/test_compiler.py | 81 ++----- graph_net/path_utils.py | 22 ++ graph_net/test_compiler_utils.py | 74 ++++++ 4 files changed, 109 insertions(+), 445 deletions(-) delete mode 100644 graph_net/paddle/collect_stats.py create mode 100644 graph_net/path_utils.py create mode 100644 graph_net/test_compiler_utils.py diff --git a/graph_net/paddle/collect_stats.py b/graph_net/paddle/collect_stats.py deleted file mode 100644 index c9e12f0b05..0000000000 --- a/graph_net/paddle/collect_stats.py +++ /dev/null @@ -1,377 +0,0 @@ -import argparse -import os -import re -import sys -import ast -import math -import importlib -import inspect -import subprocess -from datetime import datetime -from typing import Type -from dataclasses import dataclass, field -from collections import defaultdict - -import paddle -from graph_net.paddle import utils - - -def is_single_model_dir(model_dir): - return os.path.isfile(f"{model_dir}/graph_net.json") - - -def load_class_from_file(file_path: str, class_name: str) -> Type[paddle.nn.Layer]: - spec = importlib.util.spec_from_file_location("unnamed", file_path) - unnamed = importlib.util.module_from_spec(spec) - spec.loader.exec_module(unnamed) - model_class = getattr(unnamed, class_name, None) - return model_class - - -def get_argument_name_and_types(model_class, func_name): - argument_name2types = {} - for name, func in inspect.getmembers(model_class, predicate=inspect.isfunction): - if name == func_name: - for arg_name, arg in inspect.signature(func).parameters.items(): - if arg_name != "self": - argument_name2types[arg_name] = ( - None if arg.annotation is inspect._empty else arg.annotation - ) - return argument_name2types - - -def get_number_of_returns(file_path, class_name, func_name): - source = None - with open(f"{file_path}", "r") as f: - source = f.read() - - tree = ast.parse(source) - for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == class_name: - for f in node.body: - if isinstance(f, ast.FunctionDef) and f.name == func_name: - for stmt in ast.walk(f): - if isinstance(stmt, ast.Return): - if stmt.value is None: - return 0 - elif isinstance(stmt.value, ast.Tuple): - return len(stmt.value.elts) - else: - return 1 - return 0 - - -def read_graph_source_and_tag(model_path): - try: - with open(os.path.join(model_path, "graph_net.json"), "r") as f: - data = json.load(f) - return data["source"], data["heuristic_tag"] - except Exception: - if "PaddleX" in model_path: - return "PaddleX", "computer_vision" - elif "PaddleNLP" in model_path: - return "PaddleNLP", "nlp" - elif "PaddleScience" in model_path: - return "PaddleScience", "scientific_computing" - else: - return "unknown", "unknown" - - -def get_input_spec(model_path): - inputs_params_list = utils.load_converted_list_from_text(f"{model_path}") - input_spec = [None] * len(inputs_params_list) - for i, v in enumerate(inputs_params_list): - dtype = v["info"]["dtype"] - shape = v["info"]["shape"] - input_spec[i] = paddle.static.InputSpec(shape, dtype) - return input_spec - - -@dataclass -class OpStat: - op_name: str - op_dtypes: dict[str, int] = field(default_factory=dict) - count: int = 0 - - def update(self, other): - if isinstance(other, OpStat) and self.op_name == other.op_name: - self.count += other.count - for name, count in other.op_dtypes.items(): - self.op_dtypes[name] = self.op_dtypes.get(name, 0) + count - - -class ProgramAnalyzer: - def __init__(self): - self.op_stats = {} - self.input_dict = {} - self.num_ops = 0 - self.num_ops_misses_dtypes = 0 - self.is_complete = True - - def update_op_stats(self, op_name, op_dtype): - if op_name is not None: - dtype_str = str(op_dtype).replace("paddle.", "") - if self.op_stats.get(op_name, None) is None: - self.op_stats[op_name] = OpStat(op_name, {dtype_str: 1}, 1) - else: - self.op_stats[op_name].op_dtypes[dtype_str] = ( - self.op_stats[op_name].op_dtypes.get(dtype_str, 0) + 1 - ) - self.op_stats[op_name].count += 1 - - def parse_pir_value_dtypes(self, type_str): - short_form2dtype = { - "f32": "float32", - "f16": "float16", - "bf16": "bfloat16", - "i64": "int64", - } - # type_str: "vec[tensor<1x18x13x9xf32>,tensor<1x9x13x9xf32>]" - matches = re.findall(r"tensor<([^>]+)>", type_str) - dtype_strs = [] - for s in matches: - parts = s.split("x") - assert len(parts) > 0 - - dtype = parts[-1].lower() - dtype_strs.append(short_form2dtype[dtype]) - return dtype_strs - - def __call__(self, program): - assert isinstance(program, paddle.base.libpaddle.pir.Program) - - self.op_stats = {} - self.num_ops_misses_dtypes = 0 - self.num_ops = 0 - for block in program.blocks: - for op in block.ops: - op_name = None - op_dtype = None - if op.name() == "pd_op.data": - op_name = "data" - op_attrs = op.attrs() - op_dtype = op_attrs["dtype"] - self.input_dict[op_attrs["name"]] = { - "dtype": str(op_dtype).replace("paddle.", ""), - "shape": op_attrs["shape"], - } - elif op.name().startswith("pd_op."): - self.num_ops += 1 - op_name = op.name().replace("pd_op.", "") - try: - if len(op.results()) > 0: - out = op.results()[0] - if out.is_dense_tensor_type(): - op_dtype = out.dtype - else: - # for paddle.base.libpaddle.pir.VectorType, but cannot be accurately determined - if op_name in ["split", "split_with_num", "meshgrid"]: - op_dtype = self.parse_pir_value_dtypes( - str(out.type()) - )[0] - else: - assert False, f"Unsupport op: {op}" - except Exception: - if self.num_ops_misses_dtypes == 0: - print(f"dtype inference failed for {op_name}") - if op_dtype is not None: - self.update_op_stats(op_name, op_dtype) - else: - self.num_ops_misses_dtypes += 1 - elif not op.name().startswith("builtin."): - assert False, f"Unrecognized op: {op}" - - if self.num_ops_misses_dtypes > 0: - self.is_complete = False - - def summary(self): - print( - f"Totally {self.num_ops} operators, and {self.num_ops_misses_dtypes} operators failed to inference dtypes." - ) - - -def collect_op_stats(model, model_path): - assert isinstance(model, paddle.nn.Layer), f"{type(model)=}" - try: - static_model = paddle.jit.to_static( - model, - input_spec=get_input_spec(model_path), - full_graph=True, - backend=None, - ) - static_model.eval() - program = static_model.forward.concrete_program.main_program - - program_analyzer = ProgramAnalyzer() - program_analyzer(program) - program_analyzer.summary() - return program_analyzer - except Exception: - print("Failed with to_static") - return None - - -def collect_model_stats(model_path, log_prompt): - file_path = os.path.join(model_path, "model.py") - model_class = load_class_from_file(file_path, "GraphModule") - model = model_class() - num_outputs = get_number_of_returns(file_path, "GraphModule", "forward") - - model_size = 0 - input_dtypes = {} - param_dtypes = {} - ops_count_dict = {} - op_dtypes = {} - - program_analyzer = collect_op_stats(model, model_path) - if program_analyzer is not None: - for op_name, stat in sorted(program_analyzer.op_stats.items()): - ops_count_dict[op_name] = stat.count - for dtype_str, num in stat.op_dtypes.items(): - if dtype_str is not None and dtype_str != "None": - op_dtypes[dtype_str] = op_dtypes.get(dtype_str, 0) + num - - inputs_params = utils.load_converted_from_text(f"{model_path}") - params = inputs_params["weight_info"] - inputs = inputs_params["input_info"] - - for name, value in program_analyzer.input_dict.items(): - dtype_str = value["dtype"] - if name in params.keys(): - param_numel = math.prod(value["shape"]) - model_size += param_numel - param_dtypes[dtype_str] = param_dtypes.get(dtype_str, 0) + 1 - elif name in inputs.keys(): - input_dtypes[dtype_str] = input_dtypes.get(dtype_str, 0) + 1 - - model_size_in_billion = model_size / 1e9 - num_params = sum(param_dtypes.values()) - num_inputs = sum(input_dtypes.values()) - num_ops = sum(ops_count_dict.values()) - source, heuristic_tag = read_graph_source_and_tag(model_path) - method = "to_static" - is_complete = ( - program_analyzer.is_complete if program_analyzer is not None else False - ) - - def dict_to_string(d): - kv_list = [f"{k}:{v}" for k, v in d.items()] - return " ".join(kv_list) - - def print_with_log_prompt(key, value): - print( - f"{log_prompt} [ModelStats.{key}] model_path:{model_path} {value}", - flush=True, - ) - - print_with_log_prompt("num_inputs", num_inputs) - print_with_log_prompt("num_params", num_params) - print_with_log_prompt("num_outputs", num_outputs) - print_with_log_prompt("num_ops", num_ops) - print_with_log_prompt("model_size", f"{model_size_in_billion}B") - print_with_log_prompt("input_dtypes", dict_to_string(input_dtypes)) - print_with_log_prompt("param_dtypes", dict_to_string(param_dtypes)) - print_with_log_prompt("op_dtypes", dict_to_string(op_dtypes)) - print_with_log_prompt("ops", dict_to_string(ops_count_dict)) - print_with_log_prompt("source", source) - print_with_log_prompt("heuristic_tag", heuristic_tag) - print_with_log_prompt("method", method) - print_with_log_prompt("is_complete", is_complete) - - -def main(args): - if args.model_path is not None: - assert os.path.isdir(args.model_path) - assert is_single_model_dir(args.model_path) - timestamp_sec = datetime.now().timestamp() - dt = datetime.fromtimestamp(timestamp_sec) - formatted_dt = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - print(f"[{formatted_dt}] Collect information for {args.model_path}") - collect_model_stats(args.model_path, args.log_prompt) - else: - graph_net_samples_path = ( - (graph_net.paddle.samples_util.get_default_samples_directory()) - if args.graph_net_samples_path is None - else args.graph_net_samples_path - ) - - previous_failed_model_pathes = [] - if args.previous_collect_result_path is not None: - with open(args.previous_collect_result_path, "r") as f: - for line in f.readlines(): - if "[ModelStats]" in line: - fields = line.strip().split() - model_path = fields[2].split(":")[-1] - is_complete = fields[-1].split(":")[-1] - if is_complete == "False": - previous_failed_model_pathes.append(model_path) - - i = 0 - for root, dirs, files in os.walk(graph_net_samples_path): - if is_single_model_dir(root) and ( - args.previous_collect_result_path is None - or root in previous_failed_model_pathes - ): - print(f"[{i}] Collect information for {root}") - cmd = [ - "python", - "-m", - "graph_net.paddle.collect_stats", - f"--device={args.device}", - f"--model-path={root}", - f"--log-prompt={args.log_prompt}", - ] - result = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=600, - ) - print(result.stdout) - if result.returncode != 0: - print(result.stderr) - i += 1 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Collect stats for computation graph samples. return 0 if success" - ) - parser.add_argument( - "--device", - type=str, - required=False, - default="cuda", - help="Device for testing the compiler (e.g., 'cpu' or 'cuda')", - ) - parser.add_argument( - "--model-path", - type=str, - required=False, - default=None, - help="Computation graph sample directory. e.g '../../paddle_samples/PaddleX/ResNet18'", - ) - parser.add_argument( - "--graph-net-samples-path", - type=str, - required=False, - default=None, - help="GraphNet samples directory. e.g '../../paddle_samples'", - ) - parser.add_argument( - "--previous-collect-result-path", - type=str, - required=False, - default=None, - help="Previous collect result path, use to recollect the failed cases", - ) - parser.add_argument( - "--log-prompt", - type=str, - required=False, - default="graph-net-collect-stats-log", - help="Log prompt for stats log filtering.", - ) - args = parser.parse_args() - main(args=args) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 89a72c6705..67c1b5c2c8 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -13,6 +13,8 @@ import traceback from graph_net.paddle import utils +from graph_net import path_utils +from graph_net import test_compiler_utils from graph_net.benchmark_result import BenchmarkResult @@ -78,31 +80,6 @@ def get_compiled_model(args, model): return compiled_model -@dataclass -class DurationBox: - value: int - - -@contextmanager -def naive_timer(duration_box, synchronizer_func): - synchronizer_func() - start = time.time() - yield - synchronizer_func() - end = time.time() - duration_box.value = (end - start) * 1000 # Store in milliseconds - - -def get_timing_stats(elapsed_times): - stats = { - "mean": float(f"{np.mean(elapsed_times):.6g}"), - "std": float(f"{np.std(elapsed_times):.6g}"), - "min": float(f"{np.min(elapsed_times):.6g}"), - "max": float(f"{np.max(elapsed_times):.6g}"), - } - return stats - - def measure_performance(model_call, args, synchronizer_func): stats = {} @@ -126,8 +103,8 @@ def measure_performance(model_call, args, synchronizer_func): for i in range(args.trials): # End-to-end timing (naive_timer) - duration_box = DurationBox(-1) - with naive_timer(duration_box, synchronizer_func): + duration_box = test_compiler_utils.DurationBox(-1) + with test_compiler_utils.naive_timer(duration_box, synchronizer_func): # GPU-only timing (CUDA Events) start_event = paddle.device.Event(enable_timing=True) end_event = paddle.device.Event(enable_timing=True) @@ -143,8 +120,8 @@ def measure_performance(model_call, args, synchronizer_func): f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms" ) - stats["e2e"] = get_timing_stats(e2e_times) - stats["gpu"] = get_timing_stats(gpu_times) + stats["e2e"] = test_compiler_utils.get_timing_stats(e2e_times) + stats["gpu"] = test_compiler_utils.get_timing_stats(gpu_times) else: # CPU or other devices hardware_name = platform.processor() print( @@ -153,12 +130,12 @@ def measure_performance(model_call, args, synchronizer_func): e2e_times = [] for i in range(args.trials): - duration_box = DurationBox(-1) - with naive_timer(duration_box, compiler.synchronize): + duration_box = test_compiler_utils.DurationBox(-1) + with test_compiler_utils.naive_timer(duration_box, synchronizer_func): outs = model_call() print(f"Trial {i + 1}: e2e={duration_box.value:.4f} ms") e2e_times.append(duration_box.value) - stats["e2e"] = get_timing_stats(e2e_times) + stats["e2e"] = test_compiler_utils.get_timing_stats(e2e_times) return outs, stats @@ -284,21 +261,10 @@ def test_single_model(args): if running_eager_success and running_compiled_success: check_outputs(args, expected_out, compiled_out) - duration_log = ( - f"{args.log_prompt} [Duration] " - f"eager_e2e:{result_data.eager_e2e_time_ms:.4f} ms compiled_e2e:{result_data.compiled_e2e_time_ms:.4f} ms" - ) - speedup_log = ( - f"{args.log_prompt} [Speedup] " f"e2e_speedup:{result_data.e2e_speedup:.4f}" + test_compiler_utils.print_times_and_speedup( + args, eager_time_stats, compiled_time_stats ) - if "cuda" in args.device: - duration_log += f" eager_gpu:{result_data.eager_gpu_time_ms:.4f} ms compiled_gpu:{result_data.compiled_gpu_time_ms:.4f} ms" - speedup_log += f" gpu_speedup:{result_data.gpu_speedup:.4f}" - - print(duration_log, file=sys.stderr) - print(speedup_log, file=sys.stderr) - def get_cmp_equal(expected_out, compiled_out): return " ".join( @@ -335,7 +301,7 @@ def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): def test_multi_models(args): - for model_path in get_recursively_model_path(args.model_path): + for model_path in path_utils.get_recursively_model_path(args.model_path): cmd = "".join( [ sys.executable, @@ -353,27 +319,6 @@ def test_multi_models(args): assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" -def get_recursively_model_path(root_dir): - for sub_dir in get_immediate_subdirectory_paths(root_dir): - if is_single_model_dir(sub_dir): - yield sub_dir - else: - yield from get_recursively_model_path(sub_dir) - - -def get_immediate_subdirectory_paths(parent_dir): - return [ - sub_dir - for name in os.listdir(parent_dir) - for sub_dir in [os.path.join(parent_dir, name)] - if os.path.isdir(sub_dir) - ] - - -def is_single_model_dir(model_dir): - return os.path.isfile(f"{model_dir}/graph_net.json") - - def main(args): assert os.path.isdir(args.model_path) assert args.compiler == "cinn" @@ -383,7 +328,7 @@ def main(args): random.seed(random_seed) np.random.seed(random_seed) - if is_single_model_dir(args.model_path): + if path_utils.is_single_model_dir(args.model_path): test_single_model(args) else: test_multi_models(args) diff --git a/graph_net/path_utils.py b/graph_net/path_utils.py new file mode 100644 index 0000000000..9047168cd0 --- /dev/null +++ b/graph_net/path_utils.py @@ -0,0 +1,22 @@ +import os + + +def is_single_model_dir(model_dir): + return os.path.isfile(f"{model_dir}/graph_net.json") + + +def get_recursively_model_path(root_dir): + for sub_dir in get_immediate_subdirectory_paths(root_dir): + if is_single_model_dir(sub_dir): + yield sub_dir + else: + yield from get_recursively_model_path(sub_dir) + + +def get_immediate_subdirectory_paths(parent_dir): + return [ + sub_dir + for name in os.listdir(parent_dir) + for sub_dir in [os.path.join(parent_dir, name)] + if os.path.isdir(sub_dir) + ] diff --git a/graph_net/test_compiler_utils.py b/graph_net/test_compiler_utils.py new file mode 100644 index 0000000000..4030608a30 --- /dev/null +++ b/graph_net/test_compiler_utils.py @@ -0,0 +1,74 @@ +import sys +import json +import time +import numpy as np +from dataclasses import dataclass +from contextlib import contextmanager + + +@dataclass +class DurationBox: + value: int + + +@contextmanager +def naive_timer(duration_box, synchronizer_func): + synchronizer_func() + start = time.time() + yield + synchronizer_func() + end = time.time() + duration_box.value = (end - start) * 1000 # Store in milliseconds + + +def get_timing_stats(elapsed_times): + stats = { + "mean": float(f"{np.mean(elapsed_times):.6g}"), + "std": float(f"{np.std(elapsed_times):.6g}"), + "min": float(f"{np.min(elapsed_times):.6g}"), + "max": float(f"{np.max(elapsed_times):.6g}"), + } + return stats + + +def print_times_and_speedup(args, eager_stats, compiled_stats): + print( + f"{args.log_prompt} [Performance][eager]: {json.dumps(eager_stats)}", + file=sys.stderr, + flush=True, + ) + print( + f"{args.log_prompt} [Performance][compiled]: {json.dumps(compiled_stats)}", + file=sys.stderr, + flush=True, + ) + + e2e_speedup = 0 + gpu_speedup = 0 + + eager_e2e_time_ms = eager_stats.get("e2e", {}).get("mean", 0) + compiled_e2e_time_ms = compiled_stats.get("e2e", {}).get("mean", 0) + + if eager_e2e_time_ms > 0 and compiled_e2e_time_ms > 0: + e2e_speedup = eager_e2e_time_ms / compiled_e2e_time_ms + + if "cuda" in args.device: + eager_gpu_time_ms = eager_stats.get("gpu", {}).get("mean", 0) + compiled_gpu_time_ms = compiled_stats.get("gpu", {}).get("mean", 0) + + if eager_gpu_time_ms > 0 and compiled_gpu_time_ms > 0: + gpu_speedup = eager_gpu_time_ms / compiled_gpu_time_ms + + if e2e_speedup > 0: + print( + f"{args.log_prompt} [Speedup][e2e]: {e2e_speedup:.5f}", + file=sys.stderr, + flush=True, + ) + + if "cuda" in args.device and gpu_speedup > 0: + print( + f"{args.log_prompt} [Speedup][gpu]: {gpu_speedup:.5f}", + file=sys.stderr, + flush=True, + ) From 00b554c9703a031c39d4c514f818dbb497fcf48a Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Tue, 23 Sep 2025 15:46:01 +0800 Subject: [PATCH 06/16] Rename test_compiler_util.py. --- graph_net/paddle/test_compiler.py | 18 +++++++++--------- ...compiler_utils.py => test_compiler_util.py} | 0 2 files changed, 9 insertions(+), 9 deletions(-) rename graph_net/{test_compiler_utils.py => test_compiler_util.py} (100%) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 67c1b5c2c8..af88266d0a 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -14,7 +14,7 @@ from graph_net.paddle import utils from graph_net import path_utils -from graph_net import test_compiler_utils +from graph_net import test_compiler_util from graph_net.benchmark_result import BenchmarkResult @@ -103,8 +103,8 @@ def measure_performance(model_call, args, synchronizer_func): for i in range(args.trials): # End-to-end timing (naive_timer) - duration_box = test_compiler_utils.DurationBox(-1) - with test_compiler_utils.naive_timer(duration_box, synchronizer_func): + duration_box = test_compiler_util.DurationBox(-1) + with test_compiler_util.naive_timer(duration_box, synchronizer_func): # GPU-only timing (CUDA Events) start_event = paddle.device.Event(enable_timing=True) end_event = paddle.device.Event(enable_timing=True) @@ -120,8 +120,8 @@ def measure_performance(model_call, args, synchronizer_func): f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms" ) - stats["e2e"] = test_compiler_utils.get_timing_stats(e2e_times) - stats["gpu"] = test_compiler_utils.get_timing_stats(gpu_times) + stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) + stats["gpu"] = test_compiler_util.get_timing_stats(gpu_times) else: # CPU or other devices hardware_name = platform.processor() print( @@ -130,12 +130,12 @@ def measure_performance(model_call, args, synchronizer_func): e2e_times = [] for i in range(args.trials): - duration_box = test_compiler_utils.DurationBox(-1) - with test_compiler_utils.naive_timer(duration_box, synchronizer_func): + duration_box = test_compiler_util.DurationBox(-1) + with test_compiler_util.naive_timer(duration_box, synchronizer_func): outs = model_call() print(f"Trial {i + 1}: e2e={duration_box.value:.4f} ms") e2e_times.append(duration_box.value) - stats["e2e"] = test_compiler_utils.get_timing_stats(e2e_times) + stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) return outs, stats @@ -261,7 +261,7 @@ def test_single_model(args): if running_eager_success and running_compiled_success: check_outputs(args, expected_out, compiled_out) - test_compiler_utils.print_times_and_speedup( + test_compiler_util.print_times_and_speedup( args, eager_time_stats, compiled_time_stats ) diff --git a/graph_net/test_compiler_utils.py b/graph_net/test_compiler_util.py similarity index 100% rename from graph_net/test_compiler_utils.py rename to graph_net/test_compiler_util.py From 4bfb4433fe4a284fa0a419dcaf37a91af528f549 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Tue, 23 Sep 2025 16:46:13 +0800 Subject: [PATCH 07/16] Reorganize codes and logs. --- graph_net/benchmark_result.py | 103 ----------------------- graph_net/paddle/test_compiler.py | 135 ++++++++++++++++-------------- graph_net/test_compiler_util.py | 135 ++++++++++++++++++++++++++---- 3 files changed, 188 insertions(+), 185 deletions(-) delete mode 100644 graph_net/benchmark_result.py diff --git a/graph_net/benchmark_result.py b/graph_net/benchmark_result.py deleted file mode 100644 index a7ffbf9ab8..0000000000 --- a/graph_net/benchmark_result.py +++ /dev/null @@ -1,103 +0,0 @@ -import os -import sys -import json -import re - - -class BenchmarkResult: - def __init__(self, args, framework, hardware, compile_framework_version): - self.configuration = { - "model_name": self.get_model_name(args), - "subgraph_tag": self.get_subgraph_tag(args), - "model_path": args.model_path, - "device": args.device, - "hardware": hardware, - "framework": framework, - "compiler": args.compiler, - "compile_framework_version": compile_framework_version, - "warmup": args.warmup, - "trials": args.trials, - } - self.model_info = { - "num_ops": -1, - "input_dtypes": None, - "param_dtypes": None, - } - self.correctness = {} - self.performance = { - "eager": None, - "compiled": None, - "speedup": {}, - } - - self.device = args.device - - self.eager_e2e_time_ms = -1 - self.compiled_e2e_time_ms = -1 - self.e2e_speedup = -1 - - self.eager_gpu_time_ms = -1 - self.compiled_gpu_time_ms = -1 - self.gpu_speedup = -1 - - def get_model_name(self, args): - model_name = None - with open(os.path.join(args.model_path, "graph_net.json"), "r") as f: - data = json.load(f) - model_name = data.get("model_name", None) - - if model_name is not None: - fields = args.model_path.split(os.sep) - pattern = rf"^subgraph(_\d+)?$" - model_name = fields[-2] if re.match(pattern, fields[-1]) else fields[-1] - return model_name - - def get_subgraph_tag(self, args): - fields = args.model_path.split(os.sep) - pattern = rf"^subgraph(_\d+)?$" - return fields[-1] if re.match(pattern, fields[-1]) else "" - - def update_model_info(self, num_ops, input_dtypes, param_dtypes): - self.model_info["num_ops"] = num_ops - self.model_info["input_dtypes"] = input_dtypes - self.model_info["param_dtypes"] = param_dtypes - - def update_corrrectness(self, key, cmp_ret): - self.correctness[key] = cmp_ret - - def update_performance(self, eager_stats, compiled_stats): - self.performance["eager"] = eager_stats - self.performance["compiled"] = compiled_stats - - self.eager_e2e_time_ms = eager_stats.get("e2e", {}).get("mean", -1) - self.compiled_e2e_time_ms = compiled_stats.get("e2e", {}).get("mean", -1) - if self.eager_e2e_time_ms > 0 and self.compiled_e2e_time_ms > 0: - self.e2e_speedup = self.eager_e2e_time_ms / self.compiled_e2e_time_ms - self.performance["speedup"]["e2e"] = float(f"{self.e2e_speedup:.6g}") - - if "cuda" in self.device: - self.eager_gpu_time_ms = eager_stats.get("gpu", {}).get("mean", -1) - self.compiled_gpu_time_ms = compiled_stats.get("gpu", {}).get("mean", -1) - if self.eager_gpu_time_ms > 0 and self.compiled_gpu_time_ms > 0: - self.gpu_speedup = self.eager_gpu_time_ms / self.compiled_gpu_time_ms - self.performance["speedup"]["gpu"] = float(f"{self.gpu_speedup:.6g}") - - def write_to_json(self, output_dir): - assert output_dir is not None - os.makedirs(output_dir, exist_ok=True) - result_data = { - "configuration": self.configuration, - "model_info": self.model_info, - "correctness": self.correctness, - "performance": self.performance, - } - model_name = self.configuration["model_name"] - subgraph_tag = self.configuration["subgraph_tag"] - compiler_name = self.configuration["compiler"] - file_path = os.path.join( - output_dir, f"{model_name}_{subgraph_tag}_{compiler_name}.json" - ) - with open(file_path, "w") as f: - json.dump(result_data, f, indent=4) - print(f"Result saved to {file_path}", file=sys.stderr) - print(result_data) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index af88266d0a..20607ad6d2 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -15,7 +15,24 @@ from graph_net.paddle import utils from graph_net import path_utils from graph_net import test_compiler_util -from graph_net.benchmark_result import BenchmarkResult + + +def get_hardward_name(args): + if args.device == "cuda": + hardware = paddle.device.cuda.get_device_name(0) + elif args.device == "cpu": + hardware = platform.processor() + else: + hardware = "unknown" + return hardware + + +def get_compile_framework_version(args): + if args.compiler == "cinn": + compile_framework_version = paddle.__version__ + else: + compile_framework_version = "unknown" + return compile_framework_version def load_class_from_file(file_path: str, class_name: str): @@ -88,15 +105,18 @@ def measure_performance(model_call, args, synchronizer_func): outs = model_call() synchronizer_func() + hardware_name = get_hardward_name(args) + print( + f"[Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}", + file=sys.stderr, + flush=True, + ) + if "cuda" in args.device: """ Acknowledgement: We evaluate the performance on both end-to-end and GPU-only timings, With reference to methods only based on CUDA events from KernelBench in https://github.com/ScalingIntelligence/KernelBench """ - hardware_name = paddle.device.cuda.get_device_name(0) - print( - f"{args.log_prompt} [Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}" - ) e2e_times = [] gpu_times = [] @@ -117,17 +137,14 @@ def measure_performance(model_call, args, synchronizer_func): e2e_times.append(duration_box.value) gpu_times.append(gpu_time_ms) print( - f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms" + f"Trial {i + 1}: e2e={duration_box.value:.5f} ms, gpu={gpu_time_ms:.5f} ms", + file=sys.stderr, + flush=True, ) stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) stats["gpu"] = test_compiler_util.get_timing_stats(gpu_times) else: # CPU or other devices - hardware_name = platform.processor() - print( - f"[Profiling] Using device: {args.device} {hardware_name}, warm up {args.warmup}, trials {args.trials}" - ) - e2e_times = [] for i in range(args.trials): duration_box = test_compiler_util.DurationBox(-1) @@ -140,50 +157,27 @@ def measure_performance(model_call, args, synchronizer_func): return outs, stats -def init_benchmark_result(args): - if args.device == "cuda": - hardware = paddle.device.cuda.get_device_name(0) - elif args.device == "cpu": - hardware = platform.processor() - else: - hardware = "unknown" - - if args.compiler == "cinn": - compile_framework_version = paddle.__version__ - else: - compile_framework_version = "unknown" - - result_data = BenchmarkResult( - args=args, - framework="PaddlePaddle", - hardware=hardware, - compile_framework_version=compile_framework_version, - ) - return result_data - - def check_outputs(args, expected_out, compiled_out): if isinstance(expected_out, paddle.Tensor): expected_out = [expected_out] if isinstance(compiled_out, paddle.Tensor): compiled_out = [compiled_out] - eager_output_dtypes = [None] * len(expected_out) + eager_dtypes = [None] * len(expected_out) for i, tensor in enumerate(expected_out): - if tensor is not None: - eager_output_dtypes[i] = str(tensor.dtype) + eager_dtypes[i] = ( + str(tensor.dtype).replace("paddle.", "") if tensor is not None else "none" + ) - compiled_output_dtypes = [None] * len(compiled_out) + compiled_dtypes = [None] * len(compiled_out) for i, tensor in enumerate(compiled_out): - if tensor is not None: - compiled_output_dtypes[i] = str(tensor.dtype) + compiled_dtypes[i] = ( + str(tensor.dtype).replace("paddle.", "") if tensor is not None else "none" + ) - is_output_consistent = len(expected_out) == len(compiled_out) - for a, b in zip(expected_out, compiled_out): - if (a is None and b is not None) or (a is not None and b is None): - is_output_consistent = False - if a is not None and b is not None and a.dtype != b.dtype: - is_output_consistent = False + type_match = test_compiler_util.check_output_datatype( + args, eager_dtypes, compiled_dtypes + ) def regular_outputs(origin_outputs): outputs = [] @@ -197,9 +191,6 @@ def regular_outputs(origin_outputs): outputs.append(item) return outputs - expected_out = regular_outputs(expected_out) - compiled_out = regular_outputs(compiled_out) - def print_cmp(key, func, **kwargs): try: cmp_ret = func(expected_out, compiled_out, **kwargs) @@ -210,23 +201,33 @@ def print_cmp(key, func, **kwargs): file=sys.stderr, ) - print( - f"{args.log_prompt} output_dtypes model_path:{args.model_path} eager:{eager_output_dtypes} compiled:{compiled_output_dtypes}", - file=sys.stderr, - ) - print_cmp("cmp.equal", get_cmp_equal) - print_cmp("cmp.all_close_atol8_rtol8", get_cmp_all_close, atol=1e-8, rtol=1e-8) - print_cmp("cmp.all_close_atol8_rtol5", get_cmp_all_close, atol=1e-8, rtol=1e-5) - print_cmp("cmp.all_close_atol5_rtol5", get_cmp_all_close, atol=1e-5, rtol=1e-5) - print_cmp("cmp.all_close_atol3_rtol2", get_cmp_all_close, atol=1e-3, rtol=1e-2) - print_cmp("cmp.all_close_atol2_rtol1", get_cmp_all_close, atol=1e-2, rtol=1e-1) - print_cmp("cmp.max_diff", get_cmp_max_diff) - print_cmp("cmp.mean_diff", get_cmp_mean_diff) - print_cmp("cmp.diff_count_atol8_rtol8", get_cmp_diff_count, atol=1e-8, rtol=1e-8) - print_cmp("cmp.diff_count_atol8_rtol5", get_cmp_diff_count, atol=1e-8, rtol=1e-5) - print_cmp("cmp.diff_count_atol5_rtol5", get_cmp_diff_count, atol=1e-5, rtol=1e-5) - print_cmp("cmp.diff_count_atol3_rtol2", get_cmp_diff_count, atol=1e-3, rtol=1e-2) - print_cmp("cmp.diff_count_atol2_rtol1", get_cmp_diff_count, atol=1e-2, rtol=1e-1) + if type_match: + expected_out = regular_outputs(expected_out) + compiled_out = regular_outputs(compiled_out) + + print_cmp("cmp.equal", get_cmp_equal) + print_cmp("cmp.all_close_atol8_rtol8", get_cmp_all_close, atol=1e-8, rtol=1e-8) + print_cmp("cmp.all_close_atol8_rtol5", get_cmp_all_close, atol=1e-8, rtol=1e-5) + print_cmp("cmp.all_close_atol5_rtol5", get_cmp_all_close, atol=1e-5, rtol=1e-5) + print_cmp("cmp.all_close_atol3_rtol2", get_cmp_all_close, atol=1e-3, rtol=1e-2) + print_cmp("cmp.all_close_atol2_rtol1", get_cmp_all_close, atol=1e-2, rtol=1e-1) + print_cmp("cmp.max_diff", get_cmp_max_diff) + print_cmp("cmp.mean_diff", get_cmp_mean_diff) + print_cmp( + "cmp.diff_count_atol8_rtol8", get_cmp_diff_count, atol=1e-8, rtol=1e-8 + ) + print_cmp( + "cmp.diff_count_atol8_rtol5", get_cmp_diff_count, atol=1e-8, rtol=1e-5 + ) + print_cmp( + "cmp.diff_count_atol5_rtol5", get_cmp_diff_count, atol=1e-5, rtol=1e-5 + ) + print_cmp( + "cmp.diff_count_atol3_rtol2", get_cmp_diff_count, atol=1e-3, rtol=1e-2 + ) + print_cmp( + "cmp.diff_count_atol2_rtol1", get_cmp_diff_count, atol=1e-2, rtol=1e-1 + ) def test_single_model(args): @@ -235,6 +236,10 @@ def test_single_model(args): model = get_model(args) model.eval() + test_compiler_util.print_basic_config( + args, get_hardward_name(args), get_compile_framework_version(args) + ) + # Run on eager mode running_eager_success = False try: diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index 4030608a30..a0659f07e3 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -1,3 +1,5 @@ +import os +import re import sys import json import time @@ -31,16 +33,54 @@ def get_timing_stats(elapsed_times): return stats +def get_model_name(model_path): + model_name = None + with open(os.path.join(model_path, "graph_net.json"), "r") as f: + data = json.load(f) + model_name = data.get("model_name", None) + + if model_name is not None: + fields = model_path.split(os.sep) + pattern = rf"^subgraph(_\d+)?$" + model_name = fields[-2] if re.match(pattern, fields[-1]) else fields[-1] + return model_name + + +def get_subgraph_tag(model_path): + fields = model_path.split(os.sep) + pattern = rf"^subgraph(_\d+)?$" + return fields[-1] if re.match(pattern, fields[-1]) else "" + + +def print_with_log_prompt(key, value, log_prompt): + print(f"{log_prompt} {key} {value}", file=sys.stderr, flush=True) + + +def print_basic_config(args, hardware_name, compile_framework_version): + model_path = os.path.normpath(args.model_path) + print_with_log_prompt("[Processing]", model_path, args.log_prompt) + + model_name = get_model_name(model_path) + print_with_log_prompt("[Config] model:", model_name, args.log_prompt) + + print_with_log_prompt("[Config] device:", args.device, args.log_prompt) + print_with_log_prompt("[Config] hardware:", hardware_name, args.log_prompt) + print_with_log_prompt("[Config] compiler:", args.compiler, args.log_prompt) + print_with_log_prompt("[Config] warmup:", args.warmup, args.log_prompt) + print_with_log_prompt("[Config] trials:", args.trials, args.log_prompt) + print_with_log_prompt( + "[Config] compile_framework_version:", + compile_framework_version, + args.log_prompt, + ) + + def print_times_and_speedup(args, eager_stats, compiled_stats): - print( - f"{args.log_prompt} [Performance][eager]: {json.dumps(eager_stats)}", - file=sys.stderr, - flush=True, + print_with_log_prompt( + "[Performance][eager]:", json.dumps(eager_stats), args.log_prompt ) - print( - f"{args.log_prompt} [Performance][compiled]: {json.dumps(compiled_stats)}", - file=sys.stderr, - flush=True, + print_with_log_prompt( + "[Performance][compiled]:", json.dumps(compiled_stats), args.log_prompt ) e2e_speedup = 0 @@ -60,15 +100,76 @@ def print_times_and_speedup(args, eager_stats, compiled_stats): gpu_speedup = eager_gpu_time_ms / compiled_gpu_time_ms if e2e_speedup > 0: - print( - f"{args.log_prompt} [Speedup][e2e]: {e2e_speedup:.5f}", - file=sys.stderr, - flush=True, - ) + print_with_log_prompt("[Speedup][e2e]:", f"{e2e_speedup:.5f}", args.log_prompt) if "cuda" in args.device and gpu_speedup > 0: - print( - f"{args.log_prompt} [Speedup][gpu]: {gpu_speedup:.5f}", - file=sys.stderr, - flush=True, + print_with_log_prompt("[Speedup][gpu]:", f"{gpu_speedup:.5f}", args.log_prompt) + + +def check_type_match(eager_dtypes, compiled_dtypes): + if len(eager_dtypes) != len(compiled_dtypes): + type_match = False + else: + type_match = all( + eager == compiled for eager, compiled in zip(eager_dtypes, compiled_dtypes) + ) + return type_match + + +def check_output_datatype(args, eager_dtypes, compiled_dtypes): + print_with_log_prompt("[Datatype][eager]:", " ".join(eager_dtypes), args.log_prompt) + print_with_log_prompt( + "[Datatype][compiled]:", " ".join(compiled_dtypes), args.log_prompt + ) + + # datatype check + type_match = check_type_match(eager_dtypes, compiled_dtypes) + print_with_log_prompt( + "[DataType]", + f"eager:{eager_dtypes} compiled:{compiled_dtypes} match:{type_match}", + args.log_prompt, + ) + return type_match + + +def print_and_store_cmp(key, cmp_func, args, expected_out, compiled_out, **kwargs): + cmp_ret = cmp_func(expected_out, compiled_out, **kwargs) + print_with_log_prompt(f"[Correctness]{key}:", cmp_ret, args.log_prompt) + return cmp_ret + + +def check_correctness( + args, + expected_out, + compiled_out, + cmp_equal_func, + cmp_all_close_func, + cmp_max_diff_func, + cmp_mean_diff_func, + cmp_diff_count_func, +): + cmp_configs = [ + ("[equal]", cmp_equal_func, {}), + ("[all_close_atol8_rtol8]", cmp_all_close_func, {"atol": 1e-8, "rtol": 1e-8}), + ("[all_close_atol8_rtol5]", cmp_all_close_func, {"atol": 1e-8, "rtol": 1e-5}), + ("[all_close_atol5_rtol5]", cmp_all_close_func, {"atol": 1e-5, "rtol": 1e-5}), + ("[all_close_atol3_rtol2]", cmp_all_close_func, {"atol": 1e-3, "rtol": 1e-2}), + ("[all_close_atol2_rtol1]", cmp_all_close_func, {"atol": 1e-2, "rtol": 1e-1}), + ("[max_diff]", cmp_max_diff_func, {}), + ("[mean_diff]", cmp_mean_diff_func, {}), + ("[diff_count_atol8_rtol8]", cmp_diff_count_func, {"atol": 1e-8, "rtol": 1e-8}), + ("[diff_count_atol8_rtol5]", cmp_diff_count_func, {"atol": 1e-8, "rtol": 1e-5}), + ("[diff_count_atol5_rtol5]", cmp_diff_count_func, {"atol": 1e-5, "rtol": 1e-5}), + ("[diff_count_atol3_rtol2]", cmp_diff_count_func, {"atol": 1e-3, "rtol": 1e-2}), + ("[diff_count_atol2_rtol1]", cmp_diff_count_func, {"atol": 1e-2, "rtol": 1e-1}), + ] + + for key, func, kwargs in cmp_configs: + print_and_store_cmp( + key=key, + cmp_func=func, + args=args, + expected_out=expected_out, + compiled_out=compiled_out, + **kwargs, ) From b0ba9c63663964c15fa71481449c8253e840bd47 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 24 Sep 2025 10:49:26 +0800 Subject: [PATCH 08/16] Reorganize codes. --- graph_net/paddle/test_compiler.py | 64 +++++++++++++++++++------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 20607ad6d2..bf78acad64 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -97,12 +97,37 @@ def get_compiled_model(args, model): return compiled_model +def count_number_of_ops(args, model, eager_mode): + if eager_mode: + static_model = paddle.jit.to_static( + model, + input_spec=get_input_spec(args), + full_graph=True, + backend=None, + ) + static_model.eval() + program = static_model.forward.concrete_program.main_program + else: + program = model.forward.concrete_program.main_program + print(program) + + num_ops = 0 + for block in program.blocks: + for op in block.ops: + if op.name() != "pd_op.data" and not op.name().startswith("builtin."): + num_ops += 1 + print(f"Totally {num_ops} ops.") + print("") + return num_ops + + def measure_performance(model_call, args, synchronizer_func): stats = {} # Warmup runs + outs = model_call() for _ in range(args.warmup): - outs = model_call() + model_call() synchronizer_func() hardware_name = get_hardward_name(args) @@ -130,7 +155,7 @@ def measure_performance(model_call, args, synchronizer_func): end_event = paddle.device.Event(enable_timing=True) start_event.record() - outs = model_call() + model_call() end_event.record() gpu_time_ms = start_event.elapsed_time(end_event) @@ -149,7 +174,7 @@ def measure_performance(model_call, args, synchronizer_func): for i in range(args.trials): duration_box = test_compiler_util.DurationBox(-1) with test_compiler_util.naive_timer(duration_box, synchronizer_func): - outs = model_call() + model_call() print(f"Trial {i + 1}: e2e={duration_box.value:.4f} ms") e2e_times.append(duration_box.value) stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) @@ -205,28 +230,15 @@ def print_cmp(key, func, **kwargs): expected_out = regular_outputs(expected_out) compiled_out = regular_outputs(compiled_out) - print_cmp("cmp.equal", get_cmp_equal) - print_cmp("cmp.all_close_atol8_rtol8", get_cmp_all_close, atol=1e-8, rtol=1e-8) - print_cmp("cmp.all_close_atol8_rtol5", get_cmp_all_close, atol=1e-8, rtol=1e-5) - print_cmp("cmp.all_close_atol5_rtol5", get_cmp_all_close, atol=1e-5, rtol=1e-5) - print_cmp("cmp.all_close_atol3_rtol2", get_cmp_all_close, atol=1e-3, rtol=1e-2) - print_cmp("cmp.all_close_atol2_rtol1", get_cmp_all_close, atol=1e-2, rtol=1e-1) - print_cmp("cmp.max_diff", get_cmp_max_diff) - print_cmp("cmp.mean_diff", get_cmp_mean_diff) - print_cmp( - "cmp.diff_count_atol8_rtol8", get_cmp_diff_count, atol=1e-8, rtol=1e-8 - ) - print_cmp( - "cmp.diff_count_atol8_rtol5", get_cmp_diff_count, atol=1e-8, rtol=1e-5 - ) - print_cmp( - "cmp.diff_count_atol5_rtol5", get_cmp_diff_count, atol=1e-5, rtol=1e-5 - ) - print_cmp( - "cmp.diff_count_atol3_rtol2", get_cmp_diff_count, atol=1e-3, rtol=1e-2 - ) - print_cmp( - "cmp.diff_count_atol2_rtol1", get_cmp_diff_count, atol=1e-2, rtol=1e-1 + test_compiler_util.check_correctness( + args, + expected_out, + compiled_out, + cmp_equal_func=get_cmp_equal, + cmp_all_close_func=get_cmp_all_close, + cmp_max_diff_func=get_cmp_max_diff, + cmp_mean_diff_func=get_cmp_mean_diff, + cmp_diff_count_func=get_cmp_diff_count, ) @@ -236,6 +248,8 @@ def test_single_model(args): model = get_model(args) model.eval() + num_eager_ops = count_number_of_ops(args, model, eager_mode=True) + test_compiler_util.print_basic_config( args, get_hardward_name(args), get_compile_framework_version(args) ) From 371562b7f717f7e77c2df98832aef06cc9730834 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 24 Sep 2025 15:51:50 +0800 Subject: [PATCH 09/16] Fix seed of dropout. --- graph_net/paddle/test_compiler.py | 84 ++++++++++++++----------------- graph_net/test_compiler_util.py | 11 ++++ 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index bf78acad64..83c7403893 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -17,6 +17,12 @@ from graph_net import test_compiler_util +def set_seed(random_seed): + paddle.seed(random_seed) + random.seed(random_seed) + np.random.seed(random_seed) + + def get_hardward_name(args): if args.device == "cuda": hardware = paddle.device.cuda.get_device_name(0) @@ -94,38 +100,30 @@ def get_compiled_model(args, model): full_graph=True, ) compiled_model.eval() + program = compiled_model.forward.concrete_program.main_program return compiled_model -def count_number_of_ops(args, model, eager_mode): - if eager_mode: - static_model = paddle.jit.to_static( - model, - input_spec=get_input_spec(args), - full_graph=True, - backend=None, - ) - static_model.eval() - program = static_model.forward.concrete_program.main_program - else: - program = model.forward.concrete_program.main_program - print(program) - - num_ops = 0 - for block in program.blocks: - for op in block.ops: - if op.name() != "pd_op.data" and not op.name().startswith("builtin."): - num_ops += 1 - print(f"Totally {num_ops} ops.") - print("") - return num_ops +def get_static_model(args, model): + static_model = paddle.jit.to_static( + model, + input_spec=get_input_spec(args), + full_graph=True, + backend=None, + ) + static_model.eval() + program = static_model.forward.concrete_program.main_program + return static_model -def measure_performance(model_call, args, synchronizer_func): +def measure_performance(model_call, args, synchronizer_func, profile=False): + runtime_seed = 1024 stats = {} - # Warmup runs + paddle.seed(runtime_seed) outs = model_call() + + # Warmup runs for _ in range(args.warmup): model_call() synchronizer_func() @@ -146,6 +144,8 @@ def measure_performance(model_call, args, synchronizer_func): e2e_times = [] gpu_times = [] + if profile: + paddle.base.core.nvprof_start() for i in range(args.trials): # End-to-end timing (naive_timer) duration_box = test_compiler_util.DurationBox(-1) @@ -166,6 +166,8 @@ def measure_performance(model_call, args, synchronizer_func): file=sys.stderr, flush=True, ) + if profile: + paddle.base.core.nvprof_stop() stats["e2e"] = test_compiler_util.get_timing_stats(e2e_times) stats["gpu"] = test_compiler_util.get_timing_stats(gpu_times) @@ -216,16 +218,6 @@ def regular_outputs(origin_outputs): outputs.append(item) return outputs - def print_cmp(key, func, **kwargs): - try: - cmp_ret = func(expected_out, compiled_out, **kwargs) - except Exception as e: - cmp_ret = f"{key} failed: {str(e)}\n{traceback.format_exc()}" - print( - f"{args.log_prompt} {key} model_path:{args.model_path} {cmp_ret}", - file=sys.stderr, - ) - if type_match: expected_out = regular_outputs(expected_out) compiled_out = regular_outputs(compiled_out) @@ -248,36 +240,38 @@ def test_single_model(args): model = get_model(args) model.eval() - num_eager_ops = count_number_of_ops(args, model, eager_mode=True) + # num_eager_ops = count_number_of_ops(args, model, eager_mode=True) test_compiler_util.print_basic_config( args, get_hardward_name(args), get_compile_framework_version(args) ) # Run on eager mode - running_eager_success = False + eager_success = False try: print("Run model in eager mode.") + static_model = get_static_model(args, model) expected_out, eager_time_stats = measure_performance( - lambda: model(**input_dict), args, synchronizer_func + lambda: static_model(**input_dict), args, synchronizer_func, profile=False ) - running_eager_success = True + eager_success = True except Exception as e: print(f"Run model in eager mode failed: {str(e)}\n{traceback.format_exc()}") # Run on compiling mode - running_compiled_success = False + compiled_success = False try: print("Run model in compiled mode.") compiled_model = get_compiled_model(args, model) compiled_out, compiled_time_stats = measure_performance( - lambda: compiled_model(**input_dict), args, synchronizer_func + lambda: compiled_model(**input_dict), args, synchronizer_func, profile=False ) - running_compiled_success = True + compiled_success = True except Exception as e: print(f"Run model in compiled mode failed: {str(e)}\n{traceback.format_exc()}") - if running_eager_success and running_compiled_success: + test_compiler_util.print_running_status(args, eager_success, compiled_success) + if eager_success and compiled_success: check_outputs(args, expected_out, compiled_out) test_compiler_util.print_times_and_speedup( @@ -342,10 +336,8 @@ def main(args): assert os.path.isdir(args.model_path) assert args.compiler == "cinn" - random_seed = 123 - paddle.seed(random_seed) - random.seed(random_seed) - np.random.seed(random_seed) + initalize_seed = 123 + set_seed(random_seed=initalize_seed) if path_utils.is_single_model_dir(args.model_path): test_single_model(args) diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index a0659f07e3..cb42cf83d9 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -75,6 +75,17 @@ def print_basic_config(args, hardware_name, compile_framework_version): ) +def print_running_status(args, eager_success, compiled_success): + def convert_to_str(b): + return "success" if b else "failed" + + print_with_log_prompt( + "[Result][status]", + f"eager:{convert_to_str(eager_success)} compiled:{convert_to_str(compiled_success)}", + args.log_prompt, + ) + + def print_times_and_speedup(args, eager_stats, compiled_stats): print_with_log_prompt( "[Performance][eager]:", json.dumps(eager_stats), args.log_prompt From 07e43404d4cc9aaf25fa4c17701a3bad17c3f471 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Wed, 24 Sep 2025 21:00:15 +0800 Subject: [PATCH 10/16] Apply the new tolerance. --- graph_net/test_compiler_util.py | 89 +++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index cb42cf83d9..a03a73ae35 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -161,18 +161,87 @@ def check_correctness( ): cmp_configs = [ ("[equal]", cmp_equal_func, {}), - ("[all_close_atol8_rtol8]", cmp_all_close_func, {"atol": 1e-8, "rtol": 1e-8}), - ("[all_close_atol8_rtol5]", cmp_all_close_func, {"atol": 1e-8, "rtol": 1e-5}), - ("[all_close_atol5_rtol5]", cmp_all_close_func, {"atol": 1e-5, "rtol": 1e-5}), - ("[all_close_atol3_rtol2]", cmp_all_close_func, {"atol": 1e-3, "rtol": 1e-2}), - ("[all_close_atol2_rtol1]", cmp_all_close_func, {"atol": 1e-2, "rtol": 1e-1}), + ("[all_close_atol_1.00E-06_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.00E-06, "rtol": 1.00E-10}), + ("[all_close_atol_2.56E-04_rtol_1.00E-10]", cmp_all_close_func, {"atol": 2.56E-04, "rtol": 1.00E-10}), + ("[all_close_atol_1.69E-12_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.69E-12, "rtol": 1.00E-10}), + ("[all_close_atol_1.00E-14_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.00E-14, "rtol": 1.00E-10}), + + ("[all_close_atol_3.98E-06_rtol_1.00E-09]", cmp_all_close_func, {"atol": 3.98E-06, "rtol": 1.00E-09}), + ("[all_close_atol_5.85E-04_rtol_1.00E-09]", cmp_all_close_func, {"atol": 5.85E-04, "rtol": 1.00E-09}), + ("[all_close_atol_2.54E-11_rtol_1.00E-09]", cmp_all_close_func, {"atol": 2.54E-11, "rtol": 1.00E-09}), + ("[all_close_atol_2.51E-13_rtol_1.00E-09]", cmp_all_close_func, {"atol": 2.51E-13, "rtol": 1.00E-09}), + + ("[all_close_atol_1.58E-05_rtol_1.00E-08]", cmp_all_close_func, {"atol": 1.58E-05, "rtol": 1.00E-08}), + ("[all_close_atol_1.34E-03_rtol_1.00E-08]", cmp_all_close_func, {"atol": 1.34E-03, "rtol": 1.00E-08}), + ("[all_close_atol_3.82E-10_rtol_1.00E-08]", cmp_all_close_func, {"atol": 3.82E-10, "rtol": 1.00E-08}), + ("[all_close_atol_6.31E-12_rtol_1.00E-08]", cmp_all_close_func, {"atol": 6.31E-12, "rtol": 1.00E-08}), + + ("[all_close_atol_6.31E-05_rtol_1.00E-07]", cmp_all_close_func, {"atol": 6.31E-05, "rtol": 1.00E-07}), + ("[all_close_atol_3.06E-03_rtol_1.00E-07]", cmp_all_close_func, {"atol": 3.06E-03, "rtol": 1.00E-07}), + ("[all_close_atol_5.75E-09_rtol_1.00E-07]", cmp_all_close_func, {"atol": 5.75E-09, "rtol": 1.00E-07}), + ("[all_close_atol_1.58E-10_rtol_1.00E-07]", cmp_all_close_func, {"atol": 1.58E-10, "rtol": 1.00E-07}), + + ("[all_close_atol_2.51E-04_rtol_1.00E-06]", cmp_all_close_func, {"atol": 2.51E-04, "rtol": 1.00E-06}), + ("[all_close_atol_7.00E-03_rtol_1.00E-06]", cmp_all_close_func, {"atol": 7.00E-03, "rtol": 1.00E-06}), + ("[all_close_atol_8.65E-08_rtol_1.00E-06]", cmp_all_close_func, {"atol": 8.65E-08, "rtol": 1.00E-06}), + ("[all_close_atol_3.98E-09_rtol_1.00E-06]", cmp_all_close_func, {"atol": 3.98E-09, "rtol": 1.00E-06}), + + ("[all_close_atol_1.00E-03_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.00E-03, "rtol": 1.00E-05}), + ("[all_close_atol_1.60E-02_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.60E-02, "rtol": 1.00E-05}), + ("[all_close_atol_1.30E-06_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.30E-06, "rtol": 1.00E-05}), + ("[all_close_atol_1.00E-07_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.00E-07, "rtol": 1.00E-05}), + + ("[all_close_atol_3.98E-03_rtol_1.00E-04]", cmp_all_close_func, {"atol": 3.98E-03, "rtol": 1.00E-04}), + ("[all_close_atol_3.66E-02_rtol_1.00E-04]", cmp_all_close_func, {"atol": 3.66E-02, "rtol": 1.00E-04}), + ("[all_close_atol_1.96E-05_rtol_1.00E-04]", cmp_all_close_func, {"atol": 1.96E-05, "rtol": 1.00E-04}), + ("[all_close_atol_2.51E-06_rtol_1.00E-04]", cmp_all_close_func, {"atol": 2.51E-06, "rtol": 1.00E-04}), + + ("[all_close_atol_1.58E-02_rtol_1.00E-03]", cmp_all_close_func, {"atol": 1.58E-02, "rtol": 1.00E-03}), + ("[all_close_atol_8.36E-02_rtol_1.00E-03]", cmp_all_close_func, {"atol": 8.36E-02, "rtol": 1.00E-03}), + ("[all_close_atol_2.94E-04_rtol_1.00E-03]", cmp_all_close_func, {"atol": 2.94E-04, "rtol": 1.00E-03}), + ("[all_close_atol_6.31E-05_rtol_1.00E-03]", cmp_all_close_func, {"atol": 6.31E-05, "rtol": 1.00E-03}), + + ("[all_close_atol_6.31E-02_rtol_1.00E-02]", cmp_all_close_func, {"atol": 6.31E-02, "rtol": 1.00E-02}), + ("[all_close_atol_1.91E-01_rtol_1.00E-02]", cmp_all_close_func, {"atol": 1.91E-01, "rtol": 1.00E-02}), + ("[all_close_atol_4.42E-03_rtol_1.00E-02]", cmp_all_close_func, {"atol": 4.42E-03, "rtol": 1.00E-02}), + ("[all_close_atol_1.58E-03_rtol_1.00E-02]", cmp_all_close_func, {"atol": 1.58E-03, "rtol": 1.00E-02}), + + ("[all_close_atol_2.51E-01_rtol_1.00E-01]", cmp_all_close_func, {"atol": 2.51E-01, "rtol": 1.00E-01}), + ("[all_close_atol_4.37E-01_rtol_1.00E-01]", cmp_all_close_func, {"atol": 4.37E-01, "rtol": 1.00E-01}), + ("[all_close_atol_6.65E-02_rtol_1.00E-01]", cmp_all_close_func, {"atol": 6.65E-02, "rtol": 1.00E-01}), + ("[all_close_atol_3.98E-02_rtol_1.00E-01]", cmp_all_close_func, {"atol": 3.98E-02, "rtol": 1.00E-01}), + + ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), + ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), + ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), + ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), + + ("[all_close_atol_3.98E+00_rtol_1.00E+01]", cmp_all_close_func, {"atol": 3.98E+00, "rtol": 1.00E+01}), + ("[all_close_atol_2.29E+00_rtol_1.00E+01]", cmp_all_close_func, {"atol": 2.29E+00, "rtol": 1.00E+01}), + ("[all_close_atol_1.50E+01_rtol_1.00E+01]", cmp_all_close_func, {"atol": 1.50E+01, "rtol": 1.00E+01}), + ("[all_close_atol_2.51E+01_rtol_1.00E+01]", cmp_all_close_func, {"atol": 2.51E+01, "rtol": 1.00E+01}), + + ("[all_close_atol_1.58E+01_rtol_1.00E+02]", cmp_all_close_func, {"atol": 1.58E+01, "rtol": 1.00E+02}), + ("[all_close_atol_5.23E+00_rtol_1.00E+02]", cmp_all_close_func, {"atol": 5.23E+00, "rtol": 1.00E+02}), + ("[all_close_atol_2.26E+02_rtol_1.00E+02]", cmp_all_close_func, {"atol": 2.26E+02, "rtol": 1.00E+02}), + ("[all_close_atol_6.31E+02_rtol_1.00E+02]", cmp_all_close_func, {"atol": 6.31E+02, "rtol": 1.00E+02}), + + ("[all_close_atol_6.31E+01_rtol_1.00E+03]", cmp_all_close_func, {"atol": 6.31E+01, "rtol": 1.00E+03}), + ("[all_close_atol_1.20E+01_rtol_1.00E+03]", cmp_all_close_func, {"atol": 1.20E+01, "rtol": 1.00E+03}), + ("[all_close_atol_3.40E+03_rtol_1.00E+03]", cmp_all_close_func, {"atol": 3.40E+03, "rtol": 1.00E+03}), + ("[all_close_atol_1.58E+04_rtol_1.00E+03]", cmp_all_close_func, {"atol": 1.58E+04, "rtol": 1.00E+03}), + + ("[all_close_atol_2.51E+02_rtol_1.00E+04]", cmp_all_close_func, {"atol": 2.51E+02, "rtol": 1.00E+04}), + ("[all_close_atol_2.73E+01_rtol_1.00E+04]", cmp_all_close_func, {"atol": 2.73E+01, "rtol": 1.00E+04}), + ("[all_close_atol_5.11E+04_rtol_1.00E+04]", cmp_all_close_func, {"atol": 5.11E+04, "rtol": 1.00E+04}), + ("[all_close_atol_3.98E+05_rtol_1.00E+04]", cmp_all_close_func, {"atol": 3.98E+05, "rtol": 1.00E+04}), + + ("[all_close_atol_1.00E+03_rtol_1.00E+05]", cmp_all_close_func, {"atol": 1.00E+03, "rtol": 1.00E+05}), + ("[all_close_atol_6.25E+01_rtol_1.00E+05]", cmp_all_close_func, {"atol": 6.25E+01, "rtol": 1.00E+05}), + ("[all_close_atol_7.69E+05_rtol_1.00E+05]", cmp_all_close_func, {"atol": 7.69E+05, "rtol": 1.00E+05}), + ("[all_close_atol_1.00E+07_rtol_1.00E+05]", cmp_all_close_func, {"atol": 1.00E+07, "rtol": 1.00E+05}), ("[max_diff]", cmp_max_diff_func, {}), ("[mean_diff]", cmp_mean_diff_func, {}), - ("[diff_count_atol8_rtol8]", cmp_diff_count_func, {"atol": 1e-8, "rtol": 1e-8}), - ("[diff_count_atol8_rtol5]", cmp_diff_count_func, {"atol": 1e-8, "rtol": 1e-5}), - ("[diff_count_atol5_rtol5]", cmp_diff_count_func, {"atol": 1e-5, "rtol": 1e-5}), - ("[diff_count_atol3_rtol2]", cmp_diff_count_func, {"atol": 1e-3, "rtol": 1e-2}), - ("[diff_count_atol2_rtol1]", cmp_diff_count_func, {"atol": 1e-2, "rtol": 1e-1}), ] for key, func, kwargs in cmp_configs: From f65c53ef80e1d8fb49983c8b6e7e2ec7f923af78 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 25 Sep 2025 10:04:05 +0800 Subject: [PATCH 11/16] Fix batched testing. --- graph_net/paddle/test_compiler.py | 38 +-- graph_net/test_compiler_util.py | 413 ++++++++++++++++++++++++------ 2 files changed, 353 insertions(+), 98 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 83c7403893..81dcbd127c 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -206,7 +206,7 @@ def check_outputs(args, expected_out, compiled_out): args, eager_dtypes, compiled_dtypes ) - def regular_outputs(origin_outputs): + def transfer_to_float(origin_outputs): outputs = [] for item in origin_outputs: if ( @@ -219,14 +219,19 @@ def regular_outputs(origin_outputs): return outputs if type_match: - expected_out = regular_outputs(expected_out) - compiled_out = regular_outputs(compiled_out) - - test_compiler_util.check_correctness( + test_compiler_util.check_equal( args, expected_out, compiled_out, cmp_equal_func=get_cmp_equal, + ) + + expected_out_fp32 = transfer_to_float(expected_out) + compiled_out_fp32 = transfer_to_float(compiled_out) + test_compiler_util.check_allclose( + args, + expected_out_fp32, + compiled_out_fp32, cmp_all_close_func=get_cmp_all_close, cmp_max_diff_func=get_cmp_max_diff, cmp_mean_diff_func=get_cmp_mean_diff, @@ -240,8 +245,6 @@ def test_single_model(args): model = get_model(args) model.eval() - # num_eager_ops = count_number_of_ops(args, model, eager_mode=True) - test_compiler_util.print_basic_config( args, get_hardward_name(args), get_compile_framework_version(args) ) @@ -314,8 +317,11 @@ def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): def test_multi_models(args): + sample_idx = 0 + failed_samples = [] for model_path in path_utils.get_recursively_model_path(args.model_path): - cmd = "".join( + print(f"[{sample_idx}] test_compiler, model_path: {model_path}") + cmd = " ".join( [ sys.executable, "-m graph_net.paddle.test_compiler", @@ -329,7 +335,14 @@ def test_multi_models(args): ] ) cmd_ret = os.system(cmd) - assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" + # assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" + if cmd_ret != 0: + failed_samples.append(model_path) + sample_idx += 1 + + print(f"Totally {sample_idx} samples, failed {len(failed_samples)} samples.") + for model_path in failed_samples: + print(f"- {model_path}") def main(args): @@ -380,12 +393,5 @@ def main(args): default="graph-net-test-compiler-log", help="Log prompt for performance log filtering.", ) - parser.add_argument( - "--output-dir", - type=str, - required=False, - default=None, - help="Directory to save the structured JSON result file.", - ) args = parser.parse_args() main(args=args) diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index a03a73ae35..66887d06a6 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -149,97 +149,346 @@ def print_and_store_cmp(key, cmp_func, args, expected_out, compiled_out, **kwarg return cmp_ret -def check_correctness( +def check_equal(args, expected_out, compiled_out, cmp_equal_func): + print_and_store_cmp( + key="[equal]", + cmp_func=cmp_equal_func, + args=args, + expected_out=expected_out, + compiled_out=compiled_out, + ) + + +def check_allclose( args, expected_out, compiled_out, - cmp_equal_func, cmp_all_close_func, cmp_max_diff_func, cmp_mean_diff_func, cmp_diff_count_func, ): cmp_configs = [ - ("[equal]", cmp_equal_func, {}), - ("[all_close_atol_1.00E-06_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.00E-06, "rtol": 1.00E-10}), - ("[all_close_atol_2.56E-04_rtol_1.00E-10]", cmp_all_close_func, {"atol": 2.56E-04, "rtol": 1.00E-10}), - ("[all_close_atol_1.69E-12_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.69E-12, "rtol": 1.00E-10}), - ("[all_close_atol_1.00E-14_rtol_1.00E-10]", cmp_all_close_func, {"atol": 1.00E-14, "rtol": 1.00E-10}), - - ("[all_close_atol_3.98E-06_rtol_1.00E-09]", cmp_all_close_func, {"atol": 3.98E-06, "rtol": 1.00E-09}), - ("[all_close_atol_5.85E-04_rtol_1.00E-09]", cmp_all_close_func, {"atol": 5.85E-04, "rtol": 1.00E-09}), - ("[all_close_atol_2.54E-11_rtol_1.00E-09]", cmp_all_close_func, {"atol": 2.54E-11, "rtol": 1.00E-09}), - ("[all_close_atol_2.51E-13_rtol_1.00E-09]", cmp_all_close_func, {"atol": 2.51E-13, "rtol": 1.00E-09}), - - ("[all_close_atol_1.58E-05_rtol_1.00E-08]", cmp_all_close_func, {"atol": 1.58E-05, "rtol": 1.00E-08}), - ("[all_close_atol_1.34E-03_rtol_1.00E-08]", cmp_all_close_func, {"atol": 1.34E-03, "rtol": 1.00E-08}), - ("[all_close_atol_3.82E-10_rtol_1.00E-08]", cmp_all_close_func, {"atol": 3.82E-10, "rtol": 1.00E-08}), - ("[all_close_atol_6.31E-12_rtol_1.00E-08]", cmp_all_close_func, {"atol": 6.31E-12, "rtol": 1.00E-08}), - - ("[all_close_atol_6.31E-05_rtol_1.00E-07]", cmp_all_close_func, {"atol": 6.31E-05, "rtol": 1.00E-07}), - ("[all_close_atol_3.06E-03_rtol_1.00E-07]", cmp_all_close_func, {"atol": 3.06E-03, "rtol": 1.00E-07}), - ("[all_close_atol_5.75E-09_rtol_1.00E-07]", cmp_all_close_func, {"atol": 5.75E-09, "rtol": 1.00E-07}), - ("[all_close_atol_1.58E-10_rtol_1.00E-07]", cmp_all_close_func, {"atol": 1.58E-10, "rtol": 1.00E-07}), - - ("[all_close_atol_2.51E-04_rtol_1.00E-06]", cmp_all_close_func, {"atol": 2.51E-04, "rtol": 1.00E-06}), - ("[all_close_atol_7.00E-03_rtol_1.00E-06]", cmp_all_close_func, {"atol": 7.00E-03, "rtol": 1.00E-06}), - ("[all_close_atol_8.65E-08_rtol_1.00E-06]", cmp_all_close_func, {"atol": 8.65E-08, "rtol": 1.00E-06}), - ("[all_close_atol_3.98E-09_rtol_1.00E-06]", cmp_all_close_func, {"atol": 3.98E-09, "rtol": 1.00E-06}), - - ("[all_close_atol_1.00E-03_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.00E-03, "rtol": 1.00E-05}), - ("[all_close_atol_1.60E-02_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.60E-02, "rtol": 1.00E-05}), - ("[all_close_atol_1.30E-06_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.30E-06, "rtol": 1.00E-05}), - ("[all_close_atol_1.00E-07_rtol_1.00E-05]", cmp_all_close_func, {"atol": 1.00E-07, "rtol": 1.00E-05}), - - ("[all_close_atol_3.98E-03_rtol_1.00E-04]", cmp_all_close_func, {"atol": 3.98E-03, "rtol": 1.00E-04}), - ("[all_close_atol_3.66E-02_rtol_1.00E-04]", cmp_all_close_func, {"atol": 3.66E-02, "rtol": 1.00E-04}), - ("[all_close_atol_1.96E-05_rtol_1.00E-04]", cmp_all_close_func, {"atol": 1.96E-05, "rtol": 1.00E-04}), - ("[all_close_atol_2.51E-06_rtol_1.00E-04]", cmp_all_close_func, {"atol": 2.51E-06, "rtol": 1.00E-04}), - - ("[all_close_atol_1.58E-02_rtol_1.00E-03]", cmp_all_close_func, {"atol": 1.58E-02, "rtol": 1.00E-03}), - ("[all_close_atol_8.36E-02_rtol_1.00E-03]", cmp_all_close_func, {"atol": 8.36E-02, "rtol": 1.00E-03}), - ("[all_close_atol_2.94E-04_rtol_1.00E-03]", cmp_all_close_func, {"atol": 2.94E-04, "rtol": 1.00E-03}), - ("[all_close_atol_6.31E-05_rtol_1.00E-03]", cmp_all_close_func, {"atol": 6.31E-05, "rtol": 1.00E-03}), - - ("[all_close_atol_6.31E-02_rtol_1.00E-02]", cmp_all_close_func, {"atol": 6.31E-02, "rtol": 1.00E-02}), - ("[all_close_atol_1.91E-01_rtol_1.00E-02]", cmp_all_close_func, {"atol": 1.91E-01, "rtol": 1.00E-02}), - ("[all_close_atol_4.42E-03_rtol_1.00E-02]", cmp_all_close_func, {"atol": 4.42E-03, "rtol": 1.00E-02}), - ("[all_close_atol_1.58E-03_rtol_1.00E-02]", cmp_all_close_func, {"atol": 1.58E-03, "rtol": 1.00E-02}), - - ("[all_close_atol_2.51E-01_rtol_1.00E-01]", cmp_all_close_func, {"atol": 2.51E-01, "rtol": 1.00E-01}), - ("[all_close_atol_4.37E-01_rtol_1.00E-01]", cmp_all_close_func, {"atol": 4.37E-01, "rtol": 1.00E-01}), - ("[all_close_atol_6.65E-02_rtol_1.00E-01]", cmp_all_close_func, {"atol": 6.65E-02, "rtol": 1.00E-01}), - ("[all_close_atol_3.98E-02_rtol_1.00E-01]", cmp_all_close_func, {"atol": 3.98E-02, "rtol": 1.00E-01}), - - ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), - ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), - ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), - ("[all_close_atol_1.00E+00_rtol_1.00E+00]", cmp_all_close_func, {"atol": 1.00E+00, "rtol": 1.00E+00}), - - ("[all_close_atol_3.98E+00_rtol_1.00E+01]", cmp_all_close_func, {"atol": 3.98E+00, "rtol": 1.00E+01}), - ("[all_close_atol_2.29E+00_rtol_1.00E+01]", cmp_all_close_func, {"atol": 2.29E+00, "rtol": 1.00E+01}), - ("[all_close_atol_1.50E+01_rtol_1.00E+01]", cmp_all_close_func, {"atol": 1.50E+01, "rtol": 1.00E+01}), - ("[all_close_atol_2.51E+01_rtol_1.00E+01]", cmp_all_close_func, {"atol": 2.51E+01, "rtol": 1.00E+01}), - - ("[all_close_atol_1.58E+01_rtol_1.00E+02]", cmp_all_close_func, {"atol": 1.58E+01, "rtol": 1.00E+02}), - ("[all_close_atol_5.23E+00_rtol_1.00E+02]", cmp_all_close_func, {"atol": 5.23E+00, "rtol": 1.00E+02}), - ("[all_close_atol_2.26E+02_rtol_1.00E+02]", cmp_all_close_func, {"atol": 2.26E+02, "rtol": 1.00E+02}), - ("[all_close_atol_6.31E+02_rtol_1.00E+02]", cmp_all_close_func, {"atol": 6.31E+02, "rtol": 1.00E+02}), - - ("[all_close_atol_6.31E+01_rtol_1.00E+03]", cmp_all_close_func, {"atol": 6.31E+01, "rtol": 1.00E+03}), - ("[all_close_atol_1.20E+01_rtol_1.00E+03]", cmp_all_close_func, {"atol": 1.20E+01, "rtol": 1.00E+03}), - ("[all_close_atol_3.40E+03_rtol_1.00E+03]", cmp_all_close_func, {"atol": 3.40E+03, "rtol": 1.00E+03}), - ("[all_close_atol_1.58E+04_rtol_1.00E+03]", cmp_all_close_func, {"atol": 1.58E+04, "rtol": 1.00E+03}), - - ("[all_close_atol_2.51E+02_rtol_1.00E+04]", cmp_all_close_func, {"atol": 2.51E+02, "rtol": 1.00E+04}), - ("[all_close_atol_2.73E+01_rtol_1.00E+04]", cmp_all_close_func, {"atol": 2.73E+01, "rtol": 1.00E+04}), - ("[all_close_atol_5.11E+04_rtol_1.00E+04]", cmp_all_close_func, {"atol": 5.11E+04, "rtol": 1.00E+04}), - ("[all_close_atol_3.98E+05_rtol_1.00E+04]", cmp_all_close_func, {"atol": 3.98E+05, "rtol": 1.00E+04}), - - ("[all_close_atol_1.00E+03_rtol_1.00E+05]", cmp_all_close_func, {"atol": 1.00E+03, "rtol": 1.00E+05}), - ("[all_close_atol_6.25E+01_rtol_1.00E+05]", cmp_all_close_func, {"atol": 6.25E+01, "rtol": 1.00E+05}), - ("[all_close_atol_7.69E+05_rtol_1.00E+05]", cmp_all_close_func, {"atol": 7.69E+05, "rtol": 1.00E+05}), - ("[all_close_atol_1.00E+07_rtol_1.00E+05]", cmp_all_close_func, {"atol": 1.00E+07, "rtol": 1.00E+05}), + ( + "[all_close_atol_1.00E-06_rtol_1.00E-10]", + cmp_all_close_func, + {"atol": 1.00e-06, "rtol": 1.00e-10}, + ), + ( + "[all_close_atol_2.56E-04_rtol_1.00E-10]", + cmp_all_close_func, + {"atol": 2.56e-04, "rtol": 1.00e-10}, + ), + ( + "[all_close_atol_1.69E-12_rtol_1.00E-10]", + cmp_all_close_func, + {"atol": 1.69e-12, "rtol": 1.00e-10}, + ), + ( + "[all_close_atol_1.00E-14_rtol_1.00E-10]", + cmp_all_close_func, + {"atol": 1.00e-14, "rtol": 1.00e-10}, + ), + ( + "[all_close_atol_3.98E-06_rtol_1.00E-09]", + cmp_all_close_func, + {"atol": 3.98e-06, "rtol": 1.00e-09}, + ), + ( + "[all_close_atol_5.85E-04_rtol_1.00E-09]", + cmp_all_close_func, + {"atol": 5.85e-04, "rtol": 1.00e-09}, + ), + ( + "[all_close_atol_2.54E-11_rtol_1.00E-09]", + cmp_all_close_func, + {"atol": 2.54e-11, "rtol": 1.00e-09}, + ), + ( + "[all_close_atol_2.51E-13_rtol_1.00E-09]", + cmp_all_close_func, + {"atol": 2.51e-13, "rtol": 1.00e-09}, + ), + ( + "[all_close_atol_1.58E-05_rtol_1.00E-08]", + cmp_all_close_func, + {"atol": 1.58e-05, "rtol": 1.00e-08}, + ), + ( + "[all_close_atol_1.34E-03_rtol_1.00E-08]", + cmp_all_close_func, + {"atol": 1.34e-03, "rtol": 1.00e-08}, + ), + ( + "[all_close_atol_3.82E-10_rtol_1.00E-08]", + cmp_all_close_func, + {"atol": 3.82e-10, "rtol": 1.00e-08}, + ), + ( + "[all_close_atol_6.31E-12_rtol_1.00E-08]", + cmp_all_close_func, + {"atol": 6.31e-12, "rtol": 1.00e-08}, + ), + ( + "[all_close_atol_6.31E-05_rtol_1.00E-07]", + cmp_all_close_func, + {"atol": 6.31e-05, "rtol": 1.00e-07}, + ), + ( + "[all_close_atol_3.06E-03_rtol_1.00E-07]", + cmp_all_close_func, + {"atol": 3.06e-03, "rtol": 1.00e-07}, + ), + ( + "[all_close_atol_5.75E-09_rtol_1.00E-07]", + cmp_all_close_func, + {"atol": 5.75e-09, "rtol": 1.00e-07}, + ), + ( + "[all_close_atol_1.58E-10_rtol_1.00E-07]", + cmp_all_close_func, + {"atol": 1.58e-10, "rtol": 1.00e-07}, + ), + ( + "[all_close_atol_2.51E-04_rtol_1.00E-06]", + cmp_all_close_func, + {"atol": 2.51e-04, "rtol": 1.00e-06}, + ), + ( + "[all_close_atol_7.00E-03_rtol_1.00E-06]", + cmp_all_close_func, + {"atol": 7.00e-03, "rtol": 1.00e-06}, + ), + ( + "[all_close_atol_8.65E-08_rtol_1.00E-06]", + cmp_all_close_func, + {"atol": 8.65e-08, "rtol": 1.00e-06}, + ), + ( + "[all_close_atol_3.98E-09_rtol_1.00E-06]", + cmp_all_close_func, + {"atol": 3.98e-09, "rtol": 1.00e-06}, + ), + ( + "[all_close_atol_1.00E-03_rtol_1.00E-05]", + cmp_all_close_func, + {"atol": 1.00e-03, "rtol": 1.00e-05}, + ), + ( + "[all_close_atol_1.60E-02_rtol_1.00E-05]", + cmp_all_close_func, + {"atol": 1.60e-02, "rtol": 1.00e-05}, + ), + ( + "[all_close_atol_1.30E-06_rtol_1.00E-05]", + cmp_all_close_func, + {"atol": 1.30e-06, "rtol": 1.00e-05}, + ), + ( + "[all_close_atol_1.00E-07_rtol_1.00E-05]", + cmp_all_close_func, + {"atol": 1.00e-07, "rtol": 1.00e-05}, + ), + ( + "[all_close_atol_3.98E-03_rtol_1.00E-04]", + cmp_all_close_func, + {"atol": 3.98e-03, "rtol": 1.00e-04}, + ), + ( + "[all_close_atol_3.66E-02_rtol_1.00E-04]", + cmp_all_close_func, + {"atol": 3.66e-02, "rtol": 1.00e-04}, + ), + ( + "[all_close_atol_1.96E-05_rtol_1.00E-04]", + cmp_all_close_func, + {"atol": 1.96e-05, "rtol": 1.00e-04}, + ), + ( + "[all_close_atol_2.51E-06_rtol_1.00E-04]", + cmp_all_close_func, + {"atol": 2.51e-06, "rtol": 1.00e-04}, + ), + ( + "[all_close_atol_1.58E-02_rtol_1.00E-03]", + cmp_all_close_func, + {"atol": 1.58e-02, "rtol": 1.00e-03}, + ), + ( + "[all_close_atol_8.36E-02_rtol_1.00E-03]", + cmp_all_close_func, + {"atol": 8.36e-02, "rtol": 1.00e-03}, + ), + ( + "[all_close_atol_2.94E-04_rtol_1.00E-03]", + cmp_all_close_func, + {"atol": 2.94e-04, "rtol": 1.00e-03}, + ), + ( + "[all_close_atol_6.31E-05_rtol_1.00E-03]", + cmp_all_close_func, + {"atol": 6.31e-05, "rtol": 1.00e-03}, + ), + ( + "[all_close_atol_6.31E-02_rtol_1.00E-02]", + cmp_all_close_func, + {"atol": 6.31e-02, "rtol": 1.00e-02}, + ), + ( + "[all_close_atol_1.91E-01_rtol_1.00E-02]", + cmp_all_close_func, + {"atol": 1.91e-01, "rtol": 1.00e-02}, + ), + ( + "[all_close_atol_4.42E-03_rtol_1.00E-02]", + cmp_all_close_func, + {"atol": 4.42e-03, "rtol": 1.00e-02}, + ), + ( + "[all_close_atol_1.58E-03_rtol_1.00E-02]", + cmp_all_close_func, + {"atol": 1.58e-03, "rtol": 1.00e-02}, + ), + ( + "[all_close_atol_2.51E-01_rtol_1.00E-01]", + cmp_all_close_func, + {"atol": 2.51e-01, "rtol": 1.00e-01}, + ), + ( + "[all_close_atol_4.37E-01_rtol_1.00E-01]", + cmp_all_close_func, + {"atol": 4.37e-01, "rtol": 1.00e-01}, + ), + ( + "[all_close_atol_6.65E-02_rtol_1.00E-01]", + cmp_all_close_func, + {"atol": 6.65e-02, "rtol": 1.00e-01}, + ), + ( + "[all_close_atol_3.98E-02_rtol_1.00E-01]", + cmp_all_close_func, + {"atol": 3.98e-02, "rtol": 1.00e-01}, + ), + ( + "[all_close_atol_1.00E+00_rtol_1.00E+00]", + cmp_all_close_func, + {"atol": 1.00e00, "rtol": 1.00e00}, + ), + ( + "[all_close_atol_1.00E+00_rtol_1.00E+00]", + cmp_all_close_func, + {"atol": 1.00e00, "rtol": 1.00e00}, + ), + ( + "[all_close_atol_1.00E+00_rtol_1.00E+00]", + cmp_all_close_func, + {"atol": 1.00e00, "rtol": 1.00e00}, + ), + ( + "[all_close_atol_1.00E+00_rtol_1.00E+00]", + cmp_all_close_func, + {"atol": 1.00e00, "rtol": 1.00e00}, + ), + ( + "[all_close_atol_3.98E+00_rtol_1.00E+01]", + cmp_all_close_func, + {"atol": 3.98e00, "rtol": 1.00e01}, + ), + ( + "[all_close_atol_2.29E+00_rtol_1.00E+01]", + cmp_all_close_func, + {"atol": 2.29e00, "rtol": 1.00e01}, + ), + ( + "[all_close_atol_1.50E+01_rtol_1.00E+01]", + cmp_all_close_func, + {"atol": 1.50e01, "rtol": 1.00e01}, + ), + ( + "[all_close_atol_2.51E+01_rtol_1.00E+01]", + cmp_all_close_func, + {"atol": 2.51e01, "rtol": 1.00e01}, + ), + ( + "[all_close_atol_1.58E+01_rtol_1.00E+02]", + cmp_all_close_func, + {"atol": 1.58e01, "rtol": 1.00e02}, + ), + ( + "[all_close_atol_5.23E+00_rtol_1.00E+02]", + cmp_all_close_func, + {"atol": 5.23e00, "rtol": 1.00e02}, + ), + ( + "[all_close_atol_2.26E+02_rtol_1.00E+02]", + cmp_all_close_func, + {"atol": 2.26e02, "rtol": 1.00e02}, + ), + ( + "[all_close_atol_6.31E+02_rtol_1.00E+02]", + cmp_all_close_func, + {"atol": 6.31e02, "rtol": 1.00e02}, + ), + ( + "[all_close_atol_6.31E+01_rtol_1.00E+03]", + cmp_all_close_func, + {"atol": 6.31e01, "rtol": 1.00e03}, + ), + ( + "[all_close_atol_1.20E+01_rtol_1.00E+03]", + cmp_all_close_func, + {"atol": 1.20e01, "rtol": 1.00e03}, + ), + ( + "[all_close_atol_3.40E+03_rtol_1.00E+03]", + cmp_all_close_func, + {"atol": 3.40e03, "rtol": 1.00e03}, + ), + ( + "[all_close_atol_1.58E+04_rtol_1.00E+03]", + cmp_all_close_func, + {"atol": 1.58e04, "rtol": 1.00e03}, + ), + ( + "[all_close_atol_2.51E+02_rtol_1.00E+04]", + cmp_all_close_func, + {"atol": 2.51e02, "rtol": 1.00e04}, + ), + ( + "[all_close_atol_2.73E+01_rtol_1.00E+04]", + cmp_all_close_func, + {"atol": 2.73e01, "rtol": 1.00e04}, + ), + ( + "[all_close_atol_5.11E+04_rtol_1.00E+04]", + cmp_all_close_func, + {"atol": 5.11e04, "rtol": 1.00e04}, + ), + ( + "[all_close_atol_3.98E+05_rtol_1.00E+04]", + cmp_all_close_func, + {"atol": 3.98e05, "rtol": 1.00e04}, + ), + ( + "[all_close_atol_1.00E+03_rtol_1.00E+05]", + cmp_all_close_func, + {"atol": 1.00e03, "rtol": 1.00e05}, + ), + ( + "[all_close_atol_6.25E+01_rtol_1.00E+05]", + cmp_all_close_func, + {"atol": 6.25e01, "rtol": 1.00e05}, + ), + ( + "[all_close_atol_7.69E+05_rtol_1.00E+05]", + cmp_all_close_func, + {"atol": 7.69e05, "rtol": 1.00e05}, + ), + ( + "[all_close_atol_1.00E+07_rtol_1.00E+05]", + cmp_all_close_func, + {"atol": 1.00e07, "rtol": 1.00e05}, + ), ("[max_diff]", cmp_max_diff_func, {}), ("[mean_diff]", cmp_mean_diff_func, {}), ] From 4f8d8dab0a4ca7ab65ab095884bb81ad8150c37c Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 25 Sep 2025 13:56:08 +0800 Subject: [PATCH 12/16] Correct the tolerance. --- graph_net/test_compiler_util.py | 365 ++++---------------------------- 1 file changed, 41 insertions(+), 324 deletions(-) diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index 66887d06a6..fac833537d 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -159,6 +159,44 @@ def check_equal(args, expected_out, compiled_out, cmp_equal_func): ) +def tolerance_generator(t): + # for float16 + yield 10 ** (t * 3 / 5), 10**t + # for bfloat16 + yield 10 ** (t * 1.796 / 5), 10**t + # yield float32 + yield 10 ** (t * 5.886 / 5), 10**t + # yield float64 + yield 10 ** (t * 7 / 5), 10 ** (t * 7 / 5) + + +def compute_tolerance_pair(begin, end): + tolerance_pair_list = [] + for t in range(begin, end + 1): + for rtol, atol in tolerance_generator(t): + effective_atol = float(f"{atol:.3g}") + effective_rtol = float(f"{rtol:.3g}") + tolerance_pair_list.append( + { + "atol": effective_atol, + "rtol": effective_rtol, + } + ) + return tolerance_pair_list + + +def generate_allclose_configs(cmp_all_close_func): + tolerance_pair_list = compute_tolerance_pair(-10, 5) + + cmp_configs = [] + for pair in tolerance_pair_list: + atol, rtol = pair["atol"], pair["rtol"] + cmp_configs.append( + (f"[all_close_atol_{atol:.2E}_rtol_{rtol:.2E}]", cmp_all_close_func, pair) + ) + return cmp_configs + + def check_allclose( args, expected_out, @@ -168,330 +206,9 @@ def check_allclose( cmp_mean_diff_func, cmp_diff_count_func, ): - cmp_configs = [ - ( - "[all_close_atol_1.00E-06_rtol_1.00E-10]", - cmp_all_close_func, - {"atol": 1.00e-06, "rtol": 1.00e-10}, - ), - ( - "[all_close_atol_2.56E-04_rtol_1.00E-10]", - cmp_all_close_func, - {"atol": 2.56e-04, "rtol": 1.00e-10}, - ), - ( - "[all_close_atol_1.69E-12_rtol_1.00E-10]", - cmp_all_close_func, - {"atol": 1.69e-12, "rtol": 1.00e-10}, - ), - ( - "[all_close_atol_1.00E-14_rtol_1.00E-10]", - cmp_all_close_func, - {"atol": 1.00e-14, "rtol": 1.00e-10}, - ), - ( - "[all_close_atol_3.98E-06_rtol_1.00E-09]", - cmp_all_close_func, - {"atol": 3.98e-06, "rtol": 1.00e-09}, - ), - ( - "[all_close_atol_5.85E-04_rtol_1.00E-09]", - cmp_all_close_func, - {"atol": 5.85e-04, "rtol": 1.00e-09}, - ), - ( - "[all_close_atol_2.54E-11_rtol_1.00E-09]", - cmp_all_close_func, - {"atol": 2.54e-11, "rtol": 1.00e-09}, - ), - ( - "[all_close_atol_2.51E-13_rtol_1.00E-09]", - cmp_all_close_func, - {"atol": 2.51e-13, "rtol": 1.00e-09}, - ), - ( - "[all_close_atol_1.58E-05_rtol_1.00E-08]", - cmp_all_close_func, - {"atol": 1.58e-05, "rtol": 1.00e-08}, - ), - ( - "[all_close_atol_1.34E-03_rtol_1.00E-08]", - cmp_all_close_func, - {"atol": 1.34e-03, "rtol": 1.00e-08}, - ), - ( - "[all_close_atol_3.82E-10_rtol_1.00E-08]", - cmp_all_close_func, - {"atol": 3.82e-10, "rtol": 1.00e-08}, - ), - ( - "[all_close_atol_6.31E-12_rtol_1.00E-08]", - cmp_all_close_func, - {"atol": 6.31e-12, "rtol": 1.00e-08}, - ), - ( - "[all_close_atol_6.31E-05_rtol_1.00E-07]", - cmp_all_close_func, - {"atol": 6.31e-05, "rtol": 1.00e-07}, - ), - ( - "[all_close_atol_3.06E-03_rtol_1.00E-07]", - cmp_all_close_func, - {"atol": 3.06e-03, "rtol": 1.00e-07}, - ), - ( - "[all_close_atol_5.75E-09_rtol_1.00E-07]", - cmp_all_close_func, - {"atol": 5.75e-09, "rtol": 1.00e-07}, - ), - ( - "[all_close_atol_1.58E-10_rtol_1.00E-07]", - cmp_all_close_func, - {"atol": 1.58e-10, "rtol": 1.00e-07}, - ), - ( - "[all_close_atol_2.51E-04_rtol_1.00E-06]", - cmp_all_close_func, - {"atol": 2.51e-04, "rtol": 1.00e-06}, - ), - ( - "[all_close_atol_7.00E-03_rtol_1.00E-06]", - cmp_all_close_func, - {"atol": 7.00e-03, "rtol": 1.00e-06}, - ), - ( - "[all_close_atol_8.65E-08_rtol_1.00E-06]", - cmp_all_close_func, - {"atol": 8.65e-08, "rtol": 1.00e-06}, - ), - ( - "[all_close_atol_3.98E-09_rtol_1.00E-06]", - cmp_all_close_func, - {"atol": 3.98e-09, "rtol": 1.00e-06}, - ), - ( - "[all_close_atol_1.00E-03_rtol_1.00E-05]", - cmp_all_close_func, - {"atol": 1.00e-03, "rtol": 1.00e-05}, - ), - ( - "[all_close_atol_1.60E-02_rtol_1.00E-05]", - cmp_all_close_func, - {"atol": 1.60e-02, "rtol": 1.00e-05}, - ), - ( - "[all_close_atol_1.30E-06_rtol_1.00E-05]", - cmp_all_close_func, - {"atol": 1.30e-06, "rtol": 1.00e-05}, - ), - ( - "[all_close_atol_1.00E-07_rtol_1.00E-05]", - cmp_all_close_func, - {"atol": 1.00e-07, "rtol": 1.00e-05}, - ), - ( - "[all_close_atol_3.98E-03_rtol_1.00E-04]", - cmp_all_close_func, - {"atol": 3.98e-03, "rtol": 1.00e-04}, - ), - ( - "[all_close_atol_3.66E-02_rtol_1.00E-04]", - cmp_all_close_func, - {"atol": 3.66e-02, "rtol": 1.00e-04}, - ), - ( - "[all_close_atol_1.96E-05_rtol_1.00E-04]", - cmp_all_close_func, - {"atol": 1.96e-05, "rtol": 1.00e-04}, - ), - ( - "[all_close_atol_2.51E-06_rtol_1.00E-04]", - cmp_all_close_func, - {"atol": 2.51e-06, "rtol": 1.00e-04}, - ), - ( - "[all_close_atol_1.58E-02_rtol_1.00E-03]", - cmp_all_close_func, - {"atol": 1.58e-02, "rtol": 1.00e-03}, - ), - ( - "[all_close_atol_8.36E-02_rtol_1.00E-03]", - cmp_all_close_func, - {"atol": 8.36e-02, "rtol": 1.00e-03}, - ), - ( - "[all_close_atol_2.94E-04_rtol_1.00E-03]", - cmp_all_close_func, - {"atol": 2.94e-04, "rtol": 1.00e-03}, - ), - ( - "[all_close_atol_6.31E-05_rtol_1.00E-03]", - cmp_all_close_func, - {"atol": 6.31e-05, "rtol": 1.00e-03}, - ), - ( - "[all_close_atol_6.31E-02_rtol_1.00E-02]", - cmp_all_close_func, - {"atol": 6.31e-02, "rtol": 1.00e-02}, - ), - ( - "[all_close_atol_1.91E-01_rtol_1.00E-02]", - cmp_all_close_func, - {"atol": 1.91e-01, "rtol": 1.00e-02}, - ), - ( - "[all_close_atol_4.42E-03_rtol_1.00E-02]", - cmp_all_close_func, - {"atol": 4.42e-03, "rtol": 1.00e-02}, - ), - ( - "[all_close_atol_1.58E-03_rtol_1.00E-02]", - cmp_all_close_func, - {"atol": 1.58e-03, "rtol": 1.00e-02}, - ), - ( - "[all_close_atol_2.51E-01_rtol_1.00E-01]", - cmp_all_close_func, - {"atol": 2.51e-01, "rtol": 1.00e-01}, - ), - ( - "[all_close_atol_4.37E-01_rtol_1.00E-01]", - cmp_all_close_func, - {"atol": 4.37e-01, "rtol": 1.00e-01}, - ), - ( - "[all_close_atol_6.65E-02_rtol_1.00E-01]", - cmp_all_close_func, - {"atol": 6.65e-02, "rtol": 1.00e-01}, - ), - ( - "[all_close_atol_3.98E-02_rtol_1.00E-01]", - cmp_all_close_func, - {"atol": 3.98e-02, "rtol": 1.00e-01}, - ), - ( - "[all_close_atol_1.00E+00_rtol_1.00E+00]", - cmp_all_close_func, - {"atol": 1.00e00, "rtol": 1.00e00}, - ), - ( - "[all_close_atol_1.00E+00_rtol_1.00E+00]", - cmp_all_close_func, - {"atol": 1.00e00, "rtol": 1.00e00}, - ), - ( - "[all_close_atol_1.00E+00_rtol_1.00E+00]", - cmp_all_close_func, - {"atol": 1.00e00, "rtol": 1.00e00}, - ), - ( - "[all_close_atol_1.00E+00_rtol_1.00E+00]", - cmp_all_close_func, - {"atol": 1.00e00, "rtol": 1.00e00}, - ), - ( - "[all_close_atol_3.98E+00_rtol_1.00E+01]", - cmp_all_close_func, - {"atol": 3.98e00, "rtol": 1.00e01}, - ), - ( - "[all_close_atol_2.29E+00_rtol_1.00E+01]", - cmp_all_close_func, - {"atol": 2.29e00, "rtol": 1.00e01}, - ), - ( - "[all_close_atol_1.50E+01_rtol_1.00E+01]", - cmp_all_close_func, - {"atol": 1.50e01, "rtol": 1.00e01}, - ), - ( - "[all_close_atol_2.51E+01_rtol_1.00E+01]", - cmp_all_close_func, - {"atol": 2.51e01, "rtol": 1.00e01}, - ), - ( - "[all_close_atol_1.58E+01_rtol_1.00E+02]", - cmp_all_close_func, - {"atol": 1.58e01, "rtol": 1.00e02}, - ), - ( - "[all_close_atol_5.23E+00_rtol_1.00E+02]", - cmp_all_close_func, - {"atol": 5.23e00, "rtol": 1.00e02}, - ), - ( - "[all_close_atol_2.26E+02_rtol_1.00E+02]", - cmp_all_close_func, - {"atol": 2.26e02, "rtol": 1.00e02}, - ), - ( - "[all_close_atol_6.31E+02_rtol_1.00E+02]", - cmp_all_close_func, - {"atol": 6.31e02, "rtol": 1.00e02}, - ), - ( - "[all_close_atol_6.31E+01_rtol_1.00E+03]", - cmp_all_close_func, - {"atol": 6.31e01, "rtol": 1.00e03}, - ), - ( - "[all_close_atol_1.20E+01_rtol_1.00E+03]", - cmp_all_close_func, - {"atol": 1.20e01, "rtol": 1.00e03}, - ), - ( - "[all_close_atol_3.40E+03_rtol_1.00E+03]", - cmp_all_close_func, - {"atol": 3.40e03, "rtol": 1.00e03}, - ), - ( - "[all_close_atol_1.58E+04_rtol_1.00E+03]", - cmp_all_close_func, - {"atol": 1.58e04, "rtol": 1.00e03}, - ), - ( - "[all_close_atol_2.51E+02_rtol_1.00E+04]", - cmp_all_close_func, - {"atol": 2.51e02, "rtol": 1.00e04}, - ), - ( - "[all_close_atol_2.73E+01_rtol_1.00E+04]", - cmp_all_close_func, - {"atol": 2.73e01, "rtol": 1.00e04}, - ), - ( - "[all_close_atol_5.11E+04_rtol_1.00E+04]", - cmp_all_close_func, - {"atol": 5.11e04, "rtol": 1.00e04}, - ), - ( - "[all_close_atol_3.98E+05_rtol_1.00E+04]", - cmp_all_close_func, - {"atol": 3.98e05, "rtol": 1.00e04}, - ), - ( - "[all_close_atol_1.00E+03_rtol_1.00E+05]", - cmp_all_close_func, - {"atol": 1.00e03, "rtol": 1.00e05}, - ), - ( - "[all_close_atol_6.25E+01_rtol_1.00E+05]", - cmp_all_close_func, - {"atol": 6.25e01, "rtol": 1.00e05}, - ), - ( - "[all_close_atol_7.69E+05_rtol_1.00E+05]", - cmp_all_close_func, - {"atol": 7.69e05, "rtol": 1.00e05}, - ), - ( - "[all_close_atol_1.00E+07_rtol_1.00E+05]", - cmp_all_close_func, - {"atol": 1.00e07, "rtol": 1.00e05}, - ), - ("[max_diff]", cmp_max_diff_func, {}), - ("[mean_diff]", cmp_mean_diff_func, {}), - ] + cmp_configs = generate_allclose_configs(cmp_all_close_func) + cmp_configs.append(("[max_diff]", cmp_max_diff_func, {})) + cmp_configs.append(("[mean_diff]", cmp_mean_diff_func, {})) for key, func, kwargs in cmp_configs: print_and_store_cmp( From 70386d1517de8f88fa1a021b97ebaa2bb9add4f5 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Thu, 25 Sep 2025 14:32:45 +0800 Subject: [PATCH 13/16] Allow to specify the verified samples. --- graph_net/paddle/test_compiler.py | 59 ++++++++++++++++++++----------- graph_net/path_utils.py | 5 +++ 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 81dcbd127c..d527d2fc06 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -317,30 +317,42 @@ def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): def test_multi_models(args): + verified_samples = None + if args.verified_samples_path is not None: + assert os.path.isfile(args.verified_samples_path) + graphnet_root = path_utils.get_graphnet_root() + print(f"graphnet_root: {graphnet_root}") + verified_samples = [] + with open(args.verified_samples_path, "r") as f: + for line in f.readlines(): + verified_samples.append(os.path.join(graphnet_root, line.strip())) + sample_idx = 0 failed_samples = [] for model_path in path_utils.get_recursively_model_path(args.model_path): - print(f"[{sample_idx}] test_compiler, model_path: {model_path}") - cmd = " ".join( - [ - sys.executable, - "-m graph_net.paddle.test_compiler", - f"--model-path {model_path}", - f"--compiler {args.compiler}", - f"--device {args.device}", - f"--warmup {args.warmup}", - f"--trials {args.trials}", - f"--log-prompt {args.log_prompt}", - f"--output-dir {args.output_dir}", - ] - ) - cmd_ret = os.system(cmd) - # assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" - if cmd_ret != 0: - failed_samples.append(model_path) - sample_idx += 1 + if verified_samples is None or os.path.abspath(model_path) in verified_samples: + print(f"[{sample_idx}] test_compiler, model_path: {model_path}") + cmd = " ".join( + [ + sys.executable, + "-m graph_net.paddle.test_compiler", + f"--model-path {model_path}", + f"--compiler {args.compiler}", + f"--device {args.device}", + f"--warmup {args.warmup}", + f"--trials {args.trials}", + f"--log-prompt {args.log_prompt}", + ] + ) + cmd_ret = os.system(cmd) + # assert cmd_ret == 0, f"{cmd_ret=}, {cmd=}" + if cmd_ret != 0: + failed_samples.append(model_path) + sample_idx += 1 - print(f"Totally {sample_idx} samples, failed {len(failed_samples)} samples.") + print( + f"Totally {sample_idx} verified samples, failed {len(failed_samples)} samples." + ) for model_path in failed_samples: print(f"- {model_path}") @@ -393,5 +405,12 @@ def main(args): default="graph-net-test-compiler-log", help="Log prompt for performance log filtering.", ) + parser.add_argument( + "--verified-samples-path", + type=str, + required=False, + default=None, + help="Path to model file(s), each subdirectory containing graph_net.json will be regarded as a model", + ) args = parser.parse_args() main(args=args) diff --git a/graph_net/path_utils.py b/graph_net/path_utils.py index 9047168cd0..8672109e69 100644 --- a/graph_net/path_utils.py +++ b/graph_net/path_utils.py @@ -1,4 +1,9 @@ import os +import graph_net + + +def get_graphnet_root(): + return os.path.dirname(os.path.dirname(graph_net.__file__)) def is_single_model_dir(model_dir): From 5799419ec2df938365754adedfd0685368c25341 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Fri, 26 Sep 2025 17:39:10 +0800 Subject: [PATCH 14/16] Fix test_compiler and several sample errors. --- graph_net/paddle/test_compiler.py | 11 ++- .../PaddleX/Co-DINO-R50/subgraph_2/model.py | 24 ------ .../PaddleX/Co-DINO-R50/subgraph_20/model.py | 2 - .../PaddleX/FCOS-ResNet50/subgraph_1/model.py | 84 +------------------ .../PaddleX/FCOS-ResNet50/subgraph_4/model.py | 84 +------------------ .../MaskFormer_small/subgraph_2/model.py | 18 ---- .../MaskFormer_tiny/subgraph_0/model.py | 18 ---- .../PaddleX/Nonstationary/subgraph_0/model.py | 2 +- .../Nonstationary/subgraph_10/model.py | 16 ---- .../PaddleX/Nonstationary/subgraph_2/model.py | 12 +-- .../PaddleX/Nonstationary/subgraph_3/model.py | 12 +-- .../PaddleX/Nonstationary/subgraph_8/model.py | 16 ---- .../PaddleX/Nonstationary/subgraph_9/model.py | 2 +- .../Nonstationary_ad/subgraph_1/model.py | 20 +---- .../Nonstationary_ad/subgraph_2/model.py | 20 +---- .../PP-OCRv3_mobile_rec/subgraph_1/model.py | 2 +- 16 files changed, 19 insertions(+), 324 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 3495733e9d..1069219680 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from contextlib import contextmanager import time +import math import numpy as np import random import platform @@ -283,8 +284,16 @@ def test_single_model(args): def get_cmp_equal(expected_out, compiled_out): + def convert(x): + if x.dtype in [paddle.float16, paddle.bfloat16]: + return x.astype("float32") + elif x.dtype in [paddle.uint8, paddle.int8, paddle.int16]: + return x.astype("int32") + return x + return " ".join( - str(int(paddle.equal_all(a, b))) for a, b in zip(expected_out, compiled_out) + str(int(paddle.equal_all(convert(a), convert(b)))) + for a, b in zip(expected_out, compiled_out) ) diff --git a/paddle_samples/PaddleX/Co-DINO-R50/subgraph_2/model.py b/paddle_samples/PaddleX/Co-DINO-R50/subgraph_2/model.py index 61d2988b3c..8b1a8fea8b 100644 --- a/paddle_samples/PaddleX/Co-DINO-R50/subgraph_2/model.py +++ b/paddle_samples/PaddleX/Co-DINO-R50/subgraph_2/model.py @@ -645,30 +645,6 @@ def forward( ) return ( - group_norm_0, - group_norm_1, - group_norm_2, - group_norm_3, - group_norm_4, - group_norm_5, - group_norm_6, - group_norm_7, - group_norm_8, - group_norm_9, - group_norm_10, - group_norm_11, - group_norm_12, - group_norm_13, - group_norm_14, - group_norm_15, - group_norm_16, - group_norm_17, - group_norm_18, - group_norm_19, - group_norm_20, - group_norm_21, - group_norm_22, - group_norm_23, add_0, add_1, add_2, diff --git a/paddle_samples/PaddleX/Co-DINO-R50/subgraph_20/model.py b/paddle_samples/PaddleX/Co-DINO-R50/subgraph_20/model.py index 2c3763a5df..01312883c5 100644 --- a/paddle_samples/PaddleX/Co-DINO-R50/subgraph_20/model.py +++ b/paddle_samples/PaddleX/Co-DINO-R50/subgraph_20/model.py @@ -1135,8 +1135,6 @@ def forward( ) return ( - group_norm_0, - group_norm_1, stack_0, stack_1, reshape_0, diff --git a/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_1/model.py b/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_1/model.py index 69c58b39f1..fb82ff0ae8 100644 --- a/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_1/model.py +++ b/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_1/model.py @@ -5079,86 +5079,4 @@ def forward( transpose_9, ) - return ( - group_norm_0, - group_norm_1, - group_norm_2, - group_norm_3, - group_norm_4, - group_norm_5, - group_norm_6, - group_norm_7, - group_norm_8, - group_norm_9, - group_norm_10, - group_norm_11, - group_norm_12, - group_norm_13, - group_norm_14, - group_norm_15, - group_norm_16, - group_norm_17, - group_norm_18, - group_norm_19, - group_norm_20, - group_norm_21, - group_norm_22, - group_norm_23, - group_norm_24, - group_norm_25, - group_norm_26, - group_norm_27, - group_norm_28, - group_norm_29, - group_norm_30, - group_norm_31, - group_norm_32, - group_norm_33, - group_norm_34, - group_norm_35, - group_norm_36, - group_norm_37, - group_norm_38, - group_norm_39, - group_norm_40, - group_norm_41, - group_norm_42, - group_norm_43, - group_norm_44, - group_norm_45, - group_norm_46, - group_norm_47, - group_norm_48, - group_norm_49, - group_norm_50, - group_norm_51, - group_norm_52, - group_norm_53, - group_norm_54, - group_norm_55, - group_norm_56, - group_norm_57, - group_norm_58, - group_norm_59, - group_norm_60, - group_norm_61, - group_norm_62, - group_norm_63, - group_norm_64, - group_norm_65, - group_norm_66, - group_norm_67, - group_norm_68, - group_norm_69, - group_norm_70, - group_norm_71, - group_norm_72, - group_norm_73, - group_norm_74, - group_norm_75, - group_norm_76, - group_norm_77, - group_norm_78, - group_norm_79, - add_n_0, - ) + return add_n_0 diff --git a/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_4/model.py b/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_4/model.py index 9f131bed07..b3349ba53b 100644 --- a/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_4/model.py +++ b/paddle_samples/PaddleX/FCOS-ResNet50/subgraph_4/model.py @@ -5066,86 +5066,4 @@ def forward( transpose_9, ) - return ( - group_norm_0, - group_norm_1, - group_norm_2, - group_norm_3, - group_norm_4, - group_norm_5, - group_norm_6, - group_norm_7, - group_norm_8, - group_norm_9, - group_norm_10, - group_norm_11, - group_norm_12, - group_norm_13, - group_norm_14, - group_norm_15, - group_norm_16, - group_norm_17, - group_norm_18, - group_norm_19, - group_norm_20, - group_norm_21, - group_norm_22, - group_norm_23, - group_norm_24, - group_norm_25, - group_norm_26, - group_norm_27, - group_norm_28, - group_norm_29, - group_norm_30, - group_norm_31, - group_norm_32, - group_norm_33, - group_norm_34, - group_norm_35, - group_norm_36, - group_norm_37, - group_norm_38, - group_norm_39, - group_norm_40, - group_norm_41, - group_norm_42, - group_norm_43, - group_norm_44, - group_norm_45, - group_norm_46, - group_norm_47, - group_norm_48, - group_norm_49, - group_norm_50, - group_norm_51, - group_norm_52, - group_norm_53, - group_norm_54, - group_norm_55, - group_norm_56, - group_norm_57, - group_norm_58, - group_norm_59, - group_norm_60, - group_norm_61, - group_norm_62, - group_norm_63, - group_norm_64, - group_norm_65, - group_norm_66, - group_norm_67, - group_norm_68, - group_norm_69, - group_norm_70, - group_norm_71, - group_norm_72, - group_norm_73, - group_norm_74, - group_norm_75, - group_norm_76, - group_norm_77, - group_norm_78, - group_norm_79, - add_n_0, - ) + return add_n_0 diff --git a/paddle_samples/PaddleX/MaskFormer_small/subgraph_2/model.py b/paddle_samples/PaddleX/MaskFormer_small/subgraph_2/model.py index 0c33093cb7..4085d459a8 100644 --- a/paddle_samples/PaddleX/MaskFormer_small/subgraph_2/model.py +++ b/paddle_samples/PaddleX/MaskFormer_small/subgraph_2/model.py @@ -14274,24 +14274,6 @@ def forward( ) return ( - group_norm_0, - group_norm_1, - group_norm_2, - group_norm_3, - group_norm_4, - group_norm_5, - group_norm_6, - group_norm_7, - group_norm_8, - group_norm_9, - group_norm_10, - group_norm_11, - group_norm_12, - group_norm_13, - split_0, - split_1, - split_2, - split_3, slice_0, slice_1, slice_2, diff --git a/paddle_samples/PaddleX/MaskFormer_tiny/subgraph_0/model.py b/paddle_samples/PaddleX/MaskFormer_tiny/subgraph_0/model.py index 640c239c2a..49bcc32049 100644 --- a/paddle_samples/PaddleX/MaskFormer_tiny/subgraph_0/model.py +++ b/paddle_samples/PaddleX/MaskFormer_tiny/subgraph_0/model.py @@ -9546,24 +9546,6 @@ def forward( ) return ( - group_norm_0, - group_norm_1, - group_norm_2, - group_norm_3, - group_norm_4, - group_norm_5, - group_norm_6, - group_norm_7, - group_norm_8, - group_norm_9, - group_norm_10, - group_norm_11, - group_norm_12, - group_norm_13, - split_0, - split_1, - split_2, - split_3, slice_0, slice_1, slice_2, diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_0/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_0/model.py index da446a84dc..6a09c58666 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_0/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_0/model.py @@ -131,4 +131,4 @@ def forward(self, data_0, data_1, data_2): softmax_0, ) - return split_0, split_1, split_2, split_3, einsum_0 + return einsum_0 diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_10/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_10/model.py index f5eddce585..b9b27f29ac 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_10/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_10/model.py @@ -1565,22 +1565,6 @@ def forward( ) return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - split_8, - split_9, - split_10, - split_11, - split_12, - split_13, - split_14, - split_15, dropout_0, layer_norm_0, ) diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_2/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_2/model.py index 63da4ce2ae..56c1993f8e 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_2/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_2/model.py @@ -531,14 +531,4 @@ def forward( unsqueeze_7, ) - return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - layer_norm_0, - ) + return layer_norm_0 diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_3/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_3/model.py index f07f248123..cd0b379342 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_3/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_3/model.py @@ -431,14 +431,4 @@ def forward( unsqueeze_7, ) - return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - layer_norm_0, - ) + return layer_norm_0 diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_8/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_8/model.py index e764f06fd2..e68f6ecb19 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_8/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_8/model.py @@ -1344,22 +1344,6 @@ def forward( ) return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - split_8, - split_9, - split_10, - split_11, - split_12, - split_13, - split_14, - split_15, dropout_0, layer_norm_0, ) diff --git a/paddle_samples/PaddleX/Nonstationary/subgraph_9/model.py b/paddle_samples/PaddleX/Nonstationary/subgraph_9/model.py index c7e7862efe..a197fd22c6 100644 --- a/paddle_samples/PaddleX/Nonstationary/subgraph_9/model.py +++ b/paddle_samples/PaddleX/Nonstationary/subgraph_9/model.py @@ -64,4 +64,4 @@ def forward(self, data_0, data_1, data_2): unsqueeze_1, ) - return split_0, split_1, split_2, split_3, scale_0 + return scale_0 diff --git a/paddle_samples/PaddleX/Nonstationary_ad/subgraph_1/model.py b/paddle_samples/PaddleX/Nonstationary_ad/subgraph_1/model.py index e2f781b798..f93ff88abd 100644 --- a/paddle_samples/PaddleX/Nonstationary_ad/subgraph_1/model.py +++ b/paddle_samples/PaddleX/Nonstationary_ad/subgraph_1/model.py @@ -1382,22 +1382,4 @@ def forward( unsqueeze_9, ) - return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - split_8, - split_9, - split_10, - split_11, - split_12, - split_13, - split_14, - split_15, - add_0, - ) + return add_0 diff --git a/paddle_samples/PaddleX/Nonstationary_ad/subgraph_2/model.py b/paddle_samples/PaddleX/Nonstationary_ad/subgraph_2/model.py index 8b1c2e380f..f9ddb379e7 100644 --- a/paddle_samples/PaddleX/Nonstationary_ad/subgraph_2/model.py +++ b/paddle_samples/PaddleX/Nonstationary_ad/subgraph_2/model.py @@ -1181,22 +1181,4 @@ def forward( unsqueeze_9, ) - return ( - split_0, - split_1, - split_2, - split_3, - split_4, - split_5, - split_6, - split_7, - split_8, - split_9, - split_10, - split_11, - split_12, - split_13, - split_14, - split_15, - add_0, - ) + return add_0 diff --git a/paddle_samples/PaddleX/PP-OCRv3_mobile_rec/subgraph_1/model.py b/paddle_samples/PaddleX/PP-OCRv3_mobile_rec/subgraph_1/model.py index f6fb1ec264..43c0078df2 100644 --- a/paddle_samples/PaddleX/PP-OCRv3_mobile_rec/subgraph_1/model.py +++ b/paddle_samples/PaddleX/PP-OCRv3_mobile_rec/subgraph_1/model.py @@ -708,4 +708,4 @@ def forward( transpose_2, ) - return rnn_0, split_0, split_1, add_0 + return add_0 From 263709b38871a332a65fb46ec596a582d457e544 Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Mon, 29 Sep 2025 00:44:03 +0800 Subject: [PATCH 15/16] Add the compare of relative error. --- graph_net/paddle/test_compiler.py | 27 ++++++++++++++++++++++++--- graph_net/test_compiler_util.py | 8 ++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 1069219680..38f1ee45b8 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -236,7 +236,8 @@ def transfer_to_float(origin_outputs): cmp_all_close_func=get_cmp_all_close, cmp_max_diff_func=get_cmp_max_diff, cmp_mean_diff_func=get_cmp_mean_diff, - cmp_diff_count_func=get_cmp_diff_count, + cmp_max_relative_diff_func=get_cmp_max_relative_diff, + cmp_mean_relative_diff_func=get_cmp_mean_relative_diff, ) @@ -254,13 +255,14 @@ def test_single_model(args): eager_success = False try: print("Run model in eager mode.") - static_model = get_static_model(args, model) + # static_model = get_static_model(args, model) expected_out, eager_time_stats = measure_performance( - lambda: static_model(**input_dict), args, synchronizer_func, profile=False + lambda: model(**input_dict), args, synchronizer_func, profile=False ) eager_success = True except Exception as e: print(f"Run model in eager mode failed: {str(e)}\n{traceback.format_exc()}") + # sys.exit(0) # Run on compiling mode compiled_success = False @@ -276,6 +278,9 @@ def test_single_model(args): test_compiler_util.print_running_status(args, eager_success, compiled_success) if eager_success and compiled_success: + i = 0 + # print(f"expected_out[{i}]: {expected_out[i]}") + # print(f"compiled_out[{i}]: {compiled_out[i]}") check_outputs(args, expected_out, compiled_out) test_compiler_util.print_times_and_speedup( @@ -318,6 +323,22 @@ def get_cmp_mean_diff(expected_out, compiled_out): ) +def get_cmp_max_relative_diff(expected_out, compiled_out): + epsilon = 1e-8 + return " ".join( + str(paddle.max(paddle.abs(a - b) / (paddle.abs(a) + epsilon)).item()) + for a, b in zip(expected_out, compiled_out) + ) + + +def get_cmp_mean_relative_diff(expected_out, compiled_out): + epsilon = 1e-8 + return " ".join( + str(paddle.mean(paddle.abs(a - b) / (paddle.abs(a) + epsilon)).item()) + for a, b in zip(expected_out, compiled_out) + ) + + def get_cmp_diff_count(expected_out, compiled_out, atol, rtol): return " ".join( str(paddle.sum(~paddle.isclose(a, b, atol=atol, rtol=rtol)).item()) diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index 573422dd1d..59abdbcbbd 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -186,7 +186,8 @@ def calculate_tolerance_pair(begin, end): def generate_allclose_configs(cmp_all_close_func): - tolerance_pair_list = calculate_tolerance_pair(-10, 5) + # tolerance_pair_list = calculate_tolerance_pair(-10, 5) + tolerance_pair_list = calculate_tolerance_pair(-3, 1) cmp_configs = [] for pair in tolerance_pair_list: @@ -204,11 +205,14 @@ def check_allclose( cmp_all_close_func, cmp_max_diff_func, cmp_mean_diff_func, - cmp_diff_count_func, + cmp_max_relative_diff_func, + cmp_mean_relative_diff_func, ): cmp_configs = generate_allclose_configs(cmp_all_close_func) cmp_configs.append(("[max_diff]", cmp_max_diff_func, {})) cmp_configs.append(("[mean_diff]", cmp_mean_diff_func, {})) + cmp_configs.append(("[max_relative_diff]", cmp_max_relative_diff_func, {})) + cmp_configs.append(("[mean_relative_diff]", cmp_mean_relative_diff_func, {})) for key, func, kwargs in cmp_configs: print_and_store_cmp( From 666e39d0c075482f9f73bf69815707ef85a0dbdd Mon Sep 17 00:00:00 2001 From: Liu Yiqun Date: Tue, 30 Sep 2025 10:54:31 +0800 Subject: [PATCH 16/16] Remove the debugging codes. --- graph_net/paddle/test_compiler.py | 8 ++------ graph_net/test_compiler_util.py | 3 +-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/graph_net/paddle/test_compiler.py b/graph_net/paddle/test_compiler.py index 38f1ee45b8..51da7f5408 100644 --- a/graph_net/paddle/test_compiler.py +++ b/graph_net/paddle/test_compiler.py @@ -255,14 +255,13 @@ def test_single_model(args): eager_success = False try: print("Run model in eager mode.") - # static_model = get_static_model(args, model) + static_model = get_static_model(args, model) expected_out, eager_time_stats = measure_performance( - lambda: model(**input_dict), args, synchronizer_func, profile=False + lambda: static_model(**input_dict), args, synchronizer_func, profile=False ) eager_success = True except Exception as e: print(f"Run model in eager mode failed: {str(e)}\n{traceback.format_exc()}") - # sys.exit(0) # Run on compiling mode compiled_success = False @@ -278,9 +277,6 @@ def test_single_model(args): test_compiler_util.print_running_status(args, eager_success, compiled_success) if eager_success and compiled_success: - i = 0 - # print(f"expected_out[{i}]: {expected_out[i]}") - # print(f"compiled_out[{i}]: {compiled_out[i]}") check_outputs(args, expected_out, compiled_out) test_compiler_util.print_times_and_speedup( diff --git a/graph_net/test_compiler_util.py b/graph_net/test_compiler_util.py index 59abdbcbbd..2b4d083b2d 100644 --- a/graph_net/test_compiler_util.py +++ b/graph_net/test_compiler_util.py @@ -186,8 +186,7 @@ def calculate_tolerance_pair(begin, end): def generate_allclose_configs(cmp_all_close_func): - # tolerance_pair_list = calculate_tolerance_pair(-10, 5) - tolerance_pair_list = calculate_tolerance_pair(-3, 1) + tolerance_pair_list = calculate_tolerance_pair(-10, 5) cmp_configs = [] for pair in tolerance_pair_list: