jhw merge request

operation/benchmarks/drivers/calculate.py

@@ -4,16 +4,17 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
 import time
+from loguru import logger
 from triton.testing import do_bench as kernel_bench
 import os
 import subprocess
 
 
+# test operation correctness
 def do_correctness(operation):
     flaggems_dir = os.getenv("FLAGGEMS_WORK_DIR", "/")
     gems_repo = subprocess.check_output(
         ["find", flaggems_dir, "-type", "d", "-name", "FlagGems"], text=True).strip()
-
     p = subprocess.Popen(
         f"cd {os.path.join(gems_repo, 'tests')} && python3 test_named_ops.py --name {operation} --device cpu ",
         shell=True
@@ -22,8 +23,44 @@ def do_correctness(operation):
 
     return p.returncode
 
+
+# test operation performance
+def do_performance(mode, warmup, result_log_dir):
+    flaggems_dir = os.getenv("FLAGGEMS_WORK_DIR", "/")
+    gems_repo = subprocess.check_output(
+        ["find", flaggems_dir, "-type", "d", "-name", "FlagGems"], text=True).strip()
+    del_file_path = os.path.join(gems_repo, 'benchmark')
+    # 删除历史日志
+    # del_file = os.path.join(del_file_path,
+    #                       f"result_test_distribution_perf--level_core--mode_{mode}--warmup_{warmup}--record_log.log")
+    del_file = os.path.join(del_file_path,
+                            f"result--level_core--mode_{mode}--warmup_{warmup}--record_log.log")
+    del_process = subprocess.Popen(["rm", del_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    del_process.communicate()
+    p = subprocess.Popen(
+        # 执行所有算子命令
+        f"cd {os.path.join(gems_repo, 'benchmark')} && pytest --level core --mode {mode} --warmup {warmup} --record log",
+        # 执行单个算子命令
+        # f"cd {os.path.join(gems_repo, 'benchmark')} && pytest -m mm --level core --mode {mode} --warmup {warmup} --record log -s",
+        # 执行单个文件命令
+        # f"cd {os.path.join(gems_repo, 'benchmark')} && pytest test_distribution_perf.py --level core --mode {mode} --warmup {warmup} --record log",
+        shell=True
+    )
+    p.wait()
+
+    # 全量执行日志路径
+    log_dir = os.path.join(gems_repo, "benchmark",
+                           f"result--level_core--mode_{mode}--warmup_{warmup}--record_log.log")
+    # 仅执行单个文件日志路径
+    # log_dir = os.path.join(gems_repo, "benchmark",
+    #                        f"result_test_distribution_perf--level_core--mode_{mode}--warmup_{warmup}--record_log.log")
+    cp_subprocess = subprocess.run(["cp", f"{log_dir}", f"{result_log_dir}/result.log.txt"], check=True)
+    return p.returncode, cp_subprocess.returncode
+
+
 grad_outputs = None
 
+
 def do(exec_func, exec_args, bp=False):
     global grad_outputs
     if bp:

operation/benchmarks/drivers/parse_log.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2024 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# !/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+
+import json
+import os
+from collections import defaultdict
+from loguru import logger
+
+
+def parse_log_file(spectflops, mode, warmup, log_dir, result_log_path):
+    log_file = os.path.join(log_dir, "result.log.txt")
+    save_log_path = os.path.join(result_log_path, "result.json")
+    if os.path.isfile(save_log_path):
+        with open(save_log_path, 'r+', encoding='utf-8') as file_r:
+            try:
+                file_r_json = file_r.read()
+                res = json.loads(file_r_json)
+                result_data = get_result_data(log_file, res, spectflops, mode, warmup)
+                file_r.seek(0)
+                file_r.write(json.dumps(result_data, ensure_ascii=False))
+                file_r.truncate()
+            except json.decoder.JSONDecodeError:
+                print("JSONDecodeError json file content is None")
+    else:
+        with open(save_log_path, 'w') as file_w:
+            res = defaultdict(dict)
+            result_data = get_result_data(log_file, res, spectflops, mode, warmup)
+            file_w.write(json.dumps(result_data, ensure_ascii=False))
+
+
+""" 参数说明
+# 时延：1 无预热时延 Latency-No warmup：no_warmup_latency，2 预热时延 Latency-Warmup：warmup_latency
+# 吞吐率：3 Raw-Throughput原始吞吐：raw_throughput， 4 Core-Throughput是核心吞吐：core_throughput
+# 算力：5 实际算力开销：ctflops， 6 实际算力利用率：cfu， 7 实际算力开销-内核时间：ktflops， 8 实际算力利用率-内核时间：kfu
+"""
+def get_result_data(log_file, res, spectflops, mode, warmup):
+    with open(log_file, 'r') as file_r:
+        lines = file_r.readlines()
+        for line in lines:
+            if line.startswith("[INFO]"):
+                json_data = line[6:].strip()
+                try:
+                    data = json.loads(json_data)
+                    op_name = data.get("op_name")
+                    dtype = data.get("dtype")
+                    results = data.get("result")
+                    for result in results:
+                        shape_detail = result.get("shape_detail")
+                        latency_base = result.get("latency_base")
+                        if mode == "cpu" and warmup == "0":
+                            no_warmup_latency = result.get("latency")
+                            parse_data = {
+                                "op_name": op_name,
+                                "dtype": dtype,
+                                "shape_detail": shape_detail,
+                                "latency_base_cpu_nowarm": latency_base,
+                                "no_warmup_latency": no_warmup_latency
+                            }
+                            res[f"{op_name}_{dtype}_{shape_detail}"].update(parse_data)
+                        elif mode == "cpu" and warmup != "0":
+                            warmup_latency = result.get("latency")
+                            raw_throughput = 1 / float(warmup_latency)
+                            ctflops = result.get("tflops")
+                            if ctflops is None:
+                                cfu = None
+                            else:
+                                cfu = round(100.0 * float(ctflops) / 1e12 / float(spectflops), 2)
+                            parse_data = {
+                                "op_name": op_name,
+                                "dtype": dtype,
+                                "shape_detail": shape_detail,
+                                "latency_base_cpu_warm": latency_base,
+                                "warmup_latency": warmup_latency,
+                                "raw_throughput": raw_throughput,
+                                "ctflops": ctflops,
+                                "cfu": cfu
+                            }
+                            res[f"{op_name}_{dtype}_{shape_detail}"].update(parse_data)
+                        elif mode == "cuda" and warmup != "0":
+                            kerneltime = result.get("latency")
+                            core_throughput = 1 / float(kerneltime)
+                            ktflops = result.get("tflops")
+                            if ktflops is None:
+                                kfu = None
+                            else:
+                                kfu = round(100.0 * float(ktflops) / 1E12 / float(spectflops), 2)
+                            parse_data = {
+                                "op_name": op_name,
+                                "dtype": dtype,
+                                "shape_detail": shape_detail,
+                                "latency_base_cuda_warm": latency_base,
+                                "kerneltime": kerneltime,
+                                "core_throughput": core_throughput,
+                                "ktflops": ktflops,
+                                "kfu": kfu
+                            }
+                            res[f"{op_name}_{dtype}_{shape_detail}"].update(parse_data)
+                except json.JSONDecodeError as e:
+                    logger.error(f"Error decoding JSON: {e}")
+        return res

operation/benchmarks/opv2/main.py

@@ -0,0 +1,128 @@
+ # Copyright (c) 2024 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+import torch
+import os
+import time
+from argparse import ArgumentParser, Namespace
+import yaml
+import sys
+import subprocess
+
+sys.path.append("..")
+from drivers.utils import *
+from drivers.calculate import *
+from drivers.parse_log import *
+
+
+def parse_args():
+    parser = ArgumentParser(description=" ")
+
+    parser.add_argument("--vendor",
+                        type=str,
+                        required=True,
+                        help="vendor name like nvidia")
+    parser.add_argument("--case_name",
+                        type=str,
+                        required=True,
+                        help="op name like mm")
+    parser.add_argument("--spectflops",
+                        type=str,
+                        required=True,
+                        help="spectflops of current dataformat")
+
+    parser.add_argument("--dataformat",
+                        type=str,
+                        required=True,
+                        help="like FP32,FP16")
+
+    parser.add_argument("--oplib",
+                        type=str,
+                        required=True,
+                        help="impl like pytorch/flaggems/cpp")
+
+    parser.add_argument("--chip",
+                        type=str,
+                        required=True,
+                        help="chip like A100_40_SXM")
+
+    parser.add_argument("--mode",
+                        type=str,
+                        required=True,
+                        help="mode like cpu")
+
+    parser.add_argument("--warmup",
+                        type=str,
+                        required=True,
+                        help="warmup")
+
+    parser.add_argument("--log_dir",
+                        type=str,
+                        required=True,
+                        help="abs log dir")
+
+    parser.add_argument("--result_log_path",
+                        type=str,
+                        required=True,
+                        help="result log path for FlagPerf/operation/result")
+
+    args, unknown_args = parser.parse_known_args()
+    args.unknown_args = unknown_args
+    return args
+
+
+def main(config):
+    correctness = do_correctness(config.case_name)
+    correctness = correctness == 0
+
+    # test operation performance
+    performance = do_performance(config.mode, config.warmup, config.log_dir)
+    performance = performance == 0
+    parse_log_file(config.spectflops, config.mode, config.warmup, config.log_dir, config.result_log_path)
+
+    # dtype = {
+    #     "FP32": torch.float32,
+    #     "FP16": torch.float16,
+    #     "BF16": torch.bfloat16,
+    #     "INT32": torch.int32,
+    #     "INT16": torch.int16,
+    #     "BOOL": torch.bool
+    #     }
+    # set_ieee_float32(config.vendor)
+    #
+    #
+    # m = case_config.Melements
+    #
+    #
+    # a = torch.randn(m, 1024, 1024, dtype=dtype[config.dataformat]).to(0)
+    #
+    # latency_nowarm, latency_warm, cputime, kerneltime = do_test(
+    #     torch.abs, (a, ), host_device_sync, config, case_config)
+    #
+    # op2flops = lambda x: x * m * 1024 * 1024
+    #
+    # perf_result = cal_perf(cputime, kerneltime, op2flops,
+    #                        config.spectflops)
+    # print_result(config, config.case_name, *perf_result, correctness,
+    #              latency_nowarm, latency_warm)
+
+if __name__ == "__main__":
+    config = parse_args()
+    # with open("case_config.yaml", "r") as file:
+    #     case_config = yaml.safe_load(file)
+    # adapt_torch(config.vendor)
+    # with open(os.path.join(config.vendor, config.chip, "case_config.yaml"),
+    #           "r") as file:
+    #     case_config_vendor = yaml.safe_load(file)
+    # case_config.update(case_config_vendor)
+    # case_config = Namespace(**case_config)
+
+    if config.oplib == "flaggems":
+        import flag_gems
+        flag_gems.enable()
+        print("Using flaggems")
+    else:
+        print("Using nativetorch")
+    main(config)

operation/configs/host.yaml

@@ -30,8 +30,11 @@ CLEAR_CACHES: True
 ACCE_VISIBLE_DEVICE_ENV_NAME: "CUDA_VISIBLE_DEVICES"
 # "operation:dataFormat:chip": "docker_images"
 # now only support flaggems and nativepytorch
+MODE: "cpu"
+WARMUP: 1000
 CASES: 
-    "mm:FP16:312:nativetorch:A100_40_SXM": "ngctorch2403"
+#    "mm:FP16:312:nativetorch:A100_40_SXM": "ngctorch2403"
+    "opv2:mm:FP16:312:flaggems:A100_40_SXM": "ngctorch2403"
 #    "mm:FP16:flaggems:A100_40_SXM": "ngctorch2403"
 #    "mm:FP16:nativetorch:A100_40_SXM": "ngctorch2403"
 #    'exp:FP32:nativetorch:R300p" : "xpytorch029"

operation/container_main.py

@@ -39,6 +39,16 @@ def parse_args():
                         required=True,
                         help="vendor name like nvidia")
 
+    parser.add_argument("--mode",
+                        type=str,
+                        required=True,
+                        help="mode like cpu")
+
+    parser.add_argument("--warmup",
+                        type=str,
+                        required=True,
+                        help="warmup")
+
     parser.add_argument("--log_level",
                         type=str,
                         required=True,
@@ -63,6 +73,11 @@ def parse_args():
                         required=True,
                         help="abs path for FlagPerf/base")
 
+    parser.add_argument("--result_log_path",
+                        type=str,
+                        required=True,
+                        help="result log path for FlagPerf/operation/result")
+
     args, unknown_args = parser.parse_known_args()
     args.unknown_args = unknown_args
     return args
@@ -96,21 +111,22 @@ def write_pid_file(pid_file_path, pid_file):
     logger.info("Success Writing PID file at " +
                 os.path.join(config.log_dir, "start_base_task.pid"))
 
-    op, dataformat, spectflops, oplib, chip = config.case_name.split(":")
+    test_file, op, dataformat, spectflops, oplib, chip = config.case_name.split(":")
 
-    case_dir = os.path.join(config.perf_path, "benchmarks", op)
+    case_dir = os.path.join(config.perf_path, "benchmarks", test_file)
     start_cmd = "cd " + case_dir + ";python3 main.py "
     start_cmd += " --vendor=" + config.vendor
     start_cmd += " --case_name=" + op
     start_cmd += " --spectflops=" + spectflops
     start_cmd += " --dataformat=" + dataformat
     start_cmd += " --oplib=" + oplib
     start_cmd += " --chip=" + chip
-
+    start_cmd += " --mode=" + config.mode
+    start_cmd += " --warmup=" + config.warmup
+    start_cmd += " --log_dir=" + config.log_dir
+    start_cmd += " --result_log_path=" + config.result_log_path
     script_log_file = os.path.join(os.path.dirname(logfile),
                                    "operation.log.txt")
-
-    logger.info(start_cmd)
     logger.info(script_log_file)
 
     f = open(script_log_file, "w")

operation/result_data_format/formatMDfile.py

@@ -0,0 +1,21 @@
+import os
+
+
+def render(extracted_values, readme_file_path, vendor, shm_size, chip):
+    json_data = []
+    for key, value in extracted_values.items():
+        json_data.append(value)
+    dest_file_path = os.path.join(readme_file_path, "README.md")
+    markdown_table = creat_markdown_table(json_data, vendor, shm_size, chip)
+    with open(dest_file_path, 'w') as file:
+        file.write(markdown_table)
+
+
+def creat_markdown_table(data, vendor, shm_size, chip):
+    v_chip = f'{vendor}_{chip}'
+    table = f"# 参评AI芯片信息\n\n * 厂商：{vendor}\n * 产品名称：{v_chip}\n * 产品型号：{chip}\n * SHM_SIZE：{shm_size}\n\n\n\n"
+    table += "# 评测结果\n\n"
+    table += "| op_name | dtype | shape_detail | 无预热时延(Latency-No warmup) | 预热时延(Latency-Warmup) | 原始吞吐(Raw-Throughput)| 核心吞吐(Core-Throughput) | 实际算力开销 | 实际算力利用率 | 实际算力开销(内核时间) | 实际算力利用率(内核时间) |\n| --- | ---| --- | ---| --- | ---| --- | ---| --- | ---| --- |\n"
+    for row in data:
+        table += f"| {row['op_name']} | {row['dtype']} | {row['shape_detail']} | {row['no_warmup_latency']} | {row['warmup_latency']} | {row['raw_throughput']} | {row['core_throughput']} | {row['ctflops']} | {row['cfu']} | {row['ktflops']} | {row['kfu']} |\n"
+    return table