From fc0d67550a3f0743ca0cf8b1b528724dc02ab4f1 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Tue, 16 Apr 2024 16:52:14 +0000 Subject: [PATCH 01/11] preliminary ir printing in e2e --- projects/pt1/e2e_testing/main.py | 15 +++++++++++++++ .../configs/linalg_on_tensors_backend.py | 6 +++++- .../torch_mlir_e2e_test/configs/onnx_backend.py | 8 ++++++-- .../pt1/python/torch_mlir_e2e_test/framework.py | 1 + 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index d2c381d654bc..ad9e045349a6 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -4,6 +4,7 @@ # Also available under a BSD-style license. See LICENSE. import argparse +import logging import re import sys @@ -75,6 +76,10 @@ def _get_argparse(): default=False, action="store_true", help="report test results with additional detail") + parser.add_argument("--print-ir", + default=False, + action="store_true", + help="Set logging level to DEBUG and causes the IR to be printed for each test.") parser.add_argument("-s", "--sequential", default=False, action="store_true", @@ -93,6 +98,16 @@ def _get_argparse(): def main(): args = _get_argparse().parse_args() + logger = logging.getLogger("e2e_test") + if args.print_ir: + print("Setting logging level to DEBUG and enabling IR printing.") + print("This currently only affects the Linalg-on-Tensors and onnx configs.") + print("Work in progress. See https://github.com/llvm/torch-mlir/issues/3172") + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.WARNING) + + all_test_unique_names = set( test.unique_name for test in GLOBAL_TEST_REGISTRY) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py index 8c99278b0ec3..922b8e17fae4 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py @@ -2,6 +2,8 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # Also available under a BSD-style license. See LICENSE. +import logging +logger = logging.getLogger("e2e_test") from typing import Any @@ -32,7 +34,9 @@ def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile( program, example_args, output_type="linalg-on-tensors") - + logger.debug("LinalgOnTensorsBackendTestConfig compiled module:") + logger.debug(module) + logger.debug("End LinalgOnTensorsBackendTestConfig compiled module") return self.backend.compile(module) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py index e411a7cbb67f..6d1c99abb508 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py @@ -22,7 +22,8 @@ from torch_mlir.extras import onnx_importer from torch_mlir.dialects import torch as torch_d from torch_mlir.ir import Context, Module - +import logging +logger = logging.getLogger("e2e_test") def import_onnx(contents): # Import the ONNX model proto from the file contents: @@ -39,7 +40,7 @@ def import_onnx(contents): return m -def convert_onnx(model, inputs): +def convert_onnx(model: torch.nn.Module, inputs): buffer = io.BytesIO() # Process the type information so we export with the dynamic shape information @@ -82,6 +83,9 @@ def __init__(self, backend: OnnxBackend, use_make_fx: bool = False): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) onnx_module = convert_onnx(program, example_args) + logger.debug("OnnxBackendTestConfig imported module:") + logger.debug(onnx_module) + logger.debug("End OnnxBackendTestConfig imported module") compiled_module = self.backend.compile(onnx_module) return compiled_module diff --git a/projects/pt1/python/torch_mlir_e2e_test/framework.py b/projects/pt1/python/torch_mlir_e2e_test/framework.py index d3fecf54d99c..8515b578e1ea 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/framework.py +++ b/projects/pt1/python/torch_mlir_e2e_test/framework.py @@ -143,6 +143,7 @@ def compile(self, program: torch.nn.Module) -> CompiledArtifact: """Compile the provided torch.nn.Module into a compiled artifact""" pass + # Any should match result of `compile`. @abc.abstractmethod From 3558aecd017c8f1195e19cb2b37c21a49da8c149 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Tue, 16 Apr 2024 17:01:03 +0000 Subject: [PATCH 02/11] change printir message to be more informative --- projects/pt1/e2e_testing/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index ad9e045349a6..f2a567a70a72 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -100,9 +100,10 @@ def main(): logger = logging.getLogger("e2e_test") if args.print_ir: - print("Setting logging level to DEBUG and enabling IR printing.") - print("This currently only affects the Linalg-on-Tensors and onnx configs.") - print("Work in progress. See https://github.com/llvm/torch-mlir/issues/3172") + print("WARNING: --print-ir is a work in progress feature.") + print("print-ir: Setting logging level to DEBUG and enabling IR printing.") + print("print-ir: This currently only affects the Linalg-on-Tensors and onnx configs.") + print("print-ir: Work in progress. See https://github.com/llvm/torch-mlir/issues/3172") logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.WARNING) From d990e52dd0ef3e893080afdc929602bfe9df5a56 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Thu, 18 Apr 2024 21:56:46 +0000 Subject: [PATCH 03/11] rename ir printing logger + add to stablehlo_backend --- projects/pt1/e2e_testing/main.py | 7 ++++--- .../configs/linalg_on_tensors_backend.py | 8 ++++---- .../python/torch_mlir_e2e_test/configs/onnx_backend.py | 8 ++++---- .../torch_mlir_e2e_test/configs/stablehlo_backend.py | 6 +++++- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index f2a567a70a72..113907b68fde 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -98,15 +98,16 @@ def _get_argparse(): def main(): args = _get_argparse().parse_args() - logger = logging.getLogger("e2e_test") + ir_printer = logging.getLogger("ir_printer") if args.print_ir: print("WARNING: --print-ir is a work in progress feature.") print("print-ir: Setting logging level to DEBUG and enabling IR printing.") print("print-ir: This currently only affects the Linalg-on-Tensors and onnx configs.") print("print-ir: Work in progress. See https://github.com/llvm/torch-mlir/issues/3172") - logger.setLevel(logging.DEBUG) + ir_printer.setLevel(logging.DEBUG) else: - logger.setLevel(logging.WARNING) + # disable logging + ir_printer.setLevel(logging.CRITICAL+1) all_test_unique_names = set( diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py index 922b8e17fae4..d5656f740302 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # Also available under a BSD-style license. See LICENSE. import logging -logger = logging.getLogger("e2e_test") +ir_printer = logging.getLogger("ir_printer") from typing import Any @@ -34,9 +34,9 @@ def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile( program, example_args, output_type="linalg-on-tensors") - logger.debug("LinalgOnTensorsBackendTestConfig compiled module:") - logger.debug(module) - logger.debug("End LinalgOnTensorsBackendTestConfig compiled module") + ir_printer.debug("LinalgOnTensorsBackendTestConfig compiled module:") + ir_printer.debug(module) + ir_printer.debug("End LinalgOnTensorsBackendTestConfig compiled module") return self.backend.compile(module) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py index 6d1c99abb508..deb91464f7c5 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py @@ -23,7 +23,7 @@ from torch_mlir.dialects import torch as torch_d from torch_mlir.ir import Context, Module import logging -logger = logging.getLogger("e2e_test") +ir_printer = logging.getLogger("ir_printer") def import_onnx(contents): # Import the ONNX model proto from the file contents: @@ -83,9 +83,9 @@ def __init__(self, backend: OnnxBackend, use_make_fx: bool = False): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) onnx_module = convert_onnx(program, example_args) - logger.debug("OnnxBackendTestConfig imported module:") - logger.debug(onnx_module) - logger.debug("End OnnxBackendTestConfig imported module") + ir_printer.debug("OnnxBackendTestConfig imported module:") + ir_printer.debug(onnx_module) + ir_printer.debug("End OnnxBackendTestConfig imported module") compiled_module = self.backend.compile(onnx_module) return compiled_module diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py index 1ab8a8d22b4f..ec1b86e54709 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py @@ -2,6 +2,8 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # Also available under a BSD-style license. See LICENSE. +import logging +ir_printer = logging.getLogger("ir_printer") from typing import Any @@ -31,7 +33,9 @@ def __init__(self, backend: StablehloBackend): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile(program, example_args, output_type="stablehlo") - + ir_printer.debug("StablehloBackendTestConfig compiled module:") + ir_printer.debug(module) + ir_printer.debug("End StablehloBackendTestConfig compiled module") return self.backend.compile(module) def run(self, artifact: Any, trace: Trace) -> Trace: From 069e575201c9e84fb71f0844247bef4d629ed275 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Fri, 19 Apr 2024 00:06:54 +0000 Subject: [PATCH 04/11] more extensive use of logger; add args for setting log level --- projects/pt1/e2e_testing/main.py | 52 +++++++++---------- .../configs/linalg_on_tensors_backend.py | 6 +-- .../configs/onnx_backend.py | 6 +-- .../configs/stablehlo_backend.py | 6 +-- .../python/torch_mlir_e2e_test/reporting.py | 15 +++--- 5 files changed, 39 insertions(+), 46 deletions(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index 113907b68fde..5a589e4be302 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -72,14 +72,11 @@ def _get_argparse(): parser.add_argument("-f", "--filter", default=".*", help=""" Regular expression specifying which tests to include in this run. """) - parser.add_argument("-v", "--verbose", - default=False, - action="store_true", - help="report test results with additional detail") - parser.add_argument("--print-ir", - default=False, - action="store_true", - help="Set logging level to DEBUG and causes the IR to be printed for each test.") + parser.add_argument("--log_level", default="WARNING", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="set the log level") + parser.add_argument("-d", "--debug", action="store_const", dest="log_level", const="DEBUG", help="set log level to DEBUG for detailed debug output") + parser.add_argument("-v", "--verbose", action="store_const", dest="log_level", const="INFO", help="set log level to INFO to report a more detailed but still user-friendly level of verbosity") + parser.add_argument("-q", "--quiet", action="store_const", dest="log_level", const="ERROR", help="suppress all logs except errors") + parser.add_argument("-s", "--sequential", default=False, action="store_true", @@ -89,7 +86,7 @@ def _get_argparse(): parser.add_argument("--crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed", metavar="TEST", type=str, nargs="+", help="A set of tests to not attempt to run, since they crash and cannot be XFAILed.") - parser.add_argument("--ignore_failures", + parser.add_argument("--ignore_familures", default=False, action="store_true", help="return exit code 0 even if the test fails to unblock pipeline") @@ -98,17 +95,13 @@ def _get_argparse(): def main(): args = _get_argparse().parse_args() - ir_printer = logging.getLogger("ir_printer") - if args.print_ir: - print("WARNING: --print-ir is a work in progress feature.") - print("print-ir: Setting logging level to DEBUG and enabling IR printing.") - print("print-ir: This currently only affects the Linalg-on-Tensors and onnx configs.") - print("print-ir: Work in progress. See https://github.com/llvm/torch-mlir/issues/3172") - ir_printer.setLevel(logging.DEBUG) - else: - # disable logging - ir_printer.setLevel(logging.CRITICAL+1) - + logger = logging.getLogger() # use root logger by default. Easy to change later. + logger.setLevel(logging.NOTSET) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(args.log_level) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) all_test_unique_names = set( test.unique_name for test in GLOBAL_TEST_REGISTRY) @@ -160,7 +153,7 @@ def main(): if args.crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed is not None: for arg in args.crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed: if arg not in all_test_unique_names: - print(f"ERROR: --crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed argument '{arg}' is not a valid test name") + logger.error(f"ERROR: --crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed argument '{arg}' is not a valid test name") sys.exit(1) # Find the selected tests, and emit a diagnostic if none are found. @@ -169,21 +162,24 @@ def main(): if re.match(args.filter, test.unique_name) ] if len(tests) == 0: - print( + logger.error( f"ERROR: the provided filter {args.filter!r} does not match any tests" ) - print("The available tests are:") - for test in available_tests: - print(test.unique_name) + available_tests = [test for test in available_tests] + + logger.error("The available tests are:" + "\n".join(available_tests)) sys.exit(1) # Run the tests. - results = run_tests(tests, config, args.sequential, args.verbose) + results = run_tests(tests, config, args.sequential, + verbose=logger.level >= logging.INFO) # Report the test results. - failed = report_results(results, xfail_set, args.verbose, args.config) + failed = report_results(results, xfail_set, + verbose=logger.level >= logging.INFO, + config=args.config) if args.config == "torchdynamo": - print("\033[91mWarning: the TorchScript based dynamo support is deprecated. " + logger.warning("\033[91mWarning: the TorchScript based dynamo support is deprecated. " "The config for torchdynamo is planned to be removed in the future.\033[0m") if args.ignore_failures: sys.exit(0) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py index d5656f740302..57f8d930cd07 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # Also available under a BSD-style license. See LICENSE. import logging -ir_printer = logging.getLogger("ir_printer") +logger = logging.getLogger() from typing import Any @@ -34,9 +34,7 @@ def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile( program, example_args, output_type="linalg-on-tensors") - ir_printer.debug("LinalgOnTensorsBackendTestConfig compiled module:") - ir_printer.debug(module) - ir_printer.debug("End LinalgOnTensorsBackendTestConfig compiled module") + logger.debug(module) return self.backend.compile(module) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py index deb91464f7c5..9cc0bc0d099d 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py @@ -23,7 +23,7 @@ from torch_mlir.dialects import torch as torch_d from torch_mlir.ir import Context, Module import logging -ir_printer = logging.getLogger("ir_printer") +logger = logging.getLogger() def import_onnx(contents): # Import the ONNX model proto from the file contents: @@ -83,9 +83,7 @@ def __init__(self, backend: OnnxBackend, use_make_fx: bool = False): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) onnx_module = convert_onnx(program, example_args) - ir_printer.debug("OnnxBackendTestConfig imported module:") - ir_printer.debug(onnx_module) - ir_printer.debug("End OnnxBackendTestConfig imported module") + logger.debug(onnx_module) compiled_module = self.backend.compile(onnx_module) return compiled_module diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py index ec1b86e54709..3b7df92160f9 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # Also available under a BSD-style license. See LICENSE. import logging -ir_printer = logging.getLogger("ir_printer") +logger = logging.getLogger() from typing import Any @@ -33,9 +33,7 @@ def __init__(self, backend: StablehloBackend): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile(program, example_args, output_type="stablehlo") - ir_printer.debug("StablehloBackendTestConfig compiled module:") - ir_printer.debug(module) - ir_printer.debug("End StablehloBackendTestConfig compiled module") + logger.debug(module) return self.backend.compile(module) def run(self, artifact: Any, trace: Trace) -> Trace: diff --git a/projects/pt1/python/torch_mlir_e2e_test/reporting.py b/projects/pt1/python/torch_mlir_e2e_test/reporting.py index 61fc7caf9166..e29f01a81a44 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/reporting.py +++ b/projects/pt1/python/torch_mlir_e2e_test/reporting.py @@ -6,6 +6,9 @@ Utilities for reporting the results of the test framework. """ +from logging import getLogger +logger = getLogger() + from typing import Any, List, Optional, Set import collections @@ -292,17 +295,17 @@ def report_results(results: List[TestResult], expected_failure = result.unique_name in expected_failures if expected_failure: if report.failed: - print(f'XFAIL - "{result.unique_name}"') + logger.info(f'XFAIL - "{result.unique_name}"') results_by_outcome['XFAIL'].append((result, report)) else: - print(f'XPASS - "{result.unique_name}"') + logger.info(f'XPASS - "{result.unique_name}"') results_by_outcome['XPASS'].append((result, report)) else: if not report.failed: - print(f'PASS - "{result.unique_name}"') + logger.inf(f'PASS - "{result.unique_name}"') results_by_outcome['PASS'].append((result, report)) else: - print(f'FAIL - "{result.unique_name}"') + logger.info(f'FAIL - "{result.unique_name}"') results_by_outcome['FAIL'].append((result, report)) OUTCOME_MEANINGS = collections.OrderedDict() @@ -329,8 +332,8 @@ def report_results(results: List[TestResult], for result, report in results: print(f' {outcome} - "{result.unique_name}"') # If the test failed, print the error message. - if outcome == 'FAIL' and verbose: - print(textwrap.indent(report.error_str(), ' ' * 8)) + if outcome == 'FAIL': + logger.info(textwrap.indent(report.error_str(), ' ' * 8)) # Print a summary for easy scanning. print('\nSummary:') From 97ee3cf67321b93bf6f8fe11ca45561477fa0489 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Fri, 19 Apr 2024 00:17:41 +0000 Subject: [PATCH 05/11] remove unneeded newline --- projects/pt1/python/torch_mlir_e2e_test/framework.py | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/pt1/python/torch_mlir_e2e_test/framework.py b/projects/pt1/python/torch_mlir_e2e_test/framework.py index 8515b578e1ea..d3fecf54d99c 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/framework.py +++ b/projects/pt1/python/torch_mlir_e2e_test/framework.py @@ -143,7 +143,6 @@ def compile(self, program: torch.nn.Module) -> CompiledArtifact: """Compile the provided torch.nn.Module into a compiled artifact""" pass - # Any should match result of `compile`. @abc.abstractmethod From dd50041a54a5837425918d62c60fba99ea6a26fe Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Fri, 19 Apr 2024 00:23:31 +0000 Subject: [PATCH 06/11] typo --- projects/pt1/python/torch_mlir_e2e_test/reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/pt1/python/torch_mlir_e2e_test/reporting.py b/projects/pt1/python/torch_mlir_e2e_test/reporting.py index e29f01a81a44..3bd96745f4ba 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/reporting.py +++ b/projects/pt1/python/torch_mlir_e2e_test/reporting.py @@ -302,7 +302,7 @@ def report_results(results: List[TestResult], results_by_outcome['XPASS'].append((result, report)) else: if not report.failed: - logger.inf(f'PASS - "{result.unique_name}"') + logger.info(f'PASS - "{result.unique_name}"') results_by_outcome['PASS'].append((result, report)) else: logger.info(f'FAIL - "{result.unique_name}"') From ff6874590b18661e32a47a26d58fb833f1d99edf Mon Sep 17 00:00:00 2001 From: "Xida Ren (Cedar)" Date: Fri, 19 Apr 2024 13:17:55 -0700 Subject: [PATCH 07/11] Update main.py typo --- projects/pt1/e2e_testing/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index 5a589e4be302..46b7361a08a8 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -86,7 +86,7 @@ def _get_argparse(): parser.add_argument("--crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed", metavar="TEST", type=str, nargs="+", help="A set of tests to not attempt to run, since they crash and cannot be XFAILed.") - parser.add_argument("--ignore_familures", + parser.add_argument("--ignore_failures", default=False, action="store_true", help="return exit code 0 even if the test fails to unblock pipeline") From 1cfc09727af765ef4a30d858221579617ded853e Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Wed, 24 Apr 2024 18:50:44 +0000 Subject: [PATCH 08/11] address some comments from @scotttodd --- projects/pt1/e2e_testing/main.py | 12 +++++------- .../configs/linalg_on_tensors_backend.py | 2 +- .../torch_mlir_e2e_test/configs/onnx_backend.py | 2 +- .../torch_mlir_e2e_test/configs/stablehlo_backend.py | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index 46b7361a08a8..3eacf11f85ca 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -153,7 +153,7 @@ def main(): if args.crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed is not None: for arg in args.crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed: if arg not in all_test_unique_names: - logger.error(f"ERROR: --crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed argument '{arg}' is not a valid test name") + logger.error(f"--crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed argument '{arg}' is not a valid test name") sys.exit(1) # Find the selected tests, and emit a diagnostic if none are found. @@ -161,13 +161,11 @@ def main(): test for test in available_tests if re.match(args.filter, test.unique_name) ] + available_tests = [test.unique_name for test in available_tests] if len(tests) == 0: logger.error( - f"ERROR: the provided filter {args.filter!r} does not match any tests" + f"the provided filter {args.filter!r} does not match any tests. The available tests are:\n" + "\n\t".join(available_tests)" ) - available_tests = [test for test in available_tests] - - logger.error("The available tests are:" + "\n".join(available_tests)) sys.exit(1) # Run the tests. @@ -179,8 +177,8 @@ def main(): verbose=logger.level >= logging.INFO, config=args.config) if args.config == "torchdynamo": - logger.warning("\033[91mWarning: the TorchScript based dynamo support is deprecated. " - "The config for torchdynamo is planned to be removed in the future.\033[0m") + logger.warning("the TorchScript based dynamo support is deprecated. " + "The config for torchdynamo is planned to be removed in the future.") if args.ignore_failures: sys.exit(0) sys.exit(1 if failed else 0) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py index 57f8d930cd07..92c5b3f5016c 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/linalg_on_tensors_backend.py @@ -34,7 +34,7 @@ def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile( program, example_args, output_type="linalg-on-tensors") - logger.debug(module) + logger.debug("MLIR produced by LinalgOnTensorsBackendTestConfig:\n" + str(module)) return self.backend.compile(module) diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py index 9cc0bc0d099d..9dd5d23f9d8a 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/onnx_backend.py @@ -83,7 +83,7 @@ def __init__(self, backend: OnnxBackend, use_make_fx: bool = False): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) onnx_module = convert_onnx(program, example_args) - logger.debug(onnx_module) + logger.debug("MLIR produced by OnnxBackendTestConfig:\n" + str(onnx_module)) compiled_module = self.backend.compile(onnx_module) return compiled_module diff --git a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py index 3b7df92160f9..fc88e180e6a5 100644 --- a/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py +++ b/projects/pt1/python/torch_mlir_e2e_test/configs/stablehlo_backend.py @@ -33,7 +33,7 @@ def __init__(self, backend: StablehloBackend): def compile(self, program: torch.nn.Module) -> Any: example_args = convert_annotations_to_placeholders(program.forward) module = torchscript.compile(program, example_args, output_type="stablehlo") - logger.debug(module) + logger.debug("MLIR produced by StablehloBackendTestConfig:\n" + str(module)) return self.backend.compile(module) def run(self, artifact: Any, trace: Trace) -> Trace: From f3bb7b5fa727c6347990c80bd0fa9a4a11594465 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Wed, 24 Apr 2024 18:56:33 +0000 Subject: [PATCH 09/11] add loglevel in formatter; add line number and file for debug messages --- projects/pt1/e2e_testing/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index 3eacf11f85ca..d34b9f243a55 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -99,7 +99,11 @@ def main(): logger.setLevel(logging.NOTSET) handler = logging.StreamHandler(sys.stdout) handler.setLevel(args.log_level) - formatter = logging.Formatter('%(message)s') + if args.log_level != "DEBUG": + fmt = "%(levelname)s: %(message)s" + else: + fmt = "%(levelname)s: %(filename)s:%(lineno)d:\n%(message)s" + formatter = logging.Formatter(fmt) handler.setFormatter(formatter) logger.addHandler(handler) From 1797aacaccdf532c38a1cd0528a704e0a43944e5 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Thu, 25 Apr 2024 21:31:22 +0000 Subject: [PATCH 10/11] uppercase log level to accomodate user input of either case --- projects/pt1/e2e_testing/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index d34b9f243a55..f9a1eeb42c34 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -94,6 +94,7 @@ def _get_argparse(): def main(): args = _get_argparse().parse_args() + args.log_level = args.log_level.upper() logger = logging.getLogger() # use root logger by default. Easy to change later. logger.setLevel(logging.NOTSET) From 0fd918ec631d70ae8abe45407cba6397001c0b72 Mon Sep 17 00:00:00 2001 From: Xida Ren Date: Fri, 26 Apr 2024 18:50:42 +0000 Subject: [PATCH 11/11] fix typo --- projects/pt1/e2e_testing/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/pt1/e2e_testing/main.py b/projects/pt1/e2e_testing/main.py index f9a1eeb42c34..010129a104ec 100644 --- a/projects/pt1/e2e_testing/main.py +++ b/projects/pt1/e2e_testing/main.py @@ -169,7 +169,7 @@ def main(): available_tests = [test.unique_name for test in available_tests] if len(tests) == 0: logger.error( - f"the provided filter {args.filter!r} does not match any tests. The available tests are:\n" + "\n\t".join(available_tests)" + f"the provided filter {args.filter!r} does not match any tests. The available tests are:\n\t" + "\n\t".join(available_tests) ) sys.exit(1)