diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml index 62dbe4fd4a1bc..5f20c0e99ded2 100644 --- a/devops/actions/run-tests/benchmark/action.yml +++ b/devops/actions/run-tests/benchmark/action.yml @@ -161,7 +161,8 @@ runs: python3 ./devops/scripts/benchmarks/compare.py to_hist \ --name "$SAVE_NAME" \ --compare-file "./llvm-ci-perf-results/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \ - --results-dir "./llvm-ci-perf-results/results/" + --results-dir "./llvm-ci-perf-results/results/" \ + --regression-filter '^[a-z_]_sycl ' echo "-----" - name: Cache changes to benchmark folder for archival purposes diff --git a/devops/scripts/benchmarks/compare.py b/devops/scripts/benchmarks/compare.py index eea5e450e6729..31b31f05f368d 100644 --- a/devops/scripts/benchmarks/compare.py +++ b/devops/scripts/benchmarks/compare.py @@ -4,6 +4,7 @@ from options import options import os +import re import sys import json import argparse @@ -319,6 +320,12 @@ def to_hist( help="Timestamp (in YYYYMMDD_HHMMSS) of oldest result to include in historic average calculation", default="20000101_010101", ) + parser_avg.add_argument( + "--regression-filter", + type=str, + help="If provided, only regressions matching provided regex will cause exit status 1.", + default=None, + ) args = parser.parse_args() @@ -333,24 +340,40 @@ def to_hist( "median", args.name, args.compare_file, args.results_dir, args.cutoff ) + # Not all regressions are of concern: if a filter is provided, filter + # regressions using filter + regressions_ignored = [] + regressions_of_concern = [] + if args.regression_filter is not None: + filter_pattern = re.compile(args.regression_filter) + for test in regressions: + if filter_pattern.search(test["name"]): + regressions_of_concern.append(test) + else: + regressions_ignored.append(test) + def print_regression(entry: dict): """Print an entry outputted from Compare.to_hist""" print(f"Test: {entry['name']}") print(f"-- Historic {entry['avg_type']}: {entry['hist_avg']}") - print(f"-- Run result: {test['value']}") - print(f"-- Delta: {test['delta']}") + print(f"-- Run result: {entry['value']}") + print(f"-- Delta: {entry['delta']}") print("") if improvements: print("#\n# Improvements:\n#\n") for test in improvements: print_regression(test) - if regressions: + if regressions_ignored: + print("#\n# Regressions (filtered out by regression-filter):\n#\n") + for test in regressions_ignored: + print_regression(test) + if regressions_of_concern: print("#\n# Regressions:\n#\n") - for test in regressions: + for test in regressions_of_concern: print_regression(test) exit(1) # Exit 1 to trigger github test failure - print("\nNo regressions found!") + print("\nNo unexpected regressions found!") else: print("Unsupported operation: exiting.") exit(1)