diff --git a/benchmarking/run_benchmark.py b/benchmarking/run_benchmark.py index 731e6db..3829370 100644 --- a/benchmarking/run_benchmark.py +++ b/benchmarking/run_benchmark.py @@ -7,6 +7,7 @@ import shutil import subprocess import sys +import pandas as pd from typing import TypedDict @@ -92,6 +93,9 @@ class BenchmarkingResult(TypedDict): "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED", ] +# List of data structures representing the results of a benchmark +results: list[BenchmarkingResult] = [] + # Arguments DEBUG = False # Debug Mode MODULE = "All" # Refactoring Module to use @@ -179,9 +183,6 @@ def stage_one(): """ datasets_list = os.listdir(DATASETS_REFACTORED_DIR) - # List of data structures representing the results of a benchmark - results: list[BenchmarkingResult] = [] - for dataset in datasets_list: print(f"Benchmarking {dataset}...") os.makedirs(f"{OUTPUT_DIR}/{dataset}", exist_ok=True) @@ -309,7 +310,13 @@ def stage_one_refactor(dataset: str): output_file = f"{OUTPUT_DIR}/{dataset}/refactoring.txt" dataset_path = f"{DATASETS_REFACTORED_DIR}/{dataset}" - refactor_cmd: list[str] = ["./gradlew", "run", f"--args='{dataset_path} {MODULE}'"] + debugArg = "--info" if DEBUG else "" + + refactor_cmd: list[str] = [ + "./gradlew", + "run", + f"{debugArg} --args='{dataset_path} {MODULE}'", + ] with open(output_file, "w+") as f: res = subprocess.run( @@ -367,6 +374,50 @@ def stage_one_count_errors(dataset: str): return error_count +def stage_two(): + print("Summarizing results...:") + + benchmark_results = pd.DataFrame(results) + + benchmark_results["error_reduction"] = ( + benchmark_results["initial_error_count"] + - benchmark_results["refactored_error_count"] + ) + avg_diff = benchmark_results["error_reduction"].mean() + + benchmark_results["error_reduction_percent"] = benchmark_results.apply( + lambda row: ( + (row["error_reduction"] / row["initial_error_count"]) * 100 + if row["initial_error_count"] != 0 + else 0 + ), + axis=1, + ) + + benchmark_results["error_reduction_percent"] = ( + benchmark_results["error_reduction_percent"] + .astype("Float64") + .replace([float("inf"), -float("inf")], pd.NA) + ) + + # Weighted percent reduction across all benchmarks + total_initial_errors = benchmark_results["initial_error_count"].sum() + total_errors_reduced = benchmark_results["error_reduction"].sum() + weighted_percent_reduction = ( + (total_errors_reduced / total_initial_errors) * 100 + if total_initial_errors != 0 + else 0 + ) + + print("SUMMARY OF RESULTS:") + print(f"AVERAGE ERROR REDUCTION: {avg_diff}") + print(f"WEIGHTED ERROR REDUCTION (PERCENT): {weighted_percent_reduction:.2f}%") + print( + f"BENCHMARKS WITH LARGEST PERCENT REDUCTION): \n{benchmark_results.sort_values(by='error_reduction_percent', ascending=False).head(10)}" + ) + benchmark_results.to_csv(f"{OUTPUT_DIR}/summary.csv") + + def get_build_cmd(dataset: str): """ Constructs the full 'javac' build command used to compile the passed dataset. @@ -472,11 +523,13 @@ def run(): """ stage_zero() stage_one() + stage_two() def main(): """Main entry point of the script.""" global DEBUG + global MODULE argparser = argparse.ArgumentParser(description="Runs benchmark.") argparser.add_argument( "--debug", action="store_true", help="Enabling debugging statements."