From 7a4b8c133ca649c9939472c4db0394121e3bcc9c Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Tue, 10 Jun 2025 19:18:44 -0700 Subject: [PATCH 1/9] Fix image_exporter --- examples/perf_benchmark/batch_benchmark.py | 160 +++++----- examples/perf_benchmark/benchmark.py | 13 +- examples/perf_benchmark/benchmark_plotter.py | 309 ++++++------------- examples/rendering/demo.py | 21 +- examples/rigid/single_franka.py | 12 +- examples/rigid/single_franka_batch_render.py | 2 +- genesis/utils/image_exporter.py | 91 +++--- 7 files changed, 266 insertions(+), 342 deletions(-) diff --git a/examples/perf_benchmark/batch_benchmark.py b/examples/perf_benchmark/batch_benchmark.py index e4ba67227..e31849ae5 100644 --- a/examples/perf_benchmark/batch_benchmark.py +++ b/examples/perf_benchmark/batch_benchmark.py @@ -25,6 +25,7 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): # Create a list of all the possible combinations of arguments # and return them as a list of BenchmarkArgs full_mjcf_list = ["xml/franka_emika_panda/panda.xml", "xml/unitree_g1/g1.xml", "xml/unitree_go2/go2.xml"] + renderer_list = ["batch_renderer", "pyrender"] rasterizer_list = [True, False] full_batch_size_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384] square_resolution_list = [ @@ -39,12 +40,13 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): full_resolution_list = square_resolution_list + four_three_resolution_list + sixteen_nine_resolution_list # Minimal mjcf, resolution, and batch size + minimal_renderer_list = ["batch_renderer"] minimal_mjcf_list = [ "xml/franka_emika_panda/panda.xml" ] minimal_batch_size_list = [ #2048, 3072, 4096, 6144, 8192, 12288, 16384 - 2048, 4096, 8192, 16384 + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 ] #minimal_batch_size_list = full_batch_size_list minimal_resolution_list = [ @@ -75,16 +77,19 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): batch_args_dict = {} # Build hierarchical structure + for renderer in renderer_list: + batch_args_dict[renderer] = {} for rasterizer in rasterizer_list: - batch_args_dict[rasterizer] = {} - for mjcf in mjcf_list: - batch_args_dict[rasterizer][mjcf] = {} - for batch_size in batch_size_list: - batch_args_dict[rasterizer][mjcf][batch_size] = {} + batch_args_dict[renderer][rasterizer] = {} + for mjcf in mjcf_list: + batch_args_dict[renderer][rasterizer][mjcf] = {} + for batch_size in batch_size_list: + batch_args_dict[renderer][rasterizer][mjcf][batch_size] = {} for resolution in resolution_list: resX, resY = resolution # Create benchmark args for this combination args = BenchmarkArgs( + renderer_name=renderer, rasterizer=rasterizer, n_envs=batch_size, n_steps=n_steps, @@ -100,7 +105,7 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): mjcf=mjcf, benchmark_result_file_path=benchmark_result_file_path ) - batch_args_dict[rasterizer][mjcf][batch_size][(resX,resY)] = args + batch_args_dict[renderer][rasterizer][mjcf][batch_size][(resX,resY)] = args return batch_args_dict @@ -118,7 +123,7 @@ def create_benchmark_result_file(continue_from_file_path): benchmark_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") benchmark_result_file_path = f"{benchmark_data_directory}/batch_benchmark_{benchmark_timestamp}.csv" with open(benchmark_result_file_path, "w") as f: - f.write("result,mjcf,rasterizer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") + f.write("result,mjcf,renderer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") print(f"Created new benchmark result file: {benchmark_result_file_path}") return benchmark_result_file_path @@ -135,7 +140,7 @@ def get_previous_runs(continue_from_file_path): for _, row in df.iterrows(): run_info = ( row['mjcf'], - row['rasterizer'], + row['renderer'], row['n_envs'], (row['resX'], row['resY']), row['result'] # 'succeeded' or 'failed' @@ -144,67 +149,83 @@ def get_previous_runs(continue_from_file_path): return previous_runs -def run_batch_benchmark(batch_args_dict, benchmark_script_path, previous_runs=None): +def get_benchmark_script_path(renderer_name): + current_dir = os.path.dirname(os.path.abspath(__file__)) + if renderer_name == "batch_renderer": + return f"{current_dir}/benchmark.py" + elif renderer_name == "pyrender": + return f"{current_dir}/benchmark_pyrender.py" + else: + raise ValueError(f"Invalid renderer name: {renderer_name}") + +def run_batch_benchmark(batch_args_dict, previous_runs=None): if previous_runs is None: previous_runs = [] - for rasterizer in batch_args_dict: - for mjcf in batch_args_dict[rasterizer]: - for batch_size in batch_args_dict[rasterizer][mjcf]: - last_resolution_failed = False - for resolution in batch_args_dict[rasterizer][mjcf][batch_size]: - if last_resolution_failed: - break - - # Check if this run was in a previous execution - run_info = (mjcf, rasterizer, batch_size, resolution) - skip_this_run = False - - for prev_run in previous_runs: - if run_info == prev_run[:4]: # Compare only the run parameters, not the status - skip_this_run = True - if prev_run[4] == 'failed': - # Skip this and subsequent resolutions if it failed before - last_resolution_failed = True + for renderer in batch_args_dict: + benchmark_script_path = get_benchmark_script_path(renderer) + if not os.path.exists(benchmark_script_path): + raise FileNotFoundError(f"Benchmark script not found: {benchmark_script_path}") + print(f"Running benchmark for {renderer}") + + for rasterizer in batch_args_dict[renderer]: + for mjcf in batch_args_dict[renderer][rasterizer]: + for batch_size in batch_args_dict[renderer][rasterizer][mjcf]: + last_resolution_failed = False + for resolution in batch_args_dict[renderer][rasterizer][mjcf][batch_size]: + if last_resolution_failed: break - - if skip_this_run: - continue - # Run the benchmark - batch_args = batch_args_dict[rasterizer][mjcf][batch_size][resolution] - - # launch a process to run the benchmark - cmd = ["python3", benchmark_script_path] - if batch_args.rasterizer: - cmd.append("-r") - cmd.extend([ - "-n", str(batch_args.n_envs), - "-s", str(batch_args.n_steps), - "-x", str(batch_args.resX), - "-y", str(batch_args.resY), - "-i", str(batch_args.camera_posX), - "-j", str(batch_args.camera_posY), - "-k", str(batch_args.camera_posZ), - "-l", str(batch_args.camera_lookatX), - "-m", str(batch_args.camera_lookatY), - "-o", str(batch_args.camera_lookatZ), - "-v", str(batch_args.camera_fov), - "-f", batch_args.mjcf, - "-g", batch_args.benchmark_result_file_path - ]) - try: - process = subprocess.Popen(cmd) - return_code = process.wait() - if return_code != 0: - raise subprocess.CalledProcessError(return_code, cmd) - except Exception as e: - print(f"Error running benchmark: {str(e)}") - last_resolution_failed = True - # Write failed result without timing data - with open(batch_args.benchmark_result_file_path, 'a') as f: - f.write(f'failed,{batch_args.mjcf},{batch_args.rasterizer},{batch_args.n_envs},{batch_args.n_steps},{batch_args.resX},{batch_args.resY},{batch_args.camera_posX},{batch_args.camera_posY},{batch_args.camera_posZ},{batch_args.camera_lookatX},{batch_args.camera_lookatY},{batch_args.camera_lookatZ},{batch_args.camera_fov},,,,\n') - break + # Check if this run was in a previous execution + run_info = (mjcf, rasterizer, batch_size, resolution) + skip_this_run = False + + for prev_run in previous_runs: + if run_info == prev_run[:4]: # Compare only the run parameters, not the status + skip_this_run = True + if prev_run[4] == 'failed': + # Skip this and subsequent resolutions if it failed before + last_resolution_failed = True + break + + if skip_this_run: + continue + + # Run the benchmark + batch_args = batch_args_dict[renderer][rasterizer][mjcf][batch_size][resolution] + + # launch a process to run the benchmark + cmd = ["python3", benchmark_script_path] + if batch_args.rasterizer: + cmd.append("-r") + cmd.extend([ + "-d", batch_args.renderer_name, + "-n", str(batch_args.n_envs), + "-s", str(batch_args.n_steps), + "-x", str(batch_args.resX), + "-y", str(batch_args.resY), + "-i", str(batch_args.camera_posX), + "-j", str(batch_args.camera_posY), + "-k", str(batch_args.camera_posZ), + "-l", str(batch_args.camera_lookatX), + "-m", str(batch_args.camera_lookatY), + "-o", str(batch_args.camera_lookatZ), + "-v", str(batch_args.camera_fov), + "-f", batch_args.mjcf, + "-g", batch_args.benchmark_result_file_path + ]) + try: + process = subprocess.Popen(cmd) + return_code = process.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, cmd) + except Exception as e: + print(f"Error running benchmark: {str(e)}") + last_resolution_failed = True + # Write failed result without timing data + with open(batch_args.benchmark_result_file_path, 'a') as f: + f.write(f'failed,{batch_args.mjcf},{batch_args.renderer_name},{batch_args.rasterizer},{batch_args.n_envs},{batch_args.n_steps},{batch_args.resX},{batch_args.resY},{batch_args.camera_posX},{batch_args.camera_posY},{batch_args.camera_posZ},{batch_args.camera_lookatX},{batch_args.camera_lookatY},{batch_args.camera_lookatZ},{batch_args.camera_fov},,,,\n') + break def main(): batch_benchmark_args = parse_args() @@ -213,14 +234,9 @@ def main(): # Get list of previous runs if continuing from a previous run previous_runs = get_previous_runs(batch_benchmark_args.continue_from) - # Run benchmark in batch - current_dir = os.path.dirname(os.path.abspath(__file__)) - benchmark_script_path = f"{current_dir}/benchmark.py" - if not os.path.exists(benchmark_script_path): - raise FileNotFoundError(f"Benchmark script not found: {benchmark_script_path}") - + # Run benchmark in batch batch_args_dict = create_batch_args(benchmark_result_file_path, use_full_list=batch_benchmark_args.use_full_list) - run_batch_benchmark(batch_args_dict, benchmark_script_path, previous_runs) + run_batch_benchmark(batch_args_dict, previous_runs) # Generate plots plot_batch_benchmark(benchmark_result_file_path) diff --git a/examples/perf_benchmark/benchmark.py b/examples/perf_benchmark/benchmark.py index 9c474ebea..9159026f6 100644 --- a/examples/perf_benchmark/benchmark.py +++ b/examples/perf_benchmark/benchmark.py @@ -7,7 +7,8 @@ # Create a struct to store the arguments class BenchmarkArgs: - def __init__(self, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): + def __init__(self, renderer_name, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): + self.renderer_name = renderer_name self.rasterizer = rasterizer self.n_envs = n_envs self.n_steps = n_steps @@ -25,6 +26,7 @@ def __init__(self, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_ def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument("-d", "--renderer_name", type=str, default="rasterizer") parser.add_argument("-r", "--rasterizer", action="store_true", default=False) parser.add_argument("-n", "--n_envs", type=int, default=1024) parser.add_argument("-s", "--n_steps", type=int, default=1) @@ -41,6 +43,7 @@ def parse_args(): parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") args = parser.parse_args() benchmark_args = BenchmarkArgs( + renderer_name=args.renderer_name, rasterizer=args.rasterizer, n_envs=args.n_envs, n_steps=args.n_steps, @@ -57,6 +60,8 @@ def parse_args(): benchmark_result_file_path=args.benchmark_result_file_path, ) print(f"Benchmark with args:") + print(f" renderer_name: {benchmark_args.renderer_name}") + print(f" script: {benchmark_args.script}") print(f" rasterizer: {benchmark_args.rasterizer}") print(f" n_envs: {benchmark_args.n_envs}") print(f" n_steps: {benchmark_args.n_steps}") @@ -155,7 +160,7 @@ def run_benchmark(scene, benchmark_args): # warmup scene.step() - rgb, depth, _, _ = scene.batch_render() + rgb, depth, _, _ = scene.render_all_cams() # fill gpu cache with random data fill_gpu_cache_with_random_data() @@ -165,7 +170,7 @@ def run_benchmark(scene, benchmark_args): start_time = time() for i in range(n_steps): - rgb, depth, _, _ = scene.batch_render(force_render=True) + rgb, depth, _, _ = scene.render_all_cams(force_render=True) end_time = time() time_taken = end_time - start_time @@ -183,7 +188,7 @@ def run_benchmark(scene, benchmark_args): # Append a line with all args and results in csv format with open(benchmark_args.benchmark_result_file_path, 'a') as f: - f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') + f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.renderer_name},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') except Exception as e: print(f"Error during benchmark: {e}") raise diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index acfcbf720..12d1d326b 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -17,11 +17,11 @@ def generatePlotHtml(plots_dir): return # Separate regular plots from difference plots - regular_plots = [p for p in plot_files if not p.endswith('_difference.png') and not any(p.endswith(f'_difference_{ar.replace(":", "x")}.png') for ar in ["1:1", "4:3", "16:9"])] + regular_plots = [p for p in plot_files if not p.endswith('_comparison.png')] aspect_ratio_plots = { - "1:1": [p for p in plot_files if p.endswith('_difference_1x1.png')], - "4:3": [p for p in plot_files if p.endswith('_difference_4x3.png')], - "16:9": [p for p in plot_files if p.endswith('_difference_16x9.png')] + "1:1": [p for p in plot_files if p.endswith('_1x1_comparison.png')], + "4:3": [p for p in plot_files if p.endswith('_4x3_comparison.png')], + "16:9": [p for p in plot_files if p.endswith('_16x9_comparison.png')] } # Group regular plots by MJCF file @@ -70,29 +70,30 @@ def generatePlotHtml(plots_dir):

Benchmark Results

""" + # Add aspect ratio difference plots sections + for aspect_ratio, mjcf_groups in aspect_ratio_groups.items(): + if mjcf_groups: + html_content += "
\n" + html_content += f"

Performance Comparison Plots ({aspect_ratio} Resolutions Only)

\n" + for mjcf_name, plots in mjcf_groups: + html_content += f"
\n" + for plot in plots: + html_content += f"

{html.escape(mjcf_name)} - {os.path.basename(plot)}

\n" + html_content += f"{html.escape(os.path.basename(plot))}
\n" + html_content += "
\n" + html_content += "
\n" + # Add regular plots section html_content += "
\n" html_content += "

Performance Plots

\n" for mjcf_name, plots in plot_groups: html_content += f"
\n" for plot in plots: - html_content += f"

{html.escape(mjcf_name)} - {'Rasterizer' if 'rasterizer' in os.path.basename(plot) else 'Raytracer'}

\n" + html_content += f"

{html.escape(mjcf_name)} - {os.path.basename(plot)}

\n" html_content += f"{html.escape(os.path.basename(plot))}
\n" html_content += "
\n" html_content += "
\n" - # Add aspect ratio difference plots sections - for aspect_ratio, mjcf_groups in aspect_ratio_groups.items(): - if mjcf_groups: - html_content += "
\n" - html_content += f"

Performance Difference Plots (Rasterizer / Raytracer) ({aspect_ratio} Resolutions Only)

\n" - for mjcf_name, plots in mjcf_groups: - html_content += f"
\n" - html_content += f"

{html.escape(mjcf_name)}

\n" - html_content += f"{html.escape(os.path.basename(plots[0]))}
\n" - html_content += "
\n" - html_content += "
\n" - html_content += """ @@ -124,7 +125,7 @@ def plot_batch_benchmark(data_file_path, width=20, height=15): # Generate difference plots for specific aspect ratios for aspect_ratio in ["1:1", "4:3", "16:9"]: - generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=aspect_ratio) + generate_comparison_plots(df, plots_dir, width, height, ("batch_renderer", True), ("batch_renderer", False), aspect_ratio=aspect_ratio) # Generate an html page to display all the plots generatePlotHtml(plots_dir) @@ -132,181 +133,71 @@ def plot_batch_benchmark(data_file_path, width=20, height=15): def generate_individual_plots(df, plots_dir, width, height): # Get unique combinations of mjcf and rasterizer for mjcf in df['mjcf'].unique(): - for rasterizer in df[df['mjcf'] == mjcf]['rasterizer'].unique(): - # Filter data for this mjcf and rasterizer - data = df[(df['mjcf'] == mjcf) & (df['rasterizer'] == rasterizer)] - - # continue if there is no data - if len(data) == 0: - print(f"No data found for {mjcf} and {rasterizer}") - continue - - # Create new figure - plt.figure(figsize=(width, height)) - - # Plot each resolution with different color - # Sort by resX and resY before plotting - for res in sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])): - resolution = res[0] - res_data = res[1] - plt.plot(res_data['n_envs'], res_data['fps'], - marker='o', label=f'{resolution[0]}x{resolution[1]}') + for renderer in df[df['mjcf'] == mjcf]['renderer'].unique(): + for rasterizer in df[df['mjcf'] == mjcf]['rasterizer'].unique(): + # Filter data for this mjcf and rasterizer + data = df[(df['mjcf'] == mjcf) & (df['renderer'] == renderer) & (df['rasterizer'] == rasterizer)] + + # continue if there is no data + if len(data) == 0: + print(f"No data found for {mjcf} and {renderer} and rasterizer:{rasterizer}") + continue - # Add x/y value annotations near each point - for x, y in zip(res_data['n_envs'], res_data['fps']): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) - - # Customize plot - plt.title(f'Performance for {os.path.basename(mjcf)}\n{"Rasterizer" if rasterizer else "Raytracer"}') - plt.xlabel('Batch Size') - plt.ylabel('FPS') - plt.grid(True) - plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('log') - - # Force both axes to use ScalarFormatter - # Get current axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) - - # Save plot - rasterizer_str = "rasterizer" if rasterizer else "raytracer" - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{rasterizer_str}.png" - plt.savefig(filename) - plt.close() - -def generate_difference_plots(df, plots_dir, width, height, aspect_ratio=None): - # Filter by aspect ratio if specified - if aspect_ratio: - if aspect_ratio == "1:1": - df = df[df['resX'] == df['resY']] - elif aspect_ratio == "4:3": - df = df[df['resX'] * 3 == df['resY'] * 4] - elif aspect_ratio == "16:9": - df = df[df['resX'] * 9 == df['resY'] * 16] - else: - raise ValueError(f"Unsupported aspect ratio: {aspect_ratio}") - - plt.clf() - plt.cla() - - # Generate plots showing fps difference between rasterizer and raytracer - for mjcf in df['mjcf'].unique(): - mjcf_data = df[df['mjcf'] == mjcf] - - # Get resolutions available for both rasterizer and raytracer - rasterizer_res = set(zip(mjcf_data[mjcf_data['rasterizer']]['resX'], - mjcf_data[mjcf_data['rasterizer']]['resY'])) - raytracer_res = set(zip(mjcf_data[~mjcf_data['rasterizer']]['resX'], - mjcf_data[~mjcf_data['rasterizer']]['resY'])) - common_res = rasterizer_res.intersection(raytracer_res) - - # continue if there is no data - if len(common_res) == 0: - print(f"No data found for {mjcf}") - continue - - plt.figure(figsize=(width, height)) - - # Plot difference for each common resolution - max_abs_diff = 0 # Track maximum absolute difference for y-axis limits - min_abs_diff = 1e10 - for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): - rast_data = mjcf_data[(mjcf_data['rasterizer']) & - (mjcf_data['resX'] == resX) & - (mjcf_data['resY'] == resY)] - ray_data = mjcf_data[(~mjcf_data['rasterizer']) & - (mjcf_data['resX'] == resX) & - (mjcf_data['resY'] == resY)] - - # Match batch sizes and calculate difference - common_batch = set(rast_data['n_envs']).intersection(set(ray_data['n_envs'])) - batch_sizes = sorted(list(common_batch)) - - rast_fps = rast_data[rast_data['n_envs'].isin(batch_sizes)]['fps'].values - ray_fps = ray_data[ray_data['n_envs'].isin(batch_sizes)]['fps'].values - diff_fps = rast_fps - ray_fps - - # Update max absolute difference - max_abs_diff = max(max_abs_diff, diff_fps.max()) - min_abs_diff = min(min_abs_diff, diff_fps.min()) - - plt.plot(batch_sizes, diff_fps, marker='o', label=f'{resX}x{resY}') - - # Add annotations - for x, y in zip(batch_sizes, diff_fps): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) - - # Set title based on aspect ratio - subtitle1 = "FPS Difference (Rasterizer - Raytracer)" - if aspect_ratio: - subtitle2 = f"({aspect_ratio} Resolutions Only)" - else: - subtitle2 = "" - plt.title(f'{subtitle1}\n{os.path.basename(mjcf)} {subtitle2}') - plt.xlabel('Batch Size') - plt.ylabel(subtitle1) - plt.grid(True) - plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('symlog', linthresh=1.0) # Use symlog with linear threshold of 1.0 - - # Format axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) - - # Set y-axis limits symmetrically - ax.set_ylim(0, max_abs_diff*1.1) # Add 10% padding - - # Add horizontal line at y=0 to show crossover point - plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) - - # Save plot with aspect ratio in filename if specified - if aspect_ratio: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference_{aspect_ratio.replace(':', 'x')}.png" - else: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference.png" - plt.savefig(filename) - plt.close() + # Create new figure + plt.figure(figsize=(width, height)) + + # Plot each resolution with different color + # Sort by resX and resY before plotting + for res in sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])): + resolution = res[0] + res_data = res[1] + plt.plot(res_data['n_envs'], res_data['fps'], + marker='o', label=f'{resolution[0]}x{resolution[1]}') + + # Add x/y value annotations near each point + for x, y in zip(res_data['n_envs'], res_data['fps']): + plt.annotate(f'({x:.0f}, {y:.1f})', + (x, y), + xytext=(5, 5), + textcoords='offset points', + fontsize=8) + + # Customize plot + plt.title(f'Performance for {os.path.basename(mjcf)}\n{renderer} {"Rasterizer" if rasterizer else "Raytracer"}') + plt.xlabel('Batch Size') + plt.ylabel('FPS') + plt.grid(True) + plt.legend(title='Resolution') + plt.xscale('log') + plt.yscale('log') + + # Force both axes to use ScalarFormatter + # Get current axes + ax = plt.gca() + + # Set major formatter for both axes + ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) + ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) -def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): + # Disable scientific notation + ax.ticklabel_format(axis='both', style='plain') + + # Add more tick marks + ax.xaxis.set_major_locator(plt.LogLocator(base=2)) + ax.yaxis.set_major_locator(plt.LogLocator(base=10)) + + # Add minor tick marks + ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) + ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) + + # Save plot + filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer}_{'rasterizer' if rasterizer else 'raytracer'}.png" + plt.savefig(filename) + plt.close() + +def generate_comparison_plots(df, plots_dir, width, height, renderer_1, renderer_2, aspect_ratio=None): + renderer_1_name, renderer_1_is_rasterizer = renderer_1 + renderer_2_name, renderer_2_is_rasterizer = renderer_2 # Filter by aspect ratio if specified if aspect_ratio: if aspect_ratio == "1:1": @@ -321,16 +212,16 @@ def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): plt.clf() plt.cla() - # Generate plots showing fps difference between rasterizer and raytracer + # Generate plots showing fps difference between renderer_1 and renderer_2 for mjcf in df['mjcf'].unique(): mjcf_data = df[df['mjcf'] == mjcf] - # Get resolutions available for both rasterizer and raytracer - rasterizer_res = set(zip(mjcf_data[mjcf_data['rasterizer']]['resX'], - mjcf_data[mjcf_data['rasterizer']]['resY'])) - raytracer_res = set(zip(mjcf_data[~mjcf_data['rasterizer']]['resX'], - mjcf_data[~mjcf_data['rasterizer']]['resY'])) - common_res = rasterizer_res.intersection(raytracer_res) + # Get resolutions available for both renderer_1 and renderer_2 + renderer_1_res = set(zip(mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer)]['resX'], + mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer)]['resY'])) + renderer_2_res = set(zip(mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & (mjcf_data['rasterizer'] == renderer_2_is_rasterizer)]['resX'], + mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & (mjcf_data['rasterizer'] == renderer_2_is_rasterizer)]['resY'])) + common_res = renderer_1_res.intersection(renderer_2_res) # continue if there is no data if len(common_res) == 0: @@ -343,20 +234,22 @@ def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): max_abs_diff = 0 # Track maximum absolute difference for y-axis limits min_abs_diff = 1e10 for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): - rast_data = mjcf_data[(mjcf_data['rasterizer']) & + renderer_1_data = mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & + (mjcf_data['rasterizer'] == renderer_1_is_rasterizer) & (mjcf_data['resX'] == resX) & (mjcf_data['resY'] == resY)] - ray_data = mjcf_data[(~mjcf_data['rasterizer']) & + renderer_2_data = mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & + (mjcf_data['rasterizer'] == renderer_2_is_rasterizer) & (mjcf_data['resX'] == resX) & (mjcf_data['resY'] == resY)] # Match batch sizes and calculate difference - common_batch = set(rast_data['n_envs']).intersection(set(ray_data['n_envs'])) + common_batch = set(renderer_1_data['n_envs']).intersection(set(renderer_2_data['n_envs'])) batch_sizes = sorted(list(common_batch)) - rast_fps = rast_data[rast_data['n_envs'].isin(batch_sizes)]['fps'].values - ray_fps = ray_data[ray_data['n_envs'].isin(batch_sizes)]['fps'].values - diff_fps = rast_fps / ray_fps + renderer_1_fps = renderer_1_data[renderer_1_data['n_envs'].isin(batch_sizes)]['fps'].values + renderer_2_fps = renderer_2_data[renderer_2_data['n_envs'].isin(batch_sizes)]['fps'].values + diff_fps = renderer_1_fps / renderer_2_fps # Update max absolute difference max_abs_diff = max(max_abs_diff, diff_fps.max()) @@ -373,7 +266,7 @@ def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): fontsize=8) # Set title based on aspect ratio - subtitle1 = "FPS Difference (Rasterizer / Raytracer)" + subtitle1 = f"FPS Difference ({renderer_1_name} {rasterizer_1_str} / {renderer_2_name} {rasterizer_2_str})" if aspect_ratio: subtitle2 = f"({aspect_ratio} Resolutions Only)" else: @@ -412,10 +305,12 @@ def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) # Save plot with aspect ratio in filename if specified + rasterizer_1_str = 'rasterizer' if renderer_1_is_rasterizer else 'raytracer' + rasterizer_2_str = 'rasterizer' if renderer_2_is_rasterizer else 'raytracer' if aspect_ratio: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference_{aspect_ratio.replace(':', 'x')}.png" + filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{aspect_ratio.replace(':', 'x')}_comparison.png" else: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference.png" + filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_comparison.png" plt.savefig(filename) plt.close() diff --git a/examples/rendering/demo.py b/examples/rendering/demo.py index 6ada02c89..23709ebff 100644 --- a/examples/rendering/demo.py +++ b/examples/rendering/demo.py @@ -1,6 +1,7 @@ import torch import genesis as gs +from genesis.utils.image_exporter import FrameImageExporter def main(): @@ -16,18 +17,7 @@ def main(): camera_lookat=(3.0, 0.0, 0.5), camera_fov=50, ), - renderer=gs.renderers.RayTracer( # type: ignore - env_surface=gs.surfaces.Emission( - emissive_texture=gs.textures.ImageTexture( - image_path="textures/indoor_bright.png", - ), - ), - env_radius=15.0, - env_euler=(0, 0, 180), - lights=[ - {"pos": (0.0, 0.0, 10.0), "radius": 3.0, "color": (15.0, 15.0, 15.0)}, - ], - ), + renderer=gs.renderers.Rasterizer(), ) ########################## materials ########################## @@ -156,9 +146,14 @@ def main(): scene.reset() horizon = 2000 + # Create an image exporter + output_dir = 'img_output/test' + exporter = FrameImageExporter(output_dir) + for i in range(horizon): scene.step() - cam_0.render() + rgb, depth, _, _ = cam_0.render() + exporter.export_frame_single_cam(i, cam_0.idx, rgb=rgb, depth=depth) if __name__ == "__main__": diff --git a/examples/rigid/single_franka.py b/examples/rigid/single_franka.py index 14cf1f0f8..57e184d28 100644 --- a/examples/rigid/single_franka.py +++ b/examples/rigid/single_franka.py @@ -1,6 +1,7 @@ import argparse import genesis as gs +from genesis.utils.image_exporter import FrameImageExporter def main(): @@ -32,7 +33,7 @@ def main(): ) franka = scene.add_entity( gs.morphs.MJCF(file="xml/franka_emika_panda/panda.xml"), - visualize_contact=True, + visualize_contact=False, ) ########################## cameras ########################## @@ -45,9 +46,14 @@ def main(): ) ########################## build ########################## scene.build() - for i in range(1000): + + # Create an image exporter + output_dir = 'img_output/test' + exporter = FrameImageExporter(output_dir) + for i in range(2): scene.step() - # cam_0.render() + rgb, depth, _, _ = cam_0.render() + exporter.export_frame_single_cam(i, cam_0.idx, rgb=rgb, depth=depth) if __name__ == "__main__": diff --git a/examples/rigid/single_franka_batch_render.py b/examples/rigid/single_franka_batch_render.py index 774df8354..36b1dbf4b 100644 --- a/examples/rigid/single_franka_batch_render.py +++ b/examples/rigid/single_franka_batch_render.py @@ -29,7 +29,7 @@ def main(): # constraint_solver=gs.constraint_solver.Newton, ), renderer = gs.options.renderers.BatchRenderer( - use_rasterizer=False, + use_rasterizer=True, batch_render_res=(512, 512), ) ) diff --git a/genesis/utils/image_exporter.py b/genesis/utils/image_exporter.py index daea24c58..45cc845c7 100644 --- a/genesis/utils/image_exporter.py +++ b/genesis/utils/image_exporter.py @@ -11,17 +11,17 @@ def _export_frame_rgb_cam(export_dir, i_env, i_cam, camera_name, i_step, rgb): cv2.imwrite(f'{export_dir}/rgb_env{i_env}_{camera_name}_{i_step:03d}.png', rgb) @staticmethod - def _export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth_normalized): - depth_normalized = depth_normalized[i_env, i_cam].cpu().numpy() - cv2.imwrite(f'{export_dir}/depth_env{i_env}_{camera_name}_{i_step:03d}.png', depth_normalized) + def _export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth): + depth = depth[i_env, i_cam].cpu().numpy() + cv2.imwrite(f'{export_dir}/depth_env{i_env}_{camera_name}_{i_step:03d}.png', depth) @staticmethod def _worker_export_frame_cam(args): - export_dir, i_env, i_cam, camera_name, rgb, depth_normalized, i_step = args + export_dir, i_env, i_cam, camera_name, rgb, depth, i_step = args if rgb is not None: FrameImageExporter._export_frame_rgb_cam(export_dir, i_env, i_cam, camera_name, i_step, rgb) - if depth_normalized is not None: - FrameImageExporter._export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth_normalized) + if depth is not None: + FrameImageExporter._export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth) def __init__(self, export_dir, depth_clip_max=100, depth_scale='log'): self.export_dir = export_dir @@ -66,20 +66,24 @@ def export_frame_all_cams(self, i_step, camera_idx=None, rgb=None, depth=None): rgb: RGB image tensor of shape (n_envs, n_cams, H, W, 3). depth: Depth tensor of shape (n_envs, n_cams, H, W). """ + if rgb is None and depth is None: + print("No rgb or depth to export") + return - if rgb.ndim == 4: - rgb = rgb.unsqueeze(0) - if depth.ndim == 4: - depth = depth.unsqueeze(0) - assert rgb.ndim == 5 and depth.ndim == 5, "rgb and depth must be of shape (n_envs, n_cams, H, W, 3)" + if rgb is not None: + if rgb.ndim == 4: + rgb = rgb.unsqueeze(0) + assert rgb.ndim == 5, "rgb must be of shape (n_envs, n_cams, H, W, 3)" + if depth is not None: + if depth.ndim == 4: + depth = depth.unsqueeze(0) + depth = self._normalize_depth(depth) + assert depth.ndim == 5, "depth must be of shape (n_envs, n_cams, H, W, 1)" if camera_idx is None: - camera_idx = range(rgb.shape[1]) - env_idx = range(rgb.shape[0]) - - depth_normalized = self._normalize_depth(depth) if depth is not None else None - - args_list = [(self.export_dir, i_env, i_cam, self._get_camera_name(i_cam), rgb, depth_normalized, i_step) + camera_idx = range(rgb.shape[1]) if rgb is not None else range(depth.shape[1]) + env_idx = range(rgb.shape[0]) if rgb is not None else range(depth.shape[0]) + args_list = [(self.export_dir, i_env, i_cam, self._get_camera_name(i_cam), rgb, depth, i_step) for i_env in env_idx for i_cam in camera_idx] with ThreadPoolExecutor() as executor: executor.map(FrameImageExporter._worker_export_frame_cam, args_list) @@ -94,32 +98,35 @@ def export_frame_single_cam(self, i_step, i_cam, rgb=None, depth=None): rgb: RGB image tensor of shape (n_envs, H, W, 3). depth: Depth tensor of shape (n_envs, H, W). """ - # Move rgb and depth to torch tensor - if isinstance(rgb, np.ndarray): - rgb = torch.from_numpy(rgb.copy()) - if isinstance(depth, np.ndarray): - depth = torch.from_numpy(depth.copy()) - - # Unsqueeze rgb and depth to (n_envs, 1, H, W, 3) and (n_envs, 1, H, W, 1) - if rgb.ndim == 4: - rgb = rgb.unsqueeze(1) - elif rgb.ndim == 3: - rgb = rgb.unsqueeze(0).unsqueeze(0) - else: - raise ValueError(f"Invalid rgb shape: {rgb.shape}") - - if depth.ndim == 4: - depth = depth.unsqueeze(1) - elif depth.ndim == 3: - depth = depth.unsqueeze(0).unsqueeze(0) - else: - raise ValueError(f"Invalid depth shape: {depth.shape}") - assert rgb.ndim == 5 and depth.ndim == 5, "rgb and depth must be of shape (n_envs, n_cams, H, W, 3)" + if rgb is not None: + if isinstance(rgb, np.ndarray): + rgb = torch.from_numpy(rgb.copy()) - env_idx = range(rgb.shape[0]) - depth_normalized = self._normalize_depth(depth) if depth is not None else None - - args_list = [(self.export_dir, i_env, 0, self._get_camera_name(i_cam), rgb, depth_normalized, i_step) + # Unsqueeze rgb to (n_envs, 1, H, W, 3) if n_envs > 0 + if rgb.ndim == 4: + rgb = rgb.unsqueeze(1) + elif rgb.ndim == 3: + rgb = rgb.unsqueeze(0).unsqueeze(0) + else: + raise ValueError(f"Invalid rgb shape: {rgb.shape}") + assert rgb.ndim == 5, "rgb must be of shape (n_envs, H, W, 3)" + + if depth is not None: + if isinstance(depth, np.ndarray): + depth = torch.from_numpy(depth.copy()) + + # Unsqueeze depth to (n_envs, 1, H, W, 1) if n_envs > 0 + if depth.ndim == 4: + depth = depth.unsqueeze(1) + elif depth.ndim == 3: + depth = depth.unsqueeze(0).unsqueeze(0) + else: + raise ValueError(f"Invalid depth shape: {depth.shape}") + depth = self._normalize_depth(depth) + assert depth.ndim == 5, "depth must be of shape (n_envs, H, W, 1)" + + env_idx = range(rgb.shape[0]) if rgb is not None else range(depth.shape[0]) + args_list = [(self.export_dir, i_env, 0, self._get_camera_name(i_cam), rgb, depth, i_step) for i_env in env_idx] with ThreadPoolExecutor() as executor: executor.map(FrameImageExporter._worker_export_frame_cam, args_list) From 20cfb954666ecf95edcb96ffa9f05e9e16b57edd Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Tue, 10 Jun 2025 19:24:54 -0700 Subject: [PATCH 2/9] Add pyrender benchmark script --- examples/perf_benchmark/benchmark_pyrender.py | 179 ++++++++++++++++++ examples/rendering/demo.py | 21 +- examples/rigid/single_franka.py | 12 +- examples/rigid/single_franka_batch_render.py | 2 +- 4 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 examples/perf_benchmark/benchmark_pyrender.py diff --git a/examples/perf_benchmark/benchmark_pyrender.py b/examples/perf_benchmark/benchmark_pyrender.py new file mode 100644 index 000000000..14f84f4b2 --- /dev/null +++ b/examples/perf_benchmark/benchmark_pyrender.py @@ -0,0 +1,179 @@ +import argparse +import os + +import numpy as np +import genesis as gs +import torch + +# Create a struct to store the arguments +class BenchmarkArgs: + def __init__(self, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): + self.rasterizer = rasterizer + self.n_envs = n_envs + self.n_steps = n_steps + self.resX = resX + self.resY = resY + self.camera_posX = camera_posX + self.camera_posY = camera_posY + self.camera_posZ = camera_posZ + self.camera_lookatX = camera_lookatX + self.camera_lookatY = camera_lookatY + self.camera_lookatZ = camera_lookatZ + self.camera_fov = camera_fov + self.mjcf = mjcf + self.benchmark_result_file_path = benchmark_result_file_path + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--rasterizer", action="store_true", default=False) + parser.add_argument("-n", "--n_envs", type=int, default=1024) + parser.add_argument("-s", "--n_steps", type=int, default=1) + parser.add_argument("-x", "--resX", type=int, default=1024) + parser.add_argument("-y", "--resY", type=int, default=1024) + parser.add_argument("-i", "--camera_posX", type=float, default=1.5) + parser.add_argument("-j", "--camera_posY", type=float, default=0.5) + parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) + parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) + parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) + parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) + parser.add_argument("-v", "--camera_fov", type=float, default=45) + parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") + parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") + args = parser.parse_args() + benchmark_args = BenchmarkArgs( + rasterizer=args.rasterizer, + n_envs=args.n_envs, + n_steps=args.n_steps, + resX=args.resX, + resY=args.resY, + camera_posX=args.camera_posX, + camera_posY=args.camera_posY, + camera_posZ=args.camera_posZ, + camera_lookatX=args.camera_lookatX, + camera_lookatY=args.camera_lookatY, + camera_lookatZ=args.camera_lookatZ, + camera_fov=args.camera_fov, + mjcf=args.mjcf, + benchmark_result_file_path=args.benchmark_result_file_path, + ) + print(f"Benchmark with args:") + print(f" rasterizer: {benchmark_args.rasterizer}") + print(f" n_envs: {benchmark_args.n_envs}") + print(f" n_steps: {benchmark_args.n_steps}") + print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") + print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") + print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") + print(f" camera_fov: {benchmark_args.camera_fov}") + print(f" mjcf: {benchmark_args.mjcf}") + print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") + return benchmark_args + +def init_gs(benchmark_args): + ########################## init ########################## + try: + gs.init(backend=gs.gpu) + except Exception as e: + print(f"Failed to initialize GPU backend: {e}") + print("Falling back to CPU backend") + gs.init(backend=gs.cpu) + + ########################## create a scene ########################## + scene = gs.Scene( + viewer_options=gs.options.ViewerOptions( + camera_pos=(benchmark_args.camera_posX, benchmark_args.camera_posY, benchmark_args.camera_posZ), + camera_lookat=(benchmark_args.camera_lookatX, benchmark_args.camera_lookatY, benchmark_args.camera_lookatZ), + camera_fov=benchmark_args.camera_fov, + ), + vis_options=gs.options.VisOptions( + lights=[ + {"type": "directional", "dir": (1.0, 1.0, -2.0), "color": (1.0, 1.0, 1.0), "intensity": 0.5}, + {"type": "point", "pos": (4, -4, 4), "color": (1.0, 1.0, 1.0), "intensity": 1}, + ], + ), + show_viewer=False, + rigid_options=gs.options.RigidOptions( + # constraint_solver=gs.constraint_solver.Newton, + ), + renderer = benchmark_args.rasterizer and gs.options.renderers.Rasterizer() or gs.options.renderers.RayTracer() + ) + + ########################## entities ########################## + plane = scene.add_entity( + gs.morphs.Plane(), + ) + franka = scene.add_entity( + gs.morphs.MJCF(file=benchmark_args.mjcf), + visualize_contact=False, + ) + + ########################## cameras ########################## + cam_0 = scene.add_camera( + res=(benchmark_args.resX, benchmark_args.resY), + pos=(benchmark_args.camera_posX, benchmark_args.camera_posY, benchmark_args.camera_posZ), + lookat=(benchmark_args.camera_lookatX, benchmark_args.camera_lookatY, benchmark_args.camera_lookatZ), + fov=benchmark_args.camera_fov, + ) + ########################## build ########################## + scene.build() + return scene + +def fill_gpu_cache_with_random_data(): + # 100 MB of random data + dummy_data =torch.rand(100, 1024, 1024, device="cuda") + # Make some random data manipulation to the entire tensor + dummy_data = dummy_data.sqrt() + +def run_benchmark(scene, benchmark_args): + try: + n_envs = benchmark_args.n_envs + n_steps = benchmark_args.n_steps + + # warmup + scene.step() + rgb, depth, _, _ = scene.visualizer.cameras[0].render(rgb=True, depth=True) + + # fill gpu cache with random data + fill_gpu_cache_with_random_data() + + # timer + from time import time + start_time = time() + + for i in range(n_steps): + for i_env in range(n_envs): + rgb, depth, _, _ = scene.visualizer.cameras[0].render(rgb=True, depth=True) + + end_time = time() + time_taken = end_time - start_time + time_taken_per_env = time_taken / n_envs + fps = n_envs * n_steps / time_taken + fps_per_env = n_steps / time_taken + + print(f'Time taken: {time_taken} seconds') + print(f'Time taken per env: {time_taken_per_env} seconds') + print(f'FPS: {fps}') + print(f'FPS per env: {fps_per_env}') + + # Ensure the directory exists + os.makedirs(os.path.dirname(benchmark_args.benchmark_result_file_path), exist_ok=True) + + # Append a line with all args and results in csv format + with open(benchmark_args.benchmark_result_file_path, 'a') as f: + f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') + except Exception as e: + print(f"Error during benchmark: {e}") + raise + +def main(): + ######################## Parse arguments ####################### + benchmark_args = parse_args() + + ######################## Initialize scene ####################### + scene = init_gs(benchmark_args) + + ######################## Run benchmark ####################### + run_benchmark(scene, benchmark_args) + +if __name__ == "__main__": + main() + diff --git a/examples/rendering/demo.py b/examples/rendering/demo.py index 23709ebff..6ada02c89 100644 --- a/examples/rendering/demo.py +++ b/examples/rendering/demo.py @@ -1,7 +1,6 @@ import torch import genesis as gs -from genesis.utils.image_exporter import FrameImageExporter def main(): @@ -17,7 +16,18 @@ def main(): camera_lookat=(3.0, 0.0, 0.5), camera_fov=50, ), - renderer=gs.renderers.Rasterizer(), + renderer=gs.renderers.RayTracer( # type: ignore + env_surface=gs.surfaces.Emission( + emissive_texture=gs.textures.ImageTexture( + image_path="textures/indoor_bright.png", + ), + ), + env_radius=15.0, + env_euler=(0, 0, 180), + lights=[ + {"pos": (0.0, 0.0, 10.0), "radius": 3.0, "color": (15.0, 15.0, 15.0)}, + ], + ), ) ########################## materials ########################## @@ -146,14 +156,9 @@ def main(): scene.reset() horizon = 2000 - # Create an image exporter - output_dir = 'img_output/test' - exporter = FrameImageExporter(output_dir) - for i in range(horizon): scene.step() - rgb, depth, _, _ = cam_0.render() - exporter.export_frame_single_cam(i, cam_0.idx, rgb=rgb, depth=depth) + cam_0.render() if __name__ == "__main__": diff --git a/examples/rigid/single_franka.py b/examples/rigid/single_franka.py index 57e184d28..14cf1f0f8 100644 --- a/examples/rigid/single_franka.py +++ b/examples/rigid/single_franka.py @@ -1,7 +1,6 @@ import argparse import genesis as gs -from genesis.utils.image_exporter import FrameImageExporter def main(): @@ -33,7 +32,7 @@ def main(): ) franka = scene.add_entity( gs.morphs.MJCF(file="xml/franka_emika_panda/panda.xml"), - visualize_contact=False, + visualize_contact=True, ) ########################## cameras ########################## @@ -46,14 +45,9 @@ def main(): ) ########################## build ########################## scene.build() - - # Create an image exporter - output_dir = 'img_output/test' - exporter = FrameImageExporter(output_dir) - for i in range(2): + for i in range(1000): scene.step() - rgb, depth, _, _ = cam_0.render() - exporter.export_frame_single_cam(i, cam_0.idx, rgb=rgb, depth=depth) + # cam_0.render() if __name__ == "__main__": diff --git a/examples/rigid/single_franka_batch_render.py b/examples/rigid/single_franka_batch_render.py index 36b1dbf4b..774df8354 100644 --- a/examples/rigid/single_franka_batch_render.py +++ b/examples/rigid/single_franka_batch_render.py @@ -29,7 +29,7 @@ def main(): # constraint_solver=gs.constraint_solver.Newton, ), renderer = gs.options.renderers.BatchRenderer( - use_rasterizer=True, + use_rasterizer=False, batch_render_res=(512, 512), ) ) From 791a7f2094e049b928a9f670da3d41b61de8f878 Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 09:46:14 -0700 Subject: [PATCH 3/9] Simplify benchmark --- examples/perf_benchmark/batch_benchmark.py | 4 ++-- examples/perf_benchmark/benchmark_plotter.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/perf_benchmark/batch_benchmark.py b/examples/perf_benchmark/batch_benchmark.py index e31849ae5..5493de3f2 100644 --- a/examples/perf_benchmark/batch_benchmark.py +++ b/examples/perf_benchmark/batch_benchmark.py @@ -46,7 +46,7 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): ] minimal_batch_size_list = [ #2048, 3072, 4096, 6144, 8192, 12288, 16384 - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 + 1024, 2048 ] #minimal_batch_size_list = full_batch_size_list minimal_resolution_list = [ @@ -161,7 +161,7 @@ def get_benchmark_script_path(renderer_name): def run_batch_benchmark(batch_args_dict, previous_runs=None): if previous_runs is None: previous_runs = [] - + for renderer in batch_args_dict: benchmark_script_path = get_benchmark_script_path(renderer) if not os.path.exists(benchmark_script_path): diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index 12d1d326b..1c42ac972 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -195,9 +195,9 @@ def generate_individual_plots(df, plots_dir, width, height): plt.savefig(filename) plt.close() -def generate_comparison_plots(df, plots_dir, width, height, renderer_1, renderer_2, aspect_ratio=None): - renderer_1_name, renderer_1_is_rasterizer = renderer_1 - renderer_2_name, renderer_2_is_rasterizer = renderer_2 +def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=None): + renderer_1_name, renderer_1_is_rasterizer = renderer_info_1 + renderer_2_name, renderer_2_is_rasterizer = renderer_info_2 # Filter by aspect ratio if specified if aspect_ratio: if aspect_ratio == "1:1": From 432215f18f4a6d45f10971a9ccbaee0111c92882 Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 10:26:40 -0700 Subject: [PATCH 4/9] Should be working --- examples/perf_benchmark/batch_benchmark.py | 136 +++++++++++++----- examples/perf_benchmark/benchmark.py | 71 +-------- examples/perf_benchmark/benchmark_plotter.py | 13 +- examples/perf_benchmark/benchmark_pyrender.py | 68 +-------- 4 files changed, 119 insertions(+), 169 deletions(-) diff --git a/examples/perf_benchmark/batch_benchmark.py b/examples/perf_benchmark/batch_benchmark.py index 5493de3f2..e489b7d0d 100644 --- a/examples/perf_benchmark/batch_benchmark.py +++ b/examples/perf_benchmark/batch_benchmark.py @@ -1,4 +1,3 @@ -from benchmark import BenchmarkArgs from benchmark_plotter import plot_batch_benchmark import argparse import subprocess @@ -6,6 +5,74 @@ from datetime import datetime import pandas as pd +# Create a struct to store the arguments +class BenchmarkArgs: + def __init__(self, renderer_name, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): + self.renderer_name = renderer_name + self.rasterizer = rasterizer + self.n_envs = n_envs + self.n_steps = n_steps + self.resX = resX + self.resY = resY + self.camera_posX = camera_posX + self.camera_posY = camera_posY + self.camera_posZ = camera_posZ + self.camera_lookatX = camera_lookatX + self.camera_lookatY = camera_lookatY + self.camera_lookatZ = camera_lookatZ + self.camera_fov = camera_fov + self.mjcf = mjcf + self.benchmark_result_file_path = benchmark_result_file_path + + @staticmethod + def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--renderer_name", type=str, default="batch_renderer") + parser.add_argument("-r", "--rasterizer", action="store_true", default=False) + parser.add_argument("-n", "--n_envs", type=int, default=1024) + parser.add_argument("-s", "--n_steps", type=int, default=1) + parser.add_argument("-x", "--resX", type=int, default=1024) + parser.add_argument("-y", "--resY", type=int, default=1024) + parser.add_argument("-i", "--camera_posX", type=float, default=1.5) + parser.add_argument("-j", "--camera_posY", type=float, default=0.5) + parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) + parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) + parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) + parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) + parser.add_argument("-v", "--camera_fov", type=float, default=45) + parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") + parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") + args = parser.parse_args() + benchmark_args = BenchmarkArgs( + renderer_name=args.renderer_name, + rasterizer=args.rasterizer, + n_envs=args.n_envs, + n_steps=args.n_steps, + resX=args.resX, + resY=args.resY, + camera_posX=args.camera_posX, + camera_posY=args.camera_posY, + camera_posZ=args.camera_posZ, + camera_lookatX=args.camera_lookatX, + camera_lookatY=args.camera_lookatY, + camera_lookatZ=args.camera_lookatZ, + camera_fov=args.camera_fov, + mjcf=args.mjcf, + benchmark_result_file_path=args.benchmark_result_file_path, + ) + print(f"Benchmark with args:") + print(f" renderer_name: {benchmark_args.renderer_name}") + print(f" rasterizer: {benchmark_args.rasterizer}") + print(f" n_envs: {benchmark_args.n_envs}") + print(f" n_steps: {benchmark_args.n_steps}") + print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") + print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") + print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") + print(f" camera_fov: {benchmark_args.camera_fov}") + print(f" mjcf: {benchmark_args.mjcf}") + print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") + return benchmark_args + class BatchBenchmarkArgs: def __init__(self, use_full_list, continue_from): self.use_full_list = use_full_list @@ -25,8 +92,8 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): # Create a list of all the possible combinations of arguments # and return them as a list of BenchmarkArgs full_mjcf_list = ["xml/franka_emika_panda/panda.xml", "xml/unitree_g1/g1.xml", "xml/unitree_go2/go2.xml"] - renderer_list = ["batch_renderer", "pyrender"] - rasterizer_list = [True, False] + full_renderer_list = ["batch_renderer", "pyrender"] + full_rasterizer_list = [True, False] full_batch_size_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384] square_resolution_list = [ (64, 64), (128, 128), (256, 256), (512, 512), (1024, 1024), (2048, 2048), (4096, 4096), (8192, 8192) @@ -40,7 +107,8 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): full_resolution_list = square_resolution_list + four_three_resolution_list + sixteen_nine_resolution_list # Minimal mjcf, resolution, and batch size - minimal_renderer_list = ["batch_renderer"] + minimal_renderer_list = ["batch_renderer", "pyrender"] + minimal_rasterizer_list = [True] minimal_mjcf_list = [ "xml/franka_emika_panda/panda.xml" ] @@ -55,10 +123,14 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): ] if use_full_list: + renderer_list = full_renderer_list + rasterizer_list = full_rasterizer_list mjcf_list = full_mjcf_list resolution_list = full_resolution_list batch_size_list = full_batch_size_list else: + renderer_list = minimal_renderer_list + rasterizer_list = minimal_rasterizer_list mjcf_list = minimal_mjcf_list resolution_list = minimal_resolution_list batch_size_list = minimal_batch_size_list @@ -79,33 +151,33 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): # Build hierarchical structure for renderer in renderer_list: batch_args_dict[renderer] = {} - for rasterizer in rasterizer_list: - batch_args_dict[renderer][rasterizer] = {} - for mjcf in mjcf_list: - batch_args_dict[renderer][rasterizer][mjcf] = {} - for batch_size in batch_size_list: - batch_args_dict[renderer][rasterizer][mjcf][batch_size] = {} - for resolution in resolution_list: - resX, resY = resolution - # Create benchmark args for this combination - args = BenchmarkArgs( - renderer_name=renderer, - rasterizer=rasterizer, - n_envs=batch_size, - n_steps=n_steps, - resX=resX, - resY=resY, - camera_posX=camera_pos[0], - camera_posY=camera_pos[1], - camera_posZ=camera_pos[2], - camera_lookatX=camera_lookat[0], - camera_lookatY=camera_lookat[1], - camera_lookatZ=camera_lookat[2], - camera_fov=camera_fov, - mjcf=mjcf, - benchmark_result_file_path=benchmark_result_file_path - ) - batch_args_dict[renderer][rasterizer][mjcf][batch_size][(resX,resY)] = args + for rasterizer in rasterizer_list: + batch_args_dict[renderer][rasterizer] = {} + for mjcf in mjcf_list: + batch_args_dict[renderer][rasterizer][mjcf] = {} + for batch_size in batch_size_list: + batch_args_dict[renderer][rasterizer][mjcf][batch_size] = {} + for resolution in resolution_list: + resX, resY = resolution + # Create benchmark args for this combination + args = BenchmarkArgs( + renderer_name=renderer, + rasterizer=rasterizer, + n_envs=batch_size, + n_steps=n_steps, + resX=resX, + resY=resY, + camera_posX=camera_pos[0], + camera_posY=camera_pos[1], + camera_posZ=camera_pos[2], + camera_lookatX=camera_lookat[0], + camera_lookatY=camera_lookat[1], + camera_lookatZ=camera_lookat[2], + camera_fov=camera_fov, + mjcf=mjcf, + benchmark_result_file_path=benchmark_result_file_path + ) + batch_args_dict[renderer][rasterizer][mjcf][batch_size][(resX,resY)] = args return batch_args_dict @@ -123,7 +195,7 @@ def create_benchmark_result_file(continue_from_file_path): benchmark_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") benchmark_result_file_path = f"{benchmark_data_directory}/batch_benchmark_{benchmark_timestamp}.csv" with open(benchmark_result_file_path, "w") as f: - f.write("result,mjcf,renderer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") + f.write("result,mjcf,renderer,rasterizer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") print(f"Created new benchmark result file: {benchmark_result_file_path}") return benchmark_result_file_path diff --git a/examples/perf_benchmark/benchmark.py b/examples/perf_benchmark/benchmark.py index 9159026f6..4508e8f85 100644 --- a/examples/perf_benchmark/benchmark.py +++ b/examples/perf_benchmark/benchmark.py @@ -4,74 +4,7 @@ import numpy as np import genesis as gs import torch - -# Create a struct to store the arguments -class BenchmarkArgs: - def __init__(self, renderer_name, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): - self.renderer_name = renderer_name - self.rasterizer = rasterizer - self.n_envs = n_envs - self.n_steps = n_steps - self.resX = resX - self.resY = resY - self.camera_posX = camera_posX - self.camera_posY = camera_posY - self.camera_posZ = camera_posZ - self.camera_lookatX = camera_lookatX - self.camera_lookatY = camera_lookatY - self.camera_lookatZ = camera_lookatZ - self.camera_fov = camera_fov - self.mjcf = mjcf - self.benchmark_result_file_path = benchmark_result_file_path - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--renderer_name", type=str, default="rasterizer") - parser.add_argument("-r", "--rasterizer", action="store_true", default=False) - parser.add_argument("-n", "--n_envs", type=int, default=1024) - parser.add_argument("-s", "--n_steps", type=int, default=1) - parser.add_argument("-x", "--resX", type=int, default=1024) - parser.add_argument("-y", "--resY", type=int, default=1024) - parser.add_argument("-i", "--camera_posX", type=float, default=1.5) - parser.add_argument("-j", "--camera_posY", type=float, default=0.5) - parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) - parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) - parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) - parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) - parser.add_argument("-v", "--camera_fov", type=float, default=45) - parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") - parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") - args = parser.parse_args() - benchmark_args = BenchmarkArgs( - renderer_name=args.renderer_name, - rasterizer=args.rasterizer, - n_envs=args.n_envs, - n_steps=args.n_steps, - resX=args.resX, - resY=args.resY, - camera_posX=args.camera_posX, - camera_posY=args.camera_posY, - camera_posZ=args.camera_posZ, - camera_lookatX=args.camera_lookatX, - camera_lookatY=args.camera_lookatY, - camera_lookatZ=args.camera_lookatZ, - camera_fov=args.camera_fov, - mjcf=args.mjcf, - benchmark_result_file_path=args.benchmark_result_file_path, - ) - print(f"Benchmark with args:") - print(f" renderer_name: {benchmark_args.renderer_name}") - print(f" script: {benchmark_args.script}") - print(f" rasterizer: {benchmark_args.rasterizer}") - print(f" n_envs: {benchmark_args.n_envs}") - print(f" n_steps: {benchmark_args.n_steps}") - print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") - print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") - print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") - print(f" camera_fov: {benchmark_args.camera_fov}") - print(f" mjcf: {benchmark_args.mjcf}") - print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") - return benchmark_args +from batch_benchmark import BenchmarkArgs def init_gs(benchmark_args): ########################## init ########################## @@ -195,7 +128,7 @@ def run_benchmark(scene, benchmark_args): def main(): ######################## Parse arguments ####################### - benchmark_args = parse_args() + benchmark_args = BenchmarkArgs.parse_args() ######################## Initialize scene ####################### scene = init_gs(benchmark_args) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index 1c42ac972..768a5a0dc 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -103,6 +103,12 @@ def generatePlotHtml(plots_dir): with open(f"{plots_dir}/index.html", 'w') as f: f.write(html_content) +def get_comparison_data_set(): + return [ + (("batch_renderer", True), ("pyrender", True)), + (("batch_renderer", True), ("batch_renderer", False)), + ] + def plot_batch_benchmark(data_file_path, width=20, height=15): # Load the log file as csv # For each mjcf, rasterizer (rasterizer or not(=raytracer)), generate a plot image and save it to a directory. @@ -125,7 +131,8 @@ def plot_batch_benchmark(data_file_path, width=20, height=15): # Generate difference plots for specific aspect ratios for aspect_ratio in ["1:1", "4:3", "16:9"]: - generate_comparison_plots(df, plots_dir, width, height, ("batch_renderer", True), ("batch_renderer", False), aspect_ratio=aspect_ratio) + for renderer_info_1, renderer_info_2 in get_comparison_data_set(): + generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=aspect_ratio) # Generate an html page to display all the plots generatePlotHtml(plots_dir) @@ -134,7 +141,7 @@ def generate_individual_plots(df, plots_dir, width, height): # Get unique combinations of mjcf and rasterizer for mjcf in df['mjcf'].unique(): for renderer in df[df['mjcf'] == mjcf]['renderer'].unique(): - for rasterizer in df[df['mjcf'] == mjcf]['rasterizer'].unique(): + for rasterizer in df[(df['mjcf'] == mjcf) & (df['renderer'] == renderer)]['rasterizer'].unique(): # Filter data for this mjcf and rasterizer data = df[(df['mjcf'] == mjcf) & (df['renderer'] == renderer) & (df['rasterizer'] == rasterizer)] @@ -320,7 +327,7 @@ def main(): print("Script arguments:", sys.argv) # Debug print parser = argparse.ArgumentParser() - parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250524_180908.csv", + parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250611_101436.csv", help="Path to the benchmark data CSV file") parser.add_argument("-w", "--width", type=int, default=20, help="Width of the plot in inches") diff --git a/examples/perf_benchmark/benchmark_pyrender.py b/examples/perf_benchmark/benchmark_pyrender.py index 14f84f4b2..e0301c4be 100644 --- a/examples/perf_benchmark/benchmark_pyrender.py +++ b/examples/perf_benchmark/benchmark_pyrender.py @@ -4,69 +4,7 @@ import numpy as np import genesis as gs import torch - -# Create a struct to store the arguments -class BenchmarkArgs: - def __init__(self, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): - self.rasterizer = rasterizer - self.n_envs = n_envs - self.n_steps = n_steps - self.resX = resX - self.resY = resY - self.camera_posX = camera_posX - self.camera_posY = camera_posY - self.camera_posZ = camera_posZ - self.camera_lookatX = camera_lookatX - self.camera_lookatY = camera_lookatY - self.camera_lookatZ = camera_lookatZ - self.camera_fov = camera_fov - self.mjcf = mjcf - self.benchmark_result_file_path = benchmark_result_file_path - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("-r", "--rasterizer", action="store_true", default=False) - parser.add_argument("-n", "--n_envs", type=int, default=1024) - parser.add_argument("-s", "--n_steps", type=int, default=1) - parser.add_argument("-x", "--resX", type=int, default=1024) - parser.add_argument("-y", "--resY", type=int, default=1024) - parser.add_argument("-i", "--camera_posX", type=float, default=1.5) - parser.add_argument("-j", "--camera_posY", type=float, default=0.5) - parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) - parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) - parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) - parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) - parser.add_argument("-v", "--camera_fov", type=float, default=45) - parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") - parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") - args = parser.parse_args() - benchmark_args = BenchmarkArgs( - rasterizer=args.rasterizer, - n_envs=args.n_envs, - n_steps=args.n_steps, - resX=args.resX, - resY=args.resY, - camera_posX=args.camera_posX, - camera_posY=args.camera_posY, - camera_posZ=args.camera_posZ, - camera_lookatX=args.camera_lookatX, - camera_lookatY=args.camera_lookatY, - camera_lookatZ=args.camera_lookatZ, - camera_fov=args.camera_fov, - mjcf=args.mjcf, - benchmark_result_file_path=args.benchmark_result_file_path, - ) - print(f"Benchmark with args:") - print(f" rasterizer: {benchmark_args.rasterizer}") - print(f" n_envs: {benchmark_args.n_envs}") - print(f" n_steps: {benchmark_args.n_steps}") - print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") - print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") - print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") - print(f" camera_fov: {benchmark_args.camera_fov}") - print(f" mjcf: {benchmark_args.mjcf}") - print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") - return benchmark_args +from batch_benchmark import BenchmarkArgs def init_gs(benchmark_args): ########################## init ########################## @@ -159,14 +97,14 @@ def run_benchmark(scene, benchmark_args): # Append a line with all args and results in csv format with open(benchmark_args.benchmark_result_file_path, 'a') as f: - f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') + f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.renderer_name},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') except Exception as e: print(f"Error during benchmark: {e}") raise def main(): ######################## Parse arguments ####################### - benchmark_args = parse_args() + benchmark_args = BenchmarkArgs.parse_args() ######################## Initialize scene ####################### scene = init_gs(benchmark_args) From f3f67a9429ddfc5dc865b4425955739e56487340 Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 10:35:44 -0700 Subject: [PATCH 5/9] Really works --- examples/perf_benchmark/benchmark_plotter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index 768a5a0dc..de088ea3a 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -205,6 +205,9 @@ def generate_individual_plots(df, plots_dir, width, height): def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=None): renderer_1_name, renderer_1_is_rasterizer = renderer_info_1 renderer_2_name, renderer_2_is_rasterizer = renderer_info_2 + rasterizer_1_str = 'rasterizer' if renderer_1_is_rasterizer else 'raytracer' + rasterizer_2_str = 'rasterizer' if renderer_2_is_rasterizer else 'raytracer' + # Filter by aspect ratio if specified if aspect_ratio: if aspect_ratio == "1:1": @@ -219,7 +222,7 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren plt.clf() plt.cla() - # Generate plots showing fps difference between renderer_1 and renderer_2 + # Generate plots showing fps comparison between renderer_1 and renderer_2 for mjcf in df['mjcf'].unique(): mjcf_data = df[df['mjcf'] == mjcf] @@ -273,7 +276,7 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren fontsize=8) # Set title based on aspect ratio - subtitle1 = f"FPS Difference ({renderer_1_name} {rasterizer_1_str} / {renderer_2_name} {rasterizer_2_str})" + subtitle1 = f"FPS Comparison ({renderer_1_name} {rasterizer_1_str} / {renderer_2_name} {rasterizer_2_str})" if aspect_ratio: subtitle2 = f"({aspect_ratio} Resolutions Only)" else: @@ -327,7 +330,7 @@ def main(): print("Script arguments:", sys.argv) # Debug print parser = argparse.ArgumentParser() - parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250611_101436.csv", + parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250611_102653.csv", help="Path to the benchmark data CSV file") parser.add_argument("-w", "--width", type=int, default=20, help="Width of the plot in inches") From 0fa1c633f881a14a1a9437b1b4b269d49fd0d69f Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 11:15:49 -0700 Subject: [PATCH 6/9] Clean up plt --- examples/perf_benchmark/benchmark_plotter.py | 176 ++++++++++--------- 1 file changed, 92 insertions(+), 84 deletions(-) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index de088ea3a..19f11b474 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -6,6 +6,7 @@ import pandas as pd import matplotlib.pyplot as plt from matplotlib.ticker import ScalarFormatter +import numpy as np def generatePlotHtml(plots_dir): #Generate an html page to display all the plots @@ -16,17 +17,17 @@ def generatePlotHtml(plots_dir): print(f"No plot files found in {plots_dir}") return - # Separate regular plots from difference plots - regular_plots = [p for p in plot_files if not p.endswith('_comparison.png')] - aspect_ratio_plots = { - "1:1": [p for p in plot_files if p.endswith('_1x1_comparison.png')], - "4:3": [p for p in plot_files if p.endswith('_4x3_comparison.png')], - "16:9": [p for p in plot_files if p.endswith('_16x9_comparison.png')] + # Separate regular plots from comparison charts + regular_plot_files = [p for p in plot_files if p.endswith('_plot.png') and not p.endswith('_comparison_plot.png')] + aspect_ratio_plot_files = { + "1:1": [p for p in plot_files if p.endswith('_1x1_comparison_plot.png')], + "4:3": [p for p in plot_files if p.endswith('_4x3_comparison_plot.png')], + "16:9": [p for p in plot_files if p.endswith('_16x9_comparison_plot.png')] } # Group regular plots by MJCF file plot_groups = {} - for plot_file in regular_plots: + for plot_file in regular_plot_files: basename = os.path.basename(plot_file) mjcf_name = basename.split('_')[0] if mjcf_name not in plot_groups: @@ -38,7 +39,7 @@ def generatePlotHtml(plots_dir): # Group aspect ratio plots by MJCF file aspect_ratio_groups = {} - for aspect_ratio, plots in aspect_ratio_plots.items(): + for aspect_ratio, plots in aspect_ratio_plot_files.items(): aspect_ratio_groups[aspect_ratio] = {} for plot_file in plots: basename = os.path.basename(plot_file) @@ -153,53 +154,52 @@ def generate_individual_plots(df, plots_dir, width, height): # Create new figure plt.figure(figsize=(width, height)) - # Plot each resolution with different color - # Sort by resX and resY before plotting - for res in sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])): - resolution = res[0] - res_data = res[1] - plt.plot(res_data['n_envs'], res_data['fps'], - marker='o', label=f'{resolution[0]}x{resolution[1]}') + # Group data by resolution + resolutions = sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])) + + # Get all batch sizes + all_batch_sizes = sorted(data['n_envs'].unique()) + + # Set up bar chart + bar_width = 0.8 / len(resolutions) # Adjust bar width based on number of resolutions + x = np.arange(len(all_batch_sizes)) + + # Plot bars for each resolution + for i, (resolution, res_data) in enumerate(resolutions): + # Create mapping from batch size to index + batch_to_idx = {batch: idx for idx, batch in enumerate(all_batch_sizes)} - # Add x/y value annotations near each point - for x, y in zip(res_data['n_envs'], res_data['fps']): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) + # Create array of FPS for all batch sizes + fps_array = np.zeros(len(all_batch_sizes)) + for batch, fps in zip(res_data['n_envs'], res_data['fps']): + fps_array[batch_to_idx[batch]] = fps + + # Plot bars + plt.bar(x + i * bar_width, fps_array, bar_width, + label=f'{resolution[0]}x{resolution[1]}') + + # Add value labels on top of bars + for j, v in enumerate(fps_array): + if v > 0: # Only add label if there's a value + plt.text(x[j] + i * bar_width, v, f'{v:.1f}', + ha='center', va='bottom', fontsize=8) # Customize plot plt.title(f'Performance for {os.path.basename(mjcf)}\n{renderer} {"Rasterizer" if rasterizer else "Raytracer"}') plt.xlabel('Batch Size') plt.ylabel('FPS') - plt.grid(True) + plt.grid(True, axis='y') plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('log') - - # Force both axes to use ScalarFormatter - # Get current axes - ax = plt.gca() - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) + # Set x-axis ticks and labels + plt.xticks(x + bar_width * (len(resolutions) - 1) / 2, all_batch_sizes, rotation=45) - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) + # Adjust layout to prevent label cutoff + plt.tight_layout() # Save plot - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer}_{'rasterizer' if rasterizer else 'raytracer'}.png" - plt.savefig(filename) + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer}_{'rasterizer' if rasterizer else 'raytracer'}_plot.png" + plt.savefig(plot_filename) plt.close() def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=None): @@ -243,6 +243,11 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren # Plot difference for each common resolution max_abs_diff = 0 # Track maximum absolute difference for y-axis limits min_abs_diff = 1e10 + + # Prepare data for grouped bar chart + all_batch_sizes = set() + resolution_data = {} + for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): renderer_1_data = mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer) & @@ -256,24 +261,47 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren # Match batch sizes and calculate difference common_batch = set(renderer_1_data['n_envs']).intersection(set(renderer_2_data['n_envs'])) batch_sizes = sorted(list(common_batch)) + all_batch_sizes.update(batch_sizes) renderer_1_fps = renderer_1_data[renderer_1_data['n_envs'].isin(batch_sizes)]['fps'].values renderer_2_fps = renderer_2_data[renderer_2_data['n_envs'].isin(batch_sizes)]['fps'].values diff_fps = renderer_1_fps / renderer_2_fps - # Update max absolute difference + # Store data for this resolution + resolution_data[f'{resX}x{resY}'] = { + 'batch_sizes': batch_sizes, + 'diff_fps': diff_fps + } + + # Update max/min differences max_abs_diff = max(max_abs_diff, diff_fps.max()) min_abs_diff = min(min_abs_diff, diff_fps.min()) - - plt.plot(batch_sizes, diff_fps, marker='o', label=f'{resX}x{resY}') + + # Convert all_batch_sizes to sorted list + all_batch_sizes = sorted(list(all_batch_sizes)) + + # Set up bar chart + bar_width = 0.8 / len(common_res) # Adjust bar width based on number of resolutions + x = np.arange(len(all_batch_sizes)) + + # Plot bars for each resolution + for i, (res_label, data) in enumerate(resolution_data.items()): + # Create mapping from batch size to index + batch_to_idx = {batch: idx for idx, batch in enumerate(all_batch_sizes)} - # Add annotations - for x, y in zip(batch_sizes, diff_fps): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) + # Create array of differences for all batch sizes + diff_array = np.zeros(len(all_batch_sizes)) + for batch, diff in zip(data['batch_sizes'], data['diff_fps']): + diff_array[batch_to_idx[batch]] = diff + + # Plot bars + plt.bar(x + i * bar_width, diff_array, bar_width, label=res_label) + + # Add value labels on top of bars + for j, v in enumerate(diff_array): + if v > 0: # Only add label if there's a value + plt.text(x[j] + i * bar_width, v, f'{v:.1f}', + ha='center', va='bottom', fontsize=8) # Set title based on aspect ratio subtitle1 = f"FPS Comparison ({renderer_1_name} {rasterizer_1_str} / {renderer_2_name} {rasterizer_2_str})" @@ -284,44 +312,24 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren plt.title(f'{subtitle1}\n{os.path.basename(mjcf)} {subtitle2}') plt.xlabel('Batch Size') plt.ylabel(subtitle1) - plt.grid(True) + plt.grid(True, axis='y') plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('log') # Use log scale with smaller base for better resolution near 1.0 - # Format axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) - - # Set y-axis limits to ensure all points are visible - #ax.set_ylim(min_abs_diff * 0.9, max_abs_diff * 1.1) # Add 10% padding - ax.autoscale(enable=True, axis='y') + # Set x-axis ticks and labels + plt.xticks(x + bar_width * (len(common_res) - 1) / 2, all_batch_sizes, rotation=45) # Add horizontal line at y=1 to show crossover point plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) + # Adjust layout to prevent label cutoff + plt.tight_layout() + # Save plot with aspect ratio in filename if specified - rasterizer_1_str = 'rasterizer' if renderer_1_is_rasterizer else 'raytracer' - rasterizer_2_str = 'rasterizer' if renderer_2_is_rasterizer else 'raytracer' if aspect_ratio: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{aspect_ratio.replace(':', 'x')}_comparison.png" + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{aspect_ratio.replace(':', 'x')}_comparison_plot.png" else: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_comparison.png" - plt.savefig(filename) + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_comparison_plot.png" + plt.savefig(plot_filename) plt.close() def main(): @@ -330,7 +338,7 @@ def main(): print("Script arguments:", sys.argv) # Debug print parser = argparse.ArgumentParser() - parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250611_102653.csv", + parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250610_160138_combined.csv", help="Path to the benchmark data CSV file") parser.add_argument("-w", "--width", type=int, default=20, help="Width of the plot in inches") From 906da659b25acd4e783ce825227fcac16e312382 Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 11:36:22 -0700 Subject: [PATCH 7/9] Comparison bar working --- examples/perf_benchmark/benchmark_plotter.py | 109 ++++++------------- 1 file changed, 35 insertions(+), 74 deletions(-) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index 19f11b474..6eb3ecc95 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -238,17 +238,10 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren print(f"No data found for {mjcf}") continue - plt.figure(figsize=(width, height)) - - # Plot difference for each common resolution - max_abs_diff = 0 # Track maximum absolute difference for y-axis limits - min_abs_diff = 1e10 - - # Prepare data for grouped bar chart - all_batch_sizes = set() - resolution_data = {} - + # Plot comparison for each resolution for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): + plt.figure(figsize=(width, height)) + renderer_1_data = mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer) & (mjcf_data['resX'] == resX) & @@ -261,76 +254,44 @@ def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, ren # Match batch sizes and calculate difference common_batch = set(renderer_1_data['n_envs']).intersection(set(renderer_2_data['n_envs'])) batch_sizes = sorted(list(common_batch)) - all_batch_sizes.update(batch_sizes) renderer_1_fps = renderer_1_data[renderer_1_data['n_envs'].isin(batch_sizes)]['fps'].values renderer_2_fps = renderer_2_data[renderer_2_data['n_envs'].isin(batch_sizes)]['fps'].values diff_fps = renderer_1_fps / renderer_2_fps - - # Store data for this resolution - resolution_data[f'{resX}x{resY}'] = { - 'batch_sizes': batch_sizes, - 'diff_fps': diff_fps - } - - # Update max/min differences - max_abs_diff = max(max_abs_diff, diff_fps.max()) - min_abs_diff = min(min_abs_diff, diff_fps.min()) - - # Convert all_batch_sizes to sorted list - all_batch_sizes = sorted(list(all_batch_sizes)) - - # Set up bar chart - bar_width = 0.8 / len(common_res) # Adjust bar width based on number of resolutions - x = np.arange(len(all_batch_sizes)) - - # Plot bars for each resolution - for i, (res_label, data) in enumerate(resolution_data.items()): - # Create mapping from batch size to index - batch_to_idx = {batch: idx for idx, batch in enumerate(all_batch_sizes)} - - # Create array of differences for all batch sizes - diff_array = np.zeros(len(all_batch_sizes)) - for batch, diff in zip(data['batch_sizes'], data['diff_fps']): - diff_array[batch_to_idx[batch]] = diff - + + # Create bar chart + x = np.arange(len(batch_sizes)) + bar_width = 0.35 + # Plot bars - plt.bar(x + i * bar_width, diff_array, bar_width, label=res_label) - + bars1 = plt.bar(x - bar_width/2, renderer_1_fps, bar_width, label=f'{renderer_1_name} {rasterizer_1_str}') + bars2 = plt.bar(x + bar_width/2, renderer_2_fps, bar_width, label=f'{renderer_2_name} {rasterizer_2_str}') + # Add value labels on top of bars - for j, v in enumerate(diff_array): - if v > 0: # Only add label if there's a value - plt.text(x[j] + i * bar_width, v, f'{v:.1f}', - ha='center', va='bottom', fontsize=8) - - # Set title based on aspect ratio - subtitle1 = f"FPS Comparison ({renderer_1_name} {rasterizer_1_str} / {renderer_2_name} {rasterizer_2_str})" - if aspect_ratio: - subtitle2 = f"({aspect_ratio} Resolutions Only)" - else: - subtitle2 = "" - plt.title(f'{subtitle1}\n{os.path.basename(mjcf)} {subtitle2}') - plt.xlabel('Batch Size') - plt.ylabel(subtitle1) - plt.grid(True, axis='y') - plt.legend(title='Resolution') - - # Set x-axis ticks and labels - plt.xticks(x + bar_width * (len(common_res) - 1) / 2, all_batch_sizes, rotation=45) - - # Add horizontal line at y=1 to show crossover point - plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) - - # Adjust layout to prevent label cutoff - plt.tight_layout() - - # Save plot with aspect ratio in filename if specified - if aspect_ratio: - plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{aspect_ratio.replace(':', 'x')}_comparison_plot.png" - else: - plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_comparison_plot.png" - plt.savefig(plot_filename) - plt.close() + def add_labels(bars): + for bar in bars: + height = bar.get_height() + plt.annotate(f'{height:.1f}', + xy=(bar.get_x() + bar.get_width() / 2, height), + xytext=(0, 3), # 3 points vertical offset + textcoords="offset points", + ha='center', va='bottom', fontsize=8) + + add_labels(bars1) + add_labels(bars2) + + # Customize plot + plt.title(f'FPS Comparison: {renderer_1_name} {rasterizer_1_str} vs {renderer_2_name} {rasterizer_2_str}\n{os.path.basename(mjcf)} - Resolution: {resX}x{resY}') + plt.xlabel('Batch Size') + plt.ylabel('FPS') + plt.xticks(x, batch_sizes) + plt.legend() + plt.grid(True, axis='y') + + # Save plot + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{resX}x{resY}_comparison_plot.png" + plt.savefig(plot_filename, dpi=100) # Added dpi parameter for better quality + plt.close() def main(): import sys From 03dbd42573ac042cb5e1f5a0908598c0dfe15bcd Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 11:49:25 -0700 Subject: [PATCH 8/9] Per-resolution comparison --- examples/perf_benchmark/benchmark_plotter.py | 86 ++++++++++---------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index 6eb3ecc95..a6ba0cec8 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -19,11 +19,6 @@ def generatePlotHtml(plots_dir): # Separate regular plots from comparison charts regular_plot_files = [p for p in plot_files if p.endswith('_plot.png') and not p.endswith('_comparison_plot.png')] - aspect_ratio_plot_files = { - "1:1": [p for p in plot_files if p.endswith('_1x1_comparison_plot.png')], - "4:3": [p for p in plot_files if p.endswith('_4x3_comparison_plot.png')], - "16:9": [p for p in plot_files if p.endswith('_16x9_comparison_plot.png')] - } # Group regular plots by MJCF file plot_groups = {} @@ -36,20 +31,23 @@ def generatePlotHtml(plots_dir): # Sort plot groups by mjcf name and plot file name plot_groups = sorted(plot_groups.items(), key=lambda x: (x[0], x[1][0])) - - # Group aspect ratio plots by MJCF file - aspect_ratio_groups = {} - for aspect_ratio, plots in aspect_ratio_plot_files.items(): - aspect_ratio_groups[aspect_ratio] = {} - for plot_file in plots: - basename = os.path.basename(plot_file) - mjcf_name = basename.split('_')[0] - if mjcf_name not in aspect_ratio_groups[aspect_ratio]: - aspect_ratio_groups[aspect_ratio][mjcf_name] = [] - aspect_ratio_groups[aspect_ratio][mjcf_name].append(plot_file) - - # Sort each aspect ratio's plot groups - aspect_ratio_groups[aspect_ratio] = sorted(aspect_ratio_groups[aspect_ratio].items(), key=lambda x: (x[0], x[1][0])) + + # Group comparison plots by resolution + comparison_plot_files = {} + for plot_file in plot_files: + if plot_file.endswith('_comparison_plot.png'): + # Extract resolution from filename (e.g., "128x128" from "..._128x128_comparison_plot.png") + resolution = plot_file.split('_')[-3] # Get the resolution part + if resolution not in comparison_plot_files: + comparison_plot_files[resolution] = [] + comparison_plot_files[resolution].append(plot_file) + + # Sort resolutions by their dimensions + def get_resolution_dims(res): + width, height = map(int, res.split('x')) + return width * height # Sort by total pixels + + sorted_resolutions = sorted(comparison_plot_files.keys(), key=get_resolution_dims) # Create HTML file html_content = """ @@ -71,18 +69,17 @@ def generatePlotHtml(plots_dir):

Benchmark Results

""" - # Add aspect ratio difference plots sections - for aspect_ratio, mjcf_groups in aspect_ratio_groups.items(): - if mjcf_groups: - html_content += "
\n" - html_content += f"

Performance Comparison Plots ({aspect_ratio} Resolutions Only)

\n" - for mjcf_name, plots in mjcf_groups: - html_content += f"
\n" - for plot in plots: - html_content += f"

{html.escape(mjcf_name)} - {os.path.basename(plot)}

\n" - html_content += f"{html.escape(os.path.basename(plot))}
\n" - html_content += "
\n" + # Add comparison plots sections by resolution + if comparison_plot_files: + html_content += "
\n" + html_content += "

Performance Comparison Plots

\n" + for resolution in sorted_resolutions: + html_content += f"

Resolution: {resolution}

\n" + html_content += "
\n" + for plot in comparison_plot_files[resolution]: + html_content += f"{html.escape(os.path.basename(plot))}
\n" html_content += "
\n" + html_content += "
\n" # Add regular plots section html_content += "
\n" @@ -160,9 +157,9 @@ def generate_individual_plots(df, plots_dir, width, height): # Get all batch sizes all_batch_sizes = sorted(data['n_envs'].unique()) - # Set up bar chart - bar_width = 0.8 / len(resolutions) # Adjust bar width based on number of resolutions + # Create bar chart x = np.arange(len(all_batch_sizes)) + bar_width = 0.8 / len(resolutions) # Adjust bar width based on number of resolutions # Plot bars for each resolution for i, (resolution, res_data) in enumerate(resolutions): @@ -175,28 +172,27 @@ def generate_individual_plots(df, plots_dir, width, height): fps_array[batch_to_idx[batch]] = fps # Plot bars - plt.bar(x + i * bar_width, fps_array, bar_width, + bars = plt.bar(x + i * bar_width, fps_array, bar_width, label=f'{resolution[0]}x{resolution[1]}') # Add value labels on top of bars - for j, v in enumerate(fps_array): - if v > 0: # Only add label if there's a value - plt.text(x[j] + i * bar_width, v, f'{v:.1f}', - ha='center', va='bottom', fontsize=8) + for bar in bars: + bar_height = bar.get_height() + if bar_height > 0: # Only add label if there's a value + plt.annotate(f'{bar_height:.1f}', + xy=(bar.get_x() + bar.get_width() / 2, bar_height), + xytext=(0, 3), # 3 points vertical offset + textcoords="offset points", + ha='center', va='bottom', fontsize=8) # Customize plot plt.title(f'Performance for {os.path.basename(mjcf)}\n{renderer} {"Rasterizer" if rasterizer else "Raytracer"}') plt.xlabel('Batch Size') plt.ylabel('FPS') - plt.grid(True, axis='y') + plt.xticks(x + bar_width * (len(resolutions) - 1) / 2, all_batch_sizes) plt.legend(title='Resolution') - - # Set x-axis ticks and labels - plt.xticks(x + bar_width * (len(resolutions) - 1) / 2, all_batch_sizes, rotation=45) - - # Adjust layout to prevent label cutoff - plt.tight_layout() - + plt.grid(True, axis='y') + # Save plot plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer}_{'rasterizer' if rasterizer else 'raytracer'}_plot.png" plt.savefig(plot_filename) From ceda0b1a999ef947af786ec2cea1d49dd19c9a12 Mon Sep 17 00:00:00 2001 From: Hongyi Yu Date: Wed, 11 Jun 2025 13:04:08 -0700 Subject: [PATCH 9/9] Adjust default plot height --- examples/perf_benchmark/benchmark_plotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index a6ba0cec8..525dc7794 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -299,7 +299,7 @@ def main(): help="Path to the benchmark data CSV file") parser.add_argument("-w", "--width", type=int, default=20, help="Width of the plot in inches") - parser.add_argument("-y", "--height", type=int, default=15, + parser.add_argument("-y", "--height", type=int, default=8, help="Height of the plot in inches") # If no arguments provided, try to get from environment variables