diff --git a/examples/perf_benchmark/batch_benchmark.py b/examples/perf_benchmark/batch_benchmark.py index e4ba67227..e489b7d0d 100644 --- a/examples/perf_benchmark/batch_benchmark.py +++ b/examples/perf_benchmark/batch_benchmark.py @@ -1,4 +1,3 @@ -from benchmark import BenchmarkArgs from benchmark_plotter import plot_batch_benchmark import argparse import subprocess @@ -6,6 +5,74 @@ from datetime import datetime import pandas as pd +# Create a struct to store the arguments +class BenchmarkArgs: + def __init__(self, renderer_name, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): + self.renderer_name = renderer_name + self.rasterizer = rasterizer + self.n_envs = n_envs + self.n_steps = n_steps + self.resX = resX + self.resY = resY + self.camera_posX = camera_posX + self.camera_posY = camera_posY + self.camera_posZ = camera_posZ + self.camera_lookatX = camera_lookatX + self.camera_lookatY = camera_lookatY + self.camera_lookatZ = camera_lookatZ + self.camera_fov = camera_fov + self.mjcf = mjcf + self.benchmark_result_file_path = benchmark_result_file_path + + @staticmethod + def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--renderer_name", type=str, default="batch_renderer") + parser.add_argument("-r", "--rasterizer", action="store_true", default=False) + parser.add_argument("-n", "--n_envs", type=int, default=1024) + parser.add_argument("-s", "--n_steps", type=int, default=1) + parser.add_argument("-x", "--resX", type=int, default=1024) + parser.add_argument("-y", "--resY", type=int, default=1024) + parser.add_argument("-i", "--camera_posX", type=float, default=1.5) + parser.add_argument("-j", "--camera_posY", type=float, default=0.5) + parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) + parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) + parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) + parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) + parser.add_argument("-v", "--camera_fov", type=float, default=45) + parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") + parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") + args = parser.parse_args() + benchmark_args = BenchmarkArgs( + renderer_name=args.renderer_name, + rasterizer=args.rasterizer, + n_envs=args.n_envs, + n_steps=args.n_steps, + resX=args.resX, + resY=args.resY, + camera_posX=args.camera_posX, + camera_posY=args.camera_posY, + camera_posZ=args.camera_posZ, + camera_lookatX=args.camera_lookatX, + camera_lookatY=args.camera_lookatY, + camera_lookatZ=args.camera_lookatZ, + camera_fov=args.camera_fov, + mjcf=args.mjcf, + benchmark_result_file_path=args.benchmark_result_file_path, + ) + print(f"Benchmark with args:") + print(f" renderer_name: {benchmark_args.renderer_name}") + print(f" rasterizer: {benchmark_args.rasterizer}") + print(f" n_envs: {benchmark_args.n_envs}") + print(f" n_steps: {benchmark_args.n_steps}") + print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") + print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") + print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") + print(f" camera_fov: {benchmark_args.camera_fov}") + print(f" mjcf: {benchmark_args.mjcf}") + print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") + return benchmark_args + class BatchBenchmarkArgs: def __init__(self, use_full_list, continue_from): self.use_full_list = use_full_list @@ -25,7 +92,8 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): # Create a list of all the possible combinations of arguments # and return them as a list of BenchmarkArgs full_mjcf_list = ["xml/franka_emika_panda/panda.xml", "xml/unitree_g1/g1.xml", "xml/unitree_go2/go2.xml"] - rasterizer_list = [True, False] + full_renderer_list = ["batch_renderer", "pyrender"] + full_rasterizer_list = [True, False] full_batch_size_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384] square_resolution_list = [ (64, 64), (128, 128), (256, 256), (512, 512), (1024, 1024), (2048, 2048), (4096, 4096), (8192, 8192) @@ -39,12 +107,14 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): full_resolution_list = square_resolution_list + four_three_resolution_list + sixteen_nine_resolution_list # Minimal mjcf, resolution, and batch size + minimal_renderer_list = ["batch_renderer", "pyrender"] + minimal_rasterizer_list = [True] minimal_mjcf_list = [ "xml/franka_emika_panda/panda.xml" ] minimal_batch_size_list = [ #2048, 3072, 4096, 6144, 8192, 12288, 16384 - 2048, 4096, 8192, 16384 + 1024, 2048 ] #minimal_batch_size_list = full_batch_size_list minimal_resolution_list = [ @@ -53,10 +123,14 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): ] if use_full_list: + renderer_list = full_renderer_list + rasterizer_list = full_rasterizer_list mjcf_list = full_mjcf_list resolution_list = full_resolution_list batch_size_list = full_batch_size_list else: + renderer_list = minimal_renderer_list + rasterizer_list = minimal_rasterizer_list mjcf_list = minimal_mjcf_list resolution_list = minimal_resolution_list batch_size_list = minimal_batch_size_list @@ -75,32 +149,35 @@ def create_batch_args(benchmark_result_file_path, use_full_list=False): batch_args_dict = {} # Build hierarchical structure - for rasterizer in rasterizer_list: - batch_args_dict[rasterizer] = {} - for mjcf in mjcf_list: - batch_args_dict[rasterizer][mjcf] = {} - for batch_size in batch_size_list: - batch_args_dict[rasterizer][mjcf][batch_size] = {} - for resolution in resolution_list: - resX, resY = resolution - # Create benchmark args for this combination - args = BenchmarkArgs( - rasterizer=rasterizer, - n_envs=batch_size, - n_steps=n_steps, - resX=resX, - resY=resY, - camera_posX=camera_pos[0], - camera_posY=camera_pos[1], - camera_posZ=camera_pos[2], - camera_lookatX=camera_lookat[0], - camera_lookatY=camera_lookat[1], - camera_lookatZ=camera_lookat[2], - camera_fov=camera_fov, - mjcf=mjcf, - benchmark_result_file_path=benchmark_result_file_path - ) - batch_args_dict[rasterizer][mjcf][batch_size][(resX,resY)] = args + for renderer in renderer_list: + batch_args_dict[renderer] = {} + for rasterizer in rasterizer_list: + batch_args_dict[renderer][rasterizer] = {} + for mjcf in mjcf_list: + batch_args_dict[renderer][rasterizer][mjcf] = {} + for batch_size in batch_size_list: + batch_args_dict[renderer][rasterizer][mjcf][batch_size] = {} + for resolution in resolution_list: + resX, resY = resolution + # Create benchmark args for this combination + args = BenchmarkArgs( + renderer_name=renderer, + rasterizer=rasterizer, + n_envs=batch_size, + n_steps=n_steps, + resX=resX, + resY=resY, + camera_posX=camera_pos[0], + camera_posY=camera_pos[1], + camera_posZ=camera_pos[2], + camera_lookatX=camera_lookat[0], + camera_lookatY=camera_lookat[1], + camera_lookatZ=camera_lookat[2], + camera_fov=camera_fov, + mjcf=mjcf, + benchmark_result_file_path=benchmark_result_file_path + ) + batch_args_dict[renderer][rasterizer][mjcf][batch_size][(resX,resY)] = args return batch_args_dict @@ -118,7 +195,7 @@ def create_benchmark_result_file(continue_from_file_path): benchmark_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") benchmark_result_file_path = f"{benchmark_data_directory}/batch_benchmark_{benchmark_timestamp}.csv" with open(benchmark_result_file_path, "w") as f: - f.write("result,mjcf,rasterizer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") + f.write("result,mjcf,renderer,rasterizer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken,time_taken_per_env,fps,fps_per_env\n") print(f"Created new benchmark result file: {benchmark_result_file_path}") return benchmark_result_file_path @@ -135,7 +212,7 @@ def get_previous_runs(continue_from_file_path): for _, row in df.iterrows(): run_info = ( row['mjcf'], - row['rasterizer'], + row['renderer'], row['n_envs'], (row['resX'], row['resY']), row['result'] # 'succeeded' or 'failed' @@ -144,67 +221,83 @@ def get_previous_runs(continue_from_file_path): return previous_runs -def run_batch_benchmark(batch_args_dict, benchmark_script_path, previous_runs=None): +def get_benchmark_script_path(renderer_name): + current_dir = os.path.dirname(os.path.abspath(__file__)) + if renderer_name == "batch_renderer": + return f"{current_dir}/benchmark.py" + elif renderer_name == "pyrender": + return f"{current_dir}/benchmark_pyrender.py" + else: + raise ValueError(f"Invalid renderer name: {renderer_name}") + +def run_batch_benchmark(batch_args_dict, previous_runs=None): if previous_runs is None: previous_runs = [] + + for renderer in batch_args_dict: + benchmark_script_path = get_benchmark_script_path(renderer) + if not os.path.exists(benchmark_script_path): + raise FileNotFoundError(f"Benchmark script not found: {benchmark_script_path}") + print(f"Running benchmark for {renderer}") - for rasterizer in batch_args_dict: - for mjcf in batch_args_dict[rasterizer]: - for batch_size in batch_args_dict[rasterizer][mjcf]: - last_resolution_failed = False - for resolution in batch_args_dict[rasterizer][mjcf][batch_size]: - if last_resolution_failed: - break - - # Check if this run was in a previous execution - run_info = (mjcf, rasterizer, batch_size, resolution) - skip_this_run = False - - for prev_run in previous_runs: - if run_info == prev_run[:4]: # Compare only the run parameters, not the status - skip_this_run = True - if prev_run[4] == 'failed': - # Skip this and subsequent resolutions if it failed before - last_resolution_failed = True + for rasterizer in batch_args_dict[renderer]: + for mjcf in batch_args_dict[renderer][rasterizer]: + for batch_size in batch_args_dict[renderer][rasterizer][mjcf]: + last_resolution_failed = False + for resolution in batch_args_dict[renderer][rasterizer][mjcf][batch_size]: + if last_resolution_failed: break - - if skip_this_run: - continue - # Run the benchmark - batch_args = batch_args_dict[rasterizer][mjcf][batch_size][resolution] - - # launch a process to run the benchmark - cmd = ["python3", benchmark_script_path] - if batch_args.rasterizer: - cmd.append("-r") - cmd.extend([ - "-n", str(batch_args.n_envs), - "-s", str(batch_args.n_steps), - "-x", str(batch_args.resX), - "-y", str(batch_args.resY), - "-i", str(batch_args.camera_posX), - "-j", str(batch_args.camera_posY), - "-k", str(batch_args.camera_posZ), - "-l", str(batch_args.camera_lookatX), - "-m", str(batch_args.camera_lookatY), - "-o", str(batch_args.camera_lookatZ), - "-v", str(batch_args.camera_fov), - "-f", batch_args.mjcf, - "-g", batch_args.benchmark_result_file_path - ]) - try: - process = subprocess.Popen(cmd) - return_code = process.wait() - if return_code != 0: - raise subprocess.CalledProcessError(return_code, cmd) - except Exception as e: - print(f"Error running benchmark: {str(e)}") - last_resolution_failed = True - # Write failed result without timing data - with open(batch_args.benchmark_result_file_path, 'a') as f: - f.write(f'failed,{batch_args.mjcf},{batch_args.rasterizer},{batch_args.n_envs},{batch_args.n_steps},{batch_args.resX},{batch_args.resY},{batch_args.camera_posX},{batch_args.camera_posY},{batch_args.camera_posZ},{batch_args.camera_lookatX},{batch_args.camera_lookatY},{batch_args.camera_lookatZ},{batch_args.camera_fov},,,,\n') - break + # Check if this run was in a previous execution + run_info = (mjcf, rasterizer, batch_size, resolution) + skip_this_run = False + + for prev_run in previous_runs: + if run_info == prev_run[:4]: # Compare only the run parameters, not the status + skip_this_run = True + if prev_run[4] == 'failed': + # Skip this and subsequent resolutions if it failed before + last_resolution_failed = True + break + + if skip_this_run: + continue + + # Run the benchmark + batch_args = batch_args_dict[renderer][rasterizer][mjcf][batch_size][resolution] + + # launch a process to run the benchmark + cmd = ["python3", benchmark_script_path] + if batch_args.rasterizer: + cmd.append("-r") + cmd.extend([ + "-d", batch_args.renderer_name, + "-n", str(batch_args.n_envs), + "-s", str(batch_args.n_steps), + "-x", str(batch_args.resX), + "-y", str(batch_args.resY), + "-i", str(batch_args.camera_posX), + "-j", str(batch_args.camera_posY), + "-k", str(batch_args.camera_posZ), + "-l", str(batch_args.camera_lookatX), + "-m", str(batch_args.camera_lookatY), + "-o", str(batch_args.camera_lookatZ), + "-v", str(batch_args.camera_fov), + "-f", batch_args.mjcf, + "-g", batch_args.benchmark_result_file_path + ]) + try: + process = subprocess.Popen(cmd) + return_code = process.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, cmd) + except Exception as e: + print(f"Error running benchmark: {str(e)}") + last_resolution_failed = True + # Write failed result without timing data + with open(batch_args.benchmark_result_file_path, 'a') as f: + f.write(f'failed,{batch_args.mjcf},{batch_args.renderer_name},{batch_args.rasterizer},{batch_args.n_envs},{batch_args.n_steps},{batch_args.resX},{batch_args.resY},{batch_args.camera_posX},{batch_args.camera_posY},{batch_args.camera_posZ},{batch_args.camera_lookatX},{batch_args.camera_lookatY},{batch_args.camera_lookatZ},{batch_args.camera_fov},,,,\n') + break def main(): batch_benchmark_args = parse_args() @@ -213,14 +306,9 @@ def main(): # Get list of previous runs if continuing from a previous run previous_runs = get_previous_runs(batch_benchmark_args.continue_from) - # Run benchmark in batch - current_dir = os.path.dirname(os.path.abspath(__file__)) - benchmark_script_path = f"{current_dir}/benchmark.py" - if not os.path.exists(benchmark_script_path): - raise FileNotFoundError(f"Benchmark script not found: {benchmark_script_path}") - + # Run benchmark in batch batch_args_dict = create_batch_args(benchmark_result_file_path, use_full_list=batch_benchmark_args.use_full_list) - run_batch_benchmark(batch_args_dict, benchmark_script_path, previous_runs) + run_batch_benchmark(batch_args_dict, previous_runs) # Generate plots plot_batch_benchmark(benchmark_result_file_path) diff --git a/examples/perf_benchmark/benchmark.py b/examples/perf_benchmark/benchmark.py index 9c474ebea..4508e8f85 100644 --- a/examples/perf_benchmark/benchmark.py +++ b/examples/perf_benchmark/benchmark.py @@ -4,69 +4,7 @@ import numpy as np import genesis as gs import torch - -# Create a struct to store the arguments -class BenchmarkArgs: - def __init__(self, rasterizer, n_envs, n_steps, resX, resY, camera_posX, camera_posY, camera_posZ, camera_lookatX, camera_lookatY, camera_lookatZ, camera_fov, mjcf, benchmark_result_file_path): - self.rasterizer = rasterizer - self.n_envs = n_envs - self.n_steps = n_steps - self.resX = resX - self.resY = resY - self.camera_posX = camera_posX - self.camera_posY = camera_posY - self.camera_posZ = camera_posZ - self.camera_lookatX = camera_lookatX - self.camera_lookatY = camera_lookatY - self.camera_lookatZ = camera_lookatZ - self.camera_fov = camera_fov - self.mjcf = mjcf - self.benchmark_result_file_path = benchmark_result_file_path - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("-r", "--rasterizer", action="store_true", default=False) - parser.add_argument("-n", "--n_envs", type=int, default=1024) - parser.add_argument("-s", "--n_steps", type=int, default=1) - parser.add_argument("-x", "--resX", type=int, default=1024) - parser.add_argument("-y", "--resY", type=int, default=1024) - parser.add_argument("-i", "--camera_posX", type=float, default=1.5) - parser.add_argument("-j", "--camera_posY", type=float, default=0.5) - parser.add_argument("-k", "--camera_posZ", type=float, default=1.5) - parser.add_argument("-l", "--camera_lookatX", type=float, default=0.0) - parser.add_argument("-m", "--camera_lookatY", type=float, default=0.0) - parser.add_argument("-o", "--camera_lookatZ", type=float, default=0.5) - parser.add_argument("-v", "--camera_fov", type=float, default=45) - parser.add_argument("-f", "--mjcf", type=str, default="xml/franka_emika_panda/panda.xml") - parser.add_argument("-g", "--benchmark_result_file_path", type=str, default="benchmark.csv") - args = parser.parse_args() - benchmark_args = BenchmarkArgs( - rasterizer=args.rasterizer, - n_envs=args.n_envs, - n_steps=args.n_steps, - resX=args.resX, - resY=args.resY, - camera_posX=args.camera_posX, - camera_posY=args.camera_posY, - camera_posZ=args.camera_posZ, - camera_lookatX=args.camera_lookatX, - camera_lookatY=args.camera_lookatY, - camera_lookatZ=args.camera_lookatZ, - camera_fov=args.camera_fov, - mjcf=args.mjcf, - benchmark_result_file_path=args.benchmark_result_file_path, - ) - print(f"Benchmark with args:") - print(f" rasterizer: {benchmark_args.rasterizer}") - print(f" n_envs: {benchmark_args.n_envs}") - print(f" n_steps: {benchmark_args.n_steps}") - print(f" resolution: {benchmark_args.resX}x{benchmark_args.resY}") - print(f" camera_pos: ({benchmark_args.camera_posX}, {benchmark_args.camera_posY}, {benchmark_args.camera_posZ})") - print(f" camera_lookat: ({benchmark_args.camera_lookatX}, {benchmark_args.camera_lookatY}, {benchmark_args.camera_lookatZ})") - print(f" camera_fov: {benchmark_args.camera_fov}") - print(f" mjcf: {benchmark_args.mjcf}") - print(f" benchmark_result_file_path: {benchmark_args.benchmark_result_file_path}") - return benchmark_args +from batch_benchmark import BenchmarkArgs def init_gs(benchmark_args): ########################## init ########################## @@ -155,7 +93,7 @@ def run_benchmark(scene, benchmark_args): # warmup scene.step() - rgb, depth, _, _ = scene.batch_render() + rgb, depth, _, _ = scene.render_all_cams() # fill gpu cache with random data fill_gpu_cache_with_random_data() @@ -165,7 +103,7 @@ def run_benchmark(scene, benchmark_args): start_time = time() for i in range(n_steps): - rgb, depth, _, _ = scene.batch_render(force_render=True) + rgb, depth, _, _ = scene.render_all_cams(force_render=True) end_time = time() time_taken = end_time - start_time @@ -183,14 +121,14 @@ def run_benchmark(scene, benchmark_args): # Append a line with all args and results in csv format with open(benchmark_args.benchmark_result_file_path, 'a') as f: - f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') + f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.renderer_name},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') except Exception as e: print(f"Error during benchmark: {e}") raise def main(): ######################## Parse arguments ####################### - benchmark_args = parse_args() + benchmark_args = BenchmarkArgs.parse_args() ######################## Initialize scene ####################### scene = init_gs(benchmark_args) diff --git a/examples/perf_benchmark/benchmark_plotter.py b/examples/perf_benchmark/benchmark_plotter.py index acfcbf720..525dc7794 100644 --- a/examples/perf_benchmark/benchmark_plotter.py +++ b/examples/perf_benchmark/benchmark_plotter.py @@ -6,6 +6,7 @@ import pandas as pd import matplotlib.pyplot as plt from matplotlib.ticker import ScalarFormatter +import numpy as np def generatePlotHtml(plots_dir): #Generate an html page to display all the plots @@ -16,17 +17,12 @@ def generatePlotHtml(plots_dir): print(f"No plot files found in {plots_dir}") return - # Separate regular plots from difference plots - regular_plots = [p for p in plot_files if not p.endswith('_difference.png') and not any(p.endswith(f'_difference_{ar.replace(":", "x")}.png') for ar in ["1:1", "4:3", "16:9"])] - aspect_ratio_plots = { - "1:1": [p for p in plot_files if p.endswith('_difference_1x1.png')], - "4:3": [p for p in plot_files if p.endswith('_difference_4x3.png')], - "16:9": [p for p in plot_files if p.endswith('_difference_16x9.png')] - } + # Separate regular plots from comparison charts + regular_plot_files = [p for p in plot_files if p.endswith('_plot.png') and not p.endswith('_comparison_plot.png')] # Group regular plots by MJCF file plot_groups = {} - for plot_file in regular_plots: + for plot_file in regular_plot_files: basename = os.path.basename(plot_file) mjcf_name = basename.split('_')[0] if mjcf_name not in plot_groups: @@ -35,20 +31,23 @@ def generatePlotHtml(plots_dir): # Sort plot groups by mjcf name and plot file name plot_groups = sorted(plot_groups.items(), key=lambda x: (x[0], x[1][0])) - - # Group aspect ratio plots by MJCF file - aspect_ratio_groups = {} - for aspect_ratio, plots in aspect_ratio_plots.items(): - aspect_ratio_groups[aspect_ratio] = {} - for plot_file in plots: - basename = os.path.basename(plot_file) - mjcf_name = basename.split('_')[0] - if mjcf_name not in aspect_ratio_groups[aspect_ratio]: - aspect_ratio_groups[aspect_ratio][mjcf_name] = [] - aspect_ratio_groups[aspect_ratio][mjcf_name].append(plot_file) - - # Sort each aspect ratio's plot groups - aspect_ratio_groups[aspect_ratio] = sorted(aspect_ratio_groups[aspect_ratio].items(), key=lambda x: (x[0], x[1][0])) + + # Group comparison plots by resolution + comparison_plot_files = {} + for plot_file in plot_files: + if plot_file.endswith('_comparison_plot.png'): + # Extract resolution from filename (e.g., "128x128" from "..._128x128_comparison_plot.png") + resolution = plot_file.split('_')[-3] # Get the resolution part + if resolution not in comparison_plot_files: + comparison_plot_files[resolution] = [] + comparison_plot_files[resolution].append(plot_file) + + # Sort resolutions by their dimensions + def get_resolution_dims(res): + width, height = map(int, res.split('x')) + return width * height # Sort by total pixels + + sorted_resolutions = sorted(comparison_plot_files.keys(), key=get_resolution_dims) # Create HTML file html_content = """ @@ -70,29 +69,29 @@ def generatePlotHtml(plots_dir):

Benchmark Results

""" + # Add comparison plots sections by resolution + if comparison_plot_files: + html_content += "
\n" + html_content += "

Performance Comparison Plots

\n" + for resolution in sorted_resolutions: + html_content += f"

Resolution: {resolution}

\n" + html_content += "
\n" + for plot in comparison_plot_files[resolution]: + html_content += f"{html.escape(os.path.basename(plot))}
\n" + html_content += "
\n" + html_content += "
\n" + # Add regular plots section html_content += "
\n" html_content += "

Performance Plots

\n" for mjcf_name, plots in plot_groups: html_content += f"
\n" for plot in plots: - html_content += f"

{html.escape(mjcf_name)} - {'Rasterizer' if 'rasterizer' in os.path.basename(plot) else 'Raytracer'}

\n" + html_content += f"

{html.escape(mjcf_name)} - {os.path.basename(plot)}

\n" html_content += f"{html.escape(os.path.basename(plot))}
\n" html_content += "
\n" html_content += "
\n" - # Add aspect ratio difference plots sections - for aspect_ratio, mjcf_groups in aspect_ratio_groups.items(): - if mjcf_groups: - html_content += "
\n" - html_content += f"

Performance Difference Plots (Rasterizer / Raytracer) ({aspect_ratio} Resolutions Only)

\n" - for mjcf_name, plots in mjcf_groups: - html_content += f"
\n" - html_content += f"

{html.escape(mjcf_name)}

\n" - html_content += f"{html.escape(os.path.basename(plots[0]))}
\n" - html_content += "
\n" - html_content += "
\n" - html_content += """ @@ -102,6 +101,12 @@ def generatePlotHtml(plots_dir): with open(f"{plots_dir}/index.html", 'w') as f: f.write(html_content) +def get_comparison_data_set(): + return [ + (("batch_renderer", True), ("pyrender", True)), + (("batch_renderer", True), ("batch_renderer", False)), + ] + def plot_batch_benchmark(data_file_path, width=20, height=15): # Load the log file as csv # For each mjcf, rasterizer (rasterizer or not(=raytracer)), generate a plot image and save it to a directory. @@ -124,7 +129,8 @@ def plot_batch_benchmark(data_file_path, width=20, height=15): # Generate difference plots for specific aspect ratios for aspect_ratio in ["1:1", "4:3", "16:9"]: - generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=aspect_ratio) + for renderer_info_1, renderer_info_2 in get_comparison_data_set(): + generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=aspect_ratio) # Generate an html page to display all the plots generatePlotHtml(plots_dir) @@ -132,181 +138,72 @@ def plot_batch_benchmark(data_file_path, width=20, height=15): def generate_individual_plots(df, plots_dir, width, height): # Get unique combinations of mjcf and rasterizer for mjcf in df['mjcf'].unique(): - for rasterizer in df[df['mjcf'] == mjcf]['rasterizer'].unique(): - # Filter data for this mjcf and rasterizer - data = df[(df['mjcf'] == mjcf) & (df['rasterizer'] == rasterizer)] - - # continue if there is no data - if len(data) == 0: - print(f"No data found for {mjcf} and {rasterizer}") - continue - - # Create new figure - plt.figure(figsize=(width, height)) - - # Plot each resolution with different color - # Sort by resX and resY before plotting - for res in sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])): - resolution = res[0] - res_data = res[1] - plt.plot(res_data['n_envs'], res_data['fps'], - marker='o', label=f'{resolution[0]}x{resolution[1]}') + for renderer in df[df['mjcf'] == mjcf]['renderer'].unique(): + for rasterizer in df[(df['mjcf'] == mjcf) & (df['renderer'] == renderer)]['rasterizer'].unique(): + # Filter data for this mjcf and rasterizer + data = df[(df['mjcf'] == mjcf) & (df['renderer'] == renderer) & (df['rasterizer'] == rasterizer)] + + # continue if there is no data + if len(data) == 0: + print(f"No data found for {mjcf} and {renderer} and rasterizer:{rasterizer}") + continue - # Add x/y value annotations near each point - for x, y in zip(res_data['n_envs'], res_data['fps']): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) - - # Customize plot - plt.title(f'Performance for {os.path.basename(mjcf)}\n{"Rasterizer" if rasterizer else "Raytracer"}') - plt.xlabel('Batch Size') - plt.ylabel('FPS') - plt.grid(True) - plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('log') - - # Force both axes to use ScalarFormatter - # Get current axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) - - # Save plot - rasterizer_str = "rasterizer" if rasterizer else "raytracer" - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{rasterizer_str}.png" - plt.savefig(filename) - plt.close() - -def generate_difference_plots(df, plots_dir, width, height, aspect_ratio=None): - # Filter by aspect ratio if specified - if aspect_ratio: - if aspect_ratio == "1:1": - df = df[df['resX'] == df['resY']] - elif aspect_ratio == "4:3": - df = df[df['resX'] * 3 == df['resY'] * 4] - elif aspect_ratio == "16:9": - df = df[df['resX'] * 9 == df['resY'] * 16] - else: - raise ValueError(f"Unsupported aspect ratio: {aspect_ratio}") - - plt.clf() - plt.cla() - - # Generate plots showing fps difference between rasterizer and raytracer - for mjcf in df['mjcf'].unique(): - mjcf_data = df[df['mjcf'] == mjcf] - - # Get resolutions available for both rasterizer and raytracer - rasterizer_res = set(zip(mjcf_data[mjcf_data['rasterizer']]['resX'], - mjcf_data[mjcf_data['rasterizer']]['resY'])) - raytracer_res = set(zip(mjcf_data[~mjcf_data['rasterizer']]['resX'], - mjcf_data[~mjcf_data['rasterizer']]['resY'])) - common_res = rasterizer_res.intersection(raytracer_res) - - # continue if there is no data - if len(common_res) == 0: - print(f"No data found for {mjcf}") - continue - - plt.figure(figsize=(width, height)) - - # Plot difference for each common resolution - max_abs_diff = 0 # Track maximum absolute difference for y-axis limits - min_abs_diff = 1e10 - for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): - rast_data = mjcf_data[(mjcf_data['rasterizer']) & - (mjcf_data['resX'] == resX) & - (mjcf_data['resY'] == resY)] - ray_data = mjcf_data[(~mjcf_data['rasterizer']) & - (mjcf_data['resX'] == resX) & - (mjcf_data['resY'] == resY)] - - # Match batch sizes and calculate difference - common_batch = set(rast_data['n_envs']).intersection(set(ray_data['n_envs'])) - batch_sizes = sorted(list(common_batch)) - - rast_fps = rast_data[rast_data['n_envs'].isin(batch_sizes)]['fps'].values - ray_fps = ray_data[ray_data['n_envs'].isin(batch_sizes)]['fps'].values - diff_fps = rast_fps - ray_fps - - # Update max absolute difference - max_abs_diff = max(max_abs_diff, diff_fps.max()) - min_abs_diff = min(min_abs_diff, diff_fps.min()) - - plt.plot(batch_sizes, diff_fps, marker='o', label=f'{resX}x{resY}') - - # Add annotations - for x, y in zip(batch_sizes, diff_fps): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) - - # Set title based on aspect ratio - subtitle1 = "FPS Difference (Rasterizer - Raytracer)" - if aspect_ratio: - subtitle2 = f"({aspect_ratio} Resolutions Only)" - else: - subtitle2 = "" - plt.title(f'{subtitle1}\n{os.path.basename(mjcf)} {subtitle2}') - plt.xlabel('Batch Size') - plt.ylabel(subtitle1) - plt.grid(True) - plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('symlog', linthresh=1.0) # Use symlog with linear threshold of 1.0 - - # Format axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) - - # Set y-axis limits symmetrically - ax.set_ylim(0, max_abs_diff*1.1) # Add 10% padding - - # Add horizontal line at y=0 to show crossover point - plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) - - # Save plot with aspect ratio in filename if specified - if aspect_ratio: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference_{aspect_ratio.replace(':', 'x')}.png" - else: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference.png" - plt.savefig(filename) - plt.close() + # Create new figure + plt.figure(figsize=(width, height)) + + # Group data by resolution + resolutions = sorted(data.groupby(['resX', 'resY']), key=lambda x: (x[0][0], x[0][1])) + + # Get all batch sizes + all_batch_sizes = sorted(data['n_envs'].unique()) + + # Create bar chart + x = np.arange(len(all_batch_sizes)) + bar_width = 0.8 / len(resolutions) # Adjust bar width based on number of resolutions + + # Plot bars for each resolution + for i, (resolution, res_data) in enumerate(resolutions): + # Create mapping from batch size to index + batch_to_idx = {batch: idx for idx, batch in enumerate(all_batch_sizes)} + + # Create array of FPS for all batch sizes + fps_array = np.zeros(len(all_batch_sizes)) + for batch, fps in zip(res_data['n_envs'], res_data['fps']): + fps_array[batch_to_idx[batch]] = fps + + # Plot bars + bars = plt.bar(x + i * bar_width, fps_array, bar_width, + label=f'{resolution[0]}x{resolution[1]}') + + # Add value labels on top of bars + for bar in bars: + bar_height = bar.get_height() + if bar_height > 0: # Only add label if there's a value + plt.annotate(f'{bar_height:.1f}', + xy=(bar.get_x() + bar.get_width() / 2, bar_height), + xytext=(0, 3), # 3 points vertical offset + textcoords="offset points", + ha='center', va='bottom', fontsize=8) + + # Customize plot + plt.title(f'Performance for {os.path.basename(mjcf)}\n{renderer} {"Rasterizer" if rasterizer else "Raytracer"}') + plt.xlabel('Batch Size') + plt.ylabel('FPS') + plt.xticks(x + bar_width * (len(resolutions) - 1) / 2, all_batch_sizes) + plt.legend(title='Resolution') + plt.grid(True, axis='y') + + # Save plot + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer}_{'rasterizer' if rasterizer else 'raytracer'}_plot.png" + plt.savefig(plot_filename) + plt.close() + +def generate_comparison_plots(df, plots_dir, width, height, renderer_info_1, renderer_info_2, aspect_ratio=None): + renderer_1_name, renderer_1_is_rasterizer = renderer_info_1 + renderer_2_name, renderer_2_is_rasterizer = renderer_info_2 + rasterizer_1_str = 'rasterizer' if renderer_1_is_rasterizer else 'raytracer' + rasterizer_2_str = 'rasterizer' if renderer_2_is_rasterizer else 'raytracer' -def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): # Filter by aspect ratio if specified if aspect_ratio: if aspect_ratio == "1:1": @@ -321,103 +218,76 @@ def generate_ratio_plots(df, plots_dir, width, height, aspect_ratio=None): plt.clf() plt.cla() - # Generate plots showing fps difference between rasterizer and raytracer + # Generate plots showing fps comparison between renderer_1 and renderer_2 for mjcf in df['mjcf'].unique(): mjcf_data = df[df['mjcf'] == mjcf] - # Get resolutions available for both rasterizer and raytracer - rasterizer_res = set(zip(mjcf_data[mjcf_data['rasterizer']]['resX'], - mjcf_data[mjcf_data['rasterizer']]['resY'])) - raytracer_res = set(zip(mjcf_data[~mjcf_data['rasterizer']]['resX'], - mjcf_data[~mjcf_data['rasterizer']]['resY'])) - common_res = rasterizer_res.intersection(raytracer_res) + # Get resolutions available for both renderer_1 and renderer_2 + renderer_1_res = set(zip(mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer)]['resX'], + mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & (mjcf_data['rasterizer'] == renderer_1_is_rasterizer)]['resY'])) + renderer_2_res = set(zip(mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & (mjcf_data['rasterizer'] == renderer_2_is_rasterizer)]['resX'], + mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & (mjcf_data['rasterizer'] == renderer_2_is_rasterizer)]['resY'])) + common_res = renderer_1_res.intersection(renderer_2_res) # continue if there is no data if len(common_res) == 0: print(f"No data found for {mjcf}") continue - plt.figure(figsize=(width, height)) - - # Plot difference for each common resolution - max_abs_diff = 0 # Track maximum absolute difference for y-axis limits - min_abs_diff = 1e10 + # Plot comparison for each resolution for resX, resY in sorted(common_res, key=lambda x: x[0] * x[1]): - rast_data = mjcf_data[(mjcf_data['rasterizer']) & + plt.figure(figsize=(width, height)) + + renderer_1_data = mjcf_data[(mjcf_data['renderer'] == renderer_1_name) & + (mjcf_data['rasterizer'] == renderer_1_is_rasterizer) & (mjcf_data['resX'] == resX) & (mjcf_data['resY'] == resY)] - ray_data = mjcf_data[(~mjcf_data['rasterizer']) & + renderer_2_data = mjcf_data[(mjcf_data['renderer'] == renderer_2_name) & + (mjcf_data['rasterizer'] == renderer_2_is_rasterizer) & (mjcf_data['resX'] == resX) & (mjcf_data['resY'] == resY)] # Match batch sizes and calculate difference - common_batch = set(rast_data['n_envs']).intersection(set(ray_data['n_envs'])) + common_batch = set(renderer_1_data['n_envs']).intersection(set(renderer_2_data['n_envs'])) batch_sizes = sorted(list(common_batch)) - rast_fps = rast_data[rast_data['n_envs'].isin(batch_sizes)]['fps'].values - ray_fps = ray_data[ray_data['n_envs'].isin(batch_sizes)]['fps'].values - diff_fps = rast_fps / ray_fps - - # Update max absolute difference - max_abs_diff = max(max_abs_diff, diff_fps.max()) - min_abs_diff = min(min_abs_diff, diff_fps.min()) + renderer_1_fps = renderer_1_data[renderer_1_data['n_envs'].isin(batch_sizes)]['fps'].values + renderer_2_fps = renderer_2_data[renderer_2_data['n_envs'].isin(batch_sizes)]['fps'].values + diff_fps = renderer_1_fps / renderer_2_fps + + # Create bar chart + x = np.arange(len(batch_sizes)) + bar_width = 0.35 + + # Plot bars + bars1 = plt.bar(x - bar_width/2, renderer_1_fps, bar_width, label=f'{renderer_1_name} {rasterizer_1_str}') + bars2 = plt.bar(x + bar_width/2, renderer_2_fps, bar_width, label=f'{renderer_2_name} {rasterizer_2_str}') + + # Add value labels on top of bars + def add_labels(bars): + for bar in bars: + height = bar.get_height() + plt.annotate(f'{height:.1f}', + xy=(bar.get_x() + bar.get_width() / 2, height), + xytext=(0, 3), # 3 points vertical offset + textcoords="offset points", + ha='center', va='bottom', fontsize=8) + + add_labels(bars1) + add_labels(bars2) - plt.plot(batch_sizes, diff_fps, marker='o', label=f'{resX}x{resY}') - - # Add annotations - for x, y in zip(batch_sizes, diff_fps): - plt.annotate(f'({x:.0f}, {y:.1f})', - (x, y), - xytext=(5, 5), - textcoords='offset points', - fontsize=8) - - # Set title based on aspect ratio - subtitle1 = "FPS Difference (Rasterizer / Raytracer)" - if aspect_ratio: - subtitle2 = f"({aspect_ratio} Resolutions Only)" - else: - subtitle2 = "" - plt.title(f'{subtitle1}\n{os.path.basename(mjcf)} {subtitle2}') - plt.xlabel('Batch Size') - plt.ylabel(subtitle1) - plt.grid(True) - plt.legend(title='Resolution') - plt.xscale('log') - plt.yscale('log') # Use log scale with smaller base for better resolution near 1.0 - - # Format axes - ax = plt.gca() - - # Set major formatter for both axes - ax.xaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - ax.yaxis.set_major_formatter(ScalarFormatter(useOffset=False)) - - # Disable scientific notation - ax.ticklabel_format(axis='both', style='plain') - - # Add more tick marks - ax.xaxis.set_major_locator(plt.LogLocator(base=2)) - ax.yaxis.set_major_locator(plt.LogLocator(base=10)) - - # Add minor tick marks - ax.xaxis.set_minor_locator(plt.LogLocator(base=2, subs=(.5,))) - ax.yaxis.set_minor_locator(plt.LogLocator(base=10, subs=(.5,))) + # Customize plot + plt.title(f'FPS Comparison: {renderer_1_name} {rasterizer_1_str} vs {renderer_2_name} {rasterizer_2_str}\n{os.path.basename(mjcf)} - Resolution: {resX}x{resY}') + plt.xlabel('Batch Size') + plt.ylabel('FPS') + plt.xticks(x, batch_sizes) + plt.legend() + plt.grid(True, axis='y') - # Set y-axis limits to ensure all points are visible - #ax.set_ylim(min_abs_diff * 0.9, max_abs_diff * 1.1) # Add 10% padding - ax.autoscale(enable=True, axis='y') - - # Add horizontal line at y=1 to show crossover point - plt.axhline(y=1, color='k', linestyle='--', alpha=0.3) - - # Save plot with aspect ratio in filename if specified - if aspect_ratio: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference_{aspect_ratio.replace(':', 'x')}.png" - else: - filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_difference.png" - plt.savefig(filename) - plt.close() + # Save plot + plot_filename = f"{plots_dir}/{os.path.splitext(os.path.basename(mjcf))[0]}_{renderer_1_name}_{rasterizer_1_str}_{renderer_2_name}_{rasterizer_2_str}_{resX}x{resY}_comparison_plot.png" + plt.savefig(plot_filename, dpi=100) # Added dpi parameter for better quality + plt.close() def main(): import sys @@ -425,11 +295,11 @@ def main(): print("Script arguments:", sys.argv) # Debug print parser = argparse.ArgumentParser() - parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250524_180908.csv", + parser.add_argument("-d", "--data_file_path", type=str, default="logs/benchmark/batch_benchmark_20250610_160138_combined.csv", help="Path to the benchmark data CSV file") parser.add_argument("-w", "--width", type=int, default=20, help="Width of the plot in inches") - parser.add_argument("-y", "--height", type=int, default=15, + parser.add_argument("-y", "--height", type=int, default=8, help="Height of the plot in inches") # If no arguments provided, try to get from environment variables diff --git a/examples/perf_benchmark/benchmark_pyrender.py b/examples/perf_benchmark/benchmark_pyrender.py new file mode 100644 index 000000000..e0301c4be --- /dev/null +++ b/examples/perf_benchmark/benchmark_pyrender.py @@ -0,0 +1,117 @@ +import argparse +import os + +import numpy as np +import genesis as gs +import torch +from batch_benchmark import BenchmarkArgs + +def init_gs(benchmark_args): + ########################## init ########################## + try: + gs.init(backend=gs.gpu) + except Exception as e: + print(f"Failed to initialize GPU backend: {e}") + print("Falling back to CPU backend") + gs.init(backend=gs.cpu) + + ########################## create a scene ########################## + scene = gs.Scene( + viewer_options=gs.options.ViewerOptions( + camera_pos=(benchmark_args.camera_posX, benchmark_args.camera_posY, benchmark_args.camera_posZ), + camera_lookat=(benchmark_args.camera_lookatX, benchmark_args.camera_lookatY, benchmark_args.camera_lookatZ), + camera_fov=benchmark_args.camera_fov, + ), + vis_options=gs.options.VisOptions( + lights=[ + {"type": "directional", "dir": (1.0, 1.0, -2.0), "color": (1.0, 1.0, 1.0), "intensity": 0.5}, + {"type": "point", "pos": (4, -4, 4), "color": (1.0, 1.0, 1.0), "intensity": 1}, + ], + ), + show_viewer=False, + rigid_options=gs.options.RigidOptions( + # constraint_solver=gs.constraint_solver.Newton, + ), + renderer = benchmark_args.rasterizer and gs.options.renderers.Rasterizer() or gs.options.renderers.RayTracer() + ) + + ########################## entities ########################## + plane = scene.add_entity( + gs.morphs.Plane(), + ) + franka = scene.add_entity( + gs.morphs.MJCF(file=benchmark_args.mjcf), + visualize_contact=False, + ) + + ########################## cameras ########################## + cam_0 = scene.add_camera( + res=(benchmark_args.resX, benchmark_args.resY), + pos=(benchmark_args.camera_posX, benchmark_args.camera_posY, benchmark_args.camera_posZ), + lookat=(benchmark_args.camera_lookatX, benchmark_args.camera_lookatY, benchmark_args.camera_lookatZ), + fov=benchmark_args.camera_fov, + ) + ########################## build ########################## + scene.build() + return scene + +def fill_gpu_cache_with_random_data(): + # 100 MB of random data + dummy_data =torch.rand(100, 1024, 1024, device="cuda") + # Make some random data manipulation to the entire tensor + dummy_data = dummy_data.sqrt() + +def run_benchmark(scene, benchmark_args): + try: + n_envs = benchmark_args.n_envs + n_steps = benchmark_args.n_steps + + # warmup + scene.step() + rgb, depth, _, _ = scene.visualizer.cameras[0].render(rgb=True, depth=True) + + # fill gpu cache with random data + fill_gpu_cache_with_random_data() + + # timer + from time import time + start_time = time() + + for i in range(n_steps): + for i_env in range(n_envs): + rgb, depth, _, _ = scene.visualizer.cameras[0].render(rgb=True, depth=True) + + end_time = time() + time_taken = end_time - start_time + time_taken_per_env = time_taken / n_envs + fps = n_envs * n_steps / time_taken + fps_per_env = n_steps / time_taken + + print(f'Time taken: {time_taken} seconds') + print(f'Time taken per env: {time_taken_per_env} seconds') + print(f'FPS: {fps}') + print(f'FPS per env: {fps_per_env}') + + # Ensure the directory exists + os.makedirs(os.path.dirname(benchmark_args.benchmark_result_file_path), exist_ok=True) + + # Append a line with all args and results in csv format + with open(benchmark_args.benchmark_result_file_path, 'a') as f: + f.write(f'succeeded,{benchmark_args.mjcf},{benchmark_args.renderer_name},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken},{time_taken_per_env},{fps},{fps_per_env}\n') + except Exception as e: + print(f"Error during benchmark: {e}") + raise + +def main(): + ######################## Parse arguments ####################### + benchmark_args = BenchmarkArgs.parse_args() + + ######################## Initialize scene ####################### + scene = init_gs(benchmark_args) + + ######################## Run benchmark ####################### + run_benchmark(scene, benchmark_args) + +if __name__ == "__main__": + main() + diff --git a/genesis/utils/image_exporter.py b/genesis/utils/image_exporter.py index daea24c58..45cc845c7 100644 --- a/genesis/utils/image_exporter.py +++ b/genesis/utils/image_exporter.py @@ -11,17 +11,17 @@ def _export_frame_rgb_cam(export_dir, i_env, i_cam, camera_name, i_step, rgb): cv2.imwrite(f'{export_dir}/rgb_env{i_env}_{camera_name}_{i_step:03d}.png', rgb) @staticmethod - def _export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth_normalized): - depth_normalized = depth_normalized[i_env, i_cam].cpu().numpy() - cv2.imwrite(f'{export_dir}/depth_env{i_env}_{camera_name}_{i_step:03d}.png', depth_normalized) + def _export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth): + depth = depth[i_env, i_cam].cpu().numpy() + cv2.imwrite(f'{export_dir}/depth_env{i_env}_{camera_name}_{i_step:03d}.png', depth) @staticmethod def _worker_export_frame_cam(args): - export_dir, i_env, i_cam, camera_name, rgb, depth_normalized, i_step = args + export_dir, i_env, i_cam, camera_name, rgb, depth, i_step = args if rgb is not None: FrameImageExporter._export_frame_rgb_cam(export_dir, i_env, i_cam, camera_name, i_step, rgb) - if depth_normalized is not None: - FrameImageExporter._export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth_normalized) + if depth is not None: + FrameImageExporter._export_frame_depth_cam(export_dir, i_env, i_cam, camera_name, i_step, depth) def __init__(self, export_dir, depth_clip_max=100, depth_scale='log'): self.export_dir = export_dir @@ -66,20 +66,24 @@ def export_frame_all_cams(self, i_step, camera_idx=None, rgb=None, depth=None): rgb: RGB image tensor of shape (n_envs, n_cams, H, W, 3). depth: Depth tensor of shape (n_envs, n_cams, H, W). """ + if rgb is None and depth is None: + print("No rgb or depth to export") + return - if rgb.ndim == 4: - rgb = rgb.unsqueeze(0) - if depth.ndim == 4: - depth = depth.unsqueeze(0) - assert rgb.ndim == 5 and depth.ndim == 5, "rgb and depth must be of shape (n_envs, n_cams, H, W, 3)" + if rgb is not None: + if rgb.ndim == 4: + rgb = rgb.unsqueeze(0) + assert rgb.ndim == 5, "rgb must be of shape (n_envs, n_cams, H, W, 3)" + if depth is not None: + if depth.ndim == 4: + depth = depth.unsqueeze(0) + depth = self._normalize_depth(depth) + assert depth.ndim == 5, "depth must be of shape (n_envs, n_cams, H, W, 1)" if camera_idx is None: - camera_idx = range(rgb.shape[1]) - env_idx = range(rgb.shape[0]) - - depth_normalized = self._normalize_depth(depth) if depth is not None else None - - args_list = [(self.export_dir, i_env, i_cam, self._get_camera_name(i_cam), rgb, depth_normalized, i_step) + camera_idx = range(rgb.shape[1]) if rgb is not None else range(depth.shape[1]) + env_idx = range(rgb.shape[0]) if rgb is not None else range(depth.shape[0]) + args_list = [(self.export_dir, i_env, i_cam, self._get_camera_name(i_cam), rgb, depth, i_step) for i_env in env_idx for i_cam in camera_idx] with ThreadPoolExecutor() as executor: executor.map(FrameImageExporter._worker_export_frame_cam, args_list) @@ -94,32 +98,35 @@ def export_frame_single_cam(self, i_step, i_cam, rgb=None, depth=None): rgb: RGB image tensor of shape (n_envs, H, W, 3). depth: Depth tensor of shape (n_envs, H, W). """ - # Move rgb and depth to torch tensor - if isinstance(rgb, np.ndarray): - rgb = torch.from_numpy(rgb.copy()) - if isinstance(depth, np.ndarray): - depth = torch.from_numpy(depth.copy()) - - # Unsqueeze rgb and depth to (n_envs, 1, H, W, 3) and (n_envs, 1, H, W, 1) - if rgb.ndim == 4: - rgb = rgb.unsqueeze(1) - elif rgb.ndim == 3: - rgb = rgb.unsqueeze(0).unsqueeze(0) - else: - raise ValueError(f"Invalid rgb shape: {rgb.shape}") - - if depth.ndim == 4: - depth = depth.unsqueeze(1) - elif depth.ndim == 3: - depth = depth.unsqueeze(0).unsqueeze(0) - else: - raise ValueError(f"Invalid depth shape: {depth.shape}") - assert rgb.ndim == 5 and depth.ndim == 5, "rgb and depth must be of shape (n_envs, n_cams, H, W, 3)" + if rgb is not None: + if isinstance(rgb, np.ndarray): + rgb = torch.from_numpy(rgb.copy()) - env_idx = range(rgb.shape[0]) - depth_normalized = self._normalize_depth(depth) if depth is not None else None - - args_list = [(self.export_dir, i_env, 0, self._get_camera_name(i_cam), rgb, depth_normalized, i_step) + # Unsqueeze rgb to (n_envs, 1, H, W, 3) if n_envs > 0 + if rgb.ndim == 4: + rgb = rgb.unsqueeze(1) + elif rgb.ndim == 3: + rgb = rgb.unsqueeze(0).unsqueeze(0) + else: + raise ValueError(f"Invalid rgb shape: {rgb.shape}") + assert rgb.ndim == 5, "rgb must be of shape (n_envs, H, W, 3)" + + if depth is not None: + if isinstance(depth, np.ndarray): + depth = torch.from_numpy(depth.copy()) + + # Unsqueeze depth to (n_envs, 1, H, W, 1) if n_envs > 0 + if depth.ndim == 4: + depth = depth.unsqueeze(1) + elif depth.ndim == 3: + depth = depth.unsqueeze(0).unsqueeze(0) + else: + raise ValueError(f"Invalid depth shape: {depth.shape}") + depth = self._normalize_depth(depth) + assert depth.ndim == 5, "depth must be of shape (n_envs, H, W, 1)" + + env_idx = range(rgb.shape[0]) if rgb is not None else range(depth.shape[0]) + args_list = [(self.export_dir, i_env, 0, self._get_camera_name(i_cam), rgb, depth, i_step) for i_env in env_idx] with ThreadPoolExecutor() as executor: executor.map(FrameImageExporter._worker_export_frame_cam, args_list)