diff --git a/README.md b/README.md index e22ac6d7..7e90eb76 100644 --- a/README.md +++ b/README.md @@ -27,25 +27,22 @@ While gs-madrona currently depends on Genesis, we plan to decouple it in the nea - CUDA kernel caching with dirty-check rebuild - Fixed vertex normal computation in the ray tracer - Benchmark scripts comparing Madrona with other batch renderers, including IsaacLab and ManiSkill +- Support for normal and semantic/instance segmentation output +- Per-camera dynamic FOV, near/far plane control +- Light color specification and attenuation based on distance and angle ## Removed Features - Legacy depth-only rendering via color buffer - Batch rendering pipeline based on JAX ## Known Limitations -- Only color and depth outputs are currently supported - Shadows are only cast from the first light with `castshadow=true` - When rendering multiple cameras with different resolutions, the first camera’s resolution is used for the entire batch ## Roadmap / Future Plans **gs-madrona** will continue evolving to support higher-quality rendering and broader functionality. Upcoming features include: - Batch rendering support for cameras with varying resolutions -- Normal buffer and semantic/instance segmentation output -- Per-camera dynamic FOV control -- Camera-specific near/far plane configuration -- Light color specification - Dynamic light parameters (position, direction, intensity, color, enable/disable) -- Light attenuation based on distance and angle - Ambient lighting control (color and intensity) - PBR material and texture support - Output rendering results to video files diff --git a/scripts/perf_benchmark/README.md b/scripts/perf_benchmark/README.md index 9ed46651..6bb7cc6f 100644 --- a/scripts/perf_benchmark/README.md +++ b/scripts/perf_benchmark/README.md @@ -29,6 +29,7 @@ perf_benchmark/ │ ├── benchmark_config_smoke_test.yml # Quick test configuration │ ├── benchmark_config_madrona.yml # Madrona-specific config │ ├── benchmark_config_omni.yml # Omniverse-specific config +│ ├── benchmark_config_maniskill.yml # ManiSkill-specific config │ └── benchmark_config_full.yml # Comprehensive test config ``` @@ -36,14 +37,24 @@ perf_benchmark/ ### 1. Optional steps -IsaacLab and Maniskill needs to be installed if they need to be benchmarked. - -Install IsaacLab -- Download and install IsaacLab from https://developer.nvidia.com/isaac-sim -- Add IsaacLab to your PATH: - -Install Maniskill -- Install ManiSkill2 following the [official instructions](https://github.com/haosulab/ManiSkill2) +To enable benchmarking with IsaacLab and ManiSkill, follow these optional setup steps: + +- Install IsaacLab + - Download and install IsaacLab from [NVIDIA IsaacLab Documentation](https://isaac-sim.github.io/IsaacLab/main/source/setup/installation/index.html). + - Add the IsaacLab installation directory to your system `PATH`. +- Install ManiSkill + - Install ManiSkill2 by following [ManiSkill Documentation](https://maniskill.readthedocs.io/en/latest/user_guide/). +- Set Environment Variables + - Both IsaacLab and ManiSkill use the `ASSET_DIR` environment variable to locate the Genesis assets directory. + - Use `genesis.utils.misc.asset_dir()` in Genesis to retrieve the exact directory path and then set the environment variable: + ``` + export ASSET_DIR=/path/to/genesis/asset_dir + ``` +- Preprocess Required Assets + - IsaacLab benchmarks require MJCF assets to be preprocessed for compatibility with Omniverse. + ```bash + python process_xml.py --file ./configs/benchmark_config_omni.yml + ``` ### 2. Run a Quick Smoke Test @@ -63,12 +74,6 @@ python batch_benchmark.py -f benchmark_config_full.yml python batch_benchmark.py -f benchmark_config_full.yml -c /name/of/previous/run/folder ``` -### 5. Preprocess MUJUCO XML Assets to make it compatible with Omniverse (if needed) - -```bash -python process_xml.py --file ./genesis/assets/xml/franka_emika_panda/panda.xml -``` - ## Configuration Files Configuration files are YAML-based and define the test parameters. Here's an example structure: diff --git a/scripts/perf_benchmark/batch_benchmark.py b/scripts/perf_benchmark/batch_benchmark.py index e5223013..576ebdb4 100644 --- a/scripts/perf_benchmark/batch_benchmark.py +++ b/scripts/perf_benchmark/batch_benchmark.py @@ -225,6 +225,21 @@ def create_benchmark_result_file(continue_from): return benchmark_result_file +def write_benchmark_result_file(args: BenchmarkArgs, performance_results: dict): + os.makedirs(os.path.dirname(args.benchmark_result_file), exist_ok=True) + with open(args.benchmark_result_file, "a") as f: + f.write( + f"succeeded,{args.mjcf},{args.renderer}," + f"{args.rasterizer},{args.n_envs},{args.n_steps}," + f"{args.resX},{args.resY}," + f"{args.camera_posX},{args.camera_posY},{args.camera_posZ}," + f"{args.camera_lookatX},{args.camera_lookatY},{args.camera_lookatZ}," + f"{args.camera_fov}," + f"{performance_results['time_taken_gpu']},{performance_results['time_taken_per_env_gpu']},{performance_results['time_taken_cpu']}," + f"{performance_results['time_taken_per_env_cpu']},{performance_results['fps']},{performance_results['fps_per_env']}\n" + ) + + def get_previous_runs(continue_from_file): if continue_from_file is None: return [] diff --git a/scripts/perf_benchmark/benchmark_assets/plane_urdf/checker.png b/scripts/perf_benchmark/benchmark_assets/plane_urdf/checker.png new file mode 100644 index 00000000..7c57d041 Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_urdf/checker.png differ diff --git a/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.mtl b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.mtl new file mode 100644 index 00000000..12fb0655 --- /dev/null +++ b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.mtl @@ -0,0 +1,14 @@ +newmtl Material + Ns 10.0000 + Ni 1.5000 + d 1.0000 + Tr 0.0000 + Tf 1.0000 1.0000 1.0000 + illum 2 + Ka 0.0000 0.0000 0.0000 + Kd 1.0000 1.0000 1.0000 + Ks 0.0000 0.0000 0.0000 + Ke 0.0000 0.0000 0.0000 + map_Ka cube.tga + map_Kd checker.png + diff --git a/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.urdf b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.urdf new file mode 100644 index 00000000..448eabee --- /dev/null +++ b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane.urdf @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane100.obj b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane100.obj new file mode 100644 index 00000000..3a74f590 --- /dev/null +++ b/scripts/perf_benchmark/benchmark_assets/plane_urdf/plane100.obj @@ -0,0 +1,22 @@ +# Blender v2.66 (sub 1) OBJ File: '' +# www.blender.org +mtllib plane.mtl +o Plane +v 100.000000 -100.000000 0.000000 +v 100.000000 100.000000 0.000000 +v -100.000000 100.000000 0.000000 +v -100.000000 -100.000000 0.000000 + +vt 100.000000 0.000000 +vt 100.000000 100.000000 +vt 0.000000 100.000000 +vt 0.000000 0.000000 + + + +usemtl Material +s off +f 1/1 2/2 3/3 +f 1/1 3/3 4/4 + + diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/.asset_hash b/scripts/perf_benchmark/benchmark_assets/plane_usd/.asset_hash new file mode 100644 index 00000000..c2288d4a --- /dev/null +++ b/scripts/perf_benchmark/benchmark_assets/plane_usd/.asset_hash @@ -0,0 +1 @@ +b3f970c53c90c6563970b4d47b8e0bab \ No newline at end of file diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/config.yaml b/scripts/perf_benchmark/benchmark_assets/plane_usd/config.yaml new file mode 100644 index 00000000..22459db1 --- /dev/null +++ b/scripts/perf_benchmark/benchmark_assets/plane_usd/config.yaml @@ -0,0 +1,18 @@ +asset_path: genesis/assets/urdf/plane/plane.urdf +usd_dir: null +usd_file_name: null +force_usd_conversion: true +make_instanceable: true +fix_base: true +root_link_name: null +link_density: 0.0 +merge_fixed_joints: true +convert_mimic_joints_to_normal_joints: false +joint_drive: null +collider_type: convex_hull +self_collision: false +replace_cylinders_with_capsules: false +collision_from_visuals: false +## +# Generated by UrdfConverter on 2025-06-10 at 18:41:48. +## diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/materials/textures/checker.png b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/materials/textures/checker.png new file mode 100644 index 00000000..7c57d041 Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/materials/textures/checker.png differ diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_base.usd b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_base.usd new file mode 100644 index 00000000..c73a418d Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_base.usd differ diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_physics.usd b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_physics.usd new file mode 100644 index 00000000..e3d61822 Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_physics.usd differ diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_sensor.usd b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_sensor.usd new file mode 100644 index 00000000..5f117bb3 Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_usd/configuration/plane_sensor.usd differ diff --git a/scripts/perf_benchmark/benchmark_assets/plane_usd/plane.usd b/scripts/perf_benchmark/benchmark_assets/plane_usd/plane.usd new file mode 100644 index 00000000..44550cfc Binary files /dev/null and b/scripts/perf_benchmark/benchmark_assets/plane_usd/plane.usd differ diff --git a/scripts/perf_benchmark/benchmark_madrona.py b/scripts/perf_benchmark/benchmark_madrona.py index 5a09a736..dbadb69e 100644 --- a/scripts/perf_benchmark/benchmark_madrona.py +++ b/scripts/perf_benchmark/benchmark_madrona.py @@ -1,8 +1,9 @@ import os import genesis as gs +from genesis.utils.image_exporter import FrameImageExporter -from batch_benchmark import BenchmarkArgs +from batch_benchmark import BenchmarkArgs, write_benchmark_result_file from benchmark_profiler import BenchmarkProfiler @@ -64,7 +65,7 @@ def init_gs(benchmark_args): pos=(0.0, 0.0, 1.5), dir=(1.0, 1.0, -2.0), directional=True, - castshadow=True, + castshadow=False, cutoff=45.0, intensity=0.5, ) @@ -72,9 +73,9 @@ def init_gs(benchmark_args): pos=(4, -4, 4), dir=(-1, 1, -1), directional=False, - castshadow=True, + castshadow=False, cutoff=45.0, - intensity=1, + intensity=0.5, ) ########################## build ########################## scene.build(n_envs=benchmark_args.n_envs) @@ -88,33 +89,38 @@ def run_benchmark(scene, benchmark_args): # warmup scene.step() - rgb, depth, _, _ = scene.render_all_cameras() + rgb, depth, _, _ = scene.render_all_cameras(rgb=True, depth=True) # Profiler profiler = BenchmarkProfiler(n_steps, n_envs) + output_dir = os.path.dirname(benchmark_args.benchmark_result_file) + os.makedirs(output_dir, exist_ok=True) + image_dirname = f"{benchmark_args.renderer}-{benchmark_args.rasterizer}-{benchmark_args.n_envs}-{benchmark_args.resX}" + image_dir = os.path.join(output_dir, image_dirname) + if n_steps < 10: + exporter = FrameImageExporter(image_dir) + for i in range(n_steps): profiler.on_simulation_start() scene.step() profiler.on_rendering_start() - rgb, depth, _, _ = scene.render_all_cameras() + rgb, depth, _, _ = scene.render_all_cameras(rgb=True, depth=True) profiler.on_rendering_end() - + if n_steps < 10: + exporter.export_frame_all_cameras(i, rgb=rgb) profiler.end() profiler.print_summary() - time_taken_gpu = profiler.get_total_rendering_gpu_time() - time_taken_cpu = profiler.get_total_rendering_cpu_time() - time_taken_per_env_gpu = profiler.get_total_rendering_gpu_time_per_env() - time_taken_per_env_cpu = profiler.get_total_rendering_cpu_time_per_env() - fps = profiler.get_rendering_fps() - fps_per_env = profiler.get_rendering_fps_per_env() - - # Append a line with all args and results in csv format - os.makedirs(os.path.dirname(benchmark_args.benchmark_result_file), exist_ok=True) - with open(benchmark_args.benchmark_result_file, "a") as f: - f.write( - f"succeeded,{benchmark_args.mjcf},{benchmark_args.renderer},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken_gpu},{time_taken_per_env_gpu},{time_taken_cpu},{time_taken_per_env_cpu},{fps},{fps_per_env}\n" - ) + performance_results = { + "time_taken_gpu": profiler.get_total_rendering_gpu_time(), + "time_taken_cpu": profiler.get_total_rendering_cpu_time(), + "time_taken_per_env_gpu": profiler.get_total_rendering_gpu_time_per_env(), + "time_taken_per_env_cpu": profiler.get_total_rendering_cpu_time_per_env(), + "fps": profiler.get_rendering_fps(), + "fps_per_env": profiler.get_rendering_fps_per_env(), + } + write_benchmark_result_file(benchmark_args, performance_results) + except Exception as e: print(f"Error during benchmark: {e}") raise diff --git a/scripts/perf_benchmark/benchmark_maniskill.py b/scripts/perf_benchmark/benchmark_maniskill.py index cf09649d..aa283e46 100644 --- a/scripts/perf_benchmark/benchmark_maniskill.py +++ b/scripts/perf_benchmark/benchmark_maniskill.py @@ -13,8 +13,13 @@ from mani_skill.utils.registration import register_env from mani_skill.utils.wrappers.flatten import FlattenActionSpaceWrapper -from batch_benchmark import BenchmarkArgs -from benchmark_profiler import BenchmarkProfiler +from batch_benchmark import BenchmarkArgs, write_benchmark_result_file +from benchmark_profiler import BenchmarkProfiler, get_utilization_percentages, print_system_utilization + + +# Get asset directory from environment variable +asset_dir = os.path.abspath(os.getenv("ASSET_DIR")) +benchmark_dir = os.path.abspath(os.path.dirname(__file__)) @register_env("SingleRobotBenchmark-v1") @@ -63,7 +68,7 @@ def _load_agent(self, options: dict): super()._load_agent(options, sapien.Pose(p=[0, 0, 0], q=[1, 0, 0, 0])) def _load_scene(self, options: dict): - ground_path = "genesis/assets/urdf/plane/plane.urdf" + ground_path = os.path.join(benchmark_dir, "benchmark_assets/plane_urdf/plane.urdf") urdf_loader = self.scene.create_urdf_loader() urdf_loader.fix_root_link = True ground_actor = urdf_loader.parse(ground_path)["actor_builders"][0] @@ -76,12 +81,8 @@ def _initialize_episode(self, env_idx: torch.Tensor, options: dict): self.agent.robot.set_qpos(qpos) def _load_lighting(self, options: Dict): - self.scene.add_directional_light( - [ - 0.26490647, - 0.26490647, - -0.92717265, - ], # norm([1.0, 1.0, -2.0] - [0, 0, 1.5]) + self.scene.add_directional_light( # norm([1.0, 1.0, -2.0] - [0, 0, 1.5]) + [0.26490647, 0.26490647, -0.92717265], [3.0, 3.0, 3.0], shadow=False, ) @@ -123,9 +124,9 @@ def main(): camera_mode = "minimal" else: camera_mode = "rt-fast" - save_image = True obs_mode = "rgbd" # Or "rgb" - render_mode = "sensors" # "human for GUI" + # render_mode = "sensors" + render_mode = "rgb_array" # "human for GUI" env = gym.make( env_id, num_envs=n_envs, @@ -135,8 +136,8 @@ def main(): sim_config=sim_config, robot_uid=robot_uid, camera_mode=camera_mode, - camera_width=args.resY, - camera_height=args.resX, + camera_width=args.resX, + camera_height=args.resY, # parallel_in_single_scene=True, # This actually combines all environments into a single env, is it batch rendering? ) if isinstance(env.action_space, gym.spaces.Dict): @@ -144,9 +145,11 @@ def main(): base_env: BaseEnv = env.unwrapped base_env.print_sim_details() - # image_dir = "data/render/maniskill" - image_dir = os.path.splitext(args.benchmark_result_file)[0] - os.makedirs(image_dir, exist_ok=True) + # Build image output directory similar to _omni + output_dir = os.path.dirname(args.benchmark_result_file) + os.makedirs(output_dir, exist_ok=True) + image_dirname = f"{args.renderer}-{args.rasterizer}-{args.n_envs}-{args.resX}" + image_dir = os.path.join(output_dir, image_dirname) image_tiles = [] profiler = BenchmarkProfiler(n_steps, n_envs) @@ -158,57 +161,48 @@ def main(): env.reset(seed=2022) for i in range(n_steps): # obs, rew, terminated, truncated, info = env.step(actions) - profiler.on_simulation_start() - obs = env.step(None)[0] - # image_tile = obs["sensor_data"]["render_camera"]["rgb"] print(f"Step: {i}") + profiler.on_simulation_start() profiler.on_rendering_start() + obs = env.step(None)[0] + # When render_mode="sensors", speed of env.render() suffers a great decline in large batch size. + profiler.on_rendering_end() if render_mode == "human": viewer = env.render() + image_tile = None else: image_tile = env.render() - if save_image: - image_tiles.append(image_tile) - profiler.on_rendering_end() + image_tile = obs["sensor_data"]["render_camera"]["rgb"] + + if n_steps < 10: + image_tiles.append(image_tile) + + if i % 10 == 0: + system_analysis = get_utilization_percentages() + print_system_utilization(system_analysis) profiler.end() profiler.print_summary() - if save_image: + if n_steps < 10: + os.makedirs(image_dir, exist_ok=True) image_tiles = [image_tile.cpu().numpy() for image_tile in image_tiles] - for i in range(min(n_steps, 10)): - for j in range(min(n_envs, 10)): + for i in range(n_steps): + for j in range(n_envs): image_pil = Image.fromarray(image_tiles[i][j]) image_path = os.path.join(image_dir, f"step{i:02d}_env{j:02d}_{camera_mode}_{robot_uid}.png") print(f"Image saved: {image_path}") image_pil.save(image_path) - # image_tiles = [tile_images(image_tile, nrows=video_nrows) for image_tile in image_tiles] - # images_to_video( - # image_tiles, - # output_dir=image_dir, - # video_name=f"mani_skill-{env_id}-num_envs={args.num_envs}-{args.cam_mode}-{args.robot_uid}", - # fps=30, - # ) env.close() - time_taken_gpu = profiler.get_total_rendering_gpu_time() - time_taken_cpu = profiler.get_total_rendering_cpu_time() - time_taken_per_env_gpu = profiler.get_total_rendering_gpu_time_per_env() - time_taken_per_env_cpu = profiler.get_total_rendering_cpu_time_per_env() - fps = profiler.get_rendering_fps() - fps_per_env = profiler.get_rendering_fps_per_env() - - os.makedirs(os.path.dirname(args.benchmark_result_file), exist_ok=True) - with open(args.benchmark_result_file, "a") as f: - f.write( - f"succeeded,{args.mjcf},{args.renderer}," - f"{args.rasterizer},{args.n_envs},{args.n_steps}," - f"{args.resX},{args.resY}," - f"{args.camera_posX},{args.camera_posY},{args.camera_posZ}," - f"{args.camera_lookatX},{args.camera_lookatY},{args.camera_lookatZ}," - f"{args.camera_fov}," - f"{time_taken_gpu},{time_taken_per_env_gpu},{time_taken_cpu}," - f"{time_taken_per_env_cpu},{fps},{fps_per_env}\n" - ) + performance_results = { + "time_taken_gpu": profiler.get_total_rendering_gpu_time(), + "time_taken_cpu": profiler.get_total_rendering_cpu_time(), + "time_taken_per_env_gpu": profiler.get_total_rendering_gpu_time_per_env(), + "time_taken_per_env_cpu": profiler.get_total_rendering_cpu_time_per_env(), + "fps": profiler.get_rendering_fps(), + "fps_per_env": profiler.get_rendering_fps_per_env(), + } + write_benchmark_result_file(args, performance_results) if __name__ == "__main__": diff --git a/scripts/perf_benchmark/benchmark_omni.py b/scripts/perf_benchmark/benchmark_omni.py index 35896525..36b6a5a4 100644 --- a/scripts/perf_benchmark/benchmark_omni.py +++ b/scripts/perf_benchmark/benchmark_omni.py @@ -1,99 +1,152 @@ -# Before running, convert the assets like: -# python examples/perf_benchmark/process_xml.py \ -# --file ./genesis/assets/xml/franka_emika_panda/panda.xml - ######################## Parse arguments ####################### -# Create a struct to store the arguments -from batch_benchmark import BenchmarkArgs - +from batch_benchmark import BenchmarkArgs, write_benchmark_result_file benchmark_args = BenchmarkArgs.parse_benchmark_args() - ######################## Launch app ####################### from isaaclab.app import AppLauncher - app = AppLauncher( headless=not benchmark_args.gui, enable_cameras=True, device="cuda:0", - rendering_mode="performance", ).app +import os +import math +from scipy.spatial.transform import Rotation as R +import torch +from pxr import PhysxSchema +from PIL import Image + import carb +import omni.replicator.core as rep import isaaclab.sim as sim_utils -import isaacsim.core.utils.prims as prim_utils import isaacsim.core.utils.stage as stage_utils -from isaaclab.sensors.camera import TiledCamera, TiledCameraCfg -from isaaclab.sim.converters import ( - MjcfConverter, - MjcfConverterCfg, - UrdfConverter, - UrdfConverterCfg, -) -from isaaclab.utils.math import ( - create_rotation_matrix_from_view, - quat_from_matrix, -) -import omni.replicator.core as rep -from pxr import UsdLux, PhysxSchema - +import isaaclab.assets as asset_utils +import isaaclab_assets.robots as asset_robots +from isaaclab.scene.interactive_scene import InteractiveScene +from isaaclab.sensors import TiledCameraCfg +from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix +from isaaclab.utils import configclass +from isaaclab.scene import InteractiveSceneCfg +from isaaclab.sim.converters import MjcfConverter, MjcfConverterCfg from isaacsim.core.utils.extensions import enable_extension - enable_extension("isaacsim.asset.importer.mjcf") +import isaacsim.asset.importer.mjcf -import os -import math -import numpy as np -import torch -import psutil -import pynvml - -from scipy.spatial.transform import Rotation as R -from genesis.utils.image_exporter import FrameImageExporter -from benchmark_profiler import BenchmarkProfiler +from benchmark_profiler import BenchmarkProfiler, get_utilization_percentages, print_system_utilization -def load_mjcf(mjcf_path): - return MjcfConverter(MjcfConverterCfg(asset_path=mjcf_path, fix_base=True, force_usd_conversion=True)).usd_path +# Get asset directory from environment variable +asset_dir = os.path.abspath(os.getenv("ASSET_DIR")) +benchmark_dir = os.path.abspath(os.path.dirname(__file__)) -def load_urdf(urdf_path): - return UrdfConverter( - UrdfConverterCfg( - asset_path=urdf_path, - joint_drive=None, +def load_mjcf(mjcf_path: str) -> str: + return MjcfConverter( + MjcfConverterCfg( + asset_path=mjcf_path, fix_base=True, - force_usd_conversion=True, + force_usd_conversion=True ) ).usd_path -def apply_benchmark_carb_settings(print_changes=False): +def get_robot_config() -> asset_utils.AssetBaseCfg: + robot_name = f"{os.path.splitext(benchmark_args.mjcf)[0]}_new.xml" + robot_path = load_mjcf(os.path.abspath(os.path.join(asset_dir, robot_name))) + print("Robot asset:", robot_path) + + if benchmark_args.mjcf.endswith("g1.xml"): + robot_cfg = asset_utils.AssetBaseCfg( + spawn=asset_robots.unitree.G1_CFG.spawn.copy() + ) + elif benchmark_args.mjcf.endswith("go2.xml"): + robot_cfg = asset_utils.AssetBaseCfg( + spawn=asset_robots.unitree.UNITREE_GO2_CFG.spawn.copy() + ) + elif benchmark_args.mjcf.endswith("panda.xml"): + robot_cfg = asset_utils.AssetBaseCfg( + spawn=asset_robots.franka.FRANKA_PANDA_CFG.spawn.copy() + ) + else: + raise Exception(f"Invalid robot: {benchmark_args.mjcf}") + robot_cfg.spawn.usd_path = robot_path + return robot_cfg.replace(prim_path="{ENV_REGEX_NS}/Robot") + + +def get_dir_light_config() -> asset_utils.AssetBaseCfg: + dir_light_pos = torch.Tensor([[0.0, 0.0, 1.5]]) + dir_light_quat = quat_from_matrix( + create_rotation_matrix_from_view( + dir_light_pos, + torch.Tensor([[1.0, 1.0, -2.0]]), + stage_utils.get_stage_up_axis())) + dir_light_pos = tuple(dir_light_pos.detach().cpu().squeeze().numpy()) + dir_light_quat = tuple(dir_light_quat.detach().cpu().squeeze().numpy()) + dir_light_cfg = asset_utils.AssetBaseCfg( + prim_path="/World/direct_light", + spawn=sim_utils.DistantLightCfg(intensity=500.0, angle=45.0), + init_state=asset_utils.AssetBaseCfg.InitialStateCfg( + pos=dir_light_pos, rot=dir_light_quat + ) + ) + return dir_light_cfg + + +@configclass +class RobotSceneCfg(InteractiveSceneCfg): + """Configuration for a cart-pole scene.""" + ground = asset_utils.AssetBaseCfg( + # prim_path="{ENV_REGEX_NS}/ground", # Each environment should have a ground + prim_path="/World/ground", # All environment shares a ground + spawn=sim_utils.UsdFileCfg( + usd_path=os.path.abspath(os.path.join(benchmark_dir, "benchmark_assets/plane_usd/plane.usd")) + ), + ) + robot: asset_utils.ArticulationCfg = get_robot_config() + dir_light = get_dir_light_config() + + +def apply_benchmark_physics_settings(): + stage = stage_utils.get_current_stage() + physxSceneAPI = PhysxSchema.PhysxSceneAPI.Apply(stage.GetPrimAtPath("/physicsScene")) + physxSceneAPI.CreateGpuTempBufferCapacityAttr(16 * 1024 * 1024 * 2) + physxSceneAPI.CreateGpuHeapCapacityAttr(64 * 1024 * 1024 * 2) + physxSceneAPI.CreateGpuMaxRigidPatchCountAttr(8388608) + physxSceneAPI.CreateGpuMaxRigidContactCountAttr(16777216) + + +def print_render_settings(settings): + print("Render mode:", settings.get("/rtx/rendermode")) + print("Sample per pixel:", settings.get("/rtx/pathtracing/spp")) + print("Total spp:", settings.get("/rtx/pathtracing/totalSpp")) + print("Clamp spp:", settings.get("/rtx/pathtracing/clampSpp")) + print("Max bounce:", settings.get("/rtx/pathtracing/maxBounces")) + print("Optix Denoiser", settings.get("/rtx/pathtracing/optixDenoiser/enabled")) + print("Shadows", settings.get("/rtx/shadows/enabled")) + print("dlss/enabled:", settings.get("/rtx/post/dlss/enabled")) + print("dlss/auto:", settings.get("/rtx/post/dlss/auto")) + print("upscaling/enabled:", settings.get("/rtx/post/upscaling/enabled")) + print("aa/denoiser/enabled:", settings.get("/rtx/post/aa/denoiser/enabled")) + print("aa/taa/enabled:", settings.get("/rtx/post/aa/taa/enabled")) + print("motionBlur/enabled:", settings.get("/rtx/post/motionBlur/enabled")) + print("dof/enabled:", settings.get("/rtx/post/dof/enabled")) + print("bloom/enabled:", settings.get("/rtx/post/bloom/enabled")) + print("tonemap/enabled:", settings.get("/rtx/post/tonemap/enabled")) + print("exposure/enabled:", settings.get("/rtx/post/exposure/enabled")) + print("vsync:", settings.get("/app/window/vsync")) + + +def apply_benchmark_carb_settings(print_changes: bool = False) -> None: + # rep.settings.set_render_rtx_realtime() # Keep default pipeline; explicitly set below settings = carb.settings.get_settings() + # Print settings before applying the settings if print_changes: print("Before settings:") - print("Render mode:", settings.get("/rtx/rendermode")) - print("Sample per pixel:", settings.get("/rtx/pathtracing/spp")) - print("Total spp:", settings.get("/rtx/pathtracing/totalSpp")) - print("Clamp spp:", settings.get("/rtx/pathtracing/clampSpp")) - print("Max bounce:", settings.get("/rtx/pathtracing/maxBounces")) - print("Optix Denoiser", settings.get("/rtx/pathtracing/optixDenoiser/enabled")) - print("Shadows", settings.get("/rtx/shadows/enabled")) - print("dlss/enabled:", settings.get("/rtx/post/dlss/enabled")) - print("dlss/auto:", settings.get("/rtx/post/dlss/auto")) - print("upscaling/enabled:", settings.get("/rtx/post/upscaling/enabled")) - print("aa/denoiser/enabled:", settings.get("/rtx/post/aa/denoiser/enabled")) - print("aa/taa/enabled:", settings.get("/rtx/post/aa/taa/enabled")) - print("motionBlur/enabled:", settings.get("/rtx/post/motionBlur/enabled")) - print("dof/enabled:", settings.get("/rtx/post/dof/enabled")) - print("bloom/enabled:", settings.get("/rtx/post/bloom/enabled")) - print("tonemap/enabled:", settings.get("/rtx/post/tonemap/enabled")) - print("exposure/enabled:", settings.get("/rtx/post/exposure/enabled")) - print("vsync:", settings.get("/app/window/vsync")) + print_render_settings(settings) # Options: https://docs.omniverse.nvidia.com/materials-and-rendering/latest/rtx-renderer_pt.html if benchmark_args.rasterizer: - # carb_settings.set("/rtx/rendermode", "Hydra Storm") settings.set("/rtx/rendermode", "RayTracedLighting") else: settings.set("/rtx/rendermode", "PathTracing") @@ -128,281 +181,150 @@ def apply_benchmark_carb_settings(print_changes=False): # Print settings after applying the settings if print_changes: print("After settings:") - print("Render mode:", settings.get("/rtx/rendermode")) - print("Sample per pixel:", settings.get("/rtx/pathtracing/spp")) - print("Total spp:", settings.get("/rtx/pathtracing/totalSpp")) - print("Clamp spp:", settings.get("/rtx/pathtracing/clampSpp")) - print("Max bounce:", settings.get("/rtx/pathtracing/maxBounces")) - print("Optix Denoiser", settings.get("/rtx/pathtracing/optixDenoiser/enabled")) - print("Shadows", settings.get("/rtx/shadows/enabled")) - print("dlss/enabled:", settings.get("/rtx/post/dlss/enabled")) - print("dlss/auto:", settings.get("/rtx/post/dlss/auto")) - print("upscaling/enabled:", settings.get("/rtx/post/upscaling/enabled")) - print("aa/denoiser/enabled:", settings.get("/rtx/post/aa/denoiser/enabled")) - print("aa/taa/enabled:", settings.get("/rtx/post/aa/taa/enabled")) - print("motionBlur/enabled:", settings.get("/rtx/post/motionBlur/enabled")) - print("dof/enabled:", settings.get("/rtx/post/dof/enabled")) - print("bloom/enabled:", settings.get("/rtx/post/bloom/enabled")) - print("tonemap/enabled:", settings.get("/rtx/post/tonemap/enabled")) - print("exposure/enabled:", settings.get("/rtx/post/exposure/enabled")) - print("vsync:", settings.get("/app/window/vsync")) - - -def init_isaac(benchmark_args): - ########################## init ########################## - stage_utils.create_new_stage() - stage = stage_utils.get_current_stage() - scene = sim_utils.SimulationContext( - sim_utils.SimulationCfg( - device="cuda:0", - dt=0.01, - ) + print_render_settings(settings) + + +def create_scene(): + """Create simulation and scene with camera and physics/render settings applied.""" + sim_cfg = sim_utils.SimulationCfg( + device="cuda:0", dt=0.01, use_fabric=False, ) - cam_eye = ( + sim = sim_utils.SimulationContext(sim_cfg) + scene_cfg = RobotSceneCfg(num_envs=benchmark_args.n_envs, env_spacing=10.0) + + apply_benchmark_physics_settings() + apply_benchmark_carb_settings(True) + + camera_fov = math.radians(benchmark_args.camera_fov) + camera_aperture = 20.955 + camera_fol = camera_aperture / (2 * math.tan(camera_fov / 2)) + + camera_pos = torch.tensor(( benchmark_args.camera_posX, benchmark_args.camera_posY, - benchmark_args.camera_posZ, - ) - cam_target = ( + benchmark_args.camera_posZ + )).reshape(-1, 3) + camera_lookat = torch.tensor(( benchmark_args.camera_lookatX, benchmark_args.camera_lookatY, - benchmark_args.camera_lookatZ, - ) - scene.set_camera_view(eye=cam_eye, target=cam_target) - cam_eye = torch.Tensor(cam_eye).reshape(-1, 3) - cam_target = torch.Tensor(cam_target).reshape(-1, 3) - - physxSceneAPI = PhysxSchema.PhysxSceneAPI.Apply(stage.GetPrimAtPath("/physicsScene")) - physxSceneAPI.CreateGpuTempBufferCapacityAttr(16 * 1024 * 1024 * 2) - physxSceneAPI.CreateGpuHeapCapacityAttr(64 * 1024 * 1024 * 2) - physxSceneAPI.CreateGpuMaxRigidPatchCountAttr(8388608) - physxSceneAPI.CreateGpuMaxRigidContactCountAttr(16777216) - - rep.settings.set_render_rtx_realtime() - apply_benchmark_carb_settings() - - ########################## entities ########################## - spacing_row = np.array((2.0, -6.0)) - spacing_col = np.array((-6.0, -2.0)) - n_cols = int(math.sqrt(benchmark_args.n_envs)) - offsets = [] - for i in range(benchmark_args.n_envs): - col = i % n_cols - row = i // n_cols - offset_XY = row * spacing_row + col * spacing_col - offset = np.array([*offset_XY, 0.0]) - offsets.append(offset) - prim_utils.create_prim(f"/World/Origin{i:05d}", "Xform", translation=offset) - offsets = np.array(offsets) - - # load objects - plane_path = os.path.abspath(os.path.join("genesis/assets", "urdf/plane_usd/plane.usd")) - print(plane_path) - plane_cfg = sim_utils.UsdFileCfg(usd_path=plane_path) - plane_cfg.func("/World/Origin.*/plane", plane_cfg) - - robot_name = f"{os.path.splitext(benchmark_args.mjcf)[0]}_new.xml" - robot_path = load_mjcf(os.path.join("genesis/assets", robot_name)) - print("Robot asset:", robot_path) - robot_cfg = sim_utils.UsdFileCfg(usd_path=robot_path) - robot_cfg.func("/World/Origin.*/robot", robot_cfg) - - cam_fov = math.radians(benchmark_args.camera_fov) - cam_hapert = 20.955 - cam_fol = cam_hapert / (2 * math.tan(cam_fov / 2)) - cam_quat = quat_from_matrix( - create_rotation_matrix_from_view(cam_target, cam_eye, stage_utils.get_stage_up_axis()) - @ R.from_euler("z", 180, degrees=True).as_matrix() - ) - cam_eye = tuple(cam_eye.detach().cpu().squeeze().numpy()) - cam_quat = tuple(cam_quat.detach().cpu().squeeze().numpy()) - - print(cam_eye, cam_quat) - print(type(cam_eye), type(cam_quat)) - - cam_0 = TiledCamera( - TiledCameraCfg( - height=benchmark_args.resX, - width=benchmark_args.resY, - offset=TiledCameraCfg.OffsetCfg(pos=cam_eye, rot=cam_quat, convention="ros"), - prim_path="/World/Origin.*/camera", - update_period=0, - data_types=["rgb", "depth"], - spawn=sim_utils.PinholeCameraCfg( - focal_length=cam_fol, - ), - ) - ) - - ########################## cameras ########################## - dir_light_pos = torch.Tensor([[0.0, 0.0, 1.5]]) - dir_light_quat = quat_from_matrix( + benchmark_args.camera_lookatZ + )).reshape(-1, 3) + camera_quat = quat_from_matrix( create_rotation_matrix_from_view( - dir_light_pos, - torch.Tensor([[1.0, 1.0, -2.0]]), - stage_utils.get_stage_up_axis(), - ) - ) - dir_light_pos = tuple(dir_light_pos.detach().cpu().squeeze().numpy()) - dir_light_quat = tuple(dir_light_quat.detach().cpu().squeeze().numpy()) - dir_light_cfg = sim_utils.DistantLightCfg(intensity=500.0, angle=45.0) - dir_light_prim = dir_light_cfg.func( - "/World/DirectionalLight", - dir_light_cfg, - translation=dir_light_pos, - orientation=dir_light_quat, - ) - - cone_light_pos = torch.Tensor([[4, -4, 4]]) - cone_light_quat = quat_from_matrix( - create_rotation_matrix_from_view(cone_light_pos, torch.Tensor([[-1, 1, -1]]), stage_utils.get_stage_up_axis()) + camera_lookat, camera_pos, stage_utils.get_stage_up_axis() + ) @ R.from_euler('z', 180, degrees=True).as_matrix() ) - cone_light_cfg = sim_utils.SphereLightCfg(intensity=1000.0, radius=0.1) - cone_light_pos = tuple(cone_light_pos.detach().cpu().squeeze().numpy()) - cone_light_quat = tuple(cone_light_quat.detach().cpu().squeeze().numpy()) - cone_light_prim = cone_light_cfg.func( - "/World/ConeLight", - cone_light_cfg, - translation=cone_light_pos, - orientation=cone_light_quat, + camera_pos = tuple(camera_pos.detach().cpu().squeeze().numpy()) + camera_quat = tuple(camera_quat.detach().cpu().squeeze().numpy()) + camera_cfg = TiledCameraCfg( + prim_path="{ENV_REGEX_NS}/tiled_camera", + update_period=0, + height=benchmark_args.resY, + width=benchmark_args.resX, + offset=TiledCameraCfg.OffsetCfg( + pos=camera_pos, + rot=camera_quat, + convention="ros" + ), + data_types=["rgb", "depth"], + spawn=sim_utils.PinholeCameraCfg( + focal_length=camera_fol, + horizontal_aperture=camera_aperture, + ), ) - cone_light = UsdLux.LightAPI(cone_light_prim) - UsdLux.ShapingAPI.Apply(cone_light_prim) - cone_light_prim.SetTypeName("SphereLight") - - return scene, cam_0 - - -def get_utilization_percentages(reset: bool = False, max_values: list[float] = [0.0, 0.0, 0.0, 0.0]) -> list[float]: - """Get the maximum CPU, RAM, GPU utilization (processing), and - GPU memory usage percentages since the last time reset was true.""" - if reset: - max_values[:] = [0, 0, 0, 0] # Reset the max values - - # CPU utilization - cpu_usage = psutil.cpu_percent(interval=0.1) - max_values[0] = max(max_values[0], cpu_usage) - - # RAM utilization - memory_info = psutil.virtual_memory() - ram_usage = memory_info.percent - max_values[1] = max(max_values[1], ram_usage) - - # GPU utilization using pynvml - if torch.cuda.is_available(): - pynvml.nvmlInit() # Initialize NVML - for i in range(torch.cuda.device_count()): - handle = pynvml.nvmlDeviceGetHandleByIndex(i) - - # GPU Utilization - gpu_utilization = pynvml.nvmlDeviceGetUtilizationRates(handle) - gpu_processing_utilization_percent = gpu_utilization.gpu # GPU core utilization - max_values[2] = max(max_values[2], gpu_processing_utilization_percent) - - # GPU Memory Usage - memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle) - gpu_memory_total = memory_info.total - gpu_memory_used = memory_info.used - gpu_memory_utilization_percent = (gpu_memory_used / gpu_memory_total) * 100 - max_values[3] = max(max_values[3], gpu_memory_utilization_percent) - - pynvml.nvmlShutdown() # Shutdown NVML after usage - else: - gpu_processing_utilization_percent = None - gpu_memory_utilization_percent = None - return max_values - - -def fill_gpu_cache_with_random_data(): - # 100 MB of random data - dummy_data = torch.rand(100, 1024, 1024, device="cuda") - # Make some random data manipulation to the entire tensor - dummy_data = dummy_data.sqrt() - - -def run_benchmark(scene, camera, benchmark_args): - try: - n_envs = benchmark_args.n_envs - n_steps = benchmark_args.n_steps - - # warmup - system_utilization_analytics = get_utilization_percentages() - print( - f"| CPU:{system_utilization_analytics[0]}% | " - f"RAM:{system_utilization_analytics[1]}% | " - f"GPU Compute:{system_utilization_analytics[2]}% | " - f"GPU Memory: {system_utilization_analytics[3]:.2f}% |" - ) - - scene.reset() - dt = scene.get_physics_dt() - for i in range(3): - scene.step() - camera.update(dt) - _ = camera.data - print("Env and steps:", n_envs, n_steps) - - if benchmark_args.gui: - while True: - scene.step() - - # Create an image exporter - image_dir = os.path.splitext(benchmark_args.benchmark_result_file)[0] - exporter = FrameImageExporter(image_dir) - - # Profiler - profiler = BenchmarkProfiler(n_steps, n_envs) - for i in range(n_steps): - profiler.on_simulation_start() - scene.step(render=False) - profiler.on_rendering_start() - scene.render() - # camera.update(dt, force_recompute=True) - # rgb_tiles = camera.data.output.get("rgb") - # depth_tiles = camera.data.output.get("depth") - profiler.on_rendering_end() - # exporter.export_frame_single_cam(i, 0, rgb=rgb_tiles, depth=depth_tiles) - - profiler.end() - profiler.print_summary() - - time_taken_gpu = profiler.get_total_rendering_gpu_time() - time_taken_cpu = profiler.get_total_rendering_cpu_time() - time_taken_per_env_gpu = profiler.get_total_rendering_gpu_time_per_env() - time_taken_per_env_cpu = profiler.get_total_rendering_cpu_time_per_env() - fps = profiler.get_rendering_fps() - fps_per_env = profiler.get_rendering_fps_per_env() - - print( - f"| CPU:{system_utilization_analytics[0]}% | " - f"RAM:{system_utilization_analytics[1]}% | " - f"GPU Compute:{system_utilization_analytics[2]}% | " - f" GPU Memory: {system_utilization_analytics[3]:.2f}% |" - ) - - # Append a line with all args and results in csv format - os.makedirs(os.path.dirname(benchmark_args.benchmark_result_file), exist_ok=True) - with open(benchmark_args.benchmark_result_file, "a") as f: - f.write( - f"succeeded,{benchmark_args.mjcf},{benchmark_args.renderer},{benchmark_args.rasterizer},{benchmark_args.n_envs},{benchmark_args.n_steps},{benchmark_args.resX},{benchmark_args.resY},{benchmark_args.camera_posX},{benchmark_args.camera_posY},{benchmark_args.camera_posZ},{benchmark_args.camera_lookatX},{benchmark_args.camera_lookatY},{benchmark_args.camera_lookatZ},{benchmark_args.camera_fov},{time_taken_gpu},{time_taken_per_env_gpu},{time_taken_cpu},{time_taken_per_env_cpu},{fps},{fps_per_env}\n" - ) - - print("App closing..") - # app.close() - print("App closed!") - - except Exception as e: - print(f"Error during benchmark: {e}") - raise - - -def main(): - ######################## Initialize scene ####################### - scene, camera = init_isaac(benchmark_args) - - ######################## Run benchmark ####################### - run_benchmark(scene, camera, benchmark_args) + setattr(scene_cfg, "tiled_camera", camera_cfg) + scene = InteractiveScene(scene_cfg) + return sim, scene + + +def run_simulator( + sim: sim_utils.SimulationContext, + scene: InteractiveScene, +) -> None: + """Run the simulator with all cameras, and return timing analytics. Visualize if desired.""" + n_envs = benchmark_args.n_envs + n_steps = benchmark_args.n_steps + camera = scene["tiled_camera"] + camera_data_types = ["rgb", "depth"] + + # Initialize timing variables + system_utilization_analytics = get_utilization_percentages() + print_system_utilization(system_utilization_analytics) + + sim.reset() + dt = sim.get_physics_dt() + n_warm_steps = 3 + for i in range(n_warm_steps): + print(f"Warm up step {i}.") + sim.step() + camera.update(dt) + _ = camera.data + + print("Warm up finished.") + output_dir = os.path.dirname(benchmark_args.benchmark_result_file) + os.makedirs(output_dir, exist_ok=True) + image_dirname = f'{benchmark_args.renderer}-{benchmark_args.rasterizer}-{benchmark_args.n_envs}-{benchmark_args.resX}' + image_dir = os.path.join(output_dir, image_dirname) + + profiler = BenchmarkProfiler(n_steps, n_envs) + for i in range(n_steps): + print(f"Step {i}:") + get_utilization_percentages() + + # Measure the total simulation step time + profiler.on_simulation_start() + sim.step(render=False) + profiler.on_rendering_start() + sim.render() + + # Update cameras and process vision data within the simulation step + # Loop through all camera lists and their data_types + camera.update(dt=dt) + rgb_tiles = camera.data.output.get("rgb") + depth_tiles = camera.data.output.get("depth") + profiler.on_rendering_end() + + if n_steps < 10: + os.makedirs(image_dir, exist_ok=True) + rgb_tiles = rgb_tiles.detach().cpu().numpy() + for j in range(n_envs): + rgb_image = rgb_tiles[j] + rgb_image = Image.fromarray(rgb_image) + + image_name = f"rgb_step{i}_env{j}.png" + image_path = os.path.join(image_dir, image_name) + rgb_image.save(image_path) + print("Image saved:", image_path) + # End timing for the step + + profiler.end() + profiler.print_summary() + + system_utilization_analytics = get_utilization_percentages() + print_system_utilization(system_utilization_analytics) + + performance_results = { + "time_taken_gpu": profiler.get_total_rendering_gpu_time(), + "time_taken_cpu": profiler.get_total_rendering_cpu_time(), + "time_taken_per_env_gpu": profiler.get_total_rendering_gpu_time_per_env(), + "time_taken_per_env_cpu": profiler.get_total_rendering_cpu_time_per_env(), + "fps": profiler.get_rendering_fps(), + "fps_per_env": profiler.get_rendering_fps_per_env(), + } + write_benchmark_result_file(benchmark_args, performance_results) + + print("App closing..") + # app.close() + print("App closed!") + + +def main() -> None: + """Entry point for running the benchmark scene and simulator.""" + sim, scene = create_scene() + run_simulator(sim=sim, scene=scene) if __name__ == "__main__": + # run the main function main() + # simulation_app.close() diff --git a/scripts/perf_benchmark/benchmark_profiler.py b/scripts/perf_benchmark/benchmark_profiler.py index 4fb8c9a4..d288401d 100644 --- a/scripts/perf_benchmark/benchmark_profiler.py +++ b/scripts/perf_benchmark/benchmark_profiler.py @@ -1,7 +1,56 @@ -import torch -import numpy as np import time +import numpy as np +import psutil +import torch +import pynvml + + +def get_utilization_percentages(reset: bool = False, max_values: list[float] = [0.0, 0.0, 0.0, 0.0]) -> list[float]: + """Get the maximum CPU, RAM, GPU utilization (processing), and + GPU memory usage percentages since the last time reset was true.""" + if reset: + max_values[:] = [0, 0, 0, 0] # Reset the max values + + # CPU utilization + cpu_usage = psutil.cpu_percent(interval=0.1) + max_values[0] = max(max_values[0], cpu_usage) + + # RAM utilization + memory_info = psutil.virtual_memory() + ram_usage = memory_info.percent + max_values[1] = max(max_values[1], ram_usage) + + # GPU utilization using pynvml + if torch.cuda.is_available(): + pynvml.nvmlInit() # Initialize NVML + for i in range(torch.cuda.device_count()): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + + # GPU Utilization + gpu_utilization = pynvml.nvmlDeviceGetUtilizationRates(handle) + gpu_processing_utilization_percent = gpu_utilization.gpu # GPU core utilization + max_values[2] = max(max_values[2], gpu_processing_utilization_percent) + + # GPU Memory Usage + memory_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + gpu_memory_total = memory_info.total + gpu_memory_used = memory_info.used + gpu_memory_utilization_percent = (gpu_memory_used / gpu_memory_total) * 100 + max_values[3] = max(max_values[3], gpu_memory_utilization_percent) + + pynvml.nvmlShutdown() # Shutdown NVML after usage + return max_values + + +def print_system_utilization(analytics: list[float]) -> None: + print( + f"| CPU: {analytics[0]}% |" + f" RAM: {analytics[1]}% |" + f" GPU Compute: {analytics[2]}% |" + f" GPU Memory: {analytics[3]:.2f}% |" + ) + class BenchmarkProfiler: def __init__(self, n_steps, n_envs): diff --git a/scripts/perf_benchmark/configs/benchmark_config_madrona.yml b/scripts/perf_benchmark/configs/benchmark_config_madrona.yml index 24d36467..b9782c05 100644 --- a/scripts/perf_benchmark/configs/benchmark_config_madrona.yml +++ b/scripts/perf_benchmark/configs/benchmark_config_madrona.yml @@ -1,7 +1,5 @@ mjcf_list: - xml/franka_emika_panda/panda.xml - - xml/unitree_g1/g1.xml - - xml/unitree_go2/go2.xml renderer_list: - renderer: madrona @@ -57,7 +55,7 @@ raytracer: # Simulation configuration simulation: - n_steps: 1000 + n_steps: 100 # Camera configuration camera: diff --git a/scripts/perf_benchmark/configs/benchmark_config_maniskill.yml b/scripts/perf_benchmark/configs/benchmark_config_maniskill.yml new file mode 100644 index 00000000..1c37780f --- /dev/null +++ b/scripts/perf_benchmark/configs/benchmark_config_maniskill.yml @@ -0,0 +1,58 @@ +mjcf_list: + - xml/franka_emika_panda/panda.xml + +renderer_list: + - renderer: maniskill + benchmark_script: benchmark_maniskill.py + timeout: 120 + +rasterizer_list: + - true + +batch_size_list: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + - 2048 + +resolution_list: + #square: + - [128, 128] + - [256, 256] + + #four_three: + + #sixteen_nine: + +comparison_list: + - - renderer: maniskill + rasterizer: true + +# Configurations shared betwen batch_benchmark.py and benchmark_*.py +# Raytracer configuration +raytracer: + max_bounce: 2 + spp: 1 + +# Simulation configuration +simulation: + n_steps: 100 + +# Camera configuration +camera: + position: [1.5, 0.5, 1.5] # [x, y, z] + lookat: [0.0, 0.0, 0.5] # [x, y, z] + fov: 45.0 # degrees + +# Display configuration +display: + gui: false # Enable/disable GUI mode diff --git a/scripts/perf_benchmark/configs/benchmark_config_omni.yml b/scripts/perf_benchmark/configs/benchmark_config_omni.yml index d331abba..dd0386e0 100644 --- a/scripts/perf_benchmark/configs/benchmark_config_omni.yml +++ b/scripts/perf_benchmark/configs/benchmark_config_omni.yml @@ -2,19 +2,28 @@ mjcf_list: - xml/franka_emika_panda/panda.xml renderer_list: - - renderer: madrona - benchmark_script: benchmark_madrona.py - timeout: 120 - renderer: omniverse benchmark_script: benchmark_omni.py - timeout: 60 + timeout: 120 rasterizer_list: - true - false batch_size_list: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 - 256 + - 512 + - 768 + - 1024 + - 2048 resolution_list: #square: @@ -28,12 +37,8 @@ resolution_list: comparison_list: - - renderer: omniverse rasterizer: true - - renderer: madrona - rasterizer: true - renderer: omniverse rasterizer: false - - renderer: madrona - rasterizer: false # Configurations shared betwen batch_benchmark.py and benchmark_*.py # Raytracer configuration @@ -43,7 +48,7 @@ raytracer: # Simulation configuration simulation: - n_steps: 1000 + n_steps: 100 # Camera configuration camera: diff --git a/scripts/perf_benchmark/example_report/index.html b/scripts/perf_benchmark/example_report/index.html index cf7b9458..2cc6b936 100644 --- a/scripts/perf_benchmark/example_report/index.html +++ b/scripts/perf_benchmark/example_report/index.html @@ -56,12 +56,57 @@

Resolution: 128x128

} - - - - + + + +
Renderer12481632641282565127681024153620483072409661448192
madrona - rasterizer554.11095.52224.44086.97020.511627.615647.117862.819456.822208.323533.124551.726077.326885.828884.429497.930696.331509.4
madrona - raytracer491.9919.81730.93100.65565.08604.012626.715650.116656.618522.419175.619943.919366.220415.920077.021039.621619.421667.8
Speedup0.9x0.8x0.8x0.8x0.8x0.7x0.8x0.9x0.9x0.8x0.8x0.8x0.7x0.8x0.7x0.7x0.7x0.7x
Renderer12481632641282565127681024153620483072
madrona - rasterizer550.01077.72099.94002.87210.412448.219639.127706.233683.333811.828812.228596.128518.228085.528065.9
madrona - raytracer526.11025.61980.43860.07022.613338.621824.832294.740519.249755.435023.335532.336919.237179.037510.0
Speedup1.0x1.0x0.9x1.0x1.0x1.1x1.1x1.2x1.2x1.5x1.2x1.2x1.3x1.3x1.3x
panda_madrona rasterizer_ madrona raytracer_128x128_comparison_plot.png
+

Resolution: 256x256

+
+ + + + + + + +
Renderer12481632641282565127681024153620483072
madrona - rasterizer547.61066.12031.83771.66762.310946.516284.721487.813058.213682.113859.313990.213842.013674.613512.6
madrona - raytracer480.0939.41748.33265.65704.68972.412211.415166.110396.810825.110909.511107.511148.211062.111146.5
Speedup0.9x0.9x0.9x0.9x0.8x0.8x0.7x0.7x0.8x0.8x0.8x0.8x0.8x0.8x0.8x
panda_madrona rasterizer_ madrona raytracer_256x256_comparison_plot.png
+

Performance Plots

diff --git a/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_plot.png b/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_plot.png index 976728fd..9d3e9e77 100644 Binary files a/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_plot.png and b/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_plot.png differ diff --git a/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_table.png b/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_table.png index fa370974..eb00e7b8 100644 Binary files a/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_table.png and b/scripts/perf_benchmark/example_report/panda_madrona rasterizer_ madrona raytracer_128x128_comparison_table.png differ diff --git a/scripts/perf_benchmark/example_report/panda_madrona_rasterizer_plot.png b/scripts/perf_benchmark/example_report/panda_madrona_rasterizer_plot.png index c1b6c6a9..792e6be6 100644 Binary files a/scripts/perf_benchmark/example_report/panda_madrona_rasterizer_plot.png and b/scripts/perf_benchmark/example_report/panda_madrona_rasterizer_plot.png differ diff --git a/scripts/perf_benchmark/example_report/panda_madrona_raytracer_plot.png b/scripts/perf_benchmark/example_report/panda_madrona_raytracer_plot.png index 20f58015..f044f5d8 100644 Binary files a/scripts/perf_benchmark/example_report/panda_madrona_raytracer_plot.png and b/scripts/perf_benchmark/example_report/panda_madrona_raytracer_plot.png differ diff --git a/scripts/perf_benchmark/example_report/perf_data.csv b/scripts/perf_benchmark/example_report/perf_data.csv index 7c965490..752bb4ea 100644 --- a/scripts/perf_benchmark/example_report/perf_data.csv +++ b/scripts/perf_benchmark/example_report/perf_data.csv @@ -1,41 +1,71 @@ result,mjcf,renderer,rasterizer,n_envs,n_steps,resX,resY,camera_posX,camera_posY,camera_posZ,camera_lookatX,camera_lookatY,camera_lookatZ,camera_fov,time_taken_gpu,time_taken_per_env_gpu,time_taken_cpu,time_taken_per_env_cpu,fps,fps_per_env -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3609180158376693,0.3609180158376693,0.3534615039825439,0.3534615039825439,554.1424678837709,554.1424678837709 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3651205130815506,0.1825602565407753,0.3656039237976074,0.1828019618988037,1095.5286971528192,547.7643485764096 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,4,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3596400642395019,0.0899100160598754,0.3542406558990478,0.0885601639747619,2224.446271556777,556.1115678891944 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,8,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3914946557283401,0.0489368319660425,0.383368968963623,0.0479211211204528,4086.901255454805,510.8626569318506 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,16,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.4558059515953064,0.0284878719747066,0.4476048946380615,0.0279753059148788,7020.531409912708,438.78321311954426 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,32,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.5504133114814759,0.0172004159837961,0.542076587677002,0.0169398933649063,11627.625761400925,363.3633050437789 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,64,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.8180408318042756,0.0127818879969418,0.8141109943389893,0.0127204842865467,15647.140707840032,244.4865735600005 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,128,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.433148413181305,0.0111964719779789,1.4245541095733645,0.0111293289810419,17862.7696647084,139.55288800553438 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,256,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,2.631472128868103,0.010279188003391,2.624802827835083,0.0102531360462307,19456.789771139654,76.00308504351429 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,512,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,4.610898963928222,0.0090056620389223,4.602760314941406,0.0089897662401199,22208.250668924014,43.37548958774222 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,768,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,6.5269759674072265,0.0084986666242281,6.516258001327515,0.0084847109392285,23533.10334939321,30.642061652855745 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1024,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,8.341577735900879,0.0081460720077157,8.333184957504272,0.0081378759350627,24551.710297989797,23.976279587880654 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1536,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,11.780353031158448,0.0076695006713271,11.77179503440857,0.0076639290588597,26077.31696897973,16.97741990167951 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2048,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,15.23481600189209,0.0074388750009238,15.226365089416504,0.0074347485788166,26885.785817769614,13.12782510633282 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,3072,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,21.27099492645264,0.0069241519942879,21.260852813720703,0.0069208505252997,28884.40348579705,9.402475093032896 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,4096,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,27.7714227142334,0.0067801324985921,27.76191997528076,0.006777812493965,29497.948608162013,7.201647609414554 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,6144,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,40.03088591003418,0.0065154436702529,40.02158451080322,0.0065139297706385,30696.297922599504,4.99614224000643 -succeeded,xml/franka_emika_panda/panda.xml,madrona,True,8192,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,51.99713287353516,0.0063473062589764,51.98826265335083,0.0063462234684266,31509.429644608197,3.846365923414086 -failed,xml/franka_emika_panda/panda.xml,madrona,True,12288,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, -failed,xml/franka_emika_panda/panda.xml,madrona,True,16384,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.4065958400964737,0.4065958400964737,0.4070327281951904,0.4070327281951904,491.88894788629824,491.88894788629824 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.4348619196414947,0.2174309598207473,0.4366605281829834,0.2183302640914917,919.8322086462864,459.9161043231432 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,4,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.4621987514495849,0.1155496878623962,0.464426040649414,0.1161065101623535,1730.8571204291998,432.7142801073 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,8,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.5160313591957092,0.0645039198994636,0.5215775966644287,0.0651971995830535,3100.586759870124,387.5733449837655 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,16,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.5750180809497834,0.0359386300593614,0.5794672966003418,0.0362167060375213,5565.04239782237,347.8151498638981 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,32,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.7438383054733276,0.0232449470460414,0.7502567768096924,0.0234455242753028,8604.020461042914,268.87563940759105 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,64,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.0137246088981628,0.0158394470140337,1.0201406478881836,0.0159396976232528,12626.703433699386,197.2922411515529 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,128,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.6357710709571838,0.0127794614918529,1.645803689956665,0.0128578413277864,15650.11171460562,122.2664977703564 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,256,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,3.0738570852279663,0.0120072542391717,3.0736243724823,0.0120063452050089,16656.597421543058,65.06483367790257 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,512,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,5.528451875686645,0.0107977575697004,5.528745889663696,0.0107983318157494,18522.364362135595,36.17649289479608 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,768,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,8.010173919677735,0.0104299139579137,8.012150526046753,0.0104324876641233,19175.613605925253,24.96824688271517 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1024,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,10.268798721313477,0.0100281237512826,10.272852897644045,0.0100320829078555,19943.910242872516,19.47647484655519 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1536,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,15.862725067138673,0.010327294965585,15.866164684295654,0.0103295342996716,19366.15548083838,12.608174141170824 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2048,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,20.062842826843266,0.0097963099740445,20.068536043167114,0.0097990898648276,20415.8505120706,9.968677007846972 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,3072,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,30.602257125854493,0.009961672241489,30.60457801818848,0.0099624277402957,20076.950450851567,6.5354656415532455 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,4096,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,38.93611776733398,0.0095058881267905,38.94675946235657,0.0095084861968643,21039.59118100058,5.136618940673969 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,6144,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,56.83783215332031,0.0092509492437044,56.85022163391113,0.0092529657607277,21619.40301813951,3.5187830433169776 -succeeded,xml/franka_emika_panda/panda.xml,madrona,False,8192,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,75.61467504882812,0.0092303070127964,75.62709641456604,0.0092318232927937,21667.751649293004,2.6449892150015875 -failed,xml/franka_emika_panda/panda.xml,madrona,False,12288,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, -failed,xml/franka_emika_panda/panda.xml,madrona,False,16384,200,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1818104321956634,0.1818104321956634,0.1930997371673584,0.1930997371673584,550.0234436073531,550.0234436073531 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1855801277160644,0.0927900638580322,0.2083349227905273,0.1041674613952636,1077.7015969403674,538.8507984701837 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,4,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1904850237369537,0.0476212559342384,0.2134573459625244,0.0533643364906311,2099.902617816147,524.9756544540368 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,8,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.199859935760498,0.0249824919700622,0.2249636650085449,0.0281204581260681,4002.80324796401,500.35040599550126 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,16,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.221900832414627,0.0138688020259141,0.2477030754089355,0.0154814422130584,7210.428111465401,450.6517569665876 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,32,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.257064959526062,0.0080332799851894,0.2882359027862549,0.0090073719620704,12448.215446787,389.00673271209376 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,64,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3258807048797607,0.0050918860137462,0.3639366626739502,0.0056865103542804,19639.08848902665,306.8607576410414 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,128,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.4619897890090942,0.0036092952266335,0.5255374908447266,0.0041057616472244,27706.240060963843,216.45500047628 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,256,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.7600212163925171,0.0029688328765332,0.8242590427398682,0.0032197618857026,33683.27021383932,131.57527427280985 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,512,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.5142652807235717,0.0029575493764132,1.5475430488586426,0.003022545017302,33811.777006162854,66.03862696516182 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,768,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,2.6655333137512205,0.0034707465022802,2.7337162494659424,0.0035595263664921,28812.24541587849,37.51594455192512 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1024,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,3.5809054374694824,0.0034969779662787,3.6493477821350098,0.0035638161934912,28596.119553596192,27.92589800155878 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1536,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,5.386033821105957,0.0035065324356158,5.450999736785889,0.0035488279536366,28518.20190918521,18.566537701292454 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2048,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,7.29202025604248,0.0035605567656457,7.355396747589111,0.0035915023181587,28085.49521379812,13.71362070986236 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,3072,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,10.945659706115723,0.0035630402689178,11.008342027664185,0.0035834446704636,28065.919117543606,9.136041379408724 +failed,xml/franka_emika_panda/panda.xml,madrona,True,4096,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,True,6144,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,True,8192,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,True,12288,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,True,16384,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1826125756502151,0.1826125756502151,0.1937849521636963,0.1937849521636963,547.607412271238,547.607412271238 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1875923197269439,0.0937961598634719,0.2091264724731445,0.1045632362365722,1066.1417284626389,533.0708642313194 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,4,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1968726726770401,0.04921816816926,0.2214667797088623,0.0553666949272155,2031.770049955995,507.9425124889988 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,8,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2121126413345337,0.0265140801668167,0.2369122505187988,0.0296140313148498,3771.580962674823,471.4476203343529 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,16,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2366069767475128,0.0147879360467195,0.2632186412811279,0.0164511650800704,6762.268898382426,422.6418061489017 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,32,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2923321604728698,0.0091353800147771,0.3365225791931152,0.0105163305997848,10946.452127688424,342.07662899026326 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,64,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3930070729255676,0.0061407355144619,0.4455823898315429,0.0069622248411178,16284.694197379262,254.44834683405097 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,128,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.5956879329681396,0.0046538119763135,0.6594650745391846,0.0051520708948373,21487.76111045481,167.8731336754282 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,256,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.9604556407928464,0.007658029846847,2.0290844440460205,0.0079261111095547,13058.188855345312,51.008550216192624 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,512,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,3.74211616897583,0.0073088206425309,3.809530258178711,0.0074404887855052,13682.09795956516,26.722847577275704 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,768,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,5.541409248352051,0.0072153766254583,5.605247259140015,0.0072984990353385,13859.290400332624,18.04595104209977 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1024,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,7.319408500671387,0.0071478598639369,7.3833396434783936,0.0072102926205843,13990.20152934587,13.662306181001826 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,1536,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,11.096647178649905,0.0072243796736001,11.15586543083191,0.0072629332231978,13842.01890238778,9.011731056242043 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,2048,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,14.976678344726562,0.007312831223011,15.03558588027954,0.0073415946681052,13674.594278250766,6.677047987427132 +succeeded,xml/franka_emika_panda/panda.xml,madrona,True,3072,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,22.73427993774414,0.0074004817505677,22.7915723323822,0.0074191316186139,13512.633821754664,4.398643822185763 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1900674557685852,0.1900674557685852,0.2022051811218261,0.2022051811218261,526.1289977057095,526.1289977057095 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.1949984003305435,0.0974992001652717,0.2171480655670166,0.1085740327835083,1025.649439487597,512.8247197437985 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,4,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2019746559858322,0.050493663996458,0.2255463600158691,0.0563865900039672,1980.446497346967,495.11162433674167 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,8,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2072516167163849,0.0259064520895481,0.2336342334747314,0.0292042791843414,3860.042264928463,482.50528311605785 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,16,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2278363840579986,0.0142397740036249,0.2552666664123535,0.015954166650772,7022.583362246039,438.9114601403774 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,32,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2399055359363555,0.0074970479980111,0.2729389667510986,0.0085293427109718,13338.583403298066,416.83073135306455 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,64,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2932444808483124,0.0045819450132548,0.3325328826904297,0.0051958262920379,21824.792683176023,341.01238567462536 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,128,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3963496654033661,0.0030964817609637,0.4526624679565429,0.0035364255309104,32294.716300500484,252.30247109766003 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,256,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.6317999358177185,0.0024679684992879,0.6983375549316406,0.0027278810739517,40519.15574645752,158.27795213459967 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,512,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,1.029033890724182,0.0020098318178206,1.0975844860076904,0.0021437196992337,49755.40695162919,97.17852920240075 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,768,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,2.1928288612365723,0.0028552459130684,2.2612593173980717,0.0029443480695287,35023.25300328783,45.603194014697685 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1024,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,2.8818820476531983,0.0028143379371613,2.948338508605957,0.0028792368248105,35532.33557334081,34.699546458340635 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1536,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,4.1604327316284175,0.0027086150596539,4.226066112518311,0.0027513451253374,36919.23650929457,24.035961269071983 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2048,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,5.508481456756591,0.0026896882113069,5.573500156402588,0.0027214356232434,37179.03048376363,18.15382347840021 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,3072,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,8.189823631286622,0.0026659582133094,8.253234386444092,0.0026865997351706,37509.96527281002,12.21027515390951 +failed,xml/franka_emika_panda/panda.xml,madrona,False,4096,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,False,6144,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,False,8192,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,False,12288,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +failed,xml/franka_emika_panda/panda.xml,madrona,False,16384,100,128,128,1.5,0.5,1.5,0.0,0.0,0.5,45.0,,,,,, +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2083193914890289,0.2083193914890289,0.2212691307067871,0.2212691307067871,480.0321241590535,480.0321241590535 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2129035832881927,0.1064517916440963,0.2365458011627197,0.1182729005813598,939.3923620781616,469.6961810390808 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,4,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2287999367713928,0.0571999841928482,0.2518947124481201,0.06297367811203,1748.2522313791678,437.063057844792 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,8,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2449816331863403,0.0306227041482925,0.2697319984436035,0.0337164998054504,3265.55092965478,408.19386620684753 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,16,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.2804776637554169,0.0175298539847135,0.3079962730407715,0.0192497670650482,5704.554075989586,356.5346297493491 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,32,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.3566481597423553,0.0111452549919486,0.3974072933197021,0.0124189779162406,8972.428183315731,280.3883807286166 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,64,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.5240999989509583,0.0081890624836087,0.5747663974761963,0.0089807249605655,12211.410060695056,190.80328219836025 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,128,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,0.8439869766235352,0.0065936482548713,0.907806634902954,0.0070922393351793,15166.110798543172,118.48524061361852 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,256,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,2.462288450241089,0.0096183142587542,2.5315020084381104,0.0098886797204613,10396.832262886765,40.61262602690142 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,512,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,4.729760364532471,0.0092378132119774,4.79765510559082,0.009370420128107,10825.072742361026,21.14272019992388 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,768,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,7.039768127441406,0.0091663647492726,7.104008436203003,0.0092500109846393,10909.450227576295,14.205013317156634 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1024,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,9.219011352539065,0.0090029407739639,9.28283953666687,0.0090652729850262,11107.48171188632,10.847150109263984 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,1536,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,13.77803616333008,0.008970075627168,13.83764934539795,0.0090088862925767,11148.178026183645,7.25792840246331 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,2048,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,18.51371530151367,0.0090399000495672,18.572023153305054,0.0090683706803247,11062.069210022672,5.401400981456384 +succeeded,xml/franka_emika_panda/panda.xml,madrona,False,3072,100,256,256,1.5,0.5,1.5,0.0,0.0,0.5,45.0,27.56010620117188,0.0089713887373606,27.61528587341309,0.0089893508702516,11146.54630710159,3.6284330426762983 diff --git a/scripts/perf_benchmark/process_xml.py b/scripts/perf_benchmark/process_xml.py index 472fced1..a71f40b5 100644 --- a/scripts/perf_benchmark/process_xml.py +++ b/scripts/perf_benchmark/process_xml.py @@ -2,48 +2,103 @@ import argparse import os import copy -import uuid +import yaml +# Get asset directory from environment variable +asset_dir = os.getenv("ASSET_DIR") -def wrap_visual_geoms_with_bodies(input_file, output_file): + +def process_mjcf_geoms(input_file, output_file): tree = ET.parse(input_file) root = tree.getroot() + link_geoms = {} - def wrap_geom(parent): - removed_elems = [] - append_elems = [] - for i, elem in enumerate(parent): + def wrap_geom(cur, cur_link_name=None): + replace_elems = [] + remove_elems = [] + for i, elem in enumerate(cur): if elem.tag == "geom": - if elem.attrib.get("class") == "visual": - # Create a new element - body = ET.Element("body") - # Give it a name based on mesh if available - mesh_name = elem.attrib.get("mesh", f"{i}") - body.set("name", f"body_{mesh_name}_{str(uuid.uuid4())[:4]}") - # Deep copy of the geom to keep attributes - body.append(copy.deepcopy(elem)) - # Replace with in parent - removed_elems.append(elem) - append_elems.append(body) - elif elem.attrib.get("class") == "collision": - removed_elems.append(elem) + if cur_link_name is None: + continue + + if elem.get("class", None) == "collision" or \ + elem.get("conaffinity", 0) > 0 or \ + elem.get("contype", 0) > 0: + remove_elems.append(elem) + continue + + body = ET.Element("body") + body_idx = link_geoms.get(cur_link_name, 0) + link_geoms[cur_link_name] = body_idx + 1 + body.set("name", f"{cur_link_name}_geom{body_idx}") + body.append(copy.deepcopy(elem)) + replace_elems.append((elem, body)) else: - wrap_geom(elem) # Recurse into children + nex_link_name = None + if elem.tag == "body": + nex_link_name = elem.get("name", None) + # Recurse into children + wrap_geom(elem, nex_link_name if nex_link_name else cur_link_name) - for elem in removed_elems: - parent.remove(elem) - for elem in append_elems: - parent.append(elem) + for elem, body in replace_elems: + idx = list(cur).index(elem) + cur.remove(elem) + cur.insert(idx, body) + for elem in remove_elems: + cur.remove(elem) wrap_geom(root) tree.write(output_file, encoding="utf-8", xml_declaration=True) -# Example usage -parser = argparse.ArgumentParser() -parser.add_argument("--file", type=str, required=True, default="./panda.xml") -args = parser.parse_args() +def process_config_file(config_file): + """Process all robot files mentioned in a specific benchmark configuration file.""" + if not os.path.exists(config_file): + print(f"Configuration file not found: {config_file}") + return + + print(f"Processing configuration file: {config_file}") + + try: + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + + if 'mjcf_list' not in config: + print(f"No mjcf_list found in {config_file}") + return + + mjcf_list = config['mjcf_list'] + print(f"Found {len(mjcf_list)} robot files in {config_file}") + + processed_count = 0 + for mjcf_path in mjcf_list: + # Construct full path using asset directory + full_mjcf_path = os.path.join(asset_dir, mjcf_path) + + if not os.path.exists(full_mjcf_path): + print(f"Warning: Robot file not found: {full_mjcf_path}") + continue + + print(f"Processing robot file: {full_mjcf_path}") + output_file = f"{os.path.splitext(full_mjcf_path)[0]}_new.xml" + process_mjcf_geoms(full_mjcf_path, output_file) + processed_count += 1 + print(f"Created processed file: {output_file}") + + print(f"Total files processed: {processed_count}") + + except Exception as e: + print(f"Error processing {config_file}: {e}") + + +def main(): + parser = argparse.ArgumentParser(description="Process MJCF robot files from a benchmark configuration file") + parser.add_argument("--file", type=str, help="Path to the benchmark configuration YAML file") + args = parser.parse_args() + + # Process robot files from the specified configuration file + process_config_file(args.file) + -input_file = args.file -output_file = f"{os.path.splitext(input_file)[0]}_new.xml" -wrap_visual_geoms_with_bodies(input_file, output_file) +if __name__ == "__main__": + main()