Add Depth Anything V3 ONNX backend

README.md

@@ -22,6 +22,7 @@ Small Python utility to **compare and visualize** the output of various **stereo
 - [Chang et al. RealtimeStereo](https://github.com/JiaRenChang/RealtimeStereo): "Attention-Aware Feature Aggregation for Real-time Stereo Matching on Edge Devices" (ACCV 2020)
 
 - [DistDepth](https://github.com/facebookresearch/DistDepth): "Toward Practical Monocular Indoor Depth Estimation" (CVPR 2022). This one is actually a **monocular** method, only using the left image.
+- [Depth Anything V3](https://github.com/DepthAnything/Depth-Anything-V3): metric monocular depth estimation. This uses the [DA3METRIC-LARGE ONNX export](https://huggingface.co/TillBeemelmanns/Depth-Anything-V3-ONNX), only using the left image.
 
 - [DUSt3R](https://github.com/naver/dust3r): "Geometric 3D Vision Made Easy" (CVPR 2024). Dense Unconstrained Stereo 3D Reconstruction using a ViT-Large encoder/cross-attention decoder pair. Weights are ~2.3 GB and downloaded on first use. **Non-commercial only (CC BY-NC-SA 4.0).**
 
@@ -142,6 +143,14 @@ I did not implement any of these myself, but just collected pre-trained models o
   - DUSt3R predicts 3D point maps rather than stereo disparity. This adapter estimates DUSt3R's implicit focal length from the left point map and the predicted baseline from the right point map (assuming a rectified pair), then uses `f_dust3r * b_dust3r / Z_dust3r` directly as the pixel disparity. No metric-scale recovery via the user's calibration is needed for the disparity output.
   - The official DUSt3R code and checkpoints are licensed under CC BY-NC-SA 4.0.
 
+- Depth Anything V3
+  - Official project: https://github.com/DepthAnything/Depth-Anything-V3
+  - ONNX model export used by stereodemo: https://huggingface.co/TillBeemelmanns/Depth-Anything-V3-ONNX
+  - Preprocessing and metric-depth postprocessing adapted from https://github.com/ika-rwth-aachen/ros2-depth-anything-v3-trt
+  - The adapter follows that postprocessing and scales raw depth from the model's 300 px reference focal length to the input calibration focal length.
+  - This is a monocular method. It predicts metric depth from the left image, then stereodemo converts that depth to disparity using the stereo calibration so it can reuse the point-cloud visualization pipeline.
+  - The model's sky output is used conservatively: when the sky mask covers most of the image, stereodemo keeps the raw depth instead of filling sky pixels, because the sky head can otherwise flatten outdoor driving scenes.
+
 # License
 
 The code of stereodemo is MIT licensed, but the pre-trained models are subject to the license of their respective implementation.

pyproject.toml

@@ -10,7 +10,7 @@ name = "stereodemo"
 version = "0.6.2"
 description = "Compare various stereo depth estimation algorithms on image files or with an OAK-D camera."
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.8,!=3.12.*" # 3.12 is not supported by onnxruntime
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [

stereodemo/main.py

@@ -25,6 +25,7 @@
 from .method_sttr import StereoTransformers
 from stereodemo.method_dist_depth import DistDepth
 from stereodemo.method_dust3r import DUSt3R
+from .method_depth_anything_v3 import DepthAnythingV3
 
 def parse_args():
     import argparse
@@ -114,7 +115,9 @@ def load_image(path):
         if self.user_provided_calibration_path is None:
             calibration_path = left_image_path.parent / 'stereodemo_calibration.json'
             if not calibration_path.exists():
-                print (f"Warning: no calibration file found {calibration_path}. Using default calibration, the point cloud won't be accurate.")
+                calibration_path = left_image_path.parent / 'stereo_calibration.json'
+            if not calibration_path.exists():
+                print (f"Warning: no calibration file found {left_image_path.parent / 'stereodemo_calibration.json'} or {left_image_path.parent / 'stereo_calibration.json'}. Using default calibration, the point cloud won't be accurate.")
                 calibration_path = None
         else:
             calibration_path = self.user_provided_calibration_path
@@ -171,8 +174,9 @@ def main():
         HitnetStereo(config),
         StereoTransformers(config),
         ChangRealtimeStereo(config),
-        DistDepth(config),
         DUSt3R(config),
+        DistDepth(config),
+        DepthAnythingV3(config),
     ]
 
     if args.images:
@@ -202,5 +206,3 @@ def main():
         time_to_sleep = 1/30.0 - elapsed
         if time_to_sleep > 0:
             time.sleep (time_to_sleep)
-
-

stereodemo/method_depth_anything_v3.py

@@ -0,0 +1,157 @@
+from pathlib import Path
+import time
+
+import cv2
+import numpy as np
+import onnxruntime
+
+from .methods import Config, EnumParameter, StereoMethod, InputPair, StereoOutput
+from . import utils
+
+
+MODEL_NAME = "DA3METRIC-LARGE.onnx"
+# Fixed ONNX export shape, in OpenCV (width, height) order. The model was
+# exported for [1, 3, 280, 504], so inputs are resized to this shape even when
+# that changes the source aspect ratio.
+MODEL_INPUT_SIZE = (504, 280)
+SKY_THRESHOLD = 0.3
+SKY_DEPTH_CAP_METERS = 200.0
+MIN_NON_SKY_FRACTION = 0.05
+MAX_SKY_FRACTION_FOR_FILL = 0.6
+REFERENCE_FOCAL_LENGTH_PIXELS = 300.0
+
+urls = {
+    MODEL_NAME: "https://huggingface.co/TillBeemelmanns/Depth-Anything-V3-ONNX/resolve/main/DA3METRIC-LARGE.onnx",
+}
+
+
+# ONNX model from https://huggingface.co/TillBeemelmanns/Depth-Anything-V3-ONNX
+# Pre/post-processing follows https://github.com/ika-rwth-aachen/ros2-depth-anything-v3-trt
+class DepthAnythingV3(StereoMethod):
+    def __init__(self, config: Config):
+        super().__init__("[Monocular] Depth Anything V3 Metric",
+                         "Depth Anything V3 metric monocular depth estimation.",
+                         {},
+                         config)
+        self.reset_defaults()
+
+        self._loaded_session = None
+        self._loaded_model_path = None
+        self._loaded_providers = None
+
+    def reset_defaults(self):
+        self.parameters.update({
+            "Device": EnumParameter("Device", 0, ["Auto", "CPU"]),
+            "Sky handling": EnumParameter("Sky handling", 0, ["Fill", "Ignore"]),
+        })
+
+    def compute_disparity(self, input: InputPair) -> StereoOutput:
+        model_path = self.config.models_path / MODEL_NAME
+        providers = self._selected_providers()
+        self._load_model(model_path, providers)
+
+        input_tensor = self._preprocess_input(input.left_image)
+        model_inputs = self._loaded_session.get_inputs()
+        model_outputs = self._loaded_session.get_outputs()
+        output_names = [output.name for output in model_outputs]
+
+        start = time.time()
+        outputs = self._loaded_session.run(output_names, {model_inputs[0].name: input_tensor})
+        elapsed_time = time.time() - start
+
+        output_by_name = dict(zip(output_names, outputs))
+        depth = output_by_name.get("depth", outputs[0])
+        sky = output_by_name.get("sky", outputs[1] if len(outputs) > 1 else None)
+
+        depth_meters = self._process_depth(depth, sky, input.calibration)
+        invalid_mask = depth_meters <= 0.0
+        if depth_meters.shape[:2] != input.left_image.shape[:2]:
+            depth_meters = cv2.resize(
+                depth_meters,
+                (input.left_image.shape[1], input.left_image.shape[0]),
+                interpolation=cv2.INTER_CUBIC,
+            )
+            invalid_mask = cv2.resize(
+                invalid_mask.astype(np.uint8),
+                (input.left_image.shape[1], input.left_image.shape[0]),
+                interpolation=cv2.INTER_NEAREST,
+            ).astype(bool)
+
+        depth_meters[invalid_mask] = 0.0
+        disparity_map = StereoMethod.disparity_from_depth_meters(depth_meters, input.calibration)
+        disparity_map[depth_meters <= 0.0] = 0.0
+        disparity_map = disparity_map.astype(np.float32)
+        return StereoOutput(disparity_map, input.left_image, elapsed_time)
+
+    def _preprocess_input(self, image_bgr: np.ndarray):
+        image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
+        image_rgb = cv2.resize(image_rgb, MODEL_INPUT_SIZE, interpolation=cv2.INTER_AREA)
+        image = image_rgb.astype(np.float32) / 255.0
+        mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+        std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+        image = (image - mean) / std
+        image = image.transpose(2, 0, 1)
+        return image[np.newaxis, :, :, :].astype(np.float32)
+
+    def _process_depth(self, depth_output, sky_output, calibration):
+        depth = np.squeeze(depth_output).astype(np.float32)
+        depth[depth < 0.0] = 0.0
+
+        focal_pixels = np.float32((calibration.fx + calibration.fy) / 2.0)
+        # Follow the ika-rwth-aachen ROS2/TensorRT adapter for the
+        # DA3METRIC-LARGE ONNX export: the raw depth is scaled from the
+        # model's 300 px reference focal length to the calibrated camera focal.
+        # Source: https://github.com/ika-rwth-aachen/ros2-depth-anything-v3-trt
+        # documents metric_depth = depth * ((fx + fy) / 2) / 300.0.
+        depth_meters = depth * (focal_pixels / np.float32(REFERENCE_FOCAL_LENGTH_PIXELS))
+
+        if sky_output is None:
+            return depth_meters
+
+        sky = np.squeeze(sky_output).astype(np.float32)
+        # The ONNX sky head uses low values for sky and high values for non-sky.
+        sky_mask = sky < np.float32(SKY_THRESHOLD)
+        if self.parameters["Sky handling"].value == "Ignore":
+            depth_meters[sky_mask] = 0.0
+            return depth_meters
+
+        if np.mean(sky_mask) > MAX_SKY_FRACTION_FOR_FILL:
+            # Treat large sky masks as unreliable. On some outdoor stereo
+            # samples the sky head also marks road and background as sky,
+            # and filling would collapse most of the disparity map.
+            return depth_meters
+
+        valid_non_sky = depth_meters[(~sky_mask) & (depth_meters > 0.0)]
+        min_non_sky_pixels = int(depth_meters.size * MIN_NON_SKY_FRACTION)
+        if valid_non_sky.size < min_non_sky_pixels:
+            # The ONNX sky head can classify non-sky indoor scenes as mostly sky.
+            # In that case a percentile fill would collapse the whole depth map.
+            return depth_meters
+        else:
+            sky_depth = np.float32(min(np.percentile(valid_non_sky, 99), SKY_DEPTH_CAP_METERS))
+        depth_meters[sky_mask] = sky_depth
+        return depth_meters
+
+    def _selected_providers(self):
+        if self.parameters["Device"].value == "CPU":
+            return ["CPUExecutionProvider"]
+
+        available = onnxruntime.get_available_providers()
+        preferred = ["CUDAExecutionProvider", "CoreMLExecutionProvider", "CPUExecutionProvider"]
+        providers = [provider for provider in preferred if provider in available]
+        if "CPUExecutionProvider" not in providers:
+            providers.append("CPUExecutionProvider")
+        return providers
+
+    def _load_model(self, model_path: Path, providers):
+        if self._loaded_model_path == model_path and self._loaded_providers == providers:
+            return
+
+        if not model_path.exists():
+            utils.download_model(urls[model_path.name], model_path)
+
+        if not model_path.exists():
+            raise RuntimeError(f"Could not download Depth Anything V3 model to {model_path}")
+        self._loaded_model_path = model_path
+        self._loaded_providers = providers
+        self._loaded_session = onnxruntime.InferenceSession(str(model_path), providers=providers)

tests/test_methods.py

@@ -17,6 +17,7 @@
 from stereodemo import method_dust3r
 from stereodemo import method_opencv_cuda_bp
 from stereodemo import method_opencv_cuda_csbp
+from stereodemo import method_depth_anything_v3
 from stereodemo.methods import Config, InputPair, Calibration, StereoOutput, StereoMethod
 
 data_folder = Path(__file__).parent.parent / 'datasets' / 'eth3d_lowres' / 'delivery_area_1l'
@@ -97,5 +98,20 @@ def test_cuda_csbp(self):
             self.skipTest("OpenCV CUDA is not available")
         self.check_method (method_opencv_cuda_csbp.StereoCudaCSBP(config), 6.0, 0.9)
 
+    def test_depth_anything_v3(self):
+        m = method_depth_anything_v3.DepthAnythingV3(config)
+        m.parameters["Device"].set_value("CPU")
+        output = m.compute_disparity(input)
+        self.assertEqual(output.disparity_pixels.shape, input.left_image.shape[:2])
+        self.assertTrue(np.isfinite(output.disparity_pixels).all())
+        valid_pixels = output.disparity_pixels[output.disparity_pixels > 0.]
+        coverage = valid_pixels.size / output.disparity_pixels.size
+        disparity_p01 = np.percentile(valid_pixels, 1)
+        disparity_p99 = np.percentile(valid_pixels, 99)
+        disparity_spread = disparity_p99 - disparity_p01
+        self.assertAlmostEqual(np.median(valid_pixels), 3.4663, delta=0.01)
+        self.assertAlmostEqual(coverage, 1.0, delta=0.01)
+        self.assertGreater(disparity_spread, 2.0)
+
 if __name__ == '__main__':
     unittest.main()