isaac-sim · pascal-roth · Jul 2, 2024 · Jul 2, 2024 · Jul 2, 2024 · Jul 2, 2024
@@ -1,3 +1,19 @@
+Added
+^^^^^
+
+* Added option to define the clipping behavior for depth images generated by
+  :class:`omni.isaac.lab.sensors.ray_caster.RayCasterCamera` and :class:`omni.isaac.lab.sensors.ray_caster.Camera`
+
+Changed
+^^^^^^^
+
+* Unified the clipping behavior for the depth images of all camera implementations. Per default, all values exceeding
+  the range are clipped to zero for both ``distance_to_image_plane`` and ``distance_to_camera`` depth images. Prev.
+  :class:`omni.isaac.lab.sensors.ray_caster.RayCasterCamera` clipped the values to the maximum value of the depth image,
+  :class:`omni.isaac.lab.sensors.ray_caster.Camera` did not clip them and had a different behavior for both types, and
+  :class:`omni.isaac.lab.sensors.ray_caster.TiledCamera` clipped the values to zero.
+
+
 Changelog
 ---------
 

@@ -492,6 +492,19 @@ def _update_buffers_impl(self, env_ids: Sequence[int]):
                     self._data.output[name][index] = data
                     # add info to output
                     self._data.info[index][name] = info
+                # NOTE: `distance_to_camera` and `distance_to_image_plane` are not both clipped to the maximum defined
+                #       in the clipping range. The clipping is applied only to `distance_to_image_plane` and then both
+                #       outputs are only clipped where the values in `distance_to_image_plane` exceed the threshold. To
+                #       have a unified behavior between all cameras, we clip both outputs to the maximum value defined.
+                if name == "distance_to_camera":
+                    self._data.output[name][self._data.output[name] > self.cfg.spawn.clipping_range[1]] = torch.inf
+                # apply defined clipping behavior
+                if (
+                    name == "distance_to_camera" or name == "distance_to_image_plane"
+                ) and self.cfg.depth_clipping_behavior is not None:
+                    self._data.output[name][torch.isinf(self._data.output[name])] = (
+                        0.0 if self.cfg.depth_clipping_behavior == "zero" else self.cfg.spawn.clipping_range[1]
+                    )
 
     """
     Private Helpers
@@ -608,6 +621,19 @@ def _create_annotator_data(self):
                 self._data.info[index][name] = info
             # concatenate the data along the batch dimension
             self._data.output[name] = torch.stack(data_all_cameras, dim=0)
+            # NOTE: `distance_to_camera` and `distance_to_image_plane` are not both clipped to the maximum defined
+            #       in the clipping range. The clipping is applied only to `distance_to_image_plane` and then both
+            #       outputs are only clipped where the values in `distance_to_image_plane` exceed the threshold. To
+            #       have a unified behavior between all cameras, we clip both outputs to the maximum value defined.
+            if name == "distance_to_camera":
+                self._data.output[name][self._data.output[name] > self.cfg.spawn.clipping_range[1]] = torch.inf
+            # clip the data if needed
+            if (
+                name == "distance_to_camera" or name == "distance_to_image_plane"
+            ) and self.cfg.depth_clipping_behavior is not None:
+                self._data.output[name][torch.isinf(self._data.output[name])] = (
+                    0.0 if self.cfg.depth_clipping_behavior == "zero" else self.cfg.spawn.clipping_range[1]
+                )
 
     def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tensor, dict | None]:
         """Process the annotator output.

@@ -53,6 +53,14 @@ class OffsetCfg:
     asset is already present in the scene.
     """
 
+    depth_clipping_behavior: Literal["max", "zero"] | None = "zero"
+    """Clipping behavior for the camera for values exceed the maximum value. Defaults to "zero".
+
+    - ``"max"``: Values are clipped to the maximum value.
+    - ``"zero"``: Values are clipped to zero.
+    - ``None``: No clipping is applied. Values will be returned as ``inf``.
+    """
+
     data_types: list[str] = ["rgb"]
     """List of sensor names/types to enable for the camera. Defaults to ["rgb"].
 

@@ -23,7 +23,7 @@
 from .camera import Camera
 
 if TYPE_CHECKING:
-    from .camera_cfg import TiledCameraCfg
+    from .tiled_camera_cfg import TiledCameraCfg
 
 
 class TiledCamera(Camera):

diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera_cfg.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera_cfg.py
@@ -15,7 +15,13 @@
 
 @configclass
 class TiledCameraCfg(SensorBaseCfg):
-    """Configuration for a tiled rendering-based camera sensor."""
+    """Configuration for a tiled rendering-based camera sensor.
+
+    .. note::
+
+        Compared to other cameras, the clipping behavior cannot be altered as the backend implementation automatically
+        clips the values exceeding the maximum range to zero.
+    """
 
     @configclass
     class OffsetCfg:

@@ -292,12 +292,19 @@ def _update_buffers_impl(self, env_ids: Sequence[int]):
                 )
             )[:, :, 0]
             # apply the maximum distance after the transformation
-            distance_to_image_plane = torch.clip(distance_to_image_plane, max=self.cfg.max_distance)
+            if self.cfg.depth_clipping_behavior == "max":
+                distance_to_image_plane = torch.clip(distance_to_image_plane, max=self.cfg.max_distance)
+            elif self.cfg.depth_clipping_behavior == "zero":
+                distance_to_image_plane[distance_to_image_plane > self.cfg.max_distance] = 0.0
             self._data.output["distance_to_image_plane"][env_ids] = distance_to_image_plane.view(-1, *self.image_shape)
+
         if "distance_to_camera" in self.cfg.data_types:
-            self._data.output["distance_to_camera"][env_ids] = torch.clip(
-                ray_depth.view(-1, *self.image_shape), max=self.cfg.max_distance
-            )
+            if self.cfg.depth_clipping_behavior == "max":
+                ray_depth = torch.clip(ray_depth, max=self.cfg.max_distance)
+            elif self.cfg.depth_clipping_behavior == "zero":
+                ray_depth[ray_depth > self.cfg.max_distance] = 0.0
+            self._data.output["distance_to_camera"][env_ids] = ray_depth.view(-1, *self.image_shape)
+
         if "normals" in self.cfg.data_types:
             self._data.output["normals"][env_ids] = ray_normal.view(-1, *self.image_shape, 3)
 

@@ -44,6 +44,14 @@ class OffsetCfg:
     data_types: list[str] = ["distance_to_image_plane"]
     """List of sensor names/types to enable for the camera. Defaults to ["distance_to_image_plane"]."""
 
+    depth_clipping_behavior: Literal["max", "zero"] | None = "zero"
+    """Clipping behavior for the camera for values exceed the maximum value. Defaults to "zero".
+
+    - ``"max"``: Values are clipped to the maximum value.
+    - ``"zero"``: Values are clipped to zero.
+    - ``None``: No clipping is applied. Values will be returned as ``nan``.
+    """
+
     def __post_init__(self):
         # for cameras, this quantity should be False always.
         self.attach_yaw_only = False
@@ -360,6 +360,50 @@ def test_intrinsic_matrix(self):
             torch.testing.assert_close(rs_intrinsic_matrix[0, 0, 0], camera.data.intrinsic_matrices[0, 0, 0])
             # torch.testing.assert_close(rs_intrinsic_matrix[0, 1, 1], camera.data.intrinsic_matrices[0, 1, 1])
 
+    def test_depth_clipping(self):
+        """Test depth clipping.
+
+        .. note::
+
+            This test is the same for all camera models to enforce the same clipping behavior.
+        """
+        # get camera cfgs
+        camera_cfg = CameraCfg(
+            prim_path="/World/Camera",
+            offset=CameraCfg.OffsetCfg(pos=(2.5, 2.5, 6.0), rot=(-0.125, 0.362, 0.873, -0.302), convention="ros"),
+            spawn=sim_utils.PinholeCameraCfg().from_intrinsic_matrix(
+                focal_length=38.0,
+                intrinsic_matrix=[380.08, 0.0, 467.79, 0.0, 380.08, 262.05, 0.0, 0.0, 1.0],
+                height=540,
+                width=960,
+                clipping_range=(0.1, 10),
+            ),
+            height=540,
+            width=960,
+            data_types=["distance_to_image_plane", "distance_to_camera"],
+        )
+        camera = Camera(camera_cfg)
+
+        # Play sim
+        self.sim.reset()
+
+        # note: This is a workaround to ensure that the textures are loaded.
+        #   Check "Known Issues" section in the documentation for more details.
+        for _ in range(5):
+            self.sim.step()
+
+        camera.update(self.dt)
+
+        self.assertTrue(
+            camera.data.output["distance_to_camera"][camera.data.output["distance_to_camera"] != 0.0].min() >= 0.1
+        )
+        self.assertTrue(camera.data.output["distance_to_camera"].max() <= 10.0)
+        self.assertTrue(
+            camera.data.output["distance_to_image_plane"][camera.data.output["distance_to_image_plane"] != 0.0].min()
+            >= 0.1
+        )
+        self.assertTrue(camera.data.output["distance_to_image_plane"].max() <= 10.0)
+
     def test_camera_resolution_all_colorize(self):
         """Test camera resolution is correctly set for all types with colorization enabled."""
         # Add all types

@@ -150,6 +150,46 @@ def test_camera_resolution(self):
         for im_data in camera.data.output.to_dict().values():
             self.assertTrue(im_data.shape == (1, self.camera_cfg.pattern_cfg.height, self.camera_cfg.pattern_cfg.width))
 
+    def test_depth_clipping(self):
+        """Test depth clipping.
+
+        .. note::
+
+            This test is the same for all camera models to enforce the same clipping behavior.
+        """
+        # get camera cfgs
+        camera_cfg = RayCasterCameraCfg(
+            prim_path="/World/Camera",
+            mesh_prim_paths=["/World/defaultGroundPlane"],
+            offset=RayCasterCameraCfg.OffsetCfg(
+                pos=(2.5, 2.5, 6.0), rot=(-0.125, 0.362, 0.873, -0.302), convention="ros"
+            ),
+            pattern_cfg=patterns.PinholeCameraPatternCfg().from_intrinsic_matrix(
+                focal_length=38.0,
+                intrinsic_matrix=[380.08, 0.0, 467.79, 0.0, 380.08, 262.05, 0.0, 0.0, 1.0],
+                height=540,
+                width=960,
+            ),
+            max_distance=10.0,
+            data_types=["distance_to_image_plane", "distance_to_camera"],
+        )
+        camera = RayCasterCamera(camera_cfg)
+
+        # Play sim
+        self.sim.reset()
+
+        camera.update(self.dt)
+
+        self.assertTrue(
+            camera.data.output["distance_to_camera"][camera.data.output["distance_to_camera"] != 0.0].min() >= 0.1
+        )
+        self.assertTrue(camera.data.output["distance_to_camera"].max() <= 10.0)
+        self.assertTrue(
+            camera.data.output["distance_to_image_plane"][camera.data.output["distance_to_image_plane"] != 0.0].min()
+            >= 0.1
+        )
+        self.assertTrue(camera.data.output["distance_to_image_plane"].max() <= 10.0)
+
     def test_camera_init_offset(self):
         """Test camera initialization with offset using different conventions."""
         # define the same offset in all conventions

@@ -119,6 +119,45 @@ def test_single_camera_init(self):
                 self.assertGreater(im_data.mean().item(), 0.0)
         del camera
 
+    def test_depth_clipping(self):
+        """Test depth clipping.
+
+        .. note::
+
+            This test is the same for all camera models to enforce the same clipping behavior.
+        """
+        # get camera cfgs
+        camera_cfg = TiledCameraCfg(
+            prim_path="/World/Camera",
+            offset=TiledCameraCfg.OffsetCfg(pos=(2.5, 2.5, 6.0), rot=(-0.125, 0.362, 0.873, -0.302), convention="ros"),
+            spawn=sim_utils.PinholeCameraCfg().from_intrinsic_matrix(
+                focal_length=38.0,
+                intrinsic_matrix=[380.08, 0.0, 467.79, 0.0, 380.08, 262.05, 0.0, 0.0, 1.0],
+                height=540,
+                width=960,
+                clipping_range=(0.1, 10),
+            ),
+            height=540,
+            width=960,
+            data_types=["depth"],
+        )
+        camera = TiledCamera(camera_cfg)
+
+        # Play sim
+        self.sim.reset()
+
+        # note: This is a workaround to ensure that the textures are loaded.
+        #   Check "Known Issues" section in the documentation for more details.
+        for _ in range(5):
+            self.sim.step()
+
+        camera.update(self.dt)
+
+        self.assertTrue(camera.data.output["depth"][camera.data.output["depth"] != 0.0].min() >= 0.1)
+        self.assertTrue(camera.data.output["depth"].max() <= 10.0)
+
+        del camera
+
     def test_multi_camera_init(self):
         """Test multi-camera initialization."""