Support subset for KITTI 3D format (#1621)

<!-- Contributing guide:
https://github.com/openvinotoolkit/datumaro/blob/develop/CONTRIBUTING.md
-->

### Summary

<!--
Resolves #111 and #222.
Depends on #1000 (for series of dependent commits).

This PR introduces this capability to make the project better in this
and that.

- Added this feature
- Removed that feature
- Fixed the problem #1234
-->
- Support subset structure for KITTI 3d format
- Update media type from PointCloud to Image

### How to test
<!-- Describe the testing procedure for reviewers, if changes are
not fully covered by unit tests or manual testing can be complicated.
-->
- Add unit test

### Checklist
<!-- Put an 'x' in all the boxes that apply -->
- [X] I have added unit tests to cover my changes.​
- [ ] I have added integration tests to cover my changes.​
- [X] I have added the description of my changes into
[CHANGELOG](https://github.com/openvinotoolkit/datumaro/blob/develop/CHANGELOG.md).​
- [ ] I have updated the
[documentation](https://github.com/openvinotoolkit/datumaro/tree/develop/docs)
accordingly

### License

- [ ] I submit _my code changes_ under the same [MIT
License](https://github.com/openvinotoolkit/datumaro/blob/develop/LICENSE)
that covers the project.
  Feel free to contact the maintainers if that's a concern.
- [ ] I have updated the license header for each file (see an example
below).

```python
# Copyright (C) 2024 Intel Corporation
#
# SPDX-License-Identifier: MIT
```

Author: sooahleex

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### New features
 - Support KITTI 3D format
   (<https://github.com/openvinotoolkit/datumaro/pull/1619>)
+  (<https://github.com/openvinotoolkit/datumaro/pull/1621>)
 - Add PseudoLabeling transform for unlabeled dataset
   (<https://github.com/openvinotoolkit/datumaro/pull/1594>)
 

src/datumaro/plugins/data_formats/kitti_3d/base.py

@@ -4,17 +4,18 @@
 
 import glob
 import logging
+import os
 import os.path as osp
 from typing import List, Optional, Type, TypeVar
 
-from datumaro.components.annotation import AnnotationType, Bbox, LabelCategories
+from datumaro.components.annotation import AnnotationType, Bbox
 from datumaro.components.dataset_base import DatasetItem, SubsetBase
 from datumaro.components.errors import InvalidAnnotationError
 from datumaro.components.importer import ImportContext
-from datumaro.components.media import Image, PointCloud
+from datumaro.components.media import Image
 from datumaro.util.image import find_images
 
-from .format import Kitti3dPath
+from .format import Kitti3DLabelMap, Kitti3dPath, make_kitti3d_categories
 
 T = TypeVar("T")
 
@@ -30,26 +31,37 @@ def __init__(
         ctx: Optional[ImportContext] = None,
     ):
         assert osp.isdir(path), path
-        super().__init__(subset=subset, media_type=PointCloud, ctx=ctx)
 
         self._path = path
 
-        common_attrs = {"truncated", "occluded", "alpha", "dimensions", "location", "rotation_y"}
-        self._categories = {AnnotationType.label: LabelCategories(attributes=common_attrs)}
+        if not subset:
+            folder_path = path.rsplit(Kitti3dPath.LABEL_DIR, 1)[0]
+            img_dir = osp.join(folder_path, Kitti3dPath.IMAGE_DIR)
+            if any(os.path.isdir(os.path.join(img_dir, item)) for item in os.listdir(img_dir)):
+                subset = osp.split(path)[-1]
+                self._path = folder_path
+        super().__init__(subset=subset, ctx=ctx)
+
+        self._categories = make_kitti3d_categories(Kitti3DLabelMap)
         self._items = self._load_items()
 
     def _load_items(self) -> List[DatasetItem]:
         items = []
+
         image_dir = osp.join(self._path, Kitti3dPath.IMAGE_DIR)
         image_path_by_id = {
-            osp.splitext(osp.relpath(p, image_dir))[0]: p
+            osp.split(osp.splitext(osp.relpath(p, image_dir))[0])[-1]: p
             for p in find_images(image_dir, recursive=True)
         }
 
-        ann_dir = osp.join(self._path, Kitti3dPath.LABEL_DIR)
+        if self._subset == "default":
+            ann_dir = osp.join(self._path, Kitti3dPath.LABEL_DIR)
+        else:
+            ann_dir = osp.join(self._path, Kitti3dPath.LABEL_DIR, self._subset)
+
         label_categories = self._categories[AnnotationType.label]
 
-        for labels_path in sorted(glob.glob(osp.join(ann_dir, "*.txt"), recursive=True)):
+        for labels_path in sorted(glob.glob(osp.join(ann_dir, "**", "*.txt"), recursive=True)):
             item_id = osp.splitext(osp.relpath(labels_path, ann_dir))[0]
             anns = []
 
@@ -116,17 +128,18 @@ def _load_items(self) -> List[DatasetItem]:
             if image:
                 image = Image.from_file(path=image)
 
+            if self._subset == "default":
+                calib_path = osp.join(self._path, Kitti3dPath.CALIB_DIR, item_id + ".txt")
+            else:
+                calib_path = osp.join(
+                    self._path, Kitti3dPath.CALIB_DIR, self._subset, item_id + ".txt"
+                )
             items.append(
                 DatasetItem(
                     id=item_id,
                     subset=self._subset,
-                    media=PointCloud.from_file(
-                        path=osp.join(self._path, Kitti3dPath.PCD_DIR, item_id + ".bin"),
-                        extra_images=[image],
-                    ),
-                    attributes={
-                        "calib_path": osp.join(self._path, Kitti3dPath.CALIB_DIR, item_id + ".txt")
-                    },
+                    media=image,
+                    attributes={"calib_path": calib_path},
                     annotations=anns,
                 )
             )

src/datumaro/plugins/data_formats/kitti_3d/format.py

@@ -4,9 +4,40 @@
 
 import os.path as osp
 
+from datumaro.components.annotation import AnnotationType, LabelCategories
+
 
 class Kitti3dPath:
     PCD_DIR = osp.join("velodyne")
     IMAGE_DIR = "image_2"
     LABEL_DIR = "label_2"
     CALIB_DIR = "calib"
+
+
+Kitti3DLabelMap = [
+    "DontCare",
+    "Car",
+    "Pedestrian",
+    "Van",
+    "Truck",
+    "Cyclist",
+    "Sitter",
+    "Train",
+    "Motorcycle",
+    "Bus",
+    "Misc",
+]
+
+
+def make_kitti3d_categories(label_map=None):
+    if label_map is None:
+        label_map = Kitti3DLabelMap
+
+    categories = {}
+    common_attrs = {"truncated", "occluded", "alpha", "dimensions", "location", "rotation_y"}
+    label_categories = LabelCategories(attributes=common_attrs)
+    for label in label_map:
+        label_categories.add(label)
+    categories[AnnotationType.label] = label_categories
+
+    return categories

src/datumaro/plugins/data_formats/kitti_3d/importer.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import os.path as osp
 from typing import List
 
 from datumaro.components.errors import DatasetImportError
@@ -16,7 +17,7 @@ class Kitti3dImporter(Importer):
 
     @classmethod
     def detect(cls, context: FormatDetectionContext) -> FormatDetectionConfidence:
-        context.require_file(f"{Kitti3dPath.PCD_DIR}/*.bin")
+        context.require_file(f"{Kitti3dPath.CALIB_DIR}/*.txt")
         cls._check_ann_file(context.require_file(f"{Kitti3dPath.LABEL_DIR}/*.txt"), context)
         return FormatDetectionConfidence.MEDIUM
 
@@ -42,4 +43,11 @@ def get_file_extensions(cls) -> List[str]:
 
     @classmethod
     def find_sources(cls, path):
-        return [{"url": path, "format": "kitti3d"}]
+        # return [{"url": path, "format": "kitti3d"}]
+        sources = cls._find_sources_recursive(
+            path, "", "kitti3d", dirname=Kitti3dPath.LABEL_DIR, file_filter=lambda p: osp.isdir(p)
+        )
+        if len(sources) == 0:
+            return [{"url": path, "format": "kitti3d"}]
+        else:
+            return sources