[Datumaro] CVAT format import (#974)

* Add label-specific attributes
* Add CVAT format import
* Register CVAT format
* Add little more logs
* Little refactoring for tests
* Cvat format checks
* Add missing check
* Refactor datumaro format
* Little refactoring
* Regularize dataset importer logic
* Fix project import issue
* Refactor coco extractor
* Refactor tests
* Codacy

Author: zhiltsov-max

datumaro/datumaro/cli/project/__init__.py

@@ -7,6 +7,7 @@
 import logging as log
 import os
 import os.path as osp
+import shutil
 
 from datumaro.components.project import Project
 from datumaro.components.comparator import Comparator
@@ -27,10 +28,13 @@ def create_command(args):
     project_dir = osp.abspath(args.dst_dir)
     project_path = make_project_path(project_dir)
 
-    if not args.overwrite and osp.isdir(project_dir) and os.listdir(project_dir):
-        log.error("Directory '%s' already exists "
-            "(pass --overwrite to force creation)" % project_dir)
-        return 1
+    if osp.isdir(project_dir) and os.listdir(project_dir):
+        if not args.overwrite:
+            log.error("Directory '%s' already exists "
+                "(pass --overwrite to force creation)" % project_dir)
+            return 1
+        else:
+            shutil.rmtree(project_dir)
     os.makedirs(project_dir, exist_ok=args.overwrite)
 
     if not args.overwrite and osp.isfile(project_path):
@@ -78,10 +82,13 @@ def import_command(args):
     project_dir = osp.abspath(args.dst_dir)
     project_path = make_project_path(project_dir)
 
-    if not args.overwrite and osp.isdir(project_dir) and os.listdir(project_dir):
-        log.error("Directory '%s' already exists "
-            "(pass --overwrite to force creation)" % project_dir)
-        return 1
+    if osp.isdir(project_dir) and os.listdir(project_dir):
+        if not args.overwrite:
+            log.error("Directory '%s' already exists "
+                "(pass --overwrite to force creation)" % project_dir)
+            return 1
+        else:
+            shutil.rmtree(project_dir)
     os.makedirs(project_dir, exist_ok=args.overwrite)
 
     if not args.overwrite and osp.isfile(project_path):
@@ -147,7 +154,11 @@ def export_command(args):
         return 1
     os.makedirs(dst_dir, exist_ok=args.overwrite)
 
-    project.make_dataset().export(
+    log.info("Loading the project...")
+    dataset = project.make_dataset()
+
+    log.info("Exporting the project...")
+    dataset.export(
         save_dir=dst_dir,
         output_format=args.output_format,
         filter_expr=args.filter,

datumaro/datumaro/cli/source/__init__.py

@@ -210,8 +210,12 @@ def export_command(args):
         return 1
     os.makedirs(dst_dir, exist_ok=args.overwrite)
 
+    log.info("Loading the project...")
     source_project = project.make_source_project(args.name)
-    source_project.make_dataset().export(
+    dataset = source_project.make_dataset()
+
+    log.info("Exporting the project...")
+    dataset.export(
         save_dir=dst_dir,
         output_format=args.output_format,
         filter_expr=args.filter,

datumaro/datumaro/components/converters/__init__.py

@@ -24,10 +24,7 @@
 )
 
 from datumaro.components.converters.yolo import YoloConverter
-
-from datumaro.components.converters.tfrecord import (
-    DetectionApiConverter,
-)
+from datumaro.components.converters.tfrecord import DetectionApiConverter
 
 
 items = [

datumaro/datumaro/components/converters/datumaro.py

@@ -11,8 +11,7 @@
 
 from datumaro.components.converter import Converter
 from datumaro.components.extractor import (
-    DEFAULT_SUBSET_NAME,
-    AnnotationType, Annotation,
+    DEFAULT_SUBSET_NAME, Annotation,
     LabelObject, MaskObject, PointsObject, PolygonObject,
     PolyLineObject, BboxObject, CaptionObject,
     LabelCategories, MaskCategories, PointsCategories
@@ -52,11 +51,13 @@ def items(self):
 
     def write_item(self, item):
         annotations = []
-        self.items.append({
+        item_desc = {
             'id': item.id,
-            'path': item.path,
             'annotations': annotations,
-        })
+        }
+        if item.path:
+            item_desc['path'] = item.path
+        self.items.append(item_desc)
 
         for ann in item.annotations:
             if isinstance(ann, LabelObject):

datumaro/datumaro/components/extractor.py

@@ -476,6 +476,7 @@ def __eq__(self, other):
             (self.id == other.id) and \
             (self.subset == other.subset) and \
             (self.annotations == other.annotations) and \
+            (self.path == other.path) and \
             (self.has_image == other.has_image) and \
             (self.has_image and np.all(self.image == other.image) or \
                 not self.has_image)

datumaro/datumaro/components/extractors/__init__.py

@@ -26,14 +26,9 @@
     VocComp_9_10_Extractor,
 )
 
-from datumaro.components.extractors.yolo import (
-    YoloExtractor,
-)
-
-from datumaro.components.extractors.tfrecord import (
-    DetectionApiExtractor,
-)
-
+from datumaro.components.extractors.yolo import YoloExtractor
+from datumaro.components.extractors.tfrecord import DetectionApiExtractor
+from datumaro.components.extractors.cvat import CvatExtractor
 
 items = [
     ('datumaro', DatumaroExtractor),
@@ -59,4 +54,6 @@
     ('yolo', YoloExtractor),
 
     ('tf_detection_api', DetectionApiExtractor),
+
+    ('cvat', CvatExtractor),
 ]