Support relative paths in import and export (#1463)

zhiltsov-max · nmanovic · web-flow · commit 0eb005c9f2a0 · 2020-06-19T14:10:30.000+03:00
* Move annotations to dm

* Refactor dm

* Rename data manager

* Move anno dump and upload functions

* Join server host and port in cvat cli

* Move export templates dir

* add dm project exporter

* update mask format support

* Use decorators for formats definition

* Update formats

* Update format implementations

* remove parameter

* Add dm views

* Move annotation components to dm

* restore extension for export formats

* update rest api

* use serializers, update views

* merge develop

* Update format names

* Update docs

* Update tests

* move test

* fix import

* Extend format tests

* django compatibility for directory access

* move tests

* update module links

* fixes

* fix git application

* fixes

* add extension recommentation

* fixes

* api

* join api methods

* Add trim whitespace to workspace config

* update tests

* fixes

* Update format docs

* join format queries

* fixes

* update new ui

* ui tests

* old ui

* update js bundles

* linter fixes

* add image with loader tests

* fix linter

* fix frame step and frame access

* use server file name for annotations export

* update cvat core

* add import hack for rest api tests

* move cli tests

* fix cvat format converter args parsing

* remove folder on extract error

* print error message on incorrect xpath expression

* use own categories when no others exist

* update changelog

* really add text to changelog

* Fix annotation window menu

* fix ui

* fix replace

* update extra apps

* format readme

* readme

* linter

* Fix old ui

* Update CHANGELOG.md

* update user guide

* linter

* more linter fixes

* update changelog

* Add image attributes

* add directory check in save image

* update image tests

* update image dir format with relative paths

* update datumaro format

* update coco format

* update cvat format

* update labelme format

* update mot format

* update image dir format

* update voc format

* update mot format

* update yolo format

* update labelme test

* update voc format

* update tfrecord format

* fixes

* update save_image usage

* remove item name conversion

* fix merge

* fix export

* prohibit relative paths in labelme format

* Add test for relative name matching

* move code

* implement frame matching

* fix yolo

* fix merge

* fix merge

* prettify code

* fix methid call

* fix frame matching in yolo

* add tests

* regularize function output

* update changelog

* fixes

* fix z_order use

* fix slash replacement

* linter

* t

* t2

Co-authored-by: Nikita Manovich &lt;40690625+nmanovic@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Removed information about e-mail from the basic user information (<https://github.com/opencv/cvat/pull/1627>)
 - Update https install manual. Makes it easier and more robust. Includes automatic renewing of lets encrypt certificates.
+- Implemented import and export of annotations with relative image paths (<https://github.com/opencv/cvat/pull/1463>)
 
 ### Deprecated
 -
diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py
@@ -5,6 +5,7 @@
 
 import os.path as osp
 from collections import OrderedDict, namedtuple
+from pathlib import Path
 
 from django.utils import timezone
 
@@ -125,8 +126,8 @@ def _init_frame_info(self):
             } for db_image in self._db_task.data.images.all()}
 
         self._frame_mapping = {
-            self._get_filename(info["path"]): frame
-            for frame, info in self._frame_info.items()
+            self._get_filename(info["path"]): frame_number
+            for frame_number, info in self._frame_info.items()
         }
 
     def _init_meta(self):
@@ -398,16 +399,27 @@ def db_task(self):
 
     @staticmethod
     def _get_filename(path):
-        return osp.splitext(osp.basename(path))[0]
-
-    def match_frame(self, filename):
-        # try to match by filename
-        _filename = self._get_filename(filename)
-        if _filename in self._frame_mapping:
-            return self._frame_mapping[_filename]
-
-        raise Exception(
-            "Cannot match filename or determine frame number for {} filename".format(filename))
+        return osp.splitext(path)[0]
+
+    def match_frame(self, path, root_hint=None):
+        path = self._get_filename(path)
+        match = self._frame_mapping.get(path)
+        if not match and root_hint and not path.startswith(root_hint):
+            path = osp.join(root_hint, path)
+            match = self._frame_mapping.get(path)
+        return match
+
+    def match_frame_fuzzy(self, path):
+        # Preconditions:
+        # - The input dataset is full, i.e. all items present. Partial dataset
+        # matching can't be correct for all input cases.
+        # - path is the longest path of input dataset in terms of path parts
+
+        path = Path(self._get_filename(path)).parts
+        for p, v in self._frame_mapping.items():
+            if Path(p).parts[-len(path):] == path: # endswith() for paths
+                return v
+        return None
 
 class CvatTaskDataExtractor(datumaro.SourceExtractor):
     def __init__(self, task_data, include_images=False):
@@ -450,8 +462,7 @@ def categories(self):
     def _load_categories(cvat_anno):
         categories = {}
 
-        label_categories = datumaro.LabelCategories(
-            attributes=['occluded', 'z_order'])
+        label_categories = datumaro.LabelCategories(attributes=['occluded'])
 
         for _, label in cvat_anno.meta['task']['labels']:
             label_categories.add(label['name'])
@@ -537,20 +548,14 @@ def convert_attrs(label, cvat_attrs):
 
         return item_anno
 
-def match_frame(item, task_data):
+def match_dm_item(item, task_data, root_hint=None):
     is_video = task_data.meta['task']['mode'] == 'interpolation'
 
     frame_number = None
     if frame_number is None and item.has_image:
-        try:
-            frame_number = task_data.match_frame(item.image.path)
-        except Exception:
-            pass
+        frame_number = task_data.match_frame(item.image.path, root_hint)
     if frame_number is None:
-        try:
-            frame_number = task_data.match_frame(item.id)
-        except Exception:
-            pass
+        frame_number = task_data.match_frame(item.id, root_hint)
     if frame_number is None:
         frame_number = cast(item.attributes.get('frame', item.id), int)
     if frame_number is None and is_video:
@@ -561,6 +566,19 @@ def match_frame(item, task_data):
             item.id)
     return frame_number
 
+def find_dataset_root(dm_dataset, task_data):
+    longest_path = max(dm_dataset, key=lambda x: len(Path(x.id).parts)).id
+    longest_match = task_data.match_frame_fuzzy(longest_path)
+    if longest_match is None:
+        return None
+
+    longest_match = osp.dirname(task_data.frame_info[longest_match]['path'])
+    prefix = longest_match[:-len(osp.dirname(longest_path)) or None]
+    if prefix.endswith('/'):
+        prefix = prefix[:-1]
+    return prefix
+
+
 def import_dm_annotations(dm_dataset, task_data):
     shapes = {
         datumaro.AnnotationType.bbox: ShapeType.RECTANGLE,
@@ -569,10 +587,16 @@ def import_dm_annotations(dm_dataset, task_data):
         datumaro.AnnotationType.points: ShapeType.POINTS,
     }
 
+    if len(dm_dataset) == 0:
+        return
+
     label_cat = dm_dataset.categories()[datumaro.AnnotationType.label]
 
+    root_hint = find_dataset_root(dm_dataset, task_data)
+
     for item in dm_dataset:
-        frame_number = task_data.abs_frame_id(match_frame(item, task_data))
+        frame_number = task_data.abs_frame_id(
+            match_dm_item(item, task_data, root_hint=root_hint))
 
         # do not store one-item groups
         group_map = {0: 0}
diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py
@@ -9,10 +9,11 @@
 from pyunpack import Archive
 
 from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
-    import_dm_annotations, match_frame)
+    import_dm_annotations, match_dm_item, find_dataset_root)
 from cvat.apps.dataset_manager.util import make_zip_archive
 from datumaro.components.extractor import DatasetItem
 from datumaro.components.project import Dataset
+from datumaro.plugins.yolo_format.extractor import YoloExtractor
 
 from .registry import dm_env, exporter, importer
 
@@ -33,17 +34,20 @@ def _import(src_file, task_data):
         Archive(src_file.name).extractall(tmp_dir)
 
         image_info = {}
-        anno_files = glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)
-        for filename in anno_files:
-            filename = osp.splitext(osp.basename(filename))[0]
+        frames = [YoloExtractor.name_from_path(osp.relpath(p, tmp_dir))
+            for p in glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)]
+        root_hint = find_dataset_root(
+            [DatasetItem(id=frame) for frame in frames], task_data)
+        for frame in frames:
             frame_info = None
             try:
-                frame_id = match_frame(DatasetItem(id=filename), task_data)
+                frame_id = match_dm_item(DatasetItem(id=frame), task_data,
+                    root_hint=root_hint)
                 frame_info = task_data.frame_info[frame_id]
             except Exception:
                 pass
             if frame_info is not None:
-                image_info[filename] = (frame_info['height'], frame_info['width'])
+                image_info[frame] = (frame_info['height'], frame_info['width'])
 
         dataset = dm_env.make_importer('yolo')(tmp_dir, image_info=image_info) \
             .make_dataset()
diff --git a/cvat/apps/dataset_manager/tests/_test_formats.py b/cvat/apps/dataset_manager/tests/_test_formats.py
@@ -70,6 +70,10 @@ def _setUpModule():
 from rest_framework.test import APITestCase, APIClient
 from rest_framework import status
 
+from cvat.apps.dataset_manager.annotation import AnnotationIR
+from cvat.apps.dataset_manager.bindings import TaskData, find_dataset_root
+from cvat.apps.engine.models import Task
+
 _setUpModule()
 
 from cvat.apps.dataset_manager.annotation import AnnotationIR
@@ -256,7 +260,7 @@ def _generate_annotations(self, task):
         self._put_api_v1_task_id_annotations(task["id"], annotations)
         return annotations
 
-    def _generate_task_images(self, count):
+    def _generate_task_images(self, count): # pylint: disable=no-self-use
         images = {
             "client_files[%d]" % i: generate_image_file("image_%d.jpg" % i)
             for i in range(count)
@@ -385,6 +389,7 @@ def load_dataset(src):
 
                             # NOTE: can't import cvat.utils.cli
                             # for whatever reason, so remove the dependency
+                            #
                             project.config.remove('sources')
 
                             return project.make_dataset()
@@ -436,3 +441,97 @@ def test_can_make_abs_frame_id_from_known(self):
         task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task['id']))
 
         self.assertEqual(5, task_data.abs_frame_id(2))
+
+class FrameMatchingTest(_DbTestBase):
+    def _generate_task_images(self, paths): # pylint: disable=no-self-use
+        f = BytesIO()
+        with zipfile.ZipFile(f, 'w') as archive:
+            for path in paths:
+                archive.writestr(path, generate_image_file(path).getvalue())
+        f.name = 'images.zip'
+        f.seek(0)
+
+        return {
+            'client_files[0]': f,
+            'image_quality': 75,
+        }
+
+    def _generate_task(self, images):
+        task = {
+            "name": "my task #1",
+            "owner": '',
+            "assignee": '',
+            "overlap": 0,
+            "segment_size": 100,
+            "z_order": False,
+            "labels": [
+                {
+                    "name": "car",
+                    "attributes": [
+                        {
+                            "name": "model",
+                            "mutable": False,
+                            "input_type": "select",
+                            "default_value": "mazda",
+                            "values": ["bmw", "mazda", "renault"]
+                        },
+                        {
+                            "name": "parked",
+                            "mutable": True,
+                            "input_type": "checkbox",
+                            "default_value": False
+                        },
+                    ]
+                },
+                {"name": "person"},
+            ]
+        }
+        return self._create_task(task, images)
+
+    def test_frame_matching(self):
+        task_paths = [
+            'a.jpg',
+            'a/a.jpg',
+            'a/b.jpg',
+            'b/a.jpg',
+            'b/c.jpg',
+            'a/b/c.jpg',
+            'a/b/d.jpg',
+        ]
+
+        images = self._generate_task_images(task_paths)
+        task = self._generate_task(images)
+        task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task["id"]))
+
+        for input_path, expected, root in [
+            ('z.jpg', None, ''), # unknown item
+            ('z/a.jpg', None, ''), # unknown item
+
+            ('d.jpg', 'a/b/d.jpg', 'a/b'), # match with root hint
+            ('b/d.jpg', 'a/b/d.jpg', 'a'), # match with root hint
+        ] + list(zip(task_paths, task_paths, [None] * len(task_paths))): # exact matches
+            with self.subTest(input=input_path):
+                actual = task_data.match_frame(input_path, root)
+                if actual is not None:
+                    actual = task_data.frame_info[actual]['path']
+                self.assertEqual(expected, actual)
+
+    def test_dataset_root(self):
+        for task_paths, dataset_paths, expected in [
+            ([ 'a.jpg', 'b/c/a.jpg' ], [ 'a.jpg', 'b/c/a.jpg' ], ''),
+            ([ 'b/a.jpg', 'b/c/a.jpg' ], [ 'a.jpg', 'c/a.jpg' ], 'b'), # 'images from share' case
+            ([ 'b/c/a.jpg' ], [ 'a.jpg' ], 'b/c'), # 'images from share' case
+            ([ 'a.jpg' ], [ 'z.jpg' ], None),
+        ]:
+            with self.subTest(expected=expected):
+                images = self._generate_task_images(task_paths)
+                task = self._generate_task(images)
+                task_data = TaskData(AnnotationIR(),
+                    Task.objects.get(pk=task["id"]))
+                dataset = [
+                    datumaro.components.extractor.DatasetItem(
+                        id=osp.splitext(p)[0])
+                    for p in dataset_paths]
+
+                root = find_dataset_root(dataset, task_data)
+                self.assertEqual(expected, root)

Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`
`19`	`19`	`### Changed`
`20`	`20`	`- Removed information about e-mail from the basic user information (<https://github.com/opencv/cvat/pull/1627>)`
`21`	`21`	`- Update https install manual. Makes it easier and more robust. Includes automatic renewing of lets encrypt certificates.`
	`22`	`+- Implemented import and export of annotations with relative image paths (<https://github.com/opencv/cvat/pull/1463>)`
`22`	`23`
`23`	`24`	`### Deprecated`
`24`	`25`	`-`