Merge pull request #121 from openvinotoolkit/develop

Release 0.1.6

Author: Maxim Zhiltsov

CHANGELOG.md

@@ -6,6 +6,51 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [Unreleased]
+### Added
+-
+
+### Changed
+-
+
+### Deprecated
+-
+
+### Removed
+-
+
+### Fixed
+-
+
+### Security
+-
+
+## 02/26/2021 - Release v0.1.6
+### Added
+- `Icdar13/15` dataset format (<https://github.com/openvinotoolkit/datumaro/pull/96>)
+- Laziness, source caching, tracking of changes and partial updating for `Dataset` (<https://github.com/openvinotoolkit/datumaro/pull/102>)
+- `Market-1501` dataset format (<https://github.com/openvinotoolkit/datumaro/pull/108>)
+- `LFW` dataset format (<https://github.com/openvinotoolkit/datumaro/pull/110>)
+- Support of polygons' and masks' confusion matrices and mismathing classes in `diff` command (<https://github.com/openvinotoolkit/datumaro/pull/117>)
+- Add near duplicate image removal plugin (<https://github.com/openvinotoolkit/datumaro/pull/113>)
+
+### Changed
+- OpenVINO model launcher is updated for OpenVINO r2021.1 (<https://github.com/openvinotoolkit/datumaro/pull/100>)
+
+### Deprecated
+-
+
+### Removed
+-
+
+### Fixed
+- High memory consumption and low performance of mask import/export, #53 (<https://github.com/openvinotoolkit/datumaro/pull/101>)
+- Masks, covered by class 0 (background), should be exported with holes inside (<https://github.com/openvinotoolkit/datumaro/pull/104>)
+- `diff` command invocation problem with missing class methods (<https://github.com/openvinotoolkit/datumaro/pull/117>)
+
+### Security
+-
+
 ## 01/23/2021 - Release v0.1.5
 ### Added
 - `WiderFace` dataset format (<https://github.com/openvinotoolkit/datumaro/pull/65>, <https://github.com/openvinotoolkit/datumaro/pull/90>)

README.md

@@ -135,6 +135,9 @@ CVAT annotations                             ---> Publication, statistics etc.
   - [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/)
   - [CVAT](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md)
   - [LabelMe](http://labelme.csail.mit.edu/Release3.0)
+  - [ICDAR13/15](https://rrc.cvc.uab.es/?ch=2) (`word_recognition`, `text_localization`, `text_segmentation`)
+  - [Market-1501](https://www.aitribune.com/dataset/2018051063) (`person re-identification`)
+  - [LFW](http://vis-www.cs.umass.edu/lfw/) (`person re-identification`, `landmarks`)
 - Dataset building
   - Merging multiple datasets into one
   - Dataset filtering by a custom criteria:
@@ -217,8 +220,7 @@ reading, exporting and iteration capabilities, simplifying integration of custom
 formats and providing high performance operations:
 
 ``` python
-from datumaro.components.project import Project # project-related things
-import datumaro.components.extractor # annotations and high-level interfaces
+from datumaro.components.project import Project
 
 # load a Datumaro project
 project = Project.load('directory')
@@ -227,10 +229,10 @@ project = Project.load('directory')
 dataset = project.make_dataset()
 
 # keep only annotated images
-dataset = dataset.select(lambda item: len(item.annotations) != 0)
+dataset.select(lambda item: len(item.annotations) != 0)
 
 # change dataset labels
-dataset = dataset.transform(project.env.transforms.get('remap_labels'),
+dataset.transform('remap_labels',
   {'cat': 'dog', # rename cat to dog
     'truck': 'car', # rename truck to car
     'person': '', # remove this label

datumaro/cli/commands/merge.py

@@ -9,8 +9,8 @@
 from collections import OrderedDict
 
 from datumaro.components.project import Project
-from datumaro.components.operations import (IntersectMerge,
-    QualityError, MergeError)
+from datumaro.components.operations import IntersectMerge
+from datumaro.components.errors import QualityError, MergeError
 
 from ..util import at_least, MultilineFormatter, CliException
 from ..util.project import generate_next_file_name, load_project

datumaro/cli/contexts/model.py

@@ -6,8 +6,10 @@
 import logging as log
 import os
 import os.path as osp
+import shutil
 
 from datumaro.components.project import Environment
+from datumaro.util import error_rollback
 
 from ..util import CliException, MultilineFormatter, add_subparser
 from ..util.project import load_project, \
@@ -43,6 +45,7 @@ def build_add_parser(parser_ctor=argparse.ArgumentParser):
 
     return parser
 
+@error_rollback('on_error', implicit=True)
 def add_command(args):
     project = load_project(args.project_dir)
 
@@ -69,6 +72,7 @@ def add_command(args):
         model_dir = osp.join(project.config.project_dir,
             project.local_model_dir(args.name))
         os.makedirs(model_dir, exist_ok=False)
+        on_error.do(shutil.rmtree, model_dir, ignore_errors=True)
 
         try:
             cli_plugin.copy_model(model_dir, model_args)

datumaro/cli/contexts/project/__init__.py

@@ -18,11 +18,12 @@
 from datumaro.components.project import \
     PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
 from datumaro.components.project import Environment, Project
+from datumaro.util import error_rollback
 
 from ...util import (CliException, MultilineFormatter, add_subparser,
     make_file_name)
 from ...util.project import generate_next_file_name, load_project
-from .diff import DiffVisualizer
+from .diff import DatasetDiffVisualizer
 
 
 def build_create_parser(parser_ctor=argparse.ArgumentParser):
@@ -506,8 +507,8 @@ def build_diff_parser(parser_ctor=argparse.ArgumentParser):
     parser.add_argument('-o', '--output-dir', dest='dst_dir', default=None,
         help="Directory to save comparison results (default: do not save)")
     parser.add_argument('-v', '--visualizer',
-        default=DiffVisualizer.DEFAULT_FORMAT,
-        choices=[f.name for f in DiffVisualizer.Format],
+        default=DatasetDiffVisualizer.DEFAULT_FORMAT.name,
+        choices=[f.name for f in DatasetDiffVisualizer.OutputFormat],
         help="Output format (default: %(default)s)")
     parser.add_argument('--iou-thresh', default=0.5, type=float,
         help="IoU match threshold for detections (default: %(default)s)")
@@ -521,6 +522,7 @@ def build_diff_parser(parser_ctor=argparse.ArgumentParser):
 
     return parser
 
+@error_rollback('on_error', implicit=True)
 def diff_command(args):
     first_project = load_project(args.project_dir)
     second_project = load_project(args.other_project_dir)
@@ -540,17 +542,14 @@ def diff_command(args):
     dst_dir = osp.abspath(dst_dir)
     log.info("Saving diff to '%s'" % dst_dir)
 
-    dst_dir_existed = osp.exists(dst_dir)
-    try:
-        visualizer = DiffVisualizer(save_dir=dst_dir, comparator=comparator,
-            output_format=args.visualizer)
-        visualizer.save_dataset_diff(
+    if not osp.exists(dst_dir):
+        on_error.do(shutil.rmtree, dst_dir, ignore_errors=True)
+
+    with DatasetDiffVisualizer(save_dir=dst_dir, comparator=comparator,
+            output_format=args.visualizer) as visualizer:
+        visualizer.save(
             first_project.make_dataset(),
             second_project.make_dataset())
-    except BaseException:
-        if not dst_dir_existed and osp.isdir(dst_dir):
-            shutil.rmtree(dst_dir, ignore_errors=True)
-        raise
 
     return 0