Deprecate NDR, DSA, and some filters. Remove deprecation from tiling. (#1780)

gdlg · web-flow · commit 0f10fc3c3b14 · 2025-07-09T13:26:25.000+01:00
### Summary This PR deprecates NDR, DSA (data shift analysis), model transform, xpath filters. It also remove the deprecation of tiling as it is used by OTX.  ### How to test  ### Checklist  - [ ] I have added unit tests to cover my changes.​ - [ ] I have added integration tests to cover my changes.​ - [ ] I have added the description of my changes into [CHANGELOG](https://github.com/open-edge-platform/datumaro/blob/develop/CHANGELOG.md).​ - [ ] I have updated the [documentation](https://github.com/open-edge-platform/datumaro/tree/develop/docs) accordingly ### License - [ ] I submit _my code changes_ under the same [MIT License](https://github.com/open-edge-platform/datumaro/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. - [ ] I have updated the license header for each file (see an example below). ```python # Copyright (C) 2025 Intel Corporation # # SPDX-License-Identifier: MIT ```
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,21 +31,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Deprecations
 - Added deprecation notices to the following features that will soon be removed:
+  - Inference
+  - Model-based transformations
   - Crypter
-  - Advanced dataset comparison based on similarity search with OpenVINO
   - Synthetic dataset generation
   - Data exploration
   - BBox to mask using SAM
-  - Tiling
-  - Dataset versioning with DVCS
   - Telemetry
   - Anchor generation
   - Missing annotation detection
   - Model inference explanation
   - Near-duplicate removal
   - Pruning
   - Pseudo-labels
-  (<https://github.com/open-edge-platform/datumaro/pull/1776>)
+  (<https://github.com/open-edge-platform/datumaro/pull/1776>, <https://github.com/open-edge-platform/datumaro/pull/1780>)
 
 ## Q1 2025 Release 1.10.0
 
diff --git a/src/datumaro/components/abstracts/model_interpreter.py b/src/datumaro/components/abstracts/model_interpreter.py
@@ -9,6 +9,7 @@
 
 from datumaro.components.annotation import Annotation
 from datumaro.components.dataset_base import DatasetItem
+from datumaro.util.deprecation import deprecated
 
 __all__ = ["IModelInterpreter", "PrepInfo", "ModelPred", "LauncherInputType"]
 
@@ -17,6 +18,7 @@
 LauncherInputType = Union[np.ndarray, Dict[str, np.ndarray]]
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class IModelInterpreter(ABC):
     @abstractmethod
     def preprocess(self, inp: DatasetItem) -> Tuple[LauncherInputType, PrepInfo]:
diff --git a/src/datumaro/components/algorithms/hash_key_inference/base.py b/src/datumaro/components/algorithms/hash_key_inference/base.py
@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING, Sequence
 
 from datumaro.components.dataset import Dataset
+from datumaro.util.deprecation import deprecated
 
 if TYPE_CHECKING:
     import datumaro.plugins.explorer as explorer
@@ -14,6 +15,7 @@
     explorer = lazy_import("datumaro.plugins.explorer")
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class HashInference:
     def __init__(self, *datasets: Sequence[Dataset]) -> None:
         pass
diff --git a/src/datumaro/components/algorithms/hash_key_inference/prune.py b/src/datumaro/components/algorithms/hash_key_inference/prune.py
@@ -81,6 +81,7 @@ def base(
         raise NotImplementedError
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class RandomSelect(PruneBase):
     """
     Select items randomly from the dataset.
@@ -95,6 +96,7 @@ def base(self, ratio, num_centers, labels, database_keys, item_list, source):
         return selected_items, None
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class Centroid(PruneBase):
     """
     Select items through clustering with centers targeting the desired number.
@@ -127,6 +129,7 @@ def base(self, ratio, num_centers, labels, database_keys, item_list, source):
         return selected_items, dist_tuples
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class ClusteredRandom(PruneBase):
     """
     Select items through clustering and choose randomly within each cluster.
@@ -153,6 +156,7 @@ def base(self, ratio, num_centers, labels, database_keys, item_list, source):
         return selected_items, None
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class QueryClust(PruneBase):
     """
     Select items through clustering with inits that imply each label.
@@ -206,6 +210,7 @@ def base(self, ratio, num_centers, labels, database_keys, item_list, source):
         return selected_items, dist_tuples
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class Entropy(PruneBase):
     """
     Select items through clustering and choose them based on label entropy in each cluster.
diff --git a/src/datumaro/components/annotation.py b/src/datumaro/components/annotation.py
@@ -33,6 +33,7 @@
 
 from datumaro.components.media import Image
 from datumaro.util.attrs_util import default_if_none, not_empty
+from datumaro.util.deprecation import deprecated
 from datumaro.util.points_util import normalize_points
 
 
@@ -257,6 +258,7 @@ class Label(Annotation):
     label: int = field(converter=int)
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 @attrs(slots=True, eq=False, order=False)
 class HashKey(Annotation):
     _type = AnnotationType.hash_key
@@ -276,6 +278,7 @@ def __eq__(self, other):
         return np.array_equal(self.hash_key, other.hash_key)
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 @attrs(eq=False, order=False)
 class FeatureVector(Annotation):
     _type = AnnotationType.feature_vector
diff --git a/src/datumaro/components/filter.py b/src/datumaro/components/filter.py
@@ -24,6 +24,7 @@
 )
 from datumaro.components.media import Image
 from datumaro.components.transformer import ItemTransform
+from datumaro.util.deprecation import deprecated
 
 if TYPE_CHECKING:
     from datumaro.components.dataset_base import CategoriesInfo, DatasetItem, IDataset
@@ -36,6 +37,7 @@
 ]
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class DatasetItemEncoder:
     @classmethod
     def encode(
@@ -268,6 +270,7 @@ def to_string(encoded_item: ET.ElementBase) -> str:
         return ET.tostring(encoded_item, encoding="unicode", pretty_print=True)
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class XPathDatasetFilter(ItemTransform):
     def __init__(self, extractor: IDataset, xpath: str) -> None:
         super().__init__(extractor)
@@ -289,6 +292,7 @@ def transform_item(self, item: DatasetItem) -> Optional[DatasetItem]:
         return item
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class XPathAnnotationsFilter(ItemTransform):
     def __init__(self, extractor: IDataset, xpath: str, remove_empty: bool = False) -> None:
         super().__init__(extractor)
diff --git a/src/datumaro/components/hl_ops/__init__.py b/src/datumaro/components/hl_ops/__init__.py
@@ -26,6 +26,7 @@
 from datumaro.components.transformer import ModelTransform, Transform
 from datumaro.components.validator import TaskType, Validator
 from datumaro.util import parse_str_enum_value
+from datumaro.util.deprecation import deprecated
 from datumaro.util.scope import on_error_do, scoped
 
 if TYPE_CHECKING:
@@ -296,6 +297,7 @@ def merge(
         return Dataset(source=merged, env=env)
 
     @staticmethod
+    @deprecated(deprecated_version="1.11", removed_version="1.12")
     def run_model(
         dataset: IDataset,
         model: Union[Launcher, Type[ModelTransform]],
diff --git a/src/datumaro/components/shift_analyzer.py b/src/datumaro/components/shift_analyzer.py
@@ -13,6 +13,7 @@
 from datumaro.components.dataset import IDataset
 from datumaro.components.launcher import LauncherWithModelInterpreter
 from datumaro.util import take_by
+from datumaro.util.deprecation import deprecated
 
 if TYPE_CHECKING:
     from scipy import linalg, stats
@@ -26,6 +27,7 @@
     shift_launcher = lazy_import("datumaro.plugins.openvino_plugin.shift_launcher")
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class RunningStats1D:
     def __init__(self):
         self.running_mean = None
@@ -67,6 +69,7 @@ def cov(self) -> np.ndarray:
         return self.running_sq_mean - np.matmul(mean, mean.transpose())
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class FeatureAccumulator:
     def __init__(self, model: LauncherWithModelInterpreter):
         self.model = model
@@ -83,6 +86,7 @@ def get_activation_stats(self, dataset: IDataset) -> RunningStats1D:
         return running_stats
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class FeatureAccumulatorByLabel(FeatureAccumulator):
     def __init__(self, model):
         super().__init__(model)
@@ -106,6 +110,7 @@ def get_activation_stats(self, dataset: IDataset) -> Dict[int, RunningStats1D]:
         return running_stats
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class ShiftAnalyzer:
     def __init__(self) -> None:
         """
diff --git a/src/datumaro/components/transformer.py b/src/datumaro/components/transformer.py
@@ -11,6 +11,7 @@
 from datumaro.components.dataset_base import DatasetBase, DatasetItem, IDataset
 from datumaro.components.launcher import Launcher
 from datumaro.util import is_method_redefined, take_by
+from datumaro.util.deprecation import deprecated
 from datumaro.util.multi_procs_util import consumer_generator
 
 
@@ -146,6 +147,7 @@ def _process_batch(
         return results
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class ModelTransform(Transform):
     """A transformation class for applying a model's inference to dataset items.
 
diff --git a/src/datumaro/plugins/ndr.py b/src/datumaro/plugins/ndr.py
@@ -13,6 +13,7 @@
 from datumaro.components.dataset_base import DEFAULT_SUBSET_NAME
 from datumaro.components.transformer import Transform
 from datumaro.util import parse_str_enum_value
+from datumaro.util.deprecation import deprecated
 
 
 class Algorithm(Enum):
@@ -30,6 +31,7 @@ class UnderSamplingMethod(Enum):
     inverse = auto()
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class NDR(Transform, CliPlugin):
     """
     Removes near-duplicated images in subset|n
diff --git a/src/datumaro/plugins/sam_transforms/automatic_mask_gen.py b/src/datumaro/plugins/sam_transforms/automatic_mask_gen.py
@@ -20,10 +20,12 @@
     TLSConfig,
 )
 from datumaro.plugins.sam_transforms.interpreters.sam_decoder_for_amg import AMGMasks, AMGPoints
+from datumaro.util.deprecation import deprecated
 
 __all__ = ["SAMAutomaticMaskGeneration"]
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 class SAMAutomaticMaskGeneration(ModelTransform, CliPlugin):
     """Produce instance segmentation masks automatically using Segment Anything Model (SAM).
 
diff --git a/src/datumaro/plugins/tiling/merge_tile.py b/src/datumaro/plugins/tiling/merge_tile.py
@@ -29,7 +29,6 @@
 from datumaro.components.media import BboxIntCoords, MosaicImage
 from datumaro.components.transformer import Transform
 from datumaro.plugins.tiling.util import x1y1x2y2_to_xywh, xywh_to_x1y1x2y2
-from datumaro.util.deprecation import deprecated
 
 AnnotationsForMerge = List[Tuple[Annotation, BboxIntCoords, sg.Polygon]]
 
@@ -194,7 +193,6 @@ def _merge_not_support(ann_type: AnnotationType, *args, **kwargs) -> None:
     raise DatumaroError(f"type(ann)={ann_type} is not support tiling.")
 
 
-@deprecated(deprecated_version="1.11", removed_version="1.12")
 class MergeTile(Transform, CliPlugin):
     """
     Transformation to merge the previously tiled dataset. It can generally
diff --git a/src/datumaro/plugins/tiling/tile.py b/src/datumaro/plugins/tiling/tile.py
@@ -30,7 +30,6 @@
     x1y1x2y2_to_xywh,
     xywh_to_x1y1x2y2,
 )
-from datumaro.util.deprecation import deprecated
 
 
 def _apply_offset(geom: sg.base.BaseGeometry, roi_box: sg.Polygon) -> sg.base.BaseGeometry:
@@ -133,7 +132,6 @@ def _tile_not_support(ann: Annotation, *args, **kwargs) -> None:
     raise DatumaroError(f"type(ann)={type(ann)} is not support tiling.")
 
 
-@deprecated(deprecated_version="1.11", removed_version="1.12")
 class Tile(Transform, CliPlugin):
     """
     Apply tile tranformation to items in the dataset.
diff --git a/src/datumaro/util/meta_file_util.py b/src/datumaro/util/meta_file_util.py
@@ -10,6 +10,7 @@
 
 from datumaro.components.annotation import AnnotationType, HashKey
 from datumaro.util import dump_json_file, find, parse_json_file
+from datumaro.util.deprecation import deprecated
 
 DATASET_META_FILE = "dataset_meta.json"
 DATASET_HASHKEY_FILE = "hash_keys.json"
@@ -105,6 +106,7 @@ def parse_hashkey_file(path):
     return hashkey_dict
 
 
+@deprecated(deprecated_version="1.11", removed_version="1.12")
 def save_hashkey_file(path, item_list):
     dataset_hashkey = {}
 
@@ -132,12 +134,17 @@ def save_hashkey_file(path, item_list):
     dump_json_file(meta_file, dataset_hashkey, indent=True)
 
 
-def load_hash_key(path, dataset):
-    if not os.path.isdir(path) or not has_hashkey_file(path):
-        return dataset
-
+@deprecated(deprecated_version="1.11", removed_version="1.12")
+def _load_hash_key(path, dataset):
     hashkey_dict = parse_hashkey_file(path)
     for item in dataset:
         hash_key = hashkey_dict[item.subset + "/" + item.id]
         item.annotations.append(HashKey(hash_key=np.asarray(hash_key, dtype=np.uint8)))
     return dataset
+
+
+def load_hash_key(path, dataset):
+    if not os.path.isdir(path) or not has_hashkey_file(path):
+        return dataset
+
+    return _load_hash_key(path, dataset)
