Metrics redesign (#2326)

* refactor metrics callback

* add Evaluator class

* remove unused import

* add AnomalibMetric base class

* cleanup

* add create_anomalib_metric function

* use new AnomalibMetric class

* verify keys when updating metric

* use anomalib PR curve for f1max

* replace evaluator with metrics arg

* revert removing exportable transform method

* fix unit tests

* add kwargs to lightning models

* fix pre-commit issues

* remove kwargs key from update config

* add Evaluator to metrics init

* set default evaluator for image-only models

* fix cfa tests

* fix model tests

* update changelog

* remove tests of deprecated callback modules

* fix engine tests

* fix 201 notebook

* fix 600 notebook

* add default evaluator to fastflow

* revert cfa changes and fix pred score computation

* simplify evaluator argument

* validate metrics in evaluator

* remove unused import

* fix pred_label shape validation

* fix method name

* fix pred_label validator

* update validator tests

* fix method name in fastflow

* add AnomalibMetric functionality to PIMO

* add optional default fields and implement for pimo

* update aupimo notebooks

* use explicit args in lightning model signature

* remove kwargs from evaluator

* add _resolve_evaluator method

* fix config upgrade test

* do not force pypi install in mlflow notebook

* do not force pypi install in mlflow notebook

Author: djdameln

CHANGELOG.md

@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- Implement new design for metrics by @djdameln https://github.com/openvinotoolkit/anomalib/pull/2326
 - Set permissions for github workflows by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/2127
 - Update timm requirement from <=1.0.3,>=0.5.4 to >=0.5.4,<=1.0.7 by @dependabot in https://github.com/openvinotoolkit/anomalib/pull/2151
 - 🚀 Use gh actions runners for pre-commit checks by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2160

notebooks/200_models/201_fastflow.ipynb

@@ -233,7 +233,6 @@
    "source": [
     "engine = Engine(\n",
     "    callbacks=callbacks,\n",
-    "    pixel_metrics=\"AUROC\",\n",
     "    accelerator=\"auto\",  # \\<\"cpu\", \"gpu\", \"tpu\", \"ipu\", \"hpu\", \"auto\">,\n",
     "    devices=1,\n",
     "    logger=False,\n",

notebooks/600_loggers/601_mlflow_logging.ipynb

@@ -38,7 +38,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install -qU anomalib"
+    "%pip install anomalib"
    ]
   },
   {
@@ -56,7 +56,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install -qU anomalib[loggers]"
+    "%pip install anomalib[loggers]"
    ]
   },
   {
@@ -307,7 +307,6 @@
     "\n",
     "engine = Engine(\n",
     "    callbacks=callbacks,\n",
-    "    pixel_metrics=\"AUROC\",\n",
     "    accelerator=\"auto\",\n",
     "    devices=1,\n",
     "    logger=mlflow_logger,  # Logger is set here\n",

notebooks/700_metrics/701a_aupimo.ipynb

@@ -148,8 +148,6 @@ def add_arguments_to_parser(parser: ArgumentParser) -> None:
             ``Engine`` class should be reflected manually.
         """
         parser.add_argument("--task", type=TaskType | str, default=TaskType.SEGMENTATION)
-        parser.add_argument("--metrics.image", type=list[str] | str | None, default=None)
-        parser.add_argument("--metrics.pixel", type=list[str] | str | None, default=None, required=False)
         parser.add_argument("--logging.log_graph", type=bool, help="Log the model to the logger", default=False)
         if hasattr(parser, "subcommand") and parser.subcommand not in {"export", "predict"}:
             parser.link_arguments("task", "data.init_args.task")
@@ -323,8 +321,6 @@ def instantiate_engine(self) -> None:
 
         engine_args = {
             "task": self._get(self.config_init, "task"),
-            "image_metrics": self._get(self.config_init, "metrics.image"),
-            "pixel_metrics": self._get(self.config_init, "metrics.pixel"),
         }
         trainer_config = {**self._get(self.config_init, "trainer", default={}), **engine_args}
         key = "callbacks"

notebooks/700_metrics/701b_aupimo_advanced_i.ipynb

@@ -585,10 +585,16 @@ def validate_pred_label(pred_label: torch.Tensor | None) -> torch.Tensor | None:
         if pred_label.ndim > 2:
             msg = f"Predicted label must be 1-dimensional or 2-dimensional, got shape {pred_label.shape}."
             raise ValueError(msg)
-        if pred_label.ndim == 2 and pred_label.shape[1] != 1:
-            msg = f"Predicted label with 2 dimensions must have shape [N, 1], got shape {pred_label.shape}."
-            raise ValueError(msg)
-
+        if pred_label.ndim == 2:
+            if pred_label.shape[0] == 1:
+                pred_label = pred_label.squeeze(0)
+            elif pred_label.shape[1] == 1:
+                pred_label = pred_label.squeeze(1)
+            else:
+                msg = (
+                    f"Predicted label with 2 dimensions must have shape [N, 1] or [1, N], got shape {pred_label.shape}."
+                )
+                raise ValueError(msg)
         return pred_label.to(torch.bool)
 
     @staticmethod

notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb

@@ -18,7 +18,6 @@
 
 from anomalib import LearningType, TaskType
 from anomalib.callbacks.checkpoint import ModelCheckpoint
-from anomalib.callbacks.metrics import _MetricsCallback
 from anomalib.callbacks.timer import TimerCallback
 from anomalib.data import AnomalibDataModule, AnomalibDataset, PredictDataset
 from anomalib.deploy import CompressionType, ExportType
@@ -116,8 +115,6 @@ def __init__(
         self,
         callbacks: list[Callback] | None = None,
         task: TaskType | str = TaskType.SEGMENTATION,
-        image_metrics: list[str] | str | dict[str, dict[str, Any]] | None = None,
-        pixel_metrics: list[str] | str | dict[str, dict[str, Any]] | None = None,
         logger: Logger | Iterable[Logger] | bool | None = None,
         default_root_dir: str | Path = "results",
         **kwargs,
@@ -137,12 +134,6 @@ def __init__(
         )
 
         self.task = TaskType(task)
-        self.image_metric_names = image_metrics if image_metrics else ["AUROC", "F1Max"]
-
-        # pixel metrics are only used for segmentation tasks.
-        self.pixel_metric_names = None
-        if self.task == TaskType.SEGMENTATION:
-            self.pixel_metric_names = pixel_metrics if pixel_metrics is not None else ["AUROC", "F1Max"]
 
         self._trainer: Trainer | None = None
 
@@ -375,7 +366,8 @@ def _setup_anomalib_callbacks(self, model: AnomalyModule) -> None:
             _callbacks.append(model.post_processor)
 
         # Add the metrics callback.
-        _callbacks.append(_MetricsCallback(self.task, self.image_metric_names, self.pixel_metric_names))
+        if isinstance(model.evaluator, Callback):
+            _callbacks.append(model.evaluator)
 
         # Add the image visualizer callback if it is passed by the user.
         if not any(isinstance(callback, ImageVisualizer) for callback in self._cache.args["callbacks"]):