|
| 1 | +"""FUVAS: Few-shot Unsupervised Video Anomaly Segmentation via Low-Rank Factorization of Spatio-Temporal Features. |
| 2 | +
|
| 3 | +This module provides a PyTorch Lightning implementation of the FUVAS model for |
| 4 | +video anomaly detection and segmentation. The model extracts deep features from video clips |
| 5 | +using a pre-trained 3D CNN/transformer backbone and fits a PCA-based reconstruction model |
| 6 | +to detect anomalies. |
| 7 | +
|
| 8 | +Paper: https://ieeexplore.ieee.org/abstract/document/10887597 |
| 9 | +
|
| 10 | +Example: |
| 11 | + >>> from anomalib.models.video import fuvas |
| 12 | + >>> model = fuvas( |
| 13 | + ... backbone="x3d_s", |
| 14 | + ... layer="blocks.4", |
| 15 | + ... pre_trained=True |
| 16 | + ... ) |
| 17 | +
|
| 18 | +Notes: |
| 19 | + The model uses a pre-trained backbone to extract features and fits a PCA |
| 20 | + transformation during training. No gradient updates are performed on the backbone. |
| 21 | + Anomaly detection is based on feature reconstruction error. |
| 22 | +
|
| 23 | +See Also: |
| 24 | + :class:`anomalib.models.video.fuvas.torch_model.FUVASModel`: |
| 25 | + PyTorch implementation of the FUVAS model. |
| 26 | +""" |
| 27 | + |
| 28 | +# Copyright (C) 2025 Intel Corporation |
| 29 | +# SPDX-License-Identifier: Apache-2.0 |
| 30 | + |
| 31 | +import logging |
| 32 | +from typing import Any |
| 33 | + |
| 34 | +import torch |
| 35 | +from lightning.pytorch.utilities.types import STEP_OUTPUT |
| 36 | + |
| 37 | +from anomalib import LearningType |
| 38 | +from anomalib.data import Batch |
| 39 | +from anomalib.metrics import Evaluator |
| 40 | +from anomalib.models.components import AnomalibModule, MemoryBankMixin |
| 41 | +from anomalib.post_processing import PostProcessor |
| 42 | +from anomalib.pre_processing import PreProcessor |
| 43 | +from anomalib.visualization import Visualizer |
| 44 | + |
| 45 | +from .torch_model import FUVASModel |
| 46 | + |
| 47 | +logger = logging.getLogger(__name__) |
| 48 | + |
| 49 | + |
| 50 | +class Fuvas(MemoryBankMixin, AnomalibModule): |
| 51 | + """FUVAS Lightning Module. |
| 52 | +
|
| 53 | + Args: |
| 54 | + backbone (str): Name of the backbone 3D CNN/transformer network. |
| 55 | + Defaults to ``"x3d_s"``. |
| 56 | + layer (str): Name of the layer to extract features from the backbone. |
| 57 | + Defaults to ``"blocks.4"``. |
| 58 | + pre_trained (bool, optional): Whether to use a pre-trained backbone. |
| 59 | + Defaults to ``True``. |
| 60 | + spatial_pool (bool, optional): Whether to use spatial pooling on features. |
| 61 | + Defaults to ``True``. |
| 62 | + pooling_kernel_size (int, optional): Kernel size for pooling features. |
| 63 | + Defaults to ``1``. |
| 64 | + pca_level (float, optional): Ratio of variance to preserve in PCA. |
| 65 | + Must be between 0 and 1. |
| 66 | + Defaults to ``0.98``. |
| 67 | + pre_processor (PreProcessor | bool, optional): Pre-processor to use. |
| 68 | + If ``True``, uses the default pre-processor. |
| 69 | + If ``False``, no pre-processing is performed. |
| 70 | + Defaults to ``True``. |
| 71 | + post_processor (PostProcessor | bool, optional): Post-processor to use. |
| 72 | + If ``True``, uses the default post-processor. |
| 73 | + If ``False``, no post-processing is performed. |
| 74 | + Defaults to ``True``. |
| 75 | + evaluator (Evaluator | bool, optional): Evaluator to use. |
| 76 | + If ``True``, uses the default evaluator. |
| 77 | + If ``False``, no evaluation is performed. |
| 78 | + Defaults to ``True``. |
| 79 | + visualizer (Visualizer | bool, optional): Visualizer to use. |
| 80 | + If ``True``, uses the default visualizer. |
| 81 | + If ``False``, no visualization is performed. |
| 82 | + Defaults to ``True``. |
| 83 | + """ |
| 84 | + |
| 85 | + def __init__( |
| 86 | + self, |
| 87 | + backbone: str = "x3d_s", |
| 88 | + layer: str = "blocks.4", |
| 89 | + pre_trained: bool = True, |
| 90 | + spatial_pool: bool = True, |
| 91 | + pooling_kernel_size: int = 1, |
| 92 | + pca_level: float = 0.98, |
| 93 | + pre_processor: PreProcessor | bool = True, |
| 94 | + post_processor: PostProcessor | bool = True, |
| 95 | + evaluator: Evaluator | bool = True, |
| 96 | + visualizer: Visualizer | bool = True, |
| 97 | + ) -> None: |
| 98 | + super().__init__( |
| 99 | + pre_processor=pre_processor, |
| 100 | + post_processor=post_processor, |
| 101 | + evaluator=evaluator, |
| 102 | + visualizer=visualizer, |
| 103 | + ) |
| 104 | + |
| 105 | + self.model: FUVASModel = FUVASModel( |
| 106 | + backbone=backbone, |
| 107 | + pre_trained=pre_trained, |
| 108 | + layer=layer, |
| 109 | + pooling_kernel_size=pooling_kernel_size, |
| 110 | + n_comps=pca_level, |
| 111 | + spatial_pool=spatial_pool, |
| 112 | + ) |
| 113 | + self.embeddings: list[torch.Tensor] = [] |
| 114 | + |
| 115 | + @staticmethod |
| 116 | + def configure_optimizers() -> None: # pylint: disable=arguments-differ |
| 117 | + """Configure optimizers for training. |
| 118 | +
|
| 119 | + Returns: |
| 120 | + None: FUVAS doesn't require optimization. |
| 121 | + """ |
| 122 | + return |
| 123 | + |
| 124 | + def training_step(self, batch: Batch, *args, **kwargs) -> torch.Tensor: |
| 125 | + """Extract features from the input batch during training. |
| 126 | +
|
| 127 | + Args: |
| 128 | + batch (Batch): Input batch containing video clips. |
| 129 | + *args: Additional positional arguments (unused). |
| 130 | + **kwargs: Additional keyword arguments (unused). |
| 131 | +
|
| 132 | + Returns: |
| 133 | + torch.Tensor: Dummy loss tensor for compatibility. |
| 134 | + """ |
| 135 | + del args, kwargs # These variables are not used. |
| 136 | + |
| 137 | + # Ensure batch.image is a tensor |
| 138 | + if batch.image is None or not isinstance(batch.image, torch.Tensor): |
| 139 | + msg = "Expected batch.image to be a tensor, but got None or non-tensor type" |
| 140 | + raise ValueError(msg) |
| 141 | + |
| 142 | + embedding = self.model.get_features(batch.image)[0].squeeze() |
| 143 | + self.embeddings.append(embedding) |
| 144 | + |
| 145 | + # Return a dummy loss tensor |
| 146 | + return torch.tensor(0.0, requires_grad=True, device=self.device) |
| 147 | + |
| 148 | + def fit(self) -> None: |
| 149 | + """Fit the PCA transformation to the embeddings. |
| 150 | +
|
| 151 | + The method aggregates embeddings collected during training and fits |
| 152 | + the PCA transformation used for anomaly scoring. |
| 153 | + """ |
| 154 | + logger.info("Aggregating the embedding extracted from the training set.") |
| 155 | + embeddings = torch.vstack(self.embeddings) |
| 156 | + |
| 157 | + logger.info("Fitting a PCA to dataset.") |
| 158 | + self.model.fit(embeddings) |
| 159 | + |
| 160 | + def validation_step(self, batch: Batch, *args, **kwargs) -> STEP_OUTPUT: |
| 161 | + """Compute predictions for the input batch during validation. |
| 162 | +
|
| 163 | + Args: |
| 164 | + batch (Batch): Input batch containing video clips. |
| 165 | + *args: Additional positional arguments (unused). |
| 166 | + **kwargs: Additional keyword arguments (unused). |
| 167 | +
|
| 168 | + Returns: |
| 169 | + STEP_OUTPUT: Dictionary containing anomaly scores and maps. |
| 170 | + """ |
| 171 | + del args, kwargs # These variables are not used. |
| 172 | + |
| 173 | + predictions = self.model(batch.image) |
| 174 | + return batch.update(pred_score=predictions.pred_score, anomaly_map=predictions.anomaly_map) |
| 175 | + |
| 176 | + @property |
| 177 | + def trainer_arguments(self) -> dict[str, Any]: |
| 178 | + """Get FUVAS-specific trainer arguments. |
| 179 | +
|
| 180 | + Returns: |
| 181 | + dict[str, Any]: Dictionary of trainer arguments: |
| 182 | + - ``gradient_clip_val`` (int): Disable gradient clipping |
| 183 | + - ``max_epochs`` (int): Train for one epoch only |
| 184 | + - ``num_sanity_val_steps`` (int): Skip validation sanity checks |
| 185 | + """ |
| 186 | + return {"gradient_clip_val": 0, "max_epochs": 1, "num_sanity_val_steps": 0} |
| 187 | + |
| 188 | + @property |
| 189 | + def learning_type(self) -> LearningType: |
| 190 | + """Get the learning type of the model. |
| 191 | +
|
| 192 | + Returns: |
| 193 | + LearningType: The model uses one-class learning. |
| 194 | + """ |
| 195 | + return LearningType.ONE_CLASS |
0 commit comments