Running SAM backbone on frontend (#6019)

<!-- Raise an issue to propose your change
(https://github.com/opencv/cvat/issues).
It helps to avoid duplication of efforts from multiple independent
contributors.
Discuss your ideas with maintainers to be sure that changes will be
approved and merged.
Read the [Contribution
guide](https://opencv.github.io/cvat/docs/contributing/). -->

<!-- Provide a general summary of your changes in the Title above -->

### Motivation and context
Resolved #5984 
Resolved #6049
Resolved #6041

- Compatible only with ``sam_vit_h_4b8939.pth`` weights. Need to
re-export ONNX mask decoder with some custom model changes (see below)
to support other weights (or just download them using links below)
- Need to redeploy the serverless function because its interface has
been changed.

Decoders for other weights:
sam_vit_l_0b3195.pth:
[Download](https://drive.google.com/file/d/1Nb5CJKQm_6s1n3xLSZYso6VNgljjfR-6/view?usp=sharing)
sam_vit_b_01ec64.pth:
[Download](https://drive.google.com/file/d/17cZAXBPaOABS170c9bcj9PdQsMziiBHw/view?usp=sharing)

Changes done in ONNX part:
```
git diff scripts/export_onnx_model.py
diff --git a/scripts/export_onnx_model.py b/scripts/export_onnx_model.py
index 8441258..18d5be7 100644
--- a/scripts/export_onnx_model.py
+++ b/scripts/export_onnx_model.py
@@ -138,7 +138,7 @@ def run_export(

     _ = onnx_model(**dummy_inputs)

-    output_names = ["masks", "iou_predictions", "low_res_masks"]
+    output_names = ["masks", "iou_predictions", "low_res_masks", "xtl", "ytl", "xbr", "ybr"]

     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=torch.jit.TracerWarning)
bsekachev@DESKTOP-OTBLK26:~/sam$ git diff segment_anything/utils/onnx.py
diff --git a/segment_anything/utils/onnx.py b/segment_anything/utils/onnx.py
index 3196bdf..85729c1 100644
--- a/segment_anything/utils/onnx.py
+++ b/segment_anything/utils/onnx.py
@@ -87,7 +87,15 @@ class SamOnnxModel(nn.Module):
         orig_im_size = orig_im_size.to(torch.int64)
         h, w = orig_im_size[0], orig_im_size[1]
         masks = F.interpolate(masks, size=(h, w), mode="bilinear", align_corners=False)
-        return masks
+        masks = torch.gt(masks, 0).to(torch.uint8)
+        nonzero = torch.nonzero(masks)
+        xindices = nonzero[:, 3:4]
+        yindices = nonzero[:, 2:3]
+        ytl = torch.min(yindices).to(torch.int64)
+        ybr = torch.max(yindices).to(torch.int64)
+        xtl = torch.min(xindices).to(torch.int64)
+        xbr = torch.max(xindices).to(torch.int64)
+        return masks[:, :, ytl:ybr + 1, xtl:xbr + 1], xtl, ytl, xbr, ybr

     def select_masks(
         self, masks: torch.Tensor, iou_preds: torch.Tensor, num_points: int
@@ -132,7 +140,7 @@ class SamOnnxModel(nn.Module):
         if self.return_single_mask:
             masks, scores = self.select_masks(masks, scores, point_coords.shape[1])

-        upscaled_masks = self.mask_postprocessing(masks, orig_im_size)
+        upscaled_masks, xtl, ytl, xbr, ybr = self.mask_postprocessing(masks, orig_im_size)

         if self.return_extra_metrics:
             stability_scores = calculate_stability_score(
@@ -141,4 +149,4 @@ class SamOnnxModel(nn.Module):
             areas = (upscaled_masks > self.model.mask_threshold).sum(-1).sum(-1)
             return upscaled_masks, scores, stability_scores, areas, masks

-        return upscaled_masks, scores, masks
+        return upscaled_masks, scores, masks, xtl, ytl, xbr, ybr
```

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->

### Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply.
If an item isn't applicable for some reason, then ~~explicitly
strikethrough~~ the whole
line. If you don't do that, GitHub will show incorrect progress for the
pull request.
If you're unsure about any of these, don't hesitate to ask. We're here
to help! -->
- [x] I submit my changes into the `develop` branch
- [x] I have added a description of my changes into the
[CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md)
file
- [ ] I have updated the documentation accordingly
- [ ] I have added tests to cover my changes
- [x] I have linked related issues (see [GitHub docs](

https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))
- [x] I have increased versions of npm packages if it is necessary

([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning),

[cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning),

[cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning)
and

[cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning))

### License

- [x] I submit _my code changes_ under the same [MIT License](
https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the
project.
  Feel free to contact the maintainers if that's a concern.

Author: bsekachev

CHANGELOG.md

@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - TDB
 
 ### Changed
-- TDB
+- Running SAM masks decoder on frontend (<https://github.com/opencv/cvat/pull/6019>)
 
 ### Deprecated
 - TDB

cvat-canvas/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "2.16.4",
+  "version": "2.16.5",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

cvat-canvas/src/typescript/interactionHandler.ts

@@ -1,4 +1,5 @@
 // Copyright (C) 2020-2022 Intel Corporation
+// Copyright (C) 2023 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -146,13 +147,13 @@ export class InteractionHandlerImpl implements InteractionHandler {
                         _e.stopPropagation();
                         self.remove();
                         this.shapesWereUpdated = true;
-                        const shouldRaiseEvent = this.shouldRaiseEvent(_e.ctrlKey);
                         this.interactionShapes = this.interactionShapes.filter(
                             (shape: SVG.Shape): boolean => shape !== self,
                         );
                         if (this.interactionData.startWithBox && this.interactionShapes.length === 1) {
                             this.interactionShapes[0].style({ visibility: '' });
                         }
+                        const shouldRaiseEvent = this.shouldRaiseEvent(_e.ctrlKey);
                         if (shouldRaiseEvent) {
                             this.onInteraction(this.prepareResult(), true, false);
                         }
@@ -314,7 +315,7 @@ export class InteractionHandlerImpl implements InteractionHandler {
                 'pointer-events': 'none',
                 opacity: 0.5,
             }).addClass('cvat_canvas_interact_intermediate_shape');
-            image.move(this.geometry.offset, this.geometry.offset);
+            image.move(this.geometry.offset + left, this.geometry.offset + top);
             this.drawnIntermediateShape = image;
 
             imageDataToDataURL(

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "9.0.1",
+  "version": "9.1.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",
   "scripts": {

cvat-core/src/plugins.ts

@@ -1,19 +1,32 @@
 // Copyright (C) 2019-2022 Intel Corporation
+// Copyright (C) 2023 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
 import { PluginError } from './exceptions';
 
 const plugins = [];
+
+export interface APIWrapperEnterOptions {
+    preventMethodCall?: boolean;
+}
+
 export default class PluginRegistry {
     static async apiWrapper(wrappedFunc, ...args) {
-        // I have to optimize the wrapper
         const pluginList = await PluginRegistry.list();
+        const aggregatedOptions: APIWrapperEnterOptions = {
+            preventMethodCall: false,
+        };
+
         for (const plugin of pluginList) {
             const pluginDecorators = plugin.functions.filter((obj) => obj.callback === wrappedFunc)[0];
             if (pluginDecorators && pluginDecorators.enter) {
                 try {
-                    await pluginDecorators.enter.call(this, plugin, ...args);
+                    const options: APIWrapperEnterOptions | undefined = await pluginDecorators
+                        .enter.call(this, plugin, ...args);
+                    if (options?.preventMethodCall) {
+                        aggregatedOptions.preventMethodCall = true;
+                    }
                 } catch (exception) {
                     if (exception instanceof PluginError) {
                         throw exception;
@@ -24,7 +37,10 @@ export default class PluginRegistry {
             }
         }
 
-        let result = await wrappedFunc.implementation.call(this, ...args);
+        let result = null;
+        if (!aggregatedOptions.preventMethodCall) {
+            result = await wrappedFunc.implementation.call(this, ...args);
+        }
 
         for (const plugin of pluginList) {
             const pluginDecorators = plugin.functions.filter((obj) => obj.callback === wrappedFunc)[0];

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.50.9",
+  "version": "1.51.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {
@@ -22,6 +22,7 @@
   "dependencies": {
     "@ant-design/icons": "^4.6.3",
     "@types/lodash": "^4.14.172",
+    "@types/lru-cache": "^7.10.10",
     "@types/platform": "^1.3.4",
     "@types/react": "^16.14.15",
     "@types/react-color": "^3.0.5",
@@ -41,8 +42,10 @@
     "dotenv-webpack": "^8.0.1",
     "error-stack-parser": "^2.0.6",
     "lodash": "^4.17.21",
+    "lru-cache": "^9.1.1",
     "moment": "^2.29.2",
     "mousetrap": "^1.6.5",
+    "onnxruntime-web": "^1.14.0",
     "platform": "^1.3.6",
     "prop-types": "^15.7.2",
     "react": "^16.14.0",