Add notebook example for SAMBboxToInstanceMask transform

CHANGELOG.md

@@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add SAM OVMS and Triton server Docker image builders
   (<https://github.com/openvinotoolkit/datumaro/pull/1129>)
 - Add SAMBboxToInstanceMask transform
-  (<https://github.com/openvinotoolkit/datumaro/pull/1133>)
+  (<https://github.com/openvinotoolkit/datumaro/pull/1133>, <https://github.com/openvinotoolkit/datumaro/pull/1134>)
 
 ### Enhancements
 - Remove xfail marks from the convert integration tests

docker/segment-anything/Dockerfile.exporter

@@ -39,5 +39,7 @@ FROM onnx-exporter-${model_type} AS onnx-exporter-final
 ARG model_type
 
 ADD scripts/* scripts/
-RUN python scripts/export_onnx_encoder.py --checkpoint ckpt.pth --model-type ${model_type} --output /ws/output/encoder.onnx
-RUN python scripts/export_onnx_decoder.py --checkpoint ckpt.pth --model-type ${model_type} --return-single-mask --output /ws/output/decoder.onnx
+RUN mkdir -p /ws/output/encoder
+RUN python scripts/export_onnx_encoder.py --checkpoint ckpt.pth --model-type ${model_type} --output /ws/output/encoder/model.onnx
+RUN mkdir -p /ws/output/decoder
+RUN python scripts/export_onnx_decoder.py --checkpoint ckpt.pth --model-type ${model_type} --return-single-mask --output /ws/output/decoder/model.onnx

docker/segment-anything/Dockerfile.ovms

@@ -5,8 +5,8 @@ USER root
 RUN mkdir -p /models/sam_encoder/1
 RUN mkdir -p /models/sam_decoder/1
 
-COPY --from=onnx-exporter /ws/output/encoder.onnx /models/sam_encoder/1/model.onnx
-COPY --from=onnx-exporter /ws/output/decoder.onnx /models/sam_decoder/1/model.onnx
+COPY --from=onnx-exporter /ws/output/encoder/* /models/sam_encoder/1/
+COPY --from=onnx-exporter /ws/output/decoder/* /models/sam_decoder/1/
 
 ADD ovms/model_config_list.json /models/model_config_list.json
 

docker/segment-anything/Dockerfile.triton

@@ -6,7 +6,7 @@ ADD triton/sam_decoder /models/sam_decoder
 RUN mkdir -p /models/sam_encoder/1
 RUN mkdir -p /models/sam_decoder/1
 
-COPY --from=onnx-exporter /ws/output/encoder.onnx /models/sam_encoder/1/model.onnx
-COPY --from=onnx-exporter /ws/output/decoder.onnx /models/sam_decoder/1/model.onnx
+COPY --from=onnx-exporter /ws/output/encoder/* /models/sam_encoder/1/
+COPY --from=onnx-exporter /ws/output/decoder/* /models/sam_decoder/1/
 
 ENTRYPOINT [ "tritonserver", "--model-repository=/models" ]

docker/segment-anything/scripts/export_onnx_encoder.py

@@ -57,16 +57,15 @@ def forward(self, img: torch.Tensor):
 
     output_names = ["image_embeddings"]
 
-    with open(args.output, "wb") as f:
-        torch.onnx.export(
-            Encoder(model_type=args.model_type, checkpoint=args.checkpoint),
-            inputs,
-            f,
-            export_params=True,
-            verbose=False,
-            opset_version=args.opset,
-            do_constant_folding=True,
-            input_names=list(inputs.keys()),
-            output_names=output_names,
-            dynamic_axes={"img": [2, 3]},
-        )
+    torch.onnx.export(
+        Encoder(model_type=args.model_type, checkpoint=args.checkpoint),
+        inputs,
+        args.output,
+        export_params=True,
+        verbose=False,
+        opset_version=args.opset,
+        do_constant_folding=True,
+        input_names=list(inputs.keys()),
+        output_names=output_names,
+        dynamic_axes={"img": [2, 3]},
+    )

docs/source/docs/jupyter_notebook_examples/transform.rst

@@ -14,6 +14,7 @@ Please refer `here <https://github.com/openvinotoolkit/datumaro/blob/develop/dat
 
    notebooks/05_transform
    notebooks/06_tiling
+   notebooks/18_bbox_to_instance_mask_using_sam
 
 .. grid:: 1 2 2 2
    :gutter: 2
@@ -34,8 +35,17 @@ Please refer `here <https://github.com/openvinotoolkit/datumaro/blob/develop/dat
 
       This transform is known to be useful for detecting small objects in high-resolution input images [1]_.
 
+   .. grid-item-card::
+
+      .. button-ref:: notebooks/18_bbox_to_instance_mask_using_sam
+         :color: primary
+         :outline:
+         :expand:
+
+      This transform uses Segment Anything Model [2]_ to transform bounding box annotations to instance mask annotations.
 
 References
 ^^^^^^^^^^
 
 .. [1] F., Ozge Unel, Burak O. Ozkalayci, and Cevahir Cigla. "The power of tiling for small object detection." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops. 2019.
+.. [2] Kirillov, Alexander, et al. "Segment anything." arXiv preprint arXiv:2304.02643 (2023).

src/datumaro/plugins/inference_server_plugin/__init__.py

@@ -2,8 +2,8 @@
 #
 # SPDX-License-Identifier: MIT
 
-from datumaro.plugins.inference_server_plugin.base import InferenceServerType
+from datumaro.plugins.inference_server_plugin.base import InferenceServerType, ProtocolType
 from datumaro.plugins.inference_server_plugin.ovms import OVMSLauncher
 from datumaro.plugins.inference_server_plugin.triton import TritonLauncher
 
-__all__ = ["InferenceServerType", "OVMSLauncher", "TritonLauncher"]
+__all__ = ["InferenceServerType", "OVMSLauncher", "TritonLauncher", "ProtocolType"]

src/datumaro/plugins/sam_transforms/bbox_to_inst_mask.py

@@ -21,6 +21,8 @@
 from datumaro.util import take_by
 from datumaro.util.mask_tools import extract_contours
 
+__all__ = ["SAMBboxToInstanceMask"]
+
 
 class SAMBboxToInstanceMask(ModelTransform, CliPlugin):
     """Convert bounding boxes to instance mask using Segment Anything Model.