Add a dataset export button for tasks (cvat-ai#834)

* Add dataset export button for tasks in dashboard
* Fix downloading, shrink list of export formats
* Add strict export format check
* Add strict export format check
* Change REST api paths
* Move formats declarations to server,

Author: zhiltsov-max

cvat-core/src/annotations.js

@@ -225,6 +225,25 @@
         return result;
     }
 
+    async function exportDataset(session, format) {
+        if (!(format instanceof String || typeof format === 'string')) {
+            throw new ArgumentError(
+                'Format must be a string',
+            );
+        }
+        if (!(session instanceof Task)) {
+            throw new ArgumentError(
+                'A dataset can only be created from a task',
+            );
+        }
+
+        let result = null;
+        result = await serverProxy.tasks
+            .exportDataset(session.id, format);
+
+        return result;
+    }
+
     module.exports = {
         getAnnotations,
         putAnnotations,
@@ -238,5 +257,6 @@
         selectObject,
         uploadAnnotations,
         dumpAnnotations,
+        exportDataset,
     };
 })();

cvat-core/src/api-implementation.js

@@ -70,6 +70,11 @@
             return result.map((el) => new AnnotationFormat(el));
         };
 
+        cvat.server.datasetFormats.implementation = async () => {
+            const result = await serverProxy.server.datasetFormats();
+            return result;
+        };
+
         cvat.server.register.implementation = async (username, firstName, lastName,
             email, password1, password2) => {
             await serverProxy.server.register(username, firstName, lastName, email,

cvat-core/src/api.js

@@ -115,6 +115,20 @@ function build() {
                     .apiWrapper(cvat.server.formats);
                 return result;
             },
+            /**
+                * Method returns available dataset export formats
+                * @method exportFormats
+                * @async
+                * @memberof module:API.cvat.server
+                * @returns {module:String[]}
+                * @throws {module:API.cvat.exceptions.PluginError}
+                * @throws {module:API.cvat.exceptions.ServerError}
+            */
+            async datasetFormats() {
+                const result = await PluginRegistry
+                    .apiWrapper(cvat.server.datasetFormats);
+                return result;
+            },
             /**
                 * Method allows to register on a server
                 * @method register

cvat-core/src/server-proxy.js

@@ -101,6 +101,22 @@
                 return response.data;
             }
 
+            async function datasetFormats() {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    response = await Axios.get(`${backendAPI}/server/dataset/formats`, {
+                        proxy: config.proxy,
+                    });
+                    response = JSON.parse(response.data);
+                } catch (errorData) {
+                    throw generateError(errorData, 'Could not get export formats from the server');
+                }
+
+                return response;
+            }
+
             async function register(username, firstName, lastName, email, password1, password2) {
                 let response = null;
                 try {
@@ -234,6 +250,35 @@
                 }
             }
 
+            async function exportDataset(id, format) {
+                const { backendAPI } = config;
+                let url = `${backendAPI}/tasks/${id}/dataset?format=${format}`;
+
+                return new Promise((resolve, reject) => {
+                    async function request() {
+                        try {
+                            const response = await Axios
+                                .get(`${url}`, {
+                                    proxy: config.proxy,
+                                });
+                            if (response.status === 202) {
+                                setTimeout(request, 3000);
+                            } else {
+                                url = `${url}&action=download`;
+                                resolve(url);
+                            }
+                        } catch (errorData) {
+                            reject(generateError(
+                                errorData,
+                                `Failed to export the task ${id} as a dataset`,
+                            ));
+                        }
+                    }
+
+                    setTimeout(request);
+                });
+            }
+
             async function createTask(taskData, files, onUpdate) {
                 const { backendAPI } = config;
 
@@ -566,6 +611,7 @@
                         about,
                         share,
                         formats,
+                        datasetFormats,
                         exception,
                         login,
                         logout,
@@ -582,6 +628,7 @@
                         saveTask,
                         createTask,
                         deleteTask,
+                        exportDataset,
                     }),
                     writable: false,
                 },

cvat-core/src/session.js

@@ -100,6 +100,12 @@
                                 objectStates, reset);
                         return result;
                     },
+
+                    async exportDataset(format) {
+                        const result = await PluginRegistry
+                            .apiWrapper.call(this, prototype.annotations.exportDataset, format);
+                        return result;
+                    },
                 },
                 writable: true,
             }),
@@ -367,6 +373,19 @@
                 * @instance
                 * @async
             */
+            /**
+                * Export as a dataset.
+                * Method builds a dataset in the specified format.
+                * @method exportDataset
+                * @memberof Session.annotations
+                * @param {module:String} format - a format
+                * @returns {string} An URL to the dataset file
+                * @throws {module:API.cvat.exceptions.PluginError}
+                * @throws {module:API.cvat.exceptions.ServerError}
+                * @throws {module:API.cvat.exceptions.ArgumentError}
+                * @instance
+                * @async
+            */
 
 
             /**
@@ -1132,6 +1151,8 @@
                 statistics: Object.getPrototypeOf(this).annotations.statistics.bind(this),
                 hasUnsavedChanges: Object.getPrototypeOf(this)
                     .annotations.hasUnsavedChanges.bind(this),
+                exportDataset: Object.getPrototypeOf(this)
+                    .annotations.exportDataset.bind(this),
             };
 
             this.frames = {
@@ -1195,6 +1216,7 @@
         annotationsStatistics,
         uploadAnnotations,
         dumpAnnotations,
+        exportDataset,
     } = require('./annotations');
 
     buildDublicatedAPI(Job.prototype);
@@ -1457,4 +1479,9 @@
         const result = await dumpAnnotations(this, name, dumper);
         return result;
     };
+
+    Task.prototype.annotations.exportDataset.implementation = async function (format) {
+        const result = await exportDataset(this, format);
+        return result;
+    };
 })();

cvat/apps/dashboard/static/dashboard/js/dashboard.js

@@ -13,9 +13,10 @@
 */
 
 class TaskView {
-    constructor(task, annotationFormats) {
+    constructor(task, annotationFormats, exportFormats) {
         this.init(task);
         this._annotationFormats = annotationFormats;
+        this._exportFormats = exportFormats;
 
         this._UI = null;
     }
@@ -109,6 +110,28 @@ class TaskView {
         }
     }
 
+    async _exportDataset(button, formatName) {
+        button.disabled = true;
+        try {
+            const format = this._exportFormats.find((x) => {
+                return x.name == formatName;
+            });
+            if (!format) {
+                throw `Unknown dataset export format '${formatName}'`;
+            }
+            const url = await this._task.annotations.exportDataset(format.tag);
+            const tempElem = document.createElement('a');
+            tempElem.href = `${url}`;
+            document.body.appendChild(tempElem);
+            tempElem.click();
+            tempElem.remove();
+        } catch (error) {
+            showMessage(error.message);
+        } finally {
+            button.disabled = false;
+        }
+    }
+
     init(task) {
         this._task = task;
     }
@@ -169,6 +192,22 @@ class TaskView {
         downloadButton.appendTo(buttonsContainer);
         uploadButton.appendTo(buttonsContainer);
 
+        const exportButton = $('<select class="regular dashboardButtonUI"'
+            + 'style="text-align-last: center;"> Export as Dataset </select>');
+        $('<option selected disabled> Export as Dataset </option>').appendTo(exportButton);
+        for (const format of this._exportFormats) {
+            const item = $(`<option>${format.name}</li>`);
+            if (format.is_default) {
+                item.addClass('bold');
+            }
+            item.appendTo(exportButton);
+        }
+        exportButton.on('change', (e) => {
+            this._exportDataset(e.target, e.target.value);
+            exportButton.prop('value', 'Export as Dataset');
+        });
+        exportButton.appendTo(buttonsContainer)
+
         $('<button class="regular dashboardButtonUI"> Update Task </button>').on('click', () => {
             this._update();
         }).appendTo(buttonsContainer);
@@ -207,14 +246,15 @@ class TaskView {
 
 
 class DashboardView {
-    constructor(metaData, taskData, annotationFormats) {
+    constructor(metaData, taskData, annotationFormats, exportFormats) {
         this._dashboardList = taskData.results;
         this._maxUploadSize = metaData.max_upload_size;
         this._maxUploadCount = metaData.max_upload_count;
         this._baseURL = metaData.base_url;
         this._sharePath = metaData.share_path;
         this._params = {};
         this._annotationFormats = annotationFormats;
+        this._exportFormats = exportFormats;
 
         this._setupList();
         this._setupTaskSearch();
@@ -273,7 +313,8 @@ class DashboardView {
                 }));
 
                 for (const task of tasks) {
-                    const taskView = new TaskView(task, this._annotationFormats);
+                    const taskView = new TaskView(task,
+                        this._annotationFormats, this._exportFormats);
                     dashboardList.append(taskView.render(baseURL));
                 }
 
@@ -735,9 +776,11 @@ window.addEventListener('DOMContentLoaded', () => {
         $.get('/dashboard/meta'),
         $.get(`/api/v1/tasks${window.location.search}`),
         window.cvat.server.formats(),
-    ).then((metaData, taskData, annotationFormats) => {
+        window.cvat.server.datasetFormats(),
+    ).then((metaData, taskData, annotationFormats, exportFormats) => {
         try {
-            new DashboardView(metaData[0], taskData[0], annotationFormats);
+            new DashboardView(metaData[0], taskData[0],
+                annotationFormats, exportFormats);
         } catch (exception) {
             $('#content').empty();
             const message = `Can not build CVAT dashboard. Exception: ${exception}.`;

cvat/apps/dataset_manager/task.py

@@ -36,6 +36,9 @@ def log_exception(logger=None, exc_info=True):
 def get_export_cache_dir(db_task):
     return osp.join(db_task.get_task_dirname(), 'export_cache')
 
+EXPORT_FORMAT_DATUMARO_PROJECT = "datumaro_project"
+
+
 class TaskProject:
     @staticmethod
     def _get_datumaro_project_dir(db_task):
@@ -211,9 +214,7 @@ def save(self, save_dir=None, save_images=False):
     def export(self, dst_format, save_dir, save_images=False, server_url=None):
         if self._dataset is None:
             self._init_dataset()
-        if dst_format == DEFAULT_FORMAT:
-            self._dataset.save(save_dir=save_dir, save_images=save_images)
-        elif dst_format == DEFAULT_FORMAT_REMOTE:
+        if dst_format == EXPORT_FORMAT_DATUMARO_PROJECT:
             self._remote_export(save_dir=save_dir, server_url=server_url)
         else:
             self._dataset.export(output_format=dst_format,
@@ -291,8 +292,7 @@ def _remote_export(self, save_dir, server_url=None):
             ])
 
 
-DEFAULT_FORMAT = "datumaro_project"
-DEFAULT_FORMAT_REMOTE = "datumaro_project_remote"
+DEFAULT_FORMAT = EXPORT_FORMAT_DATUMARO_PROJECT
 DEFAULT_CACHE_TTL = timedelta(hours=10)
 CACHE_TTL = DEFAULT_CACHE_TTL
 
@@ -348,4 +348,36 @@ def clear_export_cache(task_id, file_path, file_ctime):
                 .format(file_path))
     except Exception:
         log_exception(slogger.task[task_id])
-        raise
+        raise
+
+
+EXPORT_FORMATS = [
+    {
+        'name': 'Datumaro',
+        'tag': EXPORT_FORMAT_DATUMARO_PROJECT,
+        'is_default': True,
+    },
+    {
+        'name': 'PASCAL VOC 2012',
+        'tag': 'voc',
+        'is_default': False,
+    },
+    {
+        'name': 'MS COCO',
+        'tag': 'coco',
+        'is_default': False,
+    }
+]
+
+def get_export_formats():
+    from datumaro.components import converters
+
+    available_formats = set(name for name, _ in converters.items)
+    available_formats.add(EXPORT_FORMAT_DATUMARO_PROJECT)
+
+    public_formats = []
+    for fmt in EXPORT_FORMATS:
+        if fmt['tag'] in available_formats:
+            public_formats.append(fmt)
+
+    return public_formats