test: standardize retry logic (#371)

Author: bcoe

automl/test/export_dataset.test.js

@@ -27,11 +27,15 @@ const DATASET_ID = 'TRL8522556519449886720';
 const EXPORT_DATASET_REGION_TAG = 'export_dataset';
 const LOCATION = 'us-central1';
 
+const {delay} = require('./util');
+
 describe('Automl Translate Dataset Tests', () => {
   const client = new AutoMlClient();
   const prefix = 'TEST_EXPORT_OUTPUT';
 
-  it('should export a datset', async () => {
+  it('should export a datset', async function() {
+    this.retries(4);
+    await delay(this.test);
     const projectId = await client.getProjectId();
     const bucketName = `${projectId}-automl-translate`;
     const export_output = execSync(

automl/test/import_dataset.test.js

@@ -26,20 +26,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
 const IMPORT_DATASET_REGION_TAG = 'import_dataset';
 const LOCATION = 'us-central1';
 const TWENTY_MINUTES_IN_SECONDS = 60 * 20;
-
-// If two suites of tests are running parallel, importing and creating
-// datasets can fail, with:
-// No other operations should be working on projects/1046198160504/*.
-const delay = async test => {
-  const retries = test.currentRetry();
-  if (retries === 0) return; // no retry on the first failure.
-  // see: https://cloud.google.com/storage/docs/exponential-backoff:
-  const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000;
-  return new Promise(done => {
-    console.info(`retrying "${test.title}" in ${ms}ms`);
-    setTimeout(done, ms);
-  });
-};
+const {delay} = require('./util');
 
 describe('Automl Import Dataset Test', () => {
   const client = new AutoMlClient();

automl/test/translate_create_model.test.js

@@ -15,7 +15,7 @@
 'use strict';
 
 const {assert} = require('chai');
-const {describe, it, after} = require('mocha');
+const {describe, it, afterEach} = require('mocha');
 const {AutoMlClient} = require('@google-cloud/automl').v1;
 
 const cp = require('child_process');
@@ -26,11 +26,16 @@ const CREATE_MODEL_REGION_TAG = 'translate_create_model';
 const LOCATION = 'us-central1';
 const DATASET_ID = 'TRL8522556519449886720';
 
+const {delay} = require('./util');
+
 describe('Automl Translate Create Model Tests', () => {
   const client = new AutoMlClient();
   let operationId;
 
-  it('should create a model', async () => {
+  it('should create a model', async function() {
+    this.retries(5);
+    await delay(this.test);
+
     const projectId = await client.getProjectId();
     const create_output = execSync(
       `node ${CREATE_MODEL_REGION_TAG}.js ${projectId} ${LOCATION} ${DATASET_ID} translation_test_create_model`
@@ -43,7 +48,7 @@ describe('Automl Translate Create Model Tests', () => {
       .split('\n')[0];
   });
 
-  after('cancel model training', async () => {
+  afterEach('cancel model training', async () => {
     await client.operationsClient.cancelOperation({name: operationId});
   });
 });

automl/test/util.js

@@ -0,0 +1,28 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ML tests frequently run into concurrency and quota issues, for which
+// retrying with a backoff is a good strategy:
+module.exports = {
+  async delay(test) {
+    const retries = test.currentRetry();
+    if (retries === 0) return; // no retry on the first failure.
+    // see: https://cloud.google.com/storage/docs/exponential-backoff:
+    const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000;
+    return new Promise(done => {
+      console.info(`retrying "${test.title}" in ${ms}ms`);
+      setTimeout(done, ms);
+    });
+  },
+};

automl/test/vision_object_detection_predict.test.js

@@ -26,19 +26,7 @@ const MODEL_ID = 'IOD1656537412546854912';
 const PREDICT_REGION_TAG = 'vision_object_detection_predict';
 const LOCATION = 'us-central1';
 
-// If two suites of tests are running parallel, importing and creating
-// datasets can fail, with:
-// "Another DEPLOY model operation is running on the model".
-const delay = async test => {
-  const retries = test.currentRetry();
-  if (retries === 0) return; // no retry on the first failure.
-  // see: https://cloud.google.com/storage/docs/exponential-backoff:
-  const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000;
-  return new Promise(done => {
-    console.info(`retrying "${test.title}" in ${ms}ms`);
-    setTimeout(done, ms);
-  });
-};
+const {delay} = require('./util');
 
 describe('Automl Vision Object Detection Predict Test', () => {
   const client = new AutoMlClient();