Add new retry wrapper fn, retry sql database instance operations that commonly 503 (hashicorp#417)

* Retry sql database instance operations that commonly 503

* use new retry wrapper fn in resource_storage_bucket.go

Author: danawillow

google/resource_sql_database_instance.go

@@ -625,20 +625,26 @@ func resourceSqlDatabaseInstanceCreate(d *schema.ResourceData, meta interface{})
 	// If a default root user was created with a wildcard ('%') hostname, delete it. Note that if the resource is a
 	// replica, then any users are inherited from the master instance and should be left alone.
 	if !sqlResourceIsReplica(d) {
-		users, err := config.clientSqlAdmin.Users.List(project, instance.Name).Do()
+		var users *sqladmin.UsersListResponse
+		err = retry(func() error {
+			users, err = config.clientSqlAdmin.Users.List(project, instance.Name).Do()
+			return err
+		})
 		if err != nil {
 			return fmt.Errorf("Error, attempting to list users associated with instance %s: %s", instance.Name, err)
 		}
 		for _, u := range users.Items {
 			if u.Name == "root" && u.Host == "%" {
-				op, err = config.clientSqlAdmin.Users.Delete(project, instance.Name, u.Host, u.Name).Do()
+				err = retry(func() error {
+					op, err = config.clientSqlAdmin.Users.Delete(project, instance.Name, u.Host, u.Name).Do()
+					if err == nil {
+						err = sqladminOperationWait(config, op, project, "Delete default root User")
+					}
+					return err
+				})
 				if err != nil {
 					return fmt.Errorf("Error, failed to delete default 'root'@'*' user, but the database was created successfully: %s", err)
 				}
-				err = sqladminOperationWait(config, op, project, "Delete default root User")
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}

google/resource_storage_bucket.go

@@ -263,15 +263,9 @@ func resourceStorageBucketCreate(d *schema.ResourceData, meta interface{}) error
 
 	var res *storage.Bucket
 
-	err = resource.Retry(1*time.Minute, func() *resource.RetryError {
+	err = retry(func() error {
 		res, err = config.clientStorage.Buckets.Insert(project, sb).Do()
-		if err == nil {
-			return nil
-		}
-		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 429 {
-			return resource.RetryableError(gerr)
-		}
-		return resource.NonRetryableError(err)
+		return err
 	})
 
 	if err != nil {

google/utils.go

@@ -7,8 +7,10 @@ import (
 	"log"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/terraform/helper/resource"
 	"github.com/hashicorp/terraform/helper/schema"
 	"github.com/hashicorp/terraform/terraform"
 	computeBeta "google.golang.org/api/compute/v0.beta"
@@ -331,3 +333,16 @@ func convertArrToMap(ifaceArr []interface{}) map[string]struct{} {
 	}
 	return sm
 }
+
+func retry(retryFunc func() error) error {
+	return resource.Retry(1*time.Minute, func() *resource.RetryError {
+		err := retryFunc()
+		if err == nil {
+			return nil
+		}
+		if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503) {
+			return resource.RetryableError(gerr)
+		}
+		return resource.NonRetryableError(err)
+	})
+}