Add retry to ccoctl gcp create functions

The ccoct gcp create functions occassionaly fail when recently created
resources have not yet replicated in the cloud. This change adds retry
functionality to increase success rate when this happens.

Author: jstuever

pkg/cmd/provisioning/gcp/create_service_accounts.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -285,9 +286,25 @@ func createServiceAccount(ctx context.Context, client gcp.Client, name string, c
 
 		// Add member <-> role bindings for the project
 		svcAcctBindingName := actuator.ServiceAccountBindingName(serviceAccount)
-		err = actuator.EnsurePolicyBindingsForProject(client, roles, svcAcctBindingName)
-		if err != nil {
-			return "", errors.Wrap(err, fmt.Sprintf("Failed to add predefined roles for IAM service account %s", serviceAccount.DisplayName))
+		// EnsurePolicyBindingsForProject can fail due to a replication delay after service account creation
+		// Try up to 24 times with a 10 second delay between each attempt, up to 4 minutes.
+		for i := 0; i < 12; i++ {
+			err = actuator.EnsurePolicyBindingsForProject(client, roles, svcAcctBindingName)
+			if err != nil {
+				if strings.Contains(err.Error(), "Service account "+serviceAccount.Email+" does not exist") {
+					// The service account just created can't be found yet due to a replication delay so we need to retry.
+					if i >= 23 {
+						log.Fatal("Timed out adding predefined roles to IAM service account, this is most likely due to a replication delay following creation of the service account, please retry")
+						break
+					} else {
+						log.Printf("Unable to add predefined roles to IAM service account, retrying...")
+						time.Sleep(10 * time.Second)
+						continue
+					}
+				}
+
+				return "", errors.Wrap(err, fmt.Sprintf("Failed to add predefined roles for IAM service account %s", serviceAccount.DisplayName))
+			}
 		}
 
 		// Add member <-> role bindings for the IAM service account

pkg/cmd/provisioning/gcp/create_workload_identity_provider.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	iamCloud "cloud.google.com/go/iam"
 	"cloud.google.com/go/storage"
@@ -133,10 +135,27 @@ func createOIDCBucket(ctx context.Context, client gcp.Client, bucketName, region
 				}
 				log.Print("Bucket ", bucketName, " created")
 
-				policy, err := client.GetBucketPolicy(ctx, bucketName)
-				if err != nil {
-					return errors.Wrap(err, fmt.Sprintf("Failed to fetch IAM policy for bucket %s", bucketName))
+				// GetBucketPolicy can fail due to a replication delay after bucket creation
+				// Try up to 24 times with a 10 second delay between each attempt, up to 4 minutes.
+				var policy *iamCloud.Policy3
+				for i := 0; i < 12; i++ {
+					policy, err = client.GetBucketPolicy(ctx, bucketName)
+					if err != nil {
+						if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
+							// The bucket just created can't be found yet due to a replication delay so we need to retry.
+							if i >= 23 {
+								log.Fatal("Timed out fetching IAM policy for bucket, this is most likely due to a replication delay following creation of the bucket, please retry")
+								break
+							} else {
+								log.Printf("Unable to fetch IAM policy for bucket, retrying...")
+								time.Sleep(10 * time.Second)
+								continue
+							}
+						}
+						return errors.Wrap(err, fmt.Sprintf("Failed to fetch IAM policy for bucket %s", bucketName))
+					}
 				}
+
 				role := "roles/storage.objectViewer"
 				policy.Bindings = append(policy.Bindings, &iampb.Binding{
 					Role:    role,