[release tool] auth and image validation fixups [master]

go.mod

@@ -120,6 +120,8 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
+require github.com/avast/retry-go/v4 v4.6.1
+
 require (
 	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	cel.dev/expr v0.19.1 // indirect

go.sum

@@ -39,6 +39,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
+github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
 github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=

release/internal/registry/auth.go

@@ -24,17 +24,28 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/avast/retry-go/v4"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/sirupsen/logrus"
 )
 
+var defaultDockerConfigDir = filepath.Join(os.Getenv("HOME"), ".config/docker")
+
 type DockerConfig struct {
 	Auths map[string]registry.AuthConfig `json:"auths"`
 }
 
+func dockerConfigDir() string {
+	configDir := os.Getenv("DOCKER_CONFIG")
+	if configDir == "" {
+		configDir = defaultDockerConfigDir
+	}
+	return configDir
+}
+
 // readDockerConfig reads the docker config file.
 func readDockerConfig() (DockerConfig, error) {
-	dockerConfigPath := filepath.Join(os.Getenv("HOME"), ".docker", "config.json")
+	dockerConfigPath := filepath.Join(dockerConfigDir(), "config.json")
 	file, err := os.Open(dockerConfigPath)
 	if err != nil {
 		logrus.WithError(err).Error("failed to open docker config file")
@@ -89,7 +100,16 @@ func getBearerTokenWithDefaultAuth(registry Registry, scope string) (string, err
 	if err != nil {
 		return "", fmt.Errorf("failed to get auth from docker config: %w", err)
 	}
-	return getBearerTokenWithAuth(fmt.Sprintf("%s:%s", auth.Username, auth.Password), registry, scope)
+	// We use retry.DoWithData to re-attempt getting bearer tokens.
+	// We set attempts to 5, the delay between attempts to 1000ms, and we set a custom
+	// retry function. In this case, we only retry if we got a "too many requests" error
+	// from the server, but we could add in connection timeouts, etc.
+	token, err := retry.DoWithData(func() (string, error) {
+		return getBearerTokenWithAuth(fmt.Sprintf("%s:%s", auth.Username, auth.Password), registry, scope)
+	}, retry.Attempts(5), retry.Delay(1000), retry.RetryIf(func(err error) bool {
+		return err.Error() == "too many requests"
+	}))
+	return token, err
 }
 
 // getBearerTokenWithAuth retrieves a bearer token to use for the image with given authentication.
@@ -109,12 +129,23 @@ func getBearerTokenWithAuth(auth string, registry Registry, scope string) (strin
 		return "", err
 	}
 	defer res.Body.Close()
-	if res.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to get bearer token: %s", res.Status)
+
+	// If we got HTTP 200, decode the response and return the token
+	if res.StatusCode == http.StatusOK {
+		resp := map[string]interface{}{}
+		if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
+			return "", err
+		}
+		return resp["token"].(string), nil
 	}
-	resp := map[string]interface{}{}
-	if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
-		return "", err
+
+	// If we got HTTP 429, log that we got throttled and then return the error
+	if res.StatusCode == http.StatusTooManyRequests {
+		logrus.WithFields(logrus.Fields{
+			"TokenURL": tokenURL,
+			"Scope":    scope,
+		}).Error("server is throttling our requests, backing off")
+		return "", fmt.Errorf("too many requests")
 	}
-	return resp["token"].(string), nil
+	return "", fmt.Errorf("failed to get bearer token: %s", res.Status)
 }

release/pkg/manager/calico/manager.go

@@ -629,11 +629,14 @@ func (r *CalicoManager) checkHashreleaseImagesPublished() ([]registry.Component,
 	for range r.imageComponents {
 		result := <-resultsCh
 		if result.err != nil {
-			resultsErr = errors.Join(resultsErr, fmt.Errorf("error checking %s exists: %s", result.image, result.err.Error()))
+			logrus.Error(fmt.Errorf("error checking %s exists: %s", result.image, result.err.Error()))
 		} else if !result.exists {
 			missingImages = append(missingImages, r.imageComponents[result.name])
 		}
 	}
+	if len(missingImages) > 0 {
+		resultsErr = fmt.Errorf("error validating %s images", len(missingImages))
+	}
 	return missingImages, resultsErr
 }