Add retries to ILM client (#7120)

## Which problem is this PR solving?
- Resolves #6838

## Description of the changes
- Added simple retry logic with exponential backoff to the Exists method
in ILMClient to improve resilience. This helps prevent random test
failures and mitigates issues caused by transient network errors.

## How was this change tested?
- All existing tests for ILMClient passed.
- An additional unit test was added to verify that if the HTTP request
fails a few times, the client correctly retries and eventually completes
the Exists operation successfully once the service becomes healthy.

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `npm run lint` and `npm run test`

---------

Signed-off-by: Ilia Yavorov Petrov <[email protected]>
Signed-off-by: Ilia Petrov <[email protected]>

Author: iypetrov

cmd/es-rollover/main.go

@@ -54,6 +54,7 @@ func main() {
 				}
 				ilmClient := &client.ILMClient{
 					Client: c,
+					Logger: logger,
 				}
 				return &initialize.Action{
 					IndicesClient: indicesClient,

internal/storage/elasticsearch/client/ilm_client.go

@@ -4,9 +4,19 @@
 package client
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff/v5"
+	"go.uber.org/zap"
+)
+
+const (
+	maxTries       = 3
+	maxElapsedTime = 5 * time.Second
 )
 
 var _ IndexManagementLifecycleAPI = (*ILMClient)(nil)
@@ -15,22 +25,39 @@ var _ IndexManagementLifecycleAPI = (*ILMClient)(nil)
 type ILMClient struct {
 	Client
 	MasterTimeoutSeconds int
+	Logger               *zap.Logger
 }
 
 // Exists verify if a ILM policy exists
 func (i ILMClient) Exists(name string) (bool, error) {
-	_, err := i.request(elasticRequest{
-		endpoint: "_ilm/policy/" + name,
-		method:   http.MethodGet,
-	})
+	operation := func() ([]byte, error) {
+		bytes, err := i.request(elasticRequest{
+			endpoint: "_ilm/policy/" + name,
+			method:   http.MethodGet,
+		})
+		if err != nil {
+			i.Logger.Warn("Retryable error while getting ILM policy",
+				zap.String("name", name),
+				zap.Error(err),
+			)
+			return bytes, err
+		}
+		return bytes, nil
+	}
+
+	_, err := backoff.Retry(
+		context.TODO(),
+		operation,
+		backoff.WithMaxTries(maxTries),
+		backoff.WithMaxElapsedTime(maxElapsedTime),
+	)
 
 	var respError ResponseError
 	if errors.As(err, &respError) {
 		if respError.StatusCode == http.StatusNotFound {
 			return false, nil
 		}
 	}
-
 	if err != nil {
 		return false, fmt.Errorf("failed to get ILM policy: %s, %w", name, err)
 	}

internal/storage/elasticsearch/client/ilm_client_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestExists(t *testing.T) {
@@ -55,6 +56,7 @@ func TestExists(t *testing.T) {
 					Endpoint:  testServer.URL,
 					BasicAuth: "foobar",
 				},
+				Logger: zap.NewNop(),
 			}
 			result, err := c.Exists("jaeger-ilm-policy")
 			if test.errContains != "" {
@@ -64,3 +66,37 @@ func TestExists(t *testing.T) {
 		})
 	}
 }
+
+func TestExists_Retries(t *testing.T) {
+	var callCount int
+
+	testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+		callCount++
+		assert.True(t, strings.HasSuffix(req.URL.String(), "_ilm/policy/jaeger-ilm-policy"))
+		assert.Equal(t, http.MethodGet, req.Method)
+		assert.Equal(t, "Basic foobar", req.Header.Get("Authorization"))
+
+		if callCount < maxTries {
+			res.WriteHeader(http.StatusInternalServerError)
+			res.Write([]byte(`{"error": "server error"}`))
+			return
+		}
+
+		res.WriteHeader(http.StatusOK)
+	}))
+	defer testServer.Close()
+
+	c := &ILMClient{
+		Client: Client{
+			Client:    testServer.Client(),
+			Endpoint:  testServer.URL,
+			BasicAuth: "foobar",
+		},
+		Logger: zap.NewNop(),
+	}
+
+	result, err := c.Exists("jaeger-ilm-policy")
+	require.NoError(t, err)
+	assert.True(t, result)
+	assert.Equal(t, maxTries, callCount, "should retry twice before succeeding")
+}