WIP: Draft PR

test/e2e/framework/metrics/metrics_grabber.go

@@ -55,6 +55,21 @@ const (
 // the check.
 var MetricsGrabbingDisabledError = errors.New("metrics grabbing disabled")
 
+// ProxyTimeoutErrorType is returned if there's an issue in connecting to the node
+// via the proxy. Tests should treat this as a connectivity issue and may
+// choose to use another node.
+type ProxyTimeoutErrorType struct {
+	NodeName string
+}
+
+func (e *ProxyTimeoutErrorType) Error() string {
+	return fmt.Sprintf("Timed out when waiting for proxy to gather metrics from %v", e.NodeName)
+}
+
+func NewProxyTimeoutError(nodeName string) *ProxyTimeoutErrorType {
+	return &ProxyTimeoutErrorType{NodeName: nodeName}
+}
+
 // Collection is metrics collection of components
 type Collection struct {
 	APIServerMetrics          APIServerMetrics
@@ -228,7 +243,7 @@ func (g *Grabber) getMetricsFromNode(ctx context.Context, nodeName string, kubel
 	}()
 	select {
 	case <-time.After(proxyTimeout):
-		return "", fmt.Errorf("Timed out when waiting for proxy to gather metrics from %v", nodeName)
+		return "", NewProxyTimeoutError(nodeName)
 	case <-finished:
 		if err != nil {
 			return "", err

test/e2e/instrumentation/metrics.go

@@ -57,14 +57,26 @@ var _ = common.SIGDescribe("Metrics", func() {
 	   Description: Should attempt to grab all resource metrics from kubelet metrics/resource endpoint.
 	*/
 	ginkgo.It("should grab all metrics from kubelet /metrics/resource endpoint", func(ctx context.Context) {
-		ginkgo.By("Connecting to kubelet's /metrics/resource endpoint")
-		node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
-		if errors.Is(err, e2emetrics.MetricsGrabbingDisabledError) {
-			e2eskipper.Skipf("%v", err)
-		}
-		framework.ExpectNoError(err)
-		response, err := grabber.GrabResourceMetricsFromKubelet(ctx, node.Name)
-		framework.ExpectNoError(err)
-		gomega.Expect(response).NotTo(gomega.BeEmpty())
+		// Each attempt to grab metrics from the Kubelet has an internal proxyTimeout (2 minutes).
+		// The Gomega.Eventually ensures we retry the entire grab operation (including potential internal proxy timeouts)
+		// for up to 7 minutes, polling every 10 seconds. This allows for multiple retries if the Kubelet is temporarily
+		// unresponsive or has transient proxy issues. Also helps select another node.
+		gomega.Eventually(ctx, func() error {
+			ginkgo.By("Connecting to kubelet's /metrics/resource endpoint")
+			node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
+			if errors.Is(err, e2emetrics.MetricsGrabbingDisabledError) {
+				e2eskipper.Skipf("%v", err)
+			}
+			framework.ExpectNoError(err)
+			response, err := grabber.GrabResourceMetricsFromKubelet(ctx, node.Name)
+			var proxyErr *e2emetrics.ProxyTimeoutErrorType
+			if errors.As(err, &proxyErr) {
+				ginkgo.By(err.Error())
+				return proxyErr // This will cause a retry. For other error fail immediately.
+			}
+			framework.ExpectNoError(err)
+			gomega.Expect(response).NotTo(gomega.BeEmpty())
+			return nil
+		}, 7*time.Minute, 10*time.Second).Should(gomega.BeNil())
 	})
 })