Merged master into release-1.19 with master versions for all conflicts

.go-version

@@ -1 +1 @@
-1.22.12
+1.24.2

README.md

@@ -743,6 +743,11 @@ Default: `standard`
 
 Network Policy agent now supports two modes for Network Policy enforcement - Strict and Standard. By default, the Amazon VPC CNI plugin for Kubernetes configures network policies for pods in parallel with the pod provisioning. In the `standard` mode, until all of the policies are configured for the new pod, containers in the new pod will start with a default allow policy. A default allow policy means that all ingress and egress traffic is allowed to and from the new pods. However, in the `strict` mode, a new pod will start with a default deny policy and all Egress and Ingress connections will be blocked till Network Policies are configured. In Strict Mode, you must have a network policy defined for every pod in your cluster. Host Networking pods are exempted from this requirement.
 
+In standard mode, return traffic is always allowed for any packets that were initially sent under the default allow policy. However, once network policies are applied, the next outgoing packet will be evaluated against the active policies, and it will be allowed or denied accordingly.
+
+If you remove the Network Policy Agent container from the aws-node DaemonSet, you must also ensure that NETWORK_POLICY_ENFORCING_MODE environment variable is not set.
+Setting this value while the NP agent is absent can lead to failures during pod creation.
+
 ### VPC CNI Feature Matrix
 
 

charts/aws-vpc-cni/README.md

@@ -69,7 +69,7 @@ The following table lists the configurable parameters for this chart and their d
 | `originalMatchLabels`   | Use the original daemonset matchLabels                  | `false`                             |
 | `nameOverride`          | Override the name of the chart                          | `aws-node`                          |
 | `nodeAgent.enabled`     | If the Node Agent container should be created           | `true`                              |
-| `nodeAgent.image.tag`   | Image tag for Node Agent                                | `v1.1.6`                            |
+| `nodeAgent.image.tag`   | Image tag for Node Agent                                | `v1.1.5`                            |
 | `nodeAgent.image.domain`| ECR repository domain                                   | `amazonaws.com`                     |
 | `nodeAgent.image.region`| ECR repository region to use. Should match your cluster | `us-west-2`                         |
 | `nodeAgent.image.endpoint`   | ECR repository endpoint to use.                    | `ecr`                               |

charts/aws-vpc-cni/templates/daemonset.yaml

@@ -81,8 +81,13 @@ spec:
             timeoutSeconds: {{ .Values.readinessProbeTimeoutSeconds }}
           env:
 {{- range $key, $value := .Values.env }}
+  {{- $skipKey := and (eq $key "NETWORK_POLICY_ENFORCING_MODE") (not $.Values.nodeAgent.enabled) }}
+  {{- if not $skipKey }}
             - name: {{ $key }}
               value: {{ $value | quote }}
+  {{- else }}
+            # Skipping NETWORK_POLICY_ENFORCING_MODE because nodeAgent is disabled
+  {{- end }}
 {{- end }}
 {{- with .Values.extraEnv }}
       {{- toYaml .| nindent 12 }}
@@ -128,6 +133,9 @@ spec:
         - name: aws-eks-nodeagent
           image: {{ include "aws-vpc-cni.nodeAgentImage" . }}
           imagePullPolicy: {{ .Values.nodeAgent.image.pullPolicy }}
+          ports:
+            - containerPort: {{ .Values.nodeAgent.metricsBindAddr}}
+              name: agentmetrics
           env:
             - name: MY_NODE_NAME
               valueFrom:

charts/aws-vpc-cni/templates/eniconfig.yaml

@@ -3,7 +3,7 @@
 apiVersion: crd.k8s.amazonaws.com/v1alpha1
 kind: ENIConfig
 metadata:
-  name: {{ $key }}
+  name: "{{ $key }}"
 spec:
   {{- if $value.securityGroups }}
   securityGroups:

charts/aws-vpc-cni/templates/podmonitor.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.podMonitor.create }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "aws-vpc-cni.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- with .Values.podMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.podMonitor.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  jobLabel: {{ include "aws-vpc-cni.fullname" . }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  podMetricsEndpoints:
+  - interval: {{ .Values.podMonitor.interval }}
+    path: /metrics
+    port: metrics
+    {{- with .Values.podMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  {{- if .Values.nodeAgent.enabled }}
+  - interval: {{ .Values.podMonitor.interval }}
+    path: /metrics
+    port: agentmetrics
+    {{- with .Values.podMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  {{- end }}
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+{{- end }}

charts/aws-vpc-cni/values.yaml

@@ -27,7 +27,7 @@ init:
 nodeAgent:
   enabled: true
   image:
-    tag: v1.1.6
+    tag: v1.1.5
     domain: amazonaws.com
     region: us-west-2
     endpoint: ecr
@@ -231,3 +231,17 @@ eniConfig:
     #   id: subnet-789
     #   securityGroups:
     #   - sg-789
+
+podMonitor:
+  # Create Prometheus podMonitor 
+  create: false
+  # Annotations to add to the Prometheus podMonitor
+  annotations: {}
+  # Labels to add to the Prometheus podMonitor
+  labels: {}
+  # The interval to scrape metrics.
+  interval: 30s
+  # The timeout before a metrics scrape fails.
+  scrapeTimeout: 30s
+  # relabelings to apply to the podMonitor
+  relabelings: []

cmd/aws-k8s-agent/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"os"
+	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
@@ -24,6 +25,7 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/version"
 	"github.com/aws/amazon-vpc-cni-k8s/utils"
 	metrics "github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
+	"k8s.io/client-go/kubernetes"
 )
 
 const (
@@ -36,44 +38,108 @@ const (
 
 	// Environment variable to disable the IPAMD introspection endpoint on 61679
 	envDisableIntrospection = "DISABLE_INTROSPECTION"
+
+	restCfgTimeout = 5 * time.Second
+	pollInterval   = 5 * time.Second
+	pollTimeout    = 30 * time.Second
 )
 
 func main() {
 	os.Exit(_main())
 }
 
+// startBackgroundAPIServerCheck checks API connectivity in the background
+func startBackgroundAPIServerCheck(ipamContext *ipamd.IPAMContext) {
+	go func() {
+		log := logger.Get()
+		log.Info("Starting background API server connectivity check...")
+
+		// Create a new client for API server check
+		restCfg, err := k8sapi.GetRestConfig()
+		if err != nil {
+			log.Errorf("Failed to get REST config for background API check: %v", err)
+			return
+		}
+		restCfg.Timeout = restCfgTimeout
+		clientSet, err := kubernetes.NewForConfig(restCfg)
+		if err != nil {
+			log.Errorf("Failed to create k8s client for background API check: %v", err)
+			return
+		}
+
+		// Keep checking until connection is established
+		for {
+			version, err := clientSet.Discovery().ServerVersion()
+			if err == nil {
+				log.Infof("API server connectivity established in background! Cluster Version is: %s", version.GitVersion)
+
+				// Update IPAM context with new API server connectivity
+				ipamContext.SetAPIServerConnectivity(true)
+
+				// Exit the goroutine after successful connection
+				log.Info("Background API server check completed successfully")
+				return
+			}
+
+			log.Debugf("Still waiting for API server connectivity in background: %v", err)
+			time.Sleep(pollInterval)
+		}
+	}()
+}
+
 func _main() int {
 	// Do not add anything before initializing logger
 	log := logger.Get()
 
 	log.Infof("Starting L-IPAMD %s  ...", version.Version)
 	version.RegisterMetric()
 
+	enabledPodEni := ipamd.EnablePodENI()
+	enabledCustomNetwork := ipamd.UseCustomNetworkCfg()
+	enabledPodAnnotation := ipamd.EnablePodIPAnnotation()
+	withApiServer := false
 	// Check API Server Connectivity
-	if err := k8sapi.CheckAPIServerConnectivity(); err != nil {
-		log.Errorf("Failed to check API server connectivity: %s", err)
-		return 1
+	if enabledPodEni || enabledCustomNetwork || enabledPodAnnotation {
+		log.Info("SGP, custom networking or pod annotation feature is in use, waiting for API server connectivity to start IPAMD")
+		if err := k8sapi.CheckAPIServerConnectivity(); err != nil {
+			log.Errorf("Failed to check API server connectivity: %s", err)
+			return 1
+		} else {
+			log.Info("API server connectivity established.")
+			withApiServer = true
+		}
+	} else {
+		log.Infof("Waiting to connect API server for upto %s...", pollTimeout)
+		// Try a quick check first
+		if err := k8sapi.CheckAPIServerConnectivityWithTimeout(pollInterval, pollTimeout); err != nil {
+			log.Warn("Proceeding without API server connectivity, will run background API server connectivity check")
+			withApiServer = false
+		} else {
+			log.Info("API server connectivity established.")
+			withApiServer = true
+		}
 	}
-
 	// Create Kubernetes client for API server requests
 	k8sClient, err := k8sapi.CreateKubeClient(appName)
 	if err != nil {
 		log.Errorf("Failed to create kube client: %s", err)
-		return 1
 	}
-
 	// Create EventRecorder for use by IPAMD
-	if err := eventrecorder.Init(k8sClient); err != nil {
+	if err := eventrecorder.Init(k8sClient, withApiServer); err != nil {
 		log.Errorf("Failed to create event recorder: %s", err)
-		return 1
+		log.Warn("Skipping event recorder initialization")
 	}
-
-	ipamContext, err := ipamd.New(k8sClient)
+	ipamContext, err := ipamd.New(k8sClient, withApiServer)
 	if err != nil {
 		log.Errorf("Initialization failure: %v", err)
 		return 1
 	}
 
+	// If not connected to API server yet, start background checks
+	if !withApiServer {
+		startBackgroundAPIServerCheck(ipamContext)
+	}
+
 	// Pool manager
 	go ipamContext.StartNodeIPPoolManager()
 

cmd/cni-metrics-helper/metrics/metrics.go

@@ -17,6 +17,7 @@ package metrics
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -358,7 +359,7 @@ func producePrometheusMetrics(t metricsTarget, families map[string]*dto.MetricFa
 	if len(prometheusCNIMetrics) == 0 {
 		errorMsg := "Skipping since prometheus mapping is missing"
 		t.getLogger().Infof(errorMsg)
-		return fmt.Errorf(errorMsg)
+		return errors.New(errorMsg)
 	}
 	for key, family := range families {
 		convertMetrics := convertDef[key]

cmd/routed-eni-cni-plugin/cni.go

@@ -282,15 +282,18 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	result.Interfaces = append(result.Interfaces, dummyInterface)
 
 	// Set up a connection to the network policy agent
-	// Cx might have removed np container if they are not using network policies
-	// If we are not able to connect to np agent we do not return return error here. If NP agent grpc is not up
-	// and listening, NP agent will be in crash loop and we will catch the issue there
+	// NP container might have been removed if network policies are not being used
+	// If NETWORK_POLICY_ENFORCING_MODE is not set, we will not configure anything related to NP
+	if r.NetworkPolicyMode == "" {
+		log.Infof("NETWORK_POLICY_ENFORCING_MODE is not set")
+		return cniTypes.PrintResult(result, conf.CNIVersion)
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), npAgentConnTimeout*time.Second) // Set timeout
 	defer cancel()
 	npConn, err := grpcClient.DialContext(ctx, npAgentAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
 	if err != nil {
-		log.Infof("Failed to connect to network policy agent: %v. Network Policy agent might not be running", err)
-		return cniTypes.PrintResult(result, conf.CNIVersion)
+		log.Errorf("Failed to connect to network policy agent: %v", err)
+		return errors.New("add cmd: failed to setup network policy")
 	}
 	defer npConn.Close()
 
@@ -451,13 +454,17 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		log.Warnf("Container %s did not have a valid IP %s", args.ContainerID, r.IPv4Addr)
 	}
 
+	if r.NetworkPolicyMode == "" {
+		log.Infof("NETWORK_POLICY_ENFORCING_MODE is not set")
+		return nil
+	}
 	// Set up a connection to the network policy agent
 	ctx, cancel := context.WithTimeout(context.Background(), npAgentConnTimeout*time.Second) // Set timeout
 	defer cancel()
 	npConn, err := grpcClient.DialContext(ctx, npAgentAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
 	if err != nil {
-		log.Infof("Failed to connect to network policy agent: %v. Network Policy agent might not be running", err)
-		return nil
+		log.Errorf("Failed to connect to network policy agent: %v. Network Policy agent might not be running", err)
+		return errors.Wrap(err, "del cmd: failed to connect to network policy agent")
 	}
 	defer npConn.Close()
 	//Make a GRPC call for network policy agent