diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 1df9dfb1..0557b973 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -43,7 +43,29 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Setup Go Version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: "**/go.sum" - name: Install `govulncheck` run: go install golang.org/x/vuln/cmd/govulncheck@latest - name: Run `govulncheck` run: ~/go/bin/govulncheck ./... + static-security-analysis: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Go Version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: "**/go.sum" + - name: Install `gosec` + run: go install github.com/securego/gosec/v2/cmd/gosec@latest + - name: Run Gosec Security Scanner + run: ~/go/bin/gosec -exclude-dir test -exclude-generated -severity medium -exclude=G108,G114 ./... + diff --git a/.go-version b/.go-version new file mode 100644 index 00000000..f124bfa1 --- /dev/null +++ b/.go-version @@ -0,0 +1 @@ +1.21.9 diff --git a/Dockerfile b/Dockerfile index 286596ca..c2e1c0d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE ARG BUILD_IMAGE -ARG ARCH=amd64 +ARG ARCH # Build the controller binary FROM $BUILD_IMAGE as builder @@ -24,17 +24,18 @@ COPY webhooks/ webhooks/ # Version package for passing the ldflags ENV VERSION_PKG=github.com/aws/amazon-vpc-resource-controller-k8s/pkg/version +ENV GOARCH $ARCH # Build RUN GIT_VERSION=$(git describe --tags --always) && \ GIT_COMMIT=$(git rev-parse HEAD) && \ BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) && \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build \ + CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build \ -ldflags="-X ${VERSION_PKG}.GitVersion=${GIT_VERSION} -X ${VERSION_PKG}.GitCommit=${GIT_COMMIT} -X ${VERSION_PKG}.BuildDate=${BUILD_DATE}" -a -o controller main.go FROM $BASE_IMAGE WORKDIR / -COPY --from=public.ecr.aws/eks-distro/kubernetes/go-runner:v0.9.0-eks-1-21-4 /usr/local/bin/go-runner /usr/local/bin/go-runner +COPY --from=public.ecr.aws/eks-distro/kubernetes/go-runner:v0.15.0-eks-1-27-3 /go-runner /usr/local/bin/go-runner COPY --from=builder /workspace/controller . ENTRYPOINT ["/controller"] diff --git a/Makefile b/Makefile index 5221e283..7f5e38aa 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,14 @@ MAKEFILE_PATH = $(dir $(realpath -s $(firstword $(MAKEFILE_LIST)))) VERSION ?= $(GIT_VERSION) IMAGE ?= $(REPO):$(VERSION) BASE_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-nonroot:latest.2 -BUILD_IMAGE ?= public.ecr.aws/bitnami/golang:1.21.3 +GOLANG_VERSION ?= $(shell cat .go-version) +BUILD_IMAGE ?= public.ecr.aws/bitnami/golang:$(GOLANG_VERSION) GOARCH ?= amd64 PLATFORM ?= linux/amd64 +export GOSUMDB = sum.golang.org +export GOTOOLCHAIN = go$(GOLANG_VERSION) + help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -28,7 +32,7 @@ verify: go generate ./... go vet ./... go fmt ./... - controller-gen crd:trivialVersions=true rbac:roleName=controller-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + controller-gen crd rbac:roleName=controller-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases controller-gen object:headerFile="scripts/templates/boilerplate.go.txt" paths="./..." @git diff --quiet ||\ { echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print " - " $$0}'; \ @@ -38,7 +42,7 @@ verify: ## Run unit tests test: verify - go test ./pkg/... ./controllers/... ./webhooks/... -coverprofile cover.out + go test -race ./pkg/... ./controllers/... ./webhooks/... -coverprofile cover.out test-e2e: KUBE_CONFIG_PATH=${KUBE_CONFIG_PATH} REGION=${AWS_REGION} CLUSTER_NAME=${CLUSTER_NAME} ./scripts/test/run-integration-tests.sh @@ -50,7 +54,7 @@ toolchain: ## Install developer toolchain ./hack/toolchain.sh apply: image check-deployment-env check-env ## Deploy controller to ~/.kube/config - eksctl create iamserviceaccount vpc-resource-controller --namespace kube-system --cluster ${CLUSTER_NAME} \ + eksctl create iamserviceaccount vpc-resource-controller --namespace kube-system --cluster ${CLUSTER_NAME} --region ${AWS_REGION} \ --role-name VPCResourceControllerRole \ --attach-policy-arn=arn:aws:iam::aws:policy/AdministratorAccess \ --override-existing-serviceaccounts \ @@ -63,7 +67,7 @@ apply: image check-deployment-env check-env ## Deploy controller to ~/.kube/conf delete: ## Delete controller from ~/.kube/config kustomize build config/default | kubectl delete --ignore-not-found -f - - eksctl delete iamserviceaccount vpc-resource-controller --namespace kube-system --cluster ${CLUSTER_NAME} + eksctl delete iamserviceaccount vpc-resource-controller --namespace kube-system --cluster ${CLUSTER_NAME} --region ${AWS_REGION} kubectl patch rolebinding eks-vpc-resource-controller-rolebinding -n kube-system --patch '{"subjects":[{"kind":"ServiceAccount","name":"eks-vpc-resource-controller","namespace":"kube-system"},{"apiGroup":"rbac.authorization.k8s.io","kind":"User","name":"eks:vpc-resource-controller"}]}' kubectl create clusterrolebinding vpc-resource-controller-rolebinding --clusterrole vpc-resource-controller-role --serviceaccount kube-system:eks-vpc-resource-controller --user eks:vpc-resource-controller @@ -73,7 +77,7 @@ docker-buildx: check-env test # Build the docker image docker-build: check-env test - docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) . -t ${IMAGE} + docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg ARCH=$(GOARCH) --build-arg BUILD_IMAGE=$(BUILD_IMAGE) . -t ${IMAGE} # Push the docker image docker-push: check-env diff --git a/PROJECT b/PROJECT index 006e6030..b9b09ae0 100644 --- a/PROJECT +++ b/PROJECT @@ -1,8 +1,27 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: k8s.aws +layout: +- go.kubebuilder.io/v3 multigroup: true +projectName: amazon-vpc-resource-controller-k8s repo: github.com/aws/amazon-vpc-resource-controller-k8s resources: -- group: vpcresources +- api: + crdVersion: v1 + namespaced: true + domain: k8s.aws + group: vpcresources kind: SecurityGroupPolicy + path: github.com/aws/amazon-vpc-resource-controller-k8s/apis/v1beta1 version: v1beta1 -version: "2" +- api: + crdVersion: v1 + domain: k8s.aws + group: vpcresources + kind: CNINode + path: github.com/aws/amazon-vpc-resource-controller-k8s/apis/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 5b83952a..a1d19d30 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Controller running on EKS Control Plane for managing Branch & Trunk Network Interface for [Kubernetes Pod](https://kubernetes.io/docs/concepts/workloads/pods/) using the [Security Group for Pod](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) feature and IPv4 Address Management(IPAM) of [Windows Nodes](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html). +The controller broadcasts its version to nodes. Describing any node will provide the version information in node `Events`. The mapping between the controller's version and the cluster's platform version is also available in release notes. + ## Security Group for Pods The controller only manages the Trunk/Branch Network Interface for EKS Cluster using the Security Group for Pods feature. The Networking on the host is setup by [amazon-vpc-cni-k8s](https://github.com/aws/amazon-vpc-cni-k8s) plugin. @@ -40,6 +42,10 @@ The controller supports the following modes for IPv4 address management on Windo Please follow this [guide](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html) for enabling Windows Support on your EKS cluster. +## Configuring the controller via amazon-vpc-cni configmap + +The controller supports various configuration options for managing security groups for pods and Windows nodes which can be set via the EKS-managed configmap `amazon-vpc-cni`. For more details, refer to the security group for pods configuration options [here](docs/sgp/sgp_config_options.md) and Windows IPAM/PD related configuration options [here](docs/windows/prefix_delegation_config_options.md) + ## Troubleshooting For troubleshooting issues related to Security group for pods or Windows IPv4 address management, please visit our troubleshooting guide [here](docs/troubleshooting.md). diff --git a/apis/vpcresources/v1alpha1/cninode_types.go b/apis/vpcresources/v1alpha1/cninode_types.go index 09f8bfd0..8555f14a 100644 --- a/apis/vpcresources/v1alpha1/cninode_types.go +++ b/apis/vpcresources/v1alpha1/cninode_types.go @@ -39,8 +39,7 @@ type CNINodeSpec struct { // CNINodeStatus defines the managed VPC resources. type CNINodeStatus struct { - //TODO: add VPS resources which will be managed by this CRD and its finalizer - + //TODO: add VPC resources which will be managed by this CRD and its finalizer } // +kubebuilder:object:root=true diff --git a/config/controller/controller.yaml b/config/controller/controller.yaml index bf2fea23..951daf12 100644 --- a/config/controller/controller.yaml +++ b/config/controller/controller.yaml @@ -31,8 +31,8 @@ spec: - args: - --cluster-name=CLUSTER_NAME - --role-arn=USER_ROLE_ARN - - --enable-leader-election - - --metrics-addr=:8443 + - --leader-elect + - --metrics-bind-address=:8443 image: controller:latest name: controller resources: diff --git a/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml b/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml index 5530cc64..393a50ab 100644 --- a/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml +++ b/config/crd/bases/vpcresources.k8s.aws_cninodes.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.6.2 + controller-gen.kubebuilder.io/version: v0.9.0 creationTimestamp: null name: cninodes.vpcresources.k8s.aws spec: @@ -65,9 +64,3 @@ spec: served: true storage: true subresources: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/vpcresources.k8s.aws_securitygrouppolicies.yaml b/config/crd/bases/vpcresources.k8s.aws_securitygrouppolicies.yaml index 5e0abff0..64d4aac0 100644 --- a/config/crd/bases/vpcresources.k8s.aws_securitygrouppolicies.yaml +++ b/config/crd/bases/vpcresources.k8s.aws_securitygrouppolicies.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.6.2 + controller-gen.kubebuilder.io/version: v0.9.0 creationTimestamp: null name: securitygrouppolicies.vpcresources.k8s.aws spec: @@ -156,9 +155,3 @@ spec: served: true storage: true subresources: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/default/controller_auth_proxy_patch.yaml b/config/default/controller_auth_proxy_patch.yaml index c48639d2..14202b6a 100644 --- a/config/default/controller_auth_proxy_patch.yaml +++ b/config/default/controller_auth_proxy_patch.yaml @@ -21,5 +21,5 @@ spec: name: https - name: controller args: - - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9b9d5b3c..b292d57b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,4 +1,3 @@ - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -71,7 +70,6 @@ rules: - get - list - watch - --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 7b8701c0..78f57a87 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -1,4 +1,3 @@ - --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration @@ -26,7 +25,6 @@ webhooks: resources: - pods sideEffects: None - --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration diff --git a/controllers/core/configmap_controller.go b/controllers/core/configmap_controller.go index 5391526d..e56e3847 100644 --- a/controllers/core/configmap_controller.go +++ b/controllers/core/configmap_controller.go @@ -22,9 +22,12 @@ import ( rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" + cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -73,6 +76,24 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } } + // Check if branch ENI cooldown period is updated + curCoolDownPeriod := cooldown.GetCoolDown().GetCoolDownPeriod() + if newCoolDownPeriod, err := cooldown.GetVpcCniConfigMapCoolDownPeriodOrDefault(r.K8sAPI, r.Log); err == nil { + if curCoolDownPeriod != newCoolDownPeriod { + r.Log.Info("Branch ENI cool down period has been updated", "newCoolDownPeriod", newCoolDownPeriod, "OldCoolDownPeriod", curCoolDownPeriod) + cooldown.GetCoolDown().SetCoolDownPeriod(newCoolDownPeriod) + utils.SendBroadcastNodeEvent( + r.K8sAPI, + utils.BranchENICoolDownUpdateReason, + fmt.Sprintf("Branch ENI cool down period has been updated to %s", cooldown.GetCoolDown().GetCoolDownPeriod()), + v1.EventTypeNormal, + r.Log, + ) + } + } else { + r.Log.Error(err, "failed to retrieve branch ENI cool down period from amazon-vpc-cni configmap, will retain the current cooldown period", "cool down period", curCoolDownPeriod) + } + // Check if the Windows IPAM flag has changed newWinIPAMEnabledCond := r.Condition.IsWindowsIPAMEnabled() diff --git a/controllers/core/configmap_controller_test.go b/controllers/core/configmap_controller_test.go index a4bd66a7..34635b3c 100644 --- a/controllers/core/configmap_controller_test.go +++ b/controllers/core/configmap_controller_test.go @@ -23,9 +23,11 @@ import ( mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node" mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -112,6 +114,9 @@ func Test_Reconcile_ConfigMap_Updated(t *testing.T) { mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true) mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil) + mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes() + + cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq) assert.NoError(t, err) assert.Equal(t, res, reconcile.Result{}) @@ -125,6 +130,9 @@ func Test_Reconcile_ConfigMap_PD_Disabled_If_IPAM_Disabled(t *testing.T) { mock := NewConfigMapMock(ctrl, mockConfigMapPD) mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false) mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false) + mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes() + + cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq) assert.NoError(t, err) @@ -142,6 +150,9 @@ func Test_Reconcile_ConfigMap_NoData(t *testing.T) { mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false) mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false) + mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes() + + cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq) assert.NoError(t, err) assert.Equal(t, res, reconcile.Result{}) @@ -154,7 +165,9 @@ func Test_Reconcile_ConfigMap_Deleted(t *testing.T) { mock := NewConfigMapMock(ctrl) mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false) mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false) + mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes() + cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq) assert.NoError(t, err) assert.Equal(t, res, reconcile.Result{}) @@ -170,9 +183,23 @@ func Test_Reconcile_UpdateNode_Error(t *testing.T) { mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil) mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true) mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(errMock) + mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes() + cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq) assert.Error(t, err) assert.Equal(t, res, reconcile.Result{}) } + +func createCoolDownMockCM(cooldownTime string) *v1.ConfigMap { + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.VpcCniConfigMapName, + Namespace: config.KubeSystemNamespace, + }, + Data: map[string]string{ + config.BranchENICooldownPeriodKey: cooldownTime, + }, + } +} diff --git a/controllers/core/node_controller.go b/controllers/core/node_controller.go index 8a1f8b0e..7440939b 100644 --- a/controllers/core/node_controller.go +++ b/controllers/core/node_controller.go @@ -15,6 +15,7 @@ package controllers import ( "context" + "fmt" "net/http" "time" @@ -24,6 +25,8 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -33,9 +36,21 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/healthz" ) +var ( + leakedCNINodeResourceCount = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "orphaned_cninode_objects", + Help: "The number of CNINode objects that do not have a parent Node object (likely indicating a leak from a deleted node)", + }, + ) + + prometheusRegistered = false +) + // MaxNodeConcurrentReconciles is the number of go routines that can invoke // Reconcile in parallel. Since Node Reconciler, performs local operation // on cache only a single go routine should be sufficient. Using more than @@ -43,6 +58,7 @@ import ( // when the controller has to be restarted for various reasons. const ( MaxNodeConcurrentReconciles = 10 + NodeTerminationFinalizer = "networking.k8s.aws/resource-cleanup" ) // NodeReconciler reconciles a Node object @@ -73,27 +89,45 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } node := &corev1.Node{} - var err error logger := r.Log.WithValues("node", req.NamespacedName) - if err := r.Client.Get(ctx, req.NamespacedName, node); err != nil { - if errors.IsNotFound(err) { - r.Log.V(1).Info("the requested node couldn't be found by k8s client", "Node", req.NamespacedName) + if nodeErr := r.Client.Get(ctx, req.NamespacedName, node); nodeErr != nil { + if errors.IsNotFound(nodeErr) { + // clean up CNINode finalizer + cniNode := &v1alpha1.CNINode{} + if cninodeErr := r.Client.Get(ctx, req.NamespacedName, cniNode); cninodeErr == nil { + if yes := controllerutil.ContainsFinalizer(cniNode, NodeTerminationFinalizer); yes { + updated := cniNode.DeepCopy() + if yes = controllerutil.RemoveFinalizer(updated, NodeTerminationFinalizer); yes { + if err := r.Client.Patch(ctx, updated, client.MergeFrom(cniNode)); err != nil { + return ctrl.Result{}, err + } + r.Log.Info("removed leaked CNINode resource's finalizer", "cninode", cniNode.Name) + } + leakedCNINodeResourceCount.Inc() + } + } else if !errors.IsNotFound(cninodeErr) { + return ctrl.Result{}, fmt.Errorf("failed getting CNINode %s from cached client, %w", cniNode.Name, cninodeErr) + } + + // clean up local cached nodes _, found := r.Manager.GetNode(req.Name) if found { - err := r.Manager.DeleteNode(req.Name) - if err != nil { + cacheErr := r.Manager.DeleteNode(req.Name) + if cacheErr != nil { // The request is not retryable so not returning the error - logger.Error(err, "failed to delete node from manager") + logger.Error(cacheErr, "failed to delete node from manager") return ctrl.Result{}, nil } logger.V(1).Info("deleted the node from manager") } } - return ctrl.Result{}, client.IgnoreNotFound(err) + return ctrl.Result{}, client.IgnoreNotFound(nodeErr) } + var err error + _, found := r.Manager.GetNode(req.Name) if found { logger.V(1).Info("updating node") @@ -115,6 +149,8 @@ func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHe map[string]healthz.Checker{"health-node-controller": r.Check()}, ) + prometheusRegister() + return ctrl.NewControllerManagedBy(mgr). For(&corev1.Node{}). WithOptions(controller.Options{MaxConcurrentReconciles: MaxNodeConcurrentReconciles}). @@ -152,3 +188,11 @@ func (r *NodeReconciler) Check() healthz.Checker { return err } } + +func prometheusRegister() { + if !prometheusRegistered { + metrics.Registry.MustRegister(leakedCNINodeResourceCount) + + prometheusRegistered = true + } +} diff --git a/controllers/core/node_controller_test.go b/controllers/core/node_controller_test.go index c592dccc..311a35b6 100644 --- a/controllers/core/node_controller_test.go +++ b/controllers/core/node_controller_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition" mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node" mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager" @@ -25,11 +26,13 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -61,6 +64,7 @@ func NewNodeMock(ctrl *gomock.Controller, mockObjects ...client.Object) NodeMock scheme := runtime.NewScheme() _ = corev1.AddToScheme(scheme) + _ = v1alpha1.AddToScheme(scheme) client := fakeClient.NewClientBuilder().WithScheme(scheme).WithObjects(mockObjects...).Build() return NodeMock{ @@ -139,6 +143,43 @@ func TestNodeReconciler_Reconcile_DeleteNonExistentNode(t *testing.T) { assert.Equal(t, res, reconcile.Result{}) } +func TestNodeReconciler_Reconcile_DeleteNonExistentNodesCNINode(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mock := NewNodeMock(ctrl) + cniNode := &v1alpha1.CNINode{ + ObjectMeta: v1.ObjectMeta{ + Name: mockNodeName, + Finalizers: []string{NodeTerminationFinalizer}, + }, + } + mock.Reconciler.Client = fakeClient.NewClientBuilder().WithScheme(mock.Reconciler.Scheme).WithObjects(cniNode).Build() + + mock.Conditions.EXPECT().GetPodDataStoreSyncStatus().Return(true) + mock.Manager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, false) + + original := &v1alpha1.CNINode{} + err := mock.Reconciler.Client.Get(context.TODO(), types.NamespacedName{Name: cniNode.Name}, original) + assert.NoError(t, err) + assert.True(t, controllerutil.ContainsFinalizer(original, NodeTerminationFinalizer), "the CNINode has finalizer") + + res, err := mock.Reconciler.Reconcile(context.TODO(), reconcileRequest) + assert.NoError(t, err) + assert.Equal(t, res, reconcile.Result{}) + + node := &corev1.Node{} + updated := &v1alpha1.CNINode{} + err = mock.Reconciler.Client.Get(context.TODO(), types.NamespacedName{Name: cniNode.Name}, node) + assert.Error(t, err, "the node shouldn't existing") + assert.True(t, errors.IsNotFound(err)) + + err = mock.Reconciler.Client.Get(context.TODO(), types.NamespacedName{Name: cniNode.Name}, updated) + assert.NoError(t, err) + assert.True(t, updated.Name == mockNodeName, "the CNINode should existing and waiting for finalizer removal") + assert.False(t, controllerutil.ContainsFinalizer(updated, NodeTerminationFinalizer), "CNINode finalizer should be removed when the node is gone") +} + func TestNodeReconciler_Reconcile_DeleteNonExistentUnmanagedNode(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/docs/sgp/sgp_config_options.md b/docs/sgp/sgp_config_options.md new file mode 100644 index 00000000..dbb1e509 --- /dev/null +++ b/docs/sgp/sgp_config_options.md @@ -0,0 +1,26 @@ +# Configuration options for Security groups for pods + +Users are able to configure the controller functionality related to security group for pods by updating the `data` fields in EKS-managed configmap `amazon-vpc-cni`. + +* **branch-eni-cooldown**: Cooldown period for the branch ENIs, the period of time to wait before deleting the branch ENI for propagation of iptables rules for the deleted pod. The default cooldown period is 60s, and the minimum value for the cool period is 30s. If user updates configmap to a lower value than 30s, this will be overridden and set to 30s. + +Add `branch-eni-cooldown` field in the configmap to set the cooldown period, example: +``` +apiVersion: v1 +data: + branch-eni-cooldown: "60" +kind: ConfigMap +metadata: + name: amazon-vpc-cni + namespace: kube-system +``` + +After changing the value of `branch-eni-cooldown`, you can verify if the change has been applied by the controller. You need describe any node in your cluster and check node events in Events list. Note: this value is applied to the cluster instead of only certain nodes. + +For example, after setting the value to `90`, the change will be reflected immediately in node events: +``` +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal BranchENICoolDownPeriodUpdated 18s vpc-resource-controller Branch ENI cool down period has been updated to 1m30s +``` \ No newline at end of file diff --git a/docs/sgp/workflow.md b/docs/sgp/workflow.md index 6e41cbf8..3251ff37 100644 --- a/docs/sgp/workflow.md +++ b/docs/sgp/workflow.md @@ -3,14 +3,14 @@ This document presents high level workflow diagram for Events associated with No ## Adding a supported Node to Cluster -Security Group for Pods is suported only on Nitro Based Instnaces. +Security Group for Pods is supported only on Nitro Based Instances. ![New Nitro Based Node Create Event Diagram](../images/sgp-node-create.png) -1. User adds a new supported Node or enables ENI Trunking with existing nodes present in the cluster. -2. VPC CNI Plugin adds label `vpc.amazonaws.com/has-trunk-attached: false` if the Node has capacity to create 1 additional ENI. -3. Controller watches for Node events and acts on node with the above label by creating a Trunk ENI. -4. Controller updates the resource capacity on this node to `vpc.amazonaws.com/pod-eni: # Supported Branch ENI`. +1. User adds a new supported node or enables ENI Trunking with existing nodes present in the cluster. +2. VPC CNI Plugin updates EKS-managed CRD `CNINode ` to add feature `SecurityGroupsForPods` if the node has capacity to create 1 additional ENI. +3. Controller watches for node events and acts on node if the feature is added in `CNINode` CRD by creating a Trunk ENI. +4. Controller updates the resource capacity on this node to `vpc.amazonaws.com/pod-eni: # Supported Branch ENI`. Controller also publishes an event on the node upon successful trunk ENI creation. ## Creating a Pod using Security Groups diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 46a79439..2c40e089 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -14,6 +14,8 @@ - [Verify Pod has the resource limit](#verify-pod-has-the-resource-limit) - [Verify Pod has the pod-eni annotation](#verify-pod-has-the-pod-eni-annotation) - [Check Issues with VPC CNI](#check-issues-with-vpc-cni) + - [Connection timeouts](#connection-timeouts) + - [IP starvation issue](#ip-starvation-issue) - [Troubleshooting Prefix Delegation for Windows](#troubleshooting-prefix-delegation-for-windows) - [Verify Windows prefix delegation is enabled in the ConfigMap](#verify-windows-prefix-delegation-is-enabled-in-the-configmap) - [Check both pod events and node events for any specific error](#check-both-pod-events-and-node-events-for-any-specific-error) @@ -38,7 +40,7 @@ To get the Platform Version of your EKS cluster aws eks describe-cluster --name cluster-name --region us-west-2 | jq .cluster.platformVersion ``` -Your Platform Version should be equal to or greater than Platfrom Version [specified here](https://github.com/aws/amazon-vpc-resource-controller-k8s/releases/tag/v1.1.0). +Your Platform Version should be equal to or greater than Platform Version [specified here](https://github.com/aws/amazon-vpc-resource-controller-k8s/releases/tag/v1.1.0). **Resolution** @@ -161,6 +163,7 @@ containers: - name: ENABLE_POD_ENI value: "true" ``` +If you are using ConfigMaps that are referred from VPC CNI containers' `env`, you need have the same key/value pair setup in the referred ConfigMap. **Resolution** If the environment variable is not set, @@ -168,15 +171,29 @@ If the environment variable is not set, - Follow the guide to [enable SGP feature](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). ### Verify Trunk ENI is created +Get the EKS managed CRD CNINode +``` +kubectl get cninode +``` +The CNINode's FEATURE column should have +``` +[{"name":"SecurityGroupsForPods"}] +``` -Describe the Node, +Alternatively, you can check node for further confirming. +Describe the Node ``` -kubectl describe node node-name +kubectl describe node ``` -The following label will be set if Trunk ENI is created, +The following annotation will be added in node's `Capacity` and `Allocatable` if Trunk ENI is created successfully ``` -Labels: vpc.amazonaws.com/has-trunk-attached=true +vpc.amazonaws.com/pod-eni: 9 (could be other values depending on your instance type) +``` + +Your node should also receive an event like the following: +``` +Normal NodeTrunkInitiated 5m12s vpc-resource-controller The node has trunk interface initialized successfully ``` **Resolution** @@ -257,6 +274,31 @@ If the Pod is still stuck in `ContainerCreating` you can, - Check the CNI Logs from the collected logs. - Open an [Issue](https://github.com/aws/amazon-vpc-resource-controller-k8s/issues/new/choose) in this repository if the problem still persists. +### Connection Timeouts + +If you observe connection failures like intermittent DNS timeouts on pods using security groups, you might need to update the branch ENI cooldown period or kernel ARP cache timeout so the **values are equal**. Else this could result in re-use of IP address of a recently terminated pod by a new pod before the kernel's ARP cache is updated, which causes DNS failures or general packet drops. + +The branch ENI cooldown period is the period of time to wait before deleting the branch ENI for propagation of iptables rules for the deleted pod. This can be set on the `amazon-vpc-cni` configmap. See more details [here](../docs/sgp/sgp_config_options.md). + +To update the kernel ARP cache timeout, set the following parameters for each existing interface on the node. If the branch ENI cooldown period is 30s, set: +``` +sudo sysctl -w net.ipv4.neigh.eth0.gc_stale_time=30 +sudo sysctl -w net.ipv4.neigh.eth0.base_reachable_time_ms=15000 +``` + +Also set the default so all new interfaces created are configured with these values: +``` +sudo sysctl -w net.ipv4.neigh.default.gc_stale_time=30 +sudo sysctl -w net.ipv4.neigh.default.base_reachable_time_ms=15000 +``` + +### IP starvation issue + +If the pods are not `Running` due to IP addresses being unavailable, but you have few pods running and expect to have IP address available, tune the branch ENI cooldown period accordingly. +The branch ENI cooldown period is the period of time to wait before deleting the branch ENI for propagation of iptables rules for the deleted pod. The default value is 60s, so IP addresses are not released for atleast 60s. This can be configured via the `amazon-vpc-cni` configmap as described [here](../docs/sgp/sgp_config_options.md). Note that the minimum cooldown period is 30s. + +Be sure to also update the kernel ARP cache timeouts if you notice DNS issues as outlined in the [above section](#intermittent-dns-failures). + ## Troubleshooting Prefix Delegation for Windows Please follow the troubleshooting steps here for issues with Windows Node and Pods when using `prefix delegation` mode. diff --git a/go.mod b/go.mod index cae8ac1d..1c748137 100644 --- a/go.mod +++ b/go.mod @@ -3,81 +3,84 @@ module github.com/aws/amazon-vpc-resource-controller-k8s go 1.21 require ( - github.com/aws/amazon-vpc-cni-k8s v1.15.1 - github.com/aws/aws-sdk-go v1.45.19 - github.com/go-logr/logr v1.3.0 - github.com/go-logr/zapr v1.2.4 + github.com/aws/amazon-vpc-cni-k8s v1.17.1 + github.com/aws/aws-sdk-go v1.51.12 + github.com/go-logr/logr v1.4.1 + github.com/go-logr/zapr v1.3.0 github.com/golang/mock v1.6.0 - github.com/google/uuid v1.4.0 - github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.28.0 + github.com/google/uuid v1.6.0 + github.com/onsi/ginkgo/v2 v2.15.0 + github.com/onsi/gomega v1.31.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.17.0 - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 - github.com/prometheus/common v0.44.0 + github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_model v0.6.0 + github.com/prometheus/common v0.51.1 github.com/stretchr/testify v1.8.4 - go.uber.org/zap v1.25.0 - golang.org/x/time v0.3.0 + go.uber.org/zap v1.26.0 + golang.org/x/time v0.5.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - k8s.io/api v0.28.3 - k8s.io/apimachinery v0.28.3 - k8s.io/client-go v0.28.3 - sigs.k8s.io/controller-runtime v0.16.3 + k8s.io/api v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + sigs.k8s.io/controller-runtime v0.17.0 ) -require github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect +require ( + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect +) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/samber/lo v1.38.1 + github.com/prometheus/procfs v0.12.0 // indirect + github.com/samber/lo v1.39.0 github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.12.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.1 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.3 // indirect - k8s.io/component-base v0.28.3 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect + k8s.io/component-base v0.29.0 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index c44158f7..f15bc12f 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,30 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/amazon-vpc-cni-k8s v1.15.1 h1:zKhJ58AoFj+QaZfo768mSVFpLr3qeSVV0Qn0aeV2fhE= -github.com/aws/amazon-vpc-cni-k8s v1.15.1/go.mod h1:VjgdEc3U5d05RY5Jnovqt6pLbHmnIkzsgX6sDC6I4II= -github.com/aws/aws-sdk-go v1.45.19 h1:+4yXWhldhCVXWFOQRF99ZTJ92t4DtoHROZIbN7Ujk/U= -github.com/aws/aws-sdk-go v1.45.19/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/aws/amazon-vpc-cni-k8s v1.17.1 h1:pF+AmlGbgK8/e58LbtOsLUzDy2hqI8Ug/D8Xxx7+Sis= +github.com/aws/amazon-vpc-cni-k8s v1.17.1/go.mod h1:fNfKsEUNrAj+046SGML0UQWLcsF7hAsKRqnvwIcflvw= +github.com/aws/aws-sdk-go v1.51.12 h1:DvuhIHZXwnjaR1/Gu19gUe1EGPw4J0qSJw4Qs/5PA8g= +github.com/aws/aws-sdk-go v1.51.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -46,28 +39,28 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -78,7 +71,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -88,8 +80,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -99,27 +89,28 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= -github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= -github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.51.1 h1:eIjN50Bwglz6a/c3hAgSMcofL3nD+nFQkV6Dd4DsQCw= +github.com/prometheus/common v0.51.1/go.mod h1:lrWtQx+iDfn2mbH5GUzlH9TSHyfZpHkSiG1W7y3sF2Q= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -127,7 +118,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -137,44 +127,33 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -182,7 +161,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -190,48 +168,41 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -243,27 +214,27 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= -k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= -k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= -k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= -k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/toolchain.sh b/hack/toolchain.sh index 939fa3d8..7bc8231d 100755 --- a/hack/toolchain.sh +++ b/hack/toolchain.sh @@ -10,8 +10,8 @@ main() { } tools() { - go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20220421205612-c162794a9b12 - go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2 + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20230216140739-c98506dc3b8e + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.0 go install github.com/google/ko@latest if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then diff --git a/main.go b/main.go index fb4366b5..c034481b 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s/pod" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/version" @@ -106,8 +107,9 @@ func main() { var healthCheckTimeout int var enableWindowsPrefixDelegation bool var region string + var vpcID string - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&roleARN, "role-arn", "", "Role ARN that will be assumed to make EC2 API calls "+ @@ -115,7 +117,7 @@ func main() { "controller on your worker node.") flag.StringVar(&logLevel, "log-level", "info", "Set the controller log level - info(default), debug") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.IntVar(&leaderLeaseDurationSeconds, "leader-lease-duration-seconds", 30, @@ -140,6 +142,7 @@ func main() { flag.BoolVar(&enableWindowsPrefixDelegation, "enable-windows-prefix-delegation", false, "Enable the feature flag for Windows prefix delegation") flag.StringVar(®ion, "aws-region", "", "The aws region of the k8s cluster") + flag.StringVar(&vpcID, "vpc-id", "", "The VPC ID where EKS cluster is deployed") flag.Parse() @@ -182,6 +185,11 @@ func main() { os.Exit(1) } + if vpcID == "" { + setupLog.Error(fmt.Errorf("vpc-id is a required parameter"), "unable to start the controller") + os.Exit(1) + } + // Profiler disabled by default, to enable set the enableProfiling argument if enableProfiling { // To use the profiler - https://golang.org/pkg/net/http/pprof/ @@ -290,6 +298,9 @@ func main() { controllerConditions := condition.NewControllerConditions( ctrl.Log.WithName("controller conditions"), k8sApi, enableWindowsPrefixDelegation) + // initialize the branch ENI cool down period + cooldown.InitCoolDownPeriod(k8sApi, ctrl.Log) + // when Windows PD feature flag is OFF, do not initialize resource for prefix IPs var supportedResources []string if enableWindowsPrefixDelegation { @@ -332,6 +343,7 @@ func main() { EC2Wrapper: ec2Wrapper, ClusterName: clusterName, Log: ctrl.Log.WithName("eni cleaner"), + VPCID: vpcID, }).SetupWithManager(ctx, mgr, healthzHandler); err != nil { setupLog.Error(err, "unable to start eni cleaner") os.Exit(1) diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api/mock_ec2_apihelper.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api/mock_ec2_apihelper.go index 18f1c8f4..19f7e104 100644 --- a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api/mock_ec2_apihelper.go +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api/mock_ec2_apihelper.go @@ -196,18 +196,18 @@ func (mr *MockEC2APIHelperMockRecorder) DetachNetworkInterfaceFromInstance(arg0 } // GetBranchNetworkInterface mocks base method. -func (m *MockEC2APIHelper) GetBranchNetworkInterface(arg0 *string) ([]*ec2.NetworkInterface, error) { +func (m *MockEC2APIHelper) GetBranchNetworkInterface(arg0, arg1 *string) ([]*ec2.NetworkInterface, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBranchNetworkInterface", arg0) + ret := m.ctrl.Call(m, "GetBranchNetworkInterface", arg0, arg1) ret0, _ := ret[0].([]*ec2.NetworkInterface) ret1, _ := ret[1].(error) return ret0, ret1 } // GetBranchNetworkInterface indicates an expected call of GetBranchNetworkInterface. -func (mr *MockEC2APIHelperMockRecorder) GetBranchNetworkInterface(arg0 interface{}) *gomock.Call { +func (mr *MockEC2APIHelperMockRecorder) GetBranchNetworkInterface(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBranchNetworkInterface", reflect.TypeOf((*MockEC2APIHelper)(nil).GetBranchNetworkInterface), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBranchNetworkInterface", reflect.TypeOf((*MockEC2APIHelper)(nil).GetBranchNetworkInterface), arg0, arg1) } // GetInstanceDetails mocks base method. diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go index d287cff4..92015b49 100644 --- a/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/mock_instance.go @@ -73,6 +73,21 @@ func (mr *MockEC2InstanceMockRecorder) FreeDeviceIndex(arg0 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FreeDeviceIndex", reflect.TypeOf((*MockEC2Instance)(nil).FreeDeviceIndex), arg0) } +// GetCustomNetworkingSpec mocks base method. +func (m *MockEC2Instance) GetCustomNetworkingSpec() (string, []string) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCustomNetworkingSpec") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].([]string) + return ret0, ret1 +} + +// GetCustomNetworkingSpec indicates an expected call of GetCustomNetworkingSpec. +func (mr *MockEC2InstanceMockRecorder) GetCustomNetworkingSpec() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCustomNetworkingSpec", reflect.TypeOf((*MockEC2Instance)(nil).GetCustomNetworkingSpec)) +} + // GetHighestUnusedDeviceIndex mocks base method. func (m *MockEC2Instance) GetHighestUnusedDeviceIndex() (int64, error) { m.ctrl.T.Helper() diff --git a/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown/mock_cooldown.go b/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown/mock_cooldown.go new file mode 100644 index 00000000..ba1f1427 --- /dev/null +++ b/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown/mock_cooldown.go @@ -0,0 +1,74 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown (interfaces: CoolDown) + +// Package mock_cooldown is a generated GoMock package. +package mock_cooldown + +import ( + reflect "reflect" + time "time" + + gomock "github.com/golang/mock/gomock" +) + +// MockCoolDown is a mock of CoolDown interface. +type MockCoolDown struct { + ctrl *gomock.Controller + recorder *MockCoolDownMockRecorder +} + +// MockCoolDownMockRecorder is the mock recorder for MockCoolDown. +type MockCoolDownMockRecorder struct { + mock *MockCoolDown +} + +// NewMockCoolDown creates a new mock instance. +func NewMockCoolDown(ctrl *gomock.Controller) *MockCoolDown { + mock := &MockCoolDown{ctrl: ctrl} + mock.recorder = &MockCoolDownMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCoolDown) EXPECT() *MockCoolDownMockRecorder { + return m.recorder +} + +// GetCoolDownPeriod mocks base method. +func (m *MockCoolDown) GetCoolDownPeriod() time.Duration { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCoolDownPeriod") + ret0, _ := ret[0].(time.Duration) + return ret0 +} + +// GetCoolDownPeriod indicates an expected call of GetCoolDownPeriod. +func (mr *MockCoolDownMockRecorder) GetCoolDownPeriod() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCoolDownPeriod", reflect.TypeOf((*MockCoolDown)(nil).GetCoolDownPeriod)) +} + +// SetCoolDownPeriod mocks base method. +func (m *MockCoolDown) SetCoolDownPeriod(arg0 time.Duration) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetCoolDownPeriod", arg0) +} + +// SetCoolDownPeriod indicates an expected call of SetCoolDownPeriod. +func (mr *MockCoolDownMockRecorder) SetCoolDownPeriod(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCoolDownPeriod", reflect.TypeOf((*MockCoolDown)(nil).SetCoolDownPeriod), arg0) +} diff --git a/pkg/aws/ec2/api/eni_cleanup.go b/pkg/aws/ec2/api/eni_cleanup.go index 6f3db155..583529a8 100644 --- a/pkg/aws/ec2/api/eni_cleanup.go +++ b/pkg/aws/ec2/api/eni_cleanup.go @@ -16,6 +16,7 @@ package api import ( "context" "fmt" + "strings" "time" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" @@ -23,6 +24,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "golang.org/x/exp/slices" + ec2Errors "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/errors" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/go-logr/logr" @@ -34,6 +36,7 @@ type ENICleaner struct { EC2Wrapper EC2Wrapper ClusterName string Log logr.Logger + VPCID string availableENIs map[string]struct{} shutdown bool @@ -42,16 +45,22 @@ type ENICleaner struct { } var ( - vpcCniLeakedENICleanupCnt = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "vpc_cni_created_leaked_eni_cleanup_count", - Help: "The number of leaked ENIs created by VPC-CNI that is cleaned up by the controller", + vpccniAvailableENICnt = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "vpc_cni_created_available_eni_count", + Help: "The number of available ENIs created by VPC-CNI that controller will try to delete in each cleanup cycle", }, ) - vpcrcLeakedENICleanupCnt = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "vpc_rc_created_leaked_eni_cleanup_count", - Help: "The number of leaked ENIs created by VPC-RC that is cleaned up by the controller", + vpcrcAvailableENICnt = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "vpc_rc_created_available_eni_count", + Help: "The number of available ENIs created by VPC-RC that controller will try to delete in each cleanup cycle", + }, + ) + leakedENICnt = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "leaked_eni_count", + Help: "The number of available ENIs that failed to be deleted by the controller in each cleanup cycle", }, ) ) @@ -101,6 +110,10 @@ func (e *ENICleaner) Start(ctx context.Context) error { // interval between cycle 1 and 2 and hence can be safely deleted. And we can also conclude that Interface 1 was // created but not attached at the the time when 1st cycle ran and hence it should not be deleted. func (e *ENICleaner) cleanUpAvailableENIs() { + vpcrcAvailableCount := 0 + vpccniAvailableCount := 0 + leakedENICount := 0 + describeNetworkInterfaceIp := &ec2.DescribeNetworkInterfacesInput{ Filters: []*ec2.Filter{ { @@ -116,6 +129,10 @@ func (e *ENICleaner) cleanUpAvailableENIs() { Values: aws.StringSlice([]string{config.NetworkInterfaceOwnerTagValue, config.NetworkInterfaceOwnerVPCCNITagValue}), }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(e.VPCID)}, + }, }, } @@ -136,12 +153,13 @@ func (e *ENICleaner) cleanUpAvailableENIs() { }); tagIdx != -1 { switch *networkInterface.TagSet[tagIdx].Value { case config.NetworkInterfaceOwnerTagValue: - vpcrcLeakedENICleanupCnt.Inc() + vpcrcAvailableCount += 1 case config.NetworkInterfaceOwnerVPCCNITagValue: - vpcCniLeakedENICleanupCnt.Inc() + vpccniAvailableCount += 1 default: - // We will not hit this case as we only filter for above two tag values, adding it for any future use cases - e.Log.Info("found available ENI not created by VPC-CNI/VPC-RC") + // We should not hit this case as we only filter for relevant tag values, log error and continue if unexpected ENIs found + e.Log.Error(fmt.Errorf("found available ENI not created by VPC-CNI/VPC-RC"), "eniID", *networkInterface.NetworkInterfaceId) + continue } } @@ -151,9 +169,13 @@ func (e *ENICleaner) cleanUpAvailableENIs() { NetworkInterfaceId: networkInterface.NetworkInterfaceId, }) if err != nil { - // Log and continue, if the ENI is still present it will be cleaned up in next 2 cycles - e.Log.Error(err, "failed to delete the dangling network interface", - "id", *networkInterface.NetworkInterfaceId) + if !strings.Contains(err.Error(), ec2Errors.NotFoundInterfaceID) { // ignore InvalidNetworkInterfaceID.NotFound error + // append err and continue, we will retry deletion in the next period/reconcile + leakedENICount += 1 + + e.Log.Error(err, "failed to delete the dangling network interface", + "id", *networkInterface.NetworkInterfaceId) + } continue } e.Log.Info("deleted dangling ENI successfully", @@ -173,6 +195,10 @@ func (e *ENICleaner) cleanUpAvailableENIs() { describeNetworkInterfaceIp.NextToken = describeNetworkInterfaceOp.NextToken } + // Update leaked ENI metrics + vpcrcAvailableENICnt.Set(float64(vpcrcAvailableCount)) + vpccniAvailableENICnt.Set(float64(vpccniAvailableCount)) + leakedENICnt.Set(float64(leakedENICount)) // Set the available ENIs to the list of ENIs seen in the current cycle e.availableENIs = availableENIs } diff --git a/pkg/aws/ec2/api/eni_cleanup_test.go b/pkg/aws/ec2/api/eni_cleanup_test.go index 199f6368..e00127c0 100644 --- a/pkg/aws/ec2/api/eni_cleanup_test.go +++ b/pkg/aws/ec2/api/eni_cleanup_test.go @@ -37,6 +37,8 @@ var ( mockNetworkInterfaceId2 = "eni-000000000000001" mockNetworkInterfaceId3 = "eni-000000000000002" + mockVPCID = "vpc-0000000000000000" + mockDescribeNetworkInterfaceIp = &ec2.DescribeNetworkInterfacesInput{ Filters: []*ec2.Filter{ { @@ -52,6 +54,10 @@ var ( Values: aws.StringSlice([]string{config.NetworkInterfaceOwnerTagValue, config.NetworkInterfaceOwnerVPCCNITagValue}), }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(mockVPCID)}, + }, }, } mockDescribeInterfaceOpWith1And2 = &ec2.DescribeNetworkInterfacesOutput{ @@ -74,6 +80,7 @@ func getMockENICleaner(ctrl *gomock.Controller) (*ENICleaner, *mock_api.MockEC2W EC2Wrapper: mockEC2Wrapper, availableENIs: map[string]struct{}{}, Log: zap.New(zap.UseDevMode(true)), + VPCID: mockVPCID, clusterNameTagKey: mockClusterNameTagKey, ctx: context.Background(), }, mockEC2Wrapper diff --git a/pkg/aws/ec2/api/helper.go b/pkg/aws/ec2/api/helper.go index 3a6cb3ea..14a7864f 100644 --- a/pkg/aws/ec2/api/helper.go +++ b/pkg/aws/ec2/api/helper.go @@ -79,7 +79,7 @@ type EC2APIHelper interface { ipResourceCount *config.IPResourceCount, interfaceType *string) (*ec2.NetworkInterface, error) DeleteNetworkInterface(interfaceId *string) error GetSubnet(subnetId *string) (*ec2.Subnet, error) - GetBranchNetworkInterface(trunkID *string) ([]*ec2.NetworkInterface, error) + GetBranchNetworkInterface(trunkID, subnetID *string) ([]*ec2.NetworkInterface, error) GetInstanceNetworkInterface(instanceId *string) ([]*ec2.InstanceNetworkInterface, error) DescribeNetworkInterfaces(nwInterfaceIds []*string) ([]*ec2.NetworkInterface, error) DescribeTrunkInterfaceAssociation(trunkInterfaceId *string) ([]*ec2.TrunkInterfaceAssociation, error) @@ -562,11 +562,17 @@ func (h *ec2APIHelper) UnassignIPv4Resources(eniID string, resourceType config.R return err } -func (h *ec2APIHelper) GetBranchNetworkInterface(trunkID *string) ([]*ec2.NetworkInterface, error) { - filters := []*ec2.Filter{{ - Name: aws.String("tag:" + config.TrunkENIIDTag), - Values: []*string{trunkID}, - }} +func (h *ec2APIHelper) GetBranchNetworkInterface(trunkID, subnetID *string) ([]*ec2.NetworkInterface, error) { + filters := []*ec2.Filter{ + { + Name: aws.String("tag:" + config.TrunkENIIDTag), + Values: []*string{trunkID}, + }, + { + Name: aws.String("subnet-id"), + Values: []*string{subnetID}, + }, + } describeNetworkInterfacesInput := &ec2.DescribeNetworkInterfacesInput{Filters: filters} var nwInterfaces []*ec2.NetworkInterface diff --git a/pkg/aws/ec2/api/helper_test.go b/pkg/aws/ec2/api/helper_test.go index 971e8211..6981c99a 100644 --- a/pkg/aws/ec2/api/helper_test.go +++ b/pkg/aws/ec2/api/helper_test.go @@ -180,16 +180,28 @@ var ( tokenID = "token" describeTrunkInterfaceInput1 = &ec2.DescribeNetworkInterfacesInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("tag:" + config.TrunkENIIDTag), - Values: []*string{&trunkInterfaceId}, - }}, + Filters: []*ec2.Filter{ + { + Name: aws.String("tag:" + config.TrunkENIIDTag), + Values: []*string{&trunkInterfaceId}, + }, + { + Name: aws.String("subnet-id"), + Values: aws.StringSlice([]string{subnetId}), + }, + }, } describeTrunkInterfaceInput2 = &ec2.DescribeNetworkInterfacesInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("tag:" + config.TrunkENIIDTag), - Values: []*string{&trunkInterfaceId}, - }}, + Filters: []*ec2.Filter{ + { + Name: aws.String("tag:" + config.TrunkENIIDTag), + Values: []*string{&trunkInterfaceId}, + }, + { + Name: aws.String("subnet-id"), + Values: aws.StringSlice([]string{subnetId}), + }, + }, NextToken: &tokenID, } @@ -1187,7 +1199,7 @@ func TestEc2APIHelper_GetBranchNetworkInterface_PaginatedResults(t *testing.T) { mockWrapper.EXPECT().DescribeNetworkInterfaces(describeTrunkInterfaceInput1).Return(describeTrunkInterfaceOutput1, nil) mockWrapper.EXPECT().DescribeNetworkInterfaces(describeTrunkInterfaceInput2).Return(describeTrunkInterfaceOutput2, nil) - branchInterfaces, err := ec2ApiHelper.GetBranchNetworkInterface(&trunkInterfaceId) + branchInterfaces, err := ec2ApiHelper.GetBranchNetworkInterface(&trunkInterfaceId, &subnetId) assert.NoError(t, err) assert.ElementsMatch(t, []*ec2.NetworkInterface{&networkInterface1, &networkInterface2}, branchInterfaces) } diff --git a/pkg/aws/ec2/api/wrapper.go b/pkg/aws/ec2/api/wrapper.go index bcf4cc74..81ca97a3 100644 --- a/pkg/aws/ec2/api/wrapper.go +++ b/pkg/aws/ec2/api/wrapper.go @@ -345,8 +345,9 @@ func prometheusRegister() { ec2modifyNetworkInterfaceAttributeAPICallCnt, ec2modifyNetworkInterfaceAttributeAPIErrCnt, ec2APICallLatencies, - vpcCniLeakedENICleanupCnt, - vpcrcLeakedENICleanupCnt, + vpccniAvailableENICnt, + vpcrcAvailableENICnt, + leakedENICnt, ) prometheusRegistered = true diff --git a/pkg/aws/ec2/instance.go b/pkg/aws/ec2/instance.go index adde9058..a513e871 100644 --- a/pkg/aws/ec2/instance.go +++ b/pkg/aws/ec2/instance.go @@ -79,6 +79,7 @@ type EC2Instance interface { PrimaryNetworkInterfaceID() string CurrentInstanceSecurityGroups() []string SetNewCustomNetworkingSpec(subnetID string, securityGroup []string) + GetCustomNetworkingSpec() (subnetID string, securityGroup []string) UpdateCurrentSubnetAndCidrBlock(helper api.EC2APIHelper) error } @@ -311,3 +312,10 @@ func (i *ec2Instance) updateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHel return nil } + +func (i *ec2Instance) GetCustomNetworkingSpec() (subnetID string, securityGroup []string) { + i.lock.RLock() + defer i.lock.RUnlock() + + return i.newCustomNetworkingSubnetID, i.newCustomNetworkingSecurityGroups +} diff --git a/pkg/aws/vpc/limits.go b/pkg/aws/vpc/limits.go index 10b14202..59bee69c 100644 --- a/pkg/aws/vpc/limits.go +++ b/pkg/aws/vpc/limits.go @@ -17,7 +17,7 @@ // so we can get this information at runtime. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2023-11-02T17:34:18Z +// This file was generated at 2024-08-03T00:54:51Z // WARNING: please add @ellistarn, @bwagner5, or @jonathan-innis from aws/karpenter to reviewers // if you are updating this file since Karpenter is depending on this file to calculate max pods. @@ -1846,19 +1846,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "c6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -1911,19 +1911,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "c6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -2365,6 +2365,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "c7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "c7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -2485,6 +2500,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "c7gn.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "c7gn.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -2905,6 +2935,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "dl2q.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "f1.16xlarge": { Interface: 8, IPv4PerInterface: 50, @@ -3400,15 +3445,165 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, - "h1.16xlarge": { + "g6.12xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 114, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.16xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.2xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 38, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.48xlarge": { Interface: 15, IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 84, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "g6.xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 18, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "gr6.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "gr6.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 84, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "h1.16xlarge": { + Interface: 8, + IPv4PerInterface: 50, IsTrunkingCompatible: false, BranchInterface: 0, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 15, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, }, @@ -6221,19 +6416,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6idn.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6286,19 +6481,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6idn.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6381,19 +6576,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6446,19 +6641,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "m6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -6900,6 +7095,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "m7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "m7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -7170,6 +7380,36 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "", IsBareMetal: true, }, + "mac2-m1ultra.metal": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 6, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "mac2-m2.metal": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 6, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "mac2-m2pro.metal": { Interface: 8, IPv4PerInterface: 30, @@ -7248,8 +7488,8 @@ var Limits = map[string]*VPCLimits{ "p3.16xlarge": { Interface: 8, IPv4PerInterface: 30, - IsTrunkingCompatible: true, - BranchInterface: 114, + IsTrunkingCompatible: false, + BranchInterface: 0, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -7263,8 +7503,8 @@ var Limits = map[string]*VPCLimits{ "p3.2xlarge": { Interface: 4, IPv4PerInterface: 15, - IsTrunkingCompatible: true, - BranchInterface: 38, + IsTrunkingCompatible: false, + BranchInterface: 0, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -7278,8 +7518,8 @@ var Limits = map[string]*VPCLimits{ "p3.8xlarge": { Interface: 8, IPv4PerInterface: 30, - IsTrunkingCompatible: true, - BranchInterface: 54, + IsTrunkingCompatible: false, + BranchInterface: 0, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { @@ -9381,19 +9621,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6idn.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9446,19 +9686,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6idn.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9541,19 +9781,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6in.32xlarge": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -9606,19 +9846,19 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "r6in.metal": { - Interface: 14, + Interface: 16, IPv4PerInterface: 50, IsTrunkingCompatible: true, - BranchInterface: 108, + BranchInterface: 106, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, { - MaximumNetworkInterfaces: 7, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 1, }, }, @@ -10060,6 +10300,21 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "r7gd.metal": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, "r7gd.xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -10390,6 +10645,186 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "r8g.12xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.16xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.2xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 38, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.48xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.large": { + Interface: 3, + IPv4PerInterface: 10, + IsTrunkingCompatible: true, + BranchInterface: 9, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 3, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.medium": { + Interface: 2, + IPv4PerInterface: 4, + IsTrunkingCompatible: true, + BranchInterface: 4, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 2, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "r8g.metal-24xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "r8g.metal-48xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "r8g.xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 18, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "t1.micro": { Interface: 2, IPv4PerInterface: 2, @@ -11085,6 +11520,81 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "u7i-12tb.224xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "u7in-16tb.224xlarge": { + Interface: 16, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 106, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 1, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "u7in-24tb.224xlarge": { + Interface: 16, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 106, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 1, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "u7in-32tb.224xlarge": { + Interface: 16, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 106, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 1, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "vt1.24xlarge": { Interface: 15, IPv4PerInterface: 50, diff --git a/pkg/config/loader.go b/pkg/config/loader.go index d2621db2..90d1b61d 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -26,7 +26,7 @@ const ( WorkQueueDefaultMaxRetries = 5 // Default Configuration for Pod ENI resource type - PodENIDefaultWorker = 20 + PodENIDefaultWorker = 30 // Default Configuration for IPv4 resource type IPv4DefaultWorker = 2 @@ -92,8 +92,17 @@ func ParseWinPDTargets(log logr.Logger, vpcCniConfigMap *v1.ConfigMap) (warmIPTa } warmIPTargetStr, foundWarmIP := vpcCniConfigMap.Data[WarmIPTarget] + if !foundWarmIP { + warmIPTargetStr, foundWarmIP = vpcCniConfigMap.Data[WinWarmIPTarget] + } minIPTargetStr, foundMinIP := vpcCniConfigMap.Data[MinimumIPTarget] + if !foundMinIP { + minIPTargetStr, foundMinIP = vpcCniConfigMap.Data[WinMinimumIPTarget] + } warmPrefixTargetStr, foundWarmPrefix := vpcCniConfigMap.Data[WarmPrefixTarget] + if !foundWarmPrefix { + warmPrefixTargetStr, foundWarmPrefix = vpcCniConfigMap.Data[WinWarmPrefixTarget] + } // If no configuration is found, return 0 if !foundWarmIP && !foundMinIP && !foundWarmPrefix { diff --git a/pkg/config/type.go b/pkg/config/type.go index 6b6a3553..d7673640 100644 --- a/pkg/config/type.go +++ b/pkg/config/type.go @@ -73,13 +73,19 @@ const ( VpcCniConfigMapName = "amazon-vpc-cni" EnableWindowsIPAMKey = "enable-windows-ipam" EnableWindowsPrefixDelegationKey = "enable-windows-prefix-delegation" - WarmPrefixTarget = "warm-prefix-target" - WarmIPTarget = "warm-ip-target" - MinimumIPTarget = "minimum-ip-target" + // TODO: we will deprecate the confusing naming of Windows flags eventually + WarmPrefixTarget = "warm-prefix-target" + WarmIPTarget = "warm-ip-target" + MinimumIPTarget = "minimum-ip-target" + // these windows prefixed flags will be used for Windows support only eventully + WinWarmPrefixTarget = "windows-warm-prefix-target" + WinWarmIPTarget = "windows-warm-ip-target" + WinMinimumIPTarget = "windows-minimum-ip-target" // Since LeaderElectionNamespace and VpcCniConfigMapName may be different in the future KubeSystemNamespace = "kube-system" VpcCNIDaemonSetName = "aws-node" OldVPCControllerDeploymentName = "vpc-resource-controller" + BranchENICooldownPeriodKey = "branch-eni-cooldown" ) type ResourceType string diff --git a/pkg/k8s/pod/converter.go b/pkg/k8s/pod/converter.go index 58c7dcad..0ec10c92 100644 --- a/pkg/k8s/pod/converter.go +++ b/pkg/k8s/pod/converter.go @@ -56,6 +56,7 @@ func (c *PodConverter) ConvertList(originalList interface{}) (convertedList inte }, } for _, pod := range podList.Items { + pod := pod // Fix gosec G601, so we can use &node strippedPod := c.StripDownPod(&pod) strippedPodList.Items = append(strippedPodList.Items, *strippedPod) } diff --git a/pkg/node/manager/manager.go b/pkg/node/manager/manager.go index c9b6f9c2..2759e775 100644 --- a/pkg/node/manager/manager.go +++ b/pkg/node/manager/manager.go @@ -228,7 +228,7 @@ func (m *manager) CreateCNINodeIfNotExisting(node *v1.Node) error { } return err } else { - m.Log.V(1).Info("The CNINode is already existing", "CNINode", cniNode) + m.Log.Info("The CNINode is already existing", "cninode", cniNode.Name, "features", cniNode.Spec.Features) return nil } } diff --git a/pkg/provider/branch/cooldown/cooldown.go b/pkg/provider/branch/cooldown/cooldown.go new file mode 100644 index 00000000..631770df --- /dev/null +++ b/pkg/provider/branch/cooldown/cooldown.go @@ -0,0 +1,87 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cooldown + +import ( + "fmt" + "strconv" + "sync" + "time" + + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" + "github.com/go-logr/logr" +) + +// Global variable for CoolDownPeriod allows packages to Get and Set the coolDown period +var coolDown *cooldown + +type cooldown struct { + mu sync.RWMutex + // CoolDownPeriod is the period to wait before deleting the branch ENI for propagation of ip tables rule for deleted pod + coolDownPeriod time.Duration +} + +type CoolDown interface { + GetCoolDownPeriod() time.Duration + SetCoolDownPeriod(time.Duration) +} + +const ( + DefaultCoolDownPeriod = time.Second * 60 + MinimalCoolDownPeriod = time.Second * 30 +) + +// Initialize coolDown period by setting the value in configmap or to default +func InitCoolDownPeriod(k8sApi k8s.K8sWrapper, log logr.Logger) { + coolDown = &cooldown{} + coolDownPeriod, err := GetVpcCniConfigMapCoolDownPeriodOrDefault(k8sApi, log) + if err != nil { + log.Info("setting coolDown period to default", "cool down period", coolDownPeriod) + } + coolDown.SetCoolDownPeriod(coolDownPeriod) +} + +func GetCoolDown() CoolDown { + return coolDown +} + +func GetVpcCniConfigMapCoolDownPeriodOrDefault(k8sApi k8s.K8sWrapper, log logr.Logger) (time.Duration, error) { + vpcCniConfigMap, err := k8sApi.GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace) + if err == nil && vpcCniConfigMap.Data != nil { + if val, ok := vpcCniConfigMap.Data[config.BranchENICooldownPeriodKey]; ok { + coolDownPeriodInt, err := strconv.Atoi(val) + if err != nil { + log.Error(err, "failed to parse branch ENI coolDown period", "cool down period", val) + } else { + return time.Second * time.Duration(coolDownPeriodInt), nil + } + } + } + // If configmap not found, or configmap data not found, or error in parsing coolDown period, return default coolDown period and error + return DefaultCoolDownPeriod, fmt.Errorf("failed to get cool down period:%v", err) +} + +func (c *cooldown) GetCoolDownPeriod() time.Duration { + if c.coolDownPeriod < 30*time.Second { + return MinimalCoolDownPeriod + } + return c.coolDownPeriod +} + +func (c *cooldown) SetCoolDownPeriod(newCoolDownPeriod time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.coolDownPeriod = newCoolDownPeriod +} diff --git a/pkg/provider/branch/cooldown/cooldown_test.go b/pkg/provider/branch/cooldown/cooldown_test.go new file mode 100644 index 00000000..ef891a47 --- /dev/null +++ b/pkg/provider/branch/cooldown/cooldown_test.go @@ -0,0 +1,110 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cooldown + +import ( + "fmt" + "testing" + "time" + + mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var log = zap.New(zap.UseDevMode(true)).WithName("cooldown test") +var ( + mockConfigMap30s = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace}, + Data: map[string]string{config.BranchENICooldownPeriodKey: "30"}, + } + mockConfigMap29s = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace}, + Data: map[string]string{config.BranchENICooldownPeriodKey: "29"}, + } + mockConfigMapNilData = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace}, + Data: map[string]string{}, + } + mockConfigMapErrData = &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace}, + Data: map[string]string{config.BranchENICooldownPeriodKey: "aaa"}, + } +) + +func TestCoolDown_InitCoolDownPeriod(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + type args struct { + vpcCniConfigMap *corev1.ConfigMap + } + tests := []struct { + name string + args args + expectedCoolDown time.Duration + err error + }{ + { + name: "VpcCniConfigMap_Exists, verifies cooldown period is set to configmap value when exists", + args: args{vpcCniConfigMap: mockConfigMap30s}, + expectedCoolDown: time.Second * 30, + err: nil, + }, + { + name: "VpcCniConfigMap_NotExists, verifies cool down period is set to default when configmap does not exist", + args: args{}, + expectedCoolDown: time.Second * 60, + err: fmt.Errorf("mock error"), + }, + { + name: "VpcCniConfigMap_Exists_NilData, verifies cool period is set to default when configmap data does not exist", + args: args{vpcCniConfigMap: mockConfigMapNilData}, + expectedCoolDown: time.Second * 60, + err: nil, + }, + { + name: "VpcCniConfigMap_Exists_ErrData, verifies cool period is set to default when configmap data is incorrect", + args: args{vpcCniConfigMap: mockConfigMapErrData}, + expectedCoolDown: time.Second * 60, + err: nil, + }, + { + // critical check to safeguard the cooldown window. at this moment we don't use any time window less than 30 seconds. + name: "VpcCniConfigMap_Force30s_If_Set_Smaller, verifies cooldown period is set to 30 seconds when setting to less than 30", + args: args{vpcCniConfigMap: mockConfigMap29s}, + expectedCoolDown: time.Second * 30, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl = gomock.NewController(t) + defer ctrl.Finish() + }) + mockK8sApi := mock_k8s.NewMockK8sWrapper(ctrl) + mockK8sApi.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(test.args.vpcCniConfigMap, test.err) + InitCoolDownPeriod(mockK8sApi, log) + assert.Equal(t, test.expectedCoolDown, coolDown.GetCoolDownPeriod()) + } +} diff --git a/pkg/provider/branch/provider.go b/pkg/provider/branch/provider.go index a7e72469..4bb3cb36 100644 --- a/pkg/provider/branch/provider.go +++ b/pkg/provider/branch/provider.go @@ -29,6 +29,7 @@ import ( rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/trunk" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/worker" @@ -71,7 +72,6 @@ var ( ReasonTrunkENICreationFailed = "TrunkENICreationFailed" - reconcileRequeueRequest = ctrl.Result{RequeueAfter: time.Minute * 30, Requeue: true} deleteQueueRequeueRequest = ctrl.Result{RequeueAfter: time.Second * 30, Requeue: true} // NodeDeleteRequeueRequestDelay represents the time after which the resources belonging to a node will be cleaned @@ -139,7 +139,7 @@ func timeSinceMs(start time.Time) float64 { // cache for use in future Create/Delete Requests func (b *branchENIProvider) InitResource(instance ec2.EC2Instance) error { nodeName := instance.Name() - log := b.log.WithValues("node name", nodeName) + log := b.log.WithValues("nodeName", nodeName) trunkENI := trunk.NewTrunkENI(log, instance, b.apiWrapper.EC2API) // Initialize the Trunk ENI @@ -358,7 +358,7 @@ func (b *branchENIProvider) CreateAndAnnotateResources(podNamespace string, podN branchENIs, err := trunkENI.CreateAndAssociateBranchENIs(pod, securityGroups, resourceCount) if err != nil { if err == trunk.ErrCurrentlyAtMaxCapacity { - return ctrl.Result{RequeueAfter: config.CoolDownPeriod, Requeue: true}, nil + return ctrl.Result{RequeueAfter: cooldown.GetCoolDown().GetCoolDownPeriod(), Requeue: true}, nil } b.apiWrapper.K8sAPI.BroadcastEvent(pod, ReasonBranchAllocationFailed, fmt.Sprintf("failed to allocate branch ENI to pod: %v", err), v1.EventTypeWarning) diff --git a/pkg/provider/branch/trunk/trunk.go b/pkg/provider/branch/trunk/trunk.go index 7b656fa5..71de6991 100644 --- a/pkg/provider/branch/trunk/trunk.go +++ b/pkg/provider/branch/trunk/trunk.go @@ -16,6 +16,7 @@ package trunk import ( "encoding/json" "fmt" + "slices" "strconv" "strings" "sync" @@ -26,6 +27,8 @@ import ( ec2Errors "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/errors" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" + "github.com/samber/lo" "github.com/aws/aws-sdk-go/aws" awsEC2 "github.com/aws/aws-sdk-go/service/ec2" @@ -38,10 +41,10 @@ import ( const ( // MaxAllocatableVlanIds is the maximum number of Vlan Ids that can be allocated per trunk. MaxAllocatableVlanIds = 121 - // CoolDownPeriod is the period to wait before deleting the branch ENI for propagation of ip tables rule for deleted pod - CoolDownPeriod = time.Second * 30 // MaxDeleteRetries is the maximum number of times the ENI will be retried before being removed from the delete queue - MaxDeleteRetries = 3 + MaxDeleteRetries = 3 + SubnetLabel = "subnet" + SecurityGroupsLabel = "security_groups" ) var ( @@ -63,6 +66,13 @@ var ( }, []string{"operation"}, ) + unreconciledTrunkENICount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "unreconciled_trunk_network_interfaces", + Help: "The number of unreconciled trunk network interfaces", + }, + []string{"attribute"}, + ) branchENIOperationsSuccessCount = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "branch_eni_opeartions_success_count", @@ -174,6 +184,7 @@ func NewTrunkENI(logger logr.Logger, instance ec2.EC2Instance, helper api.EC2API func PrometheusRegister() { if !prometheusRegistered { metrics.Registry.MustRegister(trunkENIOperationsErrCount) + metrics.Registry.MustRegister(unreconciledTrunkENICount) metrics.Registry.MustRegister(branchENIOperationsSuccessCount) metrics.Registry.MustRegister(branchENIOperationsFailureCount) @@ -193,6 +204,7 @@ func (t *trunkENI) InitTrunk(instance ec2.EC2Instance, podList []v1.Pod) error { return err } + var trunk awsEC2.InstanceNetworkInterface // Get trunk network interface for _, nwInterface := range nwInterfaces { // It's possible to get an empty network interface response if the instance is being deleted. @@ -207,6 +219,7 @@ func (t *trunkENI) InitTrunk(instance ec2.EC2Instance, podList []v1.Pod) error { } else { return fmt.Errorf("failed to verify network interface status attached for %v", *nwInterface.NetworkInterfaceId) } + trunk = *nwInterface } } @@ -232,8 +245,43 @@ func (t *trunkENI) InitTrunk(instance ec2.EC2Instance, podList []v1.Pod) error { return nil } + // the node already have trunk, let's check if its SGs and Subnets match with expected + expectedSubnetID, expectedSecurityGroups := t.instance.GetCustomNetworkingSpec() + if len(expectedSecurityGroups) > 0 || expectedSubnetID != "" { + slices.Sort(expectedSecurityGroups) + trunkSGs := lo.Map(trunk.Groups, func(g *awsEC2.GroupIdentifier, _ int) string { + return lo.FromPtr(g.GroupId) + }) + slices.Sort(trunkSGs) + + mismatchedSubnets := expectedSubnetID != lo.FromPtr(trunk.SubnetId) + mismatchedSGs := !slices.Equal(expectedSecurityGroups, trunkSGs) + + extraSGsInTrunk, missingSGsInTrunk := lo.Difference(trunkSGs, expectedSecurityGroups) + t.log.Info("Observed trunk ENI config", + "instanceID", t.instance.InstanceID(), + "trunkENIID", lo.FromPtr(trunk.NetworkInterfaceId), + "configuredTrunkSGs", trunkSGs, + "configuredTrunkSubnet", lo.FromPtr(trunk.SubnetId), + "desiredTrunkSGs", expectedSecurityGroups, + "desiredTrunkSubnet", expectedSubnetID, + "mismatchedSGs", mismatchedSGs, + "mismatchedSubnets", mismatchedSubnets, + "missingSGs", missingSGsInTrunk, + "extraSGs", extraSGsInTrunk, + ) + + if mismatchedSGs { + unreconciledTrunkENICount.WithLabelValues(SecurityGroupsLabel).Inc() + } + + if mismatchedSubnets { + unreconciledTrunkENICount.WithLabelValues(SubnetLabel).Inc() + } + } + // Get the list of branch ENIs - branchInterfaces, err := t.ec2ApiHelper.GetBranchNetworkInterface(&t.trunkENIId) + branchInterfaces, err := t.ec2ApiHelper.GetBranchNetworkInterface(&t.trunkENIId, aws.String(t.instance.SubnetID())) if err != nil { return err } @@ -246,6 +294,7 @@ func (t *trunkENI) InitTrunk(instance ec2.EC2Instance, podList []v1.Pod) error { // From the list of pods on the given node, and the branch ENIs from EC2 API call rebuild the internal cache for _, pod := range podList { + pod := pod // Fix gosec G601, so we can use &node eniListFromPod := t.getBranchInterfacesUsedByPod(&pod) if len(eniListFromPod) == 0 { continue @@ -475,7 +524,7 @@ func (t *trunkENI) PushBranchENIsToCoolDownQueue(UID string) { func (t *trunkENI) DeleteCooledDownENIs() { for eni, hasENI := t.popENIFromDeleteQueue(); hasENI; eni, hasENI = t.popENIFromDeleteQueue() { if eni.deletionTimeStamp.IsZero() || - time.Now().After(eni.deletionTimeStamp.Add(CoolDownPeriod)) { + time.Now().After(eni.deletionTimeStamp.Add(cooldown.GetCoolDown().GetCoolDownPeriod())) { err := t.deleteENI(eni) if err != nil { eni.deleteRetryCount++ diff --git a/pkg/provider/branch/trunk/trunk_test.go b/pkg/provider/branch/trunk/trunk_test.go index 021dd0b0..49dcaf0d 100644 --- a/pkg/provider/branch/trunk/trunk_test.go +++ b/pkg/provider/branch/trunk/trunk_test.go @@ -21,8 +21,12 @@ import ( mock_ec2 "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2" mock_api "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2/api" + mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s" + mock_cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" "github.com/aws/aws-sdk-go/aws" awsEc2 "github.com/aws/aws-sdk-go/service/ec2" @@ -216,6 +220,7 @@ func getMockHelperInstanceAndTrunkObject(ctrl *gomock.Controller) (*trunkENI, *m EniDetails2.deleteRetryCount = 0 return &trunkENI, mockHelper, mockInstance + } func getMockTrunk() trunkENI { @@ -436,6 +441,10 @@ func TestTrunkENI_DeleteCooledDownENIs_NotCooledDown(t *testing.T) { EniDetails2.deletionTimeStamp = time.Now() trunkENI.deleteQueue = append(trunkENI.deleteQueue, EniDetails1, EniDetails2) + mockK8sAPI := mock_k8s.NewMockK8sWrapper(ctrl) + mockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil) + cooldown.InitCoolDownPeriod(mockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) + trunkENI.DeleteCooledDownENIs() assert.Equal(t, 2, len(trunkENI.deleteQueue)) } @@ -448,7 +457,7 @@ func TestTrunkENI_DeleteCooledDownENIs_NoDeletionTimeStamp(t *testing.T) { trunkENI, ec2APIHelper, _ := getMockHelperInstanceAndTrunkObject(ctrl) EniDetails1.deletionTimeStamp = time.Time{} - EniDetails2.deletionTimeStamp = time.Now().Add(-time.Second * 34) + EniDetails2.deletionTimeStamp = time.Now().Add(-(time.Second * 62)) trunkENI.usedVlanIds[VlanId1] = true trunkENI.usedVlanIds[VlanId2] = true @@ -457,6 +466,10 @@ func TestTrunkENI_DeleteCooledDownENIs_NoDeletionTimeStamp(t *testing.T) { ec2APIHelper.EXPECT().DeleteNetworkInterface(&EniDetails1.ID).Return(nil) ec2APIHelper.EXPECT().DeleteNetworkInterface(&EniDetails2.ID).Return(nil) + mockK8sAPI := mock_k8s.NewMockK8sWrapper(ctrl) + mockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil) + cooldown.InitCoolDownPeriod(mockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) + trunkENI.DeleteCooledDownENIs() assert.Equal(t, 0, len(trunkENI.deleteQueue)) } @@ -467,7 +480,7 @@ func TestTrunkENI_DeleteCooledDownENIs_CooledDownResource(t *testing.T) { defer ctrl.Finish() trunkENI, ec2APIHelper, _ := getMockHelperInstanceAndTrunkObject(ctrl) - EniDetails1.deletionTimeStamp = time.Now().Add(-time.Second * 30) + EniDetails1.deletionTimeStamp = time.Now().Add(-time.Second * 60) EniDetails2.deletionTimeStamp = time.Now().Add(-time.Second * 24) trunkENI.usedVlanIds[VlanId1] = true trunkENI.usedVlanIds[VlanId2] = true @@ -476,6 +489,10 @@ func TestTrunkENI_DeleteCooledDownENIs_CooledDownResource(t *testing.T) { ec2APIHelper.EXPECT().DeleteNetworkInterface(&EniDetails1.ID).Return(nil) + mockK8sAPI := mock_k8s.NewMockK8sWrapper(ctrl) + mockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil) + cooldown.InitCoolDownPeriod(mockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) + trunkENI.DeleteCooledDownENIs() assert.Equal(t, 1, len(trunkENI.deleteQueue)) assert.Equal(t, EniDetails2, trunkENI.deleteQueue[0]) @@ -488,18 +505,23 @@ func TestTrunkENI_DeleteCooledDownENIs_DeleteFailed(t *testing.T) { defer ctrl.Finish() trunkENI, ec2APIHelper, _ := getMockHelperInstanceAndTrunkObject(ctrl) - EniDetails1.deletionTimeStamp = time.Now().Add(-time.Second * 31) - EniDetails2.deletionTimeStamp = time.Now().Add(-time.Second * 32) + coolDown := mock_cooldown.NewMockCoolDown(ctrl) + EniDetails1.deletionTimeStamp = time.Now().Add(-time.Second * 61) + EniDetails2.deletionTimeStamp = time.Now().Add(-time.Second * 62) trunkENI.usedVlanIds[VlanId1] = true trunkENI.usedVlanIds[VlanId2] = true trunkENI.deleteQueue = append(trunkENI.deleteQueue, EniDetails1, EniDetails2) - gomock.InOrder( + coolDown.EXPECT().GetCoolDownPeriod().Return(time.Second*60).AnyTimes(), ec2APIHelper.EXPECT().DeleteNetworkInterface(&EniDetails1.ID).Return(MockError).Times(MaxDeleteRetries), ec2APIHelper.EXPECT().DeleteNetworkInterface(&EniDetails2.ID).Return(nil), ) + mockK8sAPI := mock_k8s.NewMockK8sWrapper(ctrl) + mockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("60"), nil) + cooldown.InitCoolDownPeriod(mockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown")) + trunkENI.DeleteCooledDownENIs() assert.Zero(t, len(trunkENI.deleteQueue)) } @@ -623,9 +645,11 @@ func TestTrunkENI_InitTrunk(t *testing.T) { name: "TrunkExists_WithBranches, verifies no error when trunk exists with branches", prepare: func(f *fields) { f.mockInstance.EXPECT().InstanceID().Return(InstanceId) + f.mockInstance.EXPECT().GetCustomNetworkingSpec().Return("", []string{}) f.mockEC2APIHelper.EXPECT().GetInstanceNetworkInterface(&InstanceId).Return(instanceNwInterfaces, nil) f.mockEC2APIHelper.EXPECT().WaitForNetworkInterfaceStatusChange(&trunkId, awsEc2.AttachmentStatusAttached).Return(nil) - f.mockEC2APIHelper.EXPECT().GetBranchNetworkInterface(&trunkId).Return(branchInterfaces, nil) + f.mockInstance.EXPECT().SubnetID().Return(SubnetId) + f.mockEC2APIHelper.EXPECT().GetBranchNetworkInterface(&trunkId, &SubnetId).Return(branchInterfaces, nil) }, args: args{instance: FakeInstance, podList: []v1.Pod{*MockPod1, *MockPod2}}, wantErr: false, @@ -651,9 +675,11 @@ func TestTrunkENI_InitTrunk(t *testing.T) { name: "TrunkExists_DanglingENIs, verifies ENIs are pushed to delete queue if no pod exists", prepare: func(f *fields) { f.mockInstance.EXPECT().InstanceID().Return(InstanceId) + f.mockInstance.EXPECT().GetCustomNetworkingSpec().Return("", []string{}) f.mockEC2APIHelper.EXPECT().GetInstanceNetworkInterface(&InstanceId).Return(instanceNwInterfaces, nil) f.mockEC2APIHelper.EXPECT().WaitForNetworkInterfaceStatusChange(&trunkId, awsEc2.AttachmentStatusAttached).Return(nil) - f.mockEC2APIHelper.EXPECT().GetBranchNetworkInterface(&trunkId).Return(branchInterfaces, nil) + f.mockInstance.EXPECT().SubnetID().Return(SubnetId) + f.mockEC2APIHelper.EXPECT().GetBranchNetworkInterface(&trunkId, &SubnetId).Return(branchInterfaces, nil) }, args: args{instance: FakeInstance, podList: []v1.Pod{*MockPod2}}, wantErr: false, @@ -860,3 +886,15 @@ func TestTrunkENI_Introspect(t *testing.T) { PodToBranchENI: map[string][]ENIDetails{PodUID: {*EniDetails1}}}, ) } + +func createCoolDownMockCM(cooldownTime string) *v1.ConfigMap { + return &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.VpcCniConfigMapName, + Namespace: config.KubeSystemNamespace, + }, + Data: map[string]string{ + config.BranchENICooldownPeriodKey: cooldownTime, + }, + } +} diff --git a/pkg/provider/prefix/provider_test.go b/pkg/provider/prefix/provider_test.go index 70410aff..3daea497 100644 --- a/pkg/provider/prefix/provider_test.go +++ b/pkg/provider/prefix/provider_test.go @@ -22,9 +22,9 @@ import ( mock_ec2 "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/aws/ec2" mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition" mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/pool" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/ip/eni" - "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/worker" + mock_pool "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/pool" + mock_eni "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/provider/ip/eni" + mock_worker "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/worker" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/api" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/pool" @@ -68,6 +68,16 @@ var ( }, } + vpcCNIConfigWindows = &v1.ConfigMap{ + Data: map[string]string{ + config.EnableWindowsIPAMKey: "true", + config.EnableWindowsPrefixDelegationKey: "true", + config.WinWarmIPTarget: strconv.Itoa(config.IPv4PDDefaultWarmIPTargetSize), + config.WinMinimumIPTarget: strconv.Itoa(config.IPv4PDDefaultMinIPTargetSize), + config.WinWarmPrefixTarget: strconv.Itoa(config.IPv4PDDefaultWarmPrefixTargetSize), + }, + } + node = &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, @@ -386,23 +396,25 @@ func TestIPv4PrefixProvider_UpdateResourceCapacity_FromFromIPToPD(t *testing.T) instanceProviderAndPool: map[string]*ResourceProviderAndPool{}, log: zap.New(zap.UseDevMode(true)).WithName("prefix provider"), conditions: mockConditions} - mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(vpcCNIConfig, nil) - mockPool := mock_pool.NewMockPool(ctrl) - mockManager := mock_eni.NewMockENIManager(ctrl) - prefixProvider.putInstanceProviderAndPool(nodeName, mockPool, mockManager, nodeCapacity, true) - mockConditions.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true) + for _, c := range []*v1.ConfigMap{vpcCNIConfig, vpcCNIConfigWindows} { + mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(c, nil) + mockPool := mock_pool.NewMockPool(ctrl) + mockManager := mock_eni.NewMockENIManager(ctrl) + prefixProvider.putInstanceProviderAndPool(nodeName, mockPool, mockManager, nodeCapacity, true) + mockConditions.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true) - job := &worker.WarmPoolJob{Operations: worker.OperationCreate} - mockPool.EXPECT().SetToActive(pdWarmPoolConfig).Return(job) - mockWorker.EXPECT().SubmitJob(job) + job := &worker.WarmPoolJob{Operations: worker.OperationCreate} + mockPool.EXPECT().SetToActive(pdWarmPoolConfig).Return(job) + mockWorker.EXPECT().SubmitJob(job) - mockInstance.EXPECT().Name().Return(nodeName).Times(2) - mockInstance.EXPECT().Type().Return(instanceType) - mockInstance.EXPECT().Os().Return(config.OSWindows) - mockK8sWrapper.EXPECT().AdvertiseCapacityIfNotSet(nodeName, config.ResourceNameIPAddress, 224).Return(nil) + mockInstance.EXPECT().Name().Return(nodeName).Times(2) + mockInstance.EXPECT().Type().Return(instanceType) + mockInstance.EXPECT().Os().Return(config.OSWindows) + mockK8sWrapper.EXPECT().AdvertiseCapacityIfNotSet(nodeName, config.ResourceNameIPAddress, 224).Return(nil) - err := prefixProvider.UpdateResourceCapacity(mockInstance) - assert.NoError(t, err) + err := prefixProvider.UpdateResourceCapacity(mockInstance) + assert.NoError(t, err) + } } // TestIPv4PrefixProvider_UpdateResourceCapacity_FromFromPDToIP tests the warm pool is drained when PD is disabled @@ -449,21 +461,23 @@ func TestIPv4PrefixProvider_UpdateResourceCapacity_FromPDToPD(t *testing.T) { mockPool := mock_pool.NewMockPool(ctrl) mockManager := mock_eni.NewMockENIManager(ctrl) prefixProvider.putInstanceProviderAndPool(nodeName, mockPool, mockManager, nodeCapacity, true) - mockConditions.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true) - mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(vpcCNIConfig, nil) + for _, c := range []*v1.ConfigMap{vpcCNIConfig, vpcCNIConfigWindows} { + mockConditions.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true) + mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(c, nil) - job := &worker.WarmPoolJob{Operations: worker.OperationCreate} - mockPool.EXPECT().SetToActive(pdWarmPoolConfig).Return(job) - mockWorker.EXPECT().SubmitJob(job) + job := &worker.WarmPoolJob{Operations: worker.OperationCreate} + mockPool.EXPECT().SetToActive(pdWarmPoolConfig).Return(job) + mockWorker.EXPECT().SubmitJob(job) - mockInstance.EXPECT().Name().Return(nodeName).Times(2) - mockInstance.EXPECT().Type().Return(instanceType) - mockInstance.EXPECT().Os().Return(config.OSWindows) - mockK8sWrapper.EXPECT().AdvertiseCapacityIfNotSet(nodeName, config.ResourceNameIPAddress, 224).Return(nil) + mockInstance.EXPECT().Name().Return(nodeName).Times(2) + mockInstance.EXPECT().Type().Return(instanceType) + mockInstance.EXPECT().Os().Return(config.OSWindows) + mockK8sWrapper.EXPECT().AdvertiseCapacityIfNotSet(nodeName, config.ResourceNameIPAddress, 224).Return(nil) - err := prefixProvider.UpdateResourceCapacity(mockInstance) - assert.NoError(t, err) + err := prefixProvider.UpdateResourceCapacity(mockInstance) + assert.NoError(t, err) + } } // TestIPv4PrefixProvider_UpdateResourceCapacity_FromIPToIP tests the resource capacity is not updated when secondary IP mode stays enabled @@ -539,10 +553,12 @@ func TestGetPDWarmPoolConfig(t *testing.T) { instanceProviderAndPool: map[string]*ResourceProviderAndPool{}, log: zap.New(zap.UseDevMode(true)).WithName("prefix provider"), conditions: mockConditions} - mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(vpcCNIConfig, nil) + for _, c := range []*v1.ConfigMap{vpcCNIConfig, vpcCNIConfigWindows} { + mockK8sWrapper.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(c, nil) - config := prefixProvider.getPDWarmPoolConfig() - assert.Equal(t, pdWarmPoolConfig, config) + config := prefixProvider.getPDWarmPoolConfig() + assert.Equal(t, pdWarmPoolConfig, config) + } } // TestIsInstanceSupported tests that if the instance type is nitro, return true diff --git a/pkg/utils/events.go b/pkg/utils/events.go index 5a1f27c5..6afef7ad 100644 --- a/pkg/utils/events.go +++ b/pkg/utils/events.go @@ -28,6 +28,7 @@ const ( NodeTrunkFailedInitializationReason = "NodeTrunkFailedInit" EniConfigNameNotFoundReason = "EniConfigNameNotFound" VersionNotice = "ControllerVersionNotice" + BranchENICoolDownUpdateReason = "BranchENICoolDownPeriodUpdated" ) func SendNodeEventWithNodeName(client k8s.K8sWrapper, nodeName, reason, msg, eventType string, logger logr.Logger) { @@ -43,3 +44,14 @@ func SendNodeEventWithNodeName(client k8s.K8sWrapper, nodeName, reason, msg, eve func SendNodeEventWithNodeObject(client k8s.K8sWrapper, node *v1.Node, reason, msg, eventType string, logger logr.Logger) { client.BroadcastEvent(node, reason, msg, eventType) } + +func SendBroadcastNodeEvent(client k8s.K8sWrapper, reason, msg, eventType string, logger logr.Logger) { + if nodeList, err := client.ListNodes(); err == nil { + for _, node := range nodeList.Items { + node := node // Fix gosec G601, so we can use &node + client.BroadcastEvent(&node, reason, msg, eventType) + } + } else { + logger.Info("failed to list nodes when broadcasting node event", "Reason", reason, "Message", msg) + } +} diff --git a/pkg/worker/worker_test.go b/pkg/worker/worker_test.go index 1d1c84d6..c72e303e 100644 --- a/pkg/worker/worker_test.go +++ b/pkg/worker/worker_test.go @@ -16,6 +16,7 @@ package worker import ( "context" "fmt" + "sync" "testing" "time" @@ -34,15 +35,18 @@ var ( maxRequeue = 3 ) +var mu sync.RWMutex + func GetMockWorkerPool(ctx context.Context) Worker { log := zap.New(zap.UseDevMode(true)).WithValues("worker resource Id", resourceName) return NewDefaultWorkerPool(resourceName, workerCount, maxRequeue, log, ctx) } func MockWorkerFunc(job interface{}) (result ctrl.Result, err error) { + mu.Lock() + defer mu.Unlock() v := job.(*int) *v++ - time.Sleep(time.Millisecond * mockTimeToProcessWorkerFunc) return ctrl.Result{}, nil } @@ -75,8 +79,11 @@ func TestWorker_SubmitJob(t *testing.T) { time.Sleep(time.Millisecond * (mockTimeToProcessWorkerFunc + bufferTimeBwWorkerFuncExecution) * time.Duration(jobCount)) // Verify job completed. - assert.Equal(t, job1, 1) - assert.Equal(t, job2, 1) + mu.RLock() + defer mu.RUnlock() + for _, j := range []int{job1, job2} { + assert.Equal(t, j, 1) + } } func TestWorker_SubmitJob_RequeueOnError(t *testing.T) { @@ -84,6 +91,8 @@ func TestWorker_SubmitJob_RequeueOnError(t *testing.T) { defer cancel() workerFunc := func(job interface{}) (result ctrl.Result, err error) { + mu.Lock() + defer mu.Unlock() invoked := job.(*int) *invoked++ @@ -100,7 +109,9 @@ func TestWorker_SubmitJob_RequeueOnError(t *testing.T) { time.Sleep((mockTimeToProcessWorkerFunc + bufferTimeBwWorkerFuncExecution) * time.Millisecond * time.Duration(maxRequeue)) // expected invocation = max requeue + the first invocation + mu.RLock() assert.Equal(t, maxRequeue+1, invoked) + mu.RUnlock() } func TestWorker_SubmitJob_NotRequeueOnError(t *testing.T) { @@ -108,6 +119,8 @@ func TestWorker_SubmitJob_NotRequeueOnError(t *testing.T) { defer cancel() workerFunc := func(job interface{}) (result ctrl.Result, err error) { + mu.Lock() + defer mu.Unlock() invoked := job.(*int) *invoked++ @@ -127,5 +140,7 @@ func TestWorker_SubmitJob_NotRequeueOnError(t *testing.T) { actualInqueue := 1 // invoked should be only incremented once assert.NotEqual(t, maxRequeue, actualInqueue) + mu.RLock() assert.Equal(t, actualInqueue, invoked) + mu.RUnlock() } diff --git a/scripts/gen_mocks.sh b/scripts/gen_mocks.sh index 550c2215..c0b65e9d 100755 --- a/scripts/gen_mocks.sh +++ b/scripts/gen_mocks.sh @@ -13,6 +13,7 @@ mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/handler/moc # package provider mocks mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/provider/mock_provider.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider ResourceProvider mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/trunk/mock_trunk.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/trunk TrunkENI +mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/provider/branch/cooldown/mock_cooldown.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown CoolDown mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/provider/ip/eni/mock_eni.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/ip/eni ENIManager # package node mocks mockgen -destination=../mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager/mock_manager.go github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager Manager diff --git a/scripts/test/run-canary-test.sh b/scripts/test/run-canary-test.sh index 33826f8b..f86b9abf 100755 --- a/scripts/test/run-canary-test.sh +++ b/scripts/test/run-canary-test.sh @@ -110,7 +110,7 @@ function run_canary_tests() { # Currently the overall execution time is ~50 minutes and we will reduce it in future (CGO_ENABLED=0 ginkgo --no-color --focus="CANARY" $EXTRA_GINKGO_FLAGS -v --timeout 10m $GINKGO_TEST_BUILD_DIR/perpodsg.test -- --cluster-kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID) if [[ -z "${SKIP_WINDOWS_TEST}" ]]; then - (CGO_ENABLED=0 ginkgo --no-color --focus="CANARY" $EXTRA_GINKGO_FLAGS -v --timeout 25m $GINKGO_TEST_BUILD_DIR/windows.test -- --cluster-kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID) + (CGO_ENABLED=0 ginkgo --no-color --focus="CANARY" $EXTRA_GINKGO_FLAGS -v --timeout 35m $GINKGO_TEST_BUILD_DIR/windows.test -- --cluster-kubeconfig=$KUBE_CONFIG_PATH --cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id=$VPC_ID) else echo "skipping Windows tests" fi diff --git a/test/integration/metrics/metrics_suite_test.go b/test/integration/metrics/metrics_suite_test.go index 9cbafacf..ac09f16c 100644 --- a/test/integration/metrics/metrics_suite_test.go +++ b/test/integration/metrics/metrics_suite_test.go @@ -95,7 +95,7 @@ func ensureControllerReadyTobeScraped() error { // If the metrics endpoint is not created, we should create it for following tests. newController := deployment.DeepCopy() newController.Spec.Template.Spec.Containers[0].Args = append( - newController.Spec.Template.Spec.Containers[0].Args, "--metrics-addr=:8443") + newController.Spec.Template.Spec.Containers[0].Args, "--metrics-bind-address=:8443") port := v1.ContainerPort{ Name: "metrics", ContainerPort: 8443, diff --git a/test/integration/perpodsg/job_test.go b/test/integration/perpodsg/job_test.go index ea6b6ae9..2382f954 100644 --- a/test/integration/perpodsg/job_test.go +++ b/test/integration/perpodsg/job_test.go @@ -21,6 +21,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown" "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/manifest" "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/controller" sgpWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/sgp" @@ -321,7 +322,7 @@ func VerifyJobNetworkingRemovedOnCompletion(jobs map[string][]*batchV1.Job, By("waiting for the ENI to be cooled down and deleted") // Need to account for actual deletion of ENI + Cool down Period - time.Sleep(config.CoolDownPeriod * 2) + time.Sleep(cooldown.DefaultCoolDownPeriod * 2) By("verifying the deleted Pod have their ENI deleted") verify.VerifyPodENIDeletedForAllPods(namespace, podLabelKey, podLabelVal) diff --git a/test/integration/windows/windows_test.go b/test/integration/windows/windows_test.go index 343d729b..24962b4a 100644 --- a/test/integration/windows/windows_test.go +++ b/test/integration/windows/windows_test.go @@ -285,6 +285,7 @@ var _ = Describe("Windows Integration Test", func() { }) }) + // TODO: remove this context when VPC CNI also updates the flag name to windows prefixed. Context("When warm-prefix-target is set to 2", Label("warm-prefix-target"), func() { BeforeEach(func() { data = map[string]string{ @@ -316,6 +317,7 @@ var _ = Describe("Windows Integration Test", func() { }) }) + // TODO: remove this context when VPC CNI also updates the flag name to windows prefixed. Context("When warm-ip-target is set to 15", Label("warm-ip-target"), func() { BeforeEach(func() { data = map[string]string{ @@ -362,6 +364,7 @@ var _ = Describe("Windows Integration Test", func() { }) }) + // TODO: remove this context when VPC CNI also updates the flag name to windows prefixed. Context("When minimum-ip-target is set to 20", Label("minimum-ip-target"), func() { BeforeEach(func() { data = map[string]string{ @@ -415,6 +418,137 @@ var _ = Describe("Windows Integration Test", func() { }) }) + Context("When windows-warm-prefix-target is set to 2", Label("windows-warm-prefix-target"), func() { + BeforeEach(func() { + data = map[string]string{ + config.EnableWindowsIPAMKey: "true", + config.EnableWindowsPrefixDelegationKey: "true", + config.WinWarmPrefixTarget: "2"} + + }) + + It("two prefixes should be assigned", func() { + // allow some time for previous test pod to cool down + time.Sleep(bufferForCoolDown) + _, prefixesBefore, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesBefore)).To(Equal(2)) + + By("creating pod and waiting for ready should have 1 new prefix assigned") + // verify if ip assigned is coming from a prefix + createdPod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, testPod, utils.WindowsPodsCreationTimeout) + Expect(err).ToNot(HaveOccurred()) + verify.WindowsPodHaveIPv4AddressFromPrefixes(createdPod, prefixesBefore) + + // number of prefixes should increase by 1 since need 1 more prefix to fulfill warm-prefix-target of 2 + _, prefixesAfter, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesAfter) - len(prefixesBefore)).To(Equal(1)) + + err = frameWork.PodManager.DeleteAndWaitTillPodIsDeleted(ctx, testPod) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("When windows-warm-ip-target is set to 15", Label("windows-warm-ip-target"), func() { + BeforeEach(func() { + data = map[string]string{ + config.EnableWindowsIPAMKey: "true", + config.EnableWindowsPrefixDelegationKey: "true", + config.WinWarmIPTarget: "15"} + }) + It("should assign new prefix when 2nd pod is launched", func() { + // allow some time for previous test pod to cool down + time.Sleep(bufferForCoolDown) + // before running any pod, should have 1 prefix assigned + privateIPsBefore, prefixesBefore, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesBefore)).To(Equal(1)) + + By("creating 1 pod and waiting for ready should not create new prefix") + // verify if ip assigned is coming from a prefix + createdPod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, testPod, utils.WindowsPodsCreationTimeout) + Expect(err).ToNot(HaveOccurred()) + + _, prefixesAfterPod1, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesAfterPod1)).To(Equal(len(prefixesBefore))) + verify.WindowsPodHaveIPv4AddressFromPrefixes(createdPod, prefixesAfterPod1) + + // launch 2nd pod to trigger a new prefix to be assigned since warm-ip-target=15 + By("creating 2nd pod and waiting for ready should have 1 more prefix assigned") + createdPod, err = frameWork.PodManager.CreateAndWaitTillPodIsRunning(ctx, testPod2, utils.WindowsPodsCreationTimeout) + Expect(err).ToNot(HaveOccurred()) + verify.WindowsPodHaveResourceLimits(createdPod, true) + + privateIPsAfter, prefixesAfterPod2, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + // 1 more prefix should be created to fulfill warm-ip-target=15 + Expect(len(prefixesAfterPod2) - len(prefixesAfterPod1)).To(Equal(1)) + // number of secondary ips should not change + Expect(len(privateIPsBefore)).To(Equal(len(privateIPsAfter))) + verify.WindowsPodHaveIPv4AddressFromPrefixes(createdPod, prefixesAfterPod2) + + err = frameWork.PodManager.DeleteAndWaitTillPodIsDeleted(ctx, testPod) + Expect(err).ToNot(HaveOccurred()) + err = frameWork.PodManager.DeleteAndWaitTillPodIsDeleted(ctx, testPod2) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("When windows-minimum-ip-target is set to 20", Label("windows-minimum-ip-target"), func() { + BeforeEach(func() { + data = map[string]string{ + config.EnableWindowsIPAMKey: "true", + config.EnableWindowsPrefixDelegationKey: "true", + config.WinMinimumIPTarget: "20"} + }) + It("should have 2 prefixes to satisfy windows-minimum-ip-target when no pods running", func() { + By("adding labels to selected nodes for testing") + node := windowsNodeList.Items[0] + err = frameWork.NodeManager.AddLabels([]v1.Node{node}, map[string]string{podLabelKey: podLabelVal}) + Expect(err).ToNot(HaveOccurred()) + + // allow some time for previous test pod to cool down + time.Sleep(bufferForCoolDown) + // before running any pod, should have 2 prefixes assigned + instanceID = manager.GetNodeInstanceID(&node) + privateIPsBefore, prefixesBefore, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesBefore)).To(Equal(2)) + + By("creating 33 pods and waiting for ready should have 3 prefixes attached") + deployment := manifest.NewWindowsDeploymentBuilder(). + Replicas(33). + Container(manifest.NewWindowsContainerBuilder().Build()). + PodLabel(podLabelKey, podLabelVal). + NodeSelector(map[string]string{"kubernetes.io/os": "windows", podLabelKey: podLabelVal}). + Build() + _, err = frameWork.DeploymentManager.CreateAndWaitUntilDeploymentReady(ctx, deployment) + Expect(err).ToNot(HaveOccurred()) + + _, prefixesAfterDeployment, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesAfterDeployment)).To(Equal(3)) + + By("deleting 33 pods should still have 2 prefixes attached") + err = frameWork.DeploymentManager.DeleteAndWaitUntilDeploymentDeleted(ctx, deployment) + Expect(err).ToNot(HaveOccurred()) + + // allow some time for previous test pods to cool down since deletion of deployment doesn't wait for pods to terminate + time.Sleep(utils.WindowsPodsDeletionTimeout) + privateIPsAfter, prefixesAfterDelete, err := frameWork.EC2Manager.GetPrivateIPv4AddressAndPrefix(instanceID) + Expect(err).ToNot(HaveOccurred()) + Expect(len(prefixesAfterDelete)).To(Equal(2)) + // number of secondary ips should not change + Expect(len(privateIPsBefore)).To(Equal(len(privateIPsAfter))) + + By("removing labels on selected nodes for testing") + err = frameWork.NodeManager.RemoveLabels([]v1.Node{node}, map[string]string{podLabelKey: podLabelVal}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("[CANARY] When enable-windows-prefix-delegation is toggled to false", func() { BeforeEach(func() { data = map[string]string{