add CNINode integration tests (aws#479)

* add CNINode integration tests

* address PR comments

* updating log statements

* add retry in VerifyCNINode

Author: sushrk

pkg/utils/helper.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -237,7 +236,7 @@ func GetSourceAcctAndArn(roleARN, region, clusterName string) (string, string, s
 
 // PodHasENIRequest will return true if first container of pod spec has request for eni indicating
 // it needs trunk interface from vpc-rc
-func PodHasENIRequest(pod *v1.Pod) bool {
+func PodHasENIRequest(pod *corev1.Pod) bool {
 	if pod == nil {
 		return false
 	}

pkg/utils/set.go

@@ -13,6 +13,10 @@
 
 package utils
 
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
 // Difference returns a-b, elements present in a and not in b
 func Difference[T comparable](a, b []T) (diff []T) {
 	m := make(map[T]struct{})
@@ -35,3 +39,11 @@ func GetKeyValSlice(m map[string]string) (key []string, val []string) {
 	}
 	return
 }
+
+func GetTagKeyValueMap(tagSet []*ec2.Tag) map[string]string {
+	m := make(map[string]string)
+	for _, tag := range tagSet {
+		m[*tag.Key] = *tag.Value
+	}
+	return m
+}

scripts/test/run-integration-tests.sh

@@ -47,6 +47,7 @@ function run_integration_tests(){
     echo "skipping Windows tests"
   fi
   (cd $INTEGRATION_TEST_DIR/webhook && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=5m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
+  (cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
 
   if [[ "$TEST_RESULT" == fail ]]; then
       exit 1

test/framework/framework.go

@@ -17,6 +17,7 @@ import (
 	eniConfig "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	sgp "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1"
+	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/autoscaling"
 	ec2Manager "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/ec2"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/configmap"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/controller"
@@ -42,21 +43,22 @@ import (
 )
 
 type Framework struct {
-	Options           Options
-	K8sClient         client.Client
-	ec2Client         *ec2.EC2
-	DeploymentManager deployment.Manager
-	PodManager        pod.Manager
-	EC2Manager        *ec2Manager.Manager
-	SAManager         serviceaccount.Manager
-	NSManager         namespace.Manager
-	SGPManager        *sgpManager.Manager
-	SVCManager        service.Manager
-	JobManager        jobs.Manager
-	NodeManager       node.Manager
-	ControllerManager controller.Manager
-	RBACManager       rbac.Manager
-	ConfigMapManager  configmap.Manager
+	Options            Options
+	K8sClient          client.Client
+	ec2Client          *ec2.EC2
+	DeploymentManager  deployment.Manager
+	PodManager         pod.Manager
+	EC2Manager         *ec2Manager.Manager
+	SAManager          serviceaccount.Manager
+	NSManager          namespace.Manager
+	SGPManager         *sgpManager.Manager
+	SVCManager         service.Manager
+	JobManager         jobs.Manager
+	NodeManager        node.Manager
+	ControllerManager  controller.Manager
+	RBACManager        rbac.Manager
+	ConfigMapManager   configmap.Manager
+	AutoScalingManager autoscaling.Manager
 }
 
 func New(options Options) *Framework {
@@ -91,20 +93,21 @@ func New(options Options) *Framework {
 	ec2 := ec2.New(sess, &aws.Config{Region: aws.String(options.AWSRegion)})
 
 	return &Framework{
-		K8sClient:         k8sClient,
-		ec2Client:         ec2,
-		PodManager:        pod.NewManager(k8sClient, k8sSchema, config),
-		DeploymentManager: deployment.NewManager(k8sClient),
-		EC2Manager:        ec2Manager.NewManager(ec2, options.AWSVPCID),
-		SAManager:         serviceaccount.NewManager(k8sClient, config),
-		NSManager:         namespace.NewManager(k8sClient),
-		SGPManager:        sgpManager.NewManager(k8sClient),
-		SVCManager:        service.NewManager(k8sClient),
-		JobManager:        jobs.NewManager(k8sClient),
-		NodeManager:       node.NewManager(k8sClient),
-		ControllerManager: controller.NewManager(k8sClient),
-		RBACManager:       rbac.NewManager(k8sClient),
-		ConfigMapManager:  configmap.NewManager(k8sClient),
-		Options:           options,
+		K8sClient:          k8sClient,
+		ec2Client:          ec2,
+		PodManager:         pod.NewManager(k8sClient, k8sSchema, config),
+		DeploymentManager:  deployment.NewManager(k8sClient),
+		EC2Manager:         ec2Manager.NewManager(ec2, options.AWSVPCID),
+		SAManager:          serviceaccount.NewManager(k8sClient, config),
+		NSManager:          namespace.NewManager(k8sClient),
+		SGPManager:         sgpManager.NewManager(k8sClient),
+		SVCManager:         service.NewManager(k8sClient),
+		JobManager:         jobs.NewManager(k8sClient),
+		NodeManager:        node.NewManager(k8sClient),
+		ControllerManager:  controller.NewManager(k8sClient),
+		RBACManager:        rbac.NewManager(k8sClient),
+		ConfigMapManager:   configmap.NewManager(k8sClient),
+		AutoScalingManager: autoscaling.NewManager(sess),
+		Options:            options,
 	}
 }

test/framework/resource/aws/autoscaling/manager.go

@@ -0,0 +1,64 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package autoscaling
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
+)
+
+type Manager interface {
+	DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error)
+	UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error
+}
+
+type defaultManager struct {
+	autoscalingiface.AutoScalingAPI
+}
+
+func NewManager(session *session.Session) Manager {
+	return &defaultManager{
+		AutoScalingAPI: autoscaling.New(session),
+	}
+}
+
+func (d defaultManager) DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error) {
+	describeAutoScalingGroupIp := &autoscaling.DescribeAutoScalingGroupsInput{
+		AutoScalingGroupNames: aws.StringSlice([]string{autoScalingGroupName}),
+	}
+	asg, err := d.AutoScalingAPI.DescribeAutoScalingGroups(describeAutoScalingGroupIp)
+	if err != nil {
+		return nil, err
+	}
+	if len(asg.AutoScalingGroups) == 0 {
+		return nil, fmt.Errorf("failed to find asg %s", autoScalingGroupName)
+	}
+
+	return asg.AutoScalingGroups, nil
+}
+
+func (d defaultManager) UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error {
+	updateASGInput := &autoscaling.UpdateAutoScalingGroupInput{
+		AutoScalingGroupName: aws.String(asgName),
+		DesiredCapacity:      aws.Int64(desiredSize),
+		MaxSize:              aws.Int64(maxSize),
+		MinSize:              aws.Int64(minSize),
+	}
+	_, err := d.AutoScalingAPI.UpdateAutoScalingGroup(updateASGInput)
+	return err
+}

test/framework/resource/k8s/node/manager.go

@@ -15,6 +15,7 @@ package node
 
 import (
 	"context"
+	"strings"
 
 	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
@@ -32,6 +33,7 @@ type Manager interface {
 	GetNodeList() (*v1.NodeList, error)
 	GetCNINode(node *v1.Node) (*cninode.CNINode, error)
 	GetCNINodeList() (*cninode.CNINodeList, error)
+	GetInstanceID(node *v1.Node) string
 }
 
 type defaultManager struct {
@@ -117,3 +119,11 @@ func (d *defaultManager) GetNodeList() (*v1.NodeList, error) {
 	err := d.k8sClient.List(context.TODO(), list)
 	return list, err
 }
+
+func (d *defaultManager) GetInstanceID(node *v1.Node) string {
+	if node.Spec.ProviderID != "" {
+		id := strings.Split(node.Spec.ProviderID, "/")
+		return id[len(id)-1]
+	}
+	return ""
+}

test/framework/resource/k8s/node/wrapper.go

@@ -15,30 +15,71 @@ package node
 
 import (
 	"context"
+	"fmt"
 
+	cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-func GetNodeAndWaitTillCapacityPresent(manager Manager, ctx context.Context, os string, expectedResource string) *v1.NodeList {
-
+func GetNodeAndWaitTillCapacityPresent(manager Manager, os string, expectedResource string) *v1.NodeList {
 	observedNodeList := &v1.NodeList{}
 	var err error
-	err = wait.Poll(utils.PollIntervalShort, utils.ResourceCreationTimeout, func() (bool, error) {
-		By("checking nodes have capacity present")
-		observedNodeList, err = manager.GetNodesWithOS(os)
-		Expect(err).ToNot(HaveOccurred())
-		for _, node := range observedNodeList.Items {
-			_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
-			if !found {
-				return false, nil
+	err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.ResourceCreationTimeout, true,
+		func(ctx context.Context) (bool, error) {
+			By("checking nodes have capacity present")
+			observedNodeList, err = manager.GetNodesWithOS(os)
+			Expect(err).ToNot(HaveOccurred())
+			for _, node := range observedNodeList.Items {
+				_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
+				if !found {
+					return false, nil
+				}
 			}
-		}
-		return true, nil
-	})
+			return true, nil
+		})
 	Expect(err).ToNot(HaveOccurred())
 	return observedNodeList
 }
+
+// VerifyCNINode checks if the number of CNINodes is equal to number of nodes in the cluster, and verifies 1:1 mapping between CNINode and Node objects
+// Returns nil if count and 1:1 mapping exists, else returns error
+func VerifyCNINode(manager Manager) error {
+	var cniNodeList *cninode.CNINodeList
+	var nodeList *v1.NodeList
+	var err error
+	By("checking number of CNINodes match number of nodes in the cluster")
+	err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.PollTimeout, true,
+		func(ctx context.Context) (bool, error) {
+			if cniNodeList, err = manager.GetCNINodeList(); err != nil {
+				return false, nil
+			}
+			if nodeList, err = manager.GetNodeList(); err != nil {
+				return false, nil
+			}
+			if len(nodeList.Items) != len(cniNodeList.Items) {
+				return false, nil
+			}
+			return true, nil
+		})
+	if err != nil {
+		return fmt.Errorf("number of CNINodes does not match number of nodes in the cluster")
+	}
+	By("checking CNINode list matches node list")
+	nameMatched := true
+	for _, node := range nodeList.Items {
+		if !lo.ContainsBy(cniNodeList.Items, func(cniNode cninode.CNINode) bool {
+			return cniNode.Name == node.Name
+		}) {
+			nameMatched = false
+		}
+	}
+	if !nameMatched {
+		return fmt.Errorf("CNINode list does not match node list")
+	}
+	return nil
+}

test/framework/utils/poll.go

@@ -19,6 +19,7 @@ const (
 	PollIntervalShort  = 2 * time.Second
 	PollIntervalMedium = 10 * time.Second
 	PollIntervalLong   = 20 * time.Second
+	PollTimeout        = 30 * time.Second
 	// ResourceCreationTimeout is the number of seconds till the controller waits
 	// for the resource creation to complete
 	ResourceCreationTimeout = 120 * time.Second

test/integration/cninode/cninode_suite_test.go

@@ -0,0 +1,50 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cninode_test
+
+import (
+	"testing"
+
+	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework"
+	"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/node"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestCNINode(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CNINode Test Suite")
+}
+
+var frameWork *framework.Framework
+var _ = BeforeSuite(func() {
+	By("creating a framework")
+	frameWork = framework.New(framework.GlobalOptions)
+
+	By("verify at least 2 nodes are available")
+	nodeList, err := frameWork.NodeManager.GetNodeList()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(nodeList.Items)).To(BeNumerically(">", 1))
+
+	By("verify CNINode count")
+	err = node.VerifyCNINode(frameWork.NodeManager)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+// Verify CNINode count before and after test remains same
+var _ = AfterSuite(func() {
+	By("verify CNINode count")
+	err := node.VerifyCNINode(frameWork.NodeManager)
+	Expect(err).ToNot(HaveOccurred())
+})