Rework acceptance tests (#175)

- Require approval for running integration and acceptance tests
- Add fail handler that prints pod logs. The amount of log lines is
  configurable with an env var.
- Add env vars to control Eventually polling interval and timeout
- Run tests in parallel. This requires using SynchronizedBeforeEach to
  set up MC only once.
- Use random name for MC. The name is generated in the Makefile before
  deploying and running the tests.
- Improve cleanup logic. The test cleanup will be performed even if deletion fails. This means that even if an introduced change breaks the deletion loop, no manual cleanup will be necessary.

Author: mnitchev

.circleci/config.yml

@@ -52,17 +52,30 @@ workflows:
             tags:
               only: /^v.*/
 
+      - hold:
+          type: approval
+          requires:
+            - unit-tests
+
       - integration-tests:
+          context: architect
+          requires:
+            - hold
           filters:
             tags:
               only: /^v.*/
 
       - acceptance-tests:
+          context: architect
+          requires:
+            - hold
           filters:
             tags:
               only: /^v.*/
 
       - architect/go-build:
+          requires:
+            - hold
           context: architect
           name: go-build
           binary: aws-resolver-rules-operator

Makefile.custom.mk

@@ -7,8 +7,13 @@ IMAGE_REPO = $(word 1,$(subst :, ,$(IMG)))
 IMAGE_TAG = $(word 2,$(subst :, ,$(IMG)))
 
 CLUSTER ?= acceptance
-MANAGEMENT_CLUSTER_NAME ?= test-mc
-MANAGEMENT_CLUSTER_NAMESPACE ?= test
+
+# Basically black magic. This will generate the a new name exactly once. If you
+# don't do the eval and subsequent variable expansion, a new name will be
+# generated every time you reference it.
+# Look up "Deferred Simple Variable Expansion" for more info.
+MANAGEMENT_CLUSTER_NAME ?= $(eval MANAGEMENT_CLUSTER_NAME := $$(shell tr -dc a-z0-9 </dev/urandom | head -c 16; echo))$(MANAGEMENT_CLUSTER_NAME)
+MANAGEMENT_CLUSTER_NAMESPACE ?= $(eval MANAGEMENT_CLUSTER_NAMESPACE := $$(shell tr -dc a-z0-9 </dev/urandom | head -c 16; echo))$(MANAGEMENT_CLUSTER_NAMESPACE)
 
 DOCKER_COMPOSE = bin/docker-compose
 
@@ -59,7 +64,7 @@ run-acceptance-tests:
 	KUBECONFIG="$(KUBECONFIG)" \
 	MANAGEMENT_CLUSTER_NAME="$(MANAGEMENT_CLUSTER_NAME)" \
 	MANAGEMENT_CLUSTER_NAMESPACE="$(MANAGEMENT_CLUSTER_NAMESPACE)" \
-	$(GINKGO) -r -randomize-all --randomize-suites tests/acceptance
+	$(GINKGO) -p --nodes 4 -r -randomize-all --randomize-suites tests/acceptance
 
 .PHONY: test-acceptance
 test-acceptance: KUBECONFIG=$(HOME)/.kube/$(CLUSTER).yml

tests/acceptance/acceptance_suite_test.go

@@ -2,14 +2,24 @@ package acceptance_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+	"strconv"
 	"testing"
+	"time"
 
+	awssdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/kubernetes/scheme"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubectl/pkg/scheme"
 	capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	capi "sigs.k8s.io/cluster-api/api/v1beta1"
 	controllerruntime "sigs.k8s.io/controller-runtime"
@@ -19,6 +29,12 @@ import (
 	"github.com/aws-resolver-rules-operator/tests/acceptance/fixture"
 )
 
+const (
+	DefaultMaxLogLines     = 300
+	DefaultPollingInterval = 5 * time.Second
+	DefaultTimeout         = 3 * time.Minute
+)
+
 var (
 	k8sClient client.Client
 
@@ -28,50 +44,175 @@ var (
 	testFixture *fixture.Fixture
 )
 
+func LogCollectorFailHandler(message string, callerSkip ...int) {
+	getPodLogs()
+	Fail(message, callerSkip...)
+}
+
 func TestAcceptance(t *testing.T) {
-	RegisterFailHandler(Fail)
+	RegisterFailHandler(LogCollectorFailHandler)
 	RunSpecs(t, "Acceptance Suite")
 }
 
-var _ = BeforeSuite(func() {
-	tests.GetEnvOrSkip("KUBECONFIG")
+var _ = SynchronizedBeforeSuite(func() []byte {
+	setupEventuallyParameters()
+	k8sClient = setupKubeClient()
 
-	config, err := controllerruntime.GetConfig()
+	fixtureConfig := getFixtureConfig()
+	testFixture = fixture.NewFixture(k8sClient, fixtureConfig)
+
+	DeferCleanup(func() {
+		Expect(testFixture.Teardown()).To(Succeed())
+	})
+	err := testFixture.Setup()
 	Expect(err).NotTo(HaveOccurred())
 
-	err = capa.AddToScheme(scheme.Scheme)
+	data := fixture.Data{
+		Config:  fixtureConfig,
+		Network: testFixture.Network,
+	}
+	encoded, err := json.Marshal(data)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = capi.AddToScheme(scheme.Scheme)
+	return encoded
+}, func(jsonData []byte) {
+	data := fixture.Data{}
+	err := json.Unmarshal(jsonData, &data)
 	Expect(err).NotTo(HaveOccurred())
 
-	k8sClient, err = client.New(config, client.Options{Scheme: scheme.Scheme})
+	setupEventuallyParameters()
+	k8sClient = setupKubeClient()
+
+	testFixture = fixture.LoadFixture(k8sClient, data)
+})
+
+var _ = BeforeEach(func() {
+	namespace = uuid.New().String()
+	namespaceObj = &corev1.Namespace{}
+	namespaceObj.Name = namespace
+	Expect(k8sClient.Create(context.Background(), namespaceObj)).To(Succeed())
+})
+
+var _ = AfterEach(func() {
+	Expect(k8sClient.Delete(context.Background(), namespaceObj)).To(Succeed())
+})
+
+func getDurationFromEnvVar(env string, defaultDuration time.Duration) time.Duration {
+	stringDuration := os.Getenv(env)
+	if stringDuration == "" {
+		return defaultDuration
+	}
+
+	duration, err := time.ParseDuration(stringDuration)
+	Expect(err).NotTo(HaveOccurred())
+
+	return duration
+}
+
+func getMaxLogLines() int64 {
+	maxLogLines := os.Getenv("MAX_LOG_LINES")
+	if maxLogLines == "" {
+		return DefaultMaxLogLines
+	}
+
+	maxLogLinesInt, err := strconv.Atoi(maxLogLines)
 	Expect(err).NotTo(HaveOccurred())
 
+	return int64(maxLogLinesInt)
+}
+
+func getPodLogs() {
+	kubeConfigPath := tests.GetEnvOrSkip("KUBECONFIG")
+	maxLogLines := getMaxLogLines()
+	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "Failed to build client config: %v", err)
+		return
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "Failed to build client: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+	podsClient := clientset.CoreV1().Pods("giantswarm")
+	pods, err := podsClient.List(ctx, metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=aws-resolver-rules-operator",
+	})
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "Failed to list pods: %v", err)
+		return
+	}
+
+	if len(pods.Items) == 0 {
+		fmt.Fprintf(GinkgoWriter, "No pods found: %v", err)
+		return
+	}
+	pod := pods.Items[0]
+
+	logOptions := corev1.PodLogOptions{TailLines: awssdk.Int64(maxLogLines)}
+	req := podsClient.GetLogs(pod.Name, &logOptions)
+	logStream, err := req.Stream(context.Background())
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "Failed to get log stream: %v", err)
+		return
+	}
+	defer logStream.Close()
+
+	logBytes, err := io.ReadAll(logStream)
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "Failed to read logs: %v", err)
+		return
+	}
+
+	fmt.Fprintf(GinkgoWriter,
+		"\n\n---------------- Last %d log lines for %q pod ----------------\n%s\n--------------------------------\n\n",
+		maxLogLines,
+		pod.Name,
+		string(logBytes))
+}
+
+func getFixtureConfig() fixture.Config {
 	awsAccount := tests.GetEnvOrSkip("MC_AWS_ACCOUNT")
 	iamRoleID := tests.GetEnvOrSkip("AWS_IAM_ROLE_ID")
 	awsRegion := tests.GetEnvOrSkip("AWS_REGION")
 	managementClusterName := tests.GetEnvOrSkip("MANAGEMENT_CLUSTER_NAME")
 	managementClusterNamespace := tests.GetEnvOrSkip("MANAGEMENT_CLUSTER_NAMESPACE")
 	roleARN := fmt.Sprintf("arn:aws:iam::%s:role/%s", awsAccount, iamRoleID)
 
-	fixtureConfig := fixture.Config{
+	return fixture.Config{
 		AWSAccount:                 awsAccount,
 		AWSIAMRoleARN:              roleARN,
 		AWSRegion:                  awsRegion,
 		ManagementClusterName:      managementClusterName,
 		ManagementClusterNamespace: managementClusterNamespace,
 	}
-	testFixture = fixture.NewFixture(k8sClient, fixtureConfig)
-})
+}
 
-var _ = BeforeEach(func() {
-	namespace = uuid.New().String()
-	namespaceObj = &corev1.Namespace{}
-	namespaceObj.Name = namespace
-	Expect(k8sClient.Create(context.Background(), namespaceObj)).To(Succeed())
-})
+func setupEventuallyParameters() {
+	pollingInterval := getDurationFromEnvVar("EVENTUALLY_POLLING_INTERVAL", DefaultPollingInterval)
+	timeout := getDurationFromEnvVar("EVENTUALLY_TIMEOUT", DefaultTimeout)
 
-var _ = AfterEach(func() {
-	Expect(k8sClient.Delete(context.Background(), namespaceObj)).To(Succeed())
-})
+	SetDefaultEventuallyPollingInterval(pollingInterval)
+	SetDefaultEventuallyTimeout(timeout)
+}
+
+func setupKubeClient() client.Client {
+	tests.GetEnvOrSkip("KUBECONFIG")
+
+	config, err := controllerruntime.GetConfig()
+	Expect(err).NotTo(HaveOccurred())
+
+	err = capa.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = capi.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	k8sClient, err = client.New(config, client.Options{Scheme: scheme.Scheme})
+	Expect(err).NotTo(HaveOccurred())
+
+	return k8sClient
+}

tests/acceptance/fixture/aws.go

@@ -1,6 +1,7 @@
 package fixture
 
 import (
+	"context"
 	"fmt"
 
 	awssdk "github.com/aws/aws-sdk-go/aws"
@@ -14,6 +15,62 @@ const (
 	WorkloadClusterCIDR   = "172.96.0.0"
 )
 
+func DetachTransitGateway(ec2Client *ec2.EC2, gatewayID, vpcID string) error {
+	if gatewayID == "" || vpcID == "" {
+		return nil
+	}
+
+	describeTGWattachmentInput := &ec2.DescribeTransitGatewayVpcAttachmentsInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   awssdk.String("transit-gateway-id"),
+				Values: awssdk.StringSlice([]string{gatewayID}),
+			},
+			{
+				Name:   awssdk.String("vpc-id"),
+				Values: awssdk.StringSlice([]string{vpcID}),
+			},
+		},
+	}
+	attachments, err := ec2Client.DescribeTransitGatewayVpcAttachments(describeTGWattachmentInput)
+	if err != nil {
+		return err
+	}
+
+	for _, attachment := range attachments.TransitGatewayVpcAttachments {
+		_, err = ec2Client.DeleteTransitGatewayVpcAttachmentWithContext(context.Background(), &ec2.DeleteTransitGatewayVpcAttachmentInput{
+			TransitGatewayAttachmentId: attachment.TransitGatewayAttachmentId,
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func DeleteTransitGateway(ec2Client *ec2.EC2, gatewayID string) error {
+	if gatewayID == "" {
+		return nil
+	}
+
+	_, err := ec2Client.DeleteTransitGateway(&ec2.DeleteTransitGatewayInput{
+		TransitGatewayId: awssdk.String(gatewayID),
+	})
+	return err
+}
+
+func DeletePrefixList(ec2Client *ec2.EC2, prefixListID string) error {
+	if prefixListID == "" {
+		return nil
+	}
+
+	_, err := ec2Client.DeleteManagedPrefixList(&ec2.DeleteManagedPrefixListInput{
+		PrefixListId: awssdk.String(prefixListID),
+	})
+	return err
+}
+
 func DisassociateRouteTable(ec2Client *ec2.EC2, associationID string) error {
 	if associationID == "" {
 		return nil