Test automation: zero-downtime upgrades (#1438)

Problem: Running system tests manually is a time consuming and arduous process, and can lead to inconsistent results.

Solution: Add automation for the zero-downtime upgrades test. This test must be run on GKE. The test deploys the previous released version of NGF, then begins sending http and https traffic while performing an upgrade to NGF. Results are rendered and written.

CSV files are getting too large to commit to the repository, so not including those going forward.

Author: sjberman

.gitignore

@@ -29,6 +29,9 @@ build/.out
 build/out
 dist/
 
+# Test artifacts
+tests/**/*.csv
+
 # Node modules
 node_modules/
 

.markdownlint-cli2.yaml

@@ -18,3 +18,4 @@ config:
 # Define glob expressions to ignore
 ignores:
   - ".github/"
+  - "tests/results/"

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        exclude: (^tests/results/)
       - id: end-of-file-fixer
       - id: check-yaml
         args: [--allow-multiple-documents]

tests/Makefile

@@ -2,7 +2,8 @@ TAG = edge
 PREFIX = nginx-gateway-fabric
 NGINX_PREFIX = $(PREFIX)/nginx
 PULL_POLICY=Never
-GW_API_VERSION ?= 1.0.0
+GW_API_PREV_VERSION ?= 1.0.0 ## Supported Gateway API version from previous NGF release
+GW_API_VERSION ?= 1.0.0 ## Supported Gateway API version for NGF under test
 K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest
 GW_SERVICE_TYPE=NodePort
 GW_SVC_GKE_INTERNAL=false
@@ -30,7 +31,8 @@ load-images: ## Load NGF and NGINX images on configured kind cluster
 	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
 
 test: ## Run the system tests against your default k8s cluster
-	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
+	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) \
+		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
 		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
@@ -46,9 +48,20 @@ run-tests-on-vm: ## Run the tests on a GCP VM
 create-and-setup-vm: ## Create and setup a GCP VM for tests
 	bash scripts/create-and-setup-gcp-vm.sh
 
-.PHONY: create-vm-and-run-tests
-create-vm-and-run-tests: create-and-setup-vm run-tests-on-vm ## Create and setup a GCP VM for tests and run the tests
-
 .PHONY: cleanup-vm
 cleanup-vm: ## Delete the test GCP VM and delete the firewall rule
 	bash scripts/cleanup-vm.sh
+
+.PHONY: create-gke-router
+create-gke-router: ## Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
+	bash scripts/create-gke-router.sh
+
+.PHONY: cleanup-router
+cleanup-router: ## Delete the GKE router
+	bash scripts/cleanup-router.sh
+
+.PHONY: setup-gcp-and-run-tests
+setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the tests
+
+.PHONY: cleanup-gcp
+cleanup-gcp: cleanup-router cleanup-vm ## Cleanup all GCP resources

tests/README.md

@@ -38,14 +38,17 @@ make
 
 ```text
 build-images                   Build NGF and NGINX images
-cleanup-vm                     Delete the test GCP VM and the firewall rule
+cleanup-gcp                    Cleanup all GCP resources
+cleanup-router                 Delete the GKE router
+cleanup-vm                     Delete the test GCP VM and delete the firewall rule
 create-and-setup-vm            Create and setup a GCP VM for tests
+create-gke-router              Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
 create-kind-cluster            Create a kind cluster
-create-vm-and-run-tests        Create and setup a GCP VM for tests and run the tests
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
 load-images                    Load NGF and NGINX images on configured kind cluster
 run-tests-on-vm                Run the tests on a GCP VM
+setup-gcp-and-run-tests        Create and setup a GKE router and GCP VM for tests and run the tests
 test                           Run the system tests against your default k8s cluster
 ```
 
@@ -101,15 +104,24 @@ make test TAG=$(whoami)
 This step only applies if you would like to run the tests from a GCP based VM.
 
 Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
-required env vars. The `GKE_CLUSTER_ZONE` needs to be the zone of your GKE cluster, and `GKE_SVC_ACCOUNT` needs to be
-the name of a service account that has Kubernetes admin permissions.
+required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
 
-To create and setup the VM (including creating a firewall rule allowing SSH access from your local machine, and
-optionally adding the VM IP to the `master-authorized-networks` list of your GKE cluster if
-`ADD_VM_IP_AUTH_NETWORKS` is set to `true`) and run the tests, run the following
+In order to run the tests in GCP, you need a few things:
+
+- GKE router to allow egress traffic (used by upgrade tests for pulling images from Github)
+  - this assumes that your GKE cluster is using private nodes. If using public nodes, you don't need this.
+- GCP VM and firewall rule to send ingress traffic to GKE
+
+To set up the GCP environment with the router and VM and then run the tests, run the following command:
 
 ```makefile
-make create-vm-and-run-tests
+make setup-gcp-and-run-tests
+```
+
+If you just need a VM and no router (this will not run the tests):
+
+```makefile
+make create-and-setup-vm
 ```
 
 To use an existing VM to run the tests, run the following
@@ -179,7 +191,17 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     make delete-kind-cluster
     ```
 
-2. Delete the cloud VM and cleanup the firewall rule, if required
+2. Delete the GCP components (GKE router, VM, and firewall rule), if required
+
+    ```makefile
+    make cleanup-gcp
+    ```
+
+    or
+
+    ```makefile
+    make cleanup-router
+    ```
 
     ```makefile
     make cleanup-vm

tests/framework/common.go

@@ -1,10 +1,10 @@
 package framework
 
 import (
-	"fmt"
+	"context"
+	"crypto/tls"
+	"net"
 	"net/http"
-	"net/url"
-	"os"
 	"time"
 
 	vegeta "github.com/tsenart/vegeta/v12/lib"
@@ -31,38 +31,57 @@ func convertTargetToVegetaTarget(targets []Target) []vegeta.Target {
 	return vegTargets
 }
 
+// LoadTestConfig is the configuration to run a load test.
+type LoadTestConfig struct {
+	Description string
+	Proxy       string
+	ServerName  string
+	Targets     []Target
+	Rate        int
+	Duration    time.Duration
+}
+
+// Metrics is a wrapper around the vegeta Metrics.
+type Metrics struct {
+	vegeta.Metrics
+}
+
 // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes
 // the results to the provided file
-func RunLoadTest(
-	targets []Target,
-	rate int,
-	duration time.Duration,
-	desc string,
-	outFile *os.File,
-	proxy string,
-) error {
-	vegTargets := convertTargetToVegetaTarget(targets)
+func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) {
+	vegTargets := convertTargetToVegetaTarget(cfg.Targets)
 	targeter := vegeta.NewStaticTargeter(vegTargets...)
-	proxyURL, err := url.Parse(proxy)
-	if err != nil {
-		return fmt.Errorf("error getting proxy URL: %w", err)
+
+	dialer := &net.Dialer{
+		LocalAddr: &net.TCPAddr{IP: vegeta.DefaultLocalAddr.IP, Zone: vegeta.DefaultLocalAddr.Zone},
+		KeepAlive: 30 * time.Second,
 	}
 
-	attacker := vegeta.NewAttacker(
-		vegeta.Proxy(http.ProxyURL(proxyURL)),
-	)
+	httpClient := http.Client{
+		Timeout: vegeta.DefaultTimeout,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return dialer.DialContext(ctx, network, cfg.Proxy)
+			},
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing
+				ServerName:         cfg.ServerName,
+			},
+			MaxIdleConnsPerHost: vegeta.DefaultConnections,
+			MaxConnsPerHost:     vegeta.DefaultMaxConnections,
+		},
+	}
+
+	attacker := vegeta.NewAttacker(vegeta.Client(&httpClient))
 
-	r := vegeta.Rate{Freq: rate, Per: time.Second}
+	r := vegeta.Rate{Freq: cfg.Rate, Per: time.Second}
+	var results vegeta.Results
 	var metrics vegeta.Metrics
-	for res := range attacker.Attack(targeter, r, duration, desc) {
+	for res := range attacker.Attack(targeter, r, cfg.Duration, cfg.Description) {
+		results = append(results, *res)
 		metrics.Add(res)
 	}
 	metrics.Close()
 
-	reporter := vegeta.NewTextReporter(&metrics)
-
-	if err = reporter.Report(outFile); err != nil {
-		return fmt.Errorf("error reporting results: %w", err)
-	}
-	return nil
+	return results, Metrics{metrics}
 }

tests/framework/load.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -83,39 +84,50 @@ func InstallGatewayAPI(
 	return nil, nil
 }
 
+// UninstallGatewayAPI uninstalls the specified version of the Gateway API resources.
+func UninstallGatewayAPI(apiVersion, k8sVersion string) ([]byte, error) {
+	apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion)
+
+	if webhookRequired(k8sVersion) {
+		webhookPath := fmt.Sprintf("%s/v%s/webhook-install.yaml", gwInstallBasePath, apiVersion)
+
+		if output, err := exec.Command("kubectl", "delete", "-f", webhookPath).CombinedOutput(); err != nil {
+			return output, err
+		}
+	}
+
+	output, err := exec.Command("kubectl", "delete", "-f", apiPath).CombinedOutput()
+	if err != nil && !strings.Contains(string(output), "not found") {
+		return output, err
+	}
+
+	return nil, nil
+}
+
 // InstallNGF installs NGF.
 func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 	args := []string{
 		"install", cfg.ReleaseName, cfg.ChartPath, "--create-namespace", "--namespace", cfg.Namespace, "--wait",
 	}
 
-	if cfg.NgfImageRepository != "" {
-		args = append(args, formatValueSet("nginxGateway.image.repository", cfg.NgfImageRepository)...)
-		if cfg.ImageTag != "" {
-			args = append(args, formatValueSet("nginxGateway.image.tag", cfg.ImageTag)...)
-		}
-		if cfg.ImagePullPolicy != "" {
-			args = append(args, formatValueSet("nginxGateway.image.pullPolicy", cfg.ImagePullPolicy)...)
-		}
-	}
+	args = append(args, setImageArgs(cfg)...)
+	fullArgs := append(args, extraArgs...)
 
-	if cfg.NginxImageRepository != "" {
-		args = append(args, formatValueSet("nginx.image.repository", cfg.NginxImageRepository)...)
-		if cfg.ImageTag != "" {
-			args = append(args, formatValueSet("nginx.image.tag", cfg.ImageTag)...)
-		}
-		if cfg.ImagePullPolicy != "" {
-			args = append(args, formatValueSet("nginx.image.pullPolicy", cfg.ImagePullPolicy)...)
-		}
+	return exec.Command("helm", fullArgs...).CombinedOutput()
+}
+
+// UpgradeNGF upgrades NGF. CRD upgrades assume the chart is local.
+func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
+	crdPath := filepath.Join(cfg.ChartPath, "crds")
+	if output, err := exec.Command("kubectl", "apply", "-f", crdPath).CombinedOutput(); err != nil {
+		return output, err
 	}
 
-	if cfg.ServiceType != "" {
-		args = append(args, formatValueSet("service.type", cfg.ServiceType)...)
-		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
-			args = append(args, formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
-		}
+	args := []string{
+		"upgrade", cfg.ReleaseName, cfg.ChartPath, "--namespace", cfg.Namespace, "--wait",
 	}
 
+	args = append(args, setImageArgs(cfg)...)
 	fullArgs := append(args, extraArgs...)
 
 	return exec.Command("helm", fullArgs...).CombinedOutput()
@@ -128,7 +140,7 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro
 	}
 
 	output, err := exec.Command("helm", args...).CombinedOutput()
-	if err != nil {
+	if err != nil && !strings.Contains(string(output), "release: not found") {
 		return output, err
 	}
 
@@ -157,19 +169,37 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro
 	return nil, nil
 }
 
-// UninstallGatewayAPI uninstalls the specified version of the Gateway API resources.
-func UninstallGatewayAPI(apiVersion, k8sVersion string) ([]byte, error) {
-	apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion)
+func setImageArgs(cfg InstallationConfig) []string {
+	var args []string
 
-	if webhookRequired(k8sVersion) {
-		webhookPath := fmt.Sprintf("%s/v%s/webhook-install.yaml", gwInstallBasePath, apiVersion)
+	if cfg.NgfImageRepository != "" {
+		args = append(args, formatValueSet("nginxGateway.image.repository", cfg.NgfImageRepository)...)
+		if cfg.ImageTag != "" {
+			args = append(args, formatValueSet("nginxGateway.image.tag", cfg.ImageTag)...)
+		}
+		if cfg.ImagePullPolicy != "" {
+			args = append(args, formatValueSet("nginxGateway.image.pullPolicy", cfg.ImagePullPolicy)...)
+		}
+	}
 
-		if output, err := exec.Command("kubectl", "delete", "-f", webhookPath).CombinedOutput(); err != nil {
-			return output, err
+	if cfg.NginxImageRepository != "" {
+		args = append(args, formatValueSet("nginx.image.repository", cfg.NginxImageRepository)...)
+		if cfg.ImageTag != "" {
+			args = append(args, formatValueSet("nginx.image.tag", cfg.ImageTag)...)
+		}
+		if cfg.ImagePullPolicy != "" {
+			args = append(args, formatValueSet("nginx.image.pullPolicy", cfg.ImagePullPolicy)...)
+		}
+	}
+
+	if cfg.ServiceType != "" {
+		args = append(args, formatValueSet("service.type", cfg.ServiceType)...)
+		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
+			args = append(args, formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
 		}
 	}
 
-	return exec.Command("kubectl", "delete", "-f", apiPath).CombinedOutput()
+	return args
 }
 
 func formatValueSet(key, value string) []string {