[repo-ci-improvement] : update GHA to run e2e tests (#248)

* update GHA to run e2e tests

* simplify go versions used for ci tests

* update helm version, update Makefile

* update k8s version

* fix linting

* address review comments

Author: rahulait

.github/filters.yml

@@ -0,0 +1,3 @@
+# Any file that is not a doc *.md file
+src:
+  - "!**/**.md"

.github/workflows/ci.yml

@@ -6,37 +6,92 @@ on:
       - main
   pull_request: null
 
+permissions:
+  contents: read
+  pull-requests: read
+  actions: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  ci:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      paths: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          disable-sudo: true
+          egress-policy: block
+          allowed-endpoints: >
+            api.github.com:443
+            github.com:443
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          base: ${{ github.ref }}
+          filters: .github/filters.yml
+
+  build-test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: [ 'stable', '1.22' ]
+    needs: changes
+    if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }}
     steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          disable-sudo: true
+          egress-policy: block
+          allowed-endpoints: >
+            api.github.com:443
+            github.com:443
+            golang.org:443
+            proxy.golang.org:443
+            sum.golang.org:443
+            objects.githubusercontent.com:443
+            storage.googleapis.com:443
+            cli.codecov.io:443
+            api.codecov.io:443
+            raw.githubusercontent.com:443
+            get.helm.sh:443
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: ${{ matrix.go-version }}
+          go-version-file: go.mod
+          check-latest: true
+
       - name: Vet
         run: make vet
-      - name: Lint
-        run: make lint
+
+      - name: lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: latest
+
       - name: Helm Lint
         run: make helm-lint
+
       - name: Test
         run: make test
+
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         with:
           files: ./coverage.out
-          fail_ci_if_error: true
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: linode/linode-cloud-controller-manager
+
       - name: Build
         run: make build
+
   docker-build:
     runs-on: ubuntu-latest
     steps:
@@ -61,3 +116,45 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             REV=${{ github.ref_name }}
+
+  e2e-tests:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.github_token }}
+      LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
+      IMG: linode/linode-cloud-controller-manager:${{ github.ref == 'refs/heads/main' && 'latest' || format('pr-{0}', github.event.number) || github.ref_name }}
+      LINODE_REGION: us-lax
+      LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
+      LINODE_MACHINE_TYPE: g6-standard-2
+      WORKER_NODES: '2'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install devbox
+        uses: jetify-com/[email protected]
+
+      - name: Setup CAPL Management Kind Cluster and CAPL Child Cluster For Testing
+        run: devbox run mgmt-and-capl-cluster
+
+      - name: Run E2E Tests
+        run: devbox run e2e-test
+
+      - name: Cleanup Resources
+        if: always()
+        run: devbox run cleanup-cluster

Makefile

@@ -1,12 +1,29 @@
-IMG ?= linode/linode-cloud-controller-manager:canary
-RELEASE_DIR ?= release
-PLATFORM ?= linux/amd64
+IMG                     ?= linode/linode-cloud-controller-manager:canary
+RELEASE_DIR             ?= release
+PLATFORM                ?= linux/amd64
 
 # Use CACHE_BIN for tools that cannot use devbox and LOCALBIN for tools that can use either method
-CACHE_BIN ?= $(CURDIR)/bin
-LOCALBIN ?= $(CACHE_BIN)
-
-DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin
+CACHE_BIN               ?= $(CURDIR)/bin
+LOCALBIN                ?= $(CACHE_BIN)
+
+DEVBOX_BIN              ?= $(DEVBOX_PACKAGES_DIR)/bin
+HELM                    ?= $(LOCALBIN)/helm
+HELM_VERSION            ?= v3.16.3
+
+#####################################################################
+# Dev Setup
+#####################################################################
+CLUSTER_NAME            ?= ccm-$(shell git rev-parse --short HEAD)
+K8S_VERSION             ?= "v1.31.2"
+CAPI_VERSION            ?= "v1.6.3"
+CAAPH_VERSION           ?= "v0.2.1"
+CAPL_VERSION            ?= "v0.7.1"
+CONTROLPLANE_NODES      ?= 1
+WORKER_NODES            ?= 1
+LINODE_FIREWALL_ENABLED ?= true
+LINODE_REGION           ?= us-lax
+LINODE_OS               ?= linode/ubuntu22.04
+KUBECONFIG_PATH         ?= $(CURDIR)/test-cluster-kubeconfig.yaml
 
 # if the $DEVBOX_PACKAGES_DIR env variable exists that means we are within a devbox shell and can safely
 # use devbox's bin for our tools
@@ -41,10 +58,15 @@ vet: fmt
 
 .PHONY: lint
 lint:
-	docker run --rm -v "$(shell pwd):/var/work:ro" -w /var/work \
-		golangci/golangci-lint:v1.57.2 golangci-lint run -v --timeout=5m
-	docker run --rm -v "$(shell pwd):/var/work:ro" -w /var/work/e2e \
-		golangci/golangci-lint:v1.57.2 golangci-lint run -v --timeout=5m
+	docker run --rm -v "$(PWD):/var/work:ro" -w /var/work \
+		golangci/golangci-lint:latest golangci-lint run -v --timeout=5m
+	docker run --rm -v "$(PWD):/var/work:ro" -w /var/work/e2e \
+		golangci/golangci-lint:latest golangci-lint run -v --timeout=5m
+
+.PHONY: gosec
+gosec: ## Run gosec against code.
+	docker run --rm -v "$(PWD):/var/work:ro" -w /var/work securego/gosec:2.19.0 \
+		-exclude-dir=bin -exclude-generated ./...
 
 .PHONY: fmt
 fmt:
@@ -88,9 +110,11 @@ docker-build: build-linux
 .PHONY: docker-push
 # must run the docker build before pushing the image
 docker-push:
-	echo "[reminder] Did you run `make docker-build`?"
 	docker push ${IMG}
 
+.PHONY: docker-setup
+docker-setup: docker-build docker-push
+
 .PHONY: run
 # run the ccm locally, really only makes sense on linux anyway
 run: build
@@ -108,6 +132,66 @@ run-debug: build
 		--kubeconfig=${KUBECONFIG} \
 		--linodego-debug
 
+#####################################################################
+# E2E Test Setup
+#####################################################################
+
+.PHONY: mgmt-and-capl-cluster
+mgmt-and-capl-cluster: docker-setup mgmt-cluster capl-cluster
+
+.PHONY: capl-cluster
+capl-cluster: generate-capl-cluster-manifests create-capl-cluster patch-linode-ccm
+
+.PHONY: generate-capl-cluster-manifests
+generate-capl-cluster-manifests:
+	# Create the CAPL cluster manifests without any CSI driver stuff
+	LINODE_FIREWALL_ENABLED=$(LINODE_FIREWALL_ENABLED) LINODE_OS=$(LINODE_OS) clusterctl generate cluster $(CLUSTER_NAME) \
+		--kubernetes-version $(K8S_VERSION) --infrastructure linode-linode:$(CAPL_VERSION) \
+		--control-plane-machine-count $(CONTROLPLANE_NODES) --worker-machine-count $(WORKER_NODES) > capl-cluster-manifests.yaml
+
+.PHONY: create-capl-cluster
+create-capl-cluster:
+	# Create a CAPL cluster with updated CCM and wait for it to be ready
+	kubectl apply -f capl-cluster-manifests.yaml
+	kubectl wait --for=condition=ControlPlaneReady cluster/$(CLUSTER_NAME) --timeout=600s || (kubectl get cluster -o yaml; kubectl get linodecluster -o yaml; kubectl get linodemachines -o yaml)
+	kubectl wait --for=condition=NodeHealthy=true machines -l cluster.x-k8s.io/cluster-name=$(CLUSTER_NAME) --timeout=900s
+	clusterctl get kubeconfig $(CLUSTER_NAME) > $(KUBECONFIG_PATH)
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl wait --for=condition=Ready nodes --all --timeout=600s
+	# Remove all taints from control plane node so that pods scheduled on it by tests can run (without this, some tests fail)
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl taint nodes -l node-role.kubernetes.io/control-plane node-role.kubernetes.io/control-plane-
+
+.PHONY: patch-linode-ccm
+patch-linode-ccm:
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl patch -n kube-system daemonset ccm-linode --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/containers/0/image', 'value': '${IMG}'}]"
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl rollout status -n kube-system daemonset/ccm-linode --timeout=600s
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl -n kube-system get daemonset/ccm-linode -o yaml
+
+.PHONY: mgmt-cluster
+mgmt-cluster:
+	# Create a mgmt cluster
+	ctlptl apply -f e2e/setup/ctlptl-config.yaml
+	clusterctl init \
+		--wait-providers \
+		--wait-provider-timeout 600 \
+		--core cluster-api:$(CAPI_VERSION) \
+		--addon helm:$(CAAPH_VERSION) \
+		--infrastructure linode-linode:$(CAPL_VERSION)
+
+.PHONY: cleanup-cluster
+cleanup-cluster:
+	kubectl delete cluster -A --all --timeout=180s
+	kubectl delete linodefirewalls -A --all --timeout=60s
+	kubectl delete lvpc -A --all --timeout=60s
+	kind delete cluster -n caplccm
+
+.PHONY: e2e-test
+e2e-test:
+	$(MAKE) -C e2e test LINODE_API_TOKEN=$(LINODE_TOKEN) SUITE_ARGS="--region=$(LINODE_REGION) --use-existing --timeout=5m --kubeconfig=$(KUBECONFIG_PATH) --image=$(IMG) --linode-url https://api.linode.com/"
+
+#####################################################################
+# OS / ARCH
+#####################################################################
+
 # Set the host's OS. Only linux and darwin supported for now
 HOSTOS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
 ifeq ($(filter darwin linux,$(HOSTOS)),)
@@ -121,9 +205,6 @@ else ifeq ($(ARCH_SHORT),aarch64)
 ARCH_SHORT := arm64
 endif
 
-HELM         ?= $(LOCALBIN)/helm
-HELM_VERSION ?= v3.9.1
-
 .PHONY: helm
 helm: $(HELM) ## Download helm locally if necessary
 $(HELM): $(LOCALBIN)

cloud/linode/cloud.go

@@ -117,7 +117,7 @@ func newCloud() (cloudprovider.Interface, error) {
 	if len(Options.IpHolderSuffix) > 23 {
 		msg := fmt.Sprintf("ip-holder-suffix must be 23 characters or less: %s is %d characters\n", Options.IpHolderSuffix, len(Options.IpHolderSuffix))
 		klog.Error(msg)
-		return nil, fmt.Errorf(msg)
+		return nil, fmt.Errorf("%s", msg)
 	}
 
 	// create struct that satisfies cloudprovider.Interface

devbox.json

@@ -18,6 +18,13 @@
     "init_hook": [
       "export \"GOROOT=$(go env GOROOT)\""
     ],
-    "scripts": {}
+    "scripts": {
+      "mgmt-and-capl-cluster": "make mgmt-and-capl-cluster",
+      "e2e-test": "make e2e-test",
+      "cleanup-cluster": "make cleanup-cluster"
+    }
+  },
+  "env": {
+    "EXP_CLUSTER_RESOURCE_SET": "true"
   }
 }

e2e/setup/ctlptl-config.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+product: kind
+kindV1Alpha4Cluster:
+  name: caplccm
+  nodes:
+    - role: control-plane
+      image: kindest/node:v1.31.2

main.go

@@ -113,7 +113,7 @@ func main() {
 	linode.Options.KubeconfigFlag = command.Flags().Lookup("kubeconfig")
 	if linode.Options.KubeconfigFlag == nil {
 		msg := "kubeconfig missing from CCM flag set"
-		sentry.CaptureError(ctx, fmt.Errorf(msg))
+		sentry.CaptureError(ctx, fmt.Errorf("%s", msg))
 		fmt.Fprintf(os.Stderr, "kubeconfig missing from CCM flag set"+"\n")
 		os.Exit(1)
 	}
@@ -122,7 +122,7 @@ func main() {
 		_, network, err := net.ParseCIDR(externalSubnet)
 		if err != nil {
 			msg := fmt.Sprintf("Unable to parse %s as network subnet: %v", externalSubnet, err)
-			sentry.CaptureError(ctx, fmt.Errorf(msg))
+			sentry.CaptureError(ctx, fmt.Errorf("%s", msg))
 			fmt.Fprintf(os.Stderr, "%v\n", msg)
 			os.Exit(1)
 		}