Upgrade project to kubebuilder v3

Add full Terraform testing example (not E2E yet though) to validate Autoneg setup easier.

Author: rosmo

.dockerignore

@@ -0,0 +1,5 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore all files which are not go type
+!**/*.go
+!**/*.mod
+!**/*.sum

.github/workflows/go.yml

@@ -35,8 +35,9 @@ jobs:
 
     - name: Test
       run: |
-        sudo mkdir -p /usr/local/kubebuilder
-        curl -Ls https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz | sudo tar -xvz --strip-components=1 -C /usr/local/kubebuilder -f - 
+        curl -Ls https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.8.0/kubebuilder_linux_amd64 -o /var/tmp/kubebuilder
+        chmod +x /var/tmp/kubebuilder
+        sudo mv /var/tmp/kubebuilder /usr/local/bin/
         make test
 
     - name: Login to GitHub Container Registry

.gitignore

@@ -6,6 +6,7 @@
 *.so
 *.dylib
 bin
+testbin/*
 
 # Test binary, build with `go test -c`
 *.test

Dockerfile

@@ -28,13 +28,13 @@ COPY main.go main.go
 COPY controllers/ controllers/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-COPY --chown=nonroot:nonroot --from=builder /workspace/manager .
-# user nonroot has uid 65532
-USER 65532
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+
 ENTRYPOINT ["/manager"]

Makefile

@@ -1,8 +1,9 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true"
+# Previously we produced CRDs that work back to Kubernetes 1.11 (no version conversion),
+# but now we'll support only 1.16+.
+CRD_OPTIONS ?= "crd"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -11,63 +12,114 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-CONTROLLER_GEN ?= sigs.k8s.io/controller-tools/cmd/[email protected]
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
 
-all: manager
+all: build
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
+##@ General
 
-# Build manager binary
-manager: generate fmt vet
-	go build -o bin/manager main.go
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
 
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
-	go run ./main.go
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-# Install CRDs into a cluster
-install: manifests
-	kustomize build config/crd | kubectl apply -f -
+##@ Development
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
-# Generate manifests e.g. CRD, RBAC etc.
-manifests:
-	go run $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
-# Run go fmt against code
-fmt:
+fmt: ## Run go fmt against code.
 	go fmt ./...
 
-# Run go vet against code
-vet:
+vet: ## Run go vet against code.
 	go vet ./...
 
-# Generate code
-generate:
-	go run $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
+ENVTEST_ASSETS_DIR = $(shell pwd)/testbin
+ENVTEST = $(shell pwd)/bin/setup-envtest
+ENVTEST_K8S_VERSION ?= 1.26.1
+
+testenv:
+	mkdir -p ${ENVTEST_ASSETS_DIR}
+	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
-# Create api directory. Circumvents issue with kubebuilder without CRDs
-api:
-	mkdir api
+test: manifests generate fmt vet testenv ## Run tests.
+	KUBEBUILDER_ASSETS="$(shell ${ENVTEST} use ${ENVTEST_K8S_VERSION} --bin-dir ${ENVTEST_ASSETS_DIR} -p path)" go test ./... -coverprofile cover.out
+
+##@ Build
 
 # Build the docker image
 DOCKER_BIN ?= docker
 VERSION ?= latest
 LABELS ?= --label org.opencontainers.image.licenses="Apache-2.0" \
     --label org.opencontainers.image.vendor="Google LLC" \
     --label org.opencontainers.image.version="${VERSION}"
-docker-build: test api
-	${DOCKER_BIN} build ${LABELS} . -t ${IMG}
+
+docker-build: test
+	${DOCKER_BIN} build ${DOCKER_FLAGS} ${LABELS} . -t ${IMG}
 
 # Push the docker image
 docker-push:
-	${DOCKER_BIN} push ${IMG}
+	${DOCKER_BIN} push ${DOCKER_FLAGS} ${IMG}
+
+build: generate fmt vet ## Build manager binary.
+	go build -o bin/manager main.go
+
+run: manifests generate fmt vet ## Run a controller from your host.
+	go run ./main.go
+
+##@ Deployment
+
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef
 
 # Used for autoneg project releases
 #
@@ -77,8 +129,9 @@ RELEASE_IMG ?= ghcr.io/googlecloudplatform/gke-autoneg-controller/gke-autoneg-co
 
 # Make deployment manifests but do not deploy
 autoneg-manifests: manifests
-	cd config/manager && kustomize edit set image controller=${RELEASE_IMG}:${VERSION}
-	kustomize build config/default > deploy/autoneg.yaml
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${RELEASE_IMG}:${VERSION}
+	cp hack/boilerplate.bash.txt deploy/autoneg.yaml
+	$(KUSTOMIZE) build config/default >> deploy/autoneg.yaml
 
 # Make release image
 release-image: docker-build

PROJECT

@@ -1,3 +1,12 @@
-version: "2"
-domain: google.com
+domain: controller.autoneg.dev
+layout:
+- go.kubebuilder.io/v3
+projectName: gke-autoneg-controller
 repo: github.com/GoogleCloudPlatform/gke-autoneg-controller
+resources:
+- controller: true
+  group: core
+  kind: Service
+  path: k8s.io/api/core/v1
+  version: v1
+version: "3"

README.md

@@ -115,7 +115,7 @@ Lastly, on each cluster in your project where you'd like to install `autoneg` (v
 ```
 kubectl apply -f deploy/autoneg.yaml
 
-kubectl annotate sa -n autoneg-system autoneg \
+kubectl annotate sa -n autoneg-system autoneg-controller-manager \
   iam.gke.io/gcp-service-account=autoneg-system@${PROJECT_ID}.iam.gserviceaccount.com
 ```
 This will create all the Kubernetes resources required to support `autoneg` and annotate the default service account in the `autoneg-system` namespace to associate a GCP service account using [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity). 

config/certmanager/certificate.yaml

@@ -27,27 +27,29 @@ commonLabels:
   app: autoneg
 
 bases:
-# - ../crd
+#- ../crd
 - ../rbac
 - ../manager
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
 #- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+#- ../prometheus
 
 patchesStrategicMerge:
-  # Protect the /metrics endpoint by putting it behind auth.
-  # Only one of manager_auth_proxy_patch.yaml and
-  # manager_prometheus_metrics_patch.yaml should be enabled.
+# Protect the /metrics endpoint by putting it behind auth.
+# If you want your controller-manager to expose the /metrics
+# endpoint w/o any authn/z, please comment the following line.
 - manager_auth_proxy_patch.yaml
-  # If you want your controller-manager to expose the /metrics
-  # endpoint w/o any authn/z, uncomment the following line and
-  # comment manager_auth_proxy_patch.yaml.
-  # Only one of manager_auth_proxy_patch.yaml and
-  # manager_prometheus_metrics_patch.yaml should be enabled.
-#- manager_prometheus_metrics_patch.yaml
 
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
+# Mount the controller config file for loading manager configurations
+# through a ComponentConfig type
+#- manager_config_patch.yaml
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
 #- manager_webhook_patch.yaml
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
@@ -61,16 +63,16 @@ vars:
 #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
 #  objref:
 #    kind: Certificate
-#    group: certmanager.k8s.io
-#    version: v1alpha1
+#    group: cert-manager.io
+#    version: v1
 #    name: serving-cert # this name should match the one in certificate.yaml
 #  fieldref:
 #    fieldpath: metadata.namespace
 #- name: CERTIFICATE_NAME
 #  objref:
 #    kind: Certificate
-#    group: certmanager.k8s.io
-#    version: v1alpha1
+#    group: cert-manager.io
+#    version: v1
 #    name: serving-cert # this name should match the one in certificate.yaml
 #- name: SERVICE_NAMESPACE # namespace of the service
 #  objref:

config/default/kustomization.yaml

@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This patch inject a sidecar container which is a HTTP proxy for the controller manager,
-# it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
+# This patch inject a sidecar container which is a HTTP proxy for the
+# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -38,5 +38,6 @@ spec:
           allowPrivilegeEscalation: false
       - name: manager
         args:
-        - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
+        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--leader-elect"

config/default/manager_auth_proxy_patch.yaml

@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,16 +22,13 @@ spec:
     spec:
       containers:
       - name: manager
-        ports:
-        - containerPort: 443
-          name: webhook-server
-          protocol: TCP
+        args:
+        - "--config=controller_manager_config.yaml"
         volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
+        - name: manager-config
+          mountPath: /controller_manager_config.yaml
+          subPath: controller_manager_config.yaml
       volumes:
-      - name: cert
-        secret:
-          defaultMode: 420
-          secretName: webhook-server-cert
+      - name: manager-config
+        configMap:
+          name: manager-config