Use operator for in-repo e2e tests

lib.Makefile

@@ -1305,7 +1305,9 @@ $(REPO_ROOT)/.$(KIND_NAME).created: $(KUBECTL) $(KIND)
 	touch $@
 
 kind-cluster-destroy: $(KIND) $(KUBECTL)
-	-$(KUBECTL) --kubeconfig=$(KIND_KUBECONFIG) drain kind-control-plane kind-worker kind-worker2 kind-worker3 --ignore-daemonsets --force
+	# We need to drain the cluster gracefully when shutting down to avoid a netdev unregister error from the kernel.
+	# This requires we execute CNI del on pods with pod networking.
+	-$(KUBECTL) --kubeconfig=$(KIND_KUBECONFIG) drain kind-control-plane kind-worker kind-worker2 kind-worker3 --ignore-daemonsets --force --timeout=10m
 	-$(KIND) delete cluster --name $(KIND_NAME)
 	rm -f $(KIND_KUBECONFIG)
 	rm -f $(REPO_ROOT)/.$(KIND_NAME).created

manifests/generate.sh

@@ -124,15 +124,6 @@ done
 # Remove the last separator (last line)
 sed -i -e '$ d' tigera-operator-ocp-upgrade.yaml
 
-##########################################################################
-# Build Calico manifest used for in-repo testing. This is largely the same as the
-# one we ship, but with tweaked values.
-##########################################################################
-echo "Generating manifest from charts/values/$FILE"
-${HELM} -n kube-system template \
-	../charts/calico \
-	-f ../node/tests/k8st/infra/values.yaml > ../node/tests/k8st/infra/calico-kdd.yaml
-
 ##########################################################################
 # Replace image versions for "static" Calico manifests.
 ##########################################################################

node/.gitignore

@@ -23,5 +23,6 @@ filesystem/included-source
 config/
 kubeconfig.yaml
 ./tests/k8st/infra/calico.yaml.tmp
+tests/k8st/infra/operator/
 apiserver.crt
 apiserver.key

node/Makefile

@@ -165,9 +165,10 @@ clean: clean-windows
 	rm -rf certs *.tar $(NODE_CONTAINER_BIN_DIR)
 	rm -rf $(REMOTE_DEPS)
 	rm -rf filesystem/included-source
+	rm -rf tests/k8st/infra/operator
 	rm -rf dist
 	rm -rf bin
-	# We build felix as part of the node build, so clean it as part of the clean.
+	# We build these as part of the node build, so clean them as part of the clean.
 	make -C ../felix clean
 	# Delete images that we built in this repo
 	-docker image rm -f $$(docker images $(NODE_IMAGE) -a -q)
@@ -291,7 +292,7 @@ $(FELIX_GPL_SOURCE): .felix-gpl-source.created
 ###############################################################################
 # FV Tests
 ###############################################################################
-K8ST_IMAGE_TARS=calico-node.tar calico-typha.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar calicoctl.tar kube-controllers.tar
+K8ST_IMAGE_TARS=calico-node.tar calico-typha.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar csi.tar node-driver-registrar.tar calicoctl.tar kube-controllers.tar operator.tar
 
 ifeq ($(SEMAPHORE_GIT_REF_TYPE), pull-request)
 # Determine the tests to run using the test spider tool, which emits a list of impacted packages.
@@ -340,6 +341,10 @@ test_image: .calico_test.created
 	$(DOCKER_BUILD) --build-arg ETCD_VERSION=$(ETCD_VERSION) -f calico_test/Dockerfile -t $(TEST_CONTAINER_NAME) calico_test
 	touch $@
 
+chart: ../bin/tigera-operator-$(GIT_VERSION).tgz
+../bin/tigera-operator-$(GIT_VERSION).tgz:
+	make -C ../ bin/tigera-operator-$(GIT_VERSION).tgz
+
 calico-node.tar: $(NODE_CONTAINER_CREATED)
 	docker save --output $@ $(NODE_IMAGE):latest-$(ARCH)
 
@@ -351,11 +356,23 @@ calico-apiserver.tar: ../go.mod $(shell find ../apiserver -name '*.go') $(shell
 	make -C ../apiserver image
 	docker save --output $@ calico/apiserver:latest-$(ARCH)
 
+operator.tar: $(K8ST_IMAGE_TARS) tests/k8st/infra/calico_versions.yml
+	cd tests/k8st/infra/ && BRANCH=$(OPERATOR_BRANCH) ./build-operator.sh
+	docker save --output $@ docker.io/tigera/operator:test-build
+
 calico-cni.tar: ../go.mod $(shell find ../cni-plugin -name '*.go') $(shell find ../libcalico-go -name '*.go')
 	make -C ../cni-plugin image
 	docker save --output $@ calico/cni:latest-$(ARCH)
 
-pod2daemon.tar: ../go.mod $(shell find ../pod2daemon -name '*.go')
+csi.tar: ../go.mod $(shell find ../pod2daemon/ -name '*.go')
+	make -C ../pod2daemon image
+	docker save --output $@ calico/csi:latest-$(ARCH)
+
+node-driver-registrar.tar: ../go.mod $(shell find ../pod2daemon/ -name '*.go')
+	make -C ../pod2daemon image
+	docker save --output $@ calico/node-driver-registrar:latest-$(ARCH)
+
+pod2daemon.tar: ../go.mod $(shell find ../pod2daemon/ -name '*.go')
 	make -C ../pod2daemon image
 	docker save --output $@ calico/pod2daemon-flexvol:latest-$(ARCH)
 
@@ -399,8 +416,8 @@ k8s-test:
 	$(MAKE) kind-k8st-cleanup
 
 .PHONY: kind-k8st-setup
-kind-k8st-setup: $(K8ST_IMAGE_TARS) kind-cluster-create
-	KUBECONFIG=$(KIND_KUBECONFIG) ARCH=$(ARCH) ./tests/k8st/deploy_resources_on_kind_cluster.sh
+kind-k8st-setup: $(K8ST_IMAGE_TARS) ../bin/tigera-operator-$(GIT_VERSION).tgz kind-cluster-create
+	KUBECONFIG=$(KIND_KUBECONFIG) ARCH=$(ARCH) GIT_VERSION=$(GIT_VERSION) ./tests/k8st/deploy_resources_on_kind_cluster.sh
 
 .PHONY: kind-k8st-run-test
 kind-k8st-run-test: .calico_test.created $(KIND_KUBECONFIG)

node/tests/k8st/deploy_resources_on_kind_cluster.sh

@@ -9,76 +9,6 @@ function cleanup() {
 }
 trap 'cleanup' SIGINT SIGHUP SIGTERM EXIT
 
-# test directory.
-TEST_DIR=./tests/k8st
-ARCH=${ARCH:-amd64}
-
-# kubectl binary.
-: ${kubectl:=../hack/test/kind/kubectl}
-
-function checkModule() {
-  MODULE="$1"
-  echo "Checking kernel module $MODULE ..."
-  if lsmod | grep "$MODULE" &>/dev/null; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-function enable_dual_stack() {
-  # Based on instructions in http://docs.projectcalico.org/master/networking/dual-stack.md
-  local yaml=$1
-  # add assign_ipv4 and assign_ipv6 to CNI config
-  sed -i -e '/"type": "calico-ipam"/r /dev/stdin' "${yaml}" <<EOF
-              "assign_ipv4": "true",
-              "assign_ipv6": "true"
-EOF
-  sed -i -e 's/"type": "calico-ipam"/"type": "calico-ipam",/' "${yaml}"
-
-  sed -i -e '/"type": "calico"/r /dev/stdin' "${yaml}" <<EOF
-     "feature_control": {
-         "floating_ips": true
-     },
-EOF
-
-  # And add all the IPV6 env vars
-  sed -i '/# Enable IPIP/r /dev/stdin' "${yaml}" <<EOF
-            - name: IP6
-              value: "autodetect"
-            - name: CALICO_IPV6POOL_CIDR
-              value: "fd00:10:244::/64"
-EOF
-  # update FELIX_IPV6SUPPORT=true
-  sed -i '/FELIX_IPV6SUPPORT/!b;n;c\              value: "true"' "${yaml}"
-}
-
-echo "Set ipv6 address on each node"
-docker exec kind-control-plane ip -6 addr replace 2001:20::8/64 dev eth0
-docker exec kind-worker ip -6 addr replace 2001:20::1/64 dev eth0
-docker exec kind-worker2 ip -6 addr replace 2001:20::2/64 dev eth0
-docker exec kind-worker3 ip -6 addr replace 2001:20::3/64 dev eth0
-echo
-
-echo "Load calico/node docker images onto each node"
-$TEST_DIR/load_images_on_kind_cluster.sh
-
-echo "Install Calico and Calicoctl for dualstack"
-cp $TEST_DIR/infra/calico-kdd.yaml $TEST_DIR/infra/calico.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/calico.yaml.tmp
-enable_dual_stack $TEST_DIR/infra/calico.yaml.tmp
-${kubectl} apply -f $TEST_DIR/infra/calico.yaml.tmp
-rm $TEST_DIR/infra/calico.yaml.tmp
-cp $TEST_DIR/infra/calicoctl.yaml $TEST_DIR/infra/calicoctl.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/calicoctl.yaml.tmp
-${kubectl} apply -f $TEST_DIR/infra/calicoctl.yaml.tmp
-rm $TEST_DIR/infra/calicoctl.yaml.tmp
-echo
-
-echo "Install additional permissions for BGP password"
-${kubectl} apply -f $TEST_DIR/infra/additional-rbac.yaml
-echo
-
 function wait_pod_ready() {
   args="$@"
 
@@ -114,10 +44,44 @@ function wait_pod_ready() {
   return $rc
 }
 
+# test directory.
+TEST_DIR=./tests/k8st
+ARCH=${ARCH:-amd64}
+GIT_VERSION=${GIT_VERSION:-`git describe --tags --dirty --always --abbrev=12`}
+HELM=../bin/helm
+CHART=../bin/tigera-operator-$GIT_VERSION.tgz
+
+# kubectl binary.
+: ${kubectl:=../hack/test/kind/kubectl}
+
+echo "Set ipv6 address on each node"
+docker exec kind-control-plane ip -6 addr replace 2001:20::8/64 dev eth0
+docker exec kind-worker ip -6 addr replace 2001:20::1/64 dev eth0
+docker exec kind-worker2 ip -6 addr replace 2001:20::2/64 dev eth0
+docker exec kind-worker3 ip -6 addr replace 2001:20::3/64 dev eth0
+
+echo
+
+echo "Load calico/node docker images onto each node"
+$TEST_DIR/load_images_on_kind_cluster.sh
+
+echo "Install additional permissions for BGP password"
+${kubectl} apply -f $TEST_DIR/infra/additional-rbac.yaml
+echo
+
+echo "Install Calico using the helm chart"
+$HELM install calico $CHART -f $TEST_DIR/infra/values.yaml -n tigera-operator --create-namespace
+
+echo "Install calicoctl as a pod"
+${kubectl} apply -f $TEST_DIR/infra/calicoctl.yaml
+echo
+
 echo "Wait for Calico to be ready..."
-wait_pod_ready -l k8s-app=calico-node -n kube-system
-wait_pod_ready -l k8s-app=calico-kube-controllers -n kube-system
+wait_pod_ready -l k8s-app=calico-node -n calico-system
+wait_pod_ready -l k8s-app=calico-kube-controllers -n calico-system
+wait_pod_ready -l k8s-app=calico-apiserver -n calico-apiserver
 wait_pod_ready -l k8s-app=kube-dns -n kube-system
+wait_pod_ready calicoctl -n kube-system
 echo "Calico is running."
 echo
 
@@ -136,19 +100,7 @@ wait_pod_ready -l app=webserver
 echo "client and webserver pods are running."
 echo
 
-echo "Deploy Calico apiserver"
-cp $TEST_DIR/infra/apiserver.yaml $TEST_DIR/infra/apiserver.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/apiserver.yaml.tmp
-${kubectl} apply -f ${TEST_DIR}/infra/apiserver.yaml.tmp
-rm $TEST_DIR/infra/apiserver.yaml.tmp
-openssl req -x509 -nodes -newkey rsa:4096 -keyout apiserver.key -out apiserver.crt -days 365 -subj "/" -addext "subjectAltName = DNS:calico-api.calico-apiserver.svc"
-${kubectl} get secret -n calico-apiserver calico-apiserver-certs ||
-  ${kubectl} create secret -n calico-apiserver generic calico-apiserver-certs --from-file=apiserver.key --from-file=apiserver.crt
-${kubectl} patch apiservice v3.projectcalico.org -p \
-  "{\"spec\": {\"caBundle\": \"$(${kubectl} get secret -n calico-apiserver calico-apiserver-certs -o go-template='{{ index .data "apiserver.crt" }}')\"}}"
-wait_pod_ready -l k8s-app=calico-apiserver -n calico-apiserver
-echo "Calico apiserver is running."
-
+# Show all the pods running for diags purposes.
 ${kubectl} get po --all-namespaces -o wide
 ${kubectl} get svc
 
@@ -164,3 +116,8 @@ function test_connection() {
 }
 test_connection 4
 test_connection 6
+
+# At the end of it all, scale down the operator so that it doesn't
+# make changes to the cluster. Some of our tests modify calico/node, etc.
+# We should remove this once we fix up those tests.
+${kubectl} scale deployment -n tigera-operator tigera-operator --replicas=0

node/tests/k8st/infra/build-operator.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# This script produces a custom operator build that is used
+# in the ST and e2e tests in this repo.
+
+# Clone the repository if needed.
+set -e
+REPO=${REPO:-tigera/operator}
+BRANCH=${BRANCH:-master}
+
+rm -rf operator/
+echo "Cloning https://github.com/${REPO} @ ${BRANCH}"
+git clone --depth=1 https://github.com/${REPO} -b ${BRANCH} operator
+
+# Modify the versions that are in-use to match our locally built images.
+pushd operator
+make build/_output/bin/gen-versions
+build/_output/bin/gen-versions -os-versions=../calico_versions.yml > pkg/components/calico.go
+
+# Modify pull policy to be "Never".
+find . -name '*.go' | xargs sed -i 's/PullIfNotPresent/PullNever/g'
+
+# Build an operator image for us to use and tag it with a local-only tag.
+make image
+docker tag tigera/operator:latest docker.io/tigera/operator:test-build
+
+# Clean up after ourselves.
+popd
+rm -rf operator/
+set +e