Switch to using operator for test cluster install

Author: caseydavenport

node/.gitignore

@@ -23,5 +23,6 @@ filesystem/included-source
 config/
 kubeconfig.yaml
 ./tests/k8st/infra/calico.yaml.tmp
+tests/k8st/infra/operator/
 apiserver.crt
 apiserver.key

node/Makefile

@@ -165,9 +165,10 @@ clean: clean-windows
 	rm -rf certs *.tar $(NODE_CONTAINER_BIN_DIR)
 	rm -rf $(REMOTE_DEPS)
 	rm -rf filesystem/included-source
+	rm -rf tests/k8st/infra/operator
 	rm -rf dist
 	rm -rf bin
-	# We build felix as part of the node build, so clean it as part of the clean.
+	# We build these as part of the node build, so clean them as part of the clean.
 	make -C ../felix clean
 	# Delete images that we built in this repo
 	-docker image rm -f $$(docker images $(NODE_IMAGE) -a -q)
@@ -291,7 +292,7 @@ $(FELIX_GPL_SOURCE): .felix-gpl-source.created
 ###############################################################################
 # FV Tests
 ###############################################################################
-K8ST_IMAGE_TARS=calico-node.tar calico-typha.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar calicoctl.tar kube-controllers.tar
+K8ST_IMAGE_TARS=calico-node.tar calico-typha.tar calico-apiserver.tar calico-cni.tar pod2daemon.tar calicoctl.tar kube-controllers.tar operator.tar
 
 ifeq ($(SEMAPHORE_GIT_REF_TYPE), pull-request)
 # Determine the tests to run using the test spider tool, which emits a list of impacted packages.
@@ -340,6 +341,10 @@ test_image: .calico_test.created
 	$(DOCKER_BUILD) --build-arg ETCD_VERSION=$(ETCD_VERSION) -f calico_test/Dockerfile -t $(TEST_CONTAINER_NAME) calico_test
 	touch $@
 
+chart: ../calico/bin/tigera-operator-$(GIT_VERSION).tgz
+../calico/bin/tigera-operator-$(GIT_VERSION).tgz:
+	make -C ../ bin/tigera-operator-$(GIT_VERSION).tgz
+
 calico-node.tar: $(NODE_CONTAINER_CREATED)
 	docker save --output $@ $(NODE_IMAGE):latest-$(ARCH)
 
@@ -351,6 +356,10 @@ calico-apiserver.tar: ../go.mod $(shell find ../apiserver -name '*.go') $(shell
 	make -C ../apiserver image
 	docker save --output $@ calico/apiserver:latest-$(ARCH)
 
+operator.tar: $(K8ST_IMAGE_TARS) tests/k8st/infra/calico_versions.yml
+	cd tests/k8st/infra/ && BRANCH=$(OPERATOR_BRANCH) ./build-operator.sh
+	docker save --output $@ docker.io/tigera/operator:test-build
+
 calico-cni.tar: ../go.mod $(shell find ../cni-plugin -name '*.go') $(shell find ../libcalico-go -name '*.go')
 	make -C ../cni-plugin image
 	docker save --output $@ calico/cni:latest-$(ARCH)
@@ -399,8 +408,8 @@ k8s-test:
 	$(MAKE) kind-k8st-cleanup
 
 .PHONY: kind-k8st-setup
-kind-k8st-setup: $(K8ST_IMAGE_TARS) kind-cluster-create
-	KUBECONFIG=$(KIND_KUBECONFIG) ARCH=$(ARCH) ./tests/k8st/deploy_resources_on_kind_cluster.sh
+kind-k8st-setup: $(K8ST_IMAGE_TARS) ../calico/bin/tigera-operator-$(GIT_VERSION).tgz kind-cluster-create
+	KUBECONFIG=$(KIND_KUBECONFIG) ARCH=$(ARCH) GIT_VERSION=$(GIT_VERSION) ./tests/k8st/deploy_resources_on_kind_cluster.sh
 
 .PHONY: kind-k8st-run-test
 kind-k8st-run-test: .calico_test.created $(KIND_KUBECONFIG)

node/tests/k8st/deploy_resources_on_kind_cluster.sh

@@ -12,69 +12,24 @@ trap 'cleanup' SIGINT SIGHUP SIGTERM EXIT
 # test directory.
 TEST_DIR=./tests/k8st
 ARCH=${ARCH:-amd64}
+GIT_VERSION=${GIT_VERSION:-`git describe --tags --dirty --always --abbrev=12`}
+HELM=../bin/helm
+CHART=../bin/tigera-operator-$GIT_VERSION.tgz
 
 # kubectl binary.
 : ${kubectl:=../hack/test/kind/kubectl}
 
-function checkModule() {
-  MODULE="$1"
-  echo "Checking kernel module $MODULE ..."
-  if lsmod | grep "$MODULE" &>/dev/null; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-function enable_dual_stack() {
-  # Based on instructions in http://docs.projectcalico.org/master/networking/dual-stack.md
-  local yaml=$1
-  # add assign_ipv4 and assign_ipv6 to CNI config
-  sed -i -e '/"type": "calico-ipam"/r /dev/stdin' "${yaml}" <<EOF
-              "assign_ipv4": "true",
-              "assign_ipv6": "true"
-EOF
-  sed -i -e 's/"type": "calico-ipam"/"type": "calico-ipam",/' "${yaml}"
-
-  sed -i -e '/"type": "calico"/r /dev/stdin' "${yaml}" <<EOF
-     "feature_control": {
-         "floating_ips": true
-     },
-EOF
-
-  # And add all the IPV6 env vars
-  sed -i '/# Enable IPIP/r /dev/stdin' "${yaml}" <<EOF
-            - name: IP6
-              value: "autodetect"
-            - name: CALICO_IPV6POOL_CIDR
-              value: "fd00:10:244::/64"
-EOF
-  # update FELIX_IPV6SUPPORT=true
-  sed -i '/FELIX_IPV6SUPPORT/!b;n;c\              value: "true"' "${yaml}"
-}
-
 echo "Set ipv6 address on each node"
 docker exec kind-control-plane ip -6 addr replace 2001:20::8/64 dev eth0
 docker exec kind-worker ip -6 addr replace 2001:20::1/64 dev eth0
 docker exec kind-worker2 ip -6 addr replace 2001:20::2/64 dev eth0
 docker exec kind-worker3 ip -6 addr replace 2001:20::3/64 dev eth0
+
 echo
 
 echo "Load calico/node docker images onto each node"
 $TEST_DIR/load_images_on_kind_cluster.sh
 
-echo "Install Calico and Calicoctl for dualstack"
-cp $TEST_DIR/infra/calico-kdd.yaml $TEST_DIR/infra/calico.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/calico.yaml.tmp
-enable_dual_stack $TEST_DIR/infra/calico.yaml.tmp
-${kubectl} apply -f $TEST_DIR/infra/calico.yaml.tmp
-rm $TEST_DIR/infra/calico.yaml.tmp
-cp $TEST_DIR/infra/calicoctl.yaml $TEST_DIR/infra/calicoctl.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/calicoctl.yaml.tmp
-${kubectl} apply -f $TEST_DIR/infra/calicoctl.yaml.tmp
-rm $TEST_DIR/infra/calicoctl.yaml.tmp
-echo
-
 echo "Install additional permissions for BGP password"
 ${kubectl} apply -f $TEST_DIR/infra/additional-rbac.yaml
 echo
@@ -114,10 +69,30 @@ function wait_pod_ready() {
   return $rc
 }
 
+echo "Install Calico using the helm chart"
+$HELM install calico $CHART -f $TEST_DIR/infra/values.yaml -n tigera-operator --create-namespace
+
+echo "Install calicoctl as a pod"
+${kubectl} apply -f $TEST_DIR/infra/calicoctl.yaml
+echo
+
 echo "Wait for Calico to be ready..."
-wait_pod_ready -l k8s-app=calico-node -n kube-system
-wait_pod_ready -l k8s-app=calico-kube-controllers -n kube-system
-wait_pod_ready -l k8s-app=kube-dns -n kube-system
+while ! time ${kubectl} wait pod -l k8s-app=calico-node --for=condition=Ready -n calico-system --timeout=300s; do
+    # This happens when no matching resources exist yet, i.e., after installing the operator but before it deploys resources.
+    sleep 5
+done
+
+while ! time ${kubectl} wait pod -l k8s-app=calico-kube-controllers --for=condition=Ready -n calico-system --timeout=300s; do
+    # This happens when no matching resources exist yet, i.e., after installing the operator but before it deploys resources.
+    sleep 5
+done
+
+while ! time ${kubectl} wait pod -l k8s-app=calico-apiserver --for=condition=Ready -n calico-apiserver --timeout=30s; do
+    # This happens when no matching resources exist yet, i.e., after installing the operator but before it deploys resources.
+    sleep 5
+done
+
+time ${kubectl} wait pod -l k8s-app=kube-dns --for=condition=Ready -n kube-system --timeout=300s
 echo "Calico is running."
 echo
 
@@ -136,19 +111,7 @@ wait_pod_ready -l app=webserver
 echo "client and webserver pods are running."
 echo
 
-echo "Deploy Calico apiserver"
-cp $TEST_DIR/infra/apiserver.yaml $TEST_DIR/infra/apiserver.yaml.tmp
-sed -i "s/amd64/${ARCH}/" $TEST_DIR/infra/apiserver.yaml.tmp
-${kubectl} apply -f ${TEST_DIR}/infra/apiserver.yaml.tmp
-rm $TEST_DIR/infra/apiserver.yaml.tmp
-openssl req -x509 -nodes -newkey rsa:4096 -keyout apiserver.key -out apiserver.crt -days 365 -subj "/" -addext "subjectAltName = DNS:calico-api.calico-apiserver.svc"
-${kubectl} get secret -n calico-apiserver calico-apiserver-certs ||
-  ${kubectl} create secret -n calico-apiserver generic calico-apiserver-certs --from-file=apiserver.key --from-file=apiserver.crt
-${kubectl} patch apiservice v3.projectcalico.org -p \
-  "{\"spec\": {\"caBundle\": \"$(${kubectl} get secret -n calico-apiserver calico-apiserver-certs -o go-template='{{ index .data "apiserver.crt" }}')\"}}"
-wait_pod_ready -l k8s-app=calico-apiserver -n calico-apiserver
-echo "Calico apiserver is running."
-
+# Show all the pods running for diags purposes.
 ${kubectl} get po --all-namespaces -o wide
 ${kubectl} get svc
 
@@ -164,3 +127,8 @@ function test_connection() {
 }
 test_connection 4
 test_connection 6
+
+# At the end of it all, scale down the operator so that it doesn't
+# make changes to the cluster. Some of our tests modify calico/node, etc.
+# We should remove this once we fix up those tests.
+kubectl scale deployment -n tigera-operator tigera-operator --replicas=0

node/tests/k8st/infra/build-operator.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# This script produces a custom operator build that is used
+# in the ST and e2e tests in this repo.
+
+# Clone the repository if needed.
+set -e
+REPO=${REPO:-tigera/operator}
+BRANCH=${BRANCH:-master}
+echo "Cloning https://github.com/${REPO} @ ${BRANCH}"
+git clone --depth=1 https://github.com/${REPO} -b ${BRANCH} operator
+
+# Modify the versions that are in-use to match our locally built images.
+pushd operator
+make build/_output/bin/gen-versions
+build/_output/bin/gen-versions -os-versions=../calico_versions.yml > pkg/components/calico.go
+
+# Modify pull policy to be "Never".
+find . -name '*.go' | xargs sed -i 's/PullIfNotPresent/PullNever/g'
+
+# Build an operator image for us to use and tag it with a local-only tag.
+make image
+docker tag tigera/operator:latest docker.io/tigera/operator:test-build
+
+# Clean up after ourselves.
+popd
+rm -rf operator/
+set +e