Fix: remove all the AmbassadorInstallations

* Cleanups were not working because we were not rermoving
  all the AmbassadorInstallations.
* Simplify some code in the tests.

Signed-off-by: Alvaro Saurin <[email protected]>

Author: inercia

.travis.yml

@@ -28,12 +28,17 @@ install:
 
 jobs:
   include:
+    #####################################################
     # stage 1: lint, build and unit-tests
+    #####################################################
     - stage: Build, lint and unit tests
       name: "Build, lint and unit tests"
       script: make ci/check-format-gen ci/lint ci/build ci/test
 
+    #####################################################
     # stage 2: end-to-end tests in multiple clouds
+    #####################################################
+
     # see the ci/infra/providers/CREDENTIALS.md for details on the credentials
     - stage: End-to-end tests
       name: "e2e tests in Google Cloud (GKE)"
@@ -55,22 +60,21 @@ jobs:
       env: CLUSTER_PROVIDER="k3d"
       script: make ci/cluster-setup ci/e2e
 
+    #####################################################
     # stage 3: publish artifacts
+    #####################################################
     - stage: Publish artifacts
+      name: "Publish artifacts to GitHub"
       if: tag IS present
-      name: "Publish artifacts"
       env:
         - AMB_OPER_TAG="$TRAVIS_TAG"
-
-      # generate all the artifacts that will be published by the "deploy"
-      before_deploy:
-        - make ci/release
-        - make ci/cluster-setup
-
-      # ... and then publish those artifacts, docs, push images, etc.
       deploy:
+        - provider: script
+          on:
+            all_branches: true
+          script: make ci/release
         # publish YAMLs to GitHub releases
-        # make ci/release should have left everything in build/artifacts/*
+        # `make ci/release` should have left everything in build/artifacts/*
         - provider: releases
           on:
             all_branches: true
@@ -80,46 +84,63 @@ jobs:
           file: build/artifacts/*
           skip_cleanup: true
 
+    - name: "Publish image to Docker.io"
+      if: tag IS present
+      env:
+        - AMB_OPER_TAG="$TRAVIS_TAG"
+        - REL_REGISTRY="docker.io/datawire"
+      deploy:
         # publish images to Docker.io
         # NOTE: make sure DOCKER_USERNAME/DOCKER_PASSWORD have values for a bot
         - provider: script
           on:
             all_branches: true
-          env:
-            - REL_REGISTRY="docker.io/datawire"
-          script: make ci/publish-image
-          skip_cleanup: true
+          script: make ci/release ci/publish-image
 
+    - name: "Publish images to Quay.io"
+      if: tag IS present
+      env:
+        - AMB_OPER_TAG="$TRAVIS_TAG"
+        - DOCKER_USERNAME="$QUAY_USERNAME"
+        - DOCKER_PASSWORD="$QUAY_PASSWORD"
+        - REL_REGISTRY="quay.io/datawire"
+      deploy:
         # publish images to Quay.io
         # NOTE: make sure DOCKER_USERNAME/DOCKER_PASSWORD have values for a bot
         #       (from https://quay.io/repository/datawire/ambassador-operator?tab=settings)
         - provider: script
           on:
             all_branches: true
-          env:
-            - DOCKER_USERNAME="$QUAY_USERNAME"
-            - DOCKER_PASSWORD="$QUAY_PASSWORD"
-            - REL_REGISTRY="quay.io/datawire"
-          script: make ci/publish-image
-          skip_cleanup: true
+          script: make ci/release ci/publish-image
 
-        # publish the image to Azure
+    - name: "Publish images to Azure ACR"
+      if: tag IS present
+      env:
+        - AMB_OPER_TAG="$TRAVIS_TAG"
+        - CLUSTER_REGISTRY="datawire"
+        - CLUSTER_PROVIDER="azure"
+        - AZ_RES_GRP="DefaultResourceGroup-EUS"
+      deploy:
         - provider: script
           on:
             all_branches: true
           env:
-            - CLUSTER_REGISTRY="datawire"
-            - CLUSTER_PROVIDER="azure"
-            - AZ_RES_GRP="DefaultResourceGroup-EUS"
-          script: make ci/publish-image-cloud/azure
-          skip_cleanup: true
+          script: make ci/cluster-setup ci/release ci/publish-image-cloud/azure
 
+    - name: "Publish Helm Chart"
+      if: tag IS present
+      env:
+        - AMB_OPER_TAG="$TRAVIS_TAG"
+        - DOCKER_USERNAME="$QUAY_USERNAME"
+        - DOCKER_PASSWORD="$QUAY_PASSWORD"
+        - REL_REGISTRY="quay.io/datawire"
+      deploy:
         # publish helm chart
         # NOTE: needs AWS_ACCESS_KEY_ID, AWS_BUCKET and AWS_SECRET_ACCESS_KEY (from keybase)
         - provider: script
           on:
             all_branches: true
-          script: make ci/publish-chart
+          script: make ci/cluster-setup ci/publish-chart
 
 after_success:
   - make ci/after-success

ci/common.sh

@@ -111,7 +111,7 @@ wait_until() {
 	local start_time=$(timestamp)
 	info "Waiting for $@"
 	until timeout_from $start_time || eval "$@"; do
-		info "... still waiting"
+		info "... still waiting for condition"
 		sleep 1
 	done
 	! timeout_from $start_time
@@ -267,42 +267,30 @@ get_amb_addr() {
 	fi
 }
 
+check_amb_has_addr() {
+	test -n "$(get_amb_addr $@)"
+}
+
 # wait_amb_addr <KUBECTL_ARGS...>
 # wait untils Ambassador has an address
 wait_amb_addr() {
-	i=0
-	timeout=$DEF_WAIT_TIMEOUT
-	until [ -n "$(get_amb_addr $@)" ] || [ $i -ge $timeout ]; do
-		info "waiting for Ambassador to have an IP ($i secs elapsed)"
-		i=$((i + 1))
-		sleep 1
-	done
-
-	if [ $i -ge $timeout ]; then
+	wait_until check_amb_has_addr $@ || {
 		warn "Timeout waiting for Ambassador's IP. Current services:"
 		kubectl get services $@
-		warn "Ambassador did not get an IP after $timeout seconds"
+		warn "Ambassador IP address: $(get_amb_addr $@)"
 		return 1
-	fi
+	}
 }
 
 # wait_not_amb_addr <KUBECTL_ARGS...>
 # wait untils Ambassador does not have an address
 wait_not_amb_addr() {
-	i=0
-	timeout=$DEF_WAIT_TIMEOUT
-	until [ -z "$(get_amb_addr $@)" ] || [ $i -ge $timeout ]; do
-		info "waiting for Ambassador to have an IP ($i secs elapsed)"
-		i=$((i + 1))
-		sleep 1
-	done
-
-	if [ $i -ge $timeout ]; then
-		warn "Timeout waiting for Ambassador not not have an IP. Current services:"
+	wait_until "! check_amb_has_addr $@" || {
+		warn "Timeout waiting for Ambassador to NOT have an IP. Current services:"
 		kubectl get services $@
-		warn "Ambassador had an IP after $timeout seconds"
+		warn "Ambassador IP address: $(get_amb_addr $@)"
 		return 1
-	fi
+	}
 }
 
 # kubectl_apply_host
@@ -989,12 +977,15 @@ kube_check_resource_empty() {
 # amb_inst_delete <KUBECTL_ARGS...>
 # delete an `AmbassadorInstallation`
 amb_inst_delete() {
-	info "Removing the AmbassadorInstallation"
-	kubectl delete $@ --wait=true --ignore-not-found=true --timeout=60s \
-		ambassadorinstallations "${AMB_INSTALLATION_NAME}" ||
-		return 1
-	wait_missing "$@ ambassadorinstallations ${AMB_INSTALLATION_NAME}" || return 1
-	passed "... AmbassadorInstallation removed."
+	lst=$(kubectl get $@ ambassadorinstallations --no-headers -o custom-columns=":metadata.name" 2>/dev/null)
+	info "Removing all the AmbassadorInstallations: $lst"
+	for k in $lst; do
+		kubectl delete $KUBECTL_DELETE_ARGS $@ ambassadorinstallations "$k" || return 1
+		wait_missing "$@ ambassadorinstallations $k" || return 1
+		passed "... AmbassadorInstallation $k removed."
+	done
+	info "Current list of AmbassadorInstallations: " && kubectl get $@ ambassadorinstallations 2>/dev/null
+	return 0
 }
 
 amb_inst_describe() {
@@ -1038,21 +1029,28 @@ oper_uninstall() {
 
 	amb_inst_delete -n "$namespace" || {
 		oper_logs_dump -n "$namespace"
-		abort "could not remove AmbassadorInstallation in namespace $namespace"
+		abort "could not remove AmbassadorInstallations in namespace $namespace"
 	}
-	sleep 5
 
-	info "Removing the operator"
+	info "Removing the operator..."
 	cat_setting_image "$TOP_DIR/deploy/operator.yaml" | kubectl delete $KUBECTL_DELETE_ARGS -n "$namespace" -f -
-	kubectl delete -n "$namespace" $KUBECTL_DELETE_ARGS -f "$TOP_DIR/deploy/role_binding.yaml"
-	kubectl delete -n "$namespace" $KUBECTL_DELETE_ARGS -f "$TOP_DIR/deploy/role.yaml"
-	kubectl delete -n "$namespace" $KUBECTL_DELETE_ARGS -f "$TOP_DIR/deploy/service_account.yaml"
-	kubectl delete -n "$namespace" $KUBECTL_DELETE_ARGS -f $CRDS
+	for f in $CRDS "$TOP_DIR/deploy/role_binding.yaml" "$TOP_DIR/deploy/role.yaml" "$TOP_DIR/deploy/service_account.yaml"; do
+		info "... removing $f"
+		kubectl delete -n "$namespace" $KUBECTL_DELETE_ARGS -f $f || {
+			oper_logs_dump -n "$namespace"
+			abort "could not delete $f"
+		}
+	done
 
+	# note: it is important have deleted all the AmbassadorInstallations at this point,
+	#       otherwise the removal of the namespace will fail
 	info "Removing namespace $namespace..."
-	kubectl delete namespace $KUBECTL_DELETE_ARGS "$namespace" &&
-		wait_namespace_missing "$namespace" || abort "namespace $namespace still present"
-
+	kubectl delete namespace $KUBECTL_DELETE_ARGS "$namespace" || abort "could not delete namespace $namespace"
+	wait_namespace_missing "$namespace" || {
+		oper_logs_dump -n "$namespace"
+		kubectl get all -n "$namespace"
+		abort "namespace $namespace still present"
+	}
 	passed "... namespace $namespace removed."
 }
 
@@ -1120,6 +1118,7 @@ oper_install_helm() {
 }
 
 oper_logs_dump() {
+	kubectl logs $@ deployment/"$AMB_OPER_DEPLOY" --previous
 	kubectl logs $@ deployment/"$AMB_OPER_DEPLOY"
 }
 

tests/e2e/tests/01-install-uninstall.sh

@@ -113,7 +113,7 @@ amb_inst_check_success -n "$TEST_NAMESPACE" || {
 }
 
 info "Checking we can remove Ambassador when the AmbassadorInstallation is deleted"
-amb_inst_delete -n "$TEST_NAMESPACE" || failed "could not remove AmbassadorInstallation"
+amb_inst_delete -n "$TEST_NAMESPACE" || failed "could not remove AmbassadorInstallations"
 
 sleep 5
 

tests/e2e/tests/04-helm-install-uninstall.sh

@@ -77,7 +77,7 @@ amb_inst_check_success -n "$TEST_NAMESPACE" || failed "Success not found in Amba
 }
 
 info "Checking we can remove Ambassador when the AmbassadorInstallation is deleted"
-amb_inst_delete -n "$TEST_NAMESPACE" || failed "could not remove AmbassadorInstallation"
+amb_inst_delete -n "$TEST_NAMESPACE" || failed "could not remove the AmbassadorInstallations"
 
 sleep 5