fix: Use specific service account token secret

With `LegacyServiceAccountTokenNoAutoGeneration` feature graduating to beta in
k8s v1.24, a token is not automatically generated when a service account is created.
This commit fixes this by requesting a specific service token account secret which
is populated by the token controller. This is backwards compatible with previous
versions of Kubernetes.

This commit also upgrades to use k8s v1.24 for both KinD image and for envtest to
ensure this works with latest version of Kubernetes.

Author: jimmidyson

Makefile

@@ -76,7 +76,9 @@ all: container hyperfed controller kubefedctl webhook e2e
 
 # Unit tests
 test:
-	go test $(TEST_PKGS)
+	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	source <(setup-envtest use -i -p env 1.24.x) && \
+		go test $(TEST_PKGS)
 
 build: hyperfed controller kubefedctl webhook
 

docs/environments/kind.md

@@ -43,17 +43,17 @@ script if you'd like to change the default:
 NUM_CLUSTERS=<num> ./scripts/create-clusters.sh
 ```
 
-The `KIND_TAG` is `v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6` by default.
+The `KIND_TAG` is `v1.24.2@sha256:1f0cee2282f43150b52dc7933183ed96abdcfc8d293f30ec07082495874876f1` by default.
 Image `kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6` is used as
 node docker image for booting the cluster.
 
 You can use `KIND_IMAGE` or `KIND_TAG` to specify the image as you want.
 ```bash
-KIND_TAG=v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b ./scripts/create-clusters.sh
+KIND_TAG=v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 ./scripts/create-clusters.sh
 ```
 
 ```bash
-KIND_IMAGE=kindest/node:v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b ./scripts/create-clusters.sh
+KIND_IMAGE=kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 ./scripts/create-clusters.sh
 ```
 
 ## Delete Clusters

pkg/kubefedctl/join.go

@@ -37,6 +37,7 @@ import (
 	kubeclient "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/pointer"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1"
@@ -417,8 +418,8 @@ func createKubeFedNamespace(clusterClientset kubeclient.Interface, kubefedNamesp
 	return fedNamespace, nil
 }
 
-// createAuthorizedServiceAccount creates a service account and grants
-// the privileges required by the KubeFed control plane to manage
+// createAuthorizedServiceAccount creates a service account and service account token secret
+// and grants the privileges required by the KubeFed control plane to manage
 // resources in the joining cluster.  The name of the created service
 // account is returned on success.
 func createAuthorizedServiceAccount(joiningClusterClientset kubeclient.Interface,
@@ -436,6 +437,16 @@ func createAuthorizedServiceAccount(joiningClusterClientset kubeclient.Interface
 
 	klog.V(2).Infof("Created service account: %s in joining cluster: %s", saName, joiningClusterName)
 
+	secretName, err := createServiceAccountTokenSecret(saName, joiningClusterClientset, namespace,
+		joiningClusterName, hostClusterName, dryRun, errorOnExisting)
+	if err != nil {
+		klog.V(2).Infof("Error creating service account: %s in joining cluster: %s due to: %v",
+			saName, joiningClusterName, err)
+		return "", err
+	}
+
+	klog.V(2).Infof("Created service account token secret: %s in joining cluster: %s", secretName, joiningClusterName)
+
 	if scope == apiextv1.NamespaceScoped {
 		klog.V(2).Infof("Creating role and binding for service account: %s in joining cluster: %s", saName, joiningClusterName)
 
@@ -487,7 +498,11 @@ func createServiceAccount(clusterClientset kubeclient.Interface, namespace,
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      saName,
 			Namespace: namespace,
+			Annotations: map[string]string{
+				"kubernetes.io/enforce-mountable-secrets": "true",
+			},
 		},
+		AutomountServiceAccountToken: pointer.Bool(false),
 	}
 
 	if dryRun {
@@ -508,6 +523,43 @@ func createServiceAccount(clusterClientset kubeclient.Interface, namespace,
 	default:
 		return saName, nil
 	}
+} // createServiceAccount creates a service account in the cluster associated
+// with clusterClientset with credentials that will be used by the host cluster
+// to access its API server.
+func createServiceAccountTokenSecret(saName string, clusterClientset kubeclient.Interface, namespace,
+	joiningClusterName, hostClusterName string, dryRun, errorOnExisting bool) (string, error) {
+	saTokenSecretName := util.ClusterServiceAccountTokenSecretName(joiningClusterName, hostClusterName)
+	saTokenSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      saTokenSecretName,
+			Namespace: namespace,
+			Annotations: map[string]string{
+				"kubernetes.io/service-account.name": saName,
+			},
+		},
+		Type: corev1.SecretTypeServiceAccountToken,
+	}
+
+	if dryRun {
+		return saName, nil
+	}
+
+	// Create a new service account.
+	_, err := clusterClientset.CoreV1().Secrets(namespace).Create(
+		context.Background(), saTokenSecret, metav1.CreateOptions{},
+	)
+	switch {
+	case apierrors.IsAlreadyExists(err) && errorOnExisting:
+		klog.V(2).Infof("Service account token secret %s/%s already exists in target cluster %s",
+			namespace, saName, joiningClusterName)
+		return "", err
+	case err != nil && !apierrors.IsAlreadyExists(err):
+		klog.V(2).Infof("Could not create service account token secret %s/%s in target cluster %s due to: %v",
+			namespace, saName, joiningClusterName, err)
+		return "", err
+	default:
+		return saTokenSecretName, nil
+	}
 }
 
 func bindingSubjects(saName, namespace string) []rbacv1.Subject {
@@ -841,32 +893,20 @@ func populateSecretInHostCluster(clusterClientset, hostClientset kubeclient.Inte
 	// Get the secret from the joining cluster.
 	var secret *corev1.Secret
 	err := wait.PollImmediate(1*time.Second, serviceAccountSecretTimeout, func() (bool, error) {
-		sa, err := clusterClientset.CoreV1().ServiceAccounts(joiningNamespace).Get(
+		joiningClusterSASecret, err := clusterClientset.CoreV1().Secrets(joiningNamespace).Get(
 			context.Background(), saName, metav1.GetOptions{},
 		)
 		if err != nil {
 			return false, nil
 		}
 
-		for _, objReference := range sa.Secrets {
-			saSecretName := objReference.Name
-			var err error
-			secret, err = clusterClientset.CoreV1().Secrets(joiningNamespace).Get(
-				context.Background(), saSecretName, metav1.GetOptions{},
-			)
-			if err != nil {
-				return false, nil
-			}
-			if secret.Type == corev1.SecretTypeServiceAccountToken {
-				klog.V(2).Infof("Using secret named: %s", secret.Name)
-				return true, nil
-			}
-		}
-		return false, nil
+		secret = joiningClusterSASecret
+
+		return true, nil
 	})
 
 	if err != nil {
-		klog.V(2).Infof("Could not get service account secret from joining cluster: %v", err)
+		klog.V(2).Infof("Could not get service account token secret from joining cluster: %v", err)
 		return nil, nil, err
 	}
 

pkg/kubefedctl/util/util.go

@@ -107,6 +107,12 @@ func ClusterServiceAccountName(joiningClusterName, hostClusterName string) strin
 	return fmt.Sprintf("%s-%s", joiningClusterName, hostClusterName)
 }
 
+// ClusterServiceAccountTokenName returns the name of a service account token secret whose
+// credentials are used by the host cluster to access the client cluster.
+func ClusterServiceAccountTokenSecretName(joiningClusterName, hostClusterName string) string {
+	return fmt.Sprintf("%s-%s", joiningClusterName, hostClusterName)
+}
+
 // RoleName returns the name of a Role or ClusterRole and its
 // associated RoleBinding or ClusterRoleBinding that are used to allow
 // the service account to access necessary resources on the cluster.

scripts/create-clusters.sh

@@ -25,7 +25,7 @@ set -o pipefail
 source "${BASH_SOURCE%/*}/util.sh"
 NUM_CLUSTERS="${NUM_CLUSTERS:-2}"
 KIND_IMAGE="${KIND_IMAGE:-}"
-KIND_TAG="${KIND_TAG:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}"
+KIND_TAG="${KIND_TAG:-v1.24.2@sha256:1f0cee2282f43150b52dc7933183ed96abdcfc8d293f30ec07082495874876f1}"
 OS="$(uname)"
 
 function create-clusters() {

scripts/download-binaries.sh

@@ -43,14 +43,11 @@ mkdir -p "${dest_dir}"
 
 platform=$(uname -s|tr A-Z a-z)
 
-kb_version="2.3.1"
-kb_tgz="kubebuilder_${kb_version}_${platform}_amd64.tar.gz"
-kb_url="https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${kb_version}/${kb_tgz}"
-curl "${curl_args}" "${kb_url}" \
-  | tar xzP -C "${dest_dir}" --strip-components=2
+go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
-export KUBEBUILDER_ASSETS="${dest_dir}"
-echo "Setting to KUBEBUILDER_ASSETS ${dest_dir}"
+source <(setup-envtest use -i -p env 1.24.x)
+
+echo "KUBEBUILDER_ASSETS is set to KUBEBUILDER_ASSETS"
 
 helm_version="3.6.0"
 helm_tgz="helm-v${helm_version}-${platform}-amd64.tar.gz"

scripts/download-e2e-binaries.sh

@@ -36,7 +36,7 @@ mkdir -p "${dest_dir}"
 
 # kind
 platform="$(uname -s|tr A-Z a-z)"
-kind_version="v0.11.1"
+kind_version="v0.14.0"
 kind_path="${dest_dir}/kind"
 kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${kind_version}/kind-${platform}-amd64"
 curl -fLo "${kind_path}" "${kind_url}" && chmod +x "${kind_path}"

scripts/pre-commit.sh

@@ -57,7 +57,7 @@ function download-dependencies() {
 }
 
 function run-unit-tests() {
-  KUBEBUILDER_ASSETS=${ROOT_DIR}/bin ${MAKE_CMD} test
+  ${MAKE_CMD} test
 }
 
 function run-e2e-tests() {
@@ -188,7 +188,7 @@ run-unit-tests
 echo "Downloading e2e test dependencies"
 ./scripts/download-e2e-binaries.sh
 
-KIND_TAG="v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6" ./scripts/create-clusters.sh
+KIND_TAG="v1.24.2@sha256:1f0cee2282f43150b52dc7933183ed96abdcfc8d293f30ec07082495874876f1" ./scripts/create-clusters.sh
 
 declare -a join_cluster_list=()
 if [[ -z "${JOIN_CLUSTERS}" ]]; then