Graceful upgrade of addons-manager

- Pauses lifecycle management of addons-manager package
- NoopDelete addons-manager packageinstall
- Deletes adoons-manager addon secret

Author: Adolfo Duarte

pkg/v1/tkg/managementcomponents/management_component_install.go

@@ -5,14 +5,20 @@ package managementcomponents
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"os"
 	"time"
 
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
 	crtclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kappipkg "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
 
@@ -23,6 +29,11 @@ import (
 	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkg/tkgpackagedatamodel"
 )
 
+const (
+	addonsManagerName = "addons-manager"
+	addonFinalizer    = "tkg.tanzu.vmware.com/addon"
+)
+
 // ClusterOptions specifies cluster configuration
 type ClusterOptions struct {
 	Kubeconfig  string
@@ -49,18 +60,183 @@ type ManagementComponentsInstallOptions struct {
 	ManagementPackageRepositoryOptions ManagementPackageRepositoryOptions
 }
 
+func generateAddonsPkgiName(addonName string) string {
+	return fmt.Sprintf("tanzu-%s", addonName)
+}
+
+func generateAddonSecretName(clusterName, addonName string) string {
+	return fmt.Sprintf("%s-%s-addon", clusterName, generateAddonsPkgiName(addonName))
+}
+
+func pauseAddonSecretReconciliation(ctx context.Context, clusterClient clusterclient.Client, clusterName, addonSecreteName, namespace string) error {
+	secret := &corev1.Secret{}
+	jsonPatch := []map[string]interface{}{
+		{
+			"op":    "add",
+			"path":  fmt.Sprintf("/metadata/annotations/tkg.tanzu.vmware.com~1addon-paused"),
+			"value": "",
+		},
+	}
+	payloadBytes, err := json.Marshal(jsonPatch)
+	if err != nil {
+		return errors.Wrap(err, "unable to generate json patch")
+	}
+
+	err = clusterClient.PatchResource(secret, addonSecreteName, namespace, string(payloadBytes), types.JSONPatchType, nil)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to pause %s secret reconciliation", addonSecreteName)
+	}
+	return nil
+}
+
+func pausePackageInstallReconciliation(ctx context.Context, clusterClient clusterclient.Client, pkgiName, namespace string) error {
+	pkgi := &kappipkg.PackageInstall{}
+	jsonPatch := []map[string]interface{}{
+		{
+			"op":    "add",
+			"path":  "/spec/paused",
+			"value": true,
+		},
+	}
+	payloadBytes, err := json.Marshal(jsonPatch)
+	if err != nil {
+		return errors.Wrap(err, "unable to generate json patch")
+	}
+	err = clusterClient.PatchResource(pkgi, pkgiName, namespace, string(payloadBytes), types.JSONPatchType, nil)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to pause %s packageinstall reconciliation", pkgiName)
+	}
+	return nil
+}
+
+// PausedAdoonLifecycleManagement pauses/unpauses the lifecycle management of addon package with given name and namespace
+func PauseAddonLifecycleManagement(clusterName, addonName, namespace string, clusterClient clusterclient.Client) error {
+	pkgiName := generateAddonsPkgiName(addonsManagerName)
+	addonSecretName := generateAddonSecretName(clusterName, addonsManagerName)
+
+	err := pauseAddonSecretReconciliation(context.TODO(), clusterClient, clusterName, addonSecretName, namespace)
+	if err != nil {
+		return err
+	}
+
+	err = pausePackageInstallReconciliation(context.TODO(), clusterClient, pkgiName, namespace)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NoopDeletePackageInstall sets spec.noopdelete = true before deleting the package install
+func NoopDeletePackageInstall(addonName, namespace string, clusterClient clusterclient.Client) error {
+	pkgiName := fmt.Sprintf("tanzu-%s", addonName)
+
+	jsonPatch := []map[string]interface{}{
+		{
+			"op":    "add",
+			"path":  "/spec/noopDelete",
+			"value": true,
+		},
+	}
+	payloadBytes, err := json.Marshal(jsonPatch)
+	if err != nil {
+		return errors.Wrap(err, "unable to generate json patch")
+	}
+	pkgi := &kappipkg.PackageInstall{}
+	err = clusterClient.PatchResource(pkgi, pkgiName, namespace, string(payloadBytes), types.JSONPatchType, nil)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to patch %s packageinstall", pkgiName)
+	}
+
+	pkgi.Name = pkgiName
+	pkgi.Namespace = namespace
+	err = clusterClient.DeleteResource(pkgi)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to delete PackageInstall resource %s", pkgiName)
+	}
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	return nil
+}
+
+func DeleteAddonSecret(clusterName, addonName, namespace string, clusterClient clusterclient.Client) error {
+	addonSecret := &corev1.Secret{}
+	addonSecret.Name = generateAddonSecretName(clusterName, addonName)
+	addonSecret.Namespace = constants.TkgNamespace
+	err := clusterClient.GetResource(addonSecret, addonSecret.Name, addonSecret.Namespace, nil, nil)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	if controllerutil.ContainsFinalizer(addonSecret, addonFinalizer) {
+		controllerutil.RemoveFinalizer(addonSecret, addonFinalizer)
+	}
+	err = clusterClient.UpdateResource(addonSecret, addonSecret.Name, addonSecret.Namespace)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	err = clusterClient.DeleteResource(addonSecret)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to delete addon addonSecret %s", addonSecret)
+	}
+	return nil
+}
+
 // InstallManagementComponents installs the management component to cluster
 func InstallManagementComponents(mcip *ManagementComponentsInstallOptions) error {
 	clusterClient, err := clusterclient.NewClient(mcip.ClusterOptions.Kubeconfig, mcip.ClusterOptions.Kubecontext, clusterclient.Options{})
 	if err != nil {
 		return errors.Wrap(err, "unable to get cluster client")
 	}
+	clusterName, err := clusterClient.GetCurrentClusterName(mcip.ClusterOptions.Kubecontext)
+	if err != nil {
+		return errors.Wrap(err, "unable to get cluster name")
+	}
+	if err != nil {
+		return err
+	}
 
-	// create package client
-	pkgClient, err := tkgpackageclient.NewTKGPackageClientForContext(mcip.ClusterOptions.Kubeconfig, mcip.ClusterOptions.Kubecontext)
+	err = PauseAddonLifecycleManagement(clusterName, addonsManagerName, constants.TkgNamespace, clusterClient)
+	if err != nil {
+		panic(err)
+	}
+
+	err = NoopDeletePackageInstall(addonsManagerName, constants.TkgNamespace, clusterClient)
+	if err != nil {
+		return nil
+	}
+
+	err = DeleteAddonSecret(clusterName, addonsManagerName, constants.TkgNamespace, clusterClient)
 	if err != nil {
 		return err
 	}
+
+	// create package client
+	pkgClient, err := tkgpackageclient.NewTKGPackageClientForContext(mcip.ClusterOptions.Kubeconfig, mcip.ClusterOptions.Kubecontext)
+
 	if err = InstallManagementPackages(pkgClient, mcip.ManagementPackageRepositoryOptions); err != nil {
 		// instead of throwing error here, wait for some additional time for packages to get reconciled successfully
 		// error will be thrown at the next step if packages are not reconciled after timeout value