unregister metrics before deletion

Signed-off-by: Bruce Ma <[email protected]>

Author: mars1024

pkg/constants/finalizers.go

@@ -16,5 +16,10 @@
 
 package constants
 
-const FinalizerIPAllocated = "networking.alibaba.com/ip-allocated"
-const FinalizerManagerRuntimeRegistered = "multicluster.alibaba.com/manager-runtime-registered"
+const (
+	FinalizerIPAllocated = "networking.alibaba.com/ip-allocated"
+
+	FinalizerManagerRuntimeRegistered = "multicluster.alibaba.com/manager-runtime-registered"
+
+	FinalizerMetricsRegistered = "networking.alibaba.com/metrics-registered"
+)

pkg/controllers/networking/networkstatus_controller.go

@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
@@ -85,6 +86,17 @@ func (r *NetworkStatusReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, wrapError("unable to fetch Network", client.IgnoreNotFound(err))
 	}
 
+	if network.DeletionTimestamp != nil {
+		cleanMetrics(network.Name)
+		return ctrl.Result{}, wrapError("unable to remove finalizer", r.removeFinalizer(ctx, network))
+
+	}
+
+	// make sure metrics will be un-registered before deletion
+	if err = r.addFinalizer(ctx, network); err != nil {
+		return ctrl.Result{}, wrapError("unable to add finalizer to network", err)
+	}
+
 	nodeSelector := network.Spec.NodeSelector
 	switch networkingv1.GetNetworkType(network) {
 	case networkingv1.NetworkTypeGlobalBGP:
@@ -187,15 +199,54 @@ func updateUsageMetrics(networkName string, networkStatus *networkingv1.NetworkS
 	}
 }
 
+func cleanMetrics(networkName string) {
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv4, metrics.IPTotalUsageType)
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv4, metrics.IPUsedUsageType)
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv4, metrics.IPAvailableUsageType)
+
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv6, metrics.IPTotalUsageType)
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv6, metrics.IPUsedUsageType)
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.IPv6, metrics.IPAvailableUsageType)
+
+	_ = metrics.IPUsageGauge.DeleteLabelValues(networkName, metrics.DualStack, metrics.IPAvailableUsageType)
+}
+
+func (r *NetworkStatusReconciler) addFinalizer(ctx context.Context, network *networkingv1.Network) error {
+	if controllerutil.ContainsFinalizer(network, constants.FinalizerMetricsRegistered) {
+		return nil
+	}
+
+	patch := client.MergeFrom(network.DeepCopy())
+	controllerutil.AddFinalizer(network, constants.FinalizerMetricsRegistered)
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return r.Patch(ctx, network, patch)
+	})
+}
+
+func (r *NetworkStatusReconciler) removeFinalizer(ctx context.Context, network *networkingv1.Network) error {
+	if !controllerutil.ContainsFinalizer(network, constants.FinalizerMetricsRegistered) {
+		return nil
+	}
+
+	patch := client.MergeFrom(network.DeepCopy())
+	controllerutil.RemoveFinalizer(network, constants.FinalizerMetricsRegistered)
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return r.Patch(ctx, network, patch)
+	})
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *NetworkStatusReconciler) SetupWithManager(mgr ctrl.Manager) (err error) {
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(ControllerNetworkStatus).
 		For(&networkingv1.Network{},
 			builder.WithPredicates(
-				&predicate.GenerationChangedPredicate{},
 				&utils.IgnoreDeletePredicate{},
-				&utils.NetworkSpecChangePredicate{},
+				&predicate.ResourceVersionChangedPredicate{},
+				predicate.Or(
+					&utils.NetworkSpecChangePredicate{},
+					&utils.TerminatingPredicate{},
+				),
 			)).
 		Watches(&source.Kind{Type: &networkingv1.Subnet{}},
 			handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {

pkg/controllers/networking/subnetstatus_controller.go

@@ -30,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
@@ -38,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	networkingv1 "github.com/alibaba/hybridnet/pkg/apis/networking/v1"
+	"github.com/alibaba/hybridnet/pkg/constants"
 	"github.com/alibaba/hybridnet/pkg/controllers/concurrency"
 	"github.com/alibaba/hybridnet/pkg/controllers/utils"
 	ipamtypes "github.com/alibaba/hybridnet/pkg/ipam/types"
@@ -80,6 +82,17 @@ func (r *SubnetStatusReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, wrapError("unable to fetch Subnet", client.IgnoreNotFound(err))
 	}
 
+	if subnet.DeletionTimestamp != nil {
+		cleanSubnetMetrics(subnet.Spec.Network, subnet.Name)
+		return ctrl.Result{}, wrapError("unable to remove finalizer", r.removeFinalizer(ctx, subnet))
+
+	}
+
+	// make sure metrics will be un-registered before deletion
+	if err = r.addFinalizer(ctx, subnet); err != nil {
+		return ctrl.Result{}, wrapError("unable to add finalizer to subnet", err)
+	}
+
 	// fetch subnet usage from manager
 	var usage *ipamtypes.Usage
 	if usage, err = r.IPAMManager.GetSubnetUsage(subnet.Spec.Network, subnet.Name); err != nil {
@@ -147,15 +160,65 @@ func updateSubnetUsageMetrics(networkName, subnetName string, subnetStatus *netw
 	}
 }
 
+func cleanSubnetMetrics(networkName, subnetName string) {
+	_ = metrics.SubnetIPUsageGauge.Delete(
+		prometheus.Labels{
+			"subnetName":  subnetName,
+			"networkName": networkName,
+			"usageType":   metrics.IPTotalUsageType,
+		})
+
+	_ = metrics.SubnetIPUsageGauge.Delete(
+		prometheus.Labels{
+			"subnetName":  subnetName,
+			"networkName": networkName,
+			"usageType":   metrics.IPUsedUsageType,
+		})
+
+	_ = metrics.SubnetIPUsageGauge.Delete(
+		prometheus.Labels{
+			"subnetName":  subnetName,
+			"networkName": networkName,
+			"usageType":   metrics.IPAvailableUsageType,
+		})
+}
+
+func (r *SubnetStatusReconciler) addFinalizer(ctx context.Context, subnet *networkingv1.Subnet) error {
+	if controllerutil.ContainsFinalizer(subnet, constants.FinalizerMetricsRegistered) {
+		return nil
+	}
+
+	patch := client.MergeFrom(subnet.DeepCopy())
+	controllerutil.AddFinalizer(subnet, constants.FinalizerMetricsRegistered)
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return r.Patch(ctx, subnet, patch)
+	})
+}
+
+func (r *SubnetStatusReconciler) removeFinalizer(ctx context.Context, subnet *networkingv1.Subnet) error {
+	if !controllerutil.ContainsFinalizer(subnet, constants.FinalizerMetricsRegistered) {
+		return nil
+	}
+
+	patch := client.MergeFrom(subnet.DeepCopy())
+	controllerutil.RemoveFinalizer(subnet, constants.FinalizerMetricsRegistered)
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return r.Patch(ctx, subnet, patch)
+	})
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *SubnetStatusReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(ControllerSubnetStatus).
 		For(&networkingv1.Subnet{},
 			builder.WithPredicates(
-				&predicate.GenerationChangedPredicate{},
 				&utils.IgnoreDeletePredicate{},
-				&utils.SubnetSpecChangePredicate{},
+				&predicate.ResourceVersionChangedPredicate{},
+				predicate.Or(
+					&utils.SubnetSpecChangePredicate{},
+					&utils.TerminatingPredicate{},
+				),
 			)).
 		Watches(&source.Kind{Type: &networkingv1.IPInstance{}},
 			handler.EnqueueRequestsFromMapFunc(func(object client.Object) []reconcile.Request {