feature: Periodically clean up invalid blade

deploy/helm/chaosblade-operator-for-v2/templates/deployment.yaml

@@ -41,6 +41,9 @@ spec:
           {{- if .Values.daemonset.enable }}
           - '--daemonset-enable'
           {{- end }}
+          {{- if .Values.remove.blade.interval }}
+          - '--remove-blade-interval={{ .Values.remove.blade.interval }}'
+          {{- end }}
           imagePullPolicy: {{ .Values.operator.pullPolicy }}
           env:
             - name: WATCH_NAMESPACE

deploy/helm/chaosblade-operator-for-v2/values.yaml

@@ -21,4 +21,8 @@ webhook:
   enable: true
 
 daemonset:
-  enable: true
+  enable: true
+
+remove:
+  blade:
+    interval: 72h

deploy/helm/chaosblade-operator-for-v3/templates/deployment.yaml

@@ -41,6 +41,9 @@ spec:
           {{- if .Values.daemonset.enable }}
           - '--daemonset-enable'
           {{- end }}
+          {{- if .Values.remove.blade.interval }}
+          - '--remove-blade-interval={{ .Values.remove.blade.interval }}'
+          {{- end }}
           imagePullPolicy: {{ .Values.operator.pullPolicy }}
           env:
             - name: WATCH_NAMESPACE

deploy/helm/chaosblade-operator-for-v3/values.yaml

@@ -22,4 +22,8 @@ webhook:
   enable: true
 
 daemonset:
-  enable: true
+  enable: true
+
+remove:
+  blade:
+    interval: 72h

pkg/controller/chaosblade/controller.go

@@ -20,6 +20,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"time"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/sirupsen/logrus"
@@ -49,7 +53,14 @@ const chaosbladeFinalizer = "finalizer.chaosblade.io"
 // Add creates a new ChaosBlade Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager) error {
-	return add(mgr, newReconciler(mgr))
+	if err := add(mgr, newReconciler(mgr)); err != nil {
+		return err
+	}
+	// add periodically clean up blade ticker
+	return mgr.Add(manager.RunnableFunc(func(s <-chan struct{}) error {
+		startPeriodicallyCleanUpBlade(mgr)
+		return nil
+	}))
 }
 
 // newReconciler returns a new reconcile.Reconciler
@@ -97,6 +108,61 @@ func add(mgr manager.Manager, rcb *ReconcileChaosBlade) error {
 	return nil
 }
 
+// if blade status is destroying
+func startPeriodicallyCleanUpBlade(mgr manager.Manager) {
+	go func() {
+		cli := mgr.GetClient()
+		duration, err := time.ParseDuration(chaosblade.RemoveBladeInterval)
+		if err != nil {
+			logrus.Errorf("parse interval error: %v, use default interval: %s", err, chaosblade.DefaultRemoveBladeInterval)
+			duration, err = time.ParseDuration(chaosblade.DefaultRemoveBladeInterval)
+			chaosblade.RemoveBladeInterval = chaosblade.DefaultRemoveBladeInterval
+			if err != nil {
+				logrus.Fatalf("start periodically clean up blade, ticker error: %v", err)
+			}
+		}
+		// first clean up
+		periodicallyCleanUpBlade(cli, duration)
+
+		// ticker clean up
+		ticker := time.NewTicker(time.Second * time.Duration(duration.Seconds()))
+		logrus.Infof("start periodically clean up blade ticker, interval: %s", chaosblade.RemoveBladeInterval)
+		for range ticker.C {
+			periodicallyCleanUpBlade(cli, duration)
+		}
+	}()
+}
+
+func periodicallyCleanUpBlade(cli client.Client, interval time.Duration) {
+	results := &v1alpha1.ChaosBladeList{}
+	if err := cli.List(context.TODO(), results, &client.ListOptions{}); err != nil {
+		logrus.Errorf("periodically clean up, list blade error: %v", err)
+	}
+	logrus.Infof("periodically clean up blade, blade size: %d", len(results.Items))
+	for _, item := range results.Items {
+		if item.DeletionTimestamp == nil {
+			continue
+		}
+		sub := time.Now().Sub(item.DeletionTimestamp.Time)
+		if item.Status.Phase == v1alpha1.ClusterPhaseDestroying && sub.Seconds() > interval.Seconds() {
+			logrus.Infof("periodically clean up blade %s, deletion time: %s", item.Name, item.DeletionTimestamp.String())
+			// patch blade
+			if err := cli.Patch(context.TODO(),
+				&v1alpha1.ChaosBlade{
+					TypeMeta: metav1.TypeMeta{
+						APIVersion: "chaosblade.io/v1alpha1",
+						Kind:       "ChaosBlade",
+					},
+					ObjectMeta: metav1.ObjectMeta{Name: item.Name},
+				},
+				client.RawPatch(types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`)),
+			); err != nil {
+				logrus.Errorf("patch blade: %s, error: %v", item.Name, err)
+			}
+		}
+	}
+}
+
 // blank assignment to verify that ReconcileChaosBlade implements reconcile.Reconciler
 var _ reconcile.Reconciler = &ReconcileChaosBlade{}
 

pkg/runtime/chaosblade/chaosblade.go

@@ -23,10 +23,11 @@ import (
 )
 
 var (
-	ImageRepository string
-	Version         string
-	PullPolicy      string
-	DaemonsetEnable bool
+	ImageRepository     string
+	Version             string
+	PullPolicy          string
+	DaemonsetEnable     bool
+	RemoveBladeInterval string
 )
 
 const (
@@ -38,6 +39,7 @@ const (
 )
 
 const DaemonsetPodName = "chaosblade-tool"
+const DefaultRemoveBladeInterval = "72h"
 
 var DaemonsetPodLabels = map[string]string{
 	"app": "chaosblade-tool",
@@ -64,6 +66,7 @@ func init() {
 	f.StringVar(&ImageRepository, "chaosblade-image-repository", "chaosbladeio/chaosblade-tool", "Image repository of chaosblade tool")
 	f.StringVar(&PullPolicy, "chaosblade-image-pull-policy", "IfNotPresent", "Pulling policy of chaosblade image, default value is IfNotPresent.")
 	f.BoolVar(&DaemonsetEnable, "daemonset-enable", false, "Deploy chaosblade daemonset to resolve chaos experiment environment of network, default value is false.")
+	f.StringVar(&RemoveBladeInterval, "remove-blade-interval", DefaultRemoveBladeInterval, "Periodically clean up blade state is destroying, default value is 24h.")
 }
 
 func FlagSet() *pflag.FlagSet {