Support filtering out objects with owner refs

There's generaly no great value in archiving objects having an owner
reference (ie. pods generated from cronjobs/daemonsets/replicasets/...),
when their owner is already archived (and able to re-generate them).

This should reduce the commit churn on busy clusters, or with many
cronjobs running at high frequency.

While at it, group exclusion filters so we don't have to pass an
increasing amount of functions arguments around.

Closes #88

Author: bpineau

README.md

@@ -25,20 +25,21 @@ To continuously push changes to a remote git repository:
 katafygio --git-url https://user:[email protected]/myorg/myrepos.git --local-dir /tmp/kfdump
 ```
 
-Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-x` or `-y`
-will help to keep resources usage low, and a concise git history. Eg.:
-
+Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-w`, `-x`, `-y`
+and `-z` is useful to keep a concise git history.
 
 ```bash
-# Filtering out replicasets and pods since they are generated by Deployments
-# (already archived), endpoints (managed by Services), secrets (to keep them
-# confidential), events and node (irrelevant), and the leader-elector
-# configmap that has low value and changes a lot, causing commits churn.
-
-katafygio -e /tmp/kfdump \
-  -g https://user:[email protected]/myorg/myrepos.git \
-  -x secret,pod,event,replicaset,node,endpoint \
-  -y configmap:kube-system/leader-elector
+# Filtering out objects having an owner reference (eg. managed pods or replicasets,
+# from Deployments, Daemonsets etc that we already archive), secrets (confidential),
+# events and nodes (irrelevant), and a configmap named "leader-elector" that has
+# low value and is causing commits churn:
+
+katafygio \
+  --local-dir /tmp/kfdump \
+  --git-url https://user:[email protected]/myorg/myrepos.git \
+  --exclude-having-owner-ref \
+  --exclude-kind secrets,events,nodes,endpoints \
+  --exclude-object configmap:kube-system/leader-elector
 ```
 
 You can also use the [docker image](https://hub.docker.com/r/bpineau/katafygio/).
@@ -47,8 +48,8 @@ You can also use the [docker image](https://hub.docker.com/r/bpineau/katafygio/)
 
 ```
 Backup Kubernetes cluster as yaml files in a git repository.
---exclude-kind (-x) and --exclude-object (-y) may be specified several times,
-or once with several comma separated values.
+--exclude-kind (-x), --exclude-object (-y) and --exclude-namespaces (-z)
+may be specified several times, or once with several comma separated values.
 
 Usage:
   katafygio [flags]
@@ -64,6 +65,7 @@ Flags:
   -q, --context string               Kubernetes configuration context
   -d, --dry-run                      Dry-run mode: don't store anything
   -m, --dump-only                    Dump mode: dump everything once and exit
+  -w, --exclude-having-owner-ref     Exclude all objects having an Owner Reference
   -x, --exclude-kind strings         Ressource kind to exclude. Eg. 'deployment'
   -z, --exclude-namespaces strings   Namespaces to exclude. Eg. 'temp.*' as regexes. This collects all namespaces and then filters them. Don't use it with the namespace flag.
   -y, --exclude-object strings       Object to exclude. Eg. 'configmap:kube-system/kube-dns'

assets/helm-chart/katafygio/Chart.yaml

@@ -5,7 +5,7 @@ name: katafygio
 home: https://github.com/bpineau/katafygio
 sources:
 - https://github.com/bpineau/katafygio
-version: 0.4.3
+version: 0.5.0
 keywords:
 - backup
 - dump

assets/helm-chart/katafygio/README.md

@@ -18,7 +18,7 @@ This chart installs a [Katafygio](https://github.com/bpineau/katafygio) deployme
 
 ## Chart Details
 
-If your backups are flooded by commits from uninteresting changes, you may filter out irrelevant objects using the `excludeKind` and `excludeObject` options.
+If your backups are flooded by commits from uninteresting changes, you may filter out irrelevant objects using the `excludeKind`, `excludeObject`, `excludeNamespaces`, and `excludeHavingOwnerRef` options.
 
 By default, the chart will dump (and version) the clusters content in /tmp/kf-dump (configurable with `localDir`).
 This can be useful as is, to keep a local and ephemeral changes history. To benefit from long term, out of cluster, and centrally reachable persistence, you may provide the address of a remote git repository (with `gitUrl`), where all changes will be pushed.
@@ -59,6 +59,8 @@ The following table lists the configurable parameters of the Katafygio chart and
 | `healthcheckPort`       | The port Katafygio will listen for health checks requests   | `8080`                               |
 | `excludeKind`           | Object kinds to ignore                                      | `{"replicaset","endpoints","event"}` |
 | `excludeObject`         | Specific objects to ignore (eg. "configmap:default/foo")    | `nil`                                |
+| `excludeNamespaces`     | List of regexps matching namespaces names to ignore         | `nil`                                |
+| `excludeHavingOwnerRef` | Ignore all objects having an Owner Reference                | `false`                              |
 | `rbac.create`           | Enable or disable RBAC roles and bindings                   | `true`                               |
 | `rbac.apiVersion`       | RBAC API version                                            | `v1`                                 |
 | `serviceAccount.create` | Whether a ServiceAccount should be created                  | `true`                               |

assets/helm-chart/katafygio/templates/deployment.yaml

@@ -62,6 +62,14 @@ spec:
             - --exclude-object={{ . }}
             {{- end }}
           {{- end }}
+          {{- if .Values.excludeNamespaces }}
+            {{- range .Values.excludeNamespaces }}
+            - --exclude-namespaces={{ . }}
+            {{- end }}
+          {{- end }}
+          {{- if .Values.excludeHavingOwnerRef }}
+            - --exclude-having-owner-ref
+          {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.healthcheckPort }}

assets/helm-chart/katafygio/values.yaml

@@ -38,15 +38,21 @@ logOutput: stdout
 
 # excludeKind is an array of excluded (not backuped) Kubernetes objects kinds.
 excludeKind:
-  - replicaset
+  - replicasets
   - endpoints
-  - event
+  - events
 
 # excludeObject is an array of specific Kubernetes objects to exclude from dumps
 # (the format is: objectkind:namespace/objectname).
 # excludeObject:
 #  - "configmap:kube-system/leader-elector"
 
+# excludeNamespaces is an array of regexp matching excluded namespaces (v0.8.2+)
+#excludeNamespaces: []
+
+# excludeHavingOwnerRef defines wether we should filter out objects having an owner reference (v0.8.2+).
+excludeHavingOwnerRef: false
+
 # resyncInterval is the interval (in seconds) between full catch-up resyncs
 # (to catch possibly missed events). Set to 0 to disable resyncs.
 resyncInterval: 300

assets/katafygio.yaml

@@ -33,22 +33,31 @@ resync-interval: 900
 # or daemonsets (which are already dumped), endpoints (managed by services,
 # already dumped), and noisy stuff (events, nodes...).
 #exclude-kind:
-#  - secret
-#  - pod
-#  - replicaset
-#  - node
-#  - event
+#  - secrets
+#  - pods
+#  - replicasets
+#  - nodes
+#  - events
 #  - endpoints
 
 # Example exclusion for specific objects:
 #exclude-object:
 #  - configmap:kube-system/datadog-leader-elector
 #  - deployment:default/testdeploy
 
+# Exclude objects (like pods, replicasets) generated/managed by other
+# objects we already archive:
+#exclude-having-owner-ref: true
+
+# Exclude namespaces matching some regular expressions:
+# exclude-namespaces:
+#  - jenkins.*
+#  - temp-.*
+
 # Only dump objects belonging to a specific namespace
 #namespace:
 
-# Set to true o dump once and exit (instead of continuously dumping new changes)
+# Set to true to dump once and exit (instead of continuously dumping new changes)
 dump-only: false
 
 # Set to true to disable git versionning

cmd/execute.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"regexp"
 	"syscall"
 
 	"github.com/spf13/afero"
@@ -31,8 +32,8 @@ var (
 		Use:   appName,
 		Short: "Backup Kubernetes cluster as yaml files",
 		Long: "Backup Kubernetes cluster as yaml files in a git repository.\n" +
-			"--exclude-kind (-x) and --exclude-object (-y) may be specified several times,\n" +
-			"or once with several comma separated values.",
+			"--exclude-kind (-x), --exclude-object (-y) and --exclude-namespaces (-z)\n" +
+			"may be specified several times, or once with several comma separated values.",
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		PreRun:        bindConf,
@@ -69,8 +70,19 @@ func runE(cmd *cobra.Command, args []string) (err error) {
 		return fmt.Errorf("failed to start git repo handler: %v", err)
 	}
 
+	exclnsre := make([]*regexp.Regexp, 0, len(exclnamespaces))
+	for _, ns := range exclnamespaces {
+		exclnsre = append(exclnsre, regexp.MustCompile(ns))
+	}
+
+	exclusions := &controller.Exclusions{
+		Names:      exclobj,
+		Namespaces: exclnsre,
+		NoOwnerRef: noOwnerRef,
+	}
+
 	evts := event.New()
-	fact := controller.NewFactory(logger, filter, resyncInt, exclobj, exclnamespaces)
+	fact := controller.NewFactory(logger, filter, resyncInt, exclusions)
 	reco := recorder.New(logger, evts, localDir, resyncInt*2, dryRun).Start()
 	obsv := observer.New(logger, restcfg, evts, fact, exclkind, namespace).Start()
 

cmd/flags.go

@@ -29,6 +29,7 @@ var (
 	exclkind       []string
 	exclobj        []string
 	noGit          bool
+	noOwnerRef     bool
 )
 
 func bindPFlag(key string, cmd string) {
@@ -89,6 +90,9 @@ func init() {
 	RootCmd.PersistentFlags().StringSliceVarP(&exclobj, "exclude-object", "y", nil, "Object to exclude. Eg. 'configmap:kube-system/kube-dns'")
 	bindPFlag("exclude-object", "exclude-object")
 
+	RootCmd.PersistentFlags().BoolVarP(&noOwnerRef, "exclude-having-owner-ref", "w", false, "Exclude all objects having an Owner Reference")
+	bindPFlag("exclude-having-owner-ref", "exclude-having-owner-ref")
+
 	RootCmd.PersistentFlags().StringVarP(&filter, "filter", "l", "", "Label filter. Select only objects matching the label")
 	bindPFlag("filter", "filter")
 
@@ -123,4 +127,5 @@ func bindConf(cmd *cobra.Command, args []string) {
 	exclkind = viper.GetStringSlice("exclude-kind")
 	exclobj = viper.GetStringSlice("exclude-object")
 	noGit = viper.GetBool("no-git")
+	noOwnerRef = viper.GetBool("exclude-having-owner-ref")
 }

pkg/controller/controller.go

@@ -42,28 +42,33 @@ type logger interface {
 	Errorf(format string, args ...interface{})
 }
 
+// Exclusions groups filters used to ignore objects
+type Exclusions struct {
+	Names      []string
+	Namespaces []*regexp.Regexp
+	NoOwnerRef bool
+}
+
 // Factory generate controllers
 type Factory struct {
-	logger      logger
-	filter      string
-	resyncIntv  time.Duration
-	excludedobj []string
-	excludedns  []string
+	logger     logger
+	filter     string
+	resyncIntv time.Duration
+	exclusions *Exclusions
 }
 
 // Controller is a generic kubernetes controller
 type Controller struct {
-	name        string
-	stopCh      chan struct{}
-	doneCh      chan struct{}
-	syncCh      chan struct{}
-	notifier    event.Notifier
-	queue       workqueue.RateLimitingInterface
-	informer    cache.SharedIndexInformer
-	logger      logger
-	resyncIntv  time.Duration
-	excludedobj []string
-	excludedns  []*regexp.Regexp
+	name       string
+	stopCh     chan struct{}
+	doneCh     chan struct{}
+	syncCh     chan struct{}
+	notifier   event.Notifier
+	queue      workqueue.RateLimitingInterface
+	informer   cache.SharedIndexInformer
+	logger     logger
+	resyncIntv time.Duration
+	exclusions *Exclusions
 }
 
 // New return a kubernetes controller using the provided client
@@ -73,8 +78,7 @@ func New(client cache.ListerWatcher,
 	name string,
 	filter string,
 	resync time.Duration,
-	excludedobj []string,
-	excludednamespace []string,
+	exclusions *Exclusions,
 ) *Controller {
 
 	selector := metav1.ListOptions{LabelSelector: filter, ResourceVersion: "0", AllowWatchBookmarks: true}
@@ -117,23 +121,17 @@ func New(client cache.ListerWatcher,
 		},
 	})
 
-	exclnsre := make([]*regexp.Regexp, 0)
-	for _, ns := range excludednamespace {
-		exclnsre = append(exclnsre, regexp.MustCompile(ns))
-	}
-
 	return &Controller{
-		stopCh:      make(chan struct{}),
-		doneCh:      make(chan struct{}),
-		syncCh:      make(chan struct{}, 1),
-		notifier:    notifier,
-		name:        name,
-		queue:       queue,
-		informer:    informer,
-		logger:      log,
-		resyncIntv:  resync,
-		excludedobj: excludedobj,
-		excludedns:  exclnsre,
+		stopCh:     make(chan struct{}),
+		doneCh:     make(chan struct{}),
+		syncCh:     make(chan struct{}, 1),
+		notifier:   notifier,
+		name:       name,
+		queue:      queue,
+		informer:   informer,
+		logger:     log,
+		resyncIntv: resync,
+		exclusions: exclusions,
 	}
 }
 
@@ -208,7 +206,7 @@ func (c *Controller) processItem(key string) error {
 		return fmt.Errorf("error fetching %s from store: %v", key, err)
 	}
 
-	for _, obj := range c.excludedobj {
+	for _, obj := range c.exclusions.Names {
 		if strings.Compare(strings.ToLower(obj), strings.ToLower(c.name+":"+key)) == 0 {
 			return nil
 		}
@@ -231,7 +229,7 @@ func (c *Controller) processItem(key string) error {
 	}
 
 	if namespace, ok := md["namespace"].(string); ok {
-		for _, nsre := range c.excludedns {
+		for _, nsre := range c.exclusions.Namespaces {
 			if nsre.MatchString(namespace) {
 				// Rely on the background sync to delete these excluded files if
 				// we previously had acquired them
@@ -240,6 +238,10 @@ func (c *Controller) processItem(key string) error {
 		}
 	}
 
+	if _, ok := md["ownerReferences"]; ok && c.exclusions.NoOwnerRef {
+		return nil
+	}
+
 	yml, err := yaml.Marshal(obj)
 	if err != nil {
 		return fmt.Errorf("failed to marshal %s: %v", key, err)
@@ -254,17 +256,16 @@ func (c *Controller) enqueue(notif *event.Notification) {
 }
 
 // NewFactory create a controller factory
-func NewFactory(logger logger, filter string, resync int, excludedobj []string, excludedns []string) *Factory {
+func NewFactory(logger logger, filter string, resync int, exclusions *Exclusions) *Factory {
 	return &Factory{
-		logger:      logger,
-		filter:      filter,
-		resyncIntv:  time.Duration(resync) * time.Second,
-		excludedobj: excludedobj,
-		excludedns:  excludedns,
+		logger:     logger,
+		filter:     filter,
+		resyncIntv: time.Duration(resync) * time.Second,
+		exclusions: exclusions,
 	}
 }
 
 // NewController create a controller.Controller
 func (f *Factory) NewController(client cache.ListerWatcher, notifier event.Notifier, name string) Interface {
-	return New(client, notifier, f.logger, name, f.filter, f.resyncIntv, f.excludedobj, f.excludedns)
+	return New(client, notifier, f.logger, name, f.filter, f.resyncIntv, f.exclusions)
 }