Implement dump-only mode

And by the way, waiting for the informer's caches
to be fully dumped before stopping controllers is
much saner.

Author: bpineau

README.md

@@ -10,21 +10,27 @@ This provides real time, continuous backups, and keeps detailled changes history
 
 ## Usage
 
+Just dump the cluster's content and exit:
 ```bash
-katafygio --git-url https://user:[email protected]/myorg/myrepos.git --local-dir /tmp/kfdump
+katafygio --dump-only --local-dir /tmp/clusterdump/
+```
+
+Create a local git repository in /tmp/kfdump and continuously save the cluster content:
+```bash
+katafygio --local-dir /tmp/kfdump
 ```
 
-The remote git url is optional: if not specified, katafygio will create a local
-repository in the target dump directory. katafygio may be used that way to
-quickly dump a cluster's content.
+Same, but also continously push the local repository to a remote:
+```bash
+katafygio --git-url https://user:[email protected]/myorg/myrepos.git --local-dir /tmp/kfdump
+```
 
-Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-x` and `-y`
-will help to keep memory usage low. Same for low value and frequently changing
-objects, that may bloat the git history.
+Filtering out irrelevant objects (esp. ReplicaSets and Pods) with `-x` or `-y`
+will help to keep resources usage low and git history concise.
 
 
 ```bash
-# Filtering out replicasets and pods since they are managed by Deployments
+# Filtering out replicasets and pods since they are generated by Deployments
 # (already archived), endpoints (managed by Services), secrets (to keep them
 # confidential), events and node (irrelevant), and the leader-elector
 # configmap that has low value and changes a lot, causing commits churn.
@@ -55,6 +61,7 @@ Flags:
   -s, --api-server string            Kubernetes api-server url
   -c, --config string                Configuration file (default "/etc/katafygio/katafygio.yaml")
   -d, --dry-run                      Dry-run mode: don't store anything.
+  -m, --dump-only                    Dump mode: dump everything and exit
   -x, --exclude-kind stringSlice     Ressource kind to exclude. Eg. 'deployment'
   -y, --exclude-object stringSlice   Object to exclude. Eg. 'configmap:kube-system/kube-dns'
   -l, --filter string                Label filter. Select only objects matching the label.

cmd/execute.go

@@ -25,6 +25,7 @@ var (
 	apiServer string
 	kubeConf  string
 	dryRun    bool
+	dumpMode  bool
 	logLevel  string
 	logOutput string
 	logServer string
@@ -54,6 +55,7 @@ var (
 		RunE: func(cmd *cobra.Command, args []string) error {
 			conf := &config.KfConfig{
 				DryRun:        viper.GetBool("dry-run"),
+				DumpMode:      viper.GetBool("dump-only"),
 				Logger:        klog.New(viper.GetString("log.level"), viper.GetString("log.server"), viper.GetString("log.output")),
 				LocalDir:      viper.GetString("local-dir"),
 				GitURL:        viper.GetString("git-url"),
@@ -105,6 +107,9 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", false, "Dry-run mode: don't store anything.")
 	bindPFlag("dry-run", "dry-run")
 
+	RootCmd.PersistentFlags().BoolVarP(&dumpMode, "dump-only", "m", false, "Dump mode: dump everything and exit")
+	bindPFlag("dump-only", "dump-only")
+
 	RootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "v", "info", "Log level")
 	bindPFlag("log.level", "log-level")
 

config/config.go

@@ -15,6 +15,9 @@ type KfConfig struct {
 	// When DryRun is true, we don't write to disk and we don't commit/push
 	DryRun bool
 
+	// When DumpMode is true, we just dump everything once and exit
+	DumpMode bool
+
 	// Logger should be used to send all logs
 	Logger *logrus.Logger
 

pkg/controller/controller.go

@@ -25,7 +25,10 @@ import (
 	"github.com/ghodss/yaml"
 )
 
-const maxProcessRetry = 6
+var (
+	maxProcessRetry = 6
+	canaryKey       = "$katafygio-canary"
+)
 
 // Interface describe a standard kubernetes controller
 type Interface interface {
@@ -41,6 +44,7 @@ type Controller struct {
 	name     string
 	stopCh   chan struct{}
 	doneCh   chan struct{}
+	syncCh   chan struct{}
 	notifier event.Notifier
 	config   *config.KfConfig
 	queue    workqueue.RateLimitingInterface
@@ -93,6 +97,7 @@ func New(client cache.ListerWatcher, notifier event.Notifier, name string, confi
 	return &Controller{
 		stopCh:   make(chan struct{}),
 		doneCh:   make(chan struct{}),
+		syncCh:   make(chan struct{}, 1),
 		notifier: notifier,
 		name:     name,
 		config:   config,
@@ -113,15 +118,18 @@ func (c *Controller) Start() {
 		return
 	}
 
+	c.queue.Add(canaryKey)
+
 	go wait.Until(c.runWorker, time.Second, c.stopCh)
 }
 
 // Stop halts the controller
 func (c *Controller) Stop() {
+	c.config.Logger.Infof("Stopping %s controller", c.name)
+	<-c.syncCh
 	close(c.stopCh)
 	c.queue.ShutDown()
 	<-c.doneCh
-	c.config.Logger.Infof("Stopping %s controller", c.name)
 }
 
 func (c *Controller) runWorker() {
@@ -138,6 +146,13 @@ func (c *Controller) processNextItem() bool {
 	}
 	defer c.queue.Done(key)
 
+	if strings.Compare(key.(string), canaryKey) == 0 {
+		c.config.Logger.Infof("Initial sync completed for %s controller", c.name)
+		c.syncCh <- struct{}{}
+		c.queue.Forget(key)
+		return true
+	}
+
 	err := c.processItem(key.(string))
 
 	if err == nil {

pkg/run/run.go

@@ -31,7 +31,9 @@ func Run(config *config.KfConfig) {
 	sigterm := make(chan os.Signal, 1)
 	signal.Notify(sigterm, syscall.SIGTERM)
 	signal.Notify(sigterm, syscall.SIGINT)
-	<-sigterm
+	if !config.DumpMode {
+		<-sigterm
+	}
 
 	obsv.Stop()
 	repo.Stop()