Add minimum age check for pod candidates

Author: Brian Akins

chaoskube/chaoskube.go

@@ -35,6 +35,8 @@ type Chaoskube struct {
 	ExcludedDaysOfYear []time.Time
 	// the timezone to apply when detecting the current weekday
 	Timezone *time.Location
+	// minimum age of pods to consider
+	MinimumAge time.Duration
 	// an instance of logrus.StdLogger to write log messages to
 	Logger log.FieldLogger
 	// dry run will not allow any pod terminations
@@ -63,7 +65,7 @@ var (
 // * a time zone to apply to the aforementioned time-based filters
 // * a logger implementing logrus.FieldLogger to send log output to
 // * whether to enable/disable dry-run mode
-func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, logger log.FieldLogger, dryRun bool) *Chaoskube {
+func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool) *Chaoskube {
 	return &Chaoskube{
 		Client:             client,
 		Labels:             labels,
@@ -73,6 +75,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel
 		ExcludedTimesOfDay: excludedTimesOfDay,
 		ExcludedDaysOfYear: excludedDaysOfYear,
 		Timezone:           timezone,
+		MinimumAge:         minimumAge,
 		Logger:             logger,
 		DryRun:             dryRun,
 		Now:                time.Now,
@@ -156,6 +159,8 @@ func (c *Chaoskube) Candidates() ([]v1.Pod, error) {
 		return nil, err
 	}
 
+	pods = filterByMinimumAge(pods, c.MinimumAge, c.Now())
+
 	return pods, nil
 }
 
@@ -251,3 +256,23 @@ func filterByAnnotations(pods []v1.Pod, annotations labels.Selector) ([]v1.Pod,
 
 	return filteredList, nil
 }
+
+// filterByMinimumAge filters pods by creation time. Only pods
+// older than minimumAge are returned
+func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) []v1.Pod {
+	if minimumAge <= time.Duration(0) {
+		return pods
+	}
+
+	creationTime := now.Add(-minimumAge)
+
+	filteredList := []v1.Pod{}
+
+	for _, pod := range pods {
+		if pod.ObjectMeta.CreationTimestamp.Time.Before(creationTime) {
+			filteredList = append(filteredList, pod)
+		}
+	}
+
+	return filteredList
+}

chaoskube/chaoskube_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/sirupsen/logrus/hooks/test"
 
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes/fake"
 
@@ -40,6 +41,7 @@ func (suite *Suite) TestNew() {
 		excludedWeekdays   = []time.Weekday{time.Friday}
 		excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
 		excludedDaysOfYear = []time.Time{time.Now()}
+		minimumAge         = time.Duration(42)
 	)
 
 	chaoskube := New(
@@ -51,6 +53,7 @@ func (suite *Suite) TestNew() {
 		excludedTimesOfDay,
 		excludedDaysOfYear,
 		time.UTC,
+		minimumAge,
 		logger,
 		false,
 	)
@@ -66,6 +69,7 @@ func (suite *Suite) TestNew() {
 	suite.Equal(time.UTC, chaoskube.Timezone)
 	suite.Equal(logger, chaoskube.Logger)
 	suite.Equal(false, chaoskube.DryRun)
+	suite.Equal(minimumAge, chaoskube.MinimumAge)
 }
 
 func (suite *Suite) TestCandidates() {
@@ -108,6 +112,7 @@ func (suite *Suite) TestCandidates() {
 			[]time.Time{},
 			time.UTC,
 			false,
+			time.Duration(42),
 		)
 
 		suite.assertCandidates(chaoskube, tt.pods)
@@ -141,6 +146,7 @@ func (suite *Suite) TestVictim() {
 			[]time.Time{},
 			time.UTC,
 			false,
+			time.Duration(42),
 		)
 
 		suite.assertVictim(chaoskube, tt.victim)
@@ -157,6 +163,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
 		[]util.TimePeriod{},
 		[]time.Time{},
 		time.UTC,
+		time.Duration(42),
 		false,
 	)
 
@@ -185,6 +192,7 @@ func (suite *Suite) TestDeletePod() {
 			[]time.Time{},
 			time.UTC,
 			tt.dryRun,
+			time.Duration(42),
 		)
 
 		victim := util.NewPod("default", "foo")
@@ -414,6 +422,7 @@ func (suite *Suite) TestTerminateVictim() {
 			tt.excludedDaysOfYear,
 			tt.timezone,
 			false,
+			time.Duration(42),
 		)
 		chaoskube.Now = tt.now
 
@@ -437,6 +446,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
 		[]util.TimePeriod{},
 		[]time.Time{},
 		time.UTC,
+		time.Duration(42),
 		false,
 	)
 
@@ -486,7 +496,7 @@ func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) {
 	}
 }
 
-func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube {
+func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool, minimumAge time.Duration) *Chaoskube {
 	chaoskube := suite.setup(
 		labelSelector,
 		annotations,
@@ -495,6 +505,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
 		excludedTimesOfDay,
 		excludedDaysOfYear,
 		timezone,
+		minimumAge,
 		dryRun,
 	)
 
@@ -511,7 +522,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
 	return chaoskube
 }
 
-func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube {
+func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube {
 	logOutput.Reset()
 
 	return New(
@@ -523,6 +534,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
 		excludedTimesOfDay,
 		excludedDaysOfYear,
 		timezone,
+		minimumAge,
 		logger,
 		dryRun,
 	)
@@ -540,3 +552,103 @@ func (t ThankGodItsFriday) Now() time.Time {
 	blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 UTC")
 	return blackFriday
 }
+
+func (suite *Suite) TestMinimumAge() {
+	type pod struct {
+		name         string
+		namespace    string
+		creationTime time.Time
+	}
+
+	for _, tt := range []struct {
+		minimumAge time.Duration
+		now        func() time.Time
+		pods       []pod
+		candidates int
+	}{
+		// no minimum age set
+		{
+			time.Duration(0),
+			func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
+			[]pod{
+				{
+					name:         "test1",
+					namespace:    "test",
+					creationTime: time.Date(0, 10, 24, 9, 00, 00, 00, time.UTC),
+				},
+			},
+			1,
+		},
+		// minimum age set, but pod is too young
+		{
+			time.Hour * 1,
+			func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
+			[]pod{
+				{
+					name:         "test1",
+					namespace:    "test",
+					creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC),
+				},
+			},
+			0,
+		},
+		// one pod is too young, one matches
+		{
+			time.Hour * 1,
+			func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
+			[]pod{
+				// too young
+				{
+					name:         "test1",
+					namespace:    "test",
+					creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC),
+				},
+				// matches
+				{
+					name:         "test2",
+					namespace:    "test",
+					creationTime: time.Date(0, 10, 23, 8, 00, 00, 00, time.UTC),
+				},
+			},
+			1,
+		},
+		// exact time - should not match
+		{
+			time.Hour * 1,
+			func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
+			[]pod{
+				{
+					name:         "test1",
+					namespace:    "test",
+					creationTime: time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC),
+				},
+			},
+			0,
+		},
+	} {
+		chaoskube := suite.setup(
+			labels.Everything(),
+			labels.Everything(),
+			labels.Everything(),
+			[]time.Weekday{},
+			[]util.TimePeriod{},
+			[]time.Time{},
+			time.UTC,
+			tt.minimumAge,
+			false,
+		)
+		chaoskube.Now = tt.now
+
+		for _, p := range tt.pods {
+			pod := util.NewPod(p.namespace, p.name)
+			pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime}
+			_, err := chaoskube.Client.Core().Pods(pod.Namespace).Create(&pod)
+			suite.Require().NoError(err)
+		}
+
+		pods, err := chaoskube.Candidates()
+		suite.Require().NoError(err)
+
+		suite.Len(pods, tt.candidates)
+	}
+}

main.go

@@ -29,6 +29,7 @@ var (
 	excludedTimesOfDay string
 	excludedDaysOfYear string
 	timezone           string
+	minimumAge         time.Duration
 	master             string
 	kubeconfig         string
 	interval           time.Duration
@@ -46,6 +47,7 @@ func init() {
 	kingpin.Flag("excluded-times-of-day", "A list of time periods of a day when termination is suspended, e.g. 22:00-08:00").StringVar(&excludedTimesOfDay)
 	kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").StringVar(&excludedDaysOfYear)
 	kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone)
+	kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge)
 	kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
 	kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
 	kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval)
@@ -69,6 +71,7 @@ func main() {
 		"excludedTimesOfDay": excludedTimesOfDay,
 		"excludedDaysOfYear": excludedDaysOfYear,
 		"timezone":           timezone,
+		"minimumAge":         minimumAge,
 		"master":             master,
 		"kubeconfig":         kubeconfig,
 		"interval":           interval,
@@ -145,6 +148,7 @@ func main() {
 		parsedTimesOfDay,
 		parsedDaysOfYear,
 		parsedTimezone,
+		minimumAge,
 		log.StandardLogger(),
 		dryRun,
 	)