Skip to content

Commit ddd0564

Browse files
Felix GlaserMartin Linkhorst
Felix Glaser
authored and
Martin Linkhorst
committed
Adds percentage per pod, kill times and weekends
1 parent fc73a95 commit ddd0564

File tree

3 files changed

+285
-25
lines changed

3 files changed

+285
-25
lines changed

‎main.go

+39-21
Original file line numberDiff line numberDiff line change
@@ -13,30 +13,39 @@ import (
1313
"k8s.io/client-go/tools/clientcmd"
1414

1515
"github.com/linki/chaoskube/chaoskube"
16+
"github.com/linki/chaoskube/util"
1617
)
1718

1819
var (
19-
labelString string
20-
annString string
21-
nsString string
22-
master string
23-
kubeconfig string
24-
interval time.Duration
25-
inCluster bool
26-
dryRun bool
27-
debug bool
28-
version string
20+
annString string
21+
debug bool
22+
dryRun bool
23+
excludeWeekends bool
24+
inCluster bool
25+
interval time.Duration
26+
kubeconfig string
27+
labelString string
28+
master string
29+
nsString string
30+
percentage float64
31+
runFrom string
32+
runUntil string
33+
version string
2934
)
3035

3136
func init() {
32-
kingpin.Flag("labels", "A set of labels to restrict the list of affected pods. Defaults to everything.").StringVar(&labelString)
3337
kingpin.Flag("annotations", "A set of annotations to restrict the list of affected pods. Defaults to everything.").StringVar(&annString)
34-
kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").StringVar(&nsString)
35-
kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
36-
kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
37-
kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval)
38-
kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun)
3938
kingpin.Flag("debug", "Enable debug logging.").BoolVar(&debug)
39+
kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun)
40+
kingpin.Flag("excludeWeekends", "Do not run on weekends").BoolVar(&excludeWeekends)
41+
kingpin.Flag("interval", "Interval between Pod terminations").Default("1m").DurationVar(&interval)
42+
kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
43+
kingpin.Flag("labels", "A set of labels to restrict the list of affected pods. Defaults to everything.").StringVar(&labelString)
44+
kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
45+
kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").StringVar(&nsString)
46+
kingpin.Flag("percentage", "How likely should a pod be killed every single run").Default("0.0").Float64Var(&percentage)
47+
kingpin.Flag("run-from", "Start chaoskube daily at hours:minutes, e.g. 9:00").Default("0:00").StringVar(&runFrom)
48+
kingpin.Flag("run-until", "Stop chaoskube daily at hours:minutes, e.g. 17:00").Default("0:00").StringVar(&runUntil)
4049
}
4150

4251
func main() {
@@ -93,13 +102,22 @@ func main() {
93102
time.Now().UTC().UnixNano(),
94103
)
95104

105+
ticker := time.NewTicker(interval)
96106
for {
97-
if err := chaoskube.TerminateVictim(); err != nil {
98-
log.Fatal(err)
107+
select {
108+
case <-ticker.C:
109+
if util.ShouldRunNow(excludeWeekends, runFrom, runUntil) {
110+
candidates, err := chaoskube.Candidates()
111+
if err != nil {
112+
log.Fatal(err)
113+
}
114+
for _, candidate := range candidates {
115+
if util.PodShouldDie(candidate, interval, percentage) {
116+
chaoskube.DeletePod(candidate)
117+
}
118+
}
119+
}
99120
}
100-
101-
log.Debugf("Sleeping for %s...", interval)
102-
time.Sleep(interval)
103121
}
104122
}
105123

‎util/util.go

+118-4
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,136 @@
11
package util
22

33
import (
4+
"math/rand"
5+
"strconv"
6+
"strings"
7+
"time"
8+
9+
log "github.com/sirupsen/logrus"
10+
411
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
512
"k8s.io/client-go/pkg/api/v1"
613
)
714

15+
var timeNow = timeNowFunc
16+
17+
func init() {
18+
rand.Seed(timeNow().Unix())
19+
}
20+
21+
func timeNowFunc() time.Time {
22+
return time.Now()
23+
}
24+
825
// NewPod returns a new pod instance for testing purposes.
9-
func NewPod(namespace, name string) v1.Pod {
26+
func NewPod(namespace, name string, schedule ...string) v1.Pod {
27+
labels := map[string]string{"app": name}
28+
if len(schedule) > 0 {
29+
labels["chaos.schedule"] = schedule[0]
30+
}
1031
return v1.Pod{
1132
ObjectMeta: metav1.ObjectMeta{
1233
Namespace: namespace,
1334
Name: name,
14-
Labels: map[string]string{
15-
"app": name,
16-
},
35+
Labels: labels,
1736
Annotations: map[string]string{
1837
"chaos": name,
1938
},
2039
},
2140
}
2241
}
42+
43+
// takes a string containing a time (e.g. "23:42" and returns time object with that time today
44+
func stringToTime(str string) (time.Time, error) {
45+
now := timeNow()
46+
year, month, day := now.Date()
47+
time, err := time.Parse("15:04", str)
48+
if err != nil {
49+
return now, err
50+
}
51+
return time.AddDate(year, int(month)-1, day-1), nil
52+
}
53+
54+
// takes two strings containing a time (e.g. "09:00" and "17:00") and returns 2
55+
// time objects so that the "runFrom" one is before the "runUntil" one unless
56+
// it needs to be after because of situations like runfrom 17:00 to 05:00
57+
func startAndEndTime(runFrom string, runUntil string) (time.Time, time.Time, error) {
58+
start, err := stringToTime(runFrom)
59+
if err != nil {
60+
return timeNow(), timeNow(), err
61+
}
62+
end, err := stringToTime(runUntil)
63+
if err != nil {
64+
return timeNow(), timeNow(), err
65+
}
66+
// start this day and end the next day and be after start which means end
67+
// will have to be moved to the next day
68+
if end.Before(start) && timeNow().After(start) {
69+
return start, end.AddDate(0, 0, 1), nil
70+
}
71+
return start, end, nil
72+
}
73+
74+
// checks whether time.Now() is between runFrom and runUntil and whether it
75+
// should run during the weekend
76+
func ShouldRunNow(excludeWeekends bool, runFrom string, runUntil string) bool {
77+
now := timeNow()
78+
// Exclude weekends, sunday = day 0, saturday = day 6
79+
weekday := now.Weekday()
80+
if excludeWeekends && (weekday == 0 || weekday == 6) {
81+
return false
82+
}
83+
// no input was specified
84+
if runFrom == runUntil && runFrom == "0:00" {
85+
return true
86+
}
87+
start, end, err := startAndEndTime(runFrom, runUntil)
88+
if err != nil {
89+
log.Info("Converting times errored. No action will be taken.")
90+
return false
91+
}
92+
if now.After(start) && now.Before(end) {
93+
return true
94+
}
95+
return false
96+
}
97+
98+
func parseLabel(label string) (rate int, span int, err error) {
99+
split := strings.Split(label, ".")
100+
if len(split) != 2 {
101+
return 0, 0, err
102+
}
103+
rate_str, span_str := split[0], split[1]
104+
rate, err = strconv.Atoi(rate_str)
105+
if err != nil {
106+
return 0, 0, err
107+
}
108+
switch span_str {
109+
case "hour":
110+
span = 60
111+
case "day":
112+
span = 60 * 24
113+
case "week":
114+
span = 60 * 24 * 7
115+
}
116+
return
117+
}
118+
119+
func getOdds(p v1.Pod, interval time.Duration, percentage float64) float64 {
120+
labels := p.GetLabels()
121+
if labels["chaos.schedule"] == "" {
122+
return percentage
123+
}
124+
rate, span, err := parseLabel(labels["chaos.schedule"])
125+
if err != nil {
126+
log.Errorf("Error: %v from parsing %v's chaos.schedule, which is %s", err, p.Name, labels["chaos.schedule"])
127+
return 0.0
128+
}
129+
return (float64(rate) * interval.Minutes()) / float64(span)
130+
}
131+
132+
func PodShouldDie(p v1.Pod, interval time.Duration, percentage float64) bool {
133+
odds := getOdds(p, interval, percentage)
134+
random := rand.Float64()
135+
return (random <= odds)
136+
}

‎util/util_test.go

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package util
2+
3+
import (
4+
"testing"
5+
"time"
6+
)
7+
8+
func TestStringToTime(t *testing.T) {
9+
nine, err := stringToTime("9:00")
10+
if err != nil {
11+
t.Fatal("stringToTime errored")
12+
}
13+
if nine.Hour() != 9 {
14+
t.Fatal("stringToTime failed to parse hour")
15+
}
16+
if nine.Minute() != 0 {
17+
t.Fatal("stringToTime failed to parse minutes")
18+
}
19+
_, err = stringToTime("9:00:00")
20+
if err == nil {
21+
t.Fatal("stringToTime should have failed")
22+
}
23+
}
24+
25+
func TestStartAndEndTime(t *testing.T) {
26+
t1, t2, err := startAndEndTime("09:00", "17:00")
27+
if err != nil {
28+
t.Fatal("startAndEndTime errored")
29+
}
30+
if t1.Hour() != 9 {
31+
t.Fatal("startAndEndTime didn't parse time correctly")
32+
}
33+
if t2.Hour() != 17 {
34+
t.Fatal("startAndEndTime didn't parse time correctly")
35+
}
36+
if t1.After(t2) {
37+
t.Fatal("startAndEndTime didn't return the right times")
38+
}
39+
y_now, m_now, d_now := time.Now().Date()
40+
y_1, m_1, d_1 := t1.Date()
41+
y_2, m_2, d_2 := t2.Date()
42+
if y_now != y_1 || y_1 != y_2 {
43+
t.Fatal("startAndEndTime years are wrong", y_now, y_1, y_2)
44+
}
45+
if m_now != m_1 || m_1 != m_2 {
46+
t.Fatal("startAndEndTime months are wrong", m_now, m_1, m_2)
47+
}
48+
if d_now != d_1 || d_1 != d_2 {
49+
t.Fatal("startAndEndTime days are wrong", d_now, d_1, d_2)
50+
}
51+
}
52+
53+
func TestShouldRunNow(t *testing.T) {
54+
y_now, m_now, d_now := time.Now().Date()
55+
56+
// within the window it should run
57+
timeNow = func() time.Time { return time.Date(y_now, m_now, d_now, 11, 30, 0, 0, time.UTC) }
58+
if !ShouldRunNow(false, "9:00", "17:00") {
59+
t.Fatal("ShouldRunNow for 11:30 returned false")
60+
}
61+
62+
// outside the window it should run
63+
timeNow = func() time.Time { return time.Date(y_now, m_now, d_now, 19, 30, 0, 0, time.UTC) }
64+
if ShouldRunNow(false, "9:00", "17:00") {
65+
t.Fatal("ShouldRunNow for 19:30 returned true")
66+
}
67+
68+
// during a weekend, excludeWeekends = true, date is a this is a Sunday
69+
timeNow = func() time.Time { return time.Date(2017, 12, 31, 11, 30, 0, 0, time.UTC) }
70+
if ShouldRunNow(true, "9:00", "17:00") {
71+
t.Fatal("ShouldRunNow for excludeWeekends, but within the time window returned false")
72+
}
73+
74+
// always run, but exclude the weekend
75+
if ShouldRunNow(true, "0:00", "0:00") {
76+
t.Fatal("ShouldRunNow for excludeWeekends returned true")
77+
}
78+
79+
// always run and include the weekend
80+
if !ShouldRunNow(false, "0:00", "0:00") {
81+
t.Fatal("ShouldRunNow for excludeWeekends returned false")
82+
}
83+
}
84+
85+
func TestParseLabel(t *testing.T) {
86+
labels := map[string]map[string]int{
87+
"1.hour": {"rate": 1, "span": 60},
88+
"2.day": {"rate": 2, "span": 1440},
89+
"3.week": {"rate": 3, "span": 10080},
90+
}
91+
for k, v := range labels {
92+
rate, span, err := parseLabel(k)
93+
if err != nil {
94+
t.Fatal("parseLabel errored")
95+
}
96+
if rate != v["rate"] {
97+
t.Fatalf("parseLabel returned wrong rate want: %v got: %v", v["rate"], rate)
98+
}
99+
if span != v["span"] {
100+
t.Fatalf("parseLabel returned wrong span want: %v got: %v", v["span"], span)
101+
}
102+
}
103+
}
104+
105+
func TestGetOdds(t *testing.T) {
106+
schedules := map[string]float64{"1.hour": 1.0 / float64(60), "2.day": 2.0 / float64(60*24), "3.week": 3.0 / float64(60*24*7)}
107+
percentage := 0.5
108+
for schedule, initial_odd := range schedules {
109+
p := NewPod("default", "foo", schedule)
110+
intervalls := []time.Duration{time.Minute * 1, time.Minute * 5, time.Minute * 10, time.Minute * 60}
111+
for _, interval := range intervalls {
112+
odd := getOdds(p, interval, 0.5)
113+
target_odd := int(initial_odd * interval.Minutes() * 100)
114+
conv_odd := int(odd * 100)
115+
if conv_odd != target_odd {
116+
t.Fatalf("getOdds returned wrong odd want: %v got: %v, schedule: %v, interval: %v, percentage: %v",
117+
target_odd, odd, schedule, interval, percentage)
118+
}
119+
}
120+
}
121+
p := NewPod("default", "foo")
122+
interval := 10 * time.Minute
123+
odd := getOdds(p, interval, 0.5)
124+
if odd != 0.5 {
125+
t.Fatalf("getOdds returned wrong odd want: %v got: %v, schedule: %v, interval: %v, percentage: %v",
126+
percentage, odd, "", interval, percentage)
127+
}
128+
}

0 commit comments

Comments
 (0)