Skip to content

Commit c2bb01a

Browse files
anarcheranarcher
authored
Feature/drain node (#1)
* refactor: rename deploy.go -> deployment.go * feat: drain node & pods * feat: aws client for terminate node * chore: README
1 parent cc0f8bd commit c2bb01a

File tree

13 files changed

+481
-4
lines changed

13 files changed

+481
-4
lines changed

‎README.md

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,23 @@
1-
# kroller
1+
# kroller
2+
3+
```
4+
$ kroller
5+
6+
_ _ _
7+
| | __ _ __ ___ | || | ___ _ __
8+
| |/ /| '__| / _ \ | || | / _ \| '__|
9+
| < | | | (_) || || || __/| |
10+
|_|\_\|_| \___/ |_||_| \___||_|
11+
12+
USAGE
13+
kroller <subcommand>
14+
15+
SUBCOMMANDS
16+
restart restart all rollout resources
17+
drain drain node
18+
19+
FLAGS
20+
-kubeconfig ... kubeconfig file
21+
-v false log verbose output
22+
```
23+

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ go 1.15
44

55
require (
66
github.com/CrowdSurge/banner v0.0.0-20140923200336-8c0e79dc5ff7
7+
github.com/aws/aws-sdk-go v1.38.21
78
github.com/fatih/color v1.10.0
89
github.com/googleapis/gnostic v0.5.4 // indirect
910
github.com/imdario/mergo v0.3.11 // indirect
1011
github.com/peterbourgon/ff/v3 v3.0.0
12+
github.com/pkg/errors v0.9.1
1113
github.com/rodaine/table v1.0.1
1214
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
1315
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect

‎go.sum

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
4848
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
4949
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
5050
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
51+
github.com/aws/aws-sdk-go v1.38.21 h1:D08DXWI4QRaawLaW+OtsIEClOI90I6eheJs1GwXTQVI=
52+
github.com/aws/aws-sdk-go v1.38.21/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
5153
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
5254
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
5355
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -153,6 +155,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
153155
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
154156
github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
155157
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
158+
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
159+
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
160+
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
156161
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
157162
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
158163
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -193,6 +198,7 @@ github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t
193198
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
194199
github.com/peterbourgon/ff/v3 v3.0.0 h1:eQzEmNahuOjQXfuegsKQTSTDbf4dNvr/eNLrmJhiH7M=
195200
github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0=
201+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
196202
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
197203
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
198204
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -398,7 +404,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY
398404
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
399405
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
400406
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
401-
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
402407
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
403408
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
404409
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

‎main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ import (
1313
func main() {
1414
rootCmd, cfg := cmd.NewRootCmd()
1515
restartCmd := cmd.NewRestartCmd(cfg)
16+
drainCmd := cmd.NewDrainCmd(cfg)
1617

1718
rootCmd.Subcommands = []*ffcli.Command{
1819
restartCmd,
20+
drainCmd,
1921
}
2022

2123
if err := rootCmd.Parse(os.Args[1:]); err != nil {

‎pkg/aws/asg.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package aws
2+
3+
import (
4+
"github.com/aws/aws-sdk-go/aws"
5+
"github.com/aws/aws-sdk-go/service/autoscaling"
6+
)
7+
8+
func (c *Client) TerminateInstance(instanceID string, decrDesiredCapacity bool) error {
9+
auto := autoscaling.New(c.Session)
10+
input := &autoscaling.TerminateInstanceInAutoScalingGroupInput{
11+
InstanceId: aws.String(instanceID),
12+
ShouldDecrementDesiredCapacity: aws.Bool(decrDesiredCapacity),
13+
}
14+
_, err := auto.TerminateInstanceInAutoScalingGroup(input)
15+
if err != nil {
16+
return err
17+
}
18+
return nil
19+
}

‎pkg/aws/client.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package aws
2+
3+
import (
4+
"github.com/aws/aws-sdk-go/aws"
5+
"github.com/aws/aws-sdk-go/aws/session"
6+
)
7+
8+
type Client struct {
9+
Session *session.Session
10+
}
11+
12+
func NewClient(region string) (*Client, error) {
13+
s, err := session.NewSession(&aws.Config{
14+
Region: aws.String(region),
15+
})
16+
if err != nil {
17+
return nil, err
18+
}
19+
20+
c := Client{
21+
Session: s,
22+
}
23+
24+
return &c, nil
25+
}

‎pkg/aws/ec2.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package aws
2+
3+
import (
4+
"github.com/aws/aws-sdk-go/aws"
5+
"github.com/aws/aws-sdk-go/service/ec2"
6+
)
7+
8+
func (c *Client) GetInstanceID(nodeName string) (string, error) {
9+
svc := ec2.New(c.Session)
10+
11+
params := &ec2.DescribeInstancesInput{
12+
Filters: []*ec2.Filter{
13+
{
14+
Name: aws.String("private-dns-name"),
15+
Values: []*string{aws.String(nodeName)},
16+
},
17+
},
18+
}
19+
20+
resp, err := svc.DescribeInstances(params)
21+
if err != nil {
22+
return "", err
23+
}
24+
25+
var instanceID string
26+
for _, reservation := range resp.Reservations {
27+
for _, instance := range reservation.Instances {
28+
instanceID = *instance.InstanceId
29+
break
30+
}
31+
break
32+
}
33+
return instanceID, nil
34+
}

‎pkg/cmd/drain.go

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package cmd
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"time"
8+
9+
"github.com/anarcher/kroller/pkg/aws"
10+
"github.com/anarcher/kroller/pkg/kubernetes"
11+
"github.com/anarcher/kroller/pkg/ui"
12+
13+
"github.com/fatih/color"
14+
"github.com/peterbourgon/ff/v3"
15+
"github.com/peterbourgon/ff/v3/ffcli"
16+
"github.com/rodaine/table"
17+
18+
v1 "k8s.io/api/core/v1"
19+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20+
)
21+
22+
type DrainConfig struct {
23+
rootCfg *RootConfig
24+
awsRegion string
25+
gracePeriod time.Duration
26+
node string
27+
isTerminateNode bool
28+
decrementDesiredCapacity bool
29+
}
30+
31+
func NewDrainCmd(rootCfg *RootConfig) *ffcli.Command {
32+
cfg := &DrainConfig{
33+
rootCfg: rootCfg,
34+
}
35+
36+
fs := flag.NewFlagSet("kroller drain", flag.ExitOnError)
37+
fs.String("config", "", "config file (optional)")
38+
fs.StringVar(&cfg.awsRegion, "aws-region", "ap-northeast-2", "The region to use for node")
39+
fs.DurationVar(&cfg.gracePeriod, "grace-period", (30 * time.Second), "Pod grace-period")
40+
fs.StringVar(&cfg.node, "node", "", "The node that should drain")
41+
fs.BoolVar(&cfg.isTerminateNode, "terminate-node", false, "Terminate the AWS instance in the autoscaling group")
42+
fs.BoolVar(&cfg.decrementDesiredCapacity, "decr-desired-capacity", false, "Decrement desired capacity of the autoscaling group")
43+
rootCfg.RegisterFlags(fs)
44+
45+
c := &ffcli.Command{
46+
Name: "drain",
47+
ShortUsage: "drain node",
48+
ShortHelp: "drain node",
49+
FlagSet: fs,
50+
Options: []ff.Option{
51+
ff.WithEnvVarNoPrefix(),
52+
ff.WithConfigFileFlag("config"),
53+
ff.WithConfigFileParser(ff.PlainParser),
54+
},
55+
Exec: cfg.Exec,
56+
}
57+
return c
58+
}
59+
60+
func (c *DrainConfig) Exec(ctx context.Context, args []string) error {
61+
if c.node == "" {
62+
return fmt.Errorf("node is required")
63+
}
64+
65+
if err := c.drainNode(ctx); err != nil {
66+
return err
67+
}
68+
69+
if c.isTerminateNode == true {
70+
if err := c.terminateNode(ctx); err != nil {
71+
return err
72+
}
73+
}
74+
75+
return nil
76+
}
77+
78+
func (c *DrainConfig) drainNode(ctx context.Context) error {
79+
verbose := c.rootCfg.Verbose
80+
kubeClient := c.rootCfg.KubeClient
81+
82+
node, err := kubeClient.Node(ctx, c.node)
83+
if err != nil {
84+
return err
85+
}
86+
87+
allPods, err := kubeClient.PodsOnNode(ctx, c.node)
88+
if err != nil {
89+
return err
90+
}
91+
92+
pods := filterRollPods(allPods.Items)
93+
94+
ui.PodList(pods)
95+
fmt.Println("")
96+
fmt.Printf(color.GreenString("Do you want to continue and drain?"))
97+
ok, err := ui.AskForConfirm()
98+
if err != nil {
99+
return err
100+
}
101+
if !ok {
102+
return nil
103+
}
104+
105+
if _, err := kubeClient.CordonNode(ctx, node); err != nil {
106+
return err
107+
}
108+
109+
ui.Print("", verbose)
110+
ui.PrintTitle("Cordon\n", verbose)
111+
ui.Print(fmt.Sprintf("[✓] %s cordoned\n\n", node.ObjectMeta.Name), verbose)
112+
113+
ui.PrintTitle("Evict Pods\n", verbose)
114+
rollPods(ctx, kubeClient, pods, c.gracePeriod, verbose)
115+
116+
return nil
117+
}
118+
119+
func (c *DrainConfig) terminateNode(ctx context.Context) error {
120+
verbose := c.rootCfg.Verbose
121+
122+
fmt.Println("")
123+
fmt.Printf(color.RedString("Do you want to continue and terminate the node? "))
124+
ok, err := ui.AskForConfirm()
125+
if err != nil {
126+
return err
127+
}
128+
if !ok {
129+
return nil
130+
}
131+
132+
ui.Print("", verbose)
133+
ui.PrintTitle("Node termination:\n", verbose)
134+
135+
client, err := aws.NewClient(c.awsRegion)
136+
if err != nil {
137+
return err
138+
}
139+
140+
instanceID, err := client.GetInstanceID(c.node)
141+
if err != nil {
142+
return err
143+
}
144+
145+
ui.Print(fmt.Sprintf("%-25s %s", "Private DNS:", c.node), verbose)
146+
ui.Print(fmt.Sprintf("%-25s %s", "Instance ID:", instanceID), verbose)
147+
ui.Print(fmt.Sprintf("Decrement desired capacity: %v", c.decrementDesiredCapacity), verbose)
148+
149+
if err := client.TerminateInstance(instanceID, c.decrementDesiredCapacity); err != nil {
150+
return err
151+
}
152+
153+
ui.Print("\n", verbose)
154+
ui.Print("[✓] Node has been terminated!\n", true)
155+
return nil
156+
}
157+
158+
func filterRollPods(pods []v1.Pod) []v1.Pod {
159+
var res []v1.Pod
160+
for _, p := range pods {
161+
controllerRef := metav1.GetControllerOf(&p)
162+
if controllerRef == nil {
163+
continue
164+
}
165+
166+
if controllerRef.Kind == "DaemonSet" {
167+
continue
168+
}
169+
res = append(res, p)
170+
}
171+
return res
172+
}
173+
174+
func rollPods(ctx context.Context, kubeClient *kubernetes.Client, pods []v1.Pod, gracePeriod time.Duration, verbose bool) error {
175+
176+
graceP := int64(gracePeriod.Seconds())
177+
deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &graceP}
178+
179+
fmt.Println("")
180+
tbl := table.New(" ", "Evict pod", "New pod", "New node")
181+
headerFmt := color.New(color.FgGreen, color.Underline).SprintfFunc()
182+
columnFmt := color.New(color.FgYellow).SprintfFunc()
183+
tbl.WithHeaderFormatter(headerFmt).WithFirstColumnFormatter(columnFmt)
184+
185+
for _, pod := range pods {
186+
err := kubeClient.DeletePod(ctx, pod, deleteOptions)
187+
if err != nil {
188+
return err
189+
}
190+
newPod, err := kubeClient.DetermineNewPod(ctx, pod)
191+
if err != nil {
192+
return err
193+
}
194+
if newPod != nil {
195+
if err := kubeClient.WaitForPodToBeReady(ctx, newPod); err != nil {
196+
return err
197+
}
198+
tbl.AddRow("[✓]", pod.Name, newPod.Name, newPod.Spec.NodeName)
199+
} else {
200+
tbl.AddRow("[✓]", pod.Name, "?", "?")
201+
}
202+
203+
if verbose {
204+
fmt.Printf("Evicting pod: %s\n", pod.Name)
205+
}
206+
}
207+
tbl.Print()
208+
209+
return nil
210+
}

‎pkg/cmd/restart.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ func NewRestartCmd(rootCfg *RootConfig) *ffcli.Command {
2929
rootCfg.RegisterFlags(fs)
3030

3131
c := &ffcli.Command{
32-
Name: "restart",
33-
FlagSet: fs,
32+
Name: "restart",
33+
ShortUsage: "restart all rollout resources (deployment,statefulset)",
34+
ShortHelp: "restart all rollout resources",
35+
FlagSet: fs,
3436
Options: []ff.Option{
3537
ff.WithEnvVarNoPrefix(),
3638
ff.WithConfigFileFlag("config"),

‎pkg/kubernetes/deploy.go renamed to ‎pkg/kubernetes/deployment.go

File renamed without changes.

0 commit comments

Comments
 (0)