Monitor node status

For now, only using Kubernets provided conditions (OutOfDisk,
MemoryPressure, DiskPressure, NetworkUnavailable). We don't
check (yet) for kube-state-metrics.

Author: bpineau

Makefile

@@ -6,10 +6,10 @@ tools:
 	which glide || go get -u github.com/Masterminds/glide
 
 lint:
-	gometalinter -j 1 --vendor --disable-all \
-		--enable=errcheck \
+	gometalinter --concurrency=1 --deadline=120s --vendor --disable-all \
 		--enable=vet \
 		--enable=vetshadow \
+		--enable=errcheck \
 		--enable=structcheck \
 		--enable=aligncheck \
 		--enable=deadcode \

README.md

@@ -6,6 +6,7 @@ kube-alert monitors:
 * Pod failures (unschedulables, error pulling images, crashloop backoff, etc.)
 * Pods restarts
 * Cluster's components status (issues with etcd, scheduler, or controller-manager daemons)
+* Nodes status (out-of-disk, memory pressure, network unavailable, ...)
 
 Support alerting to Datadog and to logs (ie. syslog).
 

pkg/controllers/cs/cs.go

@@ -22,12 +22,13 @@ func (c *CsController) HandlerName() string {
 }
 
 func (c *CsController) Init(conf *config.AlertConfig, handler handlers.Handler) controllers.Controller {
-	c.CommonController = controllers.CommonController{}
-	c.Conf = conf
-	c.Handler = handler
+	c.CommonController = controllers.CommonController{
+		Conf:    conf,
+		Handler: handler,
+		Name:    "componentstatus",
+	}
 
 	client := c.Conf.ClientSet
-	c.Name = "componentstatus"
 	c.ObjType = &v1.ComponentStatus{}
 	c.ListWatch = &cache.ListWatch{
 		ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {

pkg/controllers/node/node.go

@@ -0,0 +1,43 @@
+package node
+
+import (
+	"github.com/bpineau/kube-alert/config"
+	"github.com/bpineau/kube-alert/pkg/controllers"
+	"github.com/bpineau/kube-alert/pkg/handlers"
+
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/tools/cache"
+)
+
+type NodeController struct {
+	// https://golang.org/doc/effective_go.html#embedding
+	controllers.CommonController
+}
+
+func (c *NodeController) HandlerName() string {
+	return "node"
+}
+
+func (c *NodeController) Init(conf *config.AlertConfig, handler handlers.Handler) controllers.Controller {
+	c.CommonController = controllers.CommonController{
+		Conf:    conf,
+		Handler: handler,
+		Name:    "node",
+	}
+
+	client := c.Conf.ClientSet
+	c.ObjType = &v1.Node{}
+	c.ListWatch = &cache.ListWatch{
+		ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {
+			return client.CoreV1().Nodes().List(options)
+		},
+		WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) {
+			return client.CoreV1().Nodes().Watch(options)
+		},
+	}
+
+	return c
+}

pkg/controllers/pod/pod.go

@@ -22,12 +22,13 @@ func (c *PodController) HandlerName() string {
 }
 
 func (c *PodController) Init(conf *config.AlertConfig, handler handlers.Handler) controllers.Controller {
-	c.CommonController = controllers.CommonController{}
-	c.Conf = conf
-	c.Handler = handler
+	c.CommonController = controllers.CommonController{
+		Conf:    conf,
+		Handler: handler,
+		Name:    "pod",
+	}
 
 	client := c.Conf.ClientSet
-	c.Name = "pod"
 	c.ObjType = &v1.Pod{}
 	c.ListWatch = &cache.ListWatch{
 		ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) {

pkg/handlers/cs/cs.go

@@ -20,6 +20,10 @@ func (h *CsHandler) Init(c *config.AlertConfig) error {
 func (h *CsHandler) ObjectCreated(obj interface{}) (bool, string) {
 	cs, _ := obj.(*v1.ComponentStatus)
 
+	if cs == nil || cs.Conditions == nil {
+		return true, ""
+	}
+
 	for _, c := range cs.Conditions {
 		if c.Type != "Healthy" {
 			continue

pkg/handlers/handlers.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"github.com/bpineau/kube-alert/config"
 	"github.com/bpineau/kube-alert/pkg/handlers/cs"
+	"github.com/bpineau/kube-alert/pkg/handlers/node"
 	"github.com/bpineau/kube-alert/pkg/handlers/pod"
 )
 
@@ -13,6 +14,7 @@ type Handler interface {
 }
 
 var Handlers = map[string]Handler{
-	"pod": &pod.PodHandler{},
-	"cs":  &cs.CsHandler{},
+	"cs":   &cs.CsHandler{},
+	"pod":  &pod.PodHandler{},
+	"node": &node.NodeHandler{},
 }

pkg/handlers/node/node.go

@@ -0,0 +1,45 @@
+package node
+
+import (
+	"fmt"
+
+	"github.com/bpineau/kube-alert/config"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+type NodeHandler struct {
+	conf *config.AlertConfig
+}
+
+var knownBadConditions = map[string]bool{
+	"OutOfDisk":          true,
+	"MemoryPressure":     true,
+	"DiskPressure":       true,
+	"NetworkUnavailable": true,
+}
+
+func (n *NodeHandler) Init(c *config.AlertConfig) error {
+	c.Logger.Info("node handler initialized")
+	n.conf = c
+	return nil
+}
+
+func (n *NodeHandler) ObjectCreated(obj interface{}) (bool, string) {
+	node, _ := obj.(*v1.Node)
+
+	for _, c := range node.Status.Conditions {
+		if c.Status == "False" {
+			continue
+		}
+
+		if knownBadConditions[string(c.Type)] {
+			return false, fmt.Sprintf("Node %s is unhealthy: %s", node.Name, c.Message)
+		}
+	}
+
+	return true, ""
+}
+
+func (n *NodeHandler) ObjectDeleted(obj interface{}) (bool, string) {
+	return true, ""
+}

pkg/run/run.go

@@ -8,14 +8,16 @@ import (
 	"github.com/bpineau/kube-alert/config"
 	"github.com/bpineau/kube-alert/pkg/controllers"
 	"github.com/bpineau/kube-alert/pkg/controllers/cs"
+	"github.com/bpineau/kube-alert/pkg/controllers/node"
 	"github.com/bpineau/kube-alert/pkg/controllers/pod"
 	"github.com/bpineau/kube-alert/pkg/handlers"
 	"github.com/bpineau/kube-alert/pkg/health"
 )
 
 var Controllers = []controllers.Controller{
-	&pod.PodController{},
 	&cs.CsController{},
+	&pod.PodController{},
+	&node.NodeController{},
 }
 
 func Run(config *config.AlertConfig) {