Merge pull request #23 from BlueMedoraPublic/multi-disk-alert

Multi disk alert

Author: Joseph Sirianni

.travis.yml

@@ -6,7 +6,7 @@ os:
 arch:
   - amd64
 go:
-- '1.15'
+- '1.16'
 before_script:
   - go mod download
 script:

README.md

@@ -79,7 +79,7 @@ make
 ```
 
 If you wish to avoid Make and Docker, you can build with
-Go 1.15 on your machine
+Go 1.16 on your machine
 ```
 go install github.com/mitchellh/gox
 

cmd/root.go

@@ -7,14 +7,14 @@ import (
 
 	"github.com/BlueMedoraPublic/disk-usage/internal/alert"
 	"github.com/BlueMedoraPublic/disk-usage/internal/disk"
-	"github.com/BlueMedoraPublic/disk-usage/internal/lock"
+	"github.com/BlueMedoraPublic/disk-usage/internal/backend"
 	"github.com/BlueMedoraPublic/disk-usage/internal/pkg/host"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
-const version string = "3.0.1"
+const version string = "3.1.0"
 
 // flags
 var (
@@ -91,15 +91,15 @@ func initConfig() (disk.Config, error) {
 		return disk.Config{}, err
 	}
 
-	l, err := initLock()
+	s, err := initState()
 	if err != nil {
 		return disk.Config{}, err
 	}
 
 	return disk.Config{
 		Threshold: threshold,
 		Alert:     a,
-		Lock:      l,
+		State:      s,
 		Host: disk.System{
 			Name:    hostname,
 			Address: ip,
@@ -137,13 +137,13 @@ func initAlert() (alert.Alert, error) {
 	return nil, fmt.Errorf(fmt.Sprintf("failed to set alert type %s", alertType))
 }
 
-func initLock() (lock.Lock, error) {
+func initState() (backend.State, error) {
 	if dryrun {
-		return lock.Null(), nil
+		return backend.Null(), nil
 	}
 
 	// const defined in root_unix.go / root_windows.go
-	return lock.File(lockpath)
+	return backend.File(statePath)
 }
 
 func validateFlags() error {

cmd/root_unix.go

@@ -2,4 +2,4 @@
 
 package cmd
 
-const lockpath string = "/tmp/suppress"
+const statePath string = "/tmp/suppress"

cmd/root_windows.go

@@ -2,4 +2,4 @@
 
 package cmd
 
-const lockpath string = "C:\\suppress.txt"
+const statePath string = "C:\\suppress.txt"

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15
+FROM golang:1.16
 
 ARG version
 

internal/backend/file/file.go

@@ -0,0 +1,39 @@
+package file
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+// File type manages a file at a given path
+type File struct {
+	path string
+}
+
+// New returns a new File type
+func New(p string) File {
+	return File{p}
+}
+
+// Write will lock the file
+func (f File) Write(state []byte) error {
+	if err := os.WriteFile(f.path, state, 0644); err != nil {
+		return errors.Wrap(err, "Failed to write state file")
+	}
+	return nil
+}
+
+// Read will read the file
+func (f File) Read() ([]byte, error) {
+	b, err := os.ReadFile(f.path)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to read state file")
+	}
+	return b, nil
+}
+
+// Path will return the file's path
+func (f File) Path() string {
+	return f.path
+}

internal/backend/file/file_test.go

@@ -0,0 +1,12 @@
+package file
+
+import (
+	"testing"
+)
+
+func TestPath(t *testing.T) {
+	f := New(path)
+	if f.Path() != path {
+		t.Errorf("expected f.Path() to return " + path + ", got " + f.Path())
+	}
+}

internal/lock/file/file_unix_test.go renamed to internal/backend/file/file_unix_test.go

@@ -0,0 +1,20 @@
+package null
+
+// Null type performs no operations
+type Null struct {
+}
+
+// Write will perform a no-op when called
+func (n Null) Write(state []byte) error {
+	return nil
+}
+
+// Read will perform a no-op when called
+func (n Null) Read() ([]byte, error) {
+	return nil, nil
+}
+
+// Path will perform a no-op when called
+func (n Null) Path() string {
+	return ""
+}

internal/lock/file/file_windows_test.go renamed to internal/backend/file/file_windows_test.go

@@ -0,0 +1,28 @@
+package backend
+
+import (
+	"fmt"
+
+	"github.com/BlueMedoraPublic/disk-usage/internal/backend/file"
+	"github.com/BlueMedoraPublic/disk-usage/internal/backend/null"
+)
+
+// State interface manages writing and reading state
+type State interface {
+	Write(state []byte) error
+	Read() ([]byte, error)
+	Path() string
+}
+
+// File will return a new File type as a State interface
+func File(path string) (State, error) {
+	if path == "" {
+		return nil, fmt.Errorf("file state path not set")
+	}
+	return file.New(path), nil
+}
+
+// Null will return a new Null type as a state interface
+func Null() State {
+	return null.Null{}
+}

internal/backend/null/null.go

@@ -2,11 +2,13 @@ package disk
 
 import (
 	"fmt"
+	"encoding/json"
 
 	"github.com/BlueMedoraPublic/disk-usage/internal/alert"
-	"github.com/BlueMedoraPublic/disk-usage/internal/lock"
+	"github.com/BlueMedoraPublic/disk-usage/internal/backend"
 
 	log "github.com/sirupsen/logrus"
+	"github.com/pkg/errors"
 )
 
 // INFO represents info severity
@@ -24,8 +26,8 @@ type Config struct {
 	// alert interface
 	Alert alert.Alert
 
-	// lock interface
-	Lock lock.Lock
+	// State interface
+	State backend.State
 
 	// Host is the system being managed by this config
 	Host System
@@ -44,59 +46,122 @@ type Device struct {
 	MountPoint   string `json:"mountpoint"`
 	Type         string `json:"type"`
 	UsagePercent int    `json:"usage_percent"`
+	Healthy      bool   `json:"healthy"`
+}
+
+// State represents the state written to the state backend
+type State struct {
+	Alerted []string `json:"alerted"`
+	Host  System   `json:"host"`
 }
 
 // Run will execute disk usage checks and alerts
 func (c *Config) Run() error {
 	if err := c.getDisks(); err != nil {
 		return err
 	}
-	return c.checkUsage()
+
+	if err := c.checkUsage(); err != nil {
+		return err
+	}
+
+	return c.handleState()
 }
 
-func (c Config) checkUsage() error {
+func (c *Config) checkUsage() error {
+	log.Trace(fmt.Sprintf("Checking disk usage with threshold %d%%", c.Threshold))
+
 	if err := c.getUsage(); err != nil {
 		return err
 	}
 
-	highUsage := []string{}
-	for _, device := range c.Host.Devices {
+	for i, device := range c.Host.Devices {
 		if device.UsagePercent > c.Threshold {
-			highUsage = append(highUsage, device.Name)
+			c.Host.Devices[i].Healthy = false
+			log.Trace(fmt.Sprintf("Device disk usage is unhealthy %s", device.Name))
+			continue
 		}
+		c.Host.Devices[i].Healthy = true
+		log.Trace(fmt.Sprintf("Device disk usage is healthy %s", device.Name))
 	}
+	return nil
+}
 
-	if len(highUsage) > 0 {
-		msg := fmt.Sprintf("devices have high usage: %s", highUsage)
-		return c.handleLock(true, msg)
+func (c *Config) handleState() error {
+	// Continue on failure, assume not alerted. Start fresh.
+	prevState, err := c.ReadState()
+	if err != nil {
+		log.Error(errors.Wrap(err, "Starting fresh with new state"))
 	}
-	return c.handleLock(false, "disk usage healthy")
-}
 
-func (c Config) handleLock(unhealthy bool, message string) error {
-	// If disk usage is healthy, and lock exists, clear it
-	// by removing the lock
-	if !unhealthy && c.Lock.Exists() {
-		m := message + " disk usage is healthy"
-		if err := c.message(m, INFO); err != nil {
-			return err
-		}
-		return c.Lock.Unlock()
+	newState := State{
+		Host: c.Host,
 	}
 
-	// If disk usage is not healthy and lock does not exist,
-	// fire off an alert
-	if unhealthy && !c.Lock.Exists() {
-		if err := c.message(message, FATAL); err != nil {
-			return err
+	for _, current := range c.Host.Devices {
+
+		if current.Healthy {
+			m := fmt.Sprintf("device is healthy: %s", current.Name)
+			log.Info(m)
+			if prevState.alerted(current.Name) {
+				if err := c.message(m, INFO); err != nil {
+					// if alert fails, add device to new state as alerted
+					log.Error(err)
+					newState.Alerted = append(newState.Alerted, current.Name)
+				}
+				log.Info(fmt.Sprintf("Sent 'device is healthy' notification for device %s", current.Name))
+			}
 		}
-		return c.Lock.Lock()
-	}
 
-	if unhealthy == true && c.Lock.Exists() {
-		log.Info("Lock exists, skipping alert.")
-		return nil
+		if ! current.Healthy {
+			m := fmt.Sprintf("device is unhealthy: %s", current.Name)
+			log.Warning(m)
+			if ! prevState.alerted(current.Name) {
+				if err := c.message(m, FATAL); err != nil {
+					log.Error(err)
+					// when alert fails, do not add device to state as alerted
+					// in order to force the alert attempt next time disk-usage
+					// is executed
+					continue
+				}
+				log.Info(fmt.Sprintf("Sent 'device is unhealthy' alert for device %s", current.Name))
+			}
+
+			// add the device to the new state after alerting or skipping
+			// due to already alerted
+			newState.Alerted = append(newState.Alerted, current.Name)
+		}
 	}
 
+ 	if err := c.WriteState(newState); err != nil {
+		return errors.Wrap(err, "Failed to write state")
+	}
 	return nil
 }
+
+func (c Config) ReadState() (State, error) {
+	s := State{}
+	b, err := c.State.Read()
+	if err != nil {
+		return s, err
+	}
+	err = json.Unmarshal(b, &s)
+	return s, err
+}
+
+func (c Config) WriteState(s State) error {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return err
+	}
+	return c.State.Write(b)
+}
+
+func (s State) alerted(input string) bool {
+	for _, i := range s.Alerted {
+		if input == i {
+			return true
+		}
+	}
+	return false
+}