Skip to content

Commit e61398f

Browse files
author
Joseph Sirianni
authored
Merge pull request #23 from BlueMedoraPublic/multi-disk-alert
Multi disk alert
2 parents 1666349 + bea1ac8 commit e61398f

File tree

18 files changed

+214
-207
lines changed

18 files changed

+214
-207
lines changed

.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ os:
66
arch:
77
- amd64
88
go:
9-
- '1.15'
9+
- '1.16'
1010
before_script:
1111
- go mod download
1212
script:

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ make
7979
```
8080

8181
If you wish to avoid Make and Docker, you can build with
82-
Go 1.15 on your machine
82+
Go 1.16 on your machine
8383
```
8484
go install github.com/mitchellh/gox
8585

cmd/root.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ import (
77

88
"github.com/BlueMedoraPublic/disk-usage/internal/alert"
99
"github.com/BlueMedoraPublic/disk-usage/internal/disk"
10-
"github.com/BlueMedoraPublic/disk-usage/internal/lock"
10+
"github.com/BlueMedoraPublic/disk-usage/internal/backend"
1111
"github.com/BlueMedoraPublic/disk-usage/internal/pkg/host"
1212

1313
"github.com/pkg/errors"
1414
log "github.com/sirupsen/logrus"
1515
)
1616

17-
const version string = "3.0.1"
17+
const version string = "3.1.0"
1818

1919
// flags
2020
var (
@@ -91,15 +91,15 @@ func initConfig() (disk.Config, error) {
9191
return disk.Config{}, err
9292
}
9393

94-
l, err := initLock()
94+
s, err := initState()
9595
if err != nil {
9696
return disk.Config{}, err
9797
}
9898

9999
return disk.Config{
100100
Threshold: threshold,
101101
Alert: a,
102-
Lock: l,
102+
State: s,
103103
Host: disk.System{
104104
Name: hostname,
105105
Address: ip,
@@ -137,13 +137,13 @@ func initAlert() (alert.Alert, error) {
137137
return nil, fmt.Errorf(fmt.Sprintf("failed to set alert type %s", alertType))
138138
}
139139

140-
func initLock() (lock.Lock, error) {
140+
func initState() (backend.State, error) {
141141
if dryrun {
142-
return lock.Null(), nil
142+
return backend.Null(), nil
143143
}
144144

145145
// const defined in root_unix.go / root_windows.go
146-
return lock.File(lockpath)
146+
return backend.File(statePath)
147147
}
148148

149149
func validateFlags() error {

cmd/root_unix.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
package cmd
44

5-
const lockpath string = "/tmp/suppress"
5+
const statePath string = "/tmp/suppress"

cmd/root_windows.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
package cmd
44

5-
const lockpath string = "C:\\suppress.txt"
5+
const statePath string = "C:\\suppress.txt"

docker/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.15
1+
FROM golang:1.16
22

33
ARG version
44

internal/backend/file/file.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package file
2+
3+
import (
4+
"os"
5+
6+
"github.com/pkg/errors"
7+
)
8+
9+
// File type manages a file at a given path
10+
type File struct {
11+
path string
12+
}
13+
14+
// New returns a new File type
15+
func New(p string) File {
16+
return File{p}
17+
}
18+
19+
// Write will lock the file
20+
func (f File) Write(state []byte) error {
21+
if err := os.WriteFile(f.path, state, 0644); err != nil {
22+
return errors.Wrap(err, "Failed to write state file")
23+
}
24+
return nil
25+
}
26+
27+
// Read will read the file
28+
func (f File) Read() ([]byte, error) {
29+
b, err := os.ReadFile(f.path)
30+
if err != nil {
31+
return nil, errors.Wrap(err, "Failed to read state file")
32+
}
33+
return b, nil
34+
}
35+
36+
// Path will return the file's path
37+
func (f File) Path() string {
38+
return f.path
39+
}

internal/backend/file/file_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package file
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestPath(t *testing.T) {
8+
f := New(path)
9+
if f.Path() != path {
10+
t.Errorf("expected f.Path() to return " + path + ", got " + f.Path())
11+
}
12+
}

internal/backend/null/null.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package null
2+
3+
// Null type performs no operations
4+
type Null struct {
5+
}
6+
7+
// Write will perform a no-op when called
8+
func (n Null) Write(state []byte) error {
9+
return nil
10+
}
11+
12+
// Read will perform a no-op when called
13+
func (n Null) Read() ([]byte, error) {
14+
return nil, nil
15+
}
16+
17+
// Path will perform a no-op when called
18+
func (n Null) Path() string {
19+
return ""
20+
}

internal/backend/state.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package backend
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/BlueMedoraPublic/disk-usage/internal/backend/file"
7+
"github.com/BlueMedoraPublic/disk-usage/internal/backend/null"
8+
)
9+
10+
// State interface manages writing and reading state
11+
type State interface {
12+
Write(state []byte) error
13+
Read() ([]byte, error)
14+
Path() string
15+
}
16+
17+
// File will return a new File type as a State interface
18+
func File(path string) (State, error) {
19+
if path == "" {
20+
return nil, fmt.Errorf("file state path not set")
21+
}
22+
return file.New(path), nil
23+
}
24+
25+
// Null will return a new Null type as a state interface
26+
func Null() State {
27+
return null.Null{}
28+
}

internal/disk/disk-usage.go

Lines changed: 97 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package disk
22

33
import (
44
"fmt"
5+
"encoding/json"
56

67
"github.com/BlueMedoraPublic/disk-usage/internal/alert"
7-
"github.com/BlueMedoraPublic/disk-usage/internal/lock"
8+
"github.com/BlueMedoraPublic/disk-usage/internal/backend"
89

910
log "github.com/sirupsen/logrus"
11+
"github.com/pkg/errors"
1012
)
1113

1214
// INFO represents info severity
@@ -24,8 +26,8 @@ type Config struct {
2426
// alert interface
2527
Alert alert.Alert
2628

27-
// lock interface
28-
Lock lock.Lock
29+
// State interface
30+
State backend.State
2931

3032
// Host is the system being managed by this config
3133
Host System
@@ -44,59 +46,122 @@ type Device struct {
4446
MountPoint string `json:"mountpoint"`
4547
Type string `json:"type"`
4648
UsagePercent int `json:"usage_percent"`
49+
Healthy bool `json:"healthy"`
50+
}
51+
52+
// State represents the state written to the state backend
53+
type State struct {
54+
Alerted []string `json:"alerted"`
55+
Host System `json:"host"`
4756
}
4857

4958
// Run will execute disk usage checks and alerts
5059
func (c *Config) Run() error {
5160
if err := c.getDisks(); err != nil {
5261
return err
5362
}
54-
return c.checkUsage()
63+
64+
if err := c.checkUsage(); err != nil {
65+
return err
66+
}
67+
68+
return c.handleState()
5569
}
5670

57-
func (c Config) checkUsage() error {
71+
func (c *Config) checkUsage() error {
72+
log.Trace(fmt.Sprintf("Checking disk usage with threshold %d%%", c.Threshold))
73+
5874
if err := c.getUsage(); err != nil {
5975
return err
6076
}
6177

62-
highUsage := []string{}
63-
for _, device := range c.Host.Devices {
78+
for i, device := range c.Host.Devices {
6479
if device.UsagePercent > c.Threshold {
65-
highUsage = append(highUsage, device.Name)
80+
c.Host.Devices[i].Healthy = false
81+
log.Trace(fmt.Sprintf("Device disk usage is unhealthy %s", device.Name))
82+
continue
6683
}
84+
c.Host.Devices[i].Healthy = true
85+
log.Trace(fmt.Sprintf("Device disk usage is healthy %s", device.Name))
6786
}
87+
return nil
88+
}
6889

69-
if len(highUsage) > 0 {
70-
msg := fmt.Sprintf("devices have high usage: %s", highUsage)
71-
return c.handleLock(true, msg)
90+
func (c *Config) handleState() error {
91+
// Continue on failure, assume not alerted. Start fresh.
92+
prevState, err := c.ReadState()
93+
if err != nil {
94+
log.Error(errors.Wrap(err, "Starting fresh with new state"))
7295
}
73-
return c.handleLock(false, "disk usage healthy")
74-
}
7596

76-
func (c Config) handleLock(unhealthy bool, message string) error {
77-
// If disk usage is healthy, and lock exists, clear it
78-
// by removing the lock
79-
if !unhealthy && c.Lock.Exists() {
80-
m := message + " disk usage is healthy"
81-
if err := c.message(m, INFO); err != nil {
82-
return err
83-
}
84-
return c.Lock.Unlock()
97+
newState := State{
98+
Host: c.Host,
8599
}
86100

87-
// If disk usage is not healthy and lock does not exist,
88-
// fire off an alert
89-
if unhealthy && !c.Lock.Exists() {
90-
if err := c.message(message, FATAL); err != nil {
91-
return err
101+
for _, current := range c.Host.Devices {
102+
103+
if current.Healthy {
104+
m := fmt.Sprintf("device is healthy: %s", current.Name)
105+
log.Info(m)
106+
if prevState.alerted(current.Name) {
107+
if err := c.message(m, INFO); err != nil {
108+
// if alert fails, add device to new state as alerted
109+
log.Error(err)
110+
newState.Alerted = append(newState.Alerted, current.Name)
111+
}
112+
log.Info(fmt.Sprintf("Sent 'device is healthy' notification for device %s", current.Name))
113+
}
92114
}
93-
return c.Lock.Lock()
94-
}
95115

96-
if unhealthy == true && c.Lock.Exists() {
97-
log.Info("Lock exists, skipping alert.")
98-
return nil
116+
if ! current.Healthy {
117+
m := fmt.Sprintf("device is unhealthy: %s", current.Name)
118+
log.Warning(m)
119+
if ! prevState.alerted(current.Name) {
120+
if err := c.message(m, FATAL); err != nil {
121+
log.Error(err)
122+
// when alert fails, do not add device to state as alerted
123+
// in order to force the alert attempt next time disk-usage
124+
// is executed
125+
continue
126+
}
127+
log.Info(fmt.Sprintf("Sent 'device is unhealthy' alert for device %s", current.Name))
128+
}
129+
130+
// add the device to the new state after alerting or skipping
131+
// due to already alerted
132+
newState.Alerted = append(newState.Alerted, current.Name)
133+
}
99134
}
100135

136+
if err := c.WriteState(newState); err != nil {
137+
return errors.Wrap(err, "Failed to write state")
138+
}
101139
return nil
102140
}
141+
142+
func (c Config) ReadState() (State, error) {
143+
s := State{}
144+
b, err := c.State.Read()
145+
if err != nil {
146+
return s, err
147+
}
148+
err = json.Unmarshal(b, &s)
149+
return s, err
150+
}
151+
152+
func (c Config) WriteState(s State) error {
153+
b, err := json.Marshal(s)
154+
if err != nil {
155+
return err
156+
}
157+
return c.State.Write(b)
158+
}
159+
160+
func (s State) alerted(input string) bool {
161+
for _, i := range s.Alerted {
162+
if input == i {
163+
return true
164+
}
165+
}
166+
return false
167+
}

0 commit comments

Comments
 (0)