Skip to content

Commit 5bb0edc

Browse files
dmmcquaydmmcquay
committed
add healthchecks to kube-controller
checks datastore and apiserver. Includes fv test for healthcheck Signed-off-by: derek mcquay <[email protected]>
1 parent 6feb62d commit 5bb0edc

File tree

7 files changed

+109
-16
lines changed

7 files changed

+109
-16
lines changed

‎Dockerfile.arm64

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@
1414
FROM arm64v8/alpine:3.7
1515
LABEL maintainer "Casey Davenport <[email protected]>"
1616

17-
ADD bin/kube-controllers-linux-amd64 /usr/bin/kube-controllers
18-
ADD bin/check-status-linux-amd64 /usr/bin/check-status
17+
ADD bin/kube-controllers-linux-arm64 /usr/bin/kube-controllers
18+
ADD bin/check-status-linux-arm64 /usr/bin/check-status
1919
ENTRYPOINT ["/usr/bin/kube-controllers"]

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ image-all: $(addprefix sub-image-,$(ARCHES))
138138
sub-image-%:
139139
$(MAKE) image ARCH=$*
140140

141-
image.created-$(ARCH): bin/kube-controllers-linux-$(ARCH)
141+
image.created-$(ARCH): bin/kube-controllers-linux-$(ARCH) bin/check-status-linux-$(ARCH)
142142
# Build the docker image for the policy controller.
143143
docker build -t $(CONTAINER_NAME):latest-$(ARCH) -f Dockerfile.$(ARCH) .
144144
ifeq ($(ARCH),amd64)

‎cmd/check-status/main.go

+2-10
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import (
1818
"flag"
1919
"fmt"
2020
"os"
21-
"time"
2221

2322
"github.com/projectcalico/kube-controllers/pkg/status"
2423
)
@@ -49,15 +48,8 @@ func main() {
4948
if *checkReady {
5049
var st *status.Status
5150
var err error
52-
// Try reading the status file up to 3 times.
53-
for i := 0; i < 3; i++ {
54-
st, err = status.ReadStatusFile(*file)
55-
if err != nil {
56-
time.Sleep(1 * time.Second)
57-
continue
58-
}
59-
break
60-
}
51+
52+
st, err = status.ReadStatusFile(*file)
6153
if err != nil {
6254
fmt.Printf("Failed to read status file %s: %v\n", *file, err)
6355
os.Exit(1)

‎cmd/kube-controllers/main.go

+45-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"context"
1919
"flag"
2020
"fmt"
21+
"net/http"
2122
"os"
2223
"strings"
2324
"time"
@@ -30,6 +31,7 @@ import (
3031
"github.com/projectcalico/kube-controllers/pkg/controllers/node"
3132
"github.com/projectcalico/kube-controllers/pkg/controllers/pod"
3233
"github.com/projectcalico/kube-controllers/pkg/controllers/serviceaccount"
34+
"github.com/projectcalico/kube-controllers/pkg/status"
3335
"github.com/projectcalico/libcalico-go/lib/apiconfig"
3436
client "github.com/projectcalico/libcalico-go/lib/clientv3"
3537
"github.com/projectcalico/libcalico-go/lib/logutils"
@@ -95,6 +97,12 @@ func main() {
9597
if err != nil {
9698
log.WithError(err).Fatal("Failed to initialize Calico datastore")
9799
}
100+
// Initialize readiness to false if enabled
101+
s := status.New(status.DefaultStatusFile)
102+
if config.HealthEnabled {
103+
s.SetReady("CalicoDatastore", false, "initialized to false")
104+
s.SetReady("KubeAPIServer", false, "initialized to false")
105+
}
98106

99107
for _, controllerType := range strings.Split(config.EnabledControllers, ",") {
100108
switch controllerType {
@@ -121,8 +129,43 @@ func main() {
121129
// If configured to do so, start an etcdv3 compaction.
122130
startCompactor(ctx, config)
123131

124-
// Wait forever.
125-
select {}
132+
// Wait forever and perform healthchecks.
133+
for {
134+
// skip healthchecks if configured
135+
if !config.HealthEnabled {
136+
select {}
137+
}
138+
// Datastore HealthCheck
139+
healthCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
140+
err = calicoClient.EnsureInitialized(healthCtx, "", "k8s")
141+
if err != nil {
142+
log.WithError(err).Errorf("Failed to verify datastore")
143+
s.SetReady(
144+
"CalicoDatastore",
145+
false,
146+
fmt.Sprintf("Error verifying datastore: %v", err),
147+
)
148+
} else {
149+
s.SetReady("CalicoDatastore", true, "")
150+
}
151+
cancel()
152+
153+
// Kube-apiserver HealthCheck
154+
healthStatus := 0
155+
k8sClientset.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
156+
if healthStatus != http.StatusOK {
157+
log.WithError(err).Errorf("Failed to reach apiserver")
158+
s.SetReady(
159+
"KubeAPIServer",
160+
false,
161+
fmt.Sprintf("Error reaching apiserver: %v with http status code: %d", err, healthStatus),
162+
)
163+
} else {
164+
s.SetReady("KubeAPIServer", true, "")
165+
}
166+
167+
time.Sleep(10 * time.Second)
168+
}
126169
}
127170

128171
// Starts an etcdv3 compaction goroutine with the given config.

‎pkg/config/config.go

+3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ type Config struct {
3939

4040
// Path to a kubeconfig file to use for accessing the k8s API.
4141
Kubeconfig string `default:"" split_words:"false"`
42+
43+
// Enable healthchecks
44+
HealthEnabled bool `default:"true"`
4245
}
4346

4447
// Parse parses envconfig and stores in Config struct

‎pkg/status/status.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func (s *Status) GetNotReadyConditions() string {
111111

112112
// WriteStatus writes out the status in json format.
113113
func (c *Status) WriteStatus() error {
114-
b, err := json.Marshal(*c)
114+
b, err := json.Marshal(c)
115115
if err != nil {
116116
logrus.Errorf("Failed to marshal readiness: %s", err)
117117
return err

‎tests/fv/fv_test.go

+55
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"io/ioutil"
2121
"net"
2222
"os"
23+
"os/exec"
2324
"time"
2425

2526
"k8s.io/api/core/v1"
@@ -108,6 +109,60 @@ var _ = Describe("kube-controllers FV tests", func() {
108109
Expect(*info.Spec.DatastoreReady).To(BeTrue())
109110
})
110111

112+
Context("Healthcheck FV tests", func() {
113+
It("should pass health check", func() {
114+
// wait for a health check cycle to pass
115+
Eventually(func() []byte {
116+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
117+
stdoutStderr, _ := cmd.CombinedOutput()
118+
119+
return stdoutStderr
120+
}, 20*time.Second, 500*time.Millisecond).ShouldNot(ContainSubstring("initialized to false"))
121+
Eventually(func() []byte {
122+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
123+
stdoutStderr, _ := cmd.CombinedOutput()
124+
125+
return stdoutStderr
126+
}, 20*time.Second, 500*time.Millisecond).ShouldNot(ContainSubstring("Error"))
127+
})
128+
129+
It("should fail health check if apiserver is not running", func() {
130+
// wait for a health check cycle to pass
131+
Eventually(func() []byte {
132+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
133+
stdoutStderr, _ := cmd.CombinedOutput()
134+
135+
return stdoutStderr
136+
}, 20*time.Second, 500*time.Millisecond).ShouldNot(ContainSubstring("initialized to false"))
137+
138+
apiserver.Stop()
139+
Eventually(func() []byte {
140+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
141+
stdoutStderr, _ := cmd.CombinedOutput()
142+
143+
return stdoutStderr
144+
}, 20*time.Second, 500*time.Millisecond).Should(ContainSubstring("Error reaching apiserver"))
145+
})
146+
147+
It("should fail health check if etcd not running", func() {
148+
// wait for a health check cycle to pass
149+
Eventually(func() []byte {
150+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
151+
stdoutStderr, _ := cmd.CombinedOutput()
152+
153+
return stdoutStderr
154+
}, 20*time.Second, 500*time.Millisecond).ShouldNot(ContainSubstring("initialized to false"))
155+
156+
etcd.Stop()
157+
Eventually(func() []byte {
158+
cmd := exec.Command("docker", "exec", policyController.Name, "/usr/bin/check-status", "-r")
159+
stdoutStderr, _ := cmd.CombinedOutput()
160+
161+
return stdoutStderr
162+
}, 20*time.Second, 500*time.Millisecond).Should(ContainSubstring("Error verifying datastore"))
163+
})
164+
})
165+
111166
Context("Node FV tests", func() {
112167
It("should be removed in response to a k8s node delete [Release]", func() {
113168
kn := &v1.Node{

0 commit comments

Comments
 (0)