From b88b3191ee5882b9511f75cbf7fc5a5322e06672 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 1 Dec 2018 20:25:33 +0100 Subject: [PATCH 1/6] Mounts the parent of "datadir" AND "snapdir" Fixes #89, "logs" which are actually data would end up outside the mount. Zookeeper's startup logs are more clear than the property file entries: INFO Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /var/lib/zookeeper/log/version-2 snapdir /var/lib/zookeeper/data/version-2 --- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 11407d97..bddb426d 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -68,7 +68,7 @@ spec: - name: config mountPath: /etc/kafka - name: data - mountPath: /var/lib/zookeeper/data + mountPath: /var/lib/zookeeper volumes: - name: configmap configMap: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index b8d323c7..368ee1df 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -71,7 +71,7 @@ spec: - name: config mountPath: /etc/kafka - name: data - mountPath: /var/lib/zookeeper/data + mountPath: /var/lib/zookeeper volumes: - name: configmap configMap: From 7c589ccc92bb080ce1d2a494d56721fb02eaae52 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 1 Dec 2018 21:31:47 +0100 Subject: [PATCH 2/6] The init container must have the same mount path --- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index bddb426d..854d0acf 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -31,7 +31,7 @@ spec: - name: config mountPath: /etc/kafka - name: data - mountPath: /var/lib/zookeeper/data + mountPath: /var/lib/zookeeper containers: - name: zookeeper image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 368ee1df..6945ac92 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -34,7 +34,7 @@ spec: - name: config mountPath: /etc/kafka - name: data - mountPath: /var/lib/zookeeper/data + mountPath: /var/lib/zookeeper containers: - name: zookeeper image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 From 8c2362476670afa652b6ba19e29d5a3cfce88a46 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 1 Dec 2018 21:39:37 +0100 Subject: [PATCH 3/6] Fixes start from empty volume --- zookeeper/10zookeeper-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index d5d72171..4ff7b89a 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -9,6 +9,7 @@ data: set -e set -x + [ -d /var/lib/zookeeper/data ] || mkdir /var/lib/zookeeper/data [ -z "$ID_OFFSET" ] && ID_OFFSET=1 export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid From c43c6dd984d90b4df8f7f39bd2f1dd9c4658b690 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 1 Dec 2018 21:48:06 +0100 Subject: [PATCH 4/6] Lists this fix release as it's important --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b98121aa..866b6fbd 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e | tag | k8s ≥ | highlights | | ----- | ------ | ---------- | +| v5.0.1 | 1.11+ | Fixes Zookeeper persistence [#227](https://github.com/Yolean/kubernetes-kafka/pull/227) | | v5.0 | 1.11+ | Destabilize because in Docker we want Java 11 [#197](https://github.com/Yolean/kubernetes-kafka/pull/197) [#191](https://github.com/Yolean/kubernetes-kafka/pull/191) | | v4.3 | 1.9+ | Adds a prpper shutdown hook [207](https://github.com/Yolean/kubernetes-kafka/pull/207) | | v4.2 | 1.9+ | Kafka 1.0.2 and tools upgrade | From de6e7377edbf5adb5d666b739ef13015bfb87cad Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 2 Dec 2018 06:52:35 +0100 Subject: [PATCH 5/6] Got OOMKilled a couple of times during maintenance with no load This is still an arbitrary number --- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 854d0acf..686a77d0 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -57,7 +57,7 @@ spec: cpu: 10m memory: 100Mi limits: - memory: 100Mi + memory: 120Mi readinessProbe: exec: command: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 6945ac92..295c26ed 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -60,7 +60,7 @@ spec: cpu: 10m memory: 100Mi limits: - memory: 100Mi + memory: 120Mi readinessProbe: exec: command: From 57d820011fd339ab5418eb7f4a1ad704a4b67bf3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 2 Dec 2018 16:01:26 +0100 Subject: [PATCH 6/6] Lists the 4.x zookeeper fix as well --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 866b6fbd..72fd8476 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e | ----- | ------ | ---------- | | v5.0.1 | 1.11+ | Fixes Zookeeper persistence [#227](https://github.com/Yolean/kubernetes-kafka/pull/227) | | v5.0 | 1.11+ | Destabilize because in Docker we want Java 11 [#197](https://github.com/Yolean/kubernetes-kafka/pull/197) [#191](https://github.com/Yolean/kubernetes-kafka/pull/191) | +| v4.3.1 | 1.9+ | Critical Zookeeper persistence fix [#228](https://github.com/Yolean/kubernetes-kafka/pull/228) | | v4.3 | 1.9+ | Adds a prpper shutdown hook [207](https://github.com/Yolean/kubernetes-kafka/pull/207) | | v4.2 | 1.9+ | Kafka 1.0.2 and tools upgrade | | v4.1 | 1.9+ | Kafka 1.0.1 new [default](#148) [config](#170) |