Merge pull request #197 from Yolean/kafka-2.0.x

Upgrade to Kafka 2.0.0 and Zookeeper 3.4.13

Author: solsson

README.md

@@ -27,11 +27,13 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e
 
 ## Version history
 
-| tag   | k8s ≥ | highlights |
+| tag   | k8s ≥  | highlights |
 | ----- | ------ | ---------- |
-| 4.x  | 1.9+    | Kafka 1.1 dynamic config |
-| v4.1 | 1.9+    | Kafka 1.0.1 new [default](#148) [config](#170) |
-| v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) |
+| v5.0  | 1.11+  | Destabilize because in Docker we want Java 11 [#197](https://github.com/Yolean/kubernetes-kafka/pull/197) [#191](https://github.com/Yolean/kubernetes-kafka/pull/191) |
+| v4.3  | 1.9+   | Adds a prpper shutdown hook [207](https://github.com/Yolean/kubernetes-kafka/pull/207) |
+| v4.2  | 1.9+   | Kafka 1.0.2 and tools upgrade |
+| v4.1  | 1.9+   | Kafka 1.0.1 new [default](#148) [config](#170) |
+| v3.2  | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) |
 | v3.1  | 1.8    | The painstaking path to `min.insync.replicas`=2 |
 | v3.0  | 1.8    | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) |
 | v2.1  | 1.5    | Kafka 1.0, the init script concept |

avro-tools/rest.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: avro-rest

avro-tools/schemas.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: avro-schemas

avro-tools/test/rest-curl.yml

@@ -114,7 +114,7 @@ spec:
     spec:
       containers:
       - name: topic-create
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         command:
         - ./bin/kafka-topics.sh
         - --zookeeper
@@ -129,7 +129,7 @@ spec:
         -   "1"
       restartPolicy: Never
 ---
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: rest-curl

events-kube/events-kube-kafka.yml

@@ -30,7 +30,7 @@ spec:
       serviceAccount: eventrouter
       containers:
       - name: kube-eventrouter
-        image: gcr.io/heptio-images/eventrouter@sha256:30e36ce7bad4a7c539e0a0cb1833d309089919fb0ef0c165ee28aabe97740d02
+        image: gcr.io/heptio-images/eventrouter@sha256:e613b48c6235426fa334867d661118322b4e1973c23e2e8cf5c066b982cc8596
         resources:
           requests:
             memory: "5Mi"

kafka/50kafka.yml

@@ -20,7 +20,7 @@ spec:
       terminationGracePeriodSeconds: 30
       initContainers:
       - name: init-config
-        image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2
+        image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea
         env:
         - name: NODE_NAME
           valueFrom:
@@ -42,7 +42,7 @@ spec:
           mountPath: /etc/kafka
       containers:
       - name: broker
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
@@ -65,7 +65,12 @@ spec:
         resources:
           requests:
             cpu: 100m
-            memory: 512Mi
+            memory: 100Mi
+          limits:
+            # This limit was intentionally set low as a reminder that
+            # the entire Yolean/kubernetes-kafka is meant to be tweaked
+            # before you run production workloads
+            memory: 600Mi
         readinessProbe:
           tcpSocket:
             port: 9092

kafka/test/kafkacat.yml

@@ -91,7 +91,7 @@ spec:
           memory: 100Mi
       restartPolicy: Never
 ---
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: ReplicaSet
 metadata:
   name: kafkacat

kafka/test/produce-consume.yml

@@ -74,7 +74,7 @@ spec:
           memory: 100Mi
       restartPolicy: Never
 ---
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: ReplicaSet
 metadata:
   name: produce-consume

kafka/test/replication-config.yml

@@ -45,7 +45,7 @@ data:
 
     exit 0
 ---
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: ReplicaSet
 metadata:
   name: replication-config

linkedin-burrow/burrow.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: burrow

maintenance/preferred-replica-election-job.yml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kafka
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         command:
         - ./bin/kafka-preferred-replica-election.sh
         - --zookeeper

maintenance/reassign-paritions-job.yml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kafka
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         env:
         - name: ZOOKEEPER
           value: zookeeper.kafka:2181

maintenance/replication-factor-increase-job.yml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kafka
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         env:
         - name: ZOOKEEPER
           value: zookeeper.kafka:2181

maintenance/test/replicated-partitions.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: replicated-partitions

pixy/pixy.yml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: pixy
-        image: mailgun/kafka-pixy:0.15.0@sha256:088210d53945a0db5f93921ceff3a79c012449b7845baebe8898452741764e7c
+        image: mailgun/kafka-pixy:0.16.0@sha256:2a2d426f9fe17ede9c21b2f8b7418d8787293e39327b638fd6c403f3726597fb
         ports:
         - containerPort: 80
         command:

prometheus/50-kafka-jmx-exporter-patch.yml

@@ -1,6 +1,6 @@
 # meant to be applied using
 # kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )"
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: kafka
@@ -14,13 +14,9 @@ spec:
     spec:
       containers:
       - name: metrics
-        image: solsson/kafka-prometheus-jmx-exporter@sha256:a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:d237a12cc0cde42b539bcb5efc0008ba5e6ca1351b7843ed52bd574d181c5efd
         command:
         - java
-        - -XX:+UnlockExperimentalVMOptions
-        - -XX:+UseCGroupMemoryLimitForHeap
-        - -XX:MaxRAMFraction=1
-        - -XshowSettings:vm
         - -jar
         - jmx_prometheus_httpserver.jar
         - "5556"

yahoo-kafka-manager/kafka-manager.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kafka-manager

zookeeper/50pzoo.yml

@@ -22,7 +22,7 @@ spec:
       terminationGracePeriodSeconds: 10
       initContainers:
       - name: init-config
-        image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2
+        image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea
         command: ['/bin/bash', '/etc/kafka-configmap/init.sh']
         volumeMounts:
         - name: configmap
@@ -33,7 +33,7 @@ spec:
           mountPath: /var/lib/zookeeper/data
       containers:
       - name: zookeeper
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
@@ -55,6 +55,8 @@ spec:
           requests:
             cpu: 10m
             memory: 100Mi
+          limits:
+            memory: 100Mi
         readinessProbe:
           exec:
             command:

zookeeper/51zoo.yml

@@ -22,7 +22,7 @@ spec:
       terminationGracePeriodSeconds: 10
       initContainers:
       - name: init-config
-        image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2
+        image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea
         command: ['/bin/bash', '/etc/kafka-configmap/init.sh']
         env:
         - name: ID_OFFSET
@@ -36,7 +36,7 @@ spec:
           mountPath: /var/lib/zookeeper/data
       containers:
       - name: zookeeper
-        image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789
+        image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
@@ -58,6 +58,8 @@ spec:
           requests:
             cpu: 10m
             memory: 100Mi
+          limits:
+            memory: 100Mi
         readinessProbe:
           exec:
             command: