diff --git a/README.md b/README.md index cd42b1ff..552d8fa5 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ Then pick and chose from patches our [example variants](https://github.com/Yolea | tag | k8s ≥ | highlights | | ------ | ----- | ----------- | +| TBD | 1.13+ | Kafka [2.4.0](https://github.com/Yolean/kubernetes-kafka/pull/297) + [standard storage class](https://github.com/Yolean/kubernetes-kafka/pull/294) | | v6.0.0 | 1.11+ | Kafka 2.2.0 + `apply -k` (kubectl 1.14+) + [#270](https://github.com/Yolean/kubernetes-kafka/pull/270) | | v5.1.0 | 1.11+ | Kafka 2.1.1 | | v5.0.3 | 1.11+ | Zookeeper fix [#227](https://github.com/Yolean/kubernetes-kafka/pull/227) + [maxClientCnxns=1](https://github.com/Yolean/kubernetes-kafka/pull/230#issuecomment-445953857) | diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 10f488e2..a1e35e21 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index c415ab7a..857e1f13 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -26,9 +26,10 @@ data: LABELS="$LABELS kafka-broker-rack=$ZONE" fi + [ -z "$ADVERTISE_ADDR" ] && echo "ADVERTISE_ADDR is empty, will advertise detected DNS name" OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) - SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://:9092,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") + SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://${ADVERTISE_ADDR}:9092,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" if [ ! -z "$LABELS" ]; then diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 41945f29..e0b52999 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -45,7 +45,7 @@ spec: mountPath: /opt/kafka/libs/extensions containers: - name: broker - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: CLASSPATH value: /opt/kafka/libs/extensions/* diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 749fa7fe..fad38f97 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -81,10 +81,6 @@ spec: - --if-not-exists - --topic - test-kafkacat - - --partitions - - "3" - - --replication-factor - - "2" resources: limits: cpu: 200m diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 7a03dd4f..0b29b7cc 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -99,7 +99,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -127,7 +127,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index 96f1982a..b5e1d407 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index e55c998e..b7562921 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 42985fda..9270f878 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/test/replicated-partitions.yml b/maintenance/test/replicated-partitions.yml index 3b90de8c..f4982b30 100644 --- a/maintenance/test/replicated-partitions.yml +++ b/maintenance/test/replicated-partitions.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 command: - /bin/bash - -ec diff --git a/variants/advertise-pod-ip/kafka.yaml b/variants/advertise-pod-ip/kafka.yaml new file mode 100644 index 00000000..3c9cf675 --- /dev/null +++ b/variants/advertise-pod-ip/kafka.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka +spec: + template: + spec: + initContainers: + - name: init-config + env: + - name: ADVERTISE_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/variants/advertise-pod-ip/kustomization.yaml b/variants/advertise-pod-ip/kustomization.yaml new file mode 100644 index 00000000..d6d46f41 --- /dev/null +++ b/variants/advertise-pod-ip/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../scale-3-5 +patchesStrategicMerge: +- kafka.yaml diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index ed0d538d..e2bf8cba 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -22,6 +22,7 @@ data: sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties zookeeper.properties: | + 4lw.commands.whitelist=ruok tickTime=2000 dataDir=/var/lib/zookeeper/data dataLogDir=/var/lib/zookeeper/log diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 62809eb8..c5f6d1f6 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -34,7 +34,7 @@ spec: mountPath: /var/lib/zookeeper containers: - name: zookeeper - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 5df5a57f..5aa8ec5e 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -37,7 +37,7 @@ spec: mountPath: /var/lib/zookeeper containers: - name: zookeeper - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties