apache · nicknezis · Jul 10, 2020 · Jul 3, 2020 · Jul 3, 2020 · Jul 3, 2020
diff --git a/WORKSPACE b/WORKSPACE
@@ -66,7 +66,7 @@ maven_install(
     name = "maven",
     artifacts = [
         "antlr:antlr:2.7.7",
-        "org.apache.zookeeper:zookeeper:3.4.14",
+        "org.apache.zookeeper:zookeeper:3.5.8",
         "io.kubernetes:client-java:" + kubernetes_client_version,
         "com.esotericsoftware:kryo:3.0.3",
         "org.apache.avro:avro:1.7.4",
@@ -282,11 +282,8 @@ http_archive(
 http_archive(
     name = "org_apache_zookeeper",
     build_file = "@//:third_party/zookeeper/BUILD",
-    patch_args = ["-p2"],
-    patches = ["//third_party/zookeeper:pkgconfig.patch"],
-    sha256 = "b14f7a0fece8bd34c7fffa46039e563ac5367607c612517aa7bd37306afbd1cd",
-    strip_prefix = "zookeeper-3.4.14",
-    urls = ["https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz"],
+    strip_prefix = "apache-zookeeper-3.5.8",
+    urls = ["https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8.tar.gz"],
 )
 
 http_archive(

diff --git a/deploy/kubernetes/helm/templates/bookie.yaml b/deploy/kubernetes/helm/templates/bookie.yaml
@@ -111,6 +111,21 @@ spec:
 {{- end }}
       terminationGracePeriodSeconds: 0
       initContainers:
+{{- if eq .Values.platform "minikube" }}
+        # The first time, initialize BK to wipe data for minikube
+        - name: bookie-format
+          image: {{ .Values.bookkeeper.image }}
+          args: ["/opt/bookkeeper/bin/bookkeeper", "shell", "bookieformat", "--nonInteractive", "-f", "-deleteCookie"]
+          command: [ "/bin/bash", "/opt/bookkeeper/scripts/entrypoint.sh" ]
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-bookie-config
+          volumeMounts:
+            - name: journal-disk
+              mountPath: /bookkeeper/data/journal
+            - name: ledgers-disk
+              mountPath: /bookkeeper/data/ledgers
+{{- end }}
         # Wait until the zookeeper pods are up and running
         - name: bookie-init-container
           image: {{ .Values.image }} # use heron image until bookkeeper has nc

diff --git a/deploy/kubernetes/minikube/apiserver.yaml b/deploy/kubernetes/minikube/apiserver.yaml
@@ -64,7 +64,7 @@ spec:
       initContainers:
         - name: init-heron-apiserver
           image: apache/bookkeeper:4.7.3
-          command: ['sh', '-c', '/opt/bookkeeper/bin/dlog admin bind -l /ledgers -s zookeeper:2181 -c distributedlog://zookeeper:2181/heron']
+          command: ['sh', '-c', '/opt/bookkeeper/bin/dlog admin bind -l /ledgers -s zookeeper:2181 -c distributedlog://zookeeper:2181/heronbkdl']
       containers:
         - name: heron-apiserver
           image: heron/heron:latest
@@ -79,9 +79,9 @@ spec:
               -D heron.executor.docker.image=heron/heron:latest
               -D heron.class.uploader=org.apache.heron.uploader.dlog.DLUploader
               -D heron.uploader.dlog.topologies.num.replicas=1
-              -D heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heron
+              -D heron.uploader.dlog.topologies.namespace.uri=distributedlog://zookeeper:2181/heronbkdl
               -D heron.statefulstorage.classname=org.apache.heron.statefulstorage.dlog.DlogStorage
-              -D heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heron
+              -D heron.statefulstorage.dlog.namespace.uri=distributedlog://zookeeper:2181/heronbkdl
 
 ---
 apiVersion: v1

diff --git a/deploy/kubernetes/minikube/bookkeeper.yaml b/deploy/kubernetes/minikube/bookkeeper.yaml
@@ -65,6 +65,11 @@ spec:
           envFrom:
             - configMapRef:
                 name: bookie-config
+          volumeMounts:
+            - name: journal-disk
+              mountPath: /bookkeeper/data/journal
+            - name: ledgers-disk
+              mountPath: /bookkeeper/data/ledgers
       containers:
         - name: bookie
           image: apache/bookkeeper:4.7.3
@@ -91,13 +96,11 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.hostIP
-
           volumeMounts:
             - name: journal-disk
               mountPath: /bookkeeper/data/journal
             - name: ledgers-disk
               mountPath: /bookkeeper/data/ledgers
-
       volumes:
           # Mount local disks
         - name: journal-disk

diff --git a/docker/base/Dockerfile.base.debian9 b/docker/base/Dockerfile.base.debian9
@@ -18,6 +18,7 @@
 FROM openjdk:11-jdk-slim-stretch
 
 RUN apt-get -y update && apt-get -y install \
+    ant \
     netcat-openbsd \
     python \
     unzip \
@@ -27,7 +28,7 @@ RUN apt-get -y update && apt-get -y install \
     rm -rf /var/lib/apt/lists/*
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD base/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/base/conf/zookeeper.conf b/docker/base/conf/zookeeper.conf
@@ -24,3 +24,5 @@ autopurge.snapRetainCount=3
 # Purge task interval in hours
 # Set to "0" to disable auto purge feature
 autopurge.purgeInterval=1
+# Enable all four letter word commands by default
+4lw.commands.whitelist=*
diff --git a/docker/base/scripts/install-zookeeper.sh b/docker/base/scripts/install-zookeeper.sh
@@ -19,34 +19,23 @@
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 zookeeper-<version-number>"
   echo "Example:"
-  echo "$ $0 zookeeper-3.4.14"
+  echo "$ $0 zookeeper-3.5.8"
   exit 1
 fi
 
 ZK_DIST=$1
 
-curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz"
-tar -xzf ./$ZK_DIST.tar.gz -C /opt
-rm ./$ZK_DIST.tar.gz
+curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/apache-$ZK_DIST-bin.tar.gz"
+tar -xzf ./apache-$ZK_DIST-bin.tar.gz -C /opt
+rm ./apache-$ZK_DIST-bin.tar.gz
+
+mv /opt/apache-$ZK_DIST-bin /opt/zookeeper
+rm -rf \
+  /opt/zookeeper/README.txt \
+  /opt/zookeeper/NOTICE.txt \
+  /opt/zookeeper/README_packaging.txt \
+  /opt/zookeeper/docs \
 
-mv /opt/$ZK_DIST /opt/zookeeper
-rm -rf /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README.txt \
-    /opt/zookeeper/NOTICE.txt \
-    /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README_packaging.txt \
-    /opt/zookeeper/build.xml \
-    /opt/zookeeper/config \
-    /opt/zookeeper/contrib \
-    /opt/zookeeper/dist-maven \
-    /opt/zookeeper/docs \
-    /opt/zookeeper/ivy.xml \
-    /opt/zookeeper/ivysettings.xml \
-    /opt/zookeeper/recipes \
-    /opt/zookeeper/src \
-    /opt/zookeeper/$ZK_DIST.jar.asc \
-    /opt/zookeeper/$ZK_DIST.jar.md5 \
-    /opt/zookeeper/$ZK_DIST.jar.sha1
 
 # copy zk scripts
 mkdir -p /opt/zookeeper/scripts
@@ -55,6 +44,6 @@ chmod +x /opt/heron-docker/scripts/generate-zookeeper-config.sh
 cp /opt/heron-docker/scripts/zookeeper-ruok.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/zookeeper-ruok.sh
 cp /opt/heron-docker/scripts/start-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/
+chmod +x /opt/heron-docker/scripts/start-zookeeper.sh
 cp /opt/heron-docker/scripts/wait-for-zookeeper.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/wait-for-zookeeper.sh
diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7
@@ -23,6 +23,7 @@ ENV bazelVersion 3.0.0
 
 RUN yum -y upgrade
 RUN yum -y install \
+      ant \
       automake \
       curl \
       cmake \

diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM debian
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \

diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM debian
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \

diff --git a/docker/compile/Dockerfile.ubuntu14.04 b/docker/compile/Dockerfile.ubuntu14.04
@@ -27,6 +27,7 @@ RUN add-apt-repository ppa:george-edison55/cmake-3.x
 RUN add-apt-repository ppa:openjdk-r/ppa
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       curl \

diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04
@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:openjdk-r/ppa
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \

diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM ubuntu
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       g++ \
       cmake \
       automake \

diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04
@@ -24,6 +24,7 @@ ENV bazelVersion 3.0.0
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get -y install \
+      ant \
       g++ \
       cmake \
       automake \

diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7
@@ -51,7 +51,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.debian10 b/docker/dist/Dockerfile.dist.debian10
@@ -51,7 +51,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.debian9 b/docker/dist/Dockerfile.dist.debian9
@@ -52,7 +52,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04
@@ -56,7 +56,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04
@@ -59,7 +59,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04
@@ -52,7 +52,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/Dockerfile.dist.ubuntu20.04 b/docker/dist/Dockerfile.dist.ubuntu20.04
@@ -54,7 +54,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST

diff --git a/docker/dist/conf/zookeeper.conf b/docker/dist/conf/zookeeper.conf
@@ -24,3 +24,5 @@ autopurge.snapRetainCount=3
 # Purge task interval in hours
 # Set to "0" to disable auto purge feature
 autopurge.purgeInterval=1
+# Enable all four letter word commands by default
+4lw.commands.whitelist=*
diff --git a/docker/dist/scripts/install-zookeeper.sh b/docker/dist/scripts/install-zookeeper.sh
@@ -19,34 +19,23 @@
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 zookeeper-<version-number>"
   echo "Example:"
-  echo "$ $0 zookeeper-3.4.14"
+  echo "$ $0 zookeeper-3.5.8"
   exit 1
 fi
 
 ZK_DIST=$1
 
-curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz"
-tar -xzf ./$ZK_DIST.tar.gz -C /opt
-rm ./$ZK_DIST.tar.gz
+curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/apache-$ZK_DIST-bin.tar.gz"
+tar -xzf ./apache-$ZK_DIST-bin.tar.gz -C /opt
+rm ./apache-$ZK_DIST-bin.tar.gz
+
+mv /opt/apache-$ZK_DIST-bin /opt/zookeeper
+rm -rf \
+  /opt/zookeeper/README.txt \
+  /opt/zookeeper/NOTICE.txt \
+  /opt/zookeeper/README_packaging.txt \
+  /opt/zookeeper/docs \
 
-mv /opt/$ZK_DIST /opt/zookeeper
-rm -rf /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README.txt \
-    /opt/zookeeper/NOTICE.txt \
-    /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README_packaging.txt \
-    /opt/zookeeper/build.xml \
-    /opt/zookeeper/config \
-    /opt/zookeeper/contrib \
-    /opt/zookeeper/dist-maven \
-    /opt/zookeeper/docs \
-    /opt/zookeeper/ivy.xml \
-    /opt/zookeeper/ivysettings.xml \
-    /opt/zookeeper/recipes \
-    /opt/zookeeper/src \
-    /opt/zookeeper/$ZK_DIST.jar.asc \
-    /opt/zookeeper/$ZK_DIST.jar.md5 \
-    /opt/zookeeper/$ZK_DIST.jar.sha1
 
 # copy zk scripts
 mkdir -p /opt/zookeeper/scripts
@@ -55,6 +44,6 @@ chmod +x /opt/heron-docker/scripts/generate-zookeeper-config.sh
 cp /opt/heron-docker/scripts/zookeeper-ruok.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/zookeeper-ruok.sh
 cp /opt/heron-docker/scripts/start-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/
+chmod +x /opt/heron-docker/scripts/start-zookeeper.sh
 cp /opt/heron-docker/scripts/wait-for-zookeeper.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/wait-for-zookeeper.sh
diff --git a/third_party/zookeeper/BUILD b/third_party/zookeeper/BUILD
@@ -39,6 +39,8 @@ genrule(
             "export CFLAGS=-Wno-error",
             "mkdir -p $$TMP_DIR",
             "cp -R $$(pwd)/external/org_apache_zookeeper/* $$TMP_DIR",
+            "cd $$TMP_DIR",
+            "ant compile_jute",
             "cd $$TMP_DIR/zookeeper-client/zookeeper-client-c",
             "autoreconf -if",
             "./configure --prefix=$$INSTALL_DIR --enable-shared=no",

diff --git a/third_party/zookeeper/pkgconfig.patch b/third_party/zookeeper/pkgconfig.patch
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,6 +23,7 @@ ENV bazelVersion 3.0.0 @@
     RUN yum -y upgrade
     RUN yum -y install \
+          ant \
           automake \
           curl \
           cmake \
@@ Expand Down @@