zookeeper to 3.5.8 (#3555)

huijunwu · huijunw · nicknezis · web-flow · commit 278c05a1a7f4 · 2020-07-10T17:11:09.000-04:00
* Upgrade zookeeper3.5.8

Co-authored-by: Huijun Wu &lt;huijunw@twitter.com&gt;
Co-authored-by: Nicholas Nezis &lt;nicholas.nezis@gmail.com&gt;
diff --git a/WORKSPACE b/WORKSPACE
@@ -66,7 +66,7 @@ maven_install(
     name = "maven",
     artifacts = [
         "antlr:antlr:2.7.7",
-        "org.apache.zookeeper:zookeeper:3.4.14",
+        "org.apache.zookeeper:zookeeper:3.5.8",
         "io.kubernetes:client-java:" + kubernetes_client_version,
         "com.esotericsoftware:kryo:3.0.3",
         "org.apache.avro:avro:1.7.4",
@@ -282,11 +282,8 @@ http_archive(
 http_archive(
     name = "org_apache_zookeeper",
     build_file = "@//:third_party/zookeeper/BUILD",
-    patch_args = ["-p2"],
-    patches = ["//third_party/zookeeper:pkgconfig.patch"],
-    sha256 = "b14f7a0fece8bd34c7fffa46039e563ac5367607c612517aa7bd37306afbd1cd",
-    strip_prefix = "zookeeper-3.4.14",
-    urls = ["https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz"],
+    strip_prefix = "apache-zookeeper-3.5.8",
+    urls = ["https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8.tar.gz"],
 )
 
 http_archive(
diff --git a/bazel_configure.py b/bazel_configure.py
@@ -422,6 +422,7 @@ def main():
 
   env_map['AR'] = discover_tool('ar', 'archiver', 'AR')
   env_map['GCOV']= discover_tool('gcov','coverage tool', 'GCOV')
+  env_map['ANT'] = discover_tool('ant', "ant", 'ANT')
   env_map['DWP'] = discover_tool_default('dwp', 'dwp', 'DWP', '/usr/bin/dwp')
   env_map['NM'] = discover_tool_default('nm', 'nm', 'NM', '/usr/bin/nm')
   env_map['OBJCOPY'] = discover_tool_default('objcopy', 'objcopy', 'OBJCOPY', '/usr/bin/objcopy')
diff --git a/deploy/kubernetes/helm/templates/bookie.yaml b/deploy/kubernetes/helm/templates/bookie.yaml
@@ -111,6 +111,21 @@ spec:
 {{- end }}
       terminationGracePeriodSeconds: 0
       initContainers:
+{{- if eq .Values.platform "minikube" }}
+        # The first time, initialize BK to wipe data for minikube
+        - name: bookie-format
+          image: {{ .Values.bookkeeper.image }}
+          args: ["/opt/bookkeeper/bin/bookkeeper", "shell", "bookieformat", "--nonInteractive", "-f", "-deleteCookie"]
+          command: [ "/bin/bash", "/opt/bookkeeper/scripts/entrypoint.sh" ]
+          envFrom:
+            - configMapRef:
+                name: {{ .Release.Name }}-bookie-config
+          volumeMounts:
+            - name: journal-disk
+              mountPath: /bookkeeper/data/journal
+            - name: ledgers-disk
+              mountPath: /bookkeeper/data/ledgers
+{{- end }}
         # Wait until the zookeeper pods are up and running
         - name: bookie-init-container
           image: {{ .Values.image }} # use heron image until bookkeeper has nc
diff --git a/docker/base/Dockerfile.base.debian9 b/docker/base/Dockerfile.base.debian9
@@ -18,6 +18,7 @@
 FROM openjdk:11-jdk-slim-stretch
 
 RUN apt-get -y update && apt-get -y install \
+    ant \
     netcat-openbsd \
     python3 \
     unzip \
@@ -27,7 +28,7 @@ RUN apt-get -y update && apt-get -y install \
     rm -rf /var/lib/apt/lists/*
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD base/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/base/conf/zookeeper.conf b/docker/base/conf/zookeeper.conf
@@ -24,3 +24,5 @@ autopurge.snapRetainCount=3
 # Purge task interval in hours
 # Set to "0" to disable auto purge feature
 autopurge.purgeInterval=1
+# Enable all four letter word commands by default
+4lw.commands.whitelist=*
diff --git a/docker/base/scripts/install-zookeeper.sh b/docker/base/scripts/install-zookeeper.sh
@@ -19,34 +19,23 @@
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 zookeeper-<version-number>"
   echo "Example:"
-  echo "$ $0 zookeeper-3.4.14"
+  echo "$ $0 zookeeper-3.5.8"
   exit 1
 fi
 
 ZK_DIST=$1
 
-curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz"
-tar -xzf ./$ZK_DIST.tar.gz -C /opt
-rm ./$ZK_DIST.tar.gz
+curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/apache-$ZK_DIST-bin.tar.gz"
+tar -xzf ./apache-$ZK_DIST-bin.tar.gz -C /opt
+rm ./apache-$ZK_DIST-bin.tar.gz
+
+mv /opt/apache-$ZK_DIST-bin /opt/zookeeper
+rm -rf \
+  /opt/zookeeper/README.txt \
+  /opt/zookeeper/NOTICE.txt \
+  /opt/zookeeper/README_packaging.txt \
+  /opt/zookeeper/docs \
 
-mv /opt/$ZK_DIST /opt/zookeeper
-rm -rf /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README.txt \
-    /opt/zookeeper/NOTICE.txt \
-    /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README_packaging.txt \
-    /opt/zookeeper/build.xml \
-    /opt/zookeeper/config \
-    /opt/zookeeper/contrib \
-    /opt/zookeeper/dist-maven \
-    /opt/zookeeper/docs \
-    /opt/zookeeper/ivy.xml \
-    /opt/zookeeper/ivysettings.xml \
-    /opt/zookeeper/recipes \
-    /opt/zookeeper/src \
-    /opt/zookeeper/$ZK_DIST.jar.asc \
-    /opt/zookeeper/$ZK_DIST.jar.md5 \
-    /opt/zookeeper/$ZK_DIST.jar.sha1
 
 # copy zk scripts
 mkdir -p /opt/zookeeper/scripts
@@ -55,6 +44,6 @@ chmod +x /opt/heron-docker/scripts/generate-zookeeper-config.sh
 cp /opt/heron-docker/scripts/zookeeper-ruok.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/zookeeper-ruok.sh
 cp /opt/heron-docker/scripts/start-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/
+chmod +x /opt/heron-docker/scripts/start-zookeeper.sh
 cp /opt/heron-docker/scripts/wait-for-zookeeper.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/wait-for-zookeeper.sh
diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7
@@ -23,8 +23,10 @@ ENV bazelVersion 3.0.0
 
 RUN yum -y upgrade
 RUN yum -y install \
+      ant \
       automake \
       curl \
+      cppunit-devel \
       cmake \
       openssl-devel \
       file \
diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM debian
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \
diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM debian
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \
diff --git a/docker/compile/Dockerfile.ubuntu14.04 b/docker/compile/Dockerfile.ubuntu14.04
@@ -27,6 +27,7 @@ RUN add-apt-repository ppa:george-edison55/cmake-3.x
 RUN add-apt-repository ppa:openjdk-r/ppa
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       curl \
diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04
@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:openjdk-r/ppa
 
 RUN apt-get update && apt-get -y install \
+      ant \
       automake \
       build-essential \
       cmake \
diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04
@@ -22,6 +22,7 @@ ENV TARGET_PLATFORM ubuntu
 ENV bazelVersion 3.0.0
 
 RUN apt-get update && apt-get -y install \
+      ant \
       g++ \
       cmake \
       automake \
diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04
@@ -24,6 +24,7 @@ ENV bazelVersion 3.0.0
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get -y install \
+      ant \
       g++ \
       cmake \
       automake \
diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7
@@ -52,7 +52,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.debian10 b/docker/dist/Dockerfile.dist.debian10
@@ -52,7 +52,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.debian9 b/docker/dist/Dockerfile.dist.debian9
@@ -53,7 +53,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04
@@ -57,7 +57,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04
@@ -60,7 +60,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04
@@ -53,7 +53,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/Dockerfile.dist.ubuntu20.04 b/docker/dist/Dockerfile.dist.ubuntu20.04
@@ -55,7 +55,7 @@ ENV HERON_HOME /heron/heron-core/
 RUN export HERON_HOME
 
 # install zookeeper
-ARG ZK_DIST=zookeeper-3.4.14
+ARG ZK_DIST=zookeeper-3.5.8
 ADD dist/scripts /opt/heron-docker/scripts
 
 RUN sh /opt/heron-docker/scripts/install-zookeeper.sh $ZK_DIST
diff --git a/docker/dist/conf/zookeeper.conf b/docker/dist/conf/zookeeper.conf
@@ -24,3 +24,5 @@ autopurge.snapRetainCount=3
 # Purge task interval in hours
 # Set to "0" to disable auto purge feature
 autopurge.purgeInterval=1
+# Enable all four letter word commands by default
+4lw.commands.whitelist=*
diff --git a/docker/dist/scripts/install-zookeeper.sh b/docker/dist/scripts/install-zookeeper.sh
@@ -19,34 +19,23 @@
 if [ "$#" -ne 1 ]; then
   echo "Usage: $0 zookeeper-<version-number>"
   echo "Example:"
-  echo "$ $0 zookeeper-3.4.14"
+  echo "$ $0 zookeeper-3.5.8"
   exit 1
 fi
 
 ZK_DIST=$1
 
-curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/$ZK_DIST.tar.gz"
-tar -xzf ./$ZK_DIST.tar.gz -C /opt
-rm ./$ZK_DIST.tar.gz
+curl -O "https://archive.apache.org/dist/zookeeper/$ZK_DIST/apache-$ZK_DIST-bin.tar.gz"
+tar -xzf ./apache-$ZK_DIST-bin.tar.gz -C /opt
+rm ./apache-$ZK_DIST-bin.tar.gz
+
+mv /opt/apache-$ZK_DIST-bin /opt/zookeeper
+rm -rf \
+  /opt/zookeeper/README.txt \
+  /opt/zookeeper/NOTICE.txt \
+  /opt/zookeeper/README_packaging.txt \
+  /opt/zookeeper/docs \
 
-mv /opt/$ZK_DIST /opt/zookeeper
-rm -rf /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README.txt \
-    /opt/zookeeper/NOTICE.txt \
-    /opt/zookeeper/CHANGES.txt \
-    /opt/zookeeper/README_packaging.txt \
-    /opt/zookeeper/build.xml \
-    /opt/zookeeper/config \
-    /opt/zookeeper/contrib \
-    /opt/zookeeper/dist-maven \
-    /opt/zookeeper/docs \
-    /opt/zookeeper/ivy.xml \
-    /opt/zookeeper/ivysettings.xml \
-    /opt/zookeeper/recipes \
-    /opt/zookeeper/src \
-    /opt/zookeeper/$ZK_DIST.jar.asc \
-    /opt/zookeeper/$ZK_DIST.jar.md5 \
-    /opt/zookeeper/$ZK_DIST.jar.sha1
 
 # copy zk scripts
 mkdir -p /opt/zookeeper/scripts
@@ -55,6 +44,6 @@ chmod +x /opt/heron-docker/scripts/generate-zookeeper-config.sh
 cp /opt/heron-docker/scripts/zookeeper-ruok.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/zookeeper-ruok.sh
 cp /opt/heron-docker/scripts/start-zookeeper.sh /opt/zookeeper/scripts/
-chmod +x /opt/heron-docker/scripts/
+chmod +x /opt/heron-docker/scripts/start-zookeeper.sh
 cp /opt/heron-docker/scripts/wait-for-zookeeper.sh /opt/zookeeper/scripts/
 chmod +x /opt/heron-docker/scripts/wait-for-zookeeper.sh
diff --git a/heron/statemgrs/tests/java/BUILD b/heron/statemgrs/tests/java/BUILD
@@ -22,6 +22,7 @@ zookeeper_statemgr_deps_files = [
     "//heron/statemgrs/src/java:zookeeper-statemgr-java",
     "@maven//:org_apache_curator_curator_framework",
     "@maven//:org_apache_zookeeper_zookeeper",
+    "@maven//:org_apache_zookeeper_zookeeper_jute",
 ]
 
 localfs_deps_files = \
diff --git a/maven_install.json b/maven_install.json
diff --git a/third_party/zookeeper/BUILD b/third_party/zookeeper/BUILD
diff --git a/third_party/zookeeper/pkgconfig.patch b/third_party/zookeeper/pkgconfig.patch
diff --git a/website2/docs/compiling-linux.md b/website2/docs/compiling-linux.md
diff --git a/website2/docs/compiling-osx.md b/website2/docs/compiling-osx.md
diff --git a/website2/docs/compiling-overview.md b/website2/docs/compiling-overview.md

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@ zookeeper_statemgr_deps_files = [`
`22`	`22`	`"//heron/statemgrs/src/java:zookeeper-statemgr-java",`
`23`	`23`	`"@maven//:org_apache_curator_curator_framework",`
`24`	`24`	`"@maven//:org_apache_zookeeper_zookeeper",`
	`25`	`+ "@maven//:org_apache_zookeeper_zookeeper_jute",`
`25`	`26`	`]`
`26`	`27`
`27`	`28`	`localfs_deps_files = \`