bookie.yaml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

## A reference of https://github.com/apache/incubator-pulsar/blob/master/kubernetes/google-container-engine/bookie.yaml

{{- $platform := .Values.platform }}
{{- $bookieReplicas := .Values.bookkeeper.replicas }}
{{- $bookieCpuMin := .Values.bookkeeper.resources.cpuMin }}
{{- $bookieCpuMax := .Values.bookkeeper.resources.cpuMax }}
{{- $bookieHeapMemory := .Values.bookkeeper.resources.heapMemory }}
{{- $bookieDirectMemory := .Values.bookkeeper.resources.directMemory }}
{{- $bookieJournalCapacity := .Values.bookkeeper.journal.capacity }}
{{- $bookieJournalHostPath := .Values.bookkeeper.journal.hostPath }}
{{- $bookieLedgersCapacity := .Values.bookkeeper.ledgers.capacity }}
{{- $bookieLedgersHostPath := .Values.bookkeeper.ledgers.hostPath }}

apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ .Release.Name }}-bookie-config
data:
  BK_BOOKIE_EXTRA_OPTS: "\"-Xms{{ $bookieHeapMemory }} -Xmx{{ $bookieHeapMemory }} -XX:MaxDirectMemorySize={{ $bookieDirectMemory }} -XX:+UseG1GC  -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB\""
  BK_bookiePort: "3181"
  BK_httpServerPort: "8080"
  BK_journalDirectory: "/bookkeeper/data/journal"
  BK_ledgerDirectories: "/bookkeeper/data/ledgers"
  BK_indexDirectories: "/bookkeeper/data/ledgers" 
  BK_zkServers: {{ .Release.Name }}-zookeeper:{{ .Values.zookeeper.clientPort }}
  BK_autoRecoveryDaemonEnabled: "true"
  {{- if .Values.bookkeeper.useHostNameAsBookieID }}
  BK_useHostNameAsBookieID: "true"
  {{- end }}
  # TODO: Issue 458: https://github.com/apache/bookkeeper/issues/458
  {{- if eq .Values.bookkeeper.prometheus.enabled true }}
  BK_enableStatistics: "true"
  BK_enableTaskExecutionStats: "true"
  BK_statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
  {{- end }}

---
## BookKeeper servers need to access the local disks and the pods
## cannot be moved across different nodes.
## For this reason, we run BK as a daemon set, one for each node in the
## cluster, unless restricted by label selectors
{{- if .Values.bookkeeper.useStatefulSet }}
apiVersion: apps/v1
kind: StatefulSet
{{- else }}
apiVersion: apps/v1
kind: DaemonSet
{{- end }}

metadata:
  name: {{ .Release.Name }}-bookie
  labels:
    app: {{ .Release.Name }}-bookkeeper
    component: {{ .Release.Name }}-bookie
spec:
  selector:
    matchLabels:
      app: {{ .Release.Name }}-bookkeeper
      component: {{ .Release.Name }}-bookie
      cluster: {{ .Release.Name }}-bookkeeper
{{- if .Values.bookkeeper.createReplicas }}
  serviceName: {{ .Release.Name }}-bookie
  replicas: {{ $bookieReplicas }}
{{- end }}

  template:
    metadata:
      labels:
        app: {{ .Release.Name }}-bookkeeper
        component: {{ .Release.Name }}-bookie
        # Specify cluster to allow aggregation by cluster in
        # the metrics
        cluster: {{ .Release.Name }}-bookkeeper
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"

    spec:
{{- if .Values.bookkeeper.affinityPods }}
      # Make sure multiple pods of bookkeeper don't get scheduled on the
      # same node, unless there are no other available nodes
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 1
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: "app"
                      operator: In
                      values:
                        - {{ .Release.Name }}-bookkeeper
                topologyKey: "kubernetes.io/hostname"
      terminationGracePeriodSeconds: 0
{{- end }}
      initContainers:
        # Wait until the zookeeper pods are up and running
        - name: bookie-init-container
          image: {{ .Values.image }} # use heron image until bookkeeper has nc
          command: 
            - sh
            - -c
            - >-
              while true; do
                status=$(echo ruok | nc -w 3 {{ .Release.Name }}-zookeeper {{ .Values.zookeeper.clientPort }});
                writestatus=$(echo isro | nc -w 3 {{ .Release.Name }}-zookeeper {{ .Values.zookeeper.clientPort }})
                if [ "$status" = "imok" ] && [ "$writestatus" = "rw" ]; then
                  break;
                fi;
                echo 'Zookeeper {{ .Release.Name }}-zookeeper:{{ .Values.zookeeper.clientPort }} not ready';
                sleep 4;
              done
      containers:
        - name: bookie
          image: {{ .Values.bookkeeper.image }}
          resources:
            requests:
              cpu: {{ $bookieCpuMin | quote }}
            limits:
              cpu: {{ $bookieCpuMax | quote }}
          # use the patched entrypoint.sh - it will automatically created the desired distributedlog namespace
          command: [ "/bin/bash", "/opt/bookkeeper/scripts/entrypoint.sh" ]
          args: ["/opt/bookkeeper/bin/bookkeeper", "bookie"]
          ports:
            - name: client
              containerPort: 3181
          envFrom:
            - configMapRef:
                name: {{ .Release.Name }}-bookie-config
          volumeMounts:
            - name: journal-disk
              mountPath: /bookkeeper/data/journal
            - name: ledgers-disk
              mountPath: /bookkeeper/data/ledgers

{{- if .Values.bookkeeper.useHostPath }}
      volumes:
          # Mount local disks
        - name: journal-disk
          hostPath:
            path: {{ $bookieJournalHostPath }}
        - name: ledgers-disk
          hostPath:
            path: {{ $bookieLedgersHostPath }}
{{- end }}

{{- if .Values.bookkeeper.useVolumeClaimTemplate }}
  volumeClaimTemplates:
    - metadata:
        name: journal-disk
        annotations:
          volume.alpha.kubernetes.io/storage-class: default
        labels:
          component: {{ .Release.Name }}-bookkeeper
      spec:
        accessModes: [ "ReadWriteOnce" ]
        {{- if ne .Values.bookkeeper.storageClassName "none" }}
        storageClassName: {{ .Values.bookkeeper.storageClassName }}
        {{- end }}
        resources:
          requests:
            storage: {{ $bookieJournalCapacity }}
    - metadata:
        name: ledgers-disk
        annotations:
          volume.alpha.kubernetes.io/storage-class: default
        labels:
          component: {{ .Release.Name }}-bookkeeper
      spec:
        accessModes: [ "ReadWriteOnce" ]
        {{- if ne .Values.bookkeeper.storageClassName "none" }}
        storageClassName: {{ .Values.bookkeeper.storageClassName }}
        {{- end }}
        resources:
          requests:
            storage: {{ $bookieLedgersCapacity }}
{{- end }}

---
##
## Define the Bookie headless service
## In practice, in this case, it is only useful to have a view of
## all the bookie pods that are present
##
apiVersion: v1
kind: Service
metadata:
  {{- if .Values.bookkeeper.tolerateUnreadyEndpoints }}
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
  {{- end }}
  name: {{ .Release.Name }}-bookie
  labels:
    app: {{ .Release.Name }}-bookkeeper
    component: {{ .Release.Name }}-bookie
spec:
  ports:
  - port: 3181
    name: server
  clusterIP: None
  selector:
    app: {{ .Release.Name }}-bookkeeper
    component: {{ .Release.Name }}-bookie