whereabouts watcher

we have to break the tokenwatcher section of the whereabouts script out
because it was running in an init container, so it would never
"complete", hanging the rollout

to accomplish this I created a new whereabouts watcher ds to house this
section of the script

Signed-off-by: Benjamin Pickard <[email protected]>

Author: bpickard22

bindata/network/multus/multus.yaml

@@ -567,7 +567,7 @@ spec:
 
             CNI_BIN_DIR=${CNI_BIN_DIR:-"/host/opt/cni/bin/"}
             WHEREABOUTS_KUBECONFIG_FILE_HOST=${WHEREABOUTS_KUBECONFIG_FILE_HOST:-"/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig"}
-            CNI_CONF_DIR=${CNI_CONF_DIR:-"/host/etc/cni/net.d"}
+            CNI_CONF_DIR=${CNI_CONF_DIR:-"/host{{ .SystemCNIConfDir }}"}
             WHEREABOUTS_RECONCILER_CRON=${WHEREABOUTS_RECONCILER_CRON:-30 4 * * *}
 
             # Make a whereabouts.d directory (for our kubeconfig)
@@ -655,8 +655,8 @@ spec:
             EOF
 
             # Copy the config from ConfigMap to the desired directory
-            cp /etc/whereabouts/config/whereabouts.conf $WHEREABOUTS_GLOBALCONFIG
-            chmod 600 $ WHEREABOUTS_CONF_FILE
+            cp /etc/whereabouts/config/whereabouts.conf $WHEREABOUTS_CONF_FILE
+            chmod 600 $WHEREABOUTS_CONF_FILE
 
             else
               warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
@@ -680,24 +680,12 @@ spec:
 
             }
 
-            function get_token_md5sum {
-              md5sum "$SERVICE_ACCOUNT_TOKEN_PATH" | awk '{print $1}'
-            }
-
-            function get_ca_file_md5sum {
-              if [ ! -f "$KUBE_CA_FILE" ]; then
-                echo ""
-                return
-              fi
-              md5sum "$KUBE_CA_FILE" | awk '{print $1}'
-            }
             generateKubeConfig
-            export LAST_SERVICEACCOUNT_MD5SUM="$(get_token_md5sum)"
-            export LAST_KUBE_CA_FILE_MD5SUM="$(get_ca_file_md5sum)"
             # ------------------ end Generate a "kube-config"
 
             # ----------------- Generate a whereabouts conf
-            generateWhereaboutsConf
+            # removed because we have the configmap
+            #generateWhereaboutsConf
             # ---------------- End generate a whereabouts conf
 
 
@@ -707,22 +695,6 @@ spec:
 
             # ---------------------- end generate a "kube-config".
 
-            echo "Sleep and Watching for service account token and CA file changes..."
-            # enter sleep/watch loop
-            while true; do
-              # Check the md5sum of the service account token and ca.
-              svcaccountsum="$(get_token_md5sum)"
-              casum="$(get_ca_file_md5sum)"
-              if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || ! [ "$SKIP_TLS_VERIFY" == "true" ] && [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then
-                log "Detected service account or CA file change, regenerating kubeconfig..."
-                generateKubeConfig
-                LAST_SERVICEACCOUNT_MD5SUM="$svcaccountsum"
-                LAST_KUBE_CA_FILE_MD5SUM="$casum"
-              fi
-
-              sleep 1s
-            done
-
             # Unless told otherwise, sleep forever.
             # This prevents Kubernetes from restarting the pod repeatedly.
             should_sleep=${SLEEP:-"true"}
@@ -953,5 +925,167 @@ spec:
             items:
             - key: whereabouts.conf
               path: whereabouts.conf
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: whereabouts-token-watcher
+  namespace: openshift-multus
+  annotations:
+    kubernetes.io/description: |
+      This deamon watches over the whereabouts service account token and CA
+      file for changes and will regenerate a kubeconfig if changes are seen
+spec:
+  selector:
+    matchLabels:
+      app: whereabouts-token-watcher
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 10%
+  template:
+    metadata:
+      annotations:
+        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
+          # prevent blocks when node critical pods get evicted prior to workloads
+        cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false"
+      labels:
+        app: whereabouts-token-watcher
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: "system-node-critical"
+      tolerations:
+      - operator: Exists
+      serviceAccountName: multus-ancillary-tools
+      containers:
+      - name: whereabouts-token-watcher
+        image: {{.WhereaboutsImage}}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          #!/bin/sh
+
+          set -u -e
+
+          SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+          KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
+          SERVICE_ACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+          SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token
+          SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
+
+          function generateKubeConfig {
+          # Check if we're running as a k8s pod.
+          if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
+            # We're running as a k8d pod - expect some variables.
+            if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
+              error "KUBERNETES_SERVICE_HOST not set"; exit 1;
+            fi
+            if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
+              error "KUBERNETES_SERVICE_PORT not set"; exit 1;
+            fi
+
+            if [ "$SKIP_TLS_VERIFY" == "true" ]; then
+              TLS_CFG="insecure-skip-tls-verify: true"
+            elif [ -f "$KUBE_CA_FILE" ]; then
+              TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+            fi
+
+            # Kubernetes service address must be wrapped if it is IPv6 address
+            KUBERNETES_SERVICE_HOST_WRAP=$KUBERNETES_SERVICE_HOST
+            if [ "$KUBERNETES_SERVICE_HOST_WRAP" != "${KUBERNETES_SERVICE_HOST_WRAP#*:[0-9a-fA-F]}" ]; then
+              KUBERNETES_SERVICE_HOST_WRAP=\[$KUBERNETES_SERVICE_HOST_WRAP\]
+            fi
+             # Write a kubeconfig file for the CNI plugin.  Do this
+             # to skip TLS verification for now.  We should eventually support
+             # writing more complete kubeconfig files. This is only used
+             # if the provided CNI network config references it.
+             touch $WHEREABOUTS_KUBECONFIG
+             chmod ${KUBECONFIG_MODE:-600} $WHEREABOUTS_KUBECONFIG
+             cat > $WHEREABOUTS_KUBECONFIG <<EOF
+          # Kubeconfig file for the Whereabouts CNI plugin.
+          apiVersion: v1
+          kind: Config
+          clusters:
+          - name: local
+            cluster:
+              server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST_WRAP}:${KUBERNETES_SERVICE_PORT}
+              $TLS_CFG
+          users:
+          - name: whereabouts
+            user:
+              token: "${SERVICE_ACCOUNT_TOKEN}"
+          contexts:
+          - name: whereabouts-context
+            context:
+              cluster: local
+              user: whereabouts
+              namespace: ${WHEREABOUTS_NAMESPACE}
+          current-context: whereabouts-context
+          EOF
+
+          else
+            warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
+          fi
+
+          }
+
+          function get_token_md5sum {
+            md5sum "$SERVICE_ACCOUNT_TOKEN_PATH" | awk '{print $1}'
+          }
+
+          function get_ca_file_md5sum {
+            if [ ! -f "$KUBE_CA_FILE" ]; then
+              echo ""
+              return
+            fi
+            md5sum "$KUBE_CA_FILE" | awk '{print $1}'
+          }
+
+          export LAST_SERVICEACCOUNT_MD5SUM="$(get_token_md5sum)"
+          export LAST_KUBE_CA_FILE_MD5SUM="$(get_ca_file_md5sum)"
+
+          echo "Sleep and Watching for service account token and CA file changes..."
+          # enter sleep/watch loop
+          while true; do
+            # Check the md5sum of the service account token and ca.
+            svcaccountsum="$(get_token_md5sum)"
+            casum="$(get_ca_file_md5sum)"
+            if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || ! [ "$SKIP_TLS_VERIFY" == "true" ] && [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then
+              log "Detected service account or CA file change, regenerating kubeconfig..."
+              generateKubeConfig
+              LAST_SERVICEACCOUNT_MD5SUM="$svcaccountsum"
+              LAST_KUBE_CA_FILE_MD5SUM="$casum"
+            fi
+
+            sleep 1s
+          done
+
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: whereabouts-configmap
+          mountPath: /etc/wherabouts/config
+        env:
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{.KUBERNETES_SERVICE_PORT}}"
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{.KUBERNETES_SERVICE_HOST}}"
+        - name: CNI_BIN_DIR
+          value: "/host/opt/cni/bin/"
+        - name: CNI_CONF_DIR
+          value: "/host/etc/cni/net.d"
+        - name: SLEEP
+          value: "false"
+        - name: WHEREABOUTS_NAMESPACE
+          value: "openshift-multus"
+      volumes:
+        - name: whereabouts-configmap
+          configMap:
+            name: whereabouts-config
 {{- end}}
 ---