fix: Update oss charts. Add acceptance tests for helm (airbytehq#16131)

* fix: revert extraEnv. Move updated extraEnv to env_vars

* fix: Add node pool selector feature for gke deployments

* fix: hotfix. Fix webapp values and imagepullsecrets

* fix: remove nodeSelector duplicate

* fix: revert imagePullSecrets changes

* fix nit

* fix: fix bootloader imagepullsecrets

* fix: fix image pull secrets again...

* feat: add acceptance tests for helm

* add step to install unzip

* fix incorrect command spelling(helm)

* update slack notification sending for Airbyte CI workflow

* Rename Helm E2E test report name

* fix: Update helm charts

* fix: fix temporal DB Port bug

* fix nits

* do a small testing for sake of science

* Update test chart.yaml with note block

* Revert changes for AC tests

* fix: Include airbytehq#16166

* fix: add metrics to the ci workflow

* fix: revert changes for temporal service. move them to server service;

* fix: revert fullstory deletion

* fix: change way of defining image tag and make it to be set in global values. Add values.yaml.test for integration testing and local deployment

* fix: udpate docs, update values.yaml

* fix: fix helpers.tpl in all subcharts

* Update chart docs

Co-authored-by: Davin Chia <[email protected]>

Author: 2 people

.github/workflows/gradle.yml

@@ -700,6 +700,167 @@ jobs:
           label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
 
+
+  ## Kube Acceptance Tests
+  # Docker acceptance tests run as part of the build job.
+  # In case of self-hosted EC2 errors, remove this block.
+  start-helm-acceptance-test-runner:
+    name: "Platform: Start Helm Acceptance Test Runner"
+    needs:
+      - changes
+      - find_valid_pat
+    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+          ec2-image-id: ami-0c1a9bc22624339d8
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+
+
+  helm-acceptance-test:
+    name: "Platform: Acceptance Tests (Helm)"
+    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+    needs: start-helm-acceptance-test-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+    environment: more-secrets
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - name: Cache Build Artifacts
+        uses: ./.github/actions/cache-build-artifacts
+        with:
+          cache-key: ${{ secrets.CACHE_VERSION }}
+          cache-python: "false"
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "lts/gallium"
+
+      - name: Install unzip
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y unzip
+
+      - uses: azure/setup-helm@v3
+        with:
+          version: 'latest' 
+          token: ${{ secrets.GITHUB_TOKEN }}
+        id: install
+   
+      - name: Fix EC-2 Runner
+        run: |
+          mkdir -p /home/runner
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Create cluster config file
+        run: |
+          cat > /tmp/kind-config.yaml <<EOF
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          nodes:
+          - role: control-plane
+          - role: worker
+          EOF
+
+      - name: KIND Kubernetes Cluster Setup
+        uses: helm/[email protected]
+        with:
+          node_image: kindest/node:v1.21.2
+          config: /tmp/kind-config.yaml
+        # In case of self-hosted EC2 errors, remove this env block.
+        env:
+          USER: root
+          HOME: /home/runner
+          CHANGE_MINIKUBE_NONE_USER: true
+
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+
+      - name: Run Helm End-to-End Acceptance Tests
+        env:
+          USER: root
+          HOME: /home/runner
+          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
+          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+        run: |
+          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()    # run this step even if previous step failed
+        with:
+          name: Platform Helm E2E Test Report
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: Kubernetes Logs
+          path: /tmp/kubernetes_logs/*
+
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-helm-acceptance-test-runner:
+    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
+    timeout-minutes: 10
+    needs:
+      - start-helm-acceptance-test-runner # required to get output from the start-runner job
+      - helm-acceptance-test # required to wait when the main job is done
+      - find_valid_pat
+    runs-on: ubuntu-latest
+    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+    # Thus, we check for skipped here.
+    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: supertopher/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}
+
+
+
   notify-failure-slack-channel:
     name: "Notify Slack Channel on Build Failures"
     runs-on: ubuntu-latest
@@ -709,6 +870,7 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      - helm-acceptance-test
     if: ${{ failure() && github.ref == 'refs/heads/master' }}
     steps:
       - name: Publish to OSS Build Failure Slack Channel
@@ -733,6 +895,7 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      - helm-acceptance-test
     if: success()
     steps:
       - name: Get Previous Workflow Status

charts/airbyte-bootloader/README.md

@@ -1,6 +1,6 @@
 # airbyte-bootloader
 
-![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.39.41-alpha](https://img.shields.io/badge/AppVersion-0.39.41--alpha-informational?style=flat-square)
+![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.4](https://img.shields.io/badge/AppVersion-0.40.4-informational?style=flat-square)
 
 Helm chart to deploy airbyte-bootloader
 
@@ -16,17 +16,27 @@ Helm chart to deploy airbyte-bootloader
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
 | enabled | bool | `true` |  |
+| env_vars | object | `{}` |  |
+| extraContainers | list | `[]` |  |
+| extraEnv | list | `[]` |  |
+| extraInitContainers | list | `[]` |  |
+| extraVolumeMounts | list | `[]` |  |
+| extraVolumes | list | `[]` |  |
 | global.database.secretName | string | `""` |  |
 | global.database.secretValue | string | `""` |  |
 | global.deploymentMode | string | `"oss"` |  |
+| global.env_vars | object | `{}` |  |
+| global.extraContainers | list | `[]` |  |
 | global.secretName | string | `""` |  |
+| global.secrets | object | `{}` |  |
 | global.serviceAccountName | string | `"placeholderServiceAccount"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"airbyte/bootloader"` |  |
 | nodeSelector | object | `{}` |  |
 | podAnnotations | object | `{}` |  |
 | resources.limits | object | `{}` |  |
 | resources.requests | object | `{}` |  |
+| secrets | object | `{}` |  |
 | tolerations | list | `[]` |  |
 
 ----------------------------------------------

charts/airbyte-bootloader/templates/_helpers.tpl

@@ -30,6 +30,26 @@ Create chart name and version as used by the chart label.
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Common labels
+*/}}
+{{- define "airbyte.labels" -}}
+helm.sh/chart: {{ include "airbyte.chart" . }}
+{{ include "airbyte.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "airbyte.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "airbyte.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
 {{/*
 Define db secret
 */}}
@@ -49,3 +69,17 @@ Define db secret
     {{- end -}}
 {{- end -}}
 {{- end -}}
+
+{{/* 
+Define imageTag
+*/}}
+
+{{- define "bootloader.imageTag" -}}
+{{- if ((.Values.global.image).tag) }}
+    {{- printf "%s" .Values.global.image.tag }}
+{{- else if .Values.image.tag }}
+    {{- printf "%s" .Values.image.tag }}
+{{- else }}
+    {{- printf "%s" .Chart.AppVersion }}
+{{- end }}
+{{- end }}

charts/airbyte-bootloader/templates/bootloader-secrets.yaml

@@ -5,12 +5,7 @@ kind: Secret
 metadata:
   name: bootloader-secrets
   labels:
-    app.kubernetes.io/name: {{ include "airbyte.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    helm.sh/chart: {{ include "airbyte.chart" . }}
-    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.airbyte.io/fullname: {{ include "airbyte.fullname" . }}
+    {{- include "airbyte.labels" . | nindent 4 }}
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
     helm.sh/hook-weight: "-1"

charts/airbyte-bootloader/templates/pod.yaml

@@ -4,12 +4,7 @@ kind: Pod
 metadata:
   name: {{ include "common.names.fullname" . }}
   labels:
-    app.kubernetes.io/name: {{ include "airbyte.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    helm.sh/chart: {{ include "airbyte.chart" . }}
-    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.airbyte.io/fullname: {{ include "airbyte.fullname" . }}
+    {{- include "airbyte.labels" . | nindent 4 }}
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
     helm.sh/hook-weight: "0"
@@ -36,7 +31,7 @@ spec:
   {{- end }}
   containers:
     - name: airbyte-bootloader-container
-      image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }}
+      image: {{ printf "%s:%s" .Values.image.repository (include "bootloader.imageTag" .) }}
       imagePullPolicy: "{{ .Values.image.pullPolicy }}"
       env:
         {{- if eq .Values.global.deploymentMode "oss"  }}
@@ -58,8 +53,8 @@ spec:
         - name: DATABASE_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ .Values.global.database.secretName | default (printf "%s-postgresql" .Release.Name ) }}
-              key: {{ .Values.global.database.secretValue | default "postgresql-password" }}
+              name: {{ .Values.global.database.secretName | default (printf "%s-airbyte-secrets" .Release.Name ) }}
+              key: {{ .Values.global.database.secretValue | default "DATABASE_PASSWORD" }}
         - name: DATABASE_URL
           valueFrom:
             configMapKeyRef:
@@ -68,7 +63,7 @@ spec:
         - name: DATABASE_USER
           valueFrom:
             secretKeyRef:
-              name: {{ .Values.global.configMapName | default (printf "%s-airbyte-secrets" .Release.Name) }}
+              name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }}
               key: DATABASE_USER
         {{- end }}
         # Values from secret
@@ -96,3 +91,17 @@ spec:
         {{- end }}
 
       resources: {{- toYaml .Values.resources | nindent 8 }}
+{{- if .Values.extraVolumeMounts }}
+      volumeMounts:
+{{- toYaml .Values.extraVolumeMounts | nindent 8 }}
+{{- end }}
+{{- if .Values.extraContainers }}
+{{ toYaml .Values.extraContainers | indent 4 }}
+{{- end }}
+{{- if .Values.global.extraContainers }}
+{{ toYaml .Values.global.extraContainers | indent 4 }}
+{{- end }}
+{{- if .Values.extraVolumes }}
+  volumes:
+{{- toYaml .Values.extraVolumes | nindent 2 }}
+{{- end }}