From 272e100a51a74836265bf8f21b4ec7d1c4b48c1b Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 31 Aug 2021 01:14:58 -0500 Subject: [PATCH 01/16] Initial Helm chart creation --- .github/workflows/helm.yaml | 84 ++++++ charts/airbyte/.gitignore | 2 + charts/airbyte/.helmignore | 25 ++ charts/airbyte/Chart.lock | 12 + charts/airbyte/Chart.yaml | 39 +++ charts/airbyte/Makefile | 28 ++ charts/airbyte/README.md | 121 +++++++++ charts/airbyte/files/sweep-pod.sh | 40 +++ charts/airbyte/templates/NOTES.txt | 22 ++ charts/airbyte/templates/_helpers.tpl | 167 ++++++++++++ charts/airbyte/templates/env-configmap.yaml | 44 ++++ .../templates/gcs-log-creds-secret.yaml | 7 + .../templates/pod-sweeper/configmap.yaml | 8 + .../templates/pod-sweeper/deployment.yaml | 37 +++ .../templates/scheduler/deployment.yaml | 205 ++++++++++++++ .../airbyte/templates/server/deployment.yaml | 182 +++++++++++++ .../airbyte/templates/server/pvc-configs.yaml | 14 + .../templates/server/pvc-workspace.yaml | 14 + charts/airbyte/templates/server/service.yaml | 14 + charts/airbyte/templates/serviceaccount.yaml | 34 +++ .../airbyte/templates/temporal/configmap.yaml | 48 ++++ .../templates/temporal/deployment.yaml | 52 ++++ .../airbyte/templates/temporal/service.yaml | 13 + .../airbyte/templates/tests/test-webapp.yaml | 15 ++ .../airbyte/templates/webapp/deployment.yaml | 66 +++++ charts/airbyte/templates/webapp/ingress.yaml | 61 +++++ charts/airbyte/templates/webapp/service.yaml | 14 + charts/airbyte/values.yaml | 249 ++++++++++++++++++ 28 files changed, 1617 insertions(+) create mode 100644 .github/workflows/helm.yaml create mode 100644 charts/airbyte/.gitignore create mode 100644 charts/airbyte/.helmignore create mode 100644 charts/airbyte/Chart.lock create mode 100644 charts/airbyte/Chart.yaml create mode 100644 charts/airbyte/Makefile create mode 100644 charts/airbyte/README.md create mode 100644 charts/airbyte/files/sweep-pod.sh create mode 100644 charts/airbyte/templates/NOTES.txt create mode 100644 charts/airbyte/templates/_helpers.tpl create mode 100644 charts/airbyte/templates/env-configmap.yaml create mode 100644 charts/airbyte/templates/gcs-log-creds-secret.yaml create mode 100644 charts/airbyte/templates/pod-sweeper/configmap.yaml create mode 100644 charts/airbyte/templates/pod-sweeper/deployment.yaml create mode 100644 charts/airbyte/templates/scheduler/deployment.yaml create mode 100644 charts/airbyte/templates/server/deployment.yaml create mode 100644 charts/airbyte/templates/server/pvc-configs.yaml create mode 100644 charts/airbyte/templates/server/pvc-workspace.yaml create mode 100644 charts/airbyte/templates/server/service.yaml create mode 100644 charts/airbyte/templates/serviceaccount.yaml create mode 100644 charts/airbyte/templates/temporal/configmap.yaml create mode 100644 charts/airbyte/templates/temporal/deployment.yaml create mode 100644 charts/airbyte/templates/temporal/service.yaml create mode 100644 charts/airbyte/templates/tests/test-webapp.yaml create mode 100644 charts/airbyte/templates/webapp/deployment.yaml create mode 100644 charts/airbyte/templates/webapp/ingress.yaml create mode 100644 charts/airbyte/templates/webapp/service.yaml create mode 100644 charts/airbyte/values.yaml diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 000000000000..b8e792f5845f --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,84 @@ +name: Helm +on: + push: + paths: + - '.github/workflows/helm.yaml' + - 'charts/**' + pull_request: + paths: + - '.github/workflows/helm.yaml' + - 'charts/**' +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: '3.6.3' + - name: Build Helm Dependencies + working-directory: ./charts/airbyte + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami + helm dep build + - name: Lint Chart + working-directory: ./charts/airbyte + run: make lint + + generate-docs: + name: Generate Docs Parameters + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + - name: Checkout bitnami-labs/readme-generator-for-helm + uses: actions/checkout@v2 + with: + repository: 'bitnami-labs/readme-generator-for-helm' + ref: '55cab5dd2191c4ffa7245cfefa428d4d9bb12730' + path: readme-generator-for-helm + - name: Install readme-generator-for-helm dependencies + working-directory: readme-generator-for-helm + run: npm install -g + - name: Test can update README with generated parameters + working-directory: charts/airbyte + run: readme-generator -v ./values.yaml -r README.md + + install: + name: Install + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: '3.6.3' + - name: Build Helm Dependencies + working-directory: ./charts/airbyte + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami + helm dep build + - name: Setup Kind Cluster + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.11.1" + image: "kindest/node:v1.21.1" + - name: Install airbyte chart + working-directory: ./charts/airbyte + run: make install + - if: always() + name: Print diagnostics + working-directory: ./charts/airbyte + run: make diagnostics + - if: success() + name: Test airbyte chart + working-directory: ./charts/airbyte + run: make test diff --git a/charts/airbyte/.gitignore b/charts/airbyte/.gitignore new file mode 100644 index 000000000000..28ea717bd6a2 --- /dev/null +++ b/charts/airbyte/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts \ No newline at end of file diff --git a/charts/airbyte/.helmignore b/charts/airbyte/.helmignore new file mode 100644 index 000000000000..5c836a2d796e --- /dev/null +++ b/charts/airbyte/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +Makefile \ No newline at end of file diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock new file mode 100644 index 000000000000..c5b73545be82 --- /dev/null +++ b/charts/airbyte/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.8.0 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 10.9.4 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 7.2.0 +digest: sha256:6b5428c4bffa53a16e1015635d712f28b1c1991eea4d048928e4fbb88974cf4f +generated: "2021-08-31T22:58:47.835303672-05:00" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml new file mode 100644 index 000000000000..873f69a47211 --- /dev/null +++ b/charts/airbyte/Chart.yaml @@ -0,0 +1,39 @@ +apiVersion: v2 +name: airbyte +description: Helm chart to deploy airbyte + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.29.13-alpha" + +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +- condition: postgresql.enabled + name: postgresql + version: 10.x.x + repository: https://charts.bitnami.com/bitnami +- condition: minio.enabled + name: minio + version: 7.x.x + repository: https://charts.bitnami.com/bitnami diff --git a/charts/airbyte/Makefile b/charts/airbyte/Makefile new file mode 100644 index 000000000000..e1b87246721c --- /dev/null +++ b/charts/airbyte/Makefile @@ -0,0 +1,28 @@ +RELEASE_NAME := airbyte +NAMESPACE := airbyte-test + +lint: + helm lint . + + +install: + helm upgrade --install \ + --create-namespace \ + --namespace $(NAMESPACE) \ + --debug \ + --wait \ + --timeout 300s \ + $(RELEASE_NAME) . + + +test: + helm test $(RELEASE_NAME) --logs --debug --namespace $(NAMESPACE) + + +diagnostics: + kubectl -n $(NAMESPACE) get pods -o wide + kubectl -n $(NAMESPACE) get pods --no-headers | grep -v "Running" | awk '{print $$1}' | xargs -L 1 -r kubectl -n $(NAMESPACE) logs + + +template: + helm template --release-name $(RELEASE_NAME) . diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md new file mode 100644 index 000000000000..46e24f93cb47 --- /dev/null +++ b/charts/airbyte/README.md @@ -0,0 +1,121 @@ +# airbyte + +## Parameters + +### Common Parameters + +| Name | Description | Value | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------- | --------------- | +| `nameOverride` | String to partially override airbyte.fullname template with a string (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override airbyte.fullname template with a string | `""` | +| `serviceAccount.annotations` | Annotations for service account. Evaluated as a template. Only used if `create` is `true`. | `undefined` | +| `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `serviceAccount.name` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template. | `airbyte-admin` | + + +### Webapp Parameters + +| Name | Description | Value | +| ---------------------------- | ---------------------------------------------------------------- | ---------------- | +| `webapp.replicaCount` | Number of webapp replicas | `1` | +| `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | +| `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | +| `webapp.service.port` | The service port to expose the webapp on | `80` | +| `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | +| `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | +| `webapp.ingress.hosts` | Ingress Hosts configuration | `[]` | +| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `undefined` | +| `webapp.ingress.hosts` | The list of hostnames to be covered with this ingress record. | `[]` | +| `webapp.ingress.tls` | Custom ingress TLS configuration | `undefined` | + + +### Scheduler Parameters + +| Name | Description | Value | +| ---------------------------- | ------------------------------------------------------------------- | ------------------- | +| `scheduler.replicaCount` | Number of scheduler replicas | `1` | +| `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | +| `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | +| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | + + +### Pod Sweeper parameters + +| Name | Description | Value | +| ----------------------------- | ----------------------------------------------- | ----------------- | +| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | +| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | +| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | + + +### Server parameters + +| Name | Description | Value | +| ------------------------------------------- | ---------------------------------------------------------------- | ---------------- | +| `server.replicaCount` | Number of server replicas | `1` | +| `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | +| `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | +| `server.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | +| `server.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `server.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `server.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `server.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `server.readinessProbe.enabled` | Enable readinessProbe on the server | `true` | +| `server.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `10` | +| `server.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `server.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `server.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `server.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `server.service.type` | The service type to use for the API server | `ClusterIP` | +| `server.service.port` | The service port to expose the API server on | `8001` | +| `server.configs.persistence.accessMode` | The access mode for the airbyte configs pvc | `ReadWriteOnce` | +| `server.configs.persistence.storageClass` | The storage class to use for the airbyte configs pvc | `standard` | +| `server.configs.persistence.size` | The size of the pvc to use for the airbyte configs pvc | `500Mi` | +| `server.workspace.persistence.accessMode` | The access mode for the airbyte workspace pvc | `ReadWriteOnce` | +| `server.workspace.persistence.storageClass` | The storage class to use for the airbyte workspace pvc | `standard` | +| `server.workspace.persistence.size` | The size of the pvc to use for the airbyte workspace pvc | `500Mi` | + + +### Temporal parameters + +| Name | Description | Value | +| --------------------------- | --------------------------------------------- | ----------------------- | +| `temporal.replicaCount` | The number of temporal replicas to deploy | `1` | +| `temporal.image.repository` | The temporal image repository to use | `temporalio/auto-setup` | +| `temporal.image.pullPolicy` | The pull policy for the temporal image | `IfNotPresent` | +| `temporal.image.tag` | The temporal image tag to use | `1.7.0` | +| `temporal.service.type` | The Kubernetes Service Type | `ClusterIP` | +| `temporal.service.port` | The temporal port and exposed kubernetes port | `7233` | + + +### Airbyte Database parameters + +| Name | Description | Value | +| -------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------ | +| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | +| `postgresql.postgresqlUsername` | Airbyte Postgresql username | `airbyte` | +| `postgresql.postgresqlPassword` | Airbyte Postgresql password | `airbyte` | +| `postgresql.postgresqlDatabase` | Airbyte Postgresql database | `db-airbyte` | +| `postgresql.existingSecret` | Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) | `""` | +| `externalDatabase.host` | Database host | `localhost` | +| `externalDatabase.user` | non-root Username for Airbyte Database | `airbyte` | +| `externalDatabase.password` | Database password | `""` | +| `externalDatabase.existingSecret` | Name of an existing secret resource containing the DB password | `""` | +| `externalDatabase.existingSecretPasswordKey` | Name of an existing secret key containing the DB password | `""` | +| `externalDatabase.database` | Database name | `db-airbyte` | +| `externalDatabase.port` | Database port number | `5432` | + + +### Minio parameters + +| Name | Description | Value | +| -------------------------- | ------------------------------------------------ | ----------- | +| `minio.enabled` | Switch to enable or disable the Minio helm chart | `true` | +| `minio.accessKey.password` | Minio Access Key | `minio` | +| `minio.secretKey.password` | Minio Secret Key | `minio123` | +| `externalMinio.host` | Minio Host | `localhost` | +| `externalMinio.port` | Minio Port | `9000` | diff --git a/charts/airbyte/files/sweep-pod.sh b/charts/airbyte/files/sweep-pod.sh new file mode 100644 index 000000000000..029772624cc4 --- /dev/null +++ b/charts/airbyte/files/sweep-pod.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +get_worker_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod --field-selector status.phase!=Running get pods -o go-template --template '{{range .items}} {{.metadata.name}} {{.status.phase}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' +} + +delete_worker_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod +} + +while : +do + # Shorter time window for completed pods + SUCCESS_DATE_STR=`date -d 'now - 2 hours' --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + # Longer time window for pods in error (to debug) + NON_SUCCESS_DATE_STR=`date -d 'now - 24 hours' --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + ( + IFS=$'\n' + for POD in `get_worker_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_DATE=`date -d $POD_DATE_STR '+%s'` + if [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + else + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + sleep 60 +done \ No newline at end of file diff --git a/charts/airbyte/templates/NOTES.txt b/charts/airbyte/templates/NOTES.txt new file mode 100644 index 000000000000..f39c2932cfbd --- /dev/null +++ b/charts/airbyte/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.webapp.ingress.enabled }} +{{- range $host := .Values.webapp.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.webapp.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.webapp.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airbyte.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.webapp.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "airbyte.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airbyte.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.webapp.service.port }} +{{- else if contains "ClusterIP" .Values.webapp.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "airbyte=webapp" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl new file mode 100644 index 000000000000..7a313aa77699 --- /dev/null +++ b/charts/airbyte/templates/_helpers.tpl @@ -0,0 +1,167 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "airbyte.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airbyte.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create a default fully qualified postgresql name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airbyte.postgresql.fullname" -}} +{{- $name := default "postgresql" .Values.postgresql.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Get the Postgresql credentials secret. +*/}} +{{- define "airbyte.postgresql.secretName" -}} +{{- if and (.Values.postgresql.enabled) (not .Values.postgresql.existingSecret) -}} + {{- printf "%s" (include "airbyte.postgresql.fullname" .) -}} +{{- else if and (.Values.postgresql.enabled) (.Values.postgresql.existingSecret) -}} + {{- printf "%s" .Values.postgresql.existingSecret -}} +{{- else }} + {{- if .Values.externalDatabase.existingSecret -}} + {{- printf "%s" .Values.externalDatabase.existingSecret -}} + {{- else -}} + {{ printf "%s-%s" .Release.Name "externaldb" }} + {{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.host" -}} +{{- ternary (include "airbyte.postgresql.fullname" .) .Values.externalDatabase.host .Values.postgresql.enabled | quote -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.user" -}} +{{- ternary .Values.postgresql.postgresqlUsername .Values.externalDatabase.user .Values.postgresql.enabled | quote -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.name" -}} +{{- ternary .Values.postgresql.postgresqlDatabase .Values.externalDatabase.database .Values.postgresql.enabled | quote -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.existingsecret.key" -}} +{{- if .Values.postgresql.enabled -}} + {{- printf "%s" "postgresql-password" -}} +{{- else -}} + {{- if .Values.externalDatabase.existingSecret -}} + {{- if .Values.externalDatabase.existingSecretPasswordKey -}} + {{- printf "%s" .Values.externalDatabase.existingSecretPasswordKey -}} + {{- else -}} + {{- printf "%s" "postgresql-password" -}} + {{- end -}} + {{- else -}} + {{- printf "%s" "postgresql-password" -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.port" -}} +{{- ternary "5432" .Values.externalDatabase.port .Values.postgresql.enabled | quote -}} +{{- end -}} + + +{{/* +Create a default fully qualified minio name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airbyte.minio.fullname" -}} +{{- $name := default "minio" .Values.minio.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Add environment variables to configure minio +*/}} +{{- define "airbyte.minio.host" -}} +{{- ternary (include "airbyte.minio.fullname" .) .Values.externalMinio.host .Values.minio.enabled | quote -}} +{{- end -}} + +{{/* +Add environment variables to configure minio +*/}} +{{- define "airbyte.minio.port" -}} +{{- ternary "9000" .Values.externalMinio.port .Values.minio.enabled | quote -}} +{{- end -}} + +{{- define "airbyte.minio.endpoint" -}} +{{- $host := (include "airbyte.minio.host" .) -}} +{{- $port := (include "airbyte.minio.port" .) -}} +{{- printf "http://%s:%s" $host $port -}} +{{- end -}} \ No newline at end of file diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml new file mode 100644 index 000000000000..ba72af777820 --- /dev/null +++ b/charts/airbyte/templates/env-configmap.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: airbyte-env +data: + AIRBYTE_VERSION: {{ .Chart.AppVersion }} + API_URL: /api/v1/ + AWS_ACCESS_KEY_ID: {{ .Values.minio.accessKey.password }} + AWS_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password }} + CONFIG_ROOT: /configs + DATA_DOCKER_MOUNT: airbyte_data + DATABASE_DB: {{ .Values.postgresql.postgresqlDatabase }} + DATABASE_HOST: {{ include "airbyte.database.host" . }} + DATABASE_PASSWORD: {{ .Values.postgresql.postgresqlPassword }} + DATABASE_PORT: "5432" + DATABASE_URL: jdbc:postgresql://airbyte-postgresql:5432/{{ .Values.postgresql.postgresqlDatabase }} + DATABASE_USER: {{ .Values.postgresql.postgresqlUsername }} + DB_DOCKER_MOUNT: airbyte_db + FULLSTORY: enabled + GCP_STORAGE_BUCKET: "" + GOOGLE_APPLICATION_CREDENTIALS: "" + INTERNAL_API_HOST: {{ include "common.names.fullname" . }}-server:{{ .Values.server.service.port }} + IS_DEMO: "false" + LOCAL_ROOT: /tmp/airbyte_local + LOG_LEVEL: INFO + OPENREPLAY: enabled + PAPERCUPS_STORYTIME: enabled + RESOURCE_CPU_LIMIT: "" + RESOURCE_CPU_REQUEST: "" + RESOURCE_MEMORY_LIMIT: "" + RESOURCE_MEMORY_REQUEST: "" + RUN_DATABASE_MIGRATION_ON_STARTUP: "true" + S3_LOG_BUCKET: airbyte-dev-logs + S3_LOG_BUCKET_REGION: "" + S3_MINIO_ENDPOINT: http://airbyte-minio:9000 + S3_PATH_STYLE_ACCESS: "true" + SUBMITTER_NUM_THREADS: "10" + TEMPORAL_HOST: {{ include "common.names.fullname" . }}-temporal:{{ .Values.temporal.service.port }} + TEMPORAL_WORKER_PORTS: 9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040 + TRACKING_STRATEGY: segment + WEBAPP_URL: http://{{ include "common.names.fullname" . }}-webapp:{{ .Values.webapp.service.port }} + WORKER_ENVIRONMENT: kubernetes + WORKSPACE_DOCKER_MOUNT: airbyte_workspace + WORKSPACE_ROOT: /workspace diff --git a/charts/airbyte/templates/gcs-log-creds-secret.yaml b/charts/airbyte/templates/gcs-log-creds-secret.yaml new file mode 100644 index 000000000000..27e1f7a61d0b --- /dev/null +++ b/charts/airbyte/templates/gcs-log-creds-secret.yaml @@ -0,0 +1,7 @@ +# TODO: Make this more configurable or support an existing secret +apiVersion: v1 +kind: Secret +metadata: + name: gcs-log-creds +data: + gcp.json: "" diff --git a/charts/airbyte/templates/pod-sweeper/configmap.yaml b/charts/airbyte/templates/pod-sweeper/configmap.yaml new file mode 100644 index 000000000000..cf19ce7e1340 --- /dev/null +++ b/charts/airbyte/templates/pod-sweeper/configmap.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airbyte.fullname" . }}-sweep-pod-script +data: + sweep-pod.sh: | + {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }} diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml new file mode 100644 index 000000000000..a95742877cee --- /dev/null +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airbyte.fullname" . }}-pod-sweeper + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + airbyte: pod-sweeper + template: + metadata: + labels: + airbyte: pod-sweeper + spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} + containers: + - name: airbyte-pod-sweeper + image: "{{ .Values.podSweeper.image.repository }}:{{ .Values.podSweeper.image.tag}}" + imagePullPolicy: "{{ .Values.podSweeper.image.pullPolicy }}" + env: + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /script/sweep-pod.sh + subPath: sweep-pod.sh + name: sweep-pod-script + command: ["/bin/bash", "-c", /script/sweep-pod.sh] + volumes: + - name: sweep-pod-script + configMap: + name: {{ include "airbyte.fullname" . }}-sweep-pod-script + defaultMode: 0755 diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml new file mode 100644 index 000000000000..50f4a4bf4a54 --- /dev/null +++ b/charts/airbyte/templates/scheduler/deployment.yaml @@ -0,0 +1,205 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airbyte.fullname" . }}-scheduler + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.scheduler.replicaCount }} + selector: + matchLabels: + airbyte: scheduler + template: + metadata: + labels: + airbyte: scheduler + spec: + serviceAccountName: {{ include "airbyte.serviceAccountName" . }} + automountServiceAccountToken: true + containers: + - name: airbyte-scheduler-container + image: "{{ .Values.scheduler.image.repository }}:{{ default .Chart.AppVersion .Values.scheduler.image.tag }}" + imagePullPolicy: "{{ .Values.scheduler.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + value: {{ .Chart.AppVersion }} + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONFIG_ROOT + - name: DATABASE_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_HOST + - name: DATABASE_PORT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PORT + - name: DATABASE_PASSWORD + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PASSWORD + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_USER + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: WORKSPACE_DOCKER_MOUNT + value: workspace + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKSPACE_ROOT + - name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKER_ENVIRONMENT + - name: LOCAL_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOCAL_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_HOST + - name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_WORKER_PORTS + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOG_LEVEL + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SUBMITTER_NUM_THREADS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: SUBMITTER_NUM_THREADS + - name: RESOURCE_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_REQUEST + - name: RESOURCE_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_LIMIT + - name: RESOURCE_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_REQUEST + - name: RESOURCE_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_LIMIT + - name: S3_LOG_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET + - name: S3_LOG_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET_REGION + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.minio.accessKey.password }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.minio.secretKey.password }} + - name: S3_MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_MINIO_ENDPOINT + - name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_PATH_STYLE_ACCESS + - name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS + - name: GCP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GCP_STORAGE_BUCKET + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: INTERNAL_API_HOST + ports: + - containerPort: 9000 # for heartbeat server + - containerPort: 9001 # start temporal worker port pool + - containerPort: 9002 + - containerPort: 9003 + - containerPort: 9004 + - containerPort: 9005 + - containerPort: 9006 + - containerPort: 9007 + - containerPort: 9008 + - containerPort: 9009 + - containerPort: 9010 + - containerPort: 9011 + - containerPort: 9012 + - containerPort: 9013 + - containerPort: 9014 + - containerPort: 9015 + - containerPort: 9016 + - containerPort: 9017 + - containerPort: 9018 + - containerPort: 9019 + - containerPort: 9020 + - containerPort: 9021 + - containerPort: 9022 + - containerPort: 9023 + - containerPort: 9024 + - containerPort: 9025 + - containerPort: 9026 + - containerPort: 9027 + - containerPort: 9028 + - containerPort: 9029 + - containerPort: 9030 # end temporal worker port pool + volumeMounts: + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + volumes: + - name: gcs-log-creds-volume + secret: + secretName: gcs-log-creds diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml new file mode 100644 index 000000000000..d21f4ca98843 --- /dev/null +++ b/charts/airbyte/templates/server/deployment.yaml @@ -0,0 +1,182 @@ + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.server.replicaCount }} + selector: + matchLabels: + airbyte: server + template: + metadata: + labels: + airbyte: server + spec: + containers: + - name: airbyte-server-container + image: "{{ .Values.server.image.repository }}:{{ default .Chart.AppVersion .Values.server.image.tag }}" + imagePullPolicy: "{{ .Values.server.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AIRBYTE_VERSION + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONFIG_ROOT + - name: DATABASE_PASSWORD + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PASSWORD + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_USER + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKER_ENVIRONMENT + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WORKSPACE_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TEMPORAL_HOST + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: LOG_LEVEL + - name: RESOURCE_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_REQUEST + - name: RESOURCE_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_CPU_LIMIT + - name: RESOURCE_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_REQUEST + - name: RESOURCE_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: RESOURCE_MEMORY_LIMIT + - name: S3_LOG_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET + - name: S3_LOG_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_LOG_BUCKET_REGION + - name: AWS_ACCESS_KEY_ID + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AWS_SECRET_ACCESS_KEY + - name: S3_MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_MINIO_ENDPOINT + - name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: S3_PATH_STYLE_ACCESS + - name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS + - name: GCP_STORAGE_BUCKET + valueFrom: + configMapKeyRef: + name: airbyte-env + key: GCP_STORAGE_BUCKET + {{- if .Values.server.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.server.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.server.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8001 + protocol: TCP + volumeMounts: + - name: airbyte-volume-configs + mountPath: /configs + - name: airbyte-volume-workspace + mountPath: /workspace + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + volumes: + - name: airbyte-volume-workspace + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-workspace + - name: airbyte-volume-configs + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-configs + - name: gcs-log-creds-volume + secret: + secretName: gcs-log-creds diff --git a/charts/airbyte/templates/server/pvc-configs.yaml b/charts/airbyte/templates/server/pvc-configs.yaml new file mode 100644 index 000000000000..fcabc9718424 --- /dev/null +++ b/charts/airbyte/templates/server/pvc-configs.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "common.names.fullname" . }}-configs + labels: + airbyte: volume-configs +spec: + accessModes: + - {{ .Values.server.configs.persistence.accessMode | quote }} + storageClassName: {{ .Values.server.configs.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.server.configs.persistence.size | quote }} diff --git a/charts/airbyte/templates/server/pvc-workspace.yaml b/charts/airbyte/templates/server/pvc-workspace.yaml new file mode 100644 index 000000000000..cc63658e298c --- /dev/null +++ b/charts/airbyte/templates/server/pvc-workspace.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "common.names.fullname" . }}-workspace + labels: + airbyte: volume-workspace +spec: + accessModes: + - {{ .Values.server.workspace.persistence.accessMode | quote }} + storageClassName: {{ .Values.server.workspace.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.server.workspace.persistence.size | quote }} diff --git a/charts/airbyte/templates/server/service.yaml b/charts/airbyte/templates/server/service.yaml new file mode 100644 index 000000000000..f069bedf3d79 --- /dev/null +++ b/charts/airbyte/templates/server/service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-server +spec: + type: {{ .Values.server.service.type }} + ports: + - port: {{ .Values.server.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + airbyte: server \ No newline at end of file diff --git a/charts/airbyte/templates/serviceaccount.yaml b/charts/airbyte/templates/serviceaccount.yaml new file mode 100644 index 000000000000..142191c9c9ef --- /dev/null +++ b/charts/airbyte/templates/serviceaccount.yaml @@ -0,0 +1,34 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airbyte.serviceAccountName" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-role +rules: + - apiGroups: ["*"] + resources: ["jobs", "pods", "pods/log", "pods/exec", "pods/attach"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # over-permission for now +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-binding +roleRef: + apiGroup: "" + kind: Role + name: {{ include "airbyte.serviceAccountName" . }}-role +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.serviceAccountName" . }} +{{- end }} + diff --git a/charts/airbyte/templates/temporal/configmap.yaml b/charts/airbyte/templates/temporal/configmap.yaml new file mode 100644 index 000000000000..bab965cf3fe6 --- /dev/null +++ b/charts/airbyte/templates/temporal/configmap.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: airbyte-temporal-dynamicconfig +data: + "development.yaml": | + # when modifying, remember to update the docker-compose version of this file in temporal/dynamicconfig/development.yaml + frontend.enableClientVersionCheck: + - value: true + constraints: {} + history.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.historyMgrNumConns: + - value: 30 + constraints: {} + frontend.throttledLogRPS: + - value: 20 + constraints: {} + history.historyMgrNumConns: + - value: 50 + constraints: {} + system.advancedVisibilityWritingMode: + - value: "off" + constraints: {} + history.defaultActivityRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + history.defaultWorkflowRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + # Limit for responses. This mostly impacts discovery jobs since they have the largest responses. + limit.blobSize.error: + - value: 15728640 # 15MB + constraints: {} + limit.blobSize.warn: + - value: 10485760 # 10MB + constraints: {} diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml new file mode 100644 index 000000000000..c0b86422c04b --- /dev/null +++ b/charts/airbyte/templates/temporal/deployment.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-temporal + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.temporal.replicaCount }} + selector: + matchLabels: + airbyte: temporal + template: + metadata: + labels: + airbyte: temporal + spec: + containers: + - name: airbyte-temporal + image: "{{ .Values.temporal.image.repository}}:{{ .Values.temporal.image.tag }}" + imagePullPolicy: {{ .Values.temporal.image.pullPolicy }} + env: + - name: AUTO_SETUP + value: "true" + - name: DB # The DB engine to use + value: "postgresql" + - name: DB_PORT + value: {{ include "airbyte.database.port" . }} + - name: POSTGRES_USER + value: {{ include "airbyte.database.user" . }} + - name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.postgresql.secretName" . }} + key: {{ include "airbyte.database.existingsecret.key" . }} + - name: POSTGRES_SEEDS + value: {{ include "airbyte.database.host" . }} + - name: DYNAMIC_CONFIG_FILE_PATH + value: "config/dynamicconfig/development.yaml" + + ports: + - containerPort: 7233 + volumeMounts: + - name: airbyte-temporal-dynamicconfig + mountPath: "/etc/temporal/config/dynamicconfig/" + volumes: + - name: airbyte-temporal-dynamicconfig + configMap: + name: airbyte-temporal-dynamicconfig + items: + - key: development.yaml + path: development.yaml diff --git a/charts/airbyte/templates/temporal/service.yaml b/charts/airbyte/templates/temporal/service.yaml new file mode 100644 index 000000000000..0242d2ff0cb3 --- /dev/null +++ b/charts/airbyte/templates/temporal/service.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-temporal +spec: + type: {{ .Values.temporal.service.type }} + ports: + - port: {{ .Values.temporal.service.port }} + protocol: TCP + targetPort: 7233 + selector: + airbyte: temporal \ No newline at end of file diff --git a/charts/airbyte/templates/tests/test-webapp.yaml b/charts/airbyte/templates/tests/test-webapp.yaml new file mode 100644 index 000000000000..9784832869aa --- /dev/null +++ b/charts/airbyte/templates/tests/test-webapp.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "airbyte.fullname" . }}-test-connection" + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}-webapp:{{ .Values.webapp.service.port }}'] + restartPolicy: Never diff --git a/charts/airbyte/templates/webapp/deployment.yaml b/charts/airbyte/templates/webapp/deployment.yaml new file mode 100644 index 000000000000..404319609526 --- /dev/null +++ b/charts/airbyte/templates/webapp/deployment.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-webapp + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.webapp.replicaCount }} + selector: + matchLabels: + airbyte: webapp + template: + metadata: + labels: + airbyte: webapp + spec: + containers: + - name: airbyte-webapp-container + image: "{{ .Values.webapp.image.repository }}:{{ default .Chart.AppVersion .Values.webapp.image.tag }}" + imagePullPolicy: "{{ .Values.webapp.image.pullPolicy }}" + env: + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AIRBYTE_VERSION + - name: API_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: API_URL + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: TRACKING_STRATEGY + - name: PAPERCUPS_STORYTIME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: PAPERCUPS_STORYTIME + - name: FULLSTORY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: FULLSTORY + - name: OPENREPLAY + valueFrom: + configMapKeyRef: + name: airbyte-env + key: OPENREPLAY + - name: IS_DEMO + valueFrom: + configMapKeyRef: + name: airbyte-env + key: IS_DEMO + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: INTERNAL_API_HOST + ports: + - name: http + containerPort: 80 + protocol: TCP diff --git a/charts/airbyte/templates/webapp/ingress.yaml b/charts/airbyte/templates/webapp/ingress.yaml new file mode 100644 index 000000000000..42c4e34398eb --- /dev/null +++ b/charts/airbyte/templates/webapp/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.webapp.ingress.enabled -}} +{{- $fullName := include "airbyte.fullname" . -}} +{{- $svcPort := .Values.webapp.service.port -}} +{{- if and .Values.webapp.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.webapp.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.webapp.ingress.annotations "kubernetes.io/ingress.class" .Values.webapp.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.webapp.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.webapp.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.webapp.ingress.className }} + {{- end }} + {{- if .Values.webapp.ingress.tls }} + tls: + {{- range .Values.webapp.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.webapp.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/airbyte/templates/webapp/service.yaml b/charts/airbyte/templates/webapp/service.yaml new file mode 100644 index 000000000000..925360ea77fb --- /dev/null +++ b/charts/airbyte/templates/webapp/service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-webapp +spec: + type: {{ .Values.webapp.service.type }} + ports: + - port: {{ .Values.webapp.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + airbyte: webapp diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml new file mode 100644 index 000000000000..a632340bda7f --- /dev/null +++ b/charts/airbyte/values.yaml @@ -0,0 +1,249 @@ +## @section Common Parameters + +## @param nameOverride String to partially override airbyte.fullname template with a string (will prepend the release name) +## +nameOverride: "" +## @param fullnameOverride String to fully override airbyte.fullname template with a string +## +fullnameOverride: "" + +## Pods Service Account +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## @param serviceAccount.annotations Annotations for service account. Evaluated as a template. Only used if `create` is `true`. +## @param serviceAccount.create Specifies whether a ServiceAccount should be created +## @param serviceAccount.name Name of the service account to use. If not set and create is true, a name is generated using the fullname template. +## +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: airbyte-admin + +## @section Webapp Parameters + +webapp: + ## @param webapp.replicaCount Number of webapp replicas + replicaCount: 1 + + ## @param webapp.image.repository The repository to use for the airbyte webapp image. + ## @param webapp.image.pullPolicy the pull policy to use for the airbyte webapp image + ## @param webapp.image.tag The airbyte webapp image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/webapp + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + ## @param webapp.service.type The service type to use for the webapp service + ## @param webapp.service.port The service port to expose the webapp on + service: + type: ClusterIP + port: 80 + + ## Configure the ingress resource that allows you to access the Airbyte installation. + ## ref: http://kubernetes.io/docs/user-guide/ingress/ + ## @param webapp.ingress.enabled Set to true to enable ingress record generation + ## @param webapp.ingress.className Specifies ingressClassName for clusters >= 1.18+ + ## @param webapp.ingress.hosts Ingress Hosts configuration + ## @param webapp.ingress.annotations Ingress annotations done as key:value pairs + ## @param webapp.ingress.hosts The list of hostnames to be covered with this ingress record. + ## @param webapp.ingress.tls Custom ingress TLS configuration + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + +## @section Scheduler Parameters + +scheduler: + ## @param scheduler.replicaCount Number of scheduler replicas + replicaCount: 1 + + ## @param scheduler.image.repository The repository to use for the airbyte scheduler image. + ## @param scheduler.image.pullPolicy the pull policy to use for the airbyte scheduler image + ## @param scheduler.image.tag The airbyte scheduler image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/scheduler + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + +## @section Pod Sweeper parameters + +podSweeper: + ## @param podSweeper.image.repository The image repository to use for the pod sweeper + ## @param podSweeper.image.pullPolicy The pull policy for the pod sweeper image + ## @param podSweeper.image.tag The pod sweeper image tag to use + image: + repository: bitnami/kubectl + pullPolicy: IfNotPresent + tag: latest + + +## @section Server parameters + +server: + ## @param server.replicaCount Number of server replicas + replicaCount: 1 + + ## @param server.image.repository The repository to use for the airbyte server image. + ## @param server.image.pullPolicy the pull policy to use for the airbyte server image + ## @param server.image.tag The airbyte server image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/server + pullPolicy: IfNotPresent + tag: 0.29.13-alpha + + ## Configure extra options for Serer containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## @param server.livenessProbe.enabled Enable livenessProbe on the server + ## @param server.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## @param server.livenessProbe.periodSeconds Period seconds for livenessProbe + ## @param server.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## @param server.livenessProbe.failureThreshold Failure threshold for livenessProbe + ## @param server.livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param server.readinessProbe.enabled Enable readinessProbe on the server + ## @param server.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## @param server.readinessProbe.periodSeconds Period seconds for readinessProbe + ## @param server.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## @param server.readinessProbe.failureThreshold Failure threshold for readinessProbe + ## @param server.readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## @param server.service.type The service type to use for the API server + ## @param server.service.port The service port to expose the API server on + service: + type: ClusterIP + port: 8001 + + ## @param server.configs.persistence.accessMode The access mode for the airbyte configs pvc + ## @param server.configs.persistence.storageClass The storage class to use for the airbyte configs pvc + ## @param server.configs.persistence.size The size of the pvc to use for the airbyte configs pvc + configs: + persistence: + accessMode: ReadWriteOnce + storageClass: standard + size: 500Mi + + ## @param server.workspace.persistence.accessMode The access mode for the airbyte workspace pvc + ## @param server.workspace.persistence.storageClass The storage class to use for the airbyte workspace pvc + ## @param server.workspace.persistence.size The size of the pvc to use for the airbyte workspace pvc + workspace: + persistence: + accessMode: ReadWriteOnce + storageClass: standard + size: 500Mi + + +## @section Temporal parameters +## TODO: Move to consuming temporal from a dedicated helm chart + +temporal: + ## @param temporal.replicaCount The number of temporal replicas to deploy + replicaCount: 1 + + ## @param temporal.image.repository The temporal image repository to use + ## @param temporal.image.pullPolicy The pull policy for the temporal image + ## @param temporal.image.tag The temporal image tag to use + image: + repository: temporalio/auto-setup + pullPolicy: IfNotPresent + tag: "1.7.0" + + ## @param temporal.service.type The Kubernetes Service Type + ## @param temporal.service.port The temporal port and exposed kubernetes port + service: + type: ClusterIP + port: 7233 + + +## @section Airbyte Database parameters + +## PostgreSQL chart configuration +## ref: https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml +## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart +## @param postgresql.postgresqlUsername Airbyte Postgresql username +## @param postgresql.postgresqlPassword Airbyte Postgresql password +## @param postgresql.postgresqlDatabase Airbyte Postgresql database +## @param postgresql.existingSecret Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) +## +postgresql: + enabled: true + postgresqlUsername: airbyte + postgresqlPassword: airbyte + postgresqlDatabase: db-airbyte + ## This secret is used in case of postgresql.enabled=true and we would like to specify password for newly created postgresql instance + ## + existingSecret: "" + + +## External PostgreSQL configuration +## All of these values are only used when postgresql.enabled is set to false +## @param externalDatabase.host Database host +## @param externalDatabase.user non-root Username for Airbyte Database +## @param externalDatabase.password Database password +## @param externalDatabase.existingSecret Name of an existing secret resource containing the DB password +## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the DB password +## @param externalDatabase.database Database name +## @param externalDatabase.port Database port number +## +externalDatabase: + host: localhost + user: airbyte + password: "" + existingSecret: "" + existingSecretPasswordKey: "" + database: db-airbyte + port: 5432 + +## @section Minio parameters + +## Minio chart configuration +## ref: https://github.com/bitnami/charts/blob/master/bitnami/minio/values.yaml +## @param minio.enabled Switch to enable or disable the Minio helm chart +## @param minio.accessKey.password Minio Access Key +## @param minio.secretKey.password Minio Secret Key +minio: + enabled: true + accessKey: + password: minio + secretKey: + password: minio123 + +## External Minio configuration +## All of these values are only used when minio.enabled is set to false +## @param externalMinio.host Minio Host +## @param externalMinio.port Minio Port +externalMinio: + host: localhost + port: 9000 \ No newline at end of file From 367cc74f1b70ad7f1e9f0fadf174871c5cd3ada3 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 1 Sep 2021 17:17:43 -0500 Subject: [PATCH 02/16] Fix ingress service name to match --- charts/airbyte/templates/webapp/ingress.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/airbyte/templates/webapp/ingress.yaml b/charts/airbyte/templates/webapp/ingress.yaml index 42c4e34398eb..670d8d8783a4 100644 --- a/charts/airbyte/templates/webapp/ingress.yaml +++ b/charts/airbyte/templates/webapp/ingress.yaml @@ -49,11 +49,11 @@ spec: backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ $fullName }} + name: {{ $fullName }}-webapp port: number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }} + serviceName: {{ $fullName }}-webapp servicePort: {{ $svcPort }} {{- end }} {{- end }} From 2ad7a3f4814281f4a926ef5f3303d373c97ea1cd Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 1 Sep 2021 17:19:54 -0500 Subject: [PATCH 03/16] Use a single server volume with different subPaths In EKS if you have a multi-az cluster volumes can get provisioned in seperate AZs and pods won't schedule --- .../airbyte/templates/server/deployment.yaml | 13 +++++----- .../airbyte/templates/server/pvc-configs.yaml | 14 ----------- charts/airbyte/templates/server/pvc-data.yaml | 14 +++++++++++ .../templates/server/pvc-workspace.yaml | 14 ----------- charts/airbyte/values.yaml | 25 ++++++------------- 5 files changed, 27 insertions(+), 53 deletions(-) delete mode 100644 charts/airbyte/templates/server/pvc-configs.yaml create mode 100644 charts/airbyte/templates/server/pvc-data.yaml delete mode 100644 charts/airbyte/templates/server/pvc-workspace.yaml diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index d21f4ca98843..839fa521d64b 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -163,20 +163,19 @@ spec: containerPort: 8001 protocol: TCP volumeMounts: - - name: airbyte-volume-configs + - name: airbyte-data mountPath: /configs - - name: airbyte-volume-workspace + subPath: configs + - name: airbyte-data mountPath: /workspace + subPath: workspace - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true volumes: - - name: airbyte-volume-workspace + - name: airbyte-data persistentVolumeClaim: - claimName: {{ include "common.names.fullname" . }}-workspace - - name: airbyte-volume-configs - persistentVolumeClaim: - claimName: {{ include "common.names.fullname" . }}-configs + claimName: {{ include "common.names.fullname" . }}-data - name: gcs-log-creds-volume secret: secretName: gcs-log-creds diff --git a/charts/airbyte/templates/server/pvc-configs.yaml b/charts/airbyte/templates/server/pvc-configs.yaml deleted file mode 100644 index fcabc9718424..000000000000 --- a/charts/airbyte/templates/server/pvc-configs.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "common.names.fullname" . }}-configs - labels: - airbyte: volume-configs -spec: - accessModes: - - {{ .Values.server.configs.persistence.accessMode | quote }} - storageClassName: {{ .Values.server.configs.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.server.configs.persistence.size | quote }} diff --git a/charts/airbyte/templates/server/pvc-data.yaml b/charts/airbyte/templates/server/pvc-data.yaml new file mode 100644 index 000000000000..353c538a7684 --- /dev/null +++ b/charts/airbyte/templates/server/pvc-data.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "common.names.fullname" . }}-data + labels: + airbyte: volume-data +spec: + accessModes: + - {{ .Values.server.persistence.accessMode | quote }} + storageClassName: {{ .Values.server.persistence.storageClass | quote }} + resources: + requests: + storage: {{ .Values.server.persistence.size | quote }} diff --git a/charts/airbyte/templates/server/pvc-workspace.yaml b/charts/airbyte/templates/server/pvc-workspace.yaml deleted file mode 100644 index cc63658e298c..000000000000 --- a/charts/airbyte/templates/server/pvc-workspace.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "common.names.fullname" . }}-workspace - labels: - airbyte: volume-workspace -spec: - accessModes: - - {{ .Values.server.workspace.persistence.accessMode | quote }} - storageClassName: {{ .Values.server.workspace.persistence.storageClass | quote }} - resources: - requests: - storage: {{ .Values.server.workspace.persistence.size | quote }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index a632340bda7f..655dcd3c8d17 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -146,24 +146,13 @@ server: type: ClusterIP port: 8001 - ## @param server.configs.persistence.accessMode The access mode for the airbyte configs pvc - ## @param server.configs.persistence.storageClass The storage class to use for the airbyte configs pvc - ## @param server.configs.persistence.size The size of the pvc to use for the airbyte configs pvc - configs: - persistence: - accessMode: ReadWriteOnce - storageClass: standard - size: 500Mi - - ## @param server.workspace.persistence.accessMode The access mode for the airbyte workspace pvc - ## @param server.workspace.persistence.storageClass The storage class to use for the airbyte workspace pvc - ## @param server.workspace.persistence.size The size of the pvc to use for the airbyte workspace pvc - workspace: - persistence: - accessMode: ReadWriteOnce - storageClass: standard - size: 500Mi - + ## @param server.persistence.accessMode The access mode for the airbyte server pvc + ## @param server.persistence.storageClass The storage class to use for the airbyte server pvc + ## @param server.persistence.size The size of the pvc to use for the airbyte server pvc + persistence: + accessMode: ReadWriteOnce + storageClass: standard + size: 1Gi ## @section Temporal parameters ## TODO: Move to consuming temporal from a dedicated helm chart From 2fbb54f8fd943e3812d468fa32a3334d703c2147 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 1 Sep 2021 22:18:34 -0500 Subject: [PATCH 04/16] Add resources and podAnnotations --- .github/workflows/helm.yaml | 2 +- charts/airbyte/Makefile | 6 + charts/airbyte/README.md | 51 +++++---- .../templates/pod-sweeper/deployment.yaml | 7 ++ .../templates/scheduler/deployment.yaml | 7 ++ .../airbyte/templates/server/deployment.yaml | 7 ++ .../airbyte/templates/webapp/deployment.yaml | 7 ++ charts/airbyte/values.yaml | 106 ++++++++++++++++-- 8 files changed, 165 insertions(+), 28 deletions(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index b8e792f5845f..ac5233e20249 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -47,7 +47,7 @@ jobs: run: npm install -g - name: Test can update README with generated parameters working-directory: charts/airbyte - run: readme-generator -v ./values.yaml -r README.md + run: make update-docs install: name: Install diff --git a/charts/airbyte/Makefile b/charts/airbyte/Makefile index e1b87246721c..82379e827c98 100644 --- a/charts/airbyte/Makefile +++ b/charts/airbyte/Makefile @@ -26,3 +26,9 @@ diagnostics: template: helm template --release-name $(RELEASE_NAME) . + + +# Uses the readme-generator command to update the README.md +# with changes in the values.yaml file. See: https://github.com/bitnami-labs/readme-generator-for-helm +update-docs: + readme-generator -v ./values.yaml -r README.md diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 46e24f93cb47..67110aabacaf 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -8,7 +8,7 @@ | ---------------------------- | ------------------------------------------------------------------------------------------------------------------- | --------------- | | `nameOverride` | String to partially override airbyte.fullname template with a string (will prepend the release name) | `""` | | `fullnameOverride` | String to fully override airbyte.fullname template with a string | `""` | -| `serviceAccount.annotations` | Annotations for service account. Evaluated as a template. Only used if `create` is `true`. | `undefined` | +| `serviceAccount.annotations` | Annotations for service account. Evaluated as a template. Only used if `create` is `true`. | `{}` | | `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `serviceAccount.name` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template. | `airbyte-admin` | @@ -21,33 +21,42 @@ | `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | | `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | | `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `webapp.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | | `webapp.service.type` | The service type to use for the webapp service | `ClusterIP` | | `webapp.service.port` | The service port to expose the webapp on | `80` | +| `webapp.resources.limits` | The resources limits for the Web container | `{}` | +| `webapp.resources.requests` | The requested resources for the Web container | `{}` | | `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | | `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | | `webapp.ingress.hosts` | Ingress Hosts configuration | `[]` | -| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `undefined` | +| `webapp.ingress.annotations` | Ingress annotations done as key:value pairs | `{}` | | `webapp.ingress.hosts` | The list of hostnames to be covered with this ingress record. | `[]` | -| `webapp.ingress.tls` | Custom ingress TLS configuration | `undefined` | +| `webapp.ingress.tls` | Custom ingress TLS configuration | `[]` | ### Scheduler Parameters -| Name | Description | Value | -| ---------------------------- | ------------------------------------------------------------------- | ------------------- | -| `scheduler.replicaCount` | Number of scheduler replicas | `1` | -| `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | -| `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | -| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| Name | Description | Value | +| ------------------------------ | ------------------------------------------------------------------- | ------------------- | +| `scheduler.replicaCount` | Number of scheduler replicas | `1` | +| `scheduler.image.repository` | The repository to use for the airbyte scheduler image. | `airbyte/scheduler` | +| `scheduler.image.pullPolicy` | the pull policy to use for the airbyte scheduler image | `IfNotPresent` | +| `scheduler.image.tag` | The airbyte scheduler image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | +| `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | +| `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | ### Pod Sweeper parameters -| Name | Description | Value | -| ----------------------------- | ----------------------------------------------- | ----------------- | -| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | -| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | -| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | +| Name | Description | Value | +| ------------------------------- | ---------------------------------------------------- | ----------------- | +| `podSweeper.image.repository` | The image repository to use for the pod sweeper | `bitnami/kubectl` | +| `podSweeper.image.pullPolicy` | The pull policy for the pod sweeper image | `IfNotPresent` | +| `podSweeper.image.tag` | The pod sweeper image tag to use | `latest` | +| `podSweeper.podAnnotations` | Add extra annotations to the podSweeper pod | `{}` | +| `podSweeper.resources.limits` | The resources limits for the podSweeper container | `{}` | +| `podSweeper.resources.requests` | The requested resources for the podSweeper container | `{}` | ### Server parameters @@ -58,6 +67,7 @@ | `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | | `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | | `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.29.13-alpha` | +| `server.podAnnotations` | Add extra annotations to the server pod | `{}` | | `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | | `server.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `30` | | `server.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | @@ -70,14 +80,13 @@ | `server.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | | `server.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | | `server.readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `server.resources.limits` | The resources limits for the server container | `{}` | +| `server.resources.requests` | The requested resources for the server container | `{}` | | `server.service.type` | The service type to use for the API server | `ClusterIP` | | `server.service.port` | The service port to expose the API server on | `8001` | -| `server.configs.persistence.accessMode` | The access mode for the airbyte configs pvc | `ReadWriteOnce` | -| `server.configs.persistence.storageClass` | The storage class to use for the airbyte configs pvc | `standard` | -| `server.configs.persistence.size` | The size of the pvc to use for the airbyte configs pvc | `500Mi` | -| `server.workspace.persistence.accessMode` | The access mode for the airbyte workspace pvc | `ReadWriteOnce` | -| `server.workspace.persistence.storageClass` | The storage class to use for the airbyte workspace pvc | `standard` | -| `server.workspace.persistence.size` | The size of the pvc to use for the airbyte workspace pvc | `500Mi` | +| `server.persistence.accessMode` | The access mode for the airbyte server pvc | `ReadWriteOnce` | +| `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `standard` | +| `server.persistence.size` | The size of the pvc to use for the airbyte server pvc | `1Gi` | ### Temporal parameters @@ -119,3 +128,5 @@ | `minio.secretKey.password` | Minio Secret Key | `minio123` | | `externalMinio.host` | Minio Host | `localhost` | | `externalMinio.port` | Minio Port | `9000` | + + diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml index a95742877cee..d333e0cf1c09 100644 --- a/charts/airbyte/templates/pod-sweeper/deployment.yaml +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -14,6 +14,10 @@ spec: metadata: labels: airbyte: pod-sweeper + {{- if .Values.podSweeper.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.podAnnotations "context" $) | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "airbyte.serviceAccountName" . }} containers: @@ -30,6 +34,9 @@ spec: subPath: sweep-pod.sh name: sweep-pod-script command: ["/bin/bash", "-c", /script/sweep-pod.sh] + {{- if .Values.podSweeper.resources }} + resources: {{- toYaml .Values.podSweeper.resources | nindent 10 }} + {{- end }} volumes: - name: sweep-pod-script configMap: diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml index 50f4a4bf4a54..4d4dac724b33 100644 --- a/charts/airbyte/templates/scheduler/deployment.yaml +++ b/charts/airbyte/templates/scheduler/deployment.yaml @@ -13,6 +13,10 @@ spec: metadata: labels: airbyte: scheduler + {{- if .Values.scheduler.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.podAnnotations "context" $) | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "airbyte.serviceAccountName" . }} automountServiceAccountToken: true @@ -195,6 +199,9 @@ spec: - containerPort: 9028 - containerPort: 9029 - containerPort: 9030 # end temporal worker port pool + {{- if .Values.scheduler.resources }} + resources: {{- toYaml .Values.scheduler.resources | nindent 10 }} + {{- end }} volumeMounts: - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index 839fa521d64b..90c6571b80f8 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -15,6 +15,10 @@ spec: metadata: labels: airbyte: server + {{- if .Values.server.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.server.podAnnotations "context" $) | nindent 8 }} + {{- end }} spec: containers: - name: airbyte-server-container @@ -162,6 +166,9 @@ spec: - name: http containerPort: 8001 protocol: TCP + {{- if .Values.server.resources }} + resources: {{- toYaml .Values.server.resources | nindent 10 }} + {{- end }} volumeMounts: - name: airbyte-data mountPath: /configs diff --git a/charts/airbyte/templates/webapp/deployment.yaml b/charts/airbyte/templates/webapp/deployment.yaml index 404319609526..bca9aaf1073c 100644 --- a/charts/airbyte/templates/webapp/deployment.yaml +++ b/charts/airbyte/templates/webapp/deployment.yaml @@ -14,6 +14,10 @@ spec: metadata: labels: airbyte: webapp + {{- if .Values.webapp.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.webapp.podAnnotations "context" $) | nindent 8 }} + {{- end }} spec: containers: - name: airbyte-webapp-container @@ -64,3 +68,6 @@ spec: - name: http containerPort: 80 protocol: TCP + {{- if .Values.webapp.resources }} + resources: {{- toYaml .Values.webapp.resources | nindent 10 }} + {{- end }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 655dcd3c8d17..74fd0cb29c34 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -9,17 +9,13 @@ fullnameOverride: "" ## Pods Service Account ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ -## @param serviceAccount.annotations Annotations for service account. Evaluated as a template. Only used if `create` is `true`. +## @param serviceAccount.annotations [object] Annotations for service account. Evaluated as a template. Only used if `create` is `true`. ## @param serviceAccount.create Specifies whether a ServiceAccount should be created ## @param serviceAccount.name Name of the service account to use. If not set and create is true, a name is generated using the fullname template. ## serviceAccount: - # Specifies whether a service account should be created create: true - # Annotations to add to the service account annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template name: airbyte-admin ## @section Webapp Parameters @@ -36,20 +32,44 @@ webapp: pullPolicy: IfNotPresent tag: 0.29.13-alpha + ## @param webapp.podAnnotations [object] Add extra annotations to the scheduler pod + ## + podAnnotations: {} + ## @param webapp.service.type The service type to use for the webapp service ## @param webapp.service.port The service port to expose the webapp on service: type: ClusterIP port: 80 + ## Web app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param webapp.resources.limits [object] The resources limits for the Web container + ## @param webapp.resources.requests [object] The requested resources for the Web container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Configure the ingress resource that allows you to access the Airbyte installation. ## ref: http://kubernetes.io/docs/user-guide/ingress/ ## @param webapp.ingress.enabled Set to true to enable ingress record generation ## @param webapp.ingress.className Specifies ingressClassName for clusters >= 1.18+ ## @param webapp.ingress.hosts Ingress Hosts configuration - ## @param webapp.ingress.annotations Ingress annotations done as key:value pairs + ## @param webapp.ingress.annotations [object] Ingress annotations done as key:value pairs ## @param webapp.ingress.hosts The list of hostnames to be covered with this ingress record. - ## @param webapp.ingress.tls Custom ingress TLS configuration + ## @param webapp.ingress.tls [array] Custom ingress TLS configuration ingress: enabled: false className: "" @@ -81,6 +101,30 @@ scheduler: pullPolicy: IfNotPresent tag: 0.29.13-alpha + ## @param scheduler.podAnnotations [object] Add extra annotations to the scheduler pod + ## + podAnnotations: {} + + ## Scheduler resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param scheduler.resources.limits [object] The resources limits for the scheduler container + ## @param scheduler.resources.requests [object] The requested resources for the scheduler container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## @section Pod Sweeper parameters @@ -93,6 +137,30 @@ podSweeper: pullPolicy: IfNotPresent tag: latest + ## @param podSweeper.podAnnotations [object] Add extra annotations to the podSweeper pod + ## + podAnnotations: {} + + ## Pod Sweeper app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param podSweeper.resources.limits [object] The resources limits for the podSweeper container + ## @param podSweeper.resources.requests [object] The requested resources for the podSweeper container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## @section Server parameters @@ -108,6 +176,10 @@ server: pullPolicy: IfNotPresent tag: 0.29.13-alpha + ## @param server.podAnnotations [object] Add extra annotations to the server pod + ## + podAnnotations: {} + ## Configure extra options for Serer containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param server.livenessProbe.enabled Enable livenessProbe on the server @@ -140,6 +212,26 @@ server: failureThreshold: 3 successThreshold: 1 + ## Server app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param server.resources.limits [object] The resources limits for the server container + ## @param server.resources.requests [object] The requested resources for the server container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## @param server.service.type The service type to use for the API server ## @param server.service.port The service port to expose the API server on service: From 6bac0d65325b05a9d82960a20fb6d64bfd6e7f7f Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Thu, 2 Sep 2021 13:36:16 -0500 Subject: [PATCH 05/16] Bitnami image needs a .kube folder that it can write to Prevents 'Throttling request took' error message --- charts/airbyte/templates/pod-sweeper/deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml index d333e0cf1c09..012ee868ec5c 100644 --- a/charts/airbyte/templates/pod-sweeper/deployment.yaml +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -33,11 +33,15 @@ spec: - mountPath: /script/sweep-pod.sh subPath: sweep-pod.sh name: sweep-pod-script + - mountPath: /.kube + name: kube-config command: ["/bin/bash", "-c", /script/sweep-pod.sh] {{- if .Values.podSweeper.resources }} resources: {{- toYaml .Values.podSweeper.resources | nindent 10 }} {{- end }} volumes: + - name: kube-config + emptyDir: {} - name: sweep-pod-script configMap: name: {{ include "airbyte.fullname" . }}-sweep-pod-script From fe78fabe514a957a264583c2112cc38a84e022b6 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Thu, 2 Sep 2021 13:38:33 -0500 Subject: [PATCH 06/16] Move podSweeper to recreate strategy Due to the volume mounts, it needs to be recreated so volumes can be moved to a new node. Potentially, we could add an affinity so that the volume is already attached to the node for no downtime. Another alternative would be to use object storage or something for this. --- charts/airbyte/templates/server/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index 90c6571b80f8..e63bfb5032ce 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -11,6 +11,8 @@ spec: selector: matchLabels: airbyte: server + strategy: + type: Recreate # Needed due to volume claims template: metadata: labels: From ebbc3235d2283b5b2f1d3952ba908ce66083f05b Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 7 Sep 2021 13:03:59 -0500 Subject: [PATCH 07/16] Add node selectors & tolerations to helm chart --- charts/airbyte/README.md | 10 ++++ .../templates/pod-sweeper/deployment.yaml | 6 +++ .../templates/scheduler/deployment.yaml | 6 +++ .../airbyte/templates/server/deployment.yaml | 6 +++ .../templates/temporal/deployment.yaml | 6 +++ .../airbyte/templates/webapp/deployment.yaml | 6 +++ charts/airbyte/values.yaml | 50 +++++++++++++++++++ 7 files changed, 90 insertions(+) diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 67110aabacaf..06ddca177c51 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -26,6 +26,8 @@ | `webapp.service.port` | The service port to expose the webapp on | `80` | | `webapp.resources.limits` | The resources limits for the Web container | `{}` | | `webapp.resources.requests` | The requested resources for the Web container | `{}` | +| `webapp.nodeSelector` | Node labels for pod assignment | `{}` | +| `webapp.tolerations` | Tolerations for webapp pod assignment. | `[]` | | `webapp.ingress.enabled` | Set to true to enable ingress record generation | `false` | | `webapp.ingress.className` | Specifies ingressClassName for clusters >= 1.18+ | `""` | | `webapp.ingress.hosts` | Ingress Hosts configuration | `[]` | @@ -45,6 +47,8 @@ | `scheduler.podAnnotations` | Add extra annotations to the scheduler pod | `{}` | | `scheduler.resources.limits` | The resources limits for the scheduler container | `{}` | | `scheduler.resources.requests` | The requested resources for the scheduler container | `{}` | +| `scheduler.nodeSelector` | Node labels for pod assignment | `{}` | +| `scheduler.tolerations` | Tolerations for scheduler pod assignment. | `[]` | ### Pod Sweeper parameters @@ -57,6 +61,8 @@ | `podSweeper.podAnnotations` | Add extra annotations to the podSweeper pod | `{}` | | `podSweeper.resources.limits` | The resources limits for the podSweeper container | `{}` | | `podSweeper.resources.requests` | The requested resources for the podSweeper container | `{}` | +| `podSweeper.nodeSelector` | Node labels for pod assignment | `{}` | +| `podSweeper.tolerations` | Tolerations for podSweeper pod assignment. | `[]` | ### Server parameters @@ -87,6 +93,8 @@ | `server.persistence.accessMode` | The access mode for the airbyte server pvc | `ReadWriteOnce` | | `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `standard` | | `server.persistence.size` | The size of the pvc to use for the airbyte server pvc | `1Gi` | +| `server.nodeSelector` | Node labels for pod assignment | `{}` | +| `server.tolerations` | Tolerations for server pod assignment. | `[]` | ### Temporal parameters @@ -99,6 +107,8 @@ | `temporal.image.tag` | The temporal image tag to use | `1.7.0` | | `temporal.service.type` | The Kubernetes Service Type | `ClusterIP` | | `temporal.service.port` | The temporal port and exposed kubernetes port | `7233` | +| `temporal.nodeSelector` | Node labels for pod assignment | `{}` | +| `temporal.tolerations` | Tolerations for pod assignment. | `[]` | ### Airbyte Database parameters diff --git a/charts/airbyte/templates/pod-sweeper/deployment.yaml b/charts/airbyte/templates/pod-sweeper/deployment.yaml index 012ee868ec5c..b7c0b1bab2fa 100644 --- a/charts/airbyte/templates/pod-sweeper/deployment.yaml +++ b/charts/airbyte/templates/pod-sweeper/deployment.yaml @@ -20,6 +20,12 @@ spec: {{- end }} spec: serviceAccountName: {{ include "airbyte.serviceAccountName" . }} + {{- if .Values.podSweeper.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podSweeper.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.tolerations "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-pod-sweeper image: "{{ .Values.podSweeper.image.repository }}:{{ .Values.podSweeper.image.tag}}" diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml index 4d4dac724b33..d4fde6a6742e 100644 --- a/charts/airbyte/templates/scheduler/deployment.yaml +++ b/charts/airbyte/templates/scheduler/deployment.yaml @@ -20,6 +20,12 @@ spec: spec: serviceAccountName: {{ include "airbyte.serviceAccountName" . }} automountServiceAccountToken: true + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.tolerations "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-scheduler-container image: "{{ .Values.scheduler.image.repository }}:{{ default .Chart.AppVersion .Values.scheduler.image.tag }}" diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index e63bfb5032ce..4e7cc99b0b49 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -22,6 +22,12 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.server.podAnnotations "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.server.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.server.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.server.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.server.tolerations "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-server-container image: "{{ .Values.server.image.repository }}:{{ default .Chart.AppVersion .Values.server.image.tag }}" diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml index c0b86422c04b..9402088b825a 100644 --- a/charts/airbyte/templates/temporal/deployment.yaml +++ b/charts/airbyte/templates/temporal/deployment.yaml @@ -15,6 +15,12 @@ spec: labels: airbyte: temporal spec: + {{- if .Values.temporal.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.temporal.tolerations "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-temporal image: "{{ .Values.temporal.image.repository}}:{{ .Values.temporal.image.tag }}" diff --git a/charts/airbyte/templates/webapp/deployment.yaml b/charts/airbyte/templates/webapp/deployment.yaml index bca9aaf1073c..43a524cfe7ed 100644 --- a/charts/airbyte/templates/webapp/deployment.yaml +++ b/charts/airbyte/templates/webapp/deployment.yaml @@ -19,6 +19,12 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.podAnnotations "context" $) | nindent 8 }} {{- end }} spec: + {{- if .Values.webapp.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.webapp.tolerations "context" $) | nindent 8 }} + {{- end }} containers: - name: airbyte-webapp-container image: "{{ .Values.webapp.image.repository }}:{{ default .Chart.AppVersion .Values.webapp.image.tag }}" diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 74fd0cb29c34..32e73a148094 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -62,6 +62,16 @@ webapp: ## cpu: 250m requests: {} + ## @param webapp.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param webapp.tolerations [array] Tolerations for webapp pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## Configure the ingress resource that allows you to access the Airbyte installation. ## ref: http://kubernetes.io/docs/user-guide/ingress/ ## @param webapp.ingress.enabled Set to true to enable ingress record generation @@ -125,6 +135,16 @@ scheduler: ## cpu: 250m requests: {} + ## @param scheduler.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param scheduler.tolerations [array] Tolerations for scheduler pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @section Pod Sweeper parameters @@ -161,6 +181,16 @@ podSweeper: ## cpu: 250m requests: {} + ## @param podSweeper.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param podSweeper.tolerations [array] Tolerations for podSweeper pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @section Server parameters @@ -246,6 +276,16 @@ server: storageClass: standard size: 1Gi + ## @param server.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param server.tolerations [array] Tolerations for server pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @section Temporal parameters ## TODO: Move to consuming temporal from a dedicated helm chart @@ -267,6 +307,16 @@ temporal: type: ClusterIP port: 7233 + ## @param temporal.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param temporal.tolerations [array] Tolerations for pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## @section Airbyte Database parameters From 2df0ec4e75a563e6954e51ef6002bb82c4881e1a Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 20:29:54 -0500 Subject: [PATCH 08/16] Configure server data pvc to default to using the default storage class --- charts/airbyte/README.md | 9 ++++++++- charts/airbyte/templates/server/pvc-data.yaml | 2 +- charts/airbyte/values.yaml | 19 ++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 06ddca177c51..e1d69efe702e 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -2,6 +2,13 @@ ## Parameters +### Global Parameters + +| Name | Description | Value | +| --------------------- | -------------------------------------------- | ----- | +| `global.storageClass` | Global StorageClass for Persistent Volume(s) | `""` | + + ### Common Parameters | Name | Description | Value | @@ -91,8 +98,8 @@ | `server.service.type` | The service type to use for the API server | `ClusterIP` | | `server.service.port` | The service port to expose the API server on | `8001` | | `server.persistence.accessMode` | The access mode for the airbyte server pvc | `ReadWriteOnce` | -| `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `standard` | | `server.persistence.size` | The size of the pvc to use for the airbyte server pvc | `1Gi` | +| `server.persistence.storageClass` | The storage class to use for the airbyte server pvc | `""` | | `server.nodeSelector` | Node labels for pod assignment | `{}` | | `server.tolerations` | Tolerations for server pod assignment. | `[]` | diff --git a/charts/airbyte/templates/server/pvc-data.yaml b/charts/airbyte/templates/server/pvc-data.yaml index 353c538a7684..62ef1e3ccb38 100644 --- a/charts/airbyte/templates/server/pvc-data.yaml +++ b/charts/airbyte/templates/server/pvc-data.yaml @@ -8,7 +8,7 @@ metadata: spec: accessModes: - {{ .Values.server.persistence.accessMode | quote }} - storageClassName: {{ .Values.server.persistence.storageClass | quote }} resources: requests: storage: {{ .Values.server.persistence.size | quote }} + {{- include "common.storage.class" (dict "persistence" .Values.server.persistence "global" .Values.global) | nindent 2 }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 32e73a148094..0d18c20ff120 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -1,3 +1,10 @@ +## @section Global Parameters + +## @param global.storageClass Global StorageClass for Persistent Volume(s) +## +global: + storageClass: "" + ## @section Common Parameters ## @param nameOverride String to partially override airbyte.fullname template with a string (will prepend the release name) @@ -269,12 +276,18 @@ server: port: 8001 ## @param server.persistence.accessMode The access mode for the airbyte server pvc - ## @param server.persistence.storageClass The storage class to use for the airbyte server pvc ## @param server.persistence.size The size of the pvc to use for the airbyte server pvc persistence: - accessMode: ReadWriteOnce - storageClass: standard size: 1Gi + accessMode: ReadWriteOnce + ## @param server.persistence.storageClass The storage class to use for the airbyte server pvc + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" ## @param server.nodeSelector [object] Node labels for pod assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ From 595ca5ace631346574164d33d5e0ac4c6db04b79 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 21:09:15 -0500 Subject: [PATCH 09/16] Change namespace for Make install Addresses code review comment Co-authored-by: Jared Rhizor --- charts/airbyte/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/airbyte/Makefile b/charts/airbyte/Makefile index 82379e827c98..155a1167fe2e 100644 --- a/charts/airbyte/Makefile +++ b/charts/airbyte/Makefile @@ -1,5 +1,5 @@ RELEASE_NAME := airbyte -NAMESPACE := airbyte-test +NAMESPACE := airbyte lint: helm lint . From 8166e0fb2e09c32052d740014c8d6a689e14fbb2 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 21:10:34 -0500 Subject: [PATCH 10/16] Update timeout for chart install Due to pulling images, the install can sometimes take this long. Co-authored-by: Jared Rhizor --- charts/airbyte/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/airbyte/Makefile b/charts/airbyte/Makefile index 155a1167fe2e..69793a46f11d 100644 --- a/charts/airbyte/Makefile +++ b/charts/airbyte/Makefile @@ -11,7 +11,7 @@ install: --namespace $(NAMESPACE) \ --debug \ --wait \ - --timeout 300s \ + --timeout 600s \ $(RELEASE_NAME) . From ab8a270db45b638bc113dd1b2ef8478f761ba7af Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 21:17:44 -0500 Subject: [PATCH 11/16] Fix typo in documentation --- charts/airbyte/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 0d18c20ff120..e6722599bb31 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -217,7 +217,7 @@ server: ## podAnnotations: {} - ## Configure extra options for Serer containers' liveness and readiness probes + ## Configure extra options for the server containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes ## @param server.livenessProbe.enabled Enable livenessProbe on the server ## @param server.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe From 09828e634da6786c08d06fcaad591b3f0814e073 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 21:23:15 -0500 Subject: [PATCH 12/16] Change helm action for kind cluster setup to helm/kind-action --- .github/workflows/helm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index ac5233e20249..c8caa9955324 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -67,7 +67,7 @@ jobs: helm repo add bitnami https://charts.bitnami.com/bitnami helm dep build - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 + uses: helm/kind-action@v1.2.0 with: version: "v0.11.1" image: "kindest/node:v1.21.1" From cbb0ff413977e07eba25afc938dd71793ae4d7e8 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Tue, 14 Sep 2021 22:50:25 -0500 Subject: [PATCH 13/16] Move from using make to a bash script --- .github/workflows/helm.yaml | 20 +++------ charts/airbyte/.helmignore | 2 +- charts/airbyte/Makefile | 34 -------------- charts/airbyte/ci.sh | 90 +++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 50 deletions(-) delete mode 100644 charts/airbyte/Makefile create mode 100755 charts/airbyte/ci.sh diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index c8caa9955324..6c738d4e7c00 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -21,14 +21,9 @@ jobs: uses: azure/setup-helm@v1 with: version: '3.6.3' - - name: Build Helm Dependencies - working-directory: ./charts/airbyte - run: | - helm repo add bitnami https://charts.bitnami.com/bitnami - helm dep build - name: Lint Chart working-directory: ./charts/airbyte - run: make lint + run: ./ci.sh lint generate-docs: name: Generate Docs Parameters @@ -47,7 +42,7 @@ jobs: run: npm install -g - name: Test can update README with generated parameters working-directory: charts/airbyte - run: make update-docs + run: ./ci.sh check-docs-updated install: name: Install @@ -61,11 +56,6 @@ jobs: uses: azure/setup-helm@v1 with: version: '3.6.3' - - name: Build Helm Dependencies - working-directory: ./charts/airbyte - run: | - helm repo add bitnami https://charts.bitnami.com/bitnami - helm dep build - name: Setup Kind Cluster uses: helm/kind-action@v1.2.0 with: @@ -73,12 +63,12 @@ jobs: image: "kindest/node:v1.21.1" - name: Install airbyte chart working-directory: ./charts/airbyte - run: make install + run: ./ci.sh install - if: always() name: Print diagnostics working-directory: ./charts/airbyte - run: make diagnostics + run: ./ci.sh diagnostics - if: success() name: Test airbyte chart working-directory: ./charts/airbyte - run: make test + run: ./ci.sh test diff --git a/charts/airbyte/.helmignore b/charts/airbyte/.helmignore index 5c836a2d796e..6ada380fee0a 100644 --- a/charts/airbyte/.helmignore +++ b/charts/airbyte/.helmignore @@ -22,4 +22,4 @@ *.tmproj .vscode/ -Makefile \ No newline at end of file +ci.sh \ No newline at end of file diff --git a/charts/airbyte/Makefile b/charts/airbyte/Makefile deleted file mode 100644 index 69793a46f11d..000000000000 --- a/charts/airbyte/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -RELEASE_NAME := airbyte -NAMESPACE := airbyte - -lint: - helm lint . - - -install: - helm upgrade --install \ - --create-namespace \ - --namespace $(NAMESPACE) \ - --debug \ - --wait \ - --timeout 600s \ - $(RELEASE_NAME) . - - -test: - helm test $(RELEASE_NAME) --logs --debug --namespace $(NAMESPACE) - - -diagnostics: - kubectl -n $(NAMESPACE) get pods -o wide - kubectl -n $(NAMESPACE) get pods --no-headers | grep -v "Running" | awk '{print $$1}' | xargs -L 1 -r kubectl -n $(NAMESPACE) logs - - -template: - helm template --release-name $(RELEASE_NAME) . - - -# Uses the readme-generator command to update the README.md -# with changes in the values.yaml file. See: https://github.com/bitnami-labs/readme-generator-for-helm -update-docs: - readme-generator -v ./values.yaml -r README.md diff --git a/charts/airbyte/ci.sh b/charts/airbyte/ci.sh new file mode 100755 index 000000000000..9b0bc77b3a93 --- /dev/null +++ b/charts/airbyte/ci.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e + +export RELEASE_NAME="${RELEASE_NAME:-airbyte}" +export NAMESPACE="${NAMESPACE:-airbyte}" +export INSTALL_TIMEOUT="${INSTALL_TIMEOUT:-600s}" + +usage() { + echo "Airbyte Helm Chart CI Script" + echo "" + echo "Usage:" + echo " ./ci.sh [command]" + echo "" + echo "Available Commands" + echo "" + echo "lint Lints the helm chart" + echo "test Runs 'helm test' for the helm chart" + echo "install Installs the helm chart" + echo "diagnostics Prints diagnostics & logs of pods that aren't running" + echo "template Templates out the default kubernetes manifests generated by helm" + echo "update-docs Regenerates the README.md documentation after making changes to the values.yaml" + echo "check-docs-updated Fails if changes values.yaml and README.md documentation are out of sync" + echo "help Displays help about this command" +} + +if ! helm repo list | grep "bitnami" > /dev/null 2>&1; then + helm repo add bitnami https://charts.bitnami.com/bitnami +fi + +if [ ! -d ./charts ]; then + helm dep build +fi + + +case "$1" in + lint) + helm lint . + ;; + + test) + helm test "${RELEASE_NAME}" --logs --debug --namespace "${NAMESPACE}" + ;; + + install) + helm upgrade --install \ + --create-namespace \ + --namespace "${NAMESPACE}" \ + --debug \ + --wait \ + --timeout "${INSTALL_TIMEOUT}" \ + "${RELEASE_NAME}" . + ;; + + diagnostics) + kubectl -n "${NAMESPACE}" get pods -o wide + kubectl -n "${NAMESPACE}" get pods --no-headers | grep -v "Running" | awk '{print $1}' | xargs -L 1 -r kubectl -n "${NAMESPACE}" logs + ;; + + template) + helm template --release-name "${RELEASE_NAME}" . + ;; + + # Uses the readme-generator command to update the README.md + # with changes in the values.yaml file. See: https://github.com/bitnami-labs/readme-generator-for-helm + update-docs) + readme-generator -v ./values.yaml -r README.md + ;; + + # Checks if the docs were updated as a result of changes in the values.yaml file and if they were + # fails CI with a message about needing to run the update-docs command. + check-docs-updated) + readme-generator -v ./values.yaml -r README.md + if git status --porcelain | grep "charts/airbyte/README.md"; then + echo "It appears the README.md has not been updated with changes from the values.yaml." + echo "Please run './ci.sh update-docs' locally and commit your changes." + exit 1 + else + echo "No uncommitted changes to the README.md detected" + fi + ;; + + help) + usage + ;; + + *) + usage + ;; +esac From d6f2674f2d1486a06ee099d7149e2ef891bea3bd Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 15 Sep 2021 13:09:58 -0500 Subject: [PATCH 14/16] Fix minio endpoint --- charts/airbyte/templates/_helpers.tpl | 4 ++-- charts/airbyte/templates/env-configmap.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl index 7a313aa77699..2b3aa6dfcbc5 100644 --- a/charts/airbyte/templates/_helpers.tpl +++ b/charts/airbyte/templates/_helpers.tpl @@ -150,14 +150,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Add environment variables to configure minio */}} {{- define "airbyte.minio.host" -}} -{{- ternary (include "airbyte.minio.fullname" .) .Values.externalMinio.host .Values.minio.enabled | quote -}} +{{- ternary (include "airbyte.minio.fullname" .) .Values.externalMinio.host .Values.minio.enabled -}} {{- end -}} {{/* Add environment variables to configure minio */}} {{- define "airbyte.minio.port" -}} -{{- ternary "9000" .Values.externalMinio.port .Values.minio.enabled | quote -}} +{{- ternary "9000" .Values.externalMinio.port .Values.minio.enabled -}} {{- end -}} {{- define "airbyte.minio.endpoint" -}} diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index ba72af777820..1e6710ff306c 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -32,7 +32,7 @@ data: RUN_DATABASE_MIGRATION_ON_STARTUP: "true" S3_LOG_BUCKET: airbyte-dev-logs S3_LOG_BUCKET_REGION: "" - S3_MINIO_ENDPOINT: http://airbyte-minio:9000 + S3_MINIO_ENDPOINT: {{ include "airbyte.minio.endpoint" . }} S3_PATH_STYLE_ACCESS: "true" SUBMITTER_NUM_THREADS: "10" TEMPORAL_HOST: {{ include "common.names.fullname" . }}-temporal:{{ .Values.temporal.service.port }} From c4a25d09722f697fb93a3dff9d9d6a2dc6981f62 Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Wed, 15 Sep 2021 13:27:50 -0500 Subject: [PATCH 15/16] Fix postgres environment variable generation --- charts/airbyte/templates/_helpers.tpl | 17 +++++++++++++---- charts/airbyte/templates/env-configmap.yaml | 8 ++++---- .../airbyte/templates/temporal/deployment.yaml | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl index 2b3aa6dfcbc5..e415eebbc9e2 100644 --- a/charts/airbyte/templates/_helpers.tpl +++ b/charts/airbyte/templates/_helpers.tpl @@ -93,21 +93,21 @@ Get the Postgresql credentials secret. Add environment variables to configure database values */}} {{- define "airbyte.database.host" -}} -{{- ternary (include "airbyte.postgresql.fullname" .) .Values.externalDatabase.host .Values.postgresql.enabled | quote -}} +{{- ternary (include "airbyte.postgresql.fullname" .) .Values.externalDatabase.host .Values.postgresql.enabled -}} {{- end -}} {{/* Add environment variables to configure database values */}} {{- define "airbyte.database.user" -}} -{{- ternary .Values.postgresql.postgresqlUsername .Values.externalDatabase.user .Values.postgresql.enabled | quote -}} +{{- ternary .Values.postgresql.postgresqlUsername .Values.externalDatabase.user .Values.postgresql.enabled -}} {{- end -}} {{/* Add environment variables to configure database values */}} {{- define "airbyte.database.name" -}} -{{- ternary .Values.postgresql.postgresqlDatabase .Values.externalDatabase.database .Values.postgresql.enabled | quote -}} +{{- ternary .Values.postgresql.postgresqlDatabase .Values.externalDatabase.database .Values.postgresql.enabled -}} {{- end -}} {{/* @@ -133,9 +133,18 @@ Add environment variables to configure database values Add environment variables to configure database values */}} {{- define "airbyte.database.port" -}} -{{- ternary "5432" .Values.externalDatabase.port .Values.postgresql.enabled | quote -}} +{{- ternary "5432" .Values.externalDatabase.port .Values.postgresql.enabled -}} {{- end -}} +{{/* +Add environment variables to configure database values +*/}} +{{- define "airbyte.database.url" -}} +{{- $host := (include "airbyte.database.host" .) -}} +{{- $dbName := (include "airbyte.database.name" .) -}} +{{- $port := (include "airbyte.database.port" . ) -}} +{{- printf "jdbc:postgresql://%s:%s/%s" $host $port $dbName -}} +{{- end -}} {{/* Create a default fully qualified minio name. diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 1e6710ff306c..dfc38afdebd2 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -9,12 +9,12 @@ data: AWS_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password }} CONFIG_ROOT: /configs DATA_DOCKER_MOUNT: airbyte_data - DATABASE_DB: {{ .Values.postgresql.postgresqlDatabase }} + DATABASE_DB: {{ include "airbyte.database.name" . }} DATABASE_HOST: {{ include "airbyte.database.host" . }} DATABASE_PASSWORD: {{ .Values.postgresql.postgresqlPassword }} - DATABASE_PORT: "5432" - DATABASE_URL: jdbc:postgresql://airbyte-postgresql:5432/{{ .Values.postgresql.postgresqlDatabase }} - DATABASE_USER: {{ .Values.postgresql.postgresqlUsername }} + DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} + DATABASE_URL: {{ include "airbyte.database.url" . | quote }} + DATABASE_USER: {{ include "airbyte.database.user" . }} DB_DOCKER_MOUNT: airbyte_db FULLSTORY: enabled GCP_STORAGE_BUCKET: "" diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml index 9402088b825a..48aba564aa86 100644 --- a/charts/airbyte/templates/temporal/deployment.yaml +++ b/charts/airbyte/templates/temporal/deployment.yaml @@ -31,7 +31,7 @@ spec: - name: DB # The DB engine to use value: "postgresql" - name: DB_PORT - value: {{ include "airbyte.database.port" . }} + value: {{ include "airbyte.database.port" . | quote }} - name: POSTGRES_USER value: {{ include "airbyte.database.user" . }} - name: POSTGRES_PWD From fb59cdb3520dd3c30a62bb45812fe01b221f169c Mon Sep 17 00:00:00 2001 From: Jonathan Stacks Date: Thu, 16 Sep 2021 13:08:04 -0500 Subject: [PATCH 16/16] Add final newlines at end of files per PR feedback --- charts/airbyte/.gitignore | 2 +- charts/airbyte/.helmignore | 2 +- charts/airbyte/files/sweep-pod.sh | 2 +- charts/airbyte/templates/_helpers.tpl | 2 +- charts/airbyte/templates/server/service.yaml | 2 +- charts/airbyte/templates/serviceaccount.yaml | 1 - charts/airbyte/templates/temporal/service.yaml | 2 +- charts/airbyte/values.yaml | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/charts/airbyte/.gitignore b/charts/airbyte/.gitignore index 28ea717bd6a2..88e91e8a8f34 100644 --- a/charts/airbyte/.gitignore +++ b/charts/airbyte/.gitignore @@ -1,2 +1,2 @@ # Charts are downloaded at install time with `helm dep build`. -charts \ No newline at end of file +charts diff --git a/charts/airbyte/.helmignore b/charts/airbyte/.helmignore index 6ada380fee0a..f885da3fd491 100644 --- a/charts/airbyte/.helmignore +++ b/charts/airbyte/.helmignore @@ -22,4 +22,4 @@ *.tmproj .vscode/ -ci.sh \ No newline at end of file +ci.sh diff --git a/charts/airbyte/files/sweep-pod.sh b/charts/airbyte/files/sweep-pod.sh index 029772624cc4..15422f204a2f 100644 --- a/charts/airbyte/files/sweep-pod.sh +++ b/charts/airbyte/files/sweep-pod.sh @@ -37,4 +37,4 @@ do done ) sleep 60 -done \ No newline at end of file +done diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl index e415eebbc9e2..f6f4c8fa67e8 100644 --- a/charts/airbyte/templates/_helpers.tpl +++ b/charts/airbyte/templates/_helpers.tpl @@ -173,4 +173,4 @@ Add environment variables to configure minio {{- $host := (include "airbyte.minio.host" .) -}} {{- $port := (include "airbyte.minio.port" .) -}} {{- printf "http://%s:%s" $host $port -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/airbyte/templates/server/service.yaml b/charts/airbyte/templates/server/service.yaml index f069bedf3d79..a00b7021a17f 100644 --- a/charts/airbyte/templates/server/service.yaml +++ b/charts/airbyte/templates/server/service.yaml @@ -11,4 +11,4 @@ spec: protocol: TCP name: http selector: - airbyte: server \ No newline at end of file + airbyte: server diff --git a/charts/airbyte/templates/serviceaccount.yaml b/charts/airbyte/templates/serviceaccount.yaml index 142191c9c9ef..2be26ab5e964 100644 --- a/charts/airbyte/templates/serviceaccount.yaml +++ b/charts/airbyte/templates/serviceaccount.yaml @@ -31,4 +31,3 @@ subjects: - kind: ServiceAccount name: {{ include "airbyte.serviceAccountName" . }} {{- end }} - diff --git a/charts/airbyte/templates/temporal/service.yaml b/charts/airbyte/templates/temporal/service.yaml index 0242d2ff0cb3..7c949422aeac 100644 --- a/charts/airbyte/templates/temporal/service.yaml +++ b/charts/airbyte/templates/temporal/service.yaml @@ -10,4 +10,4 @@ spec: protocol: TCP targetPort: 7233 selector: - airbyte: temporal \ No newline at end of file + airbyte: temporal diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index e6722599bb31..fe4e2d5cf7a0 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -390,4 +390,4 @@ minio: ## @param externalMinio.port Minio Port externalMinio: host: localhost - port: 9000 \ No newline at end of file + port: 9000