add secrets to kubernetes yamls (#5962)

Author: mohammad-bolt

kube/overlays/dev-integration-test/.env

@@ -1,8 +1,6 @@
 AIRBYTE_VERSION=dev
 
 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
 DATABASE_HOST=airbyte-db-svc
 DATABASE_PORT=5432
 DATABASE_DB=airbyte
@@ -43,14 +41,11 @@ LOG_LEVEL=INFO
 # S3/Minio Log Configuration
 S3_LOG_BUCKET=airbyte-dev-logs
 S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
 # GCS Log Configuration
 GCP_STORAGE_BUCKET=
-GOOGLE_APPLICATION_CREDENTIALS=
 
 # Docker Resource Limits
 RESOURCE_CPU_REQUEST=

kube/overlays/dev-integration-test/.secrets

@@ -0,0 +1,5 @@
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+GOOGLE_APPLICATION_CREDENTIALS=

kube/overlays/dev-integration-test/kustomization.yaml

@@ -24,6 +24,10 @@ configMapGenerator:
   - name: airbyte-env
     env: .env
 
+secretGenerator:
+  - name: airbyte-secrets
+    env: .secrets
+
 patchesStrategicMerge:
   - pod-antiaffinity.yaml
   - parallelize-worker.yaml

kube/overlays/dev/.env

@@ -1,8 +1,6 @@
 AIRBYTE_VERSION=dev
 
 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
 DATABASE_HOST=airbyte-db-svc
 DATABASE_PORT=5432
 DATABASE_DB=airbyte
@@ -45,14 +43,11 @@ LOG_LEVEL=INFO
 # S3/Minio Log Configuration
 S3_LOG_BUCKET=airbyte-dev-logs
 S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
 # GCS Log Configuration
 GCP_STORAGE_BUCKET=
-GOOGLE_APPLICATION_CREDENTIALS=
 
 # Docker Resource Limits
 RESOURCE_CPU_REQUEST=

kube/overlays/dev/.secrets

@@ -0,0 +1,5 @@
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+GOOGLE_APPLICATION_CREDENTIALS=

kube/overlays/dev/kustomization.yaml

@@ -23,3 +23,7 @@ images:
 configMapGenerator:
   - name: airbyte-env
     env: .env
+
+secretGenerator:
+  - name: airbyte-secrets
+    env: .secrets

kube/overlays/stable-with-resource-limits/.env

@@ -1,8 +1,6 @@
 AIRBYTE_VERSION=0.29.17-alpha
 
 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
 DATABASE_HOST=airbyte-db-svc
 DATABASE_PORT=5432
 DATABASE_DB=airbyte
@@ -45,14 +43,11 @@ LOG_LEVEL=INFO
 # S3/Minio Log Configuration
 S3_LOG_BUCKET=airbyte-dev-logs
 S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
 # GCS Log Configuration
 GCP_STORAGE_BUCKET=
-GOOGLE_APPLICATION_CREDENTIALS=
 
 # Docker Resource Limits
 RESOURCE_CPU_REQUEST=

kube/overlays/stable-with-resource-limits/.secrets

@@ -0,0 +1,5 @@
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+GOOGLE_APPLICATION_CREDENTIALS=

kube/overlays/stable-with-resource-limits/kustomization.yaml

@@ -24,5 +24,9 @@ configMapGenerator:
   - name: airbyte-env
     env: .env
 
+secretGenerator:
+  - name: airbyte-secrets
+    env: .secrets
+
 patchesStrategicMerge:
   - set-resource-limits.yaml

kube/overlays/stable/.env

@@ -1,8 +1,6 @@
 AIRBYTE_VERSION=0.29.17-alpha
 
 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
 DATABASE_HOST=airbyte-db-svc
 DATABASE_PORT=5432
 DATABASE_DB=airbyte
@@ -45,14 +43,11 @@ LOG_LEVEL=INFO
 # S3/Minio Log Configuration
 S3_LOG_BUCKET=airbyte-dev-logs
 S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=minio123
 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 S3_PATH_STYLE_ACCESS=true
 
 # GCS Log Configuration
 GCP_STORAGE_BUCKET=
-GOOGLE_APPLICATION_CREDENTIALS=
 
 # Docker Resource Limits
 RESOURCE_CPU_REQUEST=

kube/overlays/stable/.secrets

@@ -0,0 +1,5 @@
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+AWS_ACCESS_KEY_ID=minio
+AWS_SECRET_ACCESS_KEY=minio123
+GOOGLE_APPLICATION_CREDENTIALS=

kube/overlays/stable/kustomization.yaml

@@ -24,3 +24,7 @@ configMapGenerator:
   - name: airbyte-env
     envs:
       - .env
+
+secretGenerator:
+  - name: airbyte-secrets
+    env: .secrets

kube/resources/scheduler.yaml

@@ -38,8 +38,8 @@ spec:
                   key: DATABASE_PORT
             - name: DATABASE_PASSWORD
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_PASSWORD
             - name: DATABASE_URL
               valueFrom:
@@ -48,8 +48,8 @@ spec:
                   key: DATABASE_URL
             - name: DATABASE_USER
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_USER
             - name: TRACKING_STRATEGY
               valueFrom:
@@ -134,13 +134,13 @@ spec:
                   key: S3_LOG_BUCKET_REGION
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_ACCESS_KEY_ID
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_SECRET_ACCESS_KEY
             - name: S3_MINIO_ENDPOINT
               valueFrom:
@@ -154,8 +154,8 @@ spec:
                   key: S3_PATH_STYLE_ACCESS
             - name: GOOGLE_APPLICATION_CREDENTIALS
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: GOOGLE_APPLICATION_CREDENTIALS
             - name: GCP_STORAGE_BUCKET
               valueFrom:

kube/resources/server.yaml

@@ -40,8 +40,8 @@ spec:
                   key: CONFIG_ROOT
             - name: DATABASE_PASSWORD
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_PASSWORD
             - name: DATABASE_URL
               valueFrom:
@@ -50,8 +50,8 @@ spec:
                   key: DATABASE_URL
             - name: DATABASE_USER
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_USER
             - name: TRACKING_STRATEGY
               valueFrom:
@@ -115,13 +115,13 @@ spec:
                   key: S3_LOG_BUCKET_REGION
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_ACCESS_KEY_ID
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_SECRET_ACCESS_KEY
             - name: S3_MINIO_ENDPOINT
               valueFrom:
@@ -135,8 +135,8 @@ spec:
                   key: S3_PATH_STYLE_ACCESS
             - name: GOOGLE_APPLICATION_CREDENTIALS
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: GOOGLE_APPLICATION_CREDENTIALS
             - name: GCP_STORAGE_BUCKET
               valueFrom:

kube/resources/temporal.yaml

@@ -79,13 +79,13 @@ spec:
           env:
             - name: POSTGRES_USER
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_USER
             - name: POSTGRES_PWD
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_PASSWORD
             - name: DYNAMIC_CONFIG_FILE_PATH
               value: "config/dynamicconfig/development.yaml"

kube/resources/worker.yaml

@@ -40,8 +40,8 @@ spec:
                   key: DATABASE_PORT
             - name: DATABASE_PASSWORD
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_PASSWORD
             - name: DATABASE_URL
               valueFrom:
@@ -50,8 +50,8 @@ spec:
                   key: DATABASE_URL
             - name: DATABASE_USER
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: DATABASE_USER
             - name: TRACKING_STRATEGY
               valueFrom:
@@ -136,13 +136,13 @@ spec:
                   key: S3_LOG_BUCKET_REGION
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_ACCESS_KEY_ID
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: AWS_SECRET_ACCESS_KEY
             - name: S3_MINIO_ENDPOINT
               valueFrom:
@@ -156,8 +156,8 @@ spec:
                   key: S3_PATH_STYLE_ACCESS
             - name: GOOGLE_APPLICATION_CREDENTIALS
               valueFrom:
-                configMapKeyRef:
-                  name: airbyte-env
+                secretKeyRef:
+                  name: airbyte-secrets
                   key: GOOGLE_APPLICATION_CREDENTIALS
             - name: GCP_STORAGE_BUCKET
               valueFrom: