Skip to content

Commit 488ae96

Browse files
girardaSofiiaZaitseva
girarda
authored and
SofiiaZaitseva
committed
Add connector builder server to airbyte proxy, kube overlays, and helm charts (#19554)
1 parent c18c2a1 commit 488ae96

40 files changed

+697
-57
lines changed

‎.env

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
6161
WEBAPP_URL=http://localhost:8000/
6262
# Although not present as an env var, required for webapp configuration.
6363
API_URL=/api/v1/
64+
CONNECTOR_BUILDER_API_URL=/connector-builder-api
6465

6566
### JOBS ###
6667
# Relevant to scaling.

‎.github/workflows/publish-helm-charts.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070
- name: "Helm package"
7171
shell: bash
7272
run: |
73-
declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron")
73+
declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron" "airbyte-connector-builder-server")
7474
for val in ${StringArray[@]}; do
7575
cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE
7676
sed -i -E 's/version: \"[0-9]+\.[0-9]+\.[0-9]+\"/version: \"${{ needs.generate-semantic-version.outputs.next-version }}\"/' ./airbyte/charts/${val}/Chart.yaml

‎airbyte-proxy/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ ENV VERSION ${VERSION}
99

1010
RUN apt-get update -y && apt-get install -y apache2-utils && rm -rf /var/lib/apt/lists/*
1111

12-
# This variable can be used to update the destintion containers that Nginx proxies to.
12+
# This variable can be used to update the destination containers that Nginx proxies to.
1313
ENV PROXY_PASS_WEB "http://airbyte-webapp:80"
1414
ENV PROXY_PASS_API "http://airbyte-server:8001"
15+
ENV CONNECTOR_BUILDER_SERVER_API "http://airbyte-connector-builder-server:80"
1516

1617
# Nginx config file
1718
WORKDIR /

‎airbyte-proxy/nginx-auth.conf.template

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,25 @@ http {
4242
}
4343
}
4444
}
45+
46+
server {
47+
listen 8003;
48+
49+
location / {
50+
proxy_set_header Host $host;
51+
proxy_set_header X-Real-IP $remote_addr;
52+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
53+
54+
auth_basic "Welcome to Airbyte";
55+
auth_basic_user_file /etc/nginx/.htpasswd;
56+
57+
proxy_pass "${CONNECTOR_BUILDER_SERVER_API}";
58+
59+
error_page 401 /etc/nginx/401.html;
60+
location ~ (401.html)$ {
61+
alias /etc/nginx/$1;
62+
auth_basic off;
63+
}
64+
}
65+
}
4566
}

‎airbyte-proxy/nginx-no-auth.conf.template

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,16 @@ http {
2424
proxy_pass "${PROXY_PASS_API}";
2525
}
2626
}
27+
28+
server {
29+
listen 8003;
30+
31+
location / {
32+
proxy_set_header Host $host;
33+
proxy_set_header X-Real-IP $remote_addr;
34+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
35+
36+
proxy_pass "${CONNECTOR_BUILDER_SERVER_API}";
37+
}
38+
}
2739
}

‎airbyte-proxy/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ else
1616
TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template"
1717
fi
1818

19-
envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
19+
envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
2020

2121
echo "starting nginx..."
2222
nginx -v

‎airbyte-proxy/test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environmen
1111
echo "testing with proxy container airbyte/proxy:$VERSION"
1212

1313
function start_container () {
14-
CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
14+
CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
1515
echo $CMD
1616
eval $CMD
1717
wait_for_docker;

‎airbyte-webapp/.env

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,3 @@ REACT_APP_FULL_STORY_ORG=13AXQ4
33
REACT_APP_SENTRY_DSN=
44
REACT_APP_INTERCOM_APP_ID=nj1oam7s
55
REACT_APP_OSANO=16A0CTTE7vE8m1Qif/67beec9b-e563-4736-bdb4-4fe4adc39d48
6-
REACT_APP_CONNECTOR_BUILDER_API=/connector-builder-api/

‎airbyte-webapp/nginx/default.conf.template

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ server {
2525
window.TRACKING_STRATEGY = "$TRACKING_STRATEGY";
2626
window.AIRBYTE_VERSION = "$AIRBYTE_VERSION";
2727
window.API_URL = "$API_URL";
28+
window.CONNECTOR_BUILDER_API_URL = "$CONNECTOR_BUILDER_API_URL";
2829
</script>';
2930
sub_filter_once on;
3031
}

‎airbyte-webapp/src/config/configProviders.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const windowConfigProvider: ConfigProvider = async () => {
1111
token: window.SEGMENT_TOKEN,
1212
},
1313
apiUrl: window.API_URL,
14+
connectorBuilderApiUrl: window.CONNECTOR_BUILDER_API_URL,
1415
version: window.AIRBYTE_VERSION,
1516
// cloud only start
1617
// TODO: remove when infra team supports proper webapp building
@@ -22,7 +23,6 @@ const windowConfigProvider: ConfigProvider = async () => {
2223
const envConfigProvider: ConfigProvider = async () => {
2324
return {
2425
apiUrl: process.env.REACT_APP_API_URL,
25-
connectorBuilderUrl: process.env.REACT_APP_CONNECTOR_BUILDER_API,
2626
integrationUrl: process.env.REACT_APP_INTEGRATION_DOCS_URLS,
2727
segment: {
2828
token: process.env.REACT_APP_SEGMENT_TOKEN,

‎airbyte-webapp/src/config/types.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ declare global {
33
TRACKING_STRATEGY?: string;
44
AIRBYTE_VERSION?: string;
55
API_URL?: string;
6+
CONNECTOR_BUILDER_API_URL?: string;
67
CLOUD?: string;
78
REACT_APP_DATADOG_APPLICATION_ID: string;
89
REACT_APP_DATADOG_CLIENT_TOKEN: string;

‎charts/airbyte-connector-builder-server/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Charts are downloaded at install time with `helm dep build`.
2+
charts

‎charts/airbyte-connector-builder-server/Chart.lock

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dependencies:
2+
- name: common
3+
repository: https://charts.bitnami.com/bitnami
4+
version: 1.17.1
5+
digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690
6+
generated: "2022-10-17T18:35:15.123937677Z"

‎charts/airbyte-connector-builder-server/Chart.yaml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: v2
2+
name: connector-builder-server
3+
description: Helm chart to deploy airbyte-connector-builder-server
4+
5+
# A chart can be either an 'application' or a 'library' chart.
6+
#
7+
# Application charts are a collection of templates that can be packaged into versioned archives
8+
# to be deployed.
9+
#
10+
# Library charts provide useful utilities or functions for the chart developer. They're included as
11+
# a dependency of application charts to inject those utilities and functions into the rendering
12+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
13+
type: application
14+
15+
# This is the chart version. This version number should be incremented each time you make changes
16+
# to the chart and its templates, including the app version.
17+
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18+
version: "0.40.46"
19+
20+
# This is the version number of the application being deployed. This version number should be
21+
# incremented each time you make changes to the application. Versions are not expected to
22+
# follow Semantic Versioning. They should reflect the version the application is using.
23+
# It is recommended to use it with quotes.
24+
appVersion: "0.40.21"
25+
26+
dependencies:
27+
- name: common
28+
repository: https://charts.bitnami.com/bitnami
29+
tags:
30+
- bitnami-common
31+
version: 1.x.x

‎charts/airbyte-connector-builder-server/README.md

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# server
2+
3+
![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.4](https://img.shields.io/badge/AppVersion-0.40.4-informational?style=flat-square)
4+
5+
Helm chart to deploy airbyte-server
6+
7+
## Requirements
8+
9+
| Repository | Name | Version |
10+
|------------|------|---------|
11+
| https://charts.bitnami.com/bitnami | common | 1.x.x |
12+
13+
## Values
14+
15+
| Key | Type | Default | Description |
16+
|-----|------|---------|-------------|
17+
| affinity | object | `{}` | |
18+
| containerSecurityContext | object | `{}` | |
19+
| enabled | bool | `true` | |
20+
| env_vars | object | `{}` | |
21+
| extraContainers | list | `[]` | |
22+
| extraEnv | list | `[]` | |
23+
| extraInitContainers | list | `[]` | |
24+
| extraVolumeMounts | list | `[]` | |
25+
| extraVolumes | list | `[]` | |
26+
| global.configMapName | string | `""` | |
27+
| global.credVolumeOverride | string | `""` | |
28+
| global.database.host | string | `"example.com"` | |
29+
| global.database.port | string | `"5432"` | |
30+
| global.database.secretName | string | `""` | |
31+
| global.database.secretValue | string | `""` | |
32+
| global.deploymentMode | string | `"oss"` | |
33+
| global.extraContainers | list | `[]` | |
34+
| global.logs.accessKey.existingSecret | string | `""` | |
35+
| global.logs.accessKey.existingSecretKey | string | `""` | |
36+
| global.logs.accessKey.password | string | `"minio"` | |
37+
| global.logs.externalMinio.enabled | bool | `false` | |
38+
| global.logs.externalMinio.host | string | `"localhost"` | |
39+
| global.logs.externalMinio.port | int | `9000` | |
40+
| global.logs.gcs.bucket | string | `""` | |
41+
| global.logs.gcs.credentials | string | `""` | |
42+
| global.logs.gcs.credentialsJson | string | `""` | |
43+
| global.logs.minio.enabled | bool | `true` | |
44+
| global.logs.s3.bucket | string | `"airbyte-dev-logs"` | |
45+
| global.logs.s3.bucketRegion | string | `""` | |
46+
| global.logs.s3.enabled | bool | `false` | |
47+
| global.logs.secretKey.existingSecret | string | `""` | |
48+
| global.logs.secretKey.existingSecretKey | string | `""` | |
49+
| global.logs.secretKey.password | string | `"minio123"` | |
50+
| global.secretName | string | `""` | |
51+
| global.serviceAccountName | string | `"placeholderServiceAccount"` | |
52+
| image.pullPolicy | string | `"IfNotPresent"` | |
53+
| image.repository | string | `"airbyte/server"` | |
54+
| livenessProbe.enabled | bool | `true` | |
55+
| livenessProbe.failureThreshold | int | `3` | |
56+
| livenessProbe.initialDelaySeconds | int | `30` | |
57+
| livenessProbe.periodSeconds | int | `10` | |
58+
| livenessProbe.successThreshold | int | `1` | |
59+
| livenessProbe.timeoutSeconds | int | `1` | |
60+
| log.level | string | `"INFO"` | |
61+
| nodeSelector | object | `{}` | |
62+
| podAnnotations | object | `{}` | |
63+
| readinessProbe.enabled | bool | `true` | |
64+
| readinessProbe.failureThreshold | int | `3` | |
65+
| readinessProbe.initialDelaySeconds | int | `10` | |
66+
| readinessProbe.periodSeconds | int | `10` | |
67+
| readinessProbe.successThreshold | int | `1` | |
68+
| readinessProbe.timeoutSeconds | int | `1` | |
69+
| replicaCount | int | `1` | |
70+
| resources.limits | object | `{}` | |
71+
| resources.requests | object | `{}` | |
72+
| secrets | object | `{}` | |
73+
| service.annotations | object | `{}` | |
74+
| service.port | int | `8001` | |
75+
| service.type | string | `"ClusterIP"` | |
76+
| tolerations | list | `[]` | |
77+
78+
----------------------------------------------
79+
Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)

‎charts/airbyte-connector-builder-server/templates/_helpers.tpl

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
{{/*
2+
Expand the name of the chart.
3+
*/}}
4+
{{- define "airbyte.name" -}}
5+
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
6+
{{- end }}
7+
8+
{{/*
9+
Create a default fully qualified app name.
10+
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
11+
If release name contains chart name it will be used as a full name.
12+
*/}}
13+
{{- define "airbyte.fullname" -}}
14+
{{- if .Values.fullnameOverride }}
15+
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
16+
{{- else }}
17+
{{- $name := default .Chart.Name .Values.nameOverride }}
18+
{{- if contains $name .Release.Name }}
19+
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
20+
{{- else }}
21+
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
22+
{{- end }}
23+
{{- end }}
24+
{{- end }}
25+
26+
{{/*
27+
Create chart name and version as used by the chart label.
28+
*/}}
29+
{{- define "airbyte.chart" -}}
30+
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
31+
{{- end }}
32+
33+
{{/*
34+
Common labels
35+
*/}}
36+
{{- define "airbyte.labels" -}}
37+
helm.sh/chart: {{ include "airbyte.chart" . }}
38+
{{ include "airbyte.selectorLabels" . }}
39+
{{- if .Chart.AppVersion }}
40+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
41+
{{- end }}
42+
app.kubernetes.io/managed-by: {{ .Release.Service }}
43+
{{- end }}
44+
45+
{{/*
46+
Selector labels
47+
*/}}
48+
{{- define "airbyte.selectorLabels" -}}
49+
app.kubernetes.io/name: {{ include "airbyte.name" . }}
50+
app.kubernetes.io/instance: {{ .Release.Name }}
51+
{{- end }}
52+
53+
{{/*
54+
Define db secret
55+
*/}}
56+
57+
{{- define "database.secret.name" -}}
58+
{{- printf "%s-postgresql" .Release.Name }}
59+
{{- end }}
60+
61+
{{/*
62+
Define imageTag
63+
*/}}
64+
65+
{{- define "connectorBuilderServer.imageTag" -}}
66+
{{- if .Values.image.tag }}
67+
{{- printf "%s" .Values.image.tag }}
68+
{{- else if ((.Values.global.image).tag) }}
69+
{{- printf "%s" .Values.global.image.tag }}
70+
{{- else }}
71+
{{- printf "%s" .Chart.AppVersion }}
72+
{{- end }}
73+
{{- end }}

0 commit comments

Comments
 (0)