Skip to content

Commit c0debfc

Browse files
bgroffperangel
andcommitted
feat: azure key vault persistence (#13791)
Co-authored-by: perangel <[email protected]>
1 parent d7bc738 commit c0debfc

File tree

12 files changed

+278
-2
lines changed

12 files changed

+278
-2
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package secrets.persistence
6+
7+
import com.azure.identity.ClientSecretCredentialBuilder
8+
import com.azure.security.keyvault.secrets.SecretClient
9+
import com.azure.security.keyvault.secrets.SecretClientBuilder
10+
import com.azure.security.keyvault.secrets.models.KeyVaultSecret
11+
import com.azure.security.keyvault.secrets.models.SecretProperties
12+
import io.airbyte.config.secrets.SecretCoordinate
13+
import io.airbyte.config.secrets.persistence.SecretPersistence
14+
import io.micronaut.context.annotation.Requires
15+
import io.micronaut.context.annotation.Value
16+
import jakarta.inject.Named
17+
import jakarta.inject.Singleton
18+
import java.time.Duration
19+
20+
/**
21+
* SecretPersistence implementation for Azure Key Vault
22+
*/
23+
@Singleton
24+
@Requires(property = "airbyte.secret.persistence", pattern = "(?i)^azure_key_vault$")
25+
@Named("secretPersistence")
26+
class AzureKeyVaultPersistence(private val secretClient: AzureKeyVaultClient) : SecretPersistence {
27+
override fun read(coordinate: SecretCoordinate): String {
28+
val key = coordinate.fullCoordinate.replace("_", "-")
29+
return secretClient.client.getSecret(
30+
key,
31+
).value
32+
}
33+
34+
override fun write(
35+
coordinate: SecretCoordinate,
36+
payload: String,
37+
) {
38+
val key = coordinate.fullCoordinate.replace("_", "-")
39+
val secret =
40+
KeyVaultSecret(
41+
key,
42+
payload,
43+
)
44+
45+
if (secretClient.tags.isNotEmpty()) {
46+
secret.setProperties(SecretProperties().setTags(secretClient.tags))
47+
}
48+
49+
secretClient.client.setSecret(secret)
50+
}
51+
52+
override fun delete(coordinate: SecretCoordinate) {
53+
val key = coordinate.fullCoordinate.replace("_", "-")
54+
secretClient.client
55+
.beginDeleteSecret(key)
56+
.waitForCompletion(Duration.ofSeconds(5))
57+
secretClient.client
58+
.purgeDeletedSecret(key)
59+
}
60+
}
61+
62+
@Singleton
63+
class AzureKeyVaultClient(
64+
@Value("\${airbyte.secret.store.azure.vault-url}") private val vaultUrl: String,
65+
@Value("\${airbyte.secret.store.azure.tenant-id}") private val tenantId: String,
66+
@Value("\${airbyte.secret.store.azure.client-id}") private val clientId: String,
67+
@Value("\${airbyte.secret.store.azure.client-secret}") private val clientSecret: String,
68+
@Value("\${airbyte.secret.store.azure.tags}") val unparsedTags: String?,
69+
) {
70+
val tags: Map<String, String> = parseTags(unparsedTags)
71+
72+
private fun parseTags(tags: String?): Map<String, String> {
73+
// Define the regex pattern for the whole string validation
74+
val pattern = "^[\\w\\s._:/=+-@]+=[\\w\\s._:/=+-@]+(,\\s*[\\w\\s._:/=+-@]+=[\\w\\s._:/=+-@]+)*$".toRegex()
75+
76+
// Check if unparsedTags is not null, not blank, and matches the pattern
77+
return if (!tags.isNullOrBlank() && pattern.matches(tags)) {
78+
tags.split(",").associate { part ->
79+
val (key, value) = part.trim().split("=")
80+
key to value
81+
}
82+
} else if (tags.isNullOrBlank()) {
83+
emptyMap() // Return an empty map if unparsedTags is null or blank
84+
} else {
85+
// If the string doesn't match the pattern, throw an error
86+
throw IllegalArgumentException(
87+
"AB_SECRET_MANAGER_SECRET_TAGS does not match the expected format \"key1=value2,key2=value2\": $tags." +
88+
" Please update the AB_SECRET_MANAGER_SECRET_TAGS env var configurations.",
89+
)
90+
}
91+
}
92+
93+
val client: SecretClient by lazy {
94+
SecretClientBuilder()
95+
.vaultUrl(vaultUrl)
96+
.credential(
97+
ClientSecretCredentialBuilder()
98+
.clientSecret(clientSecret)
99+
.clientId(clientId)
100+
.tenantId(tenantId)
101+
.build(),
102+
)
103+
.buildClient()
104+
}
105+
}

airbyte-server/src/main/resources/application.yml

+6
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,12 @@ airbyte:
158158
address: ${VAULT_ADDRESS:}
159159
prefix: ${VAULT_PREFIX:}
160160
token: ${VAULT_AUTH_TOKEN:}
161+
azure:
162+
vault-url: ${AB_AZURE_KEY_VAULT_VAULT_URL:}
163+
tenant-id: ${AB_AZURE_KEY_VAULT_TENANT_ID:}
164+
client-id: ${AB_AZURE_KEY_VAULT_CLIENT_ID:}
165+
client-secret: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET:}
166+
tags: ${AB_AZURE_KEY_VAULT_TAGS:}
161167
role: ${AIRBYTE_ROLE:dev}
162168
tracking:
163169
strategy: ${TRACKING_STRATEGY:LOGGING}

airbyte-workers/src/main/resources/application.yml

+6
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,12 @@ airbyte:
247247
address: ${VAULT_ADDRESS:}
248248
prefix: ${VAULT_PREFIX:}
249249
token: ${VAULT_AUTH_TOKEN:}
250+
azure:
251+
vault-url: ${AB_AZURE_KEY_VAULT_VAULT_URL:}
252+
tenant-id: ${AB_AZURE_KEY_VAULT_TENANT_ID:}
253+
client-id: ${AB_AZURE_KEY_VAULT_CLIENT_ID:}
254+
client-secret: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET:}
255+
tags: ${AB_AZURE_KEY_VAULT_TAGS:}
250256
temporal:
251257
worker:
252258
ports: ${TEMPORAL_WORKER_PORTS:}

airbyte-workload-init-container/src/main/resources/application.yml

+6
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ airbyte:
5252
prefix: ${VAULT_PREFIX:}
5353
# injects the actual secret value
5454
token: ${VAULT_AUTH_TOKEN:vaultAuthTokenForTest}
55+
azure:
56+
vault-url: ${AB_AZURE_KEY_VAULT_VAULT_URL:}
57+
tenant-id: ${AB_AZURE_KEY_VAULT_TENANT_ID:}
58+
client-id: ${AB_AZURE_KEY_VAULT_CLIENT_ID:}
59+
client-secret: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET:}
60+
tags: ${AB_AZURE_KEY_VAULT_TAGS:}
5561
workload-api:
5662
base-path: ${WORKLOAD_API_HOST:}
5763
bearer-token: ${WORKLOAD_API_BEARER_TOKEN:}

airbyte-workload-launcher/src/main/kotlin/config/EnvVarConfigBeanFactory.kt

+17
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,9 @@ class EnvVarConfigBeanFactory {
400400
@Value("\${airbyte.secret.store.aws.region}") awsRegion: String,
401401
@Value("\${airbyte.secret.store.aws.kms-key-arn}") awsKmsKeyArn: String,
402402
@Value("\${airbyte.secret.store.aws.tags}") awsTags: String,
403+
@Value("\${airbyte.secret.store.azure.vault-url}") azureVaultUrl: String,
404+
@Value("\${airbyte.secret.store.azure.tenant-id}") azureTenantId: String,
405+
@Value("\${airbyte.secret.store.azure.tags}") azureTags: String,
403406
@Value("\${airbyte.secret.store.vault.address}") vaultAddress: String,
404407
@Value("\${airbyte.secret.store.vault.prefix}") vaultPrefix: String,
405408
): Map<String, String> {
@@ -409,6 +412,9 @@ class EnvVarConfigBeanFactory {
409412
put(EnvVarConstants.AWS_SECRET_MANAGER_REGION, awsRegion)
410413
put(EnvVarConstants.AWS_KMS_KEY_ARN, awsKmsKeyArn)
411414
put(EnvVarConstants.AWS_SECRET_MANAGER_SECRET_TAGS, awsTags)
415+
put(EnvVarConstants.AZURE_KEY_VAULT_VAULT_URL, azureVaultUrl)
416+
put(EnvVarConstants.AZURE_KEY_VAULT_TENANT_ID, azureTenantId)
417+
put(EnvVarConstants.AZURE_KEY_VAULT_SECRET_TAGS, azureTags)
412418
put(EnvVarConstants.VAULT_ADDRESS, vaultAddress)
413419
put(EnvVarConstants.VAULT_PREFIX, vaultPrefix)
414420
}
@@ -429,6 +435,10 @@ class EnvVarConfigBeanFactory {
429435
@Value("\${airbyte.secret.store.aws.access-key-ref-key}") awsAccessKeyRefKey: String,
430436
@Value("\${airbyte.secret.store.aws.secret-key-ref-name}") awsSecretKeyRefName: String,
431437
@Value("\${airbyte.secret.store.aws.secret-key-ref-key}") awsSecretKeyRefKey: String,
438+
@Value("\${airbyte.secret.store.azure.client-id-ref-name}") azureClientKeyRefName: String,
439+
@Value("\${airbyte.secret.store.azure.client-id-ref-key}") azureClientKeyRefKey: String,
440+
@Value("\${airbyte.secret.store.azure.client-secret-ref-name}") azureSecretKeyRefName: String,
441+
@Value("\${airbyte.secret.store.azure.client-secret-ref-key}") azureSecretKeyRefKey: String,
432442
@Value("\${airbyte.secret.store.vault.token-ref-name}") vaultTokenRefName: String,
433443
@Value("\${airbyte.secret.store.vault.token-ref-key}") vaultTokenRefKey: String,
434444
): Map<String, EnvVarSource> {
@@ -443,6 +453,13 @@ class EnvVarConfigBeanFactory {
443453
if (awsSecretKeyRefName.isNotBlank() && awsSecretKeyRefKey.isNotBlank()) {
444454
put(EnvVarConstants.AWS_SECRET_MANAGER_SECRET_ACCESS_KEY, createEnvVarSource(awsSecretKeyRefName, awsSecretKeyRefKey))
445455
}
456+
// Azure
457+
if (azureClientKeyRefName.isNotBlank() && azureClientKeyRefKey.isNotBlank()) {
458+
put(EnvVarConstants.AZURE_KEY_VAULT_CLIENT_ID, createEnvVarSource(azureClientKeyRefName, azureClientKeyRefKey))
459+
}
460+
if (azureSecretKeyRefName.isNotBlank() && azureSecretKeyRefKey.isNotBlank()) {
461+
put(EnvVarConstants.AZURE_KEY_VAULT_CLIENT_SECRET, createEnvVarSource(azureSecretKeyRefName, azureSecretKeyRefKey))
462+
}
446463
if (vaultTokenRefName.isNotBlank() && vaultTokenRefKey.isNotBlank()) {
447464
put(EnvVarConstants.VAULT_AUTH_TOKEN, createEnvVarSource(vaultTokenRefName, vaultTokenRefKey))
448465
}

airbyte-workload-launcher/src/main/kotlin/constants/EnvVarConstants.kt

+5
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ object EnvVarConstants {
3838
const val AWS_SECRET_MANAGER_REGION = "AWS_SECRET_MANAGER_REGION"
3939
const val AWS_KMS_KEY_ARN = "AWS_KMS_KEY_ARN"
4040
const val AWS_SECRET_MANAGER_SECRET_TAGS = "AWS_SECRET_MANAGER_SECRET_TAGS"
41+
const val AZURE_KEY_VAULT_VAULT_URL = "AB_AZURE_KEY_VAULT_VAULT_URL"
42+
const val AZURE_KEY_VAULT_TENANT_ID = "AB_AZURE_KEY_VAULT_TENANT_ID"
43+
const val AZURE_KEY_VAULT_SECRET_TAGS = "AB_AZURE_KEY_VAULT_TAGS"
4144
const val VAULT_ADDRESS = "VAULT_ADDRESS"
4245
const val VAULT_PREFIX = "VAULT_PREFIX"
4346
const val CONCURRENT_SOURCE_STREAM_READ_ENV_VAR = "CONCURRENT_SOURCE_STREAM_READ"
@@ -52,5 +55,7 @@ object EnvVarConstants {
5255
const val SECRET_STORE_GCP_CREDENTIALS = "SECRET_STORE_GCP_CREDENTIALS"
5356
const val AWS_SECRET_MANAGER_ACCESS_KEY_ID = "AWS_SECRET_MANAGER_ACCESS_KEY_ID"
5457
const val AWS_SECRET_MANAGER_SECRET_ACCESS_KEY = "AWS_SECRET_MANAGER_SECRET_ACCESS_KEY"
58+
const val AZURE_KEY_VAULT_CLIENT_ID = "AB_AZURE_KEY_VAULT_CLIENT_ID"
59+
const val AZURE_KEY_VAULT_CLIENT_SECRET = "AB_AZURE_KEY_VAULT_CLIENT_SECRET"
5560
const val VAULT_AUTH_TOKEN = "VAULT_AUTH_TOKEN"
5661
}

airbyte-workload-launcher/src/main/resources/application.yml

+13-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ airbyte:
7575
parallelism: ${WORKLOAD_LAUNCHER_PARALLELISM:10}
7676
workflow-parallelism: ${WORKLOAD_LAUNCHER_WORKFLOW_PARALLELISM:10}
7777
secret:
78-
persistence: ${SECRET_PERSISTENCE}
78+
persistence: ${SECRET_PERSISTENCE:TESTING_CONFIG_DB_TABLE}
7979
store:
8080
aws:
8181
region: ${AWS_SECRET_MANAGER_REGION:}
@@ -104,6 +104,18 @@ airbyte:
104104
# injects the name/key for secret reference
105105
token-ref-name: ${VAULT_AUTH_TOKEN_REF_NAME:}
106106
token-ref-key: ${VAULT_AUTH_TOKEN_REF_KEY:}
107+
azure:
108+
vault-url: ${AB_AZURE_KEY_VAULT_VAULT_URL:}
109+
tenant-id: ${AB_AZURE_KEY_VAULT_TENANT_ID:}
110+
tags: ${AB_AZURE_KEY_VAULT_TAGS:}
111+
# injects actual secret values
112+
client-id: ${AB_AZURE_KEY_VAULT_CLIENT_ID:}
113+
client-secret: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET:}
114+
# injects the name/key for secret reference
115+
client-id-ref-name: ${AB_AZURE_KEY_VAULT_CLIENT_ID_REF_NAME:}
116+
client-id-ref-key: ${AB_AZURE_KEY_VAULT_CLIENT_ID_REF_REF_KEY:}
117+
client-secret-ref-name: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_NAME:}
118+
client-secret-ref-key: ${AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_KEY:}
107119
version: ${AIRBYTE_VERSION}
108120
container.orchestrator:
109121
data-plane-creds:

charts/airbyte-server/templates/deployment.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,33 @@ spec:
269269
value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }}
270270
{{- end }}
271271

272+
# Values for Azure Key Vault
273+
{{- if (((.Values.global).secretsManager).azureKeyVault) }}
274+
275+
- name: AB_AZURE_KEY_VAULT_VAULT_URL
276+
value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }}
277+
278+
- name: AB_AZURE_KEY_VAULT_TENANT_ID
279+
value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }}
280+
281+
- name: AB_AZURE_KEY_VAULT_CLIENT_ID
282+
valueFrom:
283+
secretKeyRef:
284+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
285+
key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }}
286+
- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET
287+
valueFrom:
288+
secretKeyRef:
289+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
290+
key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }}
291+
292+
{{- end }}
293+
294+
{{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }}
295+
- name: AB_AZURE_KEY_VAULT_TAGS
296+
value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }}
297+
{{- end }}
298+
272299
# Values for googleSecretManager secrets
273300
{{- if (((.Values.global).secretsManager).googleSecretManager) }}
274301
- name: SECRET_STORE_GCP_PROJECT_ID

charts/airbyte-worker/templates/deployment.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,33 @@ spec:
325325
value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }}
326326
{{- end }}
327327

328+
# Values for Azure Key Vault
329+
{{- if (((.Values.global).secretsManager).azureKeyVault) }}
330+
331+
- name: AB_AZURE_KEY_VAULT_VAULT_URL
332+
value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }}
333+
334+
- name: AB_AZURE_KEY_VAULT_TENANT_ID
335+
value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }}
336+
337+
- name: AB_AZURE_KEY_VAULT_CLIENT_ID
338+
valueFrom:
339+
secretKeyRef:
340+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
341+
key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }}
342+
- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET
343+
valueFrom:
344+
secretKeyRef:
345+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
346+
key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }}
347+
348+
{{- end }}
349+
350+
{{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }}
351+
- name: AB_AZURE_KEY_VAULT_TAGS
352+
value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }}
353+
{{- end }}
354+
328355
# Values for googleSecretManager secrets
329356
{{- if (((.Values.global).secretsManager).googleSecretManager) }}
330357
- name: SECRET_STORE_GCP_PROJECT_ID

charts/airbyte-workload-launcher/templates/_helpers.tpl

+24
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,30 @@ Get awsSecretManager secret access key secret key or default
113113
{{- end -}}
114114
{{- end -}}
115115

116+
{{/*
117+
Get azureKeyVault clientId or default
118+
*/}}
119+
{{- define "airbyte.azureKeyVaultClientIdSecretKey" -}}
120+
{{- $azureKeyVaultClientIdSecretKey := . -}}
121+
{{- if $azureKeyVaultClientIdSecretKey -}}
122+
{{- printf "%s" $azureKeyVaultClientIdSecretKey -}}
123+
{{- else -}}
124+
{{- printf "azure-key-vault-client-id" -}}
125+
{{- end -}}
126+
{{- end -}}
127+
128+
{{/*
129+
Get azureKeyVault clientSecret or default
130+
*/}}
131+
{{- define "airbyte.azureKeyVaultClientSecretSecretKey" -}}
132+
{{- $azureKeyVaultClientSecretSecretKey := . -}}
133+
{{- if $azureKeyVaultClientSecretSecretKey -}}
134+
{{- printf "%s" $azureKeyVaultClientSecretSecretKey -}}
135+
{{- else -}}
136+
{{- printf "azure-key-vault-client-secret" -}}
137+
{{- end -}}
138+
{{- end -}}
139+
116140
{{/*
117141
Get googleSecretManager credentials secret key or default
118142
*/}}

charts/airbyte-workload-launcher/templates/deployment.yaml

+37
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,43 @@ spec:
367367
value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }}
368368
{{- end }}
369369

370+
# Values for Azure Key Vault
371+
{{- if (((.Values.global).secretsManager).azureKeyVault) }}
372+
373+
- name: AB_AZURE_KEY_VAULT_VAULT_URL
374+
value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }}
375+
376+
- name: AB_AZURE_KEY_VAULT_TENANT_ID
377+
value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }}
378+
379+
- name: AB_AZURE_KEY_VAULT_CLIENT_ID
380+
valueFrom:
381+
secretKeyRef:
382+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
383+
key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }}
384+
- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET
385+
valueFrom:
386+
secretKeyRef:
387+
name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
388+
key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }}
389+
390+
- name: AB_AZURE_KEY_VAULT_CLIENT_ID_REF_NAME
391+
value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
392+
- name: AB_AZURE_KEY_VAULT_CLIENT_ID_REF_REF_KEY
393+
value: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }}
394+
395+
- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_NAME
396+
value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }}
397+
- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_KEY
398+
value: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }}
399+
400+
{{- end }}
401+
402+
{{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }}
403+
- name: AB_AZURE_KEY_VAULT_TAGS
404+
value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }}
405+
{{- end }}
406+
370407
# Values for googleSecretManager secrets
371408
{{- if (((.Values.global).secretsManager).googleSecretManager) }}
372409
- name: SECRET_STORE_GCP_PROJECT_ID

0 commit comments

Comments
 (0)