Skip to content

Commit 34f58fa

Browse files
edgaoedgao
authored
Destination databricks: Remove user/pass auth (#40712)
1 parent adc5f19 commit 34f58fa

File tree

8 files changed

+118
-47
lines changed

8 files changed

+118
-47
lines changed

‎airbyte-integrations/connectors/destination-databricks/metadata.yaml

+6-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 072d5540-f236-4294-ba7c-ade8fd918496
5-
dockerImageTag: 3.1.0
5+
dockerImageTag: 3.2.0
66
dockerRepository: airbyte/destination-databricks
77
githubIssueLabel: destination-databricks
88
icon: databricks.svg
@@ -61,4 +61,9 @@ data:
6161
secretStore:
6262
type: GSM
6363
alias: airbyte-connector-testing-secret-store
64+
- name: SECRET_DESTINATION_DATABRICKS_PERSONAL_ACCESS_TOKEN_CONFIG
65+
fileName: pat_config.json
66+
secretStore:
67+
type: GSM
68+
alias: airbyte-connector-testing-secret-store
6469
metadataSpecVersion: "1.0"

‎airbyte-integrations/connectors/destination-databricks/src/main/kotlin/io/airbyte/integrations/destination/databricks/DatabricksConnectorClientsFactory.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ object DatabricksConnectorClientsFactory {
4848
"jdbc:databricks://${config.hostname}:${config.port}/${config.database};transportMode=http;httpPath=${config.httpPath};EnableArrow=0"
4949
when (config.authentication) {
5050
is BasicAuthentication -> {
51-
datasource.userID = config.authentication.username
52-
datasource.password = config.authentication.password
53-
datasource.setURL("$jdbcUrl;AuthMech=3")
51+
datasource.setURL(
52+
"$jdbcUrl;AuthMech=3;UID=token;PWD=${config.authentication.personalAccessToken}"
53+
)
5454
}
5555
is OAuth2Authentication -> {
5656
datasource.setURL(

‎airbyte-integrations/connectors/destination-databricks/src/main/kotlin/io/airbyte/integrations/destination/databricks/model/DatabricksConnectorConfig.kt

-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@ data class DatabricksConnectorConfig(
3636
@JsonTypeName("BASIC")
3737
data class BasicAuthentication(
3838
@JsonProperty("personal_access_token") val personalAccessToken: String,
39-
val username: String,
40-
val password: String
4139
) : Authentication
4240

4341
@JsonTypeName("OAUTH")

‎airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json

+11-25
Original file line numberDiff line numberDiff line change
@@ -67,52 +67,38 @@
6767
"order": 8,
6868
"oneOf": [
6969
{
70-
"title": "Basic Authentication",
71-
"required": [
72-
"auth_type",
73-
"personal_access_token",
74-
"username",
75-
"password"
76-
],
70+
"title": "OAuth2 (Recommended)",
71+
"required": ["auth_type", "client_id", "secret"],
7772
"properties": {
7873
"auth_type": {
7974
"type": "string",
80-
"const": "BASIC",
75+
"const": "OAUTH",
8176
"order": 0
8277
},
83-
"personal_access_token": {
78+
"client_id": {
8479
"type": "string",
85-
"airbyte_secret": true,
8680
"order": 1
8781
},
88-
"username": {
89-
"type": "string",
90-
"order": 2
91-
},
92-
"password": {
82+
"secret": {
9383
"type": "string",
9484
"airbyte_secret": true,
95-
"order": 3
85+
"order": 2
9686
}
9787
}
9888
},
9989
{
100-
"title": "OAuth2 (Recommended)",
101-
"required": ["auth_type", "client_id", "secret"],
90+
"title": "Personal Access Token",
91+
"required": ["auth_type", "personal_access_token"],
10292
"properties": {
10393
"auth_type": {
10494
"type": "string",
105-
"const": "OAUTH",
95+
"const": "BASIC",
10696
"order": 0
10797
},
108-
"client_id": {
109-
"type": "string",
110-
"order": 1
111-
},
112-
"secret": {
98+
"personal_access_token": {
11399
"type": "string",
114100
"airbyte_secret": true,
115-
"order": 2
101+
"order": 1
116102
}
117103
}
118104
}

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksTypingDedupingTest.kt ‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/AbstractDatabricksTypingDedupingTest.kt

+33-16
Original file line numberDiff line numberDiff line change
@@ -5,49 +5,66 @@
55
package io.airbyte.integrations.destination.databricks.typededupe
66

77
import com.fasterxml.jackson.databind.JsonNode
8+
import com.fasterxml.jackson.databind.node.ObjectNode
89
import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase
910
import io.airbyte.cdk.db.jdbc.JdbcDatabase
1011
import io.airbyte.cdk.db.jdbc.JdbcSourceOperations
1112
import io.airbyte.cdk.integrations.base.JavaBaseConstants
13+
import io.airbyte.commons.io.IOs
14+
import io.airbyte.commons.json.Jsons
1215
import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest
1316
import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator
1417
import io.airbyte.integrations.base.destination.typing_deduping.StreamId
1518
import io.airbyte.integrations.destination.databricks.DatabricksConnectorClientsFactory
16-
import io.airbyte.integrations.destination.databricks.DatabricksIntegrationTestUtils
1719
import io.airbyte.integrations.destination.databricks.jdbc.DatabricksNamingTransformer
1820
import io.airbyte.integrations.destination.databricks.jdbc.DatabricksSqlGenerator
1921
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
22+
import java.nio.file.Path
2023
import java.sql.Connection
2124
import java.sql.ResultSet
22-
import java.util.concurrent.TimeUnit
23-
import org.junit.jupiter.api.BeforeAll
24-
import org.junit.jupiter.api.Timeout
25-
import org.mockito.Mockito
25+
import java.util.Locale
26+
import org.apache.commons.lang3.RandomStringUtils
2627

27-
class DatabricksTypingDedupingTest : BaseTypingDedupingTest() {
28+
abstract class AbstractDatabricksTypingDedupingTest(
29+
private val jdbcDatabase: JdbcDatabase,
30+
private val jsonConfig: JsonNode,
31+
private val connectorConfig: DatabricksConnectorConfig,
32+
) : BaseTypingDedupingTest() {
2833
override val imageName: String
2934
get() = "airbyte/destination-databricks:dev"
3035

3136
companion object {
32-
private var jdbcDatabase: JdbcDatabase = Mockito.mock()
33-
private var connectorConfig: DatabricksConnectorConfig = Mockito.mock()
34-
@JvmStatic
35-
@BeforeAll
36-
@Timeout(value = 10, unit = TimeUnit.MINUTES)
37-
fun setupDatabase() {
38-
connectorConfig = DatabricksIntegrationTestUtils.oauthConfig
39-
jdbcDatabase =
37+
fun setupDatabase(
38+
connectorConfigPath: String
39+
): Triple<JdbcDatabase, JsonNode, DatabricksConnectorConfig> {
40+
var jsonConfig = Jsons.deserialize(IOs.readFile(Path.of(connectorConfigPath)))
41+
42+
// Randomize the default namespace to avoid collisions between
43+
// concurrent test runs.
44+
// Technically, we should probably do this in `generateConfig`,
45+
// because there could be concurrent test runs within a single class,
46+
// but we currently only have a single test that uses the default
47+
// namespace anyway.
48+
val uniqueSuffix = RandomStringUtils.randomAlphabetic(10).lowercase(Locale.getDefault())
49+
val defaultSchema = "typing_deduping_default_schema_$uniqueSuffix"
50+
val connectorConfig =
51+
DatabricksConnectorConfig.deserialize(jsonConfig).copy(schema = defaultSchema)
52+
(jsonConfig as ObjectNode).put("schema", defaultSchema)
53+
54+
val jdbcDatabase =
4055
DefaultJdbcDatabase(
4156
DatabricksConnectorClientsFactory.createDataSource(connectorConfig)
4257
)
4358
// This will trigger warehouse start
4459
jdbcDatabase.execute("SELECT 1")
60+
61+
return Triple(jdbcDatabase, jsonConfig, connectorConfig)
4562
}
4663
}
4764

4865
override fun generateConfig(): JsonNode {
4966
// This method is called in BeforeEach so setup any other references needed per test
50-
return DatabricksIntegrationTestUtils.oauthConfigJson.deepCopy()
67+
return jsonConfig.deepCopy()
5168
}
5269

5370
private fun rawTableIdentifier(
@@ -80,7 +97,7 @@ class DatabricksTypingDedupingTest : BaseTypingDedupingTest() {
8097
.executeQuery(
8198
"""
8299
SELECT *
83-
FROM `${connectorConfig.database}`.$tableIdentifier
100+
FROM `${connectorConfig.database}`.$tableIdentifier
84101
ORDER BY ${JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT} ASC
85102
""".trimIndent(),
86103
)

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksOauthTypingDedupingTest.kt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.databricks.typededupe
6+
7+
import com.fasterxml.jackson.databind.JsonNode
8+
import io.airbyte.cdk.db.jdbc.JdbcDatabase
9+
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
10+
import java.util.concurrent.TimeUnit
11+
import org.junit.jupiter.api.BeforeAll
12+
import org.junit.jupiter.api.Timeout
13+
14+
class DatabricksOauthTypingDedupingTest :
15+
AbstractDatabricksTypingDedupingTest(jdbcDatabase, jsonConfig, connectorConfig) {
16+
companion object {
17+
private lateinit var jdbcDatabase: JdbcDatabase
18+
private lateinit var jsonConfig: JsonNode
19+
private lateinit var connectorConfig: DatabricksConnectorConfig
20+
21+
@JvmStatic
22+
@BeforeAll
23+
@Timeout(value = 10, unit = TimeUnit.MINUTES)
24+
fun setupDatabase() {
25+
val (jdbcDatabase, jsonConfig, connectorConfig) =
26+
setupDatabase("secrets/oauth_config.json")
27+
this.jdbcDatabase = jdbcDatabase
28+
this.jsonConfig = jsonConfig
29+
this.connectorConfig = connectorConfig
30+
}
31+
}
32+
}

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksPersonalAccessTokenTypingDedupingTest.kt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.databricks.typededupe
6+
7+
import com.fasterxml.jackson.databind.JsonNode
8+
import io.airbyte.cdk.db.jdbc.JdbcDatabase
9+
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
10+
import java.util.concurrent.TimeUnit
11+
import org.junit.jupiter.api.BeforeAll
12+
import org.junit.jupiter.api.Timeout
13+
14+
class DatabricksPersonalAccessTokenTypingDedupingTest :
15+
AbstractDatabricksTypingDedupingTest(jdbcDatabase, jsonConfig, connectorConfig) {
16+
companion object {
17+
private lateinit var jdbcDatabase: JdbcDatabase
18+
private lateinit var jsonConfig: JsonNode
19+
private lateinit var connectorConfig: DatabricksConnectorConfig
20+
21+
@JvmStatic
22+
@BeforeAll
23+
@Timeout(value = 10, unit = TimeUnit.MINUTES)
24+
fun setupDatabase() {
25+
val (jdbcDatabase, jsonConfig, connectorConfig) =
26+
setupDatabase("secrets/pat_config.json")
27+
this.jdbcDatabase = jdbcDatabase
28+
this.jsonConfig = jsonConfig
29+
this.connectorConfig = connectorConfig
30+
}
31+
}
32+
}

‎docs/integrations/destinations/databricks.md

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ with the raw tables, and their format is subject to change without notice.
8181

8282
| Version | Date | Pull Request | Subject |
8383
|:--------|:-----------|:--------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
84+
| 3.2.0 | 2024-08-12 | [#40712](https://github.com/airbytehq/airbyte/pull/40712) | Rely solely on PAT, instead of also needing a user/pass |
8485
| 3.1.0 | 2024-07-22 | [#40692](https://github.com/airbytehq/airbyte/pull/40692) | Support for [refreshes](../../operator-guides/refreshes.md) and resumable full refresh. WARNING: You must upgrade to platform 0.63.7 before upgrading to this connector version. |
8586
| 3.0.0 | 2024-07-12 | [#40689](https://github.com/airbytehq/airbyte/pull/40689) | (Private release, not to be used for production) Add `_airbyte_generation_id` column, and `sync_id` entry in `_airbyte_meta` |
8687
| 2.0.0 | 2024-05-17 | [#37613](https://github.com/airbytehq/airbyte/pull/37613) | (Private release, not to be used for production) Alpha release of the connector to use Unity Catalog |

0 commit comments

Comments
 (0)