Skip to content

Commit 5f06e35

Browse files
edgaoedgao
committed
delete password auth
1 parent 8ab89d3 commit 5f06e35

File tree

8 files changed

+93
-45
lines changed

8 files changed

+93
-45
lines changed

‎airbyte-integrations/connectors/destination-databricks/metadata.yaml

+6-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 072d5540-f236-4294-ba7c-ade8fd918496
5-
dockerImageTag: 3.1.0
5+
dockerImageTag: 3.2.0
66
dockerRepository: airbyte/destination-databricks
77
githubIssueLabel: destination-databricks
88
icon: databricks.svg
@@ -61,4 +61,9 @@ data:
6161
secretStore:
6262
type: GSM
6363
alias: airbyte-connector-testing-secret-store
64+
- name: SECRET_DESTINATION_DATABRICKS_PERSONAL_ACCESS_TOKEN_CONFIG
65+
fileName: pat_config.json
66+
secretStore:
67+
type: GSM
68+
alias: airbyte-connector-testing-secret-store
6469
metadataSpecVersion: "1.0"

‎airbyte-integrations/connectors/destination-databricks/src/main/kotlin/io/airbyte/integrations/destination/databricks/DatabricksConnectorClientsFactory.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ object DatabricksConnectorClientsFactory {
4848
"jdbc:databricks://${config.hostname}:${config.port}/${config.database};transportMode=http;httpPath=${config.httpPath};EnableArrow=0"
4949
when (config.authentication) {
5050
is BasicAuthentication -> {
51-
datasource.userID = config.authentication.username
52-
datasource.password = config.authentication.password
53-
datasource.setURL("$jdbcUrl;AuthMech=3")
51+
datasource.setURL(
52+
"$jdbcUrl;AuthMech=3;UID=token;PWD=${config.authentication.personalAccessToken}"
53+
)
5454
}
5555
is OAuth2Authentication -> {
5656
datasource.setURL(

‎airbyte-integrations/connectors/destination-databricks/src/main/kotlin/io/airbyte/integrations/destination/databricks/model/DatabricksConnectorConfig.kt

-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@ data class DatabricksConnectorConfig(
3636
@JsonTypeName("BASIC")
3737
data class BasicAuthentication(
3838
@JsonProperty("personal_access_token") val personalAccessToken: String,
39-
val username: String,
40-
val password: String
4139
) : Authentication
4240

4341
@JsonTypeName("OAUTH")

‎airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json

+11-25
Original file line numberDiff line numberDiff line change
@@ -67,52 +67,38 @@
6767
"order": 8,
6868
"oneOf": [
6969
{
70-
"title": "Basic Authentication",
71-
"required": [
72-
"auth_type",
73-
"personal_access_token",
74-
"username",
75-
"password"
76-
],
70+
"title": "OAuth2 (Recommended)",
71+
"required": ["auth_type", "client_id", "secret"],
7772
"properties": {
7873
"auth_type": {
7974
"type": "string",
80-
"const": "BASIC",
75+
"const": "OAUTH",
8176
"order": 0
8277
},
83-
"personal_access_token": {
78+
"client_id": {
8479
"type": "string",
85-
"airbyte_secret": true,
8680
"order": 1
8781
},
88-
"username": {
89-
"type": "string",
90-
"order": 2
91-
},
92-
"password": {
82+
"secret": {
9383
"type": "string",
9484
"airbyte_secret": true,
95-
"order": 3
85+
"order": 2
9686
}
9787
}
9888
},
9989
{
100-
"title": "OAuth2 (Recommended)",
101-
"required": ["auth_type", "client_id", "secret"],
90+
"title": "Personal Access Token",
91+
"required": ["auth_type", "personal_access_token"],
10292
"properties": {
10393
"auth_type": {
10494
"type": "string",
105-
"const": "OAUTH",
95+
"const": "BASIC",
10696
"order": 0
10797
},
108-
"client_id": {
109-
"type": "string",
110-
"order": 1
111-
},
112-
"secret": {
98+
"personal_access_token": {
11399
"type": "string",
114100
"airbyte_secret": true,
115-
"order": 2
101+
"order": 1
116102
}
117103
}
118104
}

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksTypingDedupingTest.kt renamed to ‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/AbstractDatabricksTypingDedupingTest.kt

+16-14
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase
99
import io.airbyte.cdk.db.jdbc.JdbcDatabase
1010
import io.airbyte.cdk.db.jdbc.JdbcSourceOperations
1111
import io.airbyte.cdk.integrations.base.JavaBaseConstants
12+
import io.airbyte.commons.io.IOs
13+
import io.airbyte.commons.json.Jsons
1214
import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest
1315
import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator
1416
import io.airbyte.integrations.base.destination.typing_deduping.StreamId
@@ -17,31 +19,31 @@ import io.airbyte.integrations.destination.databricks.DatabricksIntegrationTestU
1719
import io.airbyte.integrations.destination.databricks.jdbc.DatabricksNamingTransformer
1820
import io.airbyte.integrations.destination.databricks.jdbc.DatabricksSqlGenerator
1921
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
22+
import java.nio.file.Path
2023
import java.sql.Connection
2124
import java.sql.ResultSet
22-
import java.util.concurrent.TimeUnit
23-
import org.junit.jupiter.api.BeforeAll
24-
import org.junit.jupiter.api.Timeout
25-
import org.mockito.Mockito
2625

27-
class DatabricksTypingDedupingTest : BaseTypingDedupingTest() {
26+
abstract class AbstractDatabricksTypingDedupingTest(
27+
private val jdbcDatabase: JdbcDatabase,
28+
private val connectorConfig: DatabricksConnectorConfig,
29+
) : BaseTypingDedupingTest() {
2830
override val imageName: String
2931
get() = "airbyte/destination-databricks:dev"
3032

3133
companion object {
32-
private var jdbcDatabase: JdbcDatabase = Mockito.mock()
33-
private var connectorConfig: DatabricksConnectorConfig = Mockito.mock()
34-
@JvmStatic
35-
@BeforeAll
36-
@Timeout(value = 10, unit = TimeUnit.MINUTES)
37-
fun setupDatabase() {
38-
connectorConfig = DatabricksIntegrationTestUtils.oauthConfig
39-
jdbcDatabase =
34+
fun setupDatabase(
35+
connectorConfigPath: String
36+
): Pair<JdbcDatabase, DatabricksConnectorConfig> {
37+
val config = Jsons.deserialize(IOs.readFile(Path.of(connectorConfigPath)))
38+
val connectorConfig = DatabricksConnectorConfig.deserialize(config)
39+
val jdbcDatabase =
4040
DefaultJdbcDatabase(
4141
DatabricksConnectorClientsFactory.createDataSource(connectorConfig)
4242
)
4343
// This will trigger warehouse start
4444
jdbcDatabase.execute("SELECT 1")
45+
46+
return Pair(jdbcDatabase, connectorConfig)
4547
}
4648
}
4749

@@ -80,7 +82,7 @@ class DatabricksTypingDedupingTest : BaseTypingDedupingTest() {
8082
.executeQuery(
8183
"""
8284
SELECT *
83-
FROM `${connectorConfig.database}`.$tableIdentifier
85+
FROM `${connectorConfig.database}`.$tableIdentifier
8486
ORDER BY ${JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT} ASC
8587
""".trimIndent(),
8688
)

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksOauthTypingDedupingTest.kt

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.databricks.typededupe
6+
7+
import io.airbyte.cdk.db.jdbc.JdbcDatabase
8+
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
9+
import java.util.concurrent.TimeUnit
10+
import org.junit.jupiter.api.BeforeAll
11+
import org.junit.jupiter.api.Timeout
12+
13+
class DatabricksOauthTypingDedupingTest :
14+
AbstractDatabricksTypingDedupingTest(jdbcDatabase, connectorConfig) {
15+
companion object {
16+
private lateinit var jdbcDatabase: JdbcDatabase
17+
private lateinit var connectorConfig: DatabricksConnectorConfig
18+
19+
@JvmStatic
20+
@BeforeAll
21+
@Timeout(value = 10, unit = TimeUnit.MINUTES)
22+
fun setupDatabase() {
23+
val (jdbcDatabase, connectorConfig) = setupDatabase("secrets/oauth_config.json")
24+
this.jdbcDatabase = jdbcDatabase
25+
this.connectorConfig = connectorConfig
26+
}
27+
}
28+
}

‎airbyte-integrations/connectors/destination-databricks/src/test-integration/kotlin/io/airbyte/integrations/destination/databricks/typededupe/DatabricksPersonalAccessTokenTypingDedupingTest.kt

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.destination.databricks.typededupe
6+
7+
import io.airbyte.cdk.db.jdbc.JdbcDatabase
8+
import io.airbyte.integrations.destination.databricks.model.DatabricksConnectorConfig
9+
import java.util.concurrent.TimeUnit
10+
import org.junit.jupiter.api.BeforeAll
11+
import org.junit.jupiter.api.Timeout
12+
13+
class DatabricksPersonalAccessTokenTypingDedupingTest :
14+
AbstractDatabricksTypingDedupingTest(jdbcDatabase, connectorConfig) {
15+
companion object {
16+
private lateinit var jdbcDatabase: JdbcDatabase
17+
private lateinit var connectorConfig: DatabricksConnectorConfig
18+
19+
@JvmStatic
20+
@BeforeAll
21+
@Timeout(value = 10, unit = TimeUnit.MINUTES)
22+
fun setupDatabase() {
23+
val (jdbcDatabase, connectorConfig) = setupDatabase("secrets/pat_config.json")
24+
this.jdbcDatabase = jdbcDatabase
25+
this.connectorConfig = connectorConfig
26+
}
27+
}
28+
}

‎docs/integrations/destinations/databricks.md

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ with the raw tables, and their format is subject to change without notice.
8181

8282
| Version | Date | Pull Request | Subject |
8383
|:--------|:-----------|:--------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
84+
| 3.2.0 | 2024-08-12 | [#40712](https://github.com/airbytehq/airbyte/pull/40712) | Rely solely on PAT, instead of also needing a user/pass |
8485
| 3.1.0 | 2024-07-22 | [#40692](https://github.com/airbytehq/airbyte/pull/40692) | Support for [refreshes](../../operator-guides/refreshes.md) and resumable full refresh. WARNING: You must upgrade to platform 0.63.7 before upgrading to this connector version. |
8586
| 3.0.0 | 2024-07-12 | [#40689](https://github.com/airbytehq/airbyte/pull/40689) | (Private release, not to be used for production) Add `_airbyte_generation_id` column, and `sync_id` entry in `_airbyte_meta` |
8687
| 2.0.0 | 2024-05-17 | [#37613](https://github.com/airbytehq/airbyte/pull/37613) | (Private release, not to be used for production) Alpha release of the connector to use Unity Catalog |

0 commit comments

Comments
 (0)