🧱 private preview with Unity catalog and Oauth using service principal

airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle

@@ -15,6 +15,7 @@ dependencies {
     api 'com.fasterxml.jackson.core:jackson-databind'
     api 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
+    api 'com.fasterxml.jackson.module:jackson-module-kotlin'
     api 'com.google.guava:guava:33.0.0-jre'
     api 'commons-io:commons-io:2.15.1'
     api ('io.airbyte.airbyte-protocol:protocol-models:0.7.0') { exclude group: 'com.google.api-client', module: 'google-api-client' }

airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.kt

@@ -1544,7 +1544,7 @@ abstract class BaseSqlGeneratorIntegrationTest<DestinationState : MinimumDestina
 
     @Test
     @Throws(Exception::class)
-    fun testV1V2migration() {
+    open fun testV1V2migration() {
         // This is maybe a little hacky, but it avoids having to refactor this entire class and
         // subclasses
         // for something that is going away

airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.kt

@@ -10,6 +10,7 @@ import com.google.common.collect.Streams
 import io.airbyte.commons.json.Jsons
 import java.math.BigDecimal
 import java.time.*
+import java.time.format.DateTimeParseException
 import java.util.*
 import java.util.function.Function
 import java.util.stream.Collectors
@@ -440,7 +441,16 @@ constructor(
                 Instant.ofEpochMilli(Long.MIN_VALUE)
             } else {
                 try {
-                    Instant.parse(node.asText())
+                    OffsetDateTime.parse(node.asText()).toInstant()
+                } catch (parseE: DateTimeParseException) {
+                    // Fallback to using LocalDateTime and try again
+                    // Some databases have Timestamp_TZ mapped to TIMESTAMP with no offset,
+                    // this is sketchy to assume it as always UTC
+                    try {
+                        LocalDateTime.parse(node.asText()).toInstant(ZoneOffset.UTC)
+                    } catch (e: Exception) {
+                        Instant.ofEpochMilli(Long.MIN_VALUE)
+                    }
                 } catch (e: Exception) {
                     Instant.ofEpochMilli(Long.MIN_VALUE)
                 }

airbyte-integrations/connectors/destination-databricks/build.gradle

@@ -15,13 +15,12 @@
 plugins {
     id 'application'
     id 'airbyte-java-connector'
-    id "de.undercouch.download" version "5.0.1"
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.2.0'
-    features = ['db-destinations']
-    useLocalCdk = false
+    cdkVersionRequired = '0.33.2'
+    features = ['db-destinations', 's3-destinations', 'typing-deduping']
+    useLocalCdk = true
 }
 
 //remove once upgrading the CDK version to 0.4.x or later
@@ -34,27 +33,24 @@ java {
 airbyteJavaConnector.addCdkDependencies()
 
 application {
-    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestination'
-    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestinationKt'
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0',
+                                 '-XX:NativeMemoryTracking=detail', '-XX:+UnlockDiagnosticVMOptions',
+                                 '-XX:GCLockerRetryAllocationCount=100', '-XX:+UseZGC', '-XX:+ZGenerational']
 }
 
 dependencies {
-    implementation project(':airbyte-integrations:connectors:destination-s3')
-    implementation project(':airbyte-integrations:connectors:destination-azure-blob-storage')
-    implementation group: 'com.databricks', name: 'databricks-jdbc', version: '2.6.25'
-
-    // parquet
-    implementation ('org.apache.hadoop:hadoop-common:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'}
+    // Databricks JDBC driver
+    implementation 'com.databricks:databricks-jdbc:2.6.36'
+    implementation 'com.databricks:databricks-sdk-java:0.23.0'
+
+}
 
-    implementation 'com.azure:azure-storage-blob:12.18.0'
+test {
+    // Required for Databricks JDBC. There is a pending bug to fix Apache Arrow.
+    jvmArgs = ["-Dio.netty.tryReflectionSetAccessible=true",
+               "--add-opens=java.base/java.nio=ALL-UNNAMED",
+               "--add-opens=jdk.unsupported/sun.misc=ALL-UNNAMED",
+               "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
+               "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]
 }

airbyte-integrations/connectors/destination-databricks/gradle.properties

@@ -0,0 +1,3 @@
+testExecutionConcurrency=-1
+# minimum 3 minutes timeout required during parallel workload on our small warehouse
+JunitMethodExecutionTimeout=3 m

airbyte-integrations/connectors/destination-databricks/metadata.yaml

@@ -2,11 +2,11 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 072d5540-f236-4294-ba7c-ade8fd918496
-  dockerImageTag: 1.1.2
+  dockerImageTag: 1.1.3
   dockerRepository: airbyte/destination-databricks
   githubIssueLabel: destination-databricks
   icon: databricks.svg
-  license: MIT
+  license: ELv2
   name: Databricks Lakehouse
   registries:
     cloud:
@@ -23,4 +23,5 @@ data:
     sl: 100
     ql: 100
   supportLevel: community
+  supportsDbt: true
 metadataSpecVersion: "1.0"