🧱 private preview with Unity catalog and Oauth using service principal

airbyte-integrations/connectors/destination-databricks/build.gradle

@@ -15,12 +15,11 @@
 plugins {
     id 'application'
     id 'airbyte-java-connector'
-    id "de.undercouch.download" version "5.0.1"
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.2.0'
-    features = ['db-destinations']
+    cdkVersionRequired = '0.34.3'
+    features = ['db-destinations', 's3-destinations', 'typing-deduping']
     useLocalCdk = false
 }
 
@@ -34,27 +33,25 @@ java {
 airbyteJavaConnector.addCdkDependencies()
 
 application {
-    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestination'
-    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestinationKt'
+    // Adding ZGC with Generational enabled from Java 21 hotspotVM. NativeMemory tracking if any
+    // memory leaks outside heap. Add string dedup which is supported now in ZGC from Java 18
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0',
+                                 '-XX:NativeMemoryTracking=detail', '-XX:+UseZGC', '-XX:+ZGenerational',
+                                 '-XX:+UseStringDeduplication']
 }
 
 dependencies {
-    implementation project(':airbyte-integrations:connectors:destination-s3')
-    implementation project(':airbyte-integrations:connectors:destination-azure-blob-storage')
-    implementation group: 'com.databricks', name: 'databricks-jdbc', version: '2.6.25'
-
-    // parquet
-    implementation ('org.apache.hadoop:hadoop-common:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'}
+    // Databricks JDBC driver
+    implementation 'com.databricks:databricks-jdbc:2.6.36'
+    implementation 'com.databricks:databricks-sdk-java:0.23.0'
+}
 
-    implementation 'com.azure:azure-storage-blob:12.18.0'
+test {
+    // Required for Databricks JDBC. There is a pending bug to fix Apache Arrow.
+    jvmArgs = ["-Dio.netty.tryReflectionSetAccessible=true",
+               "--add-opens=java.base/java.nio=ALL-UNNAMED",
+               "--add-opens=jdk.unsupported/sun.misc=ALL-UNNAMED",
+               "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
+               "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]
 }

airbyte-integrations/connectors/destination-databricks/gradle.properties

@@ -0,0 +1,3 @@
+testExecutionConcurrency=-1
+# minimum 3 minutes timeout required during parallel workload on our small warehouse
+JunitMethodExecutionTimeout=3 m

airbyte-integrations/connectors/destination-databricks/metadata.yaml

@@ -2,11 +2,11 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 072d5540-f236-4294-ba7c-ade8fd918496
-  dockerImageTag: 1.1.2
+  dockerImageTag: 1.1.3
   dockerRepository: airbyte/destination-databricks
   githubIssueLabel: destination-databricks
   icon: databricks.svg
-  license: MIT
+  license: ELv2
   name: Databricks Lakehouse
   registries:
     cloud:
@@ -23,4 +23,5 @@ data:
     sl: 100
     ql: 100
   supportLevel: community
+  supportsDbt: true
 metadataSpecVersion: "1.0"