🧱 private preview with Unity catalog and Oauth using service principal (#37613)

Author: gisripa

airbyte-integrations/connectors/destination-databricks/build.gradle

@@ -15,12 +15,11 @@
 plugins {
     id 'application'
     id 'airbyte-java-connector'
-    id "de.undercouch.download" version "5.0.1"
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.2.0'
-    features = ['db-destinations']
+    cdkVersionRequired = '0.35.6'
+    features = ['db-destinations', 's3-destinations', 'typing-deduping']
     useLocalCdk = false
 }
 
@@ -34,27 +33,25 @@ java {
 airbyteJavaConnector.addCdkDependencies()
 
 application {
-    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestination'
-    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+    mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestinationKt'
+    // Adding ZGC with Generational enabled from Java 21 hotspotVM. NativeMemory tracking if any
+    // memory leaks outside heap. Add string dedup which is supported now in ZGC from Java 18
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0',
+                                 '-XX:NativeMemoryTracking=detail', '-XX:+UseZGC', '-XX:+ZGenerational',
+                                 '-XX:+UseStringDeduplication']
 }
 
 dependencies {
-    implementation project(':airbyte-integrations:connectors:destination-s3')
-    implementation project(':airbyte-integrations:connectors:destination-azure-blob-storage')
-    implementation group: 'com.databricks', name: 'databricks-jdbc', version: '2.6.25'
-
-    // parquet
-    implementation ('org.apache.hadoop:hadoop-common:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
-    }
-    implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'}
-    implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'}
+    // Databricks JDBC driver
+    implementation 'com.databricks:databricks-jdbc:2.6.36'
+    implementation 'com.databricks:databricks-sdk-java:0.23.0'
+}
 
-    implementation 'com.azure:azure-storage-blob:12.18.0'
+test {
+    // Required for Databricks JDBC. There is a pending bug to fix Apache Arrow.
+    jvmArgs = ["-Dio.netty.tryReflectionSetAccessible=true",
+               "--add-opens=java.base/java.nio=ALL-UNNAMED",
+               "--add-opens=jdk.unsupported/sun.misc=ALL-UNNAMED",
+               "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
+               "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]
 }

airbyte-integrations/connectors/destination-databricks/gradle.properties

@@ -0,0 +1,3 @@
+testExecutionConcurrency=-1
+# minimum 3 minutes timeout required during parallel workload on our small warehouse
+JunitMethodExecutionTimeout=3 m

airbyte-integrations/connectors/destination-databricks/metadata.yaml

@@ -2,11 +2,11 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 072d5540-f236-4294-ba7c-ade8fd918496
-  dockerImageTag: 1.1.2
+  dockerImageTag: 2.0.0
   dockerRepository: airbyte/destination-databricks
   githubIssueLabel: destination-databricks
   icon: databricks.svg
-  license: MIT
+  license: ELv2
   name: Databricks Lakehouse
   registries:
     cloud:
@@ -16,6 +16,16 @@ data:
       enabled: true
       dockerImageTag: 1.1.0 # pinning due to CDK incompatibility, see https://github.com/airbytehq/alpha-beta-issues/issues/2596
   releaseStage: alpha
+  releases:
+    breakingChanges:
+      2.0.0:
+        message: >
+          **This is a private preview version, Do not upgrade until you review the changes**.\n
+          This version is a rewrite of the community connector with support for Unity catalog, and staging files using Unity catalog volumes.
+          This version also introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), 
+          which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. 
+          To review the breaking changes, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading).
+        upgradeDeadline: "2024-12-31"
   documentationUrl: https://docs.airbyte.com/integrations/destinations/databricks
   tags:
     - language:java