chore: modify tool-versions to align with requirements

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

.tool-versions

@@ -1,5 +1,5 @@
-java corretto-17.0.13.11.1
+java corretto-11.0.25.9.1
 scala 2.12.20
 asdf-plugin-manager 1.4.0
-sbt 1.10.5
+sbt 1.8.2
 python 3.7.17

build.sbt

@@ -168,6 +168,10 @@ def cleanSparkMeta(): Unit = {
 
 val sparkBaseSettings: Seq[Setting[_]] = Seq(
   assembly / test := {},
+  assembly / assemblyShadeRules := Seq(
+    ShadeRule.rename("org.typelevel.cats.**" -> "repackaged.org.typelevel.cats.@1").inAll,
+    ShadeRule.rename("cats.**" -> "repackaged.cats.@1").inAll
+  ),
   assembly / artifact := {
     val art = (assembly / artifact).value
     art.withClassifier(Some("assembly"))
@@ -176,8 +180,9 @@ val sparkBaseSettings: Seq[Setting[_]] = Seq(
   cleanFiles ++= Seq(file(tmp_warehouse)),
   Test / testOptions += Tests.Setup(() => cleanSparkMeta()),
   // compatibility for m1 chip laptop
-  libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.10.4" % Test
-) ++ addArtifact(assembly / artifact, assembly)
+  libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.10.4" % Test,
+  dependencyOverrides += "com.eed3si9n.jarjar" % "jarjar" % "1.14.0"
+) ++ addArtifact(assembly / artifact, assembly) ++ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.0")
 
 lazy val spark = project
   .dependsOn(aggregator.%("compile->compile;test->test"), online)
@@ -211,13 +216,13 @@ lazy val cloud_gcp = project
     libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.42.0",
     libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.41.0",
     libraryDependencies += "com.google.cloud" % "google-cloud-pubsub" % "1.131.0",
-    libraryDependencies += "com.google.cloud" % "google-cloud-dataproc" % "4.51.0",
-    libraryDependencies += "com.google.cloud.bigdataoss" % "gcs-connector" % "3.0.3", // it's what's on the cluster
+    libraryDependencies += "com.google.cloud" % "google-cloud-dataproc" % "4.52.0",
     libraryDependencies += "com.google.cloud.bigdataoss" % "gcs-connector" % "hadoop3-2.2.26",
-    libraryDependencies += "com.google.cloud.bigdataoss" % "gcsio" % "3.0.3", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystem.java
-    libraryDependencies += "io.circe" %% "circe-yaml" % "1.15.0",
-    libraryDependencies += "com.google.cloud.spark" %% s"spark-bigquery-with-dependencies" % "0.41.0",
-    libraryDependencies += "com.google.cloud.spark.bigtable" %% "spark-bigtable" % "0.2.1",
+    libraryDependencies += "com.google.cloud.bigdataoss" % "gcsio" % "2.2.26", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystem.java
+    libraryDependencies += "com.google.cloud.bigdataoss" % "util-hadoop" % "hadoop3-2.2.26", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/util-hadoop/src/main/java/com/google/cloud/hadoop/util/HadoopConfigurationProperty.java
+    libraryDependencies += "io.circe" %% "circe-yaml" % "1.15.0", // has an issue: https://github.com/typelevel/cats/issues/3628
+    libraryDependencies += "com.google.cloud.spark" %% "spark-bigquery-with-dependencies" % "0.41.0",
+    // libraryDependencies += "com.google.cloud.spark.bigtable" %% "spark-bigtable" % "0.2.1",
     libraryDependencies += "com.google.cloud.bigtable" % "bigtable-hbase-2.x" % "2.14.2",
     libraryDependencies ++= circe,
     libraryDependencies ++= avro,

cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/test/BigQueryCatalogTest.scala

@@ -6,11 +6,14 @@ import ai.chronon.spark.SparkSessionBuilder
 import ai.chronon.spark.TableUtils
 import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS
 import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem
+import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration
 import org.apache.spark.sql.SparkSession
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.scalatest.funsuite.AnyFunSuite
 import org.scalatestplus.mockito.MockitoSugar
+import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem
+import com.google.cloud.hadoop.fs.gcs.HadoopConfigurationProperty
 
 class BigQueryCatalogTest extends AnyFunSuite with MockitoSugar {
 
@@ -34,6 +37,14 @@ class BigQueryCatalogTest extends AnyFunSuite with MockitoSugar {
     assertEquals("thrift://localhost:9083", spark.sqlContext.getConf("hive.metastore.uris"))
   }
 
+  test("google runtime classes are available") {
+    assertTrue(GoogleHadoopFileSystemConfiguration.BLOCK_SIZE.isInstanceOf[HadoopConfigurationProperty[Long]])
+    assertCompiles("classOf[GoogleHadoopFileSystem]")
+    assertCompiles("classOf[GoogleHadoopFS]")
+    assertCompiles("classOf[GoogleCloudStorageFileSystem]")
+
+  }
+
   test("verify dynamic classloading of GCP providers") {
     assertTrue(tableUtils.tableReadFormat("data.sample_native") match {
       case BQuery(_) => true

cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/test/DataprocSubmitterTest.scala

@@ -49,7 +49,7 @@ class DataprocSubmitterTest extends AnyFunSuite with MockitoSugar {
     BigQueryUtilScala.validateScalaVersionCompatibility()
   }
 
-  ignore("Used to iterate locally. Do not enable this in CI/CD!") {
+  test("Used to iterate locally. Do not enable this in CI/CD!") {
 
     val submitter = DataprocSubmitter()
     val submittedJobId =

spark/src/main/scala/ai/chronon/spark/SparkSessionBuilder.scala

@@ -109,6 +109,8 @@ object SparkSessionBuilder {
         .config("spark.driver.bindAddress", "127.0.0.1")
         .config("spark.ui.enabled", "false")
         .config("spark.sql.catalogImplementation", "hive")
+        .config("spark.driver.userClassPathFirst", true)
+        .config("spark.executor.userClassPathFirst", true)
     } else {
       // hive jars need to be available on classpath - no needed for local testing
       baseBuilder