comments

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

build.sbt

@@ -190,8 +190,8 @@ lazy val spark = project
     libraryDependencies += "com.google.guava" % "guava" % "33.3.1-jre",
     libraryDependencies ++= log4j2,
     libraryDependencies ++= delta.map(_ % "provided"),
-    libraryDependencies += "org.json4s" % "json4s-jackson_2.12" % "3.7.0-M11",
-    libraryDependencies += "org.json4s" %% "json4s-native" % "3.7.0-M11", // Use json4s-native or json4s-jackson
+    libraryDependencies += "org.json4s" % "json4s-jackson_2.12" % "3.7.0-M11", // This version is pinned to the one spark uses in 3.X.X - see: https://github.com/apache/spark/pull/45838
+    libraryDependencies += "org.json4s" %% "json4s-native" % "3.7.0-M11",
     libraryDependencies += "org.json4s" %% "json4s-core" % "3.7.0-M11",
     libraryDependencies += "org.yaml" % "snakeyaml" % "2.3"
   )
@@ -220,7 +220,7 @@ lazy val cloud_gcp = project
     libraryDependencies += "com.google.cloud.bigdataoss" % "gcsio" % "3.0.3", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystem.java
     libraryDependencies += "com.google.cloud.bigdataoss" % "util-hadoop" % "3.0.0", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/util-hadoop/src/main/java/com/google/cloud/hadoop/util/HadoopConfigurationProperty.java
     libraryDependencies += "com.google.cloud.spark" %% "spark-bigquery-with-dependencies" % "0.41.0",
-    libraryDependencies += "org.json4s" % "json4s-jackson_2.12" % "3.7.0-M11", // Fixed version to follow spark: https://github.com/apache/spark/pull/45838
+    libraryDependencies += "org.json4s" % "json4s-jackson_2.12" % "3.7.0-M11", // This version is pinned to the one spark uses in 3.X.X - see: https://github.com/apache/spark/pull/45838
     libraryDependencies += "org.json4s" %% "json4s-native" % "3.7.0-M11",
     libraryDependencies += "org.json4s" %% "json4s-core" % "3.7.0-M11",
     libraryDependencies += "org.yaml" % "snakeyaml" % "2.3",

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/DataprocSubmitter.scala

@@ -89,10 +89,10 @@ object DataprocSubmitter {
   }
 
   private[cloud_gcp] def loadConfig: SubmitterConf = {
-    val isO = Option(getClass.getClassLoader.getResourceAsStream("dataproc-submitter-conf.yaml"))
+    val inputStreamOption = Option(getClass.getClassLoader.getResourceAsStream("dataproc-submitter-conf.yaml"))
     val yamlLoader = new Yaml()
     implicit val formats: Formats = DefaultFormats
-    isO
+    inputStreamOption
       .map(Source.fromInputStream)
       .map((is) =>
         try { is.mkString }

cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/DataprocSubmitterTest.scala

@@ -47,7 +47,7 @@ class DataprocSubmitterTest extends AnyFunSuite with MockitoSugar {
     BigQueryUtilScala.validateScalaVersionCompatibility()
   }
 
-  test("Used to iterate locally. Do not enable this in CI/CD!") {
+  ignore("Used to iterate locally. Do not enable this in CI/CD!") {
 
     val submitter = DataprocSubmitter()
     val submittedJobId =