update submitter conf

tchow-zlai · tchow-zlai · commit 447c1523f24c · 2025-01-01T19:05:25.000-08:00
-e 
Co-authored-by: Thomas Chow &lt;thomaschow369@gmail.com&gt;
diff --git a/cloud_gcp/src/main/resources/additional-confs.yaml b/cloud_gcp/src/main/resources/additional-confs.yaml
@@ -0,0 +1 @@
+spark.chronon.table.format_provider.class: "ai.chronon.integrations.cloud_gcp.GcpFormatProvider"
diff --git a/cloud_gcp/src/main/resources/driver-conf.yaml b/cloud_gcp/src/main/resources/driver-conf.yaml
diff --git a/spark/src/test/scala/ai/chronon/spark/test/OfflineSubcommandTest.scala b/spark/src/test/scala/ai/chronon/spark/test/OfflineSubcommandTest.scala
@@ -76,10 +76,10 @@ class OfflineSubcommandTest {
       case Right(v) => v
       case Left(e)  => throw e
     }
-    assertEquals(confs.get("spark.chronon.table.format_provider.class"),
-                 sparkSession.conf.getOption("spark.chronon.table.format_provider.class"))
-    assertEquals(Some("ai.chronon.integrations.cloud_gcp.GcpFormatProvider"),
-                 sparkSession.conf.getOption("spark.chronon.table.format_provider.class"))
+
+    val confKey = "spark.chronon.table.format_provider.class"
+    assertEquals(confs.get(confKey), sparkSession.conf.getOption(confKey))
+    assertEquals(Some("ai.chronon.integrations.cloud_gcp.GcpFormatProvider"), sparkSession.conf.getOption(confKey))
     assertTrue(sparkSession.conf.getOption("nonexistent_key").isEmpty)
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+spark.chronon.table.format_provider.class: "ai.chronon.integrations.cloud_gcp.GcpFormatProvider"`
Original file line number	Diff line number	Diff line change
`@@ -76,10 +76,10 @@ class OfflineSubcommandTest {`
`76`	`76`	`case Right(v) => v`
`77`	`77`	`case Left(e) => throw e`
`78`	`78`	`}`
`79`		`- assertEquals(confs.get("spark.chronon.table.format_provider.class"),`
`80`		`- sparkSession.conf.getOption("spark.chronon.table.format_provider.class"))`
`81`		`- assertEquals(Some("ai.chronon.integrations.cloud_gcp.GcpFormatProvider"),`
`82`		`- sparkSession.conf.getOption("spark.chronon.table.format_provider.class"))`
	`79`	`+`
	`80`	`+ val confKey = "spark.chronon.table.format_provider.class"`
	`81`	`+ assertEquals(confs.get(confKey), sparkSession.conf.getOption(confKey))`
	`82`	`+ assertEquals(Some("ai.chronon.integrations.cloud_gcp.GcpFormatProvider"), sparkSession.conf.getOption(confKey))`
`83`	`83`	`assertTrue(sparkSession.conf.getOption("nonexistent_key").isEmpty)`
`84`	`84`	`}`
`85`	`85`	`}`