Temporarily set materializationProject and materializationDataset to get bq connector to create temp tables.

Author: david-zlai

cloud_gcp/src/main/resources/additional-confs.yaml

@@ -1,3 +1,5 @@
 spark.chronon.table.format_provider.class: "ai.chronon.integrations.cloud_gcp.GcpFormatProvider"
 spark.chronon.partition.format: "yyyy-MM-dd"
-spark.chronon.table.gcs.temporary_gcs_bucket: "zl-warehouse"
+spark.chronon.table.gcs.temporary_gcs_bucket: "zl-warehouse"
+spark.chronon.table.gcs.connector_output_dataset: "data"
+spark.chronon.table.gcs.connector_output_project: "canary-443022"

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpFormatProvider.scala

@@ -43,7 +43,9 @@ case class GcpFormatProvider(sparkSession: SparkSession) extends FormatProvider
     val sparkOptions: Map[String, String] = Map(
       // todo(tchow): No longer needed after https://github.com/GoogleCloudDataproc/spark-bigquery-connector/pull/1320
       "temporaryGcsBucket" -> sparkSession.conf.get("spark.chronon.table.gcs.temporary_gcs_bucket"),
-      "writeMethod" -> "indirect"
+      "writeMethod" -> "indirect",
+      "materializationProject" -> sparkSession.conf.get("spark.chronon.table.gcs.connector_output_project"),
+      "materializationDataset" -> sparkSession.conf.get("spark.chronon.table.gcs.connector_output_dataset")
     ) ++ partitionColumnOption
 
     BigQueryFormat(tableId.getProject, sparkOptions)