repartition

Co-authored-by: Thomas Chow <[email protected]>

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

spark/src/main/scala/ai/chronon/spark/TableUtils.scala

@@ -461,13 +461,7 @@ class TableUtils(@transient val sparkSession: SparkSession) extends Serializable
         if (sparkSession.conf.get("spark.chronon.write.repartition", true.toString).toBoolean)
           repartitionInternal(df, tableName, stats, sortByCols)
         else df
-      repartitioned
-        .select(repartitioned.columns.map {
-          case c if c == partitionColumn && dataPointer.writeFormat.map(_.toUpperCase).exists("BIGQUERY".equals) =>
-            to_date(repartitioned.col(c), partitionFormat).as(partitionColumn)
-          case c => repartitioned.col(c)
-        }.toList: _*)
-        .write
+      repartitioned.write
         .mode(saveMode)
         .save(dataPointer)