add back bq filter

Co-authored-by: Thomas Chow <[email protected]>

Co-authored-by: Thomas Chow <[email protected]>

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

spark/src/main/scala/ai/chronon/spark/TableUtils.scala

@@ -330,9 +330,10 @@ class TableUtils(@transient val sparkSession: SparkSession) extends Serializable
                        stats: Option[DfStats] = None,
                        sortByCols: Seq[String] = Seq.empty): Unit = {
     // partitions to the last
+    val dataPointer = DataPointer.from(tableName, sparkSession)
     val colOrder = df.columns.diff(partitionColumns) ++ partitionColumns
     val dfRearranged: DataFrame = df.select(colOrder.map {
-      case c if c == partitionColumn =>
+      case c if c == partitionColumn && dataPointer.writeFormat.map(_.toUpperCase).exists("BIGQUERY".equals) =>
         to_date(df.col(c), partitionFormat).as(partitionColumn)
       case c => df.col(c)
     }: _*)