remove other stuff

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryFormat.scala

@@ -5,7 +5,7 @@ import ai.chronon.spark.format.Format
 import com.google.cloud.bigquery.BigQuery
 import com.google.cloud.bigquery.connector.common.BigQueryUtil
 import com.google.cloud.spark.bigquery.v2.Spark35BigQueryTableProvider
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.functions.{col, date_format, to_date}
 
 case class BigQueryFormat(project: String, bqClient: BigQuery, override val options: Map[String, String])
@@ -22,13 +22,6 @@ case class BigQueryFormat(project: String, bqClient: BigQuery, override val opti
   override def primaryPartitions(tableName: String, partitionColumn: String, subPartitionsFilter: Map[String, String])(
       implicit sparkSession: SparkSession): List[String] =
     super.primaryPartitions(tableName, partitionColumn, subPartitionsFilter)
-  override def generateTableBuilder(df: DataFrame,
-                                    tableName: String,
-                                    partitionColumns: List[String],
-                                    tableProperties: Map[String, String],
-                                    fileFormat: String): (String => Unit) => Unit = {
-    throw new UnsupportedOperationException("generateTableBuilder not supported for BigQuery")
-  }
 
   override def partitions(tableName: String)(implicit sparkSession: SparkSession): List[Map[String, String]] = {
     import sparkSession.implicits._
@@ -99,10 +92,5 @@ case class BigQueryFormat(project: String, bqClient: BigQuery, override val opti
 
   }
 
-  def createTableTypeString: String =
-    throw new UnsupportedOperationException("createTableTypeString not yet supported for BigQuery")
-  def fileFormatString(format: String): String =
-    throw new UnsupportedOperationException("fileFormatString not yet supported for BigQuery")
-
   override def supportSubPartitionsFilter: Boolean = true
 }

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GCSFormat.scala

@@ -5,7 +5,7 @@ import com.google.cloud.spark.bigquery.repackaged.com.google.cloud.bigquery._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
-import org.apache.spark.sql.{DataFrame, Encoders, Row, SparkSession}
+import org.apache.spark.sql.{Encoders, Row, SparkSession}
 import org.slf4j.LoggerFactory
 
 case class GCS(sourceUri: String, fileFormat: String) extends Format {
@@ -78,18 +78,6 @@ case class GCS(sourceUri: String, fileFormat: String) extends Format {
       .toList
   }
 
-  override def generateTableBuilder(df: DataFrame,
-                                    tableName: String,
-                                    partitionColumns: List[String],
-                                    tableProperties: Map[String, String],
-                                    fileFormat: String): (String => Unit) => Unit = {
-    throw new UnsupportedOperationException("generateTableBuilder not supported for GCS")
-  }
-
-  def createTableTypeString: String = throw new UnsupportedOperationException("GCS does not support create table")
-
-  def fileFormatString(format: String): String = ""
-
   override def supportSubPartitionsFilter: Boolean = true
 
 }

spark/src/main/scala/ai/chronon/spark/format/DeltaLake.scala

@@ -32,9 +32,5 @@ case object DeltaLake extends Format {
 
   }
 
-  def createTableTypeString: String = "DELTA"
-
-  def fileFormatString(format: String): String = ""
-
   override def supportSubPartitionsFilter: Boolean = true
 }

spark/src/main/scala/ai/chronon/spark/format/Format.scala

@@ -1,11 +1,8 @@
 package ai.chronon.spark.format
 
 import ai.chronon.spark.format.CreationUtils.alterTablePropertiesSql
-import ai.chronon.spark.format.CreationUtils.createTableSql
-import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.SparkSession
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 trait Format {
   @transient private lazy val logger: Logger = LoggerFactory.getLogger(getClass)
@@ -48,33 +45,6 @@ trait Format {
   //      )
   def partitions(tableName: String)(implicit sparkSession: SparkSession): List[Map[String, String]]
 
-  def generateTableBuilder(df: DataFrame,
-                           tableName: String,
-                           partitionColumns: List[String],
-                           tableProperties: Map[String, String],
-                           fileFormat: String): (String => Unit) => Unit = {
-
-    def inner(df: DataFrame,
-              tableName: String,
-              partitionColumns: List[String],
-              tableProperties: Map[String, String],
-              fileFormat: String)(sqlEvaluator: String => Unit): Unit = {
-      val creationSql =
-        createTableSql(tableName,
-                       df.schema,
-                       partitionColumns,
-                       tableProperties,
-                       fileFormatString(fileFormat),
-                       createTableTypeString)
-
-      sqlEvaluator(creationSql)
-      ()
-    }
-
-    inner(df, tableName, partitionColumns, tableProperties, fileFormat)
-
-  }
-
   def alterTableProperties(tableName: String, tableProperties: Map[String, String]): (String => Unit) => Unit = {
 
     def inner(tableName: String, tableProperties: Map[String, String])(sqlEvaluator: String => Unit) = {
@@ -89,16 +59,7 @@ trait Format {
 
   }
 
-  // Help specify the appropriate table type to use in the Spark create table DDL query
-  def createTableTypeString: String
-
-  // Help specify the appropriate file format to use in the Spark create table DDL query
-  def fileFormatString(format: String): String
-
   // Does this format support sub partitions filters
   def supportSubPartitionsFilter: Boolean
 
-  // TODO: remove this once all formats implement table creation
-  val canCreateTable: Boolean = false
-
 }

spark/src/main/scala/ai/chronon/spark/format/Hive.scala

@@ -32,9 +32,5 @@ case object Hive extends Format {
       .toList
   }
 
-  def createTableTypeString: String = ""
-
-  def fileFormatString(format: String): String = s"STORED AS $format"
-
   override def supportSubPartitionsFilter: Boolean = true
 }

spark/src/main/scala/ai/chronon/spark/format/Iceberg.scala

@@ -52,9 +52,5 @@ case object Iceberg extends Format {
     }
   }
 
-  def createTableTypeString: String = "ICEBERG"
-
-  def fileFormatString(format: String): String = ""
-
   override def supportSubPartitionsFilter: Boolean = false
 }