diff --git a/cloud_aws/src/test/scala/ai/chronon/integrations/aws/GlueCatalogTest.scala b/cloud_aws/src/test/scala/ai/chronon/integrations/aws/GlueCatalogTest.scala index 7e29c8f60c..6c82434457 100644 --- a/cloud_aws/src/test/scala/ai/chronon/integrations/aws/GlueCatalogTest.scala +++ b/cloud_aws/src/test/scala/ai/chronon/integrations/aws/GlueCatalogTest.scala @@ -2,7 +2,7 @@ package ai.chronon.integrations.aws import ai.chronon.spark.ChrononHudiKryoRegistrator import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SaveMode import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals diff --git a/cloud_aws/src/test/scala/ai/chronon/integrations/aws/HudiTableUtilsTest.scala b/cloud_aws/src/test/scala/ai/chronon/integrations/aws/HudiTableUtilsTest.scala index 2dbfae7740..0c345eef66 100644 --- a/cloud_aws/src/test/scala/ai/chronon/integrations/aws/HudiTableUtilsTest.scala +++ b/cloud_aws/src/test/scala/ai/chronon/integrations/aws/HudiTableUtilsTest.scala @@ -2,7 +2,7 @@ package ai.chronon.integrations.aws import ai.chronon.spark.ChrononHudiKryoRegistrator import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue diff --git a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryExternal.scala b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryExternal.scala index 919c31dfe0..a1ead4823c 100644 --- a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryExternal.scala +++ b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryExternal.scala @@ -1,6 +1,6 @@ package ai.chronon.integrations.cloud_gcp -import ai.chronon.spark.format.Format +import ai.chronon.spark.catalog.Format import com.google.cloud.bigquery._ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.execution.FileSourceScanExec diff --git a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryNative.scala b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryNative.scala index 33bc345dfb..ed8b83476f 100644 --- a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryNative.scala +++ b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigQueryNative.scala @@ -1,7 +1,7 @@ package ai.chronon.integrations.cloud_gcp -import ai.chronon.spark.TableUtils -import ai.chronon.spark.format.Format +import ai.chronon.spark.catalog.TableUtils +import ai.chronon.spark.catalog.Format import com.google.cloud.bigquery.BigQueryOptions import com.google.cloud.spark.bigquery.v2.Spark35BigQueryTableProvider import org.apache.spark.sql.{DataFrame, SparkSession} diff --git a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpFormatProvider.scala b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpFormatProvider.scala index a31328e002..dbdfd8f718 100644 --- a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpFormatProvider.scala +++ b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpFormatProvider.scala @@ -1,5 +1,6 @@ package ai.chronon.integrations.cloud_gcp -import ai.chronon.spark.format.{DefaultFormatProvider, Format, Iceberg} + +import ai.chronon.spark.catalog.{DefaultFormatProvider, Format, Iceberg} import com.google.cloud.bigquery._ import com.google.cloud.spark.bigquery.v2.Spark31BigQueryTable import org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog diff --git a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/Spark2BigTableLoader.scala b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/Spark2BigTableLoader.scala index 4207268527..e51f1f466f 100644 --- a/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/Spark2BigTableLoader.scala +++ b/cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/Spark2BigTableLoader.scala @@ -6,7 +6,7 @@ import ai.chronon.api.MetaData import ai.chronon.integrations.cloud_gcp.BigTableKVStore.ColumnFamilyQualifierString import ai.chronon.integrations.cloud_gcp.BigTableKVStore.ColumnFamilyString import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.functions import org.apache.spark.sql.functions.udf import org.rogach.scallop.ScallopConf diff --git a/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigQueryCatalogTest.scala b/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigQueryCatalogTest.scala index c7d25d003b..ab53a4b1f3 100644 --- a/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigQueryCatalogTest.scala +++ b/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigQueryCatalogTest.scala @@ -1,7 +1,7 @@ package ai.chronon.integrations.cloud_gcp -import ai.chronon.spark.format.{FormatProvider, Iceberg} -import ai.chronon.spark.{SparkSessionBuilder, TableUtils} +import ai.chronon.spark.SparkSessionBuilder +import ai.chronon.spark.catalog.{FormatProvider, Iceberg, TableUtils} import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.io.{Input, Output} import com.google.cloud.hadoop.fs.gcs.{ diff --git a/spark/BUILD.bazel b/spark/BUILD.bazel index de637a0575..98f2f645cc 100644 --- a/spark/BUILD.bazel +++ b/spark/BUILD.bazel @@ -49,6 +49,27 @@ scala_library( ], ) +scala_library( + name = "catalog_lib", + srcs = glob(["src/main/scala/ai/chronon/spark/catalog/*.scala"]), + format = select({ + "//tools/config:scala_2_13": False, + "//conditions:default": True, + }), + visibility = ["//visibility:public"], + deps = [ + ":lib", + "//api:lib", + "//api:thrift_java", + "//tools/build_rules/spark:spark-exec", + maven_artifact("org.slf4j:slf4j-api"), + maven_artifact("org.apache.thrift:libthrift"), + maven_artifact("org.apache.logging.log4j:log4j-api"), + maven_artifact("org.apache.logging.log4j:log4j-core"), + maven_artifact_with_suffix("io.delta:delta-spark"), + ], +) + scala_library( name = "submission_lib", srcs = glob(["src/main/scala/ai/chronon/spark/submission/*.scala"]), diff --git a/spark/src/main/scala/ai/chronon/spark/Analyzer.scala b/spark/src/main/scala/ai/chronon/spark/Analyzer.scala index afd8f42161..279ce8b836 100644 --- a/spark/src/main/scala/ai/chronon/spark/Analyzer.scala +++ b/spark/src/main/scala/ai/chronon/spark/Analyzer.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.{DataFrame, Row, types} import org.apache.spark.sql.types.{StringType, StructType} import org.slf4j.{Logger, LoggerFactory} +import ai.chronon.spark.catalog.TableUtils import scala.collection.{Seq, immutable, mutable} import scala.collection.mutable.ListBuffer diff --git a/spark/src/main/scala/ai/chronon/spark/BootstrapInfo.scala b/spark/src/main/scala/ai/chronon/spark/BootstrapInfo.scala index b0b2f21b1e..066045c2bd 100644 --- a/spark/src/main/scala/ai/chronon/spark/BootstrapInfo.scala +++ b/spark/src/main/scala/ai/chronon/spark/BootstrapInfo.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.functions.expr import org.apache.spark.sql.types.{StringType, StructType} import org.slf4j.{Logger, LoggerFactory} +import ai.chronon.spark.catalog.TableUtils import scala.collection.{Seq, immutable, mutable} import scala.util.Try diff --git a/spark/src/main/scala/ai/chronon/spark/Driver.scala b/spark/src/main/scala/ai/chronon/spark/Driver.scala index b748f5b521..a6dab5efd7 100644 --- a/spark/src/main/scala/ai/chronon/spark/Driver.scala +++ b/spark/src/main/scala/ai/chronon/spark/Driver.scala @@ -26,7 +26,7 @@ import ai.chronon.online.fetcher.{ConfPathOrName, FetchContext, FetcherMain, Met import ai.chronon.online.{Api, MetadataDirWalker, MetadataEndPoint, TopicChecker} import ai.chronon.orchestration.{JoinMergeNode, JoinPartNode} import ai.chronon.spark.batch._ -import ai.chronon.spark.format.Format +import ai.chronon.spark.catalog.{Format, TableUtils} import ai.chronon.spark.stats.drift.{Summarizer, SummaryPacker, SummaryUploader} import ai.chronon.spark.stats.{CompareBaseJob, CompareJob, ConsistencyJob} import ai.chronon.spark.streaming.JoinSourceRunner diff --git a/spark/src/main/scala/ai/chronon/spark/Extensions.scala b/spark/src/main/scala/ai/chronon/spark/Extensions.scala index fe16dfc690..a6a78aebb6 100644 --- a/spark/src/main/scala/ai/chronon/spark/Extensions.scala +++ b/spark/src/main/scala/ai/chronon/spark/Extensions.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.{LongType, StructType} import org.apache.spark.util.sketch.BloomFilter import org.slf4j.{Logger, LoggerFactory} +import ai.chronon.spark.catalog.TableUtils import java.util import scala.collection.Seq diff --git a/spark/src/main/scala/ai/chronon/spark/GroupBy.scala b/spark/src/main/scala/ai/chronon/spark/GroupBy.scala index 97702284a0..5afcc84138 100644 --- a/spark/src/main/scala/ai/chronon/spark/GroupBy.scala +++ b/spark/src/main/scala/ai/chronon/spark/GroupBy.scala @@ -31,6 +31,7 @@ import ai.chronon.api.{ TsUtils, TimeRange } +import ai.chronon.spark.catalog.TableUtils import ai.chronon.api.DataModel.ENTITIES import ai.chronon.api.DataModel.EVENTS import ai.chronon.api.Extensions._ diff --git a/spark/src/main/scala/ai/chronon/spark/GroupByUpload.scala b/spark/src/main/scala/ai/chronon/spark/GroupByUpload.scala index c94f8a6e5e..c0868a312b 100644 --- a/spark/src/main/scala/ai/chronon/spark/GroupByUpload.scala +++ b/spark/src/main/scala/ai/chronon/spark/GroupByUpload.scala @@ -21,6 +21,7 @@ import ai.chronon.aggregator.windowing.FiveMinuteResolution import ai.chronon.aggregator.windowing.Resolution import ai.chronon.aggregator.windowing.SawtoothOnlineAggregator import ai.chronon.api +import ai.chronon.spark.catalog.TableUtils import ai.chronon.api.Accuracy import ai.chronon.api.Constants import ai.chronon.api.DataModel diff --git a/spark/src/main/scala/ai/chronon/spark/Join.scala b/spark/src/main/scala/ai/chronon/spark/Join.scala index aef08f1816..82eac9082b 100644 --- a/spark/src/main/scala/ai/chronon/spark/Join.scala +++ b/spark/src/main/scala/ai/chronon/spark/Join.scala @@ -21,6 +21,7 @@ import ai.chronon.api.DataModel.ENTITIES import ai.chronon.api.Extensions._ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ +import ai.chronon.spark.catalog.TableUtils import ai.chronon.online.serde.SparkConversions import ai.chronon.orchestration.{JoinBootstrapNode, JoinPartNode} import ai.chronon.spark.Extensions._ diff --git a/spark/src/main/scala/ai/chronon/spark/JoinBase.scala b/spark/src/main/scala/ai/chronon/spark/JoinBase.scala index b7df362d8b..e44c9b09f9 100644 --- a/spark/src/main/scala/ai/chronon/spark/JoinBase.scala +++ b/spark/src/main/scala/ai/chronon/spark/JoinBase.scala @@ -20,6 +20,7 @@ import ai.chronon.api import ai.chronon.api.DataModel.ENTITIES import ai.chronon.api.Extensions._ import ai.chronon.api.ScalaJavaConversions._ +import ai.chronon.spark.catalog.TableUtils import ai.chronon.api.{Accuracy, Constants, DateRange, JoinPart, PartitionRange, PartitionSpec} import ai.chronon.online.metrics.Metrics import ai.chronon.orchestration.JoinBootstrapNode diff --git a/spark/src/main/scala/ai/chronon/spark/JoinDerivationJob.scala b/spark/src/main/scala/ai/chronon/spark/JoinDerivationJob.scala index c788fdb64a..7c27d86f77 100644 --- a/spark/src/main/scala/ai/chronon/spark/JoinDerivationJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/JoinDerivationJob.scala @@ -3,6 +3,7 @@ package ai.chronon.spark import ai.chronon.api.Extensions._ import ai.chronon.api.ScalaJavaConversions.ListOps import ai.chronon.api.DateRange +import ai.chronon.spark.catalog.TableUtils import ai.chronon.orchestration.JoinDerivationNode import ai.chronon.spark.Extensions._ import org.apache.spark.sql.functions.{coalesce, col, expr} diff --git a/spark/src/main/scala/ai/chronon/spark/JoinUtils.scala b/spark/src/main/scala/ai/chronon/spark/JoinUtils.scala index ffcf34a485..790780fab4 100644 --- a/spark/src/main/scala/ai/chronon/spark/JoinUtils.scala +++ b/spark/src/main/scala/ai/chronon/spark/JoinUtils.scala @@ -23,6 +23,7 @@ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ import ai.chronon.spark.Extensions._ import com.google.gson.Gson +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql import org.apache.spark.sql.DataFrame import org.apache.spark.sql.expressions.UserDefinedFunction diff --git a/spark/src/main/scala/ai/chronon/spark/LabelJoin.scala b/spark/src/main/scala/ai/chronon/spark/LabelJoin.scala index 4477f3e53b..d1f1973a01 100644 --- a/spark/src/main/scala/ai/chronon/spark/LabelJoin.scala +++ b/spark/src/main/scala/ai/chronon/spark/LabelJoin.scala @@ -21,6 +21,7 @@ import ai.chronon.api.{Builders, Constants, JoinPart, PartitionSpec, TimeUnit, W import ai.chronon.api.DataModel.ENTITIES import ai.chronon.api.DataModel.EVENTS import ai.chronon.api.Extensions._ +import ai.chronon.spark.catalog.TableUtils import ai.chronon.api.PartitionRange import ai.chronon.online.metrics.Metrics import ai.chronon.spark.Extensions._ diff --git a/spark/src/main/scala/ai/chronon/spark/LocalTableExporter.scala b/spark/src/main/scala/ai/chronon/spark/LocalTableExporter.scala index 6fb740c10e..cb7594375e 100644 --- a/spark/src/main/scala/ai/chronon/spark/LocalTableExporter.scala +++ b/spark/src/main/scala/ai/chronon/spark/LocalTableExporter.scala @@ -19,7 +19,7 @@ package ai.chronon.spark import com.google.common.io.Files import org.apache.commons.io.FileUtils import org.apache.spark.sql.SaveMode - +import ai.chronon.spark.catalog.TableUtils import java.io.File object LocalTableExporter { diff --git a/spark/src/main/scala/ai/chronon/spark/LogFlattenerJob.scala b/spark/src/main/scala/ai/chronon/spark/LogFlattenerJob.scala index 50330a0d59..4d148548d8 100644 --- a/spark/src/main/scala/ai/chronon/spark/LogFlattenerJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/LogFlattenerJob.scala @@ -25,6 +25,7 @@ import ai.chronon.online._ import ai.chronon.online.metrics._ import ai.chronon.online.serde._ import ai.chronon.spark.Extensions.{StructTypeOps, _} +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions.GenericRow import org.apache.spark.sql.functions.col diff --git a/spark/src/main/scala/ai/chronon/spark/MetadataExporter.scala b/spark/src/main/scala/ai/chronon/spark/MetadataExporter.scala index 7640ead3a4..8bdfbb7df0 100644 --- a/spark/src/main/scala/ai/chronon/spark/MetadataExporter.scala +++ b/spark/src/main/scala/ai/chronon/spark/MetadataExporter.scala @@ -18,18 +18,14 @@ package ai.chronon.spark import ai.chronon.api import ai.chronon.api.ThriftJsonCodec +import ai.chronon.spark.catalog.TableUtils import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.commons.lang.exception.ExceptionUtils -import org.slf4j.Logger -import org.slf4j.LoggerFactory +import org.slf4j.{Logger, LoggerFactory} -import java.io.BufferedWriter -import java.io.File -import java.io.FileWriter -import java.nio.file.Files -import java.nio.file.Paths -import scala.collection.immutable.Map +import java.io.{BufferedWriter, File, FileWriter} +import java.nio.file.{Files, Paths} object MetadataExporter { @transient lazy val logger: Logger = LoggerFactory.getLogger(getClass) diff --git a/spark/src/main/scala/ai/chronon/spark/batch/BootstrapJob.scala b/spark/src/main/scala/ai/chronon/spark/batch/BootstrapJob.scala index 8379b315e1..b1f3e0058f 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/BootstrapJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/BootstrapJob.scala @@ -7,11 +7,12 @@ import ai.chronon.online.serde.SparkConversions import ai.chronon.orchestration.JoinBootstrapNode import ai.chronon.spark.Extensions._ import ai.chronon.spark.JoinUtils.{coalescedJoin, set_add} -import ai.chronon.spark.{BootstrapInfo, JoinUtils, TableUtils} +import ai.chronon.spark.{BootstrapInfo, JoinUtils} import org.apache.spark.sql import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.{coalesce, col, lit, typedLit} import org.slf4j.{Logger, LoggerFactory} +import ai.chronon.spark.catalog.TableUtils import scala.collection.Seq diff --git a/spark/src/main/scala/ai/chronon/spark/batch/JoinPartJob.scala b/spark/src/main/scala/ai/chronon/spark/batch/JoinPartJob.scala index 581169a4fa..8cbf93c791 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/JoinPartJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/JoinPartJob.scala @@ -1,8 +1,5 @@ package ai.chronon.spark.batch -import ai.chronon.api -import ai.chronon.api.{Accuracy, Constants, DateRange, JoinPart, PartitionRange, PartitionSpec} -import ai.chronon.api.DataModel import ai.chronon.api.DataModel.{ENTITIES, EVENTS} import ai.chronon.api.Extensions.{DateRangeOps, DerivationOps, GroupByOps, JoinPartOps, MetadataOps} import ai.chronon.api.PartitionRange.toTimeRange @@ -10,7 +7,8 @@ import ai.chronon.api.{Accuracy, Builders, Constants, DateRange, JoinPart, Parti import ai.chronon.online.metrics.Metrics import ai.chronon.orchestration.JoinPartNode import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{GroupBy, JoinUtils, TableUtils} +import ai.chronon.spark.catalog.TableUtils +import ai.chronon.spark.{GroupBy, JoinUtils} import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.{col, date_format} import org.apache.spark.util.sketch.BloomFilter diff --git a/spark/src/main/scala/ai/chronon/spark/batch/LabelJoinV2.scala b/spark/src/main/scala/ai/chronon/spark/batch/LabelJoinV2.scala index df1e48680c..6583fd26b2 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/LabelJoinV2.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/LabelJoinV2.scala @@ -6,9 +6,9 @@ import ai.chronon.api.PartitionRange.toTimeRange import ai.chronon.api._ import ai.chronon.online.metrics.Metrics import ai.chronon.online.serde.SparkConversions -import ai.chronon.spark.Analyzer import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{GroupBy, JoinUtils, TableUtils} +import ai.chronon.spark.GroupBy +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.{col, lit} import org.apache.spark.sql.types.{DataType, StructType} diff --git a/spark/src/main/scala/ai/chronon/spark/batch/MergeJob.scala b/spark/src/main/scala/ai/chronon/spark/batch/MergeJob.scala index b5a9867122..4ae3377add 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/MergeJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/MergeJob.scala @@ -7,7 +7,8 @@ import ai.chronon.api.{Accuracy, Constants, DateRange, JoinPart, PartitionRange, import ai.chronon.orchestration.JoinMergeNode import ai.chronon.spark.Extensions._ import ai.chronon.spark.JoinUtils.coalescedJoin -import ai.chronon.spark.{JoinUtils, TableUtils} +import ai.chronon.spark.JoinUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.{col, date_add, date_format, to_date} import org.slf4j.{Logger, LoggerFactory} diff --git a/spark/src/main/scala/ai/chronon/spark/batch/SourceJob.scala b/spark/src/main/scala/ai/chronon/spark/batch/SourceJob.scala index 17a2c22cf5..3a434da481 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/SourceJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/SourceJob.scala @@ -6,7 +6,7 @@ import ai.chronon.api.ScalaJavaConversions.JListOps import ai.chronon.orchestration.SourceWithFilterNode import ai.chronon.spark.Extensions._ import ai.chronon.spark.JoinUtils.parseSkewKeys -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import scala.collection.{Map, Seq} import scala.jdk.CollectionConverters._ diff --git a/spark/src/main/scala/ai/chronon/spark/batch/StagingQuery.scala b/spark/src/main/scala/ai/chronon/spark/batch/StagingQuery.scala index c26aba6c85..865eb9da54 100644 --- a/spark/src/main/scala/ai/chronon/spark/batch/StagingQuery.scala +++ b/spark/src/main/scala/ai/chronon/spark/batch/StagingQuery.scala @@ -5,9 +5,10 @@ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api.thrift.TBase import ai.chronon.api.{EngineType, ParametricMacro, PartitionRange, ThriftJsonCodec} import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{SparkSessionBuilder, TableUtils} +import ai.chronon.spark.SparkSessionBuilder import org.rogach.scallop.{ScallopConf, ScallopOption} import org.slf4j.{Logger, LoggerFactory} +import ai.chronon.spark.catalog.TableUtils import scala.collection.mutable import scala.reflect.ClassTag diff --git a/spark/src/main/scala/ai/chronon/spark/format/CreationUtils.scala b/spark/src/main/scala/ai/chronon/spark/catalog/CreationUtils.scala similarity index 98% rename from spark/src/main/scala/ai/chronon/spark/format/CreationUtils.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/CreationUtils.scala index b9332585af..5578e45165 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/CreationUtils.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/CreationUtils.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.types.StructType diff --git a/spark/src/main/scala/ai/chronon/spark/format/DefaultFormatProvider.scala b/spark/src/main/scala/ai/chronon/spark/catalog/DefaultFormatProvider.scala similarity index 98% rename from spark/src/main/scala/ai/chronon/spark/format/DefaultFormatProvider.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/DefaultFormatProvider.scala index c9955fc99e..40595f27a1 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/DefaultFormatProvider.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/DefaultFormatProvider.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.SparkSession import org.slf4j.{Logger, LoggerFactory} diff --git a/spark/src/main/scala/ai/chronon/spark/format/DeltaLake.scala b/spark/src/main/scala/ai/chronon/spark/catalog/DeltaLake.scala similarity index 98% rename from spark/src/main/scala/ai/chronon/spark/format/DeltaLake.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/DeltaLake.scala index bbaf20e04e..3b4a8d92c2 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/DeltaLake.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/DeltaLake.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.DeltaLog diff --git a/spark/src/main/scala/ai/chronon/spark/format/Format.scala b/spark/src/main/scala/ai/chronon/spark/catalog/Format.scala similarity index 98% rename from spark/src/main/scala/ai/chronon/spark/format/Format.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/Format.scala index 95cb84aea4..157f4db658 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/Format.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/Format.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.SparkSession import org.slf4j.{Logger, LoggerFactory} diff --git a/spark/src/main/scala/ai/chronon/spark/format/FormatProvider.scala b/spark/src/main/scala/ai/chronon/spark/catalog/FormatProvider.scala similarity index 97% rename from spark/src/main/scala/ai/chronon/spark/format/FormatProvider.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/FormatProvider.scala index 2f90dfff8a..572820ec09 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/FormatProvider.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/FormatProvider.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.SparkSession diff --git a/spark/src/main/scala/ai/chronon/spark/format/Hive.scala b/spark/src/main/scala/ai/chronon/spark/catalog/Hive.scala similarity index 96% rename from spark/src/main/scala/ai/chronon/spark/format/Hive.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/Hive.scala index 303a5edddc..60359604eb 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/Hive.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/Hive.scala @@ -1,4 +1,4 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog import org.apache.spark.sql.SparkSession diff --git a/spark/src/main/scala/ai/chronon/spark/format/Iceberg.scala b/spark/src/main/scala/ai/chronon/spark/catalog/Iceberg.scala similarity index 96% rename from spark/src/main/scala/ai/chronon/spark/format/Iceberg.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/Iceberg.scala index fe1cdf9527..79d080d0e9 100644 --- a/spark/src/main/scala/ai/chronon/spark/format/Iceberg.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/Iceberg.scala @@ -1,6 +1,5 @@ -package ai.chronon.spark.format +package ai.chronon.spark.catalog -import ai.chronon.spark.TableUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{col, date_format} import org.apache.spark.sql.types.StructType diff --git a/spark/src/main/scala/ai/chronon/spark/TableUtils.scala b/spark/src/main/scala/ai/chronon/spark/catalog/TableUtils.scala similarity index 98% rename from spark/src/main/scala/ai/chronon/spark/TableUtils.scala rename to spark/src/main/scala/ai/chronon/spark/catalog/TableUtils.scala index 726e4f63dc..1c669f2534 100644 --- a/spark/src/main/scala/ai/chronon/spark/TableUtils.scala +++ b/spark/src/main/scala/ai/chronon/spark/catalog/TableUtils.scala @@ -14,30 +14,28 @@ * limitations under the License. */ -package ai.chronon.spark +package ai.chronon.spark.catalog import ai.chronon.api.ColorPrinter.ColorString import ai.chronon.api.Extensions._ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api.{Constants, PartitionRange, PartitionSpec, Query, QueryUtils, TsUtils} import ai.chronon.spark.Extensions._ -import ai.chronon.spark.format.CreationUtils.alterTablePropertiesSql -import ai.chronon.spark.format.{CreationUtils, FormatProvider, Iceberg} import org.apache.hadoop.hive.metastore.api.AlreadyExistsException -import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project} +import org.apache.spark.sql.catalyst.util.QuotingUtils import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ +import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SparkSession} import org.apache.spark.storage.StorageLevel import org.slf4j.{Logger, LoggerFactory} import java.io.{PrintWriter, StringWriter} -import java.time.{Instant, ZoneId} import java.time.format.DateTimeFormatter +import java.time.{Instant, ZoneId} import scala.collection.{Seq, mutable} import scala.util.{Failure, Success, Try} -import org.apache.spark.sql.catalyst.util.QuotingUtils /** Trait to track the table format in use by a Chronon dataset and some utility methods to help * retrieve metadata / configure it appropriately at creation time @@ -274,7 +272,7 @@ class TableUtils(@transient val sparkSession: SparkSession) extends Serializable // Run tableProperties Option(tableProperties).filter(_.nonEmpty).foreach { props => - sql(alterTablePropertiesSql(tableName, props)) + sql(CreationUtils.alterTablePropertiesSql(tableName, props)) } val finalizedDf = if (autoExpand) { @@ -564,7 +562,7 @@ class TableUtils(@transient val sparkSession: SparkSession) extends Serializable sql(expandTableQueryOpt.get) // set a flag in table props to indicate that this is a dynamic table - sql(alterTablePropertiesSql(tableName, Map(Constants.ChrononDynamicTable -> true.toString))) + sql(CreationUtils.alterTablePropertiesSql(tableName, Map(Constants.ChrononDynamicTable -> true.toString))) } } diff --git a/spark/src/main/scala/ai/chronon/spark/interactive/LocalWarehouse.scala b/spark/src/main/scala/ai/chronon/spark/interactive/LocalWarehouse.scala index af2c30a803..ebff4a2693 100644 --- a/spark/src/main/scala/ai/chronon/spark/interactive/LocalWarehouse.scala +++ b/spark/src/main/scala/ai/chronon/spark/interactive/LocalWarehouse.scala @@ -4,7 +4,7 @@ import ai.chronon.api.ColorPrinter.ColorString import ai.chronon.api.Extensions.StringOps import ai.chronon.online.CatalystUtil import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SparkSession import org.slf4j.LoggerFactory diff --git a/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemo.scala b/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemo.scala index 76db009097..c2a83f6d08 100644 --- a/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemo.scala +++ b/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemo.scala @@ -14,7 +14,7 @@ import ai.chronon.observability.TileSummarySeries import ai.chronon.online.KVStore import ai.chronon.online.stats.DriftStore import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.drift.Summarizer import ai.chronon.spark.stats.drift.SummaryUploader import ai.chronon.spark.stats.drift.scripts.PrepareData diff --git a/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemoDataLoader.scala b/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemoDataLoader.scala index 75a71a0b15..c9a21a2bc4 100644 --- a/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemoDataLoader.scala +++ b/spark/src/main/scala/ai/chronon/spark/scripts/ObservabilityDemoDataLoader.scala @@ -6,7 +6,7 @@ import ai.chronon.api.Extensions.MetadataOps import ai.chronon.online.HTTPKVStore import ai.chronon.online.KVStore import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.drift.Summarizer import ai.chronon.spark.stats.drift.SummaryUploader import ai.chronon.spark.stats.drift.scripts.PrepareData diff --git a/spark/src/main/scala/ai/chronon/spark/stats/CompareBaseJob.scala b/spark/src/main/scala/ai/chronon/spark/stats/CompareBaseJob.scala index 046a5789d6..e09583d534 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/CompareBaseJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/CompareBaseJob.scala @@ -20,7 +20,8 @@ import ai.chronon.api._ import ai.chronon.online.serde.SparkConversions import ai.chronon.online._ import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{TableUtils, TimedKvRdd} +import ai.chronon.spark.TimedKvRdd +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.types.DataType import org.slf4j.{Logger, LoggerFactory} diff --git a/spark/src/main/scala/ai/chronon/spark/stats/CompareJob.scala b/spark/src/main/scala/ai/chronon/spark/stats/CompareJob.scala index 6187adc073..bbc4c6b263 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/CompareJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/CompareJob.scala @@ -28,7 +28,7 @@ import ai.chronon.online.fetcher.DataMetrics import ai.chronon.spark.Analyzer import ai.chronon.spark.Extensions._ import ai.chronon.spark.batch.StagingQuery -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.TimedKvRdd import ai.chronon.spark.stats.CompareJob.getJoinKeys import org.apache.spark.sql.DataFrame diff --git a/spark/src/main/scala/ai/chronon/spark/stats/CompareMetrics.scala b/spark/src/main/scala/ai/chronon/spark/stats/CompareMetrics.scala index 3a9264ef86..70eda101a4 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/CompareMetrics.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/CompareMetrics.scala @@ -25,7 +25,7 @@ import ai.chronon.api._ import ai.chronon.online.serde.SparkConversions import ai.chronon.online.fetcher.DataMetrics import ai.chronon.spark.Comparison -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.TimedKvRdd import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame diff --git a/spark/src/main/scala/ai/chronon/spark/stats/ConsistencyJob.scala b/spark/src/main/scala/ai/chronon/spark/stats/ConsistencyJob.scala index 2015c31782..d215d65e15 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/ConsistencyJob.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/ConsistencyJob.scala @@ -23,7 +23,7 @@ import ai.chronon.api._ import ai.chronon.online.OnlineDerivationUtil.timeFields import ai.chronon.online.{fetcher, _} import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/spark/src/main/scala/ai/chronon/spark/stats/PartitionRunner.scala b/spark/src/main/scala/ai/chronon/spark/stats/PartitionRunner.scala index 70b2ad8d74..c75d8b303f 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/PartitionRunner.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/PartitionRunner.scala @@ -4,7 +4,7 @@ import ai.chronon.api.ColorPrinter.ColorString import ai.chronon.api.PartitionRange.{collapseToRange, collapsedPrint, rangesToString} import ai.chronon.api.{PartitionRange, PartitionSpec, Window} import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import scala.collection.mutable diff --git a/spark/src/main/scala/ai/chronon/spark/stats/StatsCompute.scala b/spark/src/main/scala/ai/chronon/spark/stats/StatsCompute.scala index f6c12ea57c..1f65066881 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/StatsCompute.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/StatsCompute.scala @@ -22,7 +22,7 @@ import ai.chronon.api import ai.chronon.api.Extensions._ import ai.chronon.online.serde.SparkConversions import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.TimedKvRdd import org.apache.datasketches.kll.KllFloatsSketch import org.apache.datasketches.memory.Memory diff --git a/spark/src/main/scala/ai/chronon/spark/stats/drift/Summarizer.scala b/spark/src/main/scala/ai/chronon/spark/stats/drift/Summarizer.scala index 6f001e04eb..a9e76331b9 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/drift/Summarizer.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/drift/Summarizer.scala @@ -10,7 +10,7 @@ import ai.chronon.observability.TileKey import ai.chronon.online.Api import ai.chronon.online.KVStore.GetRequest import ai.chronon.online.KVStore.PutRequest -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.PartitionRunner import ai.chronon.spark.stats.drift.Expressions.CardinalityExpression import ai.chronon.spark.stats.drift.Expressions.SummaryExpression diff --git a/spark/src/main/scala/ai/chronon/spark/stats/drift/SummaryUploader.scala b/spark/src/main/scala/ai/chronon/spark/stats/drift/SummaryUploader.scala index 185a29e34e..f48b47ab1a 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/drift/SummaryUploader.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/drift/SummaryUploader.scala @@ -4,7 +4,7 @@ import ai.chronon.api.Constants import ai.chronon.online.Api import ai.chronon.online.KVStore import ai.chronon.online.KVStore.PutRequest -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.Row import org.apache.spark.sql.types diff --git a/spark/src/main/scala/ai/chronon/spark/stats/drift/scripts/PrepareData.scala b/spark/src/main/scala/ai/chronon/spark/stats/drift/scripts/PrepareData.scala index 2e8c43f2e6..18f27d6542 100644 --- a/spark/src/main/scala/ai/chronon/spark/stats/drift/scripts/PrepareData.scala +++ b/spark/src/main/scala/ai/chronon/spark/stats/drift/scripts/PrepareData.scala @@ -12,7 +12,7 @@ import ai.chronon.api.Window import ai.chronon.observability.DriftMetric import ai.chronon.observability.DriftSpec import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import com.google.gson.GsonBuilder import com.google.gson.JsonParser import org.apache.spark.rdd.RDD diff --git a/spark/src/main/scala/ai/chronon/spark/streaming/JoinSourceRunner.scala b/spark/src/main/scala/ai/chronon/spark/streaming/JoinSourceRunner.scala index 88d69ae07a..b22e0975ad 100644 --- a/spark/src/main/scala/ai/chronon/spark/streaming/JoinSourceRunner.scala +++ b/spark/src/main/scala/ai/chronon/spark/streaming/JoinSourceRunner.scala @@ -24,7 +24,8 @@ import ai.chronon.online.KVStore.PutRequest import ai.chronon.online._ import ai.chronon.online.serde._ import ai.chronon.online.fetcher.Fetcher -import ai.chronon.spark.{GenericRowHandler, TableUtils} +import ai.chronon.spark.GenericRowHandler +import ai.chronon.spark.catalog.TableUtils import com.google.gson.Gson import org.apache.spark.api.java.function.{MapPartitionsFunction, VoidFunction2} import org.apache.spark.sql.{DataFrame, Dataset, Encoder, Encoders, Row, SparkSession} diff --git a/spark/src/main/scala/ai/chronon/spark/utils/InMemoryKvStore.scala b/spark/src/main/scala/ai/chronon/spark/utils/InMemoryKvStore.scala index 2421a39bee..8aef492f40 100644 --- a/spark/src/main/scala/ai/chronon/spark/utils/InMemoryKvStore.scala +++ b/spark/src/main/scala/ai/chronon/spark/utils/InMemoryKvStore.scala @@ -20,7 +20,7 @@ import ai.chronon.api.Constants import ai.chronon.online.KVStore import ai.chronon.online.KVStore.PutRequest import ai.chronon.online.KVStore.TimedValue -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.Row import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/spark/src/main/scala/ai/chronon/spark/utils/InMemoryStream.scala b/spark/src/main/scala/ai/chronon/spark/utils/InMemoryStream.scala index d5d19edd2b..c1f8c76f92 100644 --- a/spark/src/main/scala/ai/chronon/spark/utils/InMemoryStream.scala +++ b/spark/src/main/scala/ai/chronon/spark/utils/InMemoryStream.scala @@ -23,7 +23,7 @@ import ai.chronon.online.serde.AvroConversions import ai.chronon.online.serde.SparkConversions import ai.chronon.online.TileCodec import ai.chronon.spark.GenericRowHandler -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.avro.data.TimeConversions import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord diff --git a/spark/src/main/scala/ai/chronon/spark/utils/MockApi.scala b/spark/src/main/scala/ai/chronon/spark/utils/MockApi.scala index e77bcb301d..7e35a297f3 100644 --- a/spark/src/main/scala/ai/chronon/spark/utils/MockApi.scala +++ b/spark/src/main/scala/ai/chronon/spark/utils/MockApi.scala @@ -24,7 +24,7 @@ import ai.chronon.online.fetcher.Fetcher.Response import ai.chronon.online._ import ai.chronon.online.serde._ import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SparkSession diff --git a/spark/src/test/scala/ai/chronon/spark/test/CompareTest.scala b/spark/src/test/scala/ai/chronon/spark/test/CompareTest.scala index fd29daf612..a86ba9653c 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/CompareTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/CompareTest.scala @@ -19,7 +19,7 @@ package ai.chronon.spark.test import ai.chronon.api.TsUtils import ai.chronon.online.fetcher.DataMetrics import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.TimedKvRdd import ai.chronon.spark.stats.CompareBaseJob import org.apache.spark.sql.DataFrame diff --git a/spark/src/test/scala/ai/chronon/spark/test/DataFrameGen.scala b/spark/src/test/scala/ai/chronon/spark/test/DataFrameGen.scala index 1d4f0e2d41..fe9215b977 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/DataFrameGen.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/DataFrameGen.scala @@ -23,7 +23,7 @@ import ai.chronon.api.Constants import ai.chronon.api.LongType import ai.chronon.api.StringType import ai.chronon.online.serde.SparkConversions -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.rdd.RDD import org.apache.spark.sql.DataFrame import org.apache.spark.sql.Row diff --git a/spark/src/test/scala/ai/chronon/spark/test/DataRangeTest.scala b/spark/src/test/scala/ai/chronon/spark/test/DataRangeTest.scala index a654d676e8..48c1703455 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/DataRangeTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/DataRangeTest.scala @@ -19,7 +19,7 @@ package ai.chronon.spark.test import ai.chronon.api.PartitionSpec import ai.chronon.api.PartitionRange import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals import org.scalatest.flatspec.AnyFlatSpec diff --git a/spark/src/test/scala/ai/chronon/spark/test/LocalExportTableAbilityTest.scala b/spark/src/test/scala/ai/chronon/spark/test/LocalExportTableAbilityTest.scala index e78600c726..0276698c96 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/LocalExportTableAbilityTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/LocalExportTableAbilityTest.scala @@ -20,7 +20,7 @@ import ai.chronon.spark.Driver.LocalExportTableAbility import ai.chronon.spark.Driver.OfflineSubcommand import ai.chronon.spark.LocalTableExporter import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse diff --git a/spark/src/test/scala/ai/chronon/spark/test/LocalTableExporterTest.scala b/spark/src/test/scala/ai/chronon/spark/test/LocalTableExporterTest.scala index 005e0e3934..1b90403d00 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/LocalTableExporterTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/LocalTableExporterTest.scala @@ -24,7 +24,7 @@ import ai.chronon.api.LongType import ai.chronon.api.StringType import ai.chronon.spark.LocalTableExporter import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.LocalTableExporterTest.spark import ai.chronon.spark.test.LocalTableExporterTest.tmpDir import com.google.common.io.Files diff --git a/spark/src/test/scala/ai/chronon/spark/test/MetadataExporterTest.scala b/spark/src/test/scala/ai/chronon/spark/test/MetadataExporterTest.scala index f8991dbd43..45aeeb44f6 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/MetadataExporterTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/MetadataExporterTest.scala @@ -20,7 +20,7 @@ import ai.chronon.aggregator.test.Column import ai.chronon.api import ai.chronon.spark.Extensions._ import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.scalatest.flatspec.AnyFlatSpec import org.slf4j.Logger diff --git a/spark/src/test/scala/ai/chronon/spark/test/MigrationCompareTest.scala b/spark/src/test/scala/ai/chronon/spark/test/MigrationCompareTest.scala index d42724118b..bd55ddf547 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/MigrationCompareTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/MigrationCompareTest.scala @@ -25,7 +25,7 @@ import ai.chronon.online.fetcher.DataMetrics import ai.chronon.spark.Extensions._ import ai.chronon.spark.Join import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.CompareJob import org.apache.spark.sql.SparkSession import org.scalatest.flatspec.AnyFlatSpec diff --git a/spark/src/test/scala/ai/chronon/spark/test/OnlineUtils.scala b/spark/src/test/scala/ai/chronon/spark/test/OnlineUtils.scala index dc050434df..a96eebd92b 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/OnlineUtils.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/OnlineUtils.scala @@ -30,7 +30,7 @@ import ai.chronon.online.KVStore import ai.chronon.spark.GenericRowHandler import ai.chronon.spark.GroupByUpload import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.streaming.GroupBy import ai.chronon.spark.streaming.JoinSourceRunner import ai.chronon.spark.utils.InMemoryKvStore diff --git a/spark/src/test/scala/ai/chronon/spark/test/ResultValidationAbilityTest.scala b/spark/src/test/scala/ai/chronon/spark/test/ResultValidationAbilityTest.scala index 8fb0fab346..8cbcadbb63 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/ResultValidationAbilityTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/ResultValidationAbilityTest.scala @@ -22,7 +22,7 @@ import ai.chronon.api.PartitionSpec import ai.chronon.spark.Driver.OfflineSubcommand import ai.chronon.spark.Driver.ResultValidationAbility import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue diff --git a/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionTest.scala b/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionTest.scala index 37cf2159c0..2d20842fa1 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionTest.scala @@ -29,7 +29,7 @@ import ai.chronon.spark.Extensions.DataframeOps import ai.chronon.spark.LogFlattenerJob import ai.chronon.spark.LoggingSchema import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.utils.InMemoryKvStore import ai.chronon.spark.utils.MockApi import org.apache.spark.sql.DataFrame diff --git a/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionUtils.scala b/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionUtils.scala index ab1b819e37..fcb7320af1 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionUtils.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/SchemaEvolutionUtils.scala @@ -18,7 +18,7 @@ package ai.chronon.spark.test import ai.chronon.spark.LogUtils import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.utils.MockApi object SchemaEvolutionUtils { diff --git a/spark/src/test/scala/ai/chronon/spark/test/StagingQueryTest.scala b/spark/src/test/scala/ai/chronon/spark/test/StagingQueryTest.scala index 0d1596068f..99a7dc3def 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/StagingQueryTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/StagingQueryTest.scala @@ -20,7 +20,8 @@ import ai.chronon.aggregator.test.Column import ai.chronon.api.Extensions._ import ai.chronon.api._ import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{Comparison, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{Comparison, SparkSessionBuilder} +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.batch.StagingQuery import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals diff --git a/spark/src/test/scala/ai/chronon/spark/test/StatsComputeTest.scala b/spark/src/test/scala/ai/chronon/spark/test/StatsComputeTest.scala index 3a4a3aadd4..2a84e12c05 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/StatsComputeTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/StatsComputeTest.scala @@ -21,7 +21,7 @@ import ai.chronon.api._ import ai.chronon.online.serde.SparkConversions.toChrononSchema import ai.chronon.spark.Extensions._ import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.StatsCompute import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.lit diff --git a/spark/src/test/scala/ai/chronon/spark/test/TableTestUtils.scala b/spark/src/test/scala/ai/chronon/spark/test/TableTestUtils.scala index 29e1231cbf..7e1bb8d7bd 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/TableTestUtils.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/TableTestUtils.scala @@ -1,6 +1,6 @@ package ai.chronon.spark.test -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.SparkSession case class TableTestUtils(override val sparkSession: SparkSession) extends TableUtils(sparkSession: SparkSession) { diff --git a/spark/src/test/scala/ai/chronon/spark/test/TableUtilsFormatTest.scala b/spark/src/test/scala/ai/chronon/spark/test/TableUtilsFormatTest.scala index 44010e8727..c907c48ac9 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/TableUtilsFormatTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/TableUtilsFormatTest.scala @@ -6,11 +6,11 @@ import ai.chronon.api.LongType import ai.chronon.api.StringType import ai.chronon.api.StructField import ai.chronon.api.StructType -import ai.chronon.spark.IncompatibleSchemaException +import ai.chronon.spark.catalog.IncompatibleSchemaException import ai.chronon.spark.SparkSessionBuilder import ai.chronon.spark.SparkSessionBuilder.FormatTestEnvVar -import ai.chronon.spark.TableUtils -import ai.chronon.spark.format.{DefaultFormatProvider, FormatProvider} +import ai.chronon.spark.catalog.TableUtils +import ai.chronon.spark.catalog.{DefaultFormatProvider, FormatProvider} import ai.chronon.spark.test.TestUtils.makeDf import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.DataFrame diff --git a/spark/src/test/scala/ai/chronon/spark/test/TableUtilsTest.scala b/spark/src/test/scala/ai/chronon/spark/test/TableUtilsTest.scala index 64ae263765..3b5665cf69 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/TableUtilsTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/TableUtilsTest.scala @@ -18,17 +18,14 @@ package ai.chronon.spark.test import ai.chronon.api._ import ai.chronon.spark._ +import ai.chronon.spark.catalog.{DefaultFormatProvider, FormatProvider, IncompatibleSchemaException} import ai.chronon.spark.test.TestUtils.makeDf import org.apache.hadoop.hive.ql.exec.UDF -import org.apache.spark.sql.{Row, _} +import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.functions.col -import org.junit.Assert.{assertEquals, assertTrue} +import org.apache.spark.sql.{Row, _} +import org.junit.Assert.{assertEquals, assertNull, assertTrue} import org.scalatest.flatspec.AnyFlatSpec -import org.junit.Assert.assertNull - -import ai.chronon.spark.format.FormatProvider -import ai.chronon.spark.format.DefaultFormatProvider -import org.apache.spark.sql.catalyst.parser.ParseException import scala.util.Try diff --git a/spark/src/test/scala/ai/chronon/spark/test/TestUtils.scala b/spark/src/test/scala/ai/chronon/spark/test/TestUtils.scala index c43ca1ab51..b7ace29405 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/TestUtils.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/TestUtils.scala @@ -22,7 +22,7 @@ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ import ai.chronon.online.serde.SparkConversions import ai.chronon.spark.Extensions._ -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.Row import org.apache.spark.sql.SparkSession diff --git a/spark/src/test/scala/ai/chronon/spark/test/analyzer/AnalyzerTest.scala b/spark/src/test/scala/ai/chronon/spark/test/analyzer/AnalyzerTest.scala index 5c3b95042c..da823882fc 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/analyzer/AnalyzerTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/analyzer/AnalyzerTest.scala @@ -20,7 +20,8 @@ import ai.chronon.aggregator.test.Column import ai.chronon.api import ai.chronon.api._ import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{Analyzer, Join, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.catalog.TableUtils +import ai.chronon.spark.{Analyzer, Join, SparkSessionBuilder} import ai.chronon.spark.test.DataFrameGen import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{col, lit} diff --git a/spark/src/test/scala/ai/chronon/spark/test/analyzer/DerivationTest.scala b/spark/src/test/scala/ai/chronon/spark/test/analyzer/DerivationTest.scala index 3e1cf21210..6de548bfb6 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/analyzer/DerivationTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/analyzer/DerivationTest.scala @@ -23,6 +23,7 @@ import ai.chronon.api._ import ai.chronon.online.fetcher.Fetcher.Request import ai.chronon.spark.Extensions.DataframeOps import ai.chronon.spark._ +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.bootstrap.BootstrapUtils import ai.chronon.spark.test.{OnlineUtils, SchemaEvolutionUtils} import ai.chronon.spark.utils.MockApi diff --git a/spark/src/test/scala/ai/chronon/spark/test/batch/LabelJoinV2Test.scala b/spark/src/test/scala/ai/chronon/spark/test/batch/LabelJoinV2Test.scala index 73cc6f8c48..29b4e725f5 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/batch/LabelJoinV2Test.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/batch/LabelJoinV2Test.scala @@ -5,13 +5,13 @@ import ai.chronon.api import ai.chronon.api.Extensions._ import ai.chronon.api._ import ai.chronon.spark.Extensions._ +import ai.chronon.spark.batch._ import ai.chronon.spark.test.{DataFrameGen, TableTestUtils} import ai.chronon.spark.{GroupBy, Join, _} import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals import org.scalatest.flatspec.AnyFlatSpec import org.slf4j.LoggerFactory -import ai.chronon.spark.batch._ class LabelJoinV2Test extends AnyFlatSpec { @transient private lazy val logger = LoggerFactory.getLogger(getClass) diff --git a/spark/src/test/scala/ai/chronon/spark/test/batch/ModularJoinTest.scala b/spark/src/test/scala/ai/chronon/spark/test/batch/ModularJoinTest.scala index 98f6544d00..b51b1cef7a 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/batch/ModularJoinTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/batch/ModularJoinTest.scala @@ -18,6 +18,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions._ import org.junit.Assert._ import org.scalatest.flatspec.AnyFlatSpec +import ai.chronon.spark.catalog.TableUtils class ModularJoinTest extends AnyFlatSpec { diff --git a/spark/src/test/scala/ai/chronon/spark/test/bootstrap/LogBootstrapTest.scala b/spark/src/test/scala/ai/chronon/spark/test/bootstrap/LogBootstrapTest.scala index 18ec9d1a67..16034cb6d7 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/bootstrap/LogBootstrapTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/bootstrap/LogBootstrapTest.scala @@ -24,7 +24,7 @@ import ai.chronon.spark.Comparison import ai.chronon.spark.Extensions._ import ai.chronon.spark.LogFlattenerJob import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.OnlineUtils import ai.chronon.spark.test.SchemaEvolutionUtils import ai.chronon.spark.utils.MockApi diff --git a/spark/src/test/scala/ai/chronon/spark/test/bootstrap/TableBootstrapTest.scala b/spark/src/test/scala/ai/chronon/spark/test/bootstrap/TableBootstrapTest.scala index a7b6e7ad7c..67b8bdbc1f 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/bootstrap/TableBootstrapTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/bootstrap/TableBootstrapTest.scala @@ -22,7 +22,7 @@ import ai.chronon.api._ import ai.chronon.spark.Comparison import ai.chronon.spark.Extensions._ import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions._ diff --git a/spark/src/test/scala/ai/chronon/spark/test/fetcher/ChainingFetcherTest.scala b/spark/src/test/scala/ai/chronon/spark/test/fetcher/ChainingFetcherTest.scala index 1fb374304a..9d3eed8d06 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/fetcher/ChainingFetcherTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/fetcher/ChainingFetcherTest.scala @@ -21,6 +21,8 @@ import ai.chronon.api.Constants.MetadataDataset import ai.chronon.api.Extensions.{JoinOps, MetadataOps} import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ +import ai.chronon.spark.catalog.TableUtils + import ai.chronon.online.fetcher.Fetcher.Request import ai.chronon.online.fetcher.{FetchContext, MetadataStore} import ai.chronon.online.serde.SparkConversions diff --git a/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTest.scala b/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTest.scala index c6f3617698..8344c4e490 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTest.scala @@ -22,6 +22,8 @@ import ai.chronon.api.Constants.MetadataDataset import ai.chronon.api.Extensions.{JoinOps, MetadataOps} import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ +import ai.chronon.spark.catalog.TableUtils + import ai.chronon.online.KVStore.GetRequest import ai.chronon.online._ import ai.chronon.online.fetcher.FetchContext diff --git a/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTestUtil.scala b/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTestUtil.scala index 90432ad504..cbffde71dd 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTestUtil.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTestUtil.scala @@ -1,8 +1,9 @@ package ai.chronon.spark.test.fetcher import ai.chronon.api.ScalaJavaConversions._ -import ai.chronon.online.fetcher.Fetcher.{Request, Response} import ai.chronon.online._ +import ai.chronon.online.fetcher.Fetcher.{Request, Response} +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.utils.MockApi import ai.chronon.spark.{Join => _, _} import org.apache.spark.sql.{DataFrame, SparkSession} diff --git a/spark/src/test/scala/ai/chronon/spark/test/fetcher/JavaFetchTypesTest.java b/spark/src/test/scala/ai/chronon/spark/test/fetcher/JavaFetchTypesTest.java index 3d3639be54..d22a1d7b4a 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/fetcher/JavaFetchTypesTest.java +++ b/spark/src/test/scala/ai/chronon/spark/test/fetcher/JavaFetchTypesTest.java @@ -20,7 +20,7 @@ import ai.chronon.online.JavaRequest; import ai.chronon.online.JavaResponse; import ai.chronon.online.fetcher.FetchTypes; -import ai.chronon.spark.TableUtils; +import ai.chronon.spark.catalog.TableUtils; import ai.chronon.spark.SparkSessionBuilder; import ai.chronon.spark.utils.InMemoryKvStore; import ai.chronon.spark.utils.MockApi; diff --git a/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByTest.scala b/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByTest.scala index 34d148351b..87d3098479 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByTest.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.{Encoders, Row, SparkSession} import org.apache.spark.sql.types.{StructField, StructType, LongType => SparkLongType, StringType => SparkStringType} import org.junit.Assert._ import org.scalatest.flatspec.AnyFlatSpec +import ai.chronon.spark.catalog.TableUtils import scala.collection.mutable diff --git a/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByUploadTest.scala b/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByUploadTest.scala index 5dffb53187..8550e138f5 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByUploadTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/groupby/GroupByUploadTest.scala @@ -22,12 +22,14 @@ import ai.chronon.api.ScalaJavaConversions._ import ai.chronon.api._ import ai.chronon.online.fetcher.Fetcher import ai.chronon.spark.Extensions.DataframeOps -import ai.chronon.spark.{GroupByUpload, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{GroupByUpload, SparkSessionBuilder} import ai.chronon.spark.test.{DataFrameGen, OnlineUtils} import ai.chronon.spark.utils.MockApi import com.google.gson.Gson import org.apache.spark.sql.SparkSession import org.junit.Assert.assertEquals +import ai.chronon.spark.catalog.TableUtils + import org.scalatest.flatspec.AnyFlatSpec import org.slf4j.{Logger, LoggerFactory} diff --git a/spark/src/test/scala/ai/chronon/spark/test/join/FeatureWithLabelJoinTest.scala b/spark/src/test/scala/ai/chronon/spark/test/join/FeatureWithLabelJoinTest.scala index ea5c50c92c..1942751700 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/join/FeatureWithLabelJoinTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/join/FeatureWithLabelJoinTest.scala @@ -18,7 +18,8 @@ package ai.chronon.spark.test.join import ai.chronon.api.Extensions.{LabelPartsOps, MetadataOps} import ai.chronon.api._ -import ai.chronon.spark.{Comparison, LabelJoin, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{Comparison, LabelJoin, SparkSessionBuilder} +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.TestUtils import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.sql.functions.{max, min} diff --git a/spark/src/test/scala/ai/chronon/spark/test/join/JoinTest.scala b/spark/src/test/scala/ai/chronon/spark/test/join/JoinTest.scala index e7cd4b011e..a2388c043a 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/join/JoinTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/join/JoinTest.scala @@ -25,6 +25,7 @@ import ai.chronon.api.planner.RelevantLeftForJoinPart import ai.chronon.spark.Extensions._ import ai.chronon.spark._ import ai.chronon.spark.test.{DataFrameGen, TableTestUtils} +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.rdd.RDD import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SparkSession} import org.apache.spark.sql.functions._ diff --git a/spark/src/test/scala/ai/chronon/spark/test/join/JoinUtilsTest.scala b/spark/src/test/scala/ai/chronon/spark/test/join/JoinUtilsTest.scala index 2017f43676..4c2e235b41 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/join/JoinUtilsTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/join/JoinUtilsTest.scala @@ -22,7 +22,8 @@ import ai.chronon.api.{Builders, Constants, PartitionSpec} import ai.chronon.api.PartitionRange import ai.chronon.spark.Extensions._ import ai.chronon.spark.JoinUtils.{contains_any, set_add} -import ai.chronon.spark.{JoinUtils, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{JoinUtils, SparkSessionBuilder} +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.{DataFrameGen, TestUtils} import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Row, SparkSession} diff --git a/spark/src/test/scala/ai/chronon/spark/test/stats/drift/DriftTest.scala b/spark/src/test/scala/ai/chronon/spark/test/stats/drift/DriftTest.scala index 71d6b77c46..805ed1c778 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/stats/drift/DriftTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/stats/drift/DriftTest.scala @@ -12,7 +12,7 @@ import ai.chronon.observability.{DriftMetric, TileSummary, TileSummarySeries} import ai.chronon.online.KVStore import ai.chronon.online.stats.DriftStore import ai.chronon.spark.SparkSessionBuilder -import ai.chronon.spark.TableUtils +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.stats.drift.Summarizer import ai.chronon.spark.stats.drift.SummaryUploader import ai.chronon.spark.stats.drift.scripts.PrepareData diff --git a/spark/src/test/scala/ai/chronon/spark/test/streaming/AvroTest.scala b/spark/src/test/scala/ai/chronon/spark/test/streaming/AvroTest.scala index 88527a647b..26acbc6d92 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/streaming/AvroTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/streaming/AvroTest.scala @@ -19,7 +19,8 @@ package ai.chronon.spark.test.streaming import ai.chronon.aggregator.test.Column import ai.chronon.api._ import ai.chronon.spark.Extensions._ -import ai.chronon.spark.{Join, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{Join, SparkSessionBuilder} +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.DataFrameGen import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.col diff --git a/spark/src/test/scala/ai/chronon/spark/test/streaming/MutationsTest.scala b/spark/src/test/scala/ai/chronon/spark/test/streaming/MutationsTest.scala index aeb26cc008..c4948718f5 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/streaming/MutationsTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/streaming/MutationsTest.scala @@ -21,7 +21,8 @@ import ai.chronon.api import ai.chronon.api.{Builders, Operation, TimeUnit, TsUtils, Window} import ai.chronon.spark.Extensions._ import ai.chronon.spark.test.DataFrameGen -import ai.chronon.spark.{Comparison, Join, SparkSessionBuilder, TableUtils} +import ai.chronon.spark.{Comparison, Join, SparkSessionBuilder} +import ai.chronon.spark.catalog.TableUtils import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.scalatest.flatspec.AnyFlatSpec diff --git a/spark/src/test/scala/ai/chronon/spark/test/streaming/StreamingTest.scala b/spark/src/test/scala/ai/chronon/spark/test/streaming/StreamingTest.scala index fa950bfd64..cf0ff58773 100644 --- a/spark/src/test/scala/ai/chronon/spark/test/streaming/StreamingTest.scala +++ b/spark/src/test/scala/ai/chronon/spark/test/streaming/StreamingTest.scala @@ -22,6 +22,7 @@ import ai.chronon.api.Constants.MetadataDataset import ai.chronon.api._ import ai.chronon.online.fetcher.{FetchContext, MetadataStore} import ai.chronon.spark.Extensions._ +import ai.chronon.spark.catalog.TableUtils import ai.chronon.spark.test.{DataFrameGen, OnlineUtils} import ai.chronon.spark.utils.InMemoryKvStore import ai.chronon.spark.{Join => _, _}