We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 2010a0b commit ff48144Copy full SHA for ff48144
spark/src/main/scala/ai/chronon/spark/TableUtils.scala
@@ -264,11 +264,12 @@ class TableUtils(@transient val sparkSession: SparkSession) extends Serializable
264
try {
265
// retrieve one row from the table
266
val partitionFilter = lastAvailablePartition(tableName).getOrElse(fallbackPartition)
267
- sparkSession.read
+ val df = sparkSession.read
268
.load(DataPointer.from(tableName, sparkSession))
269
.where(s"$partitionColumn='$partitionFilter'")
270
.limit(1)
271
- .collect()
+ df.cache
272
+ df.collect()
273
true
274
} catch {
275
case e: SparkException =>
0 commit comments