wip

Co-authored-by: Thomas Chow <[email protected]>

Author: tchow-zlai

api/python/ai/chronon/resources/gcp/zipline-cli-install.sh

@@ -49,8 +49,8 @@ done
 
 gcloud storage cp "${ARTIFACT_PREFIX%/}/release/$VERSION/wheels/zipline_ai-$VERSION-py3-none-any.whl" .
 
+trap 'rm -f ./zipline_ai-$VERSION-py3-none-any.whl' EXIT
+
 pip3 uninstall zipline-ai
 
 pip3 install ./zipline_ai-$VERSION-py3-none-any.whl
-
-trap 'rm -f ./zipline_ai-$VERSION-py3-none-any.whl' EXIT

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/DelegatingBigQueryMetastoreCatalog.scala

@@ -24,6 +24,7 @@ import java.util
 import scala.jdk.CollectionConverters._
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import scala.util.{Failure, Success, Try}
+import org.slf4j.{Logger, LoggerFactory}
 
 /** Galactus catalog that allows us to interact with BigQuery metastore as a spark catalog. This allows for
   * querying of a variety of table types directly in spark sql or the dataframe api.
@@ -47,6 +48,8 @@ class DelegatingBigQueryMetastoreCatalog extends TableCatalog with SupportsNames
   @transient private lazy val icebergCatalog: SparkCatalog = new SparkCatalog()
   @transient private lazy val connectorCatalog: BigQueryCatalog = new BigQueryCatalog()
 
+  @transient protected lazy val logger: Logger = LoggerFactory.getLogger(getClass)
+
   private var catalogName: String =
     null // This corresponds to `spark_catalog in `spark.sql.catalog.spark_catalog`. This is necessary for spark to correctly choose which implementation to use.
 
@@ -118,14 +121,17 @@ class DelegatingBigQueryMetastoreCatalog extends TableCatalog with SupportsNames
               )
               fileBasedTable
             }
-            case _: StandardTableDefinition => {
+            case stTable: StandardTableDefinition => {
               //todo(tchow): Support partitioning
 
               // Hack because there's a bug in the BigQueryCatalog where they ignore the projectId.
               // See: https://github.com/GoogleCloudDataproc/spark-bigquery-connector/pull/1340
               // ideally it should be the below:
               // val connectorTable = connectorCatalog.loadTable(ident)
-              connectorCatalog.loadTable(Identifier.of(Array(tId.getDataset), tId.getTable))
+              val nativeTable = connectorCatalog.loadTable(Identifier.of(Array(tId.getDataset), tId.getTable))
+              logger.info("Table columns:")
+              logger.info(f"${nativeTable.columns()}")
+              nativeTable
             }
             case _ => throw new IllegalStateException(s"Cannot support table of type: ${table.getDefinition}")
           }