DLT runtimes display fix and issue #182

python/tempo/__init__.py

@@ -1,2 +1,10 @@
+from tempo.utils import __notdltruntime
+
 from tempo.tsdf import TSDF
-from tempo.utils import display
+if __notdltruntime():
+    from tempo.utils import display
+
+    '''
+    Conditonal import of display so that it doesn't gets imported in runtimes where display is not required.
+    For example in DATABRICKS Delta Live Tables Runtimes.
+    '''

python/tempo/tsdf.py

@@ -324,6 +324,7 @@ def describe(self):
     global_smry_rec = desc_stats.limit(1).select(f.lit('global').alias("summary"),f.lit(unique_ts).alias("unique_ts_count"), f.lit(min_ts).alias("min_ts"), f.lit(max_ts).alias("max_ts"), f.lit(gran).alias("granularity"), *[f.lit(" ").alias(c) for c in non_summary_cols])
 
     full_smry = global_smry_rec.union(desc_stats)
+    full_smry = full_smry.withColumnRenamed("unique_ts_count","unique_time_series_count")
 
     try:
         dbutils.fs.ls("/")

python/tempo/utils.py

@@ -17,6 +17,12 @@
 """
 
 
+def __notdltruntime():
+    user_namespace_keys = list(get_ipython().user_ns.keys())
+    check_bool = (('create_dlt_table_fn' not in user_namespace_keys) and ('dlt_sql_fn' not in user_namespace_keys))
+    return check_bool
+
+
 def __isnotebookenv():
     """
     This method returns a boolean value signifying whether the environment is a notebook environment
@@ -57,19 +63,25 @@ def display_unavailable(df):
 ENV_BOOLEAN = __isnotebookenv()
 
 if PLATFORM == "DATABRICKS":
-    method = get_ipython().user_ns['display']
 
+    # This below check is for ensuring compatibility with Databricks DLT runtimes
+    # This if logic ensures that the custom user's namespace of DLT runtimes
+    # which doesn't have PythonShell's Display object in the namespace doesn't result in an error.
+    if __notdltruntime():
+        method = get_ipython().user_ns['display']
 
-    # Under 'display' key in user_ns the original databricks display method is present
-    # to know more refer: /databricks/python_shell/scripts/db_ipykernel_launcher.py
-    def display_improvised(obj):
-        if type(obj).__name__ == 'TSDF':
-            method(obj.df)
-        else:
-            method(obj)
+        # Under 'display' key in user_ns the original databricks display method is present
+        # to know more refer: /databricks/python_shell/scripts/db_ipykernel_launcher.py
+        def display_improvised(obj):
+            if type(obj).__name__ == 'TSDF':
+                method(obj.df)
+            else:
+                method(obj)
 
 
-    display = display_improvised
+        display = display_improvised
+    else:
+        display = display_unavailable
 elif __isnotebookenv():
     def display_html_improvised(obj):
         if type(obj).__name__ == 'TSDF':

python/tests/tsdf_tests.py

@@ -156,7 +156,7 @@ def test_describe(self):
         # joined dataframe should equal the expected dataframe
         # self.assertDataFramesEqual(res, dfExpected)
         assert res.count() == 7
-        assert res.filter(F.col("unique_ts_count") != " ").select(F.max(F.col('unique_ts_count'))).collect()[0][
+        assert res.filter(F.col("unique_time_series_count") != " ").select(F.max(F.col('unique_time_series_count'))).collect()[0][
                    0] == "1"
         assert res.filter(F.col("min_ts") != " ").select(F.col('min_ts').cast("string")).collect()[0][
                    0] == '2020-08-01 00:00:10'

scala/tempo/src/main/scala/com/databrickslabs/tempo/TSDF.scala

@@ -453,6 +453,7 @@ private[tempo] sealed class BaseTSDF(val df: DataFrame,
 		val global_smry_rec = desc_stats.limit(1).select(non_summary_cols_blank: _*)
 
 		val full_smry = global_smry_rec.union(desc_stats)
+		val full_smry = full_smry.withColumnRenamed("unique_ts_count","unique_time_series_count")
 
     return(full_smry)
 	}

scala/tempo/src/test/scala/com/databrickslabs/tempo/DescribeTests.scala

@@ -42,7 +42,7 @@ class DescribeTests
 		// joined dataframe should equal the expected dataframe
 		// self.assertDataFramesEqual(res, dfExpected)
 		assert(res.count == 7)
-		assert(res.select(max(col("unique_ts_count"))).collect()(0)(0) == "1")
+		assert(res.select(max(col("unique_time_series_count"))).collect()(0)(0) == "1")
 		assert(res.select(col("min_ts").cast("string")).collect()(0)(0) == "2020-08-01 00:00:10")
 		assert(res.select(col("max_ts").cast("string")).collect()(0)(0) == "2020-09-01 00:19:12")
 	}