DLT runtimes display fix and issue #182 (#186)

* Bug fix for display in DLT runtimes

* #182

* addressing the Tristan's comments.

* addressing the describe failure comment

* correcting the import statement in the init.py

* correcting the name of the column for issue #182 in both the scala and the python code

* adding the IPython NoneType handler condition

* changed the name to snake case

* streamling by removing ipython and namespace checks

Author: souvik-databricks

python/tempo/__init__.py

@@ -1,2 +1,2 @@
 from tempo.tsdf import TSDF
-from tempo.utils import display
+from tempo.utils import display

python/tempo/tsdf.py

@@ -324,6 +324,7 @@ def describe(self):
     global_smry_rec = desc_stats.limit(1).select(f.lit('global').alias("summary"),f.lit(unique_ts).alias("unique_ts_count"), f.lit(min_ts).alias("min_ts"), f.lit(max_ts).alias("max_ts"), f.lit(gran).alias("granularity"), *[f.lit(" ").alias(c) for c in non_summary_cols])
 
     full_smry = global_smry_rec.union(desc_stats)
+    full_smry = full_smry.withColumnRenamed("unique_ts_count","unique_time_series_count")
 
     try:
         dbutils.fs.ls("/")

python/tempo/utils.py

@@ -9,15 +9,15 @@
 
 logger = logging.getLogger(__name__)
 PLATFORM = "DATABRICKS" if "DB_HOME" in os.environ.keys() else "NON_DATABRICKS"
+
 """
 DB_HOME env variable has been chosen and that's because this variable is a special variable that will be available in DBR.
 
 This constant is to ensure the correct behaviour of the show and display methods are called based on the platform 
 where the code is running from. 
 """
 
-
-def __isnotebookenv():
+def __is_capable_of_html_rendering():
     """
     This method returns a boolean value signifying whether the environment is a notebook environment
     capable of rendering HTML or not.
@@ -54,31 +54,28 @@ def display_unavailable(df):
     logger.error("'display' method not available in this environment. Use 'show' method instead.")
 
 
-ENV_BOOLEAN = __isnotebookenv()
-
-if PLATFORM == "DATABRICKS":
-    method = get_ipython().user_ns['display']
+ENV_BOOLEAN = __is_capable_of_html_rendering()
 
 
+if (PLATFORM == "DATABRICKS") and (type(get_ipython()) != type(None)) and ('display' in get_ipython().user_ns.keys()):
+    method = get_ipython().user_ns['display']
     # Under 'display' key in user_ns the original databricks display method is present
     # to know more refer: /databricks/python_shell/scripts/db_ipykernel_launcher.py
     def display_improvised(obj):
         if type(obj).__name__ == 'TSDF':
             method(obj.df)
         else:
             method(obj)
-
-
     display = display_improvised
-elif __isnotebookenv():
+
+elif ENV_BOOLEAN:
     def display_html_improvised(obj):
         if type(obj).__name__ == 'TSDF':
             display_html(obj.df)
         else:
             display_html(obj)
-
-
     display = display_html_improvised
+
 else:
     display = display_unavailable
 

python/tests/tsdf_tests.py

@@ -156,7 +156,7 @@ def test_describe(self):
         # joined dataframe should equal the expected dataframe
         # self.assertDataFramesEqual(res, dfExpected)
         assert res.count() == 7
-        assert res.filter(F.col("unique_ts_count") != " ").select(F.max(F.col('unique_ts_count'))).collect()[0][
+        assert res.filter(F.col("unique_time_series_count") != " ").select(F.max(F.col('unique_time_series_count'))).collect()[0][
                    0] == "1"
         assert res.filter(F.col("min_ts") != " ").select(F.col('min_ts').cast("string")).collect()[0][
                    0] == '2020-08-01 00:00:10'

scala/tempo/src/main/scala/com/databrickslabs/tempo/TSDF.scala

@@ -453,6 +453,7 @@ private[tempo] sealed class BaseTSDF(val df: DataFrame,
 		val global_smry_rec = desc_stats.limit(1).select(non_summary_cols_blank: _*)
 
 		val full_smry = global_smry_rec.union(desc_stats)
+		val full_smry = full_smry.withColumnRenamed("unique_ts_count","unique_time_series_count")
 
     return(full_smry)
 	}

scala/tempo/src/test/scala/com/databrickslabs/tempo/DescribeTests.scala

@@ -42,7 +42,7 @@ class DescribeTests
 		// joined dataframe should equal the expected dataframe
 		// self.assertDataFramesEqual(res, dfExpected)
 		assert(res.count == 7)
-		assert(res.select(max(col("unique_ts_count"))).collect()(0)(0) == "1")
+		assert(res.select(max(col("unique_time_series_count"))).collect()(0)(0) == "1")
 		assert(res.select(col("min_ts").cast("string")).collect()(0)(0) == "2020-08-01 00:00:10")
 		assert(res.select(col("max_ts").cast("string")).collect()(0)(0) == "2020-09-01 00:19:12")
 	}