Magic double values to support nulls in arrays of double in thrift

api/src/main/scala/ai/chronon/api/Constants.scala

@@ -76,4 +76,13 @@ object Constants {
 
   val extensionsToIgnore: Array[String] = Array(".class", ".csv", ".java", ".scala", ".py", ".DS_Store")
   val foldersToIgnore: Array[String] = Array(".git")
+
+  // import base64
+  // text_bytes = "chronon".encode('utf-8')
+  // base64_str = base64.b64encode(text_bytes)
+  // int.from_bytes(base64.b64decode(base64_str), "big")
+  //
+  // output: 27980863399423854
+
+  val magicNullDouble: java.lang.Double = -27980863399423854.0
 }

api/src/test/scala/ai/chronon/api/test/TileSeriesSerializationTest.scala

@@ -0,0 +1,34 @@
+package ai.chronon.api.test
+
+import ai.chronon.api.Constants
+import ai.chronon.api.ScalaJavaConversions.JListOps
+import ai.chronon.api.ThriftJsonCodec
+import ai.chronon.observability.TileDriftSeries
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.lang.{Double => JDouble}
+
+class TileSeriesSerializationTest extends AnyFlatSpec with Matchers {
+
+  "TileDriftSeries" should "serialize with nulls" in {
+    val tileDriftSeries = new TileDriftSeries()
+
+    val percentileDrifts: Seq[JDouble] = Seq(0.1, null, 0.3, null, 0.5, null, 0.7, null, 0.9)
+      .map(v =>
+        if (v == null)
+          Constants.magicNullDouble
+        else
+          v.asInstanceOf[JDouble]
+      )
+
+    val percentileDriftsList: java.util.List[JDouble] = percentileDrifts.toJava
+    tileDriftSeries.setPercentileDriftSeries(percentileDriftsList)
+
+    val jsonStr = ThriftJsonCodec.toJsonStr(tileDriftSeries)
+
+    jsonStr should be ("""{"percentileDriftSeries":[0.1,-2.7980863399423856E16,0.3,-2.7980863399423856E16,0.5,-2.7980863399423856E16,0.7,-2.7980863399423856E16,0.9]}""")
+  }
+
+
+}

online/src/main/scala/ai/chronon/online/stats/PivotUtils.scala

@@ -1,5 +1,6 @@
 package ai.chronon.online.stats
 
+import ai.chronon.api.Constants
 import ai.chronon.observability.TileDrift
 import ai.chronon.observability.TileDriftSeries
 import ai.chronon.observability.TileSummary
@@ -133,7 +134,7 @@ object PivotUtils {
         if (isSetFunc(drift)) {
           JDouble.valueOf(extract(drift))
         } else {
-          null
+          Constants.magicNullDouble
         }
       }
     }