Skip to content

Commit 8fd9917

Browse files
authored
Magic double values to support nulls in arrays of double in thrift (#279)
## Summary we cannot represent absent data in time series with null values because the thrift serializer doesn't allow nulls in list<double> - so we create a magic double value (based on string "chronon") to represent nulls ## Checklist - [x] Added Unit Tests - [ ] Covered by existing CI - [ ] Integration tested - [ ] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes - **New Features** - Introduced a new constant `magicNullDouble` for handling null value representations - **Bug Fixes** - Improved null value handling in serialization and data processing workflows - **Tests** - Added new test case for `TileDriftSeries` serialization to validate null value management The changes enhance data processing consistency and provide a standardized approach to managing null values across the system. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 0478853 commit 8fd9917

File tree

3 files changed

+45
-1
lines changed

3 files changed

+45
-1
lines changed

api/src/main/scala/ai/chronon/api/Constants.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,4 +76,13 @@ object Constants {
7676

7777
val extensionsToIgnore: Array[String] = Array(".class", ".csv", ".java", ".scala", ".py", ".DS_Store")
7878
val foldersToIgnore: Array[String] = Array(".git")
79+
80+
// import base64
81+
// text_bytes = "chronon".encode('utf-8')
82+
// base64_str = base64.b64encode(text_bytes)
83+
// int.from_bytes(base64.b64decode(base64_str), "big")
84+
//
85+
// output: 27980863399423854
86+
87+
val magicNullDouble: java.lang.Double = -27980863399423854.0
7988
}
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package ai.chronon.api.test
2+
3+
import ai.chronon.api.Constants
4+
import ai.chronon.api.ScalaJavaConversions.JListOps
5+
import ai.chronon.api.ThriftJsonCodec
6+
import ai.chronon.observability.TileDriftSeries
7+
import org.scalatest.flatspec.AnyFlatSpec
8+
import org.scalatest.matchers.should.Matchers
9+
10+
import java.lang.{Double => JDouble}
11+
12+
class TileSeriesSerializationTest extends AnyFlatSpec with Matchers {
13+
14+
"TileDriftSeries" should "serialize with nulls" in {
15+
val tileDriftSeries = new TileDriftSeries()
16+
17+
val percentileDrifts: Seq[JDouble] = Seq(0.1, null, 0.3, null, 0.5, null, 0.7, null, 0.9)
18+
.map(v =>
19+
if (v == null)
20+
Constants.magicNullDouble
21+
else
22+
v.asInstanceOf[JDouble]
23+
)
24+
25+
val percentileDriftsList: java.util.List[JDouble] = percentileDrifts.toJava
26+
tileDriftSeries.setPercentileDriftSeries(percentileDriftsList)
27+
28+
val jsonStr = ThriftJsonCodec.toJsonStr(tileDriftSeries)
29+
30+
jsonStr should be ("""{"percentileDriftSeries":[0.1,-2.7980863399423856E16,0.3,-2.7980863399423856E16,0.5,-2.7980863399423856E16,0.7,-2.7980863399423856E16,0.9]}""")
31+
}
32+
33+
34+
}

online/src/main/scala/ai/chronon/online/stats/PivotUtils.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package ai.chronon.online.stats
22

3+
import ai.chronon.api.Constants
34
import ai.chronon.observability.TileDrift
45
import ai.chronon.observability.TileDriftSeries
56
import ai.chronon.observability.TileSummary
@@ -133,7 +134,7 @@ object PivotUtils {
133134
if (isSetFunc(drift)) {
134135
JDouble.valueOf(extract(drift))
135136
} else {
136-
null
137+
Constants.magicNullDouble
137138
}
138139
}
139140
}

0 commit comments

Comments
 (0)