Etsy Patina / MMX shading reverts and other fixes (#544)

## Summary
We decided to push the bulk of the shading logic into a script on the
Etsy side. Shading our BT jars was painful as they were shading things
again so we decided to roll that back. We still have the two forked
bazel targets (cloud_gcp_lib and cloud_gcp_embedded_lib) with the
embedded one being used by Etsy.
Also made a couple of other updates:
* Update the thrift decoder class to use our vendored thrift -> A couple
of follow up AIs on the thrift front would be to: a) [add thrift version
guards while publishing
locally](https://linear.app/zipline-ai/issue/ZIP-651/add-thrift-version-guards-while-publishing-locally)
b) revisit using shading instead of vendoring our thrift (more long
term)
* Fetcher changes to use flag store to detect tiling / not for now - I
have a Flink PR upcoming that yanks out the untiled implementation and
can put up a follow up to drop the flag store references to use untiled
and default to tiled across the board.

## Checklist
- [ ] Added Unit Tests
- [X] Covered by existing CI
- [X] Integration tested
- [ ] Documentation update



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **Refactor**
- Updated dependency imports to use the official versions of Bigtable,
YAML, and Thrift libraries, ensuring improved stability and
compatibility.
- **New Features**
- Introduced a tiling configuration check for streaming requests,
providing more flexible data handling.
- Added a method to check if tiling is enabled in the FetchContext
class.
- **Chores**
- Streamlined build configurations by removing deprecated dependencies
and leveraging direct Maven artifacts, enabling clients to manage
specific dependency versions.
- Introduced a new flag store for managing tiling functionality during
tests.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: piyush-zlai

api/src/main/scala/ai/chronon/api/Extensions.scala

@@ -591,8 +591,6 @@ object Extensions {
       None
     }
 
-    def tilingFlag: Boolean = servingFlagValue("tiling").exists(_.toLowerCase() == "true")
-
     def dontThrowOnDecodeFailFlag: Boolean = servingFlagValue("decode.throw_on_fail").exists(_.toLowerCase() == "false")
 
     // build left streaming query for join source runner

api/src/test/scala/ai/chronon/api/test/ExtensionsTest.scala

@@ -117,30 +117,4 @@ class ExtensionsTest extends AnyFlatSpec {
     assertEquals(4, keys.size)
   }
 
-  it should "is tiling enabled" in {
-    def buildGroupByWithServingFlags(flags: Map[String, String] = null): GroupByOps = {
-
-      val execInfo: ExecutionInfo = if (flags != null) {
-        new ExecutionInfo()
-          .setConf(new ConfigProperties().setServing(flags.toJava))
-      } else {
-        null
-      }
-
-      Builders.GroupBy(
-        metaData = Builders.MetaData(name = "featureGroupName", executionInfo = execInfo)
-      )
-
-    }
-
-    // customJson not set defaults to false
-    assertFalse(buildGroupByWithServingFlags().tilingFlag)
-    assertFalse(buildGroupByWithServingFlags(Map.empty).tilingFlag)
-
-    val trueGb = buildGroupByWithServingFlags(Map("tiling" -> "true"))
-    assertTrue(trueGb.tilingFlag)
-    assertFalse(buildGroupByWithServingFlags(Map("tiling" -> "false")).tilingFlag)
-    assertFalse(buildGroupByWithServingFlags(Map("tiling" -> "invalid")).tilingFlag)
-
-  }
 }

cloud_gcp/BUILD.bazel

@@ -5,11 +5,6 @@ shared_deps = [
     "//online:lib",
     "//spark:lib",
     "//tools/build_rules/spark:spark-exec",
-    ":shaded_bigtable",
-    ":shaded_bigtable_proto",
-    ":shaded_bigtable_admin_proto",
-    ":shaded_grpc_bigtable",
-    ":shaded_snakeyaml",
     maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
     maven_artifact_with_suffix("org.json4s:json4s-core"),
     maven_artifact_with_suffix("org.json4s:json4s-jackson"),
@@ -18,6 +13,7 @@ shared_deps = [
     maven_artifact_with_suffix("org.rogach:scallop"),
     maven_artifact("com.google.cloud:google-cloud-core"),
     maven_artifact("com.google.cloud:google-cloud-bigquery"),
+    maven_artifact("com.google.cloud:google-cloud-bigtable"),
     maven_artifact("com.google.cloud:google-cloud-pubsub"),
     maven_artifact("com.google.cloud:google-cloud-dataproc"),
     maven_artifact("com.google.cloud.bigdataoss:gcsio"),
@@ -31,8 +27,10 @@ shared_deps = [
     maven_artifact("com.google.api:gax"),
     maven_artifact("com.google.guava:guava"),
     maven_artifact("com.google.protobuf:protobuf-java"),
+    maven_artifact("org.yaml:snakeyaml"),
     maven_artifact("io.grpc:grpc-netty-shaded"),
     maven_artifact("org.slf4j:slf4j-api"),
+    maven_artifact("ch.qos.reload4j:reload4j"),
     maven_artifact("org.threeten:threetenbp"),
     maven_artifact("org.apache.kafka:kafka-clients"),
     maven_artifact("com.google.cloud.spark:spark-3.5-bigquery"),

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreImpl.scala

@@ -22,16 +22,16 @@ import com.google.cloud.bigquery.Job
 import com.google.cloud.bigquery.JobId
 import com.google.cloud.bigquery.JobInfo
 import com.google.cloud.bigquery.QueryJobConfiguration
-import com.google.cloud.shaded_bigtable.admin.v2.BigtableTableAdminClient
-import com.google.cloud.shaded_bigtable.admin.v2.models.CreateTableRequest
-import com.google.cloud.shaded_bigtable.admin.v2.models.GCRules
-import com.google.cloud.shaded_bigtable.data.v2.BigtableDataClient
-import com.google.cloud.shaded_bigtable.data.v2.models.Filters
-import com.google.cloud.shaded_bigtable.data.v2.models.Query
-import com.google.cloud.shaded_bigtable.data.v2.models.Range.ByteStringRange
-import com.google.cloud.shaded_bigtable.data.v2.models.Range.TimestampRange
-import com.google.cloud.shaded_bigtable.data.v2.models.RowMutation
-import com.google.cloud.shaded_bigtable.data.v2.models.{TableId => BTTableId}
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient
+import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest
+import com.google.cloud.bigtable.admin.v2.models.GCRules
+import com.google.cloud.bigtable.data.v2.BigtableDataClient
+import com.google.cloud.bigtable.data.v2.models.Filters
+import com.google.cloud.bigtable.data.v2.models.Query
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange
+import com.google.cloud.bigtable.data.v2.models.Range.TimestampRange
+import com.google.cloud.bigtable.data.v2.models.RowMutation
+import com.google.cloud.bigtable.data.v2.models.{TableId => BTTableId}
 import com.google.protobuf.ByteString
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/DataprocSubmitter.scala

@@ -9,7 +9,7 @@ import com.google.api.gax.rpc.ApiException
 import com.google.cloud.dataproc.v1._
 import org.json4s._
 import org.json4s.jackson.JsonMethods._
-import org.yaml.shaded_snakeyaml.Yaml
+import org.yaml.snakeyaml.Yaml
 
 import scala.io.Source
 

cloud_gcp/src/main/scala/ai/chronon/integrations/cloud_gcp/GcpApiImpl.scala

@@ -11,11 +11,11 @@ import ai.chronon.online.Serde
 import ai.chronon.online.serde.AvroSerde
 import com.google.api.gax.core.NoCredentialsProvider
 import com.google.cloud.bigquery.BigQueryOptions
-import com.google.cloud.shaded_bigtable.admin.v2.BigtableTableAdminClient
-import com.google.cloud.shaded_bigtable.admin.v2.BigtableTableAdminSettings
-import com.google.cloud.shaded_bigtable.data.v2.BigtableDataClient
-import com.google.cloud.shaded_bigtable.data.v2.BigtableDataSettings
-import com.google.cloud.shaded_bigtable.data.v2.stub.metrics.NoopMetricsProvider
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings
+import com.google.cloud.bigtable.data.v2.BigtableDataClient
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings
+import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider
 
 import java.util
 

cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreTest.scala

@@ -10,13 +10,13 @@ import com.google.api.core.ApiFutures
 import com.google.api.gax.core.NoCredentialsProvider
 import com.google.api.gax.rpc.ServerStreamingCallable
 import com.google.api.gax.rpc.UnaryCallable
-import com.google.cloud.shaded_bigtable.admin.v2.BigtableTableAdminClient
-import com.google.cloud.shaded_bigtable.admin.v2.BigtableTableAdminSettings
-import com.google.cloud.shaded_bigtable.data.v2.BigtableDataClient
-import com.google.cloud.shaded_bigtable.data.v2.BigtableDataSettings
-import com.google.cloud.shaded_bigtable.data.v2.models.Query
-import com.google.cloud.shaded_bigtable.data.v2.models.Row
-import com.google.cloud.shaded_bigtable.data.v2.models.RowMutation
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings
+import com.google.cloud.bigtable.data.v2.BigtableDataClient
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings
+import com.google.cloud.bigtable.data.v2.models.Query
+import com.google.cloud.bigtable.data.v2.models.Row
+import com.google.cloud.bigtable.data.v2.models.RowMutation
 import com.google.cloud.bigtable.emulator.v2.Emulator
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito.when

online/src/main/java/ai/chronon/online/ThriftDecoder.java

@@ -17,11 +17,11 @@
 package ai.chronon.online;
 
 import ai.chronon.api.DataType;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TFieldIdEnum;
-import org.apache.thrift.meta_data.FieldMetaData;
-import org.apache.thrift.meta_data.StructMetaData;
-import org.apache.thrift.protocol.TType;
+import ai.chronon.api.thrift.TBase;
+import ai.chronon.api.thrift.TFieldIdEnum;
+import ai.chronon.api.thrift.meta_data.FieldMetaData;
+import ai.chronon.api.thrift.meta_data.StructMetaData;
+import ai.chronon.api.thrift.protocol.TType;
 
 import java.io.Serializable;
 import java.nio.ByteBuffer;

online/src/main/scala/ai/chronon/online/fetcher/FetchContext.scala

@@ -16,4 +16,10 @@ case class FetchContext(kvStore: KVStore,
   def getOrCreateExecutionContext: ExecutionContext = {
     Option(executionContextOverride).getOrElse(FlexibleExecutionContext.buildExecutionContext)
   }
+
+  def isTilingEnabled: Boolean = {
+    Option(flagStore)
+      .map(_.isSet(FlagStoreConstants.TILING_ENABLED, Map.empty[String, String].toJava))
+      .exists(_.asInstanceOf[Boolean])
+  }
 }

online/src/main/scala/ai/chronon/online/fetcher/GroupByFetcher.scala

@@ -86,7 +86,7 @@ class GroupByFetcher(fetchContext: FetchContext, metadataStore: MetadataStore)
         case Accuracy.TEMPORAL =>
           // Build a tile key for the streaming request
           // When we build support for layering, we can expand this out into a utility that builds n tile keys for n layers
-          val keyBytes = if (groupByServingInfo.groupByOps.tilingFlag) {
+          val keyBytes = if (fetchContext.isTilingEnabled) {
 
             val tileKey = TilingUtils.buildTileKey(
               groupByServingInfo.groupByOps.streamingDataset,

online/src/main/scala/ai/chronon/online/fetcher/GroupByResponseHandler.scala

@@ -112,7 +112,7 @@ class GroupByResponseHandler(fetchContext: FetchContext, metadataStore: Metadata
     val batchIr: FinalBatchIr =
       getBatchIrFromBatchResponse(batchResponses, batchBytes, servingInfo, toBatchIr, requestContext.keys)
 
-    if (servingInfo.groupByOps.tilingFlag) {
+    if (fetchContext.isTilingEnabled) {
       mergeTiledIrsFromStreaming(requestContext.queryTimeMs, servingInfo, streamingResponses, aggregator, batchIr)
     } else {
       mergeRawEventsFromStreaming(requestContext.queryTimeMs,

spark/BUILD.bazel

@@ -12,7 +12,6 @@ scala_library(
         "//api:thrift_java",
         "//online:lib",
         "//tools/build_rules/spark:spark-exec",
-        ":shaded_snakeyaml",
         maven_artifact("com.fasterxml.jackson.core:jackson-core"),
         maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
         maven_artifact_with_suffix("com.fasterxml.jackson.module:jackson-module-scala"),
@@ -34,6 +33,7 @@ scala_library(
         maven_artifact("org.apache.datasketches:datasketches-java"),
         maven_artifact_with_suffix("org.rogach:scallop"),
         maven_artifact("io.netty:netty-all"),
+        maven_artifact("org.yaml:snakeyaml"),
         maven_artifact("io.netty:netty-transport"),
         maven_artifact("io.netty:netty-handler"),
         maven_artifact("io.netty:netty-buffer"),

spark/src/main/scala/ai/chronon/spark/Driver.scala

@@ -52,7 +52,7 @@ import org.rogach.scallop.ScallopOption
 import org.rogach.scallop.Subcommand
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
-import org.yaml.shaded_snakeyaml.Yaml
+import org.yaml.snakeyaml.Yaml
 
 import java.io.File
 import java.nio.file.Files

spark/src/test/scala/ai/chronon/spark/test/fetcher/FetcherTest.scala

@@ -559,12 +559,9 @@ class FetcherTest extends AnyFlatSpec {
         name = "unit_test/fetcher_tiled_gb",
         namespace = namespace,
         team = "chronon",
-        customJson = groupByCustomJson.orNull,
-        executionInfo = new ExecutionInfo()
-          .setConf(new ConfigProperties().setServing(Map("tiling" -> "true").toJava))
+        customJson = groupByCustomJson.orNull
       )
     )
-    groupBy.tilingFlag shouldBe true
 
     val joinConf = Builders.Join(
       left = leftSource,
@@ -586,7 +583,18 @@ class FetcherTest extends AnyFlatSpec {
     val kvStoreFunc = () => OnlineUtils.buildInMemoryKVStore("FetcherTest")
     val inMemoryKvStore = kvStoreFunc()
 
+    val tilingEnabledFlagStore = new FlagStore {
+      override def isSet(flagName: String, attributes: util.Map[String, String]): lang.Boolean = {
+        if (flagName == FlagStoreConstants.TILING_ENABLED) {
+          enableTiling
+        } else {
+          false
+        }
+      }
+    }
+
     val mockApi = new MockApi(kvStoreFunc, namespace)
+    mockApi.setFlagStore(tilingEnabledFlagStore)
 
     val joinedDf = new ai.chronon.spark.Join(joinConf, endDs, tableUtils).computeJoin()
     val joinTable = s"$namespace.join_test_expected_${joinConf.metaData.cleanName}"

tools/build_rules/cloud_gcp_embedded/BUILD

@@ -8,9 +8,5 @@ java_binary(
         maven_artifact("org.yaml:snakeyaml"),
         # pull out some slf4j-impl dependencies - these can be included at the application deploy target level if needed
         maven_artifact("org.apache.logging.log4j:log4j-slf4j2-impl"),
-        # Exclude grpc deps as clients might want to use their own version
-        maven_artifact("io.grpc:grpc-core:1.69.0"),
-        maven_artifact("io.grpc:grpc-inprocess:1.69.0"),
-        maven_artifact("io.grpc:grpc-netty-shaded"),
     ],
 )