Bazel migration for spark module (#196)

## Summary
Migrated spark module to bazel.

Known issue with tests : JUnit and Scala tests using FunSuite are not
getting identified in bazel scala_test_suite. Thanks to @nikhil-zlai for
working on the script to modify our tests to use Scala FlatSpec api
which seems to be working with bazel. Will test again after his changes
and create a separate PR if additional changes are needed. Also, it's
good to be consistent with the unit tests api going forward unless we
really need others.

## Checklist
- [ ] Added Unit Tests
- [x] Covered by existing CI
- [ ] Integration tested
- [ ] Documentation update



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **Dependencies**
- Updated logging framework dependency to Log4j 2.x across various
libraries.
	- Updated ScalaTest dependencies in API testing.
	- Migrated library dependencies from Maven to Scala artifacts.
	- Updated Jackson, JSON4s, and other library versions.
	- Added new dependencies for Guava, Netty, Delta Spark, and Kafka.

- **Build Configuration**
	- Introduced new Scala library and test configurations.
	- Added JVM binary assembly configuration.
	- Refined Bazel build rule imports and loading.

- **Dependency Management**
	- Removed several unused dependencies.
	- Simplified dependency references.
	- Updated runtime dependency paths.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: kumar-zlai

aggregator/BUILD.bazel

@@ -12,7 +12,7 @@ scala_library(
         maven_artifact("org.apache.datasketches:datasketches-java"),
         maven_artifact("org.apache.commons:commons-lang3"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         scala_artifact("org.scala-lang.modules:scala-collection-compat"),
     ],
 )
@@ -28,7 +28,7 @@ scala_library(
         maven_artifact("junit:junit"),
         maven_artifact("com.novocode:junit-interface"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("com.google.code.gson:gson"),
         maven_artifact("org.apache.datasketches:datasketches-memory"),
         maven_artifact("org.apache.datasketches:datasketches-java"),
@@ -50,7 +50,7 @@ scala_test_suite(
        maven_artifact("junit:junit"),
        maven_artifact("com.novocode:junit-interface"),
        maven_artifact("org.slf4j:slf4j-api"),
-       maven_artifact("org.slf4j:slf4j-log4j12"),
+       maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
        maven_artifact("com.google.code.gson:gson"),
        maven_artifact("org.apache.datasketches:datasketches-memory"),
        maven_artifact("org.apache.datasketches:datasketches-java"),

api/BUILD.bazel

@@ -10,7 +10,7 @@ scala_library(
         maven_artifact("com.fasterxml.jackson.core:jackson-core"),
         maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("org.apache.commons:commons-lang3"),
         maven_artifact("com.google.code.gson:gson"),
         scala_artifact("org.scala-lang.modules:scala-collection-compat"),
@@ -29,12 +29,15 @@ scala_test_suite(
         maven_artifact("com.fasterxml.jackson.core:jackson-core"),
         maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("junit:junit"),
         maven_artifact("com.novocode:junit-interface"),
         maven_artifact("org.mockito:mockito-core"),
         scala_artifact("org.scala-lang.modules:scala-parser-combinators"),
         scala_artifact("org.scala-lang.modules:scala-collection-compat"),
+        scala_artifact("org.scalatest:scalatest"),
+        scala_artifact("org.scalatest:scalatest-flatspec"),
+        scala_artifact("org.scalatest:scalatest-funsuite"),
     ],
 )
 
@@ -50,7 +53,7 @@ java_library(
     deps = [
         maven_artifact("javax.annotation:javax.annotation.api"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("org.apache.commons:commons-lang3"),
         maven_artifact("com.google.code.gson:gson"),
     ],

online/BUILD.bazel

@@ -13,16 +13,16 @@ scala_library(
         scala_artifact("org.json4s:json4s-ast"),
         scala_artifact("org.scala-lang.modules:scala-collection-compat"),
         maven_artifact("com.datadoghq:java-dogstatsd-client"),
-        maven_artifact("org.rogach:scallop_2_12"),
+        scala_artifact("org.rogach:scallop"),
         maven_artifact("net.jodah:typetools"),
         maven_artifact("com.github.ben-manes.caffeine:caffeine"),
         maven_artifact("com.fasterxml.jackson.core:jackson-core"),
         maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
-        maven_artifact("com.softwaremill.sttp.client3:core_2.12"),
-        maven_artifact("com.softwaremill.sttp.model:core_2.12"),
-        maven_artifact("com.softwaremill.sttp.shared:core_2.12"),
+        scala_artifact("com.softwaremill.sttp.client3:core"),
+        scala_artifact("com.softwaremill.sttp.model:core"),
+        scala_artifact("com.softwaremill.sttp.shared:core"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("com.google.code.gson:gson"),
     ],
 )
@@ -40,7 +40,7 @@ scala_library(
         # Libraries
         maven_artifact("com.github.ben-manes.caffeine:caffeine"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("com.google.code.gson:gson"),
         # Testing
         scala_artifact("org.scalatest:scalatest-matchers-core"),
@@ -74,7 +74,7 @@ scala_test_suite(
         maven_artifact("org.mockito:mockito-core"),
         scala_artifact("org.scala-lang.modules:scala-collection-compat"),
         maven_artifact("org.slf4j:slf4j-api"),
-        maven_artifact("org.slf4j:slf4j-log4j12"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("com.google.code.gson:gson"),
         maven_artifact("com.github.ben-manes.caffeine:caffeine"),
         maven_artifact("junit:junit"),

spark/BUILD.bazel

@@ -0,0 +1,149 @@
+scala_library(
+    name = "lib",
+    srcs = glob(["src/main/**/*.scala"]),
+    resources = [
+        "src/main/resources/log4j2.properties",
+    ],
+    deps = [
+        "//aggregator:lib",
+        "//api:lib",
+        "//api:thrift",
+        "//online:lib",
+        "//tools/build_rules/spark:spark-exec",
+        maven_artifact("com.fasterxml.jackson.core:jackson-core"),
+        maven_artifact("com.fasterxml.jackson.core:jackson-databind"),
+        scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"),
+        maven_artifact("com.google.guava:guava"),
+        maven_artifact("commons-io:commons-io"),
+        maven_artifact("commons-lang:commons-lang"),
+        maven_artifact("org.apache.kafka:kafka-clients"),
+        scala_artifact("org.json4s:json4s-core"),
+        scala_artifact("org.json4s:json4s-jackson"),
+        scala_artifact("org.json4s:json4s-ast"),
+        scala_artifact("org.scala-lang.modules:scala-collection-compat"),
+        scala_artifact("org.scala-lang.modules:scala-parser-combinators"),
+        maven_artifact("org.slf4j:slf4j-api"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
+        maven_artifact("com.google.code.gson:gson"),
+        maven_artifact("jakarta.servlet:jakarta.servlet-api"),
+        maven_artifact("org.yaml:snakeyaml"),
+        maven_artifact("org.apache.datasketches:datasketches-memory"),
+        maven_artifact("org.apache.datasketches:datasketches-java"),
+        scala_artifact("org.rogach:scallop"),
+        maven_artifact("io.netty:netty-all"),
+        maven_artifact("io.netty:netty-transport"),
+        maven_artifact("io.netty:netty-handler"),
+        maven_artifact("io.netty:netty-buffer"),
+        maven_artifact("io.netty:netty-codec-http"),
+        maven_artifact("io.netty:netty-common"),
+        maven_artifact("io.netty:netty-codec"),
+        scala_artifact("io.delta:delta-spark"),
+    ],
+)
+
+scala_library(
+    name = "test-lib",
+    srcs = glob(["src/test/**/*.scala"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":lib",
+        "//aggregator:lib",
+        "//aggregator:test-lib",
+        "//api:lib",
+        "//api:thrift",
+        "//online:lib",
+        "//tools/build_rules/spark:spark-exec",
+        # Library
+        maven_artifact("com.google.code.gson:gson"),
+        maven_artifact("com.google.guava:guava"),
+        scala_artifact("org.rogach:scallop"),
+        maven_artifact("commons.io:commons-io"),
+        maven_artifact("commons.lang:commons-lang"),
+        scala_artifact("org.scala-lang.modules:scala-java8-compat"),
+        scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"),
+        maven_artifact("org.slf4j:slf4j-api"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
+        scala_artifact("org.json4s:json4s-core"),
+        scala_artifact("org.json4s:json4s-jackson"),
+        scala_artifact("org.json4s:json4s-ast"),
+        maven_artifact("org.yaml:snakeyaml"),
+        # Testing
+        scala_artifact("org.scalatest:scalatest-matchers-core"),
+        scala_artifact("org.scalatest:scalatest-core"),
+        scala_artifact("org.scalatest:scalatest"),
+        scala_artifact("org.scalatest:scalatest-flatspec"),
+        scala_artifact("org.scalatest:scalatest-funsuite"),
+        scala_artifact("org.scalatest:scalatest-shouldmatchers"),
+        scala_artifact("org.scalactic:scalactic"),
+        scala_artifact("org.scalatestplus:mockito-3-4"),
+        maven_artifact("org.scalatest:scalatest-compatible"),
+        maven_artifact("junit:junit"),
+        maven_artifact("com.novocode:junit-interface"),
+        maven_artifact("org.mockito:mockito-core"),
+    ],
+)
+
+scala_test_suite(
+    name = "test",
+    srcs = glob(["src/test/**/*.scala"]),
+    visibility = ["//visibility:public"],
+    jvm_flags = [
+        "--add-opens=java.base/java.lang=ALL-UNNAMED",
+        "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
+        "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
+        "--add-opens=java.base/java.io=ALL-UNNAMED",
+        "--add-opens=java.base/java.net=ALL-UNNAMED",
+        "--add-opens=java.base/java.nio=ALL-UNNAMED",
+        "--add-opens=java.base/java.util=ALL-UNNAMED",
+        "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+        "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+        "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+        "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
+        "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
+        "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+    ],
+    deps = [
+        ":lib",
+        ":test-lib",
+        "//aggregator:lib",
+        "//aggregator:test-lib",
+        "//api:lib",
+        "//api:thrift",
+        "//online:lib",
+        "//tools/build_rules/spark:spark-exec",
+        # Library
+        maven_artifact("com.google.code.gson:gson"),
+        maven_artifact("com.google.guava:guava"),
+        scala_artifact("org.rogach:scallop"),
+        maven_artifact("commons.io:commons-io"),
+        maven_artifact("commons.lang:commons-lang"),
+        scala_artifact("org.scala-lang.modules:scala-java8-compat"),
+        scala_artifact("com.fasterxml.jackson.module:jackson-module-scala"),
+        maven_artifact("org.slf4j:slf4j-api"),
+        maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
+        scala_artifact("org.json4s:json4s-core"),
+        scala_artifact("org.json4s:json4s-jackson"),
+        scala_artifact("org.json4s:json4s-ast"),
+        maven_artifact("org.yaml:snakeyaml"),
+        # Testing
+        scala_artifact("org.scalatest:scalatest-matchers-core"),
+        scala_artifact("org.scalatest:scalatest-core"),
+        scala_artifact("org.scalatest:scalatest"),
+        scala_artifact("org.scalatest:scalatest-flatspec"),
+        scala_artifact("org.scalatest:scalatest-funsuite"),
+        scala_artifact("org.scalatest:scalatest-shouldmatchers"),
+        scala_artifact("org.scalactic:scalactic"),
+        scala_artifact("org.scalatestplus:mockito-3-4"),
+        maven_artifact("org.scalatest:scalatest-compatible"),
+        maven_artifact("junit:junit"),
+        maven_artifact("com.novocode:junit-interface"),
+        maven_artifact("org.mockito:mockito-core"),
+    ],
+)
+
+jvm_binary(
+    name = "assembly",
+    deploy_env = ["//tools/build_rules/spark:spark"],
+    main_class = "ai.chronon.spark.Driver",
+    runtime_deps = [":lib"],
+)

tools/build_rules/dependencies/maven_repository.bzl

@@ -23,45 +23,33 @@ maven_repository = repository(
         "org.scalactic:scalactic_2.12:3.2.15",
 
         # Add other dependencies
-        "org.slf4j:slf4j-api:1.7.30",
-        "org.slf4j:slf4j-log4j12:1.7.30",
+        "org.slf4j:slf4j-api:2.0.12",
+        "org.apache.logging.log4j:log4j-slf4j-impl:2.20.0",
         "org.apache.datasketches:datasketches-memory:3.0.2",
         "org.apache.datasketches:datasketches-java:6.1.1",
-        "com.fasterxml.jackson.core:jackson-core:2.12.5",
-        "com.fasterxml.jackson.core:jackson-databind:2.12.5",
+        "com.fasterxml.jackson.core:jackson-core:2.15.2",
+        "com.fasterxml.jackson.core:jackson-databind:2.15.2",
+        "com.fasterxml.jackson.module:jackson-module-scala_2.12:2.15.2",
         "com.google.code.gson:gson:2.10.1",
         "javax.annotation:javax.annotation-api:1.3.2",
         "com.datadoghq:java-dogstatsd-client:4.4.1",
         "org.rogach:scallop_2.12:5.1.0",
         "net.jodah:typetools:0.6.3",
         "com.github.ben-manes.caffeine:caffeine:3.1.8",
         "com.softwaremill.sttp.client3:core_2.12:3.9.7",
-        "org.json4s:json4s-jackson_2.12:3.6.12",
-        "org.json4s:json4s-core_2.12:3.6.12",
-        "org.json4s:json4s-ast_2.12:3.6.12",
+        "org.json4s:json4s-jackson_2.12:3.7.0-M11",
+        "org.json4s:json4s-core_2.12:3.7.0-M11",
+        "org.json4s:json4s-ast_2.12:3.7.0-M11",
+        "jakarta.servlet:jakarta.servlet-api:4.0.3",
+        "com.google.guava:guava:33.3.1-jre",
+        "org.yaml:snakeyaml:2.3",
+        "commons-io:commons-io:2.9.0",
+        "commons-lang:commons-lang:2.6",
+        "io.netty:netty-all:4.1.99.Final",
+        "io.delta:delta-spark_2.12:3.2.0",
 
-        # Flink
-        versioned_artifacts("1.17.0", [
-            "org.apache.flink:flink-clients",
-            "org.apache.flink:flink-connector-files",
-            "org.apache.flink:flink-connector-hive_2.12",
-            "org.apache.flink:flink-csv",
-            "org.apache.flink:flink-json",
-            "org.apache.flink:flink-metrics-core",
-            "org.apache.flink:flink-metrics-prometheus:jar",
-            "org.apache.flink:flink-orc",
-            "org.apache.flink:flink-parquet",
-            "org.apache.flink:flink-protobuf",
-            "org.apache.flink:flink-scala_2.12",
-            "org.apache.flink:flink-sql-gateway-api",
-            "org.apache.flink:flink-streaming-java",
-            "org.apache.flink:flink-streaming-scala_2.12",
-            "org.apache.flink:flink-table-api-java",
-            "org.apache.flink:flink-table-planner_2.12",
-            "org.apache.flink:flink-test-utils",
-            "org.apache.flink:flink-streaming-java:jar:tests",
-            "org.apache.flink:flink-metrics-dropwizard",
-        ]),
+        # Kafka
+        "org.apache.kafka:kafka-clients:3.9.0",
     ],
     overrides = {
     },

tools/build_rules/dependencies/spark_repository.bzl

@@ -10,26 +10,9 @@ spark_repository = repository(
         "org.apache.spark:spark-hive_2.12:3.5.1",
         "org.apache.spark:spark-streaming_2.12:3.5.1",
 
-        # Other dependencies
-        "org.apache.curator:apache-curator:2.12.0",
-        "com.esotericsoftware:kryo:5.1.1",
-        "com.yahoo.datasketches:sketches-core:0.13.4",
-        "com.yahoo.datasketches:memory:0.12.2",
-        "com.yahoo.datasketches:sketches-hive:0.13.0",
-        "org.apache.datasketches:datasketches-java:2.0.0",
-        "org.apache.datasketches:datasketches-memory:1.3.0",
-
-        # Kafka dependencies - only Scala 2.12
-        "org.apache.kafka:kafka_2.12:2.6.3",
-
-        # Avro dependencies
-        "org.apache.avro:avro:1.8.2",
-        "org.apache.avro:avro-mapred:1.8.2",
+        # Hive dependencies
         "org.apache.hive:hive-metastore:2.3.9",
-        "org.apache.hive:hive-exec:3.1.2",
-
-        # Monitoring
-        "io.prometheus.jmx:jmx_prometheus_javaagent:0.20.0",
+        "org.apache.hive:hive-exec:2.3.9",
     ],
     excluded_artifacts = [
         "org.pentaho:pentaho-aggdesigner-algorithm",

tools/build_rules/jvm_binary.bzl

@@ -0,0 +1,54 @@
+load("@rules_java//java:defs.bzl", "java_binary", "java_library")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_binary", "scala_library")
+
+def jvm_binary(
+        name,
+        srcs = [],
+        deps = [],
+        runtime_deps = [],
+        services = {},
+        tags = None,
+        main_class = None,
+        visibility = None,
+        create_executable = True,
+        testonly = None,
+        # All other flags are passed to java_binary
+        **kwargs):
+    has_scala_srcs = False
+    has_java_srcs = False
+    for src in srcs:
+        if src.endswith(".scala"):
+            has_scala_srcs = True
+        if src.endswith(".java"):
+            has_java_srcs = True
+    if has_scala_srcs and has_java_srcs:
+        fail("Cannot have scala and java sources in same jvm_binary")
+
+    lib_name = name + "_lib"
+    if has_scala_srcs:
+        scala_library(
+            name = lib_name,
+            srcs = srcs,
+            deps = deps,
+            runtime_deps = runtime_deps,
+            tags = tags,
+        )
+    else:
+        java_library(
+            name = lib_name,
+            srcs = srcs,
+            deps = deps,
+            runtime_deps = runtime_deps,
+            tags = tags,
+            testonly = testonly,
+        )
+
+    java_binary(
+        name = name,
+        runtime_deps = [lib_name],
+        tags = tags,
+        main_class = main_class,
+        create_executable = create_executable,
+        testonly = testonly,
+        **kwargs
+    )