unified all artifacts to a single maven repository with pinned versions (#490)

## Summary
Unified all artifacts in to our single maven repository with pinned
versions expect for the 2 scala library artifacts whose versions need to
be determined dynamically based on config and cannot be pinned. This
should help with dependency conflicts across repos (mainly for maven and
spark we saw previously)

## Checklist
- [ ] Added Unit Tests
- [x] Covered by existing CI
- [ ] Integration tested
- [ ] Documentation update



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Chores**
- Streamlined internal build processes by consolidating dependency
management and updating repository configurations.
- Removed outdated dependency references to ensure a more consistent and
reliable build system.

- **Refactor**
- Adopted a unified Maven-based approach for artifact resolution,
enhancing version management across multiple components.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: kumar-zlai

WORKSPACE

@@ -4,7 +4,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 # Load scala version from the config
 load("//:scala_config.bzl", "scala_version")
+
 scala_version(name = "scala_config")
+
 load("@scala_config//:version.bzl", "SCALA_VERSION")
 
 # Contains useful bazel utility functions and rules
@@ -76,21 +78,28 @@ http_archive(
 
 # Initialize Scala with specific version support
 load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")
+
 scala_config(scala_version = SCALA_VERSION)
 
 load("@io_bazel_rules_scala//scala:scala.bzl", "scala_repositories")
+
 scala_repositories()
 
 load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
+
 scala_register_toolchains()
 
 load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", "scalatest_toolchain")
+
 scalatest_repositories()
+
 scalatest_toolchain()
 
 # For scalafmt
 load("@io_bazel_rules_scala//scala/scalafmt:scalafmt_repositories.bzl", "scalafmt_default_config", "scalafmt_repositories")
+
 scalafmt_default_config()
+
 scalafmt_repositories()
 
 # For Protobuf support
@@ -111,10 +120,9 @@ rules_proto_toolchains()
 
 # To load all dependencies used across our modules
 load("//tools/build_rules/dependencies:load_dependencies.bzl", "load_all_dependencies")
+
 load_all_dependencies()
 
 load("@maven//:defs.bzl", pinned_maven_repo_install = "pinned_maven_install")
-pinned_maven_repo_install()
 
-load("@spark//:defs.bzl", pinned_spark_repo_install = "pinned_maven_install")
-pinned_spark_repo_install()
+pinned_maven_repo_install()

aggregator/BUILD.bazel

@@ -17,7 +17,7 @@ scala_library(
         maven_artifact("org.apache.commons:commons-lang3"),
         maven_artifact("org.slf4j:slf4j-api"),
         maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
     ],
 )
 
@@ -33,19 +33,19 @@ test_deps = [
     maven_artifact("org.apache.datasketches:datasketches-java"),
     maven_artifact("org.apache.commons:commons-lang3"),
     maven_artifact("org.apache.commons:commons-math3"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
     # Testing
-    scala_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
-    scala_artifact_with_suffix("org.scalactic:scalactic"),
-    scala_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
+    maven_artifact_with_suffix("org.scalactic:scalactic"),
+    maven_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
     maven_artifact("org.mockito:mockito-core"),
-    scala_artifact("org.scalatest:scalatest-compatible"),
+    maven_artifact("org.scalatest:scalatest-compatible"),
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
 ]

api/BUILD.bazel

@@ -38,8 +38,8 @@ scala_library(
         maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
         maven_artifact("org.apache.commons:commons-lang3"),
         maven_artifact("com.google.code.gson:gson"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-parser-combinators"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-parser-combinators"),
     ],
 )
 
@@ -53,19 +53,19 @@ test_deps = [
     maven_artifact("org.slf4j:slf4j-api"),
     maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
     # Testing
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-parser-combinators"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
-    scala_artifact_with_suffix("org.scalactic:scalactic"),
-    scala_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-parser-combinators"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
+    maven_artifact_with_suffix("org.scalactic:scalactic"),
+    maven_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
     maven_artifact("org.mockito:mockito-core"),
-    scala_artifact("org.scalatest:scalatest-compatible"),
+    maven_artifact("org.scalatest:scalatest-compatible"),
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
 ]

cloud_aws/BUILD.bazel

@@ -11,8 +11,8 @@ shared_libs = [
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
     maven_artifact("com.fasterxml.jackson.module:jackson-module-afterburner"),
-    scala_artifact_with_suffix("org.apache.hudi:hudi-spark3.5-bundle"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.apache.hudi:hudi-spark3.5-bundle"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
     "//api:lib",
     "//api:thrift_java",
     "//online:lib",
@@ -37,9 +37,9 @@ test_deps = [
     maven_artifact("software.amazon.awssdk:auth"),
     maven_artifact("software.amazon.awssdk:identity-spi"),
     maven_artifact("software.amazon.awssdk:url-connection-client"),
-    scala_artifact_with_suffix("com.chuusai:shapeless"),
-    scala_artifact_with_suffix("org.typelevel:cats-core"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("com.chuusai:shapeless"),
+    maven_artifact_with_suffix("org.typelevel:cats-core"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
 ] + _CIRCE_DEPS + _SCALA_TEST_DEPS
 
 scala_test_suite(

cloud_gcp/BUILD.bazel

@@ -5,12 +5,12 @@ shared_deps = [
     "//online:lib",
     "//spark:lib",
     "//tools/build_rules/spark:spark-exec",
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
-    scala_artifact_with_suffix("org.json4s:json4s-core"),
-    scala_artifact_with_suffix("org.json4s:json4s-jackson"),
-    scala_artifact_with_suffix("org.json4s:json4s-ast"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
-    scala_artifact_with_suffix("org.rogach:scallop"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
+    maven_artifact_with_suffix("org.json4s:json4s-core"),
+    maven_artifact_with_suffix("org.json4s:json4s-jackson"),
+    maven_artifact_with_suffix("org.json4s:json4s-ast"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.rogach:scallop"),
     maven_artifact("com.google.cloud:google-cloud-core"),
     maven_artifact("com.google.cloud:google-cloud-bigquery"),
     maven_artifact("com.google.cloud:google-cloud-bigtable"),
@@ -35,7 +35,7 @@ shared_deps = [
     maven_artifact("org.threeten:threetenbp"),
     maven_artifact("org.apache.kafka:kafka-clients"),
     maven_artifact("com.google.cloud.spark:spark-3.5-bigquery"),
-    scala_artifact_with_suffix("org.apache.iceberg:iceberg-spark-runtime-3.5"),
+    maven_artifact_with_suffix("org.apache.iceberg:iceberg-spark-runtime-3.5"),
     maven_artifact("org.objenesis:objenesis"),
 ]
 
@@ -54,17 +54,17 @@ test_deps = [
     maven_artifact("com.google.cloud:google-cloud-bigtable-emulator"),
 
     # Testing
-    scala_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
-    scala_artifact_with_suffix("org.scalactic:scalactic"),
-    scala_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
+    maven_artifact_with_suffix("org.scalactic:scalactic"),
+    maven_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
     maven_artifact("org.mockito:mockito-core"),
-    scala_artifact("org.scalatest:scalatest-compatible"),
+    maven_artifact("org.scalatest:scalatest-compatible"),
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
 ]

flink/BUILD.bazel

@@ -12,9 +12,9 @@ scala_library(
         "//api:thrift_java",
         "//online:lib",
         "//tools/build_rules/spark:spark-exec",
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
-        scala_artifact_with_suffix("org.rogach:scallop"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+        maven_artifact_with_suffix("org.rogach:scallop"),
         maven_artifact("io.dropwizard.metrics:metrics-core"),
         maven_artifact("org.apache.kafka:kafka-clients"),
         maven_artifact("org.slf4j:slf4j-api"),
@@ -38,8 +38,8 @@ test_deps = _FLINK_TEST_DEPS + [
     "//aggregator:lib",
     "//tools/build_rules/spark:spark-exec",
     # Libraries
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
     maven_artifact("org.slf4j:slf4j-api"),
     maven_artifact("org.apache.logging.log4j:log4j-api"),
     maven_artifact("org.apache.logging.log4j:log4j-core"),
@@ -52,17 +52,17 @@ test_deps = _FLINK_TEST_DEPS + [
     maven_artifact("org.apache.hadoop:hadoop-common"),
     maven_artifact("org.apache.hadoop:hadoop-client-api"),
     # Testing
-    scala_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
-    scala_artifact_with_suffix("org.scalactic:scalactic"),
-    scala_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
+    maven_artifact_with_suffix("org.scalactic:scalactic"),
+    maven_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
     maven_artifact("org.mockito:mockito-core"),
-    scala_artifact("org.scalatest:scalatest-compatible"),
+    maven_artifact("org.scalatest:scalatest-compatible"),
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
 ]

hub/BUILD.bazel

@@ -12,12 +12,12 @@ scala_library(
         "//online:lib",
         "//service_commons:lib",
         "//spark:lib",
-        scala_artifact_with_suffix("org.typelevel:cats-core"),
-        scala_artifact_with_suffix("com.chuusai:shapeless"),
-        scala_artifact_with_suffix("org.json4s:json4s-core"),
-        scala_artifact_with_suffix("org.json4s:json4s-jackson"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
-        scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+        maven_artifact_with_suffix("org.typelevel:cats-core"),
+        maven_artifact_with_suffix("com.chuusai:shapeless"),
+        maven_artifact_with_suffix("org.json4s:json4s-core"),
+        maven_artifact_with_suffix("org.json4s:json4s-jackson"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
+        maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
         maven_artifact("ch.qos.logback:logback-classic"),
         maven_artifact("org.slf4j:slf4j-api"),
         maven_artifact("com.typesafe:config"),
@@ -35,22 +35,22 @@ test_deps = _VERTX_TEST_DEPS + _CIRCE_DEPS + [
     "//spark:lib",
     "//service_commons:lib",
     # Libraries
-    scala_artifact_with_suffix("org.typelevel:cats-core"),
-    scala_artifact_with_suffix("com.chuusai:shapeless"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
-    scala_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
+    maven_artifact_with_suffix("org.typelevel:cats-core"),
+    maven_artifact_with_suffix("com.chuusai:shapeless"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-java8-compat"),
+    maven_artifact_with_suffix("org.scala-lang.modules:scala-collection-compat"),
     # Testing
-    scala_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-core"),
-    scala_artifact_with_suffix("org.scalatest:scalatest"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
-    scala_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
-    scala_artifact_with_suffix("org.scalactic:scalactic"),
-    scala_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
-    scala_artifact_with_suffix("org.mockito:mockito-scala"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-matchers-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-core"),
+    maven_artifact_with_suffix("org.scalatest:scalatest"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-flatspec"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-funsuite"),
+    maven_artifact_with_suffix("org.scalatest:scalatest-shouldmatchers"),
+    maven_artifact_with_suffix("org.scalactic:scalactic"),
+    maven_artifact_with_suffix("org.scalatestplus:mockito-3-4"),
+    maven_artifact_with_suffix("org.mockito:mockito-scala"),
     maven_artifact("org.mockito:mockito-core"),
-    scala_artifact("org.scalatest:scalatest-compatible"),
+    maven_artifact("org.scalatest:scalatest-compatible"),
     maven_artifact("junit:junit"),
     maven_artifact("com.novocode:junit-interface"),
     maven_artifact("net.bytebuddy:byte-buddy"),