Skip to content

Commit 36a4c26

Browse files
kumar-zlaikumar-zlai
authored
Bazel migration for cloud_gcp module (#240)
## Summary Migrated `cloud_gcp` module to Bazel from sbt. Below are still pending and will fix them in a separate PR * Few tests are failing, this is actually a general problem across the project and working on it in parallel * cloud_gcp_submitter target logic also need to be migrated ## Cheour clientslist - [ ] Added Unit Tests - [x] Covered by existing CI - [ ] Integration tested - [ ] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes - **Dependency Updates** - Downgraded `rules_jvm_external` from version 6.6 to 4.5 - Updated Moour clientsito version to 5.12.0 - Added new Google Cloud service libraries (BigQuery, Bigtable, PubSub, Dataproc) - Added Apache Curator artifact - **Build Configuration** - Updated library visibility settings - Modified logging dependencies in Spark library - Enhanced Maven repository with additional artifacts - **Development Improvements** - Expanded testing and dependency management capabilities <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 2d0f252 commit 36a4c26

File tree

10 files changed

+228
-173
lines changed

10 files changed

+228
-173
lines changed

‎cloud_gcp/BUILD.bazel

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
scala_library(
2+
name = "lib",
3+
srcs = glob(["src/main/**/*.scala"]),
4+
visibility = ["//visibility:public"],
5+
deps = [
6+
"//api:thrift",
7+
"//api:lib",
8+
"//online:lib",
9+
"//spark:lib",
10+
"//tools/build_rules/spark:spark-exec",
11+
scala_artifact("org.scala-lang.modules:scala-java8-compat"),
12+
scala_artifact("org.json4s:json4s-core"),
13+
scala_artifact("org.json4s:json4s-jaour clientsson"),
14+
scala_artifact("org.json4s:json4s-ast"),
15+
scala_artifact("org.scala-lang.modules:scala-collection-compat"),
16+
scala_artifact("org.rogach:scallop"),
17+
maven_artifact("com.google.cloud:google-cloud-core"),
18+
maven_artifact("com.google.cloud:google-cloud-bigquery"),
19+
maven_artifact("com.google.cloud:google-cloud-bigtable"),
20+
maven_artifact("com.google.cloud:google-cloud-pubsub"),
21+
maven_artifact("com.google.cloud:google-cloud-dataproc"),
22+
maven_artifact("com.google.cloud.bigdataoss:gcsio"),
23+
maven_artifact("com.google.cloud.bigdataoss:util-hadoop"),
24+
scala_artifact("com.google.cloud.spark:spark-bigquery-with-dependencies"),
25+
maven_artifact("com.google.api:api-common"),
26+
maven_artifact("com.google.api.grpc:proto-google-cloud-dataproc-v1"),
27+
maven_artifact("com.google.api:gax"),
28+
maven_artifact("com.google.guava:guava"),
29+
maven_artifact("com.google.protobuf:protobuf-java"),
30+
maven_artifact("org.yaml:snakeyaml"),
31+
maven_artifact("io.grpc:grpc-netty-shaded"),
32+
maven_artifact("ch.qos.reload4j:reload4j"),
33+
maven_artifact("org.slf4j:slf4j-api"),
34+
maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
35+
maven_artifact("org.threeten:threetenbp"),
36+
],
37+
)
38+
39+
test_deps = [
40+
":lib",
41+
"//api:thrift",
42+
"//api:lib",
43+
"//online:lib",
44+
"//spark:lib",
45+
"//tools/build_rules/spark:spark-exec",
46+
# Libraries
47+
scala_artifact("org.scala-lang.modules:scala-java8-compat"),
48+
scala_artifact("org.scala-lang.modules:scala-collection-compat"),
49+
maven_artifact("com.google.cloud:google-cloud-bigquery"),
50+
maven_artifact("com.google.cloud:google-cloud-bigtable"),
51+
maven_artifact("com.google.cloud:google-cloud-dataproc"),
52+
maven_artifact("com.google.cloud.bigdataoss:gcs-connector"),
53+
maven_artifact("com.google.cloud.bigdataoss:gcsio"),
54+
maven_artifact("com.google.cloud.bigdataoss:util-hadoop"),
55+
maven_artifact("com.google.cloud:google-cloud-bigtable-emulator"),
56+
maven_artifact("com.google.api:api-common"),
57+
maven_artifact("com.google.api.grpc:proto-google-cloud-dataproc-v1"),
58+
scala_artifact("com.google.cloud.spark:spark-bigquery-with-dependencies"),
59+
maven_artifact("com.google.api:gax"),
60+
maven_artifact("com.google.protobuf:protobuf-java"),
61+
# Testing
62+
scala_artifact("org.scalatest:scalatest-matchers-core"),
63+
scala_artifact("org.scalatest:scalatest-core"),
64+
scala_artifact("org.scalatest:scalatest"),
65+
scala_artifact("org.scalatest:scalatest-flatspec"),
66+
scala_artifact("org.scalatest:scalatest-funsuite"),
67+
scala_artifact("org.scalatest:scalatest-shouldmatchers"),
68+
scala_artifact("org.scalactic:scalactic"),
69+
scala_artifact("org.scalatestplus:moour clientsito-3-4"),
70+
scala_artifact("org.moour clientsito:moour clientsito-scala"),
71+
maven_artifact("org.moour clientsito:moour clientsito-core"),
72+
maven_artifact("org.scalatest:scalatest-compatible"),
73+
maven_artifact("junit:junit"),
74+
maven_artifact("com.novocode:junit-interface"),
75+
]
76+
77+
scala_library(
78+
name = "test-lib",
79+
srcs = glob(["src/test/**/*.scala"]),
80+
visibility = ["//visibility:public"],
81+
deps = test_deps,
82+
)
83+
84+
scala_test_suite(
85+
name = "test",
86+
srcs = glob(["src/test/**/*.scala"]),
87+
visibility = ["//visibility:public"],
88+
# defined in prelude_bazel file
89+
jvm_flags = _JVM_FLAGS_FOR_ACCESSING_BASE_JAVA_CLASSES,
90+
deps = test_deps + [":test-lib"],
91+
)

‎flink/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ scala_library(
4141
test_deps = flink_test_deps + [
4242
":lib",
4343
"//online:lib",
44+
"//online:test-lib",
4445
"//api:thrift",
4546
"//api:lib",
4647
"//aggregator:lib",

‎online/BUILD.bazel

Lines changed: 29 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -27,57 +27,43 @@ scala_library(
2727
],
2828
)
2929

30+
test_deps = [
31+
":lib",
32+
"//api:thrift",
33+
"//api:lib",
34+
"//aggregator:lib",
35+
"//tools/build_rules/spark:spark-exec",
36+
# Libraries
37+
maven_artifact("com.github.ben-manes.caffeine:caffeine"),
38+
maven_artifact("org.slf4j:slf4j-api"),
39+
maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
40+
maven_artifact("com.google.code.gson:gson"),
41+
# Testing
42+
scala_artifact("org.scalatest:scalatest-matchers-core"),
43+
scala_artifact("org.scalatest:scalatest-core"),
44+
scala_artifact("org.scalatest:scalatest"),
45+
scala_artifact("org.scalatest:scalatest-flatspec"),
46+
scala_artifact("org.scalatest:scalatest-funsuite"),
47+
scala_artifact("org.scalatest:scalatest-shouldmatchers"),
48+
scala_artifact("org.scalactic:scalactic"),
49+
scala_artifact("org.scalatestplus:moour clientsito-3-4"),
50+
scala_artifact("org.moour clientsito:moour clientsito-scala"),
51+
maven_artifact("org.moour clientsito:moour clientsito-core"),
52+
maven_artifact("org.scalatest:scalatest-compatible"),
53+
maven_artifact("junit:junit"),
54+
maven_artifact("com.novocode:junit-interface"),
55+
]
56+
3057
scala_library(
3158
name = "test-lib",
3259
srcs = glob(["src/test/**/*.scala"]),
3360
visibility = ["//visibility:public"],
34-
deps = [
35-
":lib",
36-
"//api:thrift",
37-
"//api:lib",
38-
"//aggregator:lib",
39-
"//tools/build_rules/spark:spark-exec",
40-
# Libraries
41-
maven_artifact("com.github.ben-manes.caffeine:caffeine"),
42-
maven_artifact("org.slf4j:slf4j-api"),
43-
maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
44-
maven_artifact("com.google.code.gson:gson"),
45-
# Testing
46-
scala_artifact("org.scalatest:scalatest-matchers-core"),
47-
scala_artifact("org.scalatest:scalatest-core"),
48-
scala_artifact("org.scalatest:scalatest"),
49-
scala_artifact("org.scalatest:scalatest-flatspec"),
50-
scala_artifact("org.scalatest:scalatest-funsuite"),
51-
scala_artifact("org.scalatest:scalatest-shouldmatchers"),
52-
scala_artifact("org.scalactic:scalactic"),
53-
scala_artifact("org.scalatestplus:moour clientsito-3-4"),
54-
scala_artifact("org.moour clientsito:moour clientsito-scala"),
55-
maven_artifact("org.moour clientsito:moour clientsito-core"),
56-
maven_artifact("org.scalatest:scalatest-compatible"),
57-
maven_artifact("junit:junit"),
58-
maven_artifact("com.novocode:junit-interface"),
59-
],
61+
deps = test_deps,
6062
)
6163

6264
scala_test_suite(
6365
name = "test",
6466
srcs = glob(["src/test/**/*.scala"]),
6567
visibility = ["//visibility:public"],
66-
deps = [
67-
":lib",
68-
":test-lib",
69-
"//aggregator:lib",
70-
"//api:lib",
71-
"//api:thrift",
72-
"//tools/build_rules/spark:spark-exec",
73-
scala_artifact("org.scalatestplus:moour clientsito-3-4"),
74-
maven_artifact("org.moour clientsito:moour clientsito-core"),
75-
scala_artifact("org.scala-lang.modules:scala-collection-compat"),
76-
maven_artifact("org.slf4j:slf4j-api"),
77-
maven_artifact("org.apache.logging.log4j:log4j-slf4j-impl"),
78-
maven_artifact("com.google.code.gson:gson"),
79-
maven_artifact("com.github.ben-manes.caffeine:caffeine"),
80-
maven_artifact("junit:junit"),
81-
maven_artifact("com.novocode:junit-interface"),
82-
],
68+
deps = test_deps + [":test-lib"],
8369
)

‎online/src/test/scala/ai/chronon/online/test/FetcherBaseTest.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ class FetcherBaseTest extends AnyFlatSpec with Moour clientsitoSugar with Matche
6464
// Future compositions in the Fetcher so provision it in
6565
// the moour clients to prevent hanging.
6666
when(kvStore.executionContext).thenReturn(ExecutionContext.global)
67-
fetcherBase = spy(new FetcherBase(kvStore))
67+
fetcherBase = spy[FetcherBase](new FetcherBase(kvStore))
6868
}
6969

7070
it should "fetch columns single query" in {
@@ -213,7 +213,7 @@ class FetcherBaseTest extends AnyFlatSpec with Moour clientsitoSugar with Matche
213213

214214
kvStore = moour clients[KVStore](Answers.RETURNS_DEEP_STUBS)
215215
when(kvStore.executionContext).thenReturn(ExecutionContext.global)
216-
val fetcherBaseWithFlagStore = spy(new FetcherBase(kvStore, flagStore = flagStore))
216+
val fetcherBaseWithFlagStore = spy[FetcherBase](new FetcherBase(kvStore, flagStore = flagStore))
217217
when(fetcherBaseWithFlagStore.isCacheSizeConfigured).thenReturn(true)
218218

219219
def buildGroupByWithCustomJson(name: String): GroupBy = Builders.GroupBy(metaData = Builders.MetaData(name = name))

‎online/src/test/scala/ai/chronon/online/test/FetcherCacheTest.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ class FetcherCacheTest extends AnyFlatSpec with Moour clientsitoHelper {
141141

142142
it should "get cached requests does not cache when cache is disabled for group by" in {
143143
val testCache = new BatchIrCache("test", batchIrCacheMaximumSize)
144-
val spiedTestCache = spy(testCache)
144+
val spiedTestCache = spy[BatchIrCache](testCache)
145145
val fetcherCache = new TestableFetcherCache(Some(testCache)) {
146146
// Cache is enabled globally, but disabled for a specific groupBy
147147
override def isCachingEnabled(groupBy: GroupBy) = false
@@ -216,7 +216,7 @@ class FetcherCacheTest extends AnyFlatSpec with Moour clientsitoHelper {
216216
val cacheKey = BatchIrCache.Key(servingInfo.groupByOps.batchDataset, keys, servingInfo.batchEndTsMillis)
217217

218218
val fetcherCache = new TestableFetcherCache(Some(batchIrCache))
219-
val spiedFetcherCache = Moour clientsito.spy(fetcherCache)
219+
val spiedFetcherCache = Moour clientsito.spy[TestableFetcherCache](fetcherCache)
220220
doReturn(true).when(spiedFetcherCache).isCachingEnabled(any())
221221

222222
// 1. Cached BatchResponse returns the same IRs passed in
@@ -247,7 +247,7 @@ class FetcherCacheTest extends AnyFlatSpec with Moour clientsitoHelper {
247247
val toBatchIr = moour clients[(Array[Byte], GroupByServingInfoParsed) => FinalBatchIr]
248248
val kvStoreBatchResponses = BatchResponses(Success(Seq(TimedValue(batchBytes, 1000L))))
249249

250-
val spiedFetcherCache = Moour clientsito.spy(new TestableFetcherCache(None))
250+
val spiedFetcherCache = Moour clientsito.spy[TestableFetcherCache](new TestableFetcherCache(None))
251251
when(toBatchIr(any(), any())).thenReturn(finalBatchIr)
252252

253253
// When getBatchIrFromBatchResponse is called, it decodes the bytes and doesn't hit the cache
@@ -272,7 +272,7 @@ class FetcherCacheTest extends AnyFlatSpec with Moour clientsitoHelper {
272272
val keys = Map("key" -> "value")
273273
val cacheKey = BatchIrCache.Key(servingInfo.groupByOps.batchDataset, keys, servingInfo.batchEndTsMillis)
274274

275-
val spiedFetcherCache = Moour clientsito.spy(new TestableFetcherCache(Some(batchIrCache)))
275+
val spiedFetcherCache = Moour clientsito.spy[TestableFetcherCache](new TestableFetcherCache(Some(batchIrCache)))
276276
doReturn(true).when(spiedFetcherCache).isCachingEnabled(any())
277277

278278
// 1. Cached BatchResponse returns the same Map responses passed in
@@ -313,7 +313,7 @@ class FetcherCacheTest extends AnyFlatSpec with Moour clientsitoHelper {
313313
when(servingInfo.outputCodec).thenReturn(outputCodec)
314314
when(outputCodec.decodeMap(any())).thenReturn(mapResponse)
315315

316-
val spiedFetcherCache = Moour clientsito.spy(new TestableFetcherCache(None))
316+
val spiedFetcherCache = Moour clientsito.spy[TestableFetcherCache](new TestableFetcherCache(None))
317317

318318
// When getMapResponseFromBatchResponse is called, it decodes the bytes and doesn't hit the cache
319319
val decodedMapResponse = spiedFetcherCache.getMapResponseFromBatchResponse(kvStoreBatchResponses,

‎service_commons/BUILD.bazel

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ java_library(
22
name = "lib",
33
srcs = glob(["src/main/**/*.java"]),
44
visibility = ["//visibility:public"],
5-
deps = SCALA_DEPS + [
5+
deps = _SCALA_DEPS + [
66
"//api:lib",
77
"//online:lib",
88
maven_artifact("io.vertx:vertx-core"),
@@ -20,44 +20,33 @@ java_library(
2020
],
2121
)
2222

23+
test_deps = [
24+
":lib",
25+
"//api:lib",
26+
"//api:thrift",
27+
"//online:lib",
28+
# Libraries
29+
maven_artifact("io.vertx:vertx-core"),
30+
maven_artifact("io.vertx:vertx-web"),
31+
maven_artifact("io.vertx:vertx-web-client"),
32+
# Testing
33+
maven_artifact("io.vertx:vertx-junit5"),
34+
maven_artifact("org.junit.jupiter:junit-jupiter-api"),
35+
maven_artifact("org.junit.platform:junit-platform-launcher"),
36+
maven_artifact("org.junit.platform:junit-platform-reporting"),
37+
]
38+
2339
java_library(
2440
name = "test-lib",
2541
srcs = glob(["src/test/**/*.java"]),
2642
visibility = ["//visibility:public"],
27-
deps = [
28-
":lib",
29-
"//api:lib",
30-
"//api:thrift",
31-
"//online:lib",
32-
# Libraries
33-
maven_artifact("io.vertx:vertx-core"),
34-
maven_artifact("io.vertx:vertx-web"),
35-
maven_artifact("io.vertx:vertx-web-client"),
36-
# Testing
37-
maven_artifact("io.vertx:vertx-junit5"),
38-
maven_artifact("org.junit.jupiter:junit-jupiter-api"),
39-
],
43+
deps = test_deps,
4044
)
4145

4246
java_test_suite(
4347
name = "test",
4448
srcs = glob(["src/test/**/*.java"]),
4549
visibility = ["//visibility:public"],
4650
runner = "junit5",
47-
deps = [
48-
":lib",
49-
":test-lib",
50-
"//api:lib",
51-
"//api:thrift",
52-
"//online:lib",
53-
# Libraries
54-
maven_artifact("io.vertx:vertx-core"),
55-
maven_artifact("io.vertx:vertx-web"),
56-
maven_artifact("io.vertx:vertx-web-client"),
57-
# Testing
58-
maven_artifact("io.vertx:vertx-junit5"),
59-
maven_artifact("org.junit.jupiter:junit-jupiter-api"),
60-
maven_artifact("org.junit.platform:junit-platform-launcher"),
61-
maven_artifact("org.junit.platform:junit-platform-reporting"),
62-
],
51+
deps = test_deps + [":test-lib"],
6352
)

0 commit comments

Comments
 (0)