Skip to content

Commit f95dd2c

Browse files
krisnaruknarukulla
krisnaru
and
knarukulla
authored
Bazel support for scala 2.11 and spark 2.4.0 (#916)
* first stab * nit changes --------- Co-authored-by: Krish Narukulla <[email protected]>
1 parent 15291ed commit f95dd2c

File tree

16 files changed

+88
-37
lines changed

16 files changed

+88
-37
lines changed

‎.bazelrc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@ build:local --remote_cache=
66
# To set a different default Scala version, add the following to
77
# user.bazelrc:
88
# common --config scala_2.12
9+
common:scala_2.11 --repo_env=SCALA_VERSION=2.11.12
910
common:scala_2.12 --repo_env=SCALA_VERSION=2.12.18
1011
common:scala_2.13 --repo_env=SCALA_VERSION=2.13.12
1112

1213
# Default scala version to 2.12
1314
common --repo_env=SCALA_VERSION=2.12.18
1415

1516
# Spark versions
17+
common:spark_2.4 --define spark_version=2.4
1618
common:spark_3.1 --define spark_version=3.1
1719
common:spark_3.2 --define spark_version=3.2
1820
common:spark_3.5 --define spark_version=3.5

‎api/BUILD.bazel

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ scala_library(
44
name = "api-lib",
55
srcs = glob(["src/main/scala/ai/chronon/api/*.scala"]) +
66
select_for_scala_version(
7+
before_2_12 = [
8+
"//api/src/main/scala-2.11/scala/util:ScalaVersionSpecificCollectionsConverter.scala",
9+
],
710
between_2_12_and_2_13 = [
811
"//api/src/main/scala-2.12/scala/util:ScalaVersionSpecificCollectionsConverter.scala",
912
],

‎api/src/main/scala-2.11/scala/util/BUILD.bazel

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
exports_files(
2+
[
3+
"ScalaVersionSpecificCollectionsConverter.scala",
4+
],
5+
)

‎devnotes.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,8 @@ bazel build --config scala_2.12 //module:artifact_name
289289
Build a spark version specific artifact:
290290
291291
```shell
292-
bazel build --config spark_3.5 //module:artifact_name
292+
bazel build --config spark_3.5 --config scala_2.12 //module:artifact_name
293+
bazel build --config spark_2.4 --config scala_2.11 //module:artifact_name
293294
```
294295
295296
Build deploy jar aka Uber jar or fat jar:

‎jvm/maven_repo.bzl

Lines changed: 17 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
load("@rules_jvm_external//:specs.bzl", "maven")
22
load(":defs.bzl", "repo", "versioned_artifacts")
3+
load("//tools/build_rules:utils.bzl", "expand_versions")
34

45
maven_repo = repo(
56
name = "maven",
67
pinned = False,
78
artifacts = [
8-
"org.scala-lang.modules:scala-collection-compat_2.12:2.6.0",
9-
"org.scala-lang.modules:scala-collection-compat_2.13:2.6.0",
9+
expand_versions("org.scala-lang.modules:scala-collection-compat_{version}:2.6.0", versions = ["2.11", "2.12", "2.13"]),
1010
"org.scala-lang.modules:scala-parser-combinators_2.13:2.4.0",
11-
"org.scala-lang.modules:scala-java8-compat_2.12:1.0.0",
12-
"org.scala-lang.modules:scala-java8-compat_2.13:1.0.0",
11+
expand_versions("org.scala-lang.modules:scala-java8-compat_{version}:1.0.0", versions = ["2.11", "2.12", "2.13"]),
1312
"org.apache.thrift:libthrift:0.13.0",
1413
"org.scala-lang.modules:scala-parallel-collections_2.13:1.0.4",
1514
"org.apache.commons:commons-lang3:3.12.0",
@@ -23,20 +22,13 @@ maven_repo = repo(
2322
# JUnit
2423
"junit:junit:4.13.2",
2524
"com.novocode:junit-interface:0.11",
26-
"org.scalatestplus:mockito-3-4_2.12:3.2.10.0",
27-
"org.scalatestplus:mockito-3-4_2.13:3.2.10.0",
25+
expand_versions("org.scalatestplus:mockito-3-4_{version}:3.2.10.0", versions = ["2.11", "2.12", "2.13"]),
2826
"org.mockito:mockito-core:4.6.1",
29-
"org.mockito:mockito-scala_2.12:1.17.0",
30-
"org.mockito:mockito-scala_2.13:1.17.0",
31-
"org.scalatest:scalatest_2.12:3.2.15",
32-
"org.scalatest:scalatest_2.13:3.2.15",
33-
"org.scalatest:scalatest-shouldmatchers_2.12:3.2.15",
34-
"org.scalatest:scalatest-shouldmatchers_2.13:3.2.15",
35-
"org.scalatest:scalatest-matchers-core_2.12:3.2.15",
36-
"org.scalatest:scalatest-matchers-core_2.13:3.2.15",
37-
"org.scalactic:scalactic_2.12:3.2.15",
38-
"org.scalactic:scalactic_2.13:3.2.15",
39-
27+
expand_versions("org.mockito:mockito-scala_{version}:1.17.0", versions = ["2.11", "2.12", "2.13"]),
28+
expand_versions("org.scalatest:scalatest_{version}:3.2.15", versions = ["2.11", "2.12", "2.13"]),
29+
expand_versions("org.scalatest:scalatest-shouldmatchers_{version}:3.2.15", versions = ["2.11", "2.12", "2.13"]),
30+
expand_versions("org.scalatest:scalatest-matchers-core_{version}:3.2.15", versions = ["2.11", "2.12", "2.13"]),
31+
expand_versions("org.scalactic:scalactic_{version}:3.2.15", versions = ["2.11", "2.12", "2.13"]),
4032
# Add other dependencies
4133
"org.apache.hadoop:hadoop-common:3.3.1",
4234
"com.typesafe.akka:akka-actor_2.12:2.6.18",
@@ -48,8 +40,7 @@ maven_repo = repo(
4840
"com.fasterxml.jackson.core:jackson-core:2.12.5",
4941
"com.fasterxml.jackson.core:jackson-annotations:2.12.5",
5042
"com.fasterxml.jackson.core:jackson-databind:2.12.5",
51-
"com.fasterxml.jackson.module:jackson-module-scala_2.12:2.12.5",
52-
"com.fasterxml.jackson.module:jackson-module-scala_2.13:2.12.5",
43+
expand_versions("com.fasterxml.jackson.module:jackson-module-scala_{version}:2.12.5", versions = ["2.11", "2.12", "2.13"]),
5344
"org.apache.httpcomponents:httpclient:4.5.13",
5445
"org.apache.httpcomponents:httpcore:4.4.14",
5546
"com.google.guava:guava:30.1-jre",
@@ -60,20 +51,15 @@ maven_repo = repo(
6051
"commons-io:commons-io:2.11.0",
6152
"com.github.ben-manes.caffeine:caffeine:2.8.5",
6253
"net.jodah:typetools:0.4.1",
63-
"org.rogach:scallop_2.12:4.0.1",
64-
"org.rogach:scallop_2.13:4.0.1",
65-
"org.json4s:json4s-jackson_2.12:3.6.12",
66-
"org.json4s:json4s-core_2.12:3.6.12",
67-
"org.json4s:json4s-native_2.12:3.6.12",
68-
"org.json4s:json4s-ast_2.12:3.6.12",
69-
"org.json4s:json4s-jackson_2.13:3.6.12",
70-
"org.json4s:json4s-core_2.13:3.6.12",
71-
"org.json4s:json4s-native_2.13:3.6.12",
72-
"org.json4s:json4s-ast_2.13:3.6.12",
54+
expand_versions("org.rogach:scallop_{version}:4.0.1", versions = ["2.11", "2.12", "2.13"]),
55+
expand_versions("org.json4s:json4s-jackson_{version}:3.6.12", versions = ["2.11", "2.12", "2.13"]),
56+
expand_versions("org.json4s:json4s-core_{version}:3.6.12", versions = ["2.11", "2.12", "2.13"]),
57+
expand_versions("org.json4s:json4s-native_{version}:3.6.12", versions = ["2.11", "2.12", "2.13"]),
58+
expand_versions("org.json4s:json4s-ast_{version}:3.6.12", versions = ["2.11", "2.12", "2.13"]),
7359
"com.datadoghq:java-dogstatsd-client:2.7",
7460
"com.esotericsoftware:kryo:5.6.2",
75-
"io.delta:delta-core_2.12:2.0.2",
76-
"io.delta:delta-core_2.13:2.0.2",
61+
expand_versions("io.delta:delta-core_{version}:2.0.2", versions = ["2.12", "2.13"]),
62+
"io.delta:delta-core_2.11:0.6.1",
7763
"com.github.jnr:jnr-ffi:2.2.10",
7864
"org.apache.kafka:kafka-clients:3.7.1",
7965
"org.codehaus.janino:janino:3.0.9",

‎jvm/repos.bzl

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,25 @@ load(":defs.bzl", "repo", "versioned_artifacts")
55

66
# repos with artifacts defined in external files
77
load(":maven_repo.bzl", "maven_repo")
8+
load(":spark_2_4_repo.bzl", "spark_2_4_repo")
89
load(":spark_3_1_repo.bzl", "spark_3_1_repo")
910
load(":spark_3_2_repo.bzl", "spark_3_2_repo")
1011
load(":spark_3_5_repo.bzl", "spark_3_5_repo")
1112

1213
repos = [
1314
# The main repos are defined in individual files, which are loaded above and referenced here
1415
maven_repo, # defined in maven_repo.bzl
16+
spark_2_4_repo,
1517
spark_3_1_repo,
1618
spark_3_2_repo,
1719
spark_3_5_repo,
20+
repo(name = "scala_2.11", artifacts = [
21+
versioned_artifacts("2.11.12", [
22+
"org.scala-lang:scala-library",
23+
"org.scala-lang:scala-compiler",
24+
"org.scala-lang:scala-reflect",
25+
]),
26+
]),
1827
repo(name = "scala_2.12", artifacts = [
1928
versioned_artifacts("2.12.18", [
2029
"org.scala-lang:scala-library",

‎jvm/spark_2_4_repo.bzl

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
load("@rules_jvm_external//:specs.bzl", "maven")
2+
load(":defs.bzl", "repo", "versioned_artifacts")
3+
4+
spark_2_4_repo = repo(name = "spark_2_4", provided = True, artifacts = [
5+
"org.apache.avro:avro:1.8.2",
6+
"org.apache.curator:apache-curator:2.11.0",
7+
"org.apache.datasketches:datasketches-java:2.0.0",
8+
"org.apache.datasketches:datasketches-memory:1.3.0",
9+
"org.apache.hive:hive-exec:1.2.1",
10+
versioned_artifacts("2.4.0", [
11+
"org.apache.spark:spark-streaming_2.11",
12+
"org.apache.spark:spark-core_2.11:jar:tests",
13+
"org.apache.spark:spark-hive_2.11",
14+
"org.apache.spark:spark-sql-kafka-0-10_2.11",
15+
"org.apache.spark:spark-sql_2.11",
16+
"org.apache.spark:spark-streaming-kafka-0-10_2.11",
17+
]),
18+
versioned_artifacts("3.7.0-M5", [
19+
"org.json4s:json4s-ast_2.11",
20+
"org.json4s:json4s-core_2.11",
21+
"org.json4s:json4s-jackson_2.11",
22+
]),
23+
"org.apache.hive:hive-metastore:2.3.9",
24+
], excluded_artifacts = ["org.slf4j:slf4j-log4j12", "org.pentaho:pentaho-aggdesigner-algorithm"])

‎online/BUILD.bazel

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ scala_library(
44
name = "online",
55
srcs = glob(["src/main/scala/ai/chronon/online/*.scala"]) +
66
select_for_scala_version(
7+
before_2_12 = [
8+
"//online/src/main/scala-2.11/ai/chronon/online:ScalaVersionSpecificCatalystHelper.scala",
9+
"//online/src/main/scala-2.11/ai/chronon/online:FutureConverters.scala",
10+
],
711
between_2_12_and_2_13 = [
812
# For Scala 2.12.x
913
"//online/src/main/scala-2.12/ai/chronon/online:ScalaVersionSpecificCatalystHelper.scala",

‎online/src/main/scala-2.11/ai/chronon/online/BUILD

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
exports_files(
2+
[
3+
"ScalaVersionSpecificCatalystHelper.scala",
4+
"FutureConverters.scala"
5+
],
6+
)

‎spark/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ scala_library(
4141
maven_artifact("org.slf4j:slf4j-api"),
4242
maven_artifact("org.slf4j:slf4j-log4j12"),
4343
maven_artifact("com.google.code.gson:gson"),
44+
maven_artifact("com.datadoghq:java-dogstatsd-client"),
4445
],
4546
)
4647

‎spark/src/test/scala/ai/chronon/spark/test/AnalyzerTest.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ class AnalyzerTest {
452452
val testSchema = List(
453453
Column("key", api.StringType, 10),
454454
Column("col1", api.IntType, 10),
455-
Column("col2", api.IntType, 10),
455+
Column("col2", api.IntType, 10)
456456
)
457457

458458
val viewsTable = s"$namespace.test_table"

‎spark/src/test/scala/ai/chronon/spark/test/DataRangeTest.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class DataRangeTest {
3636
val testTable = s"$namespace.test_gen_scan_query"
3737
val viewsSchema = List(
3838
Column("col_1", api.StringType, 1),
39-
Column("col_2", api.StringType, 1),
39+
Column("col_2", api.StringType, 1)
4040
)
4141
DataFrameGen
4242
.events(spark, viewsSchema, count = 1000, partitions = 200)
@@ -49,7 +49,7 @@ class DataRangeTest {
4949
wheres = Seq("col_1 = 'TEST'"),
5050
timeColumn = "ts"
5151
),
52-
table = testTable,
52+
table = testTable
5353
)
5454

5555
val result: String = partitionRange.genScanQuery(

‎spark/src/test/scala/ai/chronon/spark/test/ExtensionsTest.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class ExtensionsTest {
2626
(7, "2024-01-07"),
2727
(8, "2024-01-08"),
2828
(9, "2024-01-08"),
29-
(10, "2024-01-09"),
29+
(10, "2024-01-09")
3030
).toDF("key", "ds")
3131

3232
val prunedDf = df.prunePartition(PartitionRange("2024-01-05", "2024-01-07"))
@@ -35,7 +35,7 @@ class ExtensionsTest {
3535
(4, "2024-01-05"),
3636
(5, "2024-01-05"),
3737
(6, "2024-01-06"),
38-
(7, "2024-01-07"),
38+
(7, "2024-01-07")
3939
).toDF("key", "ds")
4040
val diff = Comparison.sideBySide(expectedDf, prunedDf, List("key"))
4141
if (diff.count() != 0) {

‎third_party/java/spark/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ java_library(
9797
visibility = ["//visibility:public"],
9898
exports = select({
9999
"//conditions:default": get_jars_for_repo("spark_3_2", SPARK_JARS),
100+
"//tools/flags/spark:spark_2_4": get_jars_for_repo("spark_2_4", SPARK_JARS),
100101
"//tools/flags/spark:spark_3_1": get_jars_for_repo("spark_3_1", SPARK_JARS),
101102
"//tools/flags/spark:spark_3_2": get_jars_for_repo("spark_3_2", SPARK_JARS),
102103
"//tools/flags/spark:spark_3_5": get_jars_for_repo("spark_3_5", SPARK_3_5_JARS),

‎tools/build_rules/utils.bzl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,7 @@ def flatten(items, max_depth = 1):
2828
for i in range(max_depth):
2929
result = flat_map(identity, result)
3030
return result
31+
32+
def expand_versions(artifact, versions):
33+
expanded_versions = [artifact.format(version = version) for version in versions]
34+
return flatten(expanded_versions)

‎tools/flags/spark/BUILD

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
config_setting(
2+
name = "spark_2_4",
3+
define_values = {"spark_version": "2.4"},
4+
)
5+
16
config_setting(
27
name = "spark_3_1",
38
define_values = {"spark_version": "3.1"},

0 commit comments

Comments
 (0)