Skip to content

Commit c7afd1c

Browse files
tchow-zlaitchow-zlai
authored
feat: enable slf4j logging to work with logback (#76)
## Summary - Some of the `logger.info` invocations weren't happening. That's mainly because we don't have a logger implementation specified at least in the spark build. This PR adds a `logback` implementation and a basic configuration as such. - This will allow us to see log messages through the command line. I tested this: `sbt "testOnly ai.chronon.spark.test.FeatureWithLabelJoinTest" | grep "== Features DF =="` - I also verified that the dep tree shows the new logger deps were present: ``` sbt spark/dependencyTree [info] welcome to sbt 1.8.2 (Oracle Corporation Java 17.0.2) [info] loading settings for project chronon-build from plugins.sbt ... [info] loading project definition from /Users/thomaschow/zipline-ai/chronon/project [info] loading settings for project root from build.sbt,version.sbt ... [info] resolving key references (13698 settings) ... [info] set current project to chronon (in build file:/Users/thomaschow/zipline-ai/chronon/) [info] spark:spark_2.12:0.1.0-SNAPSHOT [S] [info] +-aggregator:aggregator_2.12:0.1.0-SNAPSHOT [S] [info] | +-api:api_2.12:0.1.0-SNAPSHOT [S] [info] | | +-org.scala-lang.modules:scala-collection-compat_2.12:2.11.0 [S] [info] | | +-org.scala-lang:scala-reflect:2.12.18 [S] [info] | | [info] | +-com.google.code.gson:gson:2.10.1 [info] | +-org.apache.datasketches:datasketches-java:6.1.0 [info] | +-org.apache.datasketches:datasketches-memory:3.0.1 [info] | [info] +-ch.qos.logback:logback-classic:1.2.11 [info] | +-ch.qos.logback:logback-core:1.2.11 [info] | +-org.slf4j:slf4j-api:1.7.36 [info] | [info] +-com.google.guava:guava:33.3.1-jre [info] | +-com.google.code.findbugs:jsr305:3.0.2 [info] | +-com.google.errorprone:error_prone_annotations:2.28.0 [info] | +-com.google.guava:failureaccess:1.0.2 [info] | +-com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-gu.. [info] | +-com.google.j2objc:j2objc-annotations:3.0.0 [info] | +-org.checkerframework:checker-qual:3.43.0 [info] | [info] +-jakarta.servlet:jakarta.servlet-api:4.0.3 [info] +-online:online_2.12:0.1.0-SNAPSHOT [S] [info] +-aggregator:aggregator_2.12:0.1.0-SNAPSHOT [S] [info] | +-api:api_2.12:0.1.0-SNAPSHOT [S] [info] | | +-org.scala-lang.modules:scala-collection-compat_2.12:2.11.0 [S] [info] | | +-org.scala-lang:scala-reflect:2.12.18 [S] [info] | | [info] | +-com.google.code.gson:gson:2.10.1 [info] | +-org.apache.datasketches:datasketches-java:6.1.0 [info] | +-org.apache.datasketches:datasketches-memory:3.0.1 [info] | [info] +-com.datadoghq:java-dogstatsd-client:4.4.1 [info] | +-com.github.jnr:jnr-unixsocket:0.36 [info] | +-com.github.jnr:jnr-constants:0.9.17 [info] | +-com.github.jnr:jnr-enxio:0.30 [info] | | +-com.github.jnr:jnr-constants:0.9.17 [info] | | +-com.github.jnr:jnr-ffi:2.1.16 [info] | | +-com.github.jnr:jffi:1.2.23 [info] | | +-com.github.jnr:jnr-a64asm:1.0.0 [info] | | +-com.github.jnr:jnr-x86asm:1.0.2 [info] | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-commons:7.1 [info] | | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-util:7.1 [info] | | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-com.github.jnr:jnr-ffi:2.1.16 [info] | | +-com.github.jnr:jffi:1.2.23 [info] | | +-com.github.jnr:jnr-a64asm:1.0.0 [info] | | +-com.github.jnr:jnr-x86asm:1.0.2 [info] | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-commons:7.1 [info] | | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-util:7.1 [info] | | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | | +-org.ow2.asm:asm:7.1 [info] | | | | [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-com.github.jnr:jnr-posix:3.0.61 [info] | +-com.github.jnr:jnr-constants:0.9.17 [info] | +-com.github.jnr:jnr-ffi:2.1.16 [info] | +-com.github.jnr:jffi:1.2.23 [info] | +-com.github.jnr:jnr-a64asm:1.0.0 [info] | +-com.github.jnr:jnr-x86asm:1.0.2 [info] | +-org.ow2.asm:asm-analysis:7.1 [info] | | +-org.ow2.asm:asm-tree:7.1 [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-org.ow2.asm:asm-commons:7.1 [info] | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-org.ow2.asm:asm-tree:7.1 [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-org.ow2.asm:asm-util:7.1 [info] | | +-org.ow2.asm:asm-analysis:7.1 [info] | | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm-tree:7.1 [info] | | | +-org.ow2.asm:asm:7.1 [info] | | | [info] | | +-org.ow2.asm:asm:7.1 [info] | | [info] | +-org.ow2.asm:asm:7.1 [info] | [info] +-com.fasterxml.jackson.core:jackson-core:2.15.2 [info] +-com.fasterxml.jackson.core:jackson-databind:2.15.2 [info] | +-com.fasterxml.jackson.core:jackson-annotations:2.15.2 [info] | +-com.fasterxml.jackson.core:jackson-core:2.15.2 [info] | [info] +-com.fasterxml.jackson.module:jackson-module-scala_2.12:2.15.2 [S] [info] | +-com.fasterxml.jackson.core:jackson-annotations:2.15.2 [info] | +-com.fasterxml.jackson.core:jackson-core:2.15.2 [info] | +-com.fasterxml.jackson.core:jackson-databind:2.15.2 [info] | | +-com.fasterxml.jackson.core:jackson-annotations:2.15.2 [info] | | +-com.fasterxml.jackson.core:jackson-core:2.15.2 [info] | | [info] | +-com.thoughtworks.paranamer:paranamer:2.8 [info] | [info] +-com.github.ben-manes.caffeine:caffeine:3.1.8 [info] | +-com.google.errorprone:error_prone_annotations:2.21.1 (evicted by: 2.28.. [info] | +-com.google.errorprone:error_prone_annotations:2.28.0 [info] | +-org.checkerframework:checker-qual:3.37.0 (evicted by: 3.43.0) [info] | +-org.checkerframework:checker-qual:3.43.0 [info] | [info] +-net.jodah:typetools:0.6.3 [info] +-org.rogach:scallop_2.12:5.1.0 [S] [info] +-org.scala-lang.modules:scala-java8-compat_2.12:1.0.2 [S] ``` - Additional steps are required for Intellij to behave the same way. I needed to configure the classpath `-cp chronon.spark` in the run configuration: <img width="953" alt="Screenshot 2024-11-21 at 3 34 50 PM" src="https://github.com/user-attachments/assets/aebbc466-a207-43d0-9f6f-a9bfa811eb66"> and same for `ScalaTest` . I updated the local setup to reflect this: https://docs.google.com/document/d/1k9_aQ3tkW5wvzKyXSsWWPK6HZxX4t8zVPVThODpZqQs/edit?tab=t.0#heading=h.en6opahtqp7u ## Checklist - [x] Added Unit Tests - [x] Covered by existing CI - [x] Integration tested - [x] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a new logging configuration with a `logback.xml` file for enhanced logging capabilities. - Added support for overriding dependencies in the build configuration for improved dependency management. - **Bug Fixes** - Ensured consistent logging library versions across the project to avoid potential conflicts. - **Chores** - Streamlined dependency declarations for better organization within the build configuration. - Improved logging feedback during the build process. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent d0cf5d9 commit c7afd1c

File tree

2 files changed

+32
-13
lines changed

2 files changed

+32
-13
lines changed

‎build.sbt

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
import sbt.Keys.{libraryDependencies, *}
1+
import sbt.Keys.{dependencyOverrides, libraryDependencies, *}
22
import sbt.{Test, *}
33
import sbt.Tests.{Group, SubProcess}
44

5+
import scala.collection.Seq
6+
57
// Notes about a few dependencies - and how we land on versions
68
// Our approach is to use the latest stable versions of deps as of today (July 24) and pin to them for a few years
79
// this should simplify our build setup, speed up CI and deployment
@@ -64,10 +66,10 @@ val spark_all = Seq(
6466
"org.apache.spark" %% "spark-hive",
6567
"org.apache.spark" %% "spark-core",
6668
"org.apache.spark" %% "spark-streaming",
67-
"org.apache.spark" %% "spark-sql-kafka-0-10",
69+
"org.apache.spark" %% "spark-sql-kafka-0-10"
6870
).map(_ % spark_3_5) :+ (
69-
"javax.servlet" % "javax.servlet-api" % "3.1.0",
70-
)
71+
"javax.servlet" % "javax.servlet-api" % "3.1.0",
72+
)
7173
val spark_all_provided = spark_all.map(_ % "provided")
7274

7375
val jackson = Seq(
@@ -106,10 +108,10 @@ lazy val aggregator = project
106108
.dependsOn(api.%("compile->compile;test->test"))
107109
.settings(
108110
libraryDependencies ++= Seq(
109-
"org.apache.datasketches" % "datasketches-java" % "6.1.0",
110-
"com.google.code.gson" % "gson" % "2.10.1"
111-
),
112-
libraryDependencies ++= spark_sql_provided,
111+
"org.apache.datasketches" % "datasketches-java" % "6.1.0",
112+
"com.google.code.gson" % "gson" % "2.10.1"
113+
),
114+
libraryDependencies ++= spark_sql_provided
113115
)
114116

115117
// todo add a service module with spark as a hard dependency
@@ -122,7 +124,7 @@ lazy val online = project
122124
"com.datadoghq" % "java-dogstatsd-client" % "4.4.1",
123125
"org.rogach" %% "scallop" % "5.1.0",
124126
"net.jodah" % "typetools" % "0.6.3",
125-
"com.github.ben-manes.caffeine" % "caffeine" % "3.1.8",
127+
"com.github.ben-manes.caffeine" % "caffeine" % "3.1.8"
126128
),
127129
libraryDependencies ++= jackson,
128130
libraryDependencies ++= spark_all.map(_ % "provided"),
@@ -158,15 +160,20 @@ lazy val spark = project
158160
libraryDependencies ++= spark_all_provided,
159161
libraryDependencies ++= spark_all.map(_ % "test"),
160162
libraryDependencies += "jakarta.servlet" % "jakarta.servlet-api" % "4.0.3",
161-
libraryDependencies += "com.google.guava" % "guava" % "33.3.1-jre"
162-
)
163+
libraryDependencies += "com.google.guava" % "guava" % "33.3.1-jre",
164+
// Ensure consistent versions of logging libraries
165+
libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.13",
166+
dependencyOverrides ++= Seq(
167+
"org.slf4j" % "slf4j-api" % "1.7.36"
168+
)
169+
)
163170

164171
lazy val flink = project
165172
.dependsOn(aggregator.%("compile->compile;test->test"), online)
166173
.settings(
167174
libraryDependencies ++= spark_all,
168175
libraryDependencies ++= flink_all,
169-
libraryDependencies += "org.apache.flink" % "flink-test-utils" % flink_1_17 % Test excludeAll(
176+
libraryDependencies += "org.apache.flink" % "flink-test-utils" % flink_1_17 % Test excludeAll (
170177
ExclusionRule(organization = "org.apache.logging.log4j", name = "log4j-api"),
171178
ExclusionRule(organization = "org.apache.logging.log4j", name = "log4j-core"),
172179
ExclusionRule(organization = "org.apache.logging.log4j", name = "log4j-slf4j-impl")
@@ -258,7 +265,7 @@ lazy val hub = (project in file("hub"))
258265
// Ensure consistent versions of logging libraries
259266
dependencyOverrides ++= Seq(
260267
"org.slf4j" % "slf4j-api" % "1.7.36",
261-
"ch.qos.logback" % "logback-classic" % "1.2.11"
268+
"ch.qos.logback" % "logback-classic" % "1.2.13"
262269
)
263270
)
264271

‎spark/src/test/resources/logback.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<configuration>
2+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
3+
<encoder>
4+
<pattern>[%date] {%logger{0}} %level - %message%n</pattern>
5+
</encoder>
6+
</appender>
7+
<logger name="ai.chronon" level="DEBUG" />
8+
9+
<root level="INFO">
10+
<appender-ref ref="CONSOLE"/>
11+
</root>
12+
</configuration>

0 commit comments

Comments
 (0)