Skip to content

[wip] use native annotation tags #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/test_scala_and_python.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ jobs:
- name: Run other spark tests
run: |
export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
sbt "spark/testOnly -- -n jointest"
sbt "spark/testOnly -- -n ai.chronon.spark.test.JoinTestTag"

mutation_spark_tests:
runs-on: ubuntu-latest
Expand All @@ -103,7 +103,7 @@ jobs:
- name: Run other spark tests
run: |
export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
sbt "spark/testOnly -- -n mutationstest"
sbt "spark/testOnly -- -n ai.chronon.spark.test.MutationsTestTag"

fetcher_spark_tests:
runs-on: ubuntu-latest
Expand All @@ -122,7 +122,7 @@ jobs:
- name: Run other spark tests
run: |
export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
sbt "spark/testOnly -- -n fetchertest"
sbt "spark/testOnly -- -n ai.chronon.spark.test.FetcherTestTag"

scala_compile_fmt_fix :
runs-on: ubuntu-latest
Expand Down
23 changes: 23 additions & 0 deletions spark/src/test/java/ai/chronon/spark/test/Tags.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package ai.chronon.spark.test;

import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.ElementType;

@org.scalatest.TagAnnotation
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@piyush-zlai wdyt about this? Reason for the change is that this more natively leverages the scalatest TagAnnotation framework and allows us to do exclusions on the commandline without additional code ie:

sbt "spark/testOnly -- -l ai.chronon.spark.test.JoinTestTag"

@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
@interface JoinTestTag {}


@org.scalatest.TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
@interface FetcherTestTag {}


@org.scalatest.TagAnnotation
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD, ElementType.TYPE})
@interface MutationsTestTag {}
7 changes: 3 additions & 4 deletions spark/src/test/scala/ai/chronon/spark/test/FetcherTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,9 @@ import scala.concurrent.duration.SECONDS
import scala.io.Source
import scala.util.ScalaJavaConversions._

// Run as follows: sbt "spark/testOnly -- -n fetchertest"
class FetcherTest extends AnyFunSuite with TaggedFilterSuite {

override def tagName: String = "fetchertest"
// Run as follows: sbt "spark/testOnly -- -n ai.chronon.spark.test.FetcherTestTag"
@FetcherTestTag
class FetcherTest extends AnyFunSuite {

@transient lazy val logger: Logger = LoggerFactory.getLogger(getClass)
val sessionName = "FetcherTest"
Expand Down
8 changes: 4 additions & 4 deletions spark/src/test/scala/ai/chronon/spark/test/JoinTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,14 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.{StringType => SparkStringType}
import org.junit.Assert._
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.{Ignore, Tag, TagAnnotation}

import scala.collection.JavaConverters._
import scala.util.ScalaJavaConversions.ListOps

// Run as follows: sbt "spark/testOnly -- -n jointest"
class JoinTest extends AnyFunSuite with TaggedFilterSuite {
// Run as follows: sbt "spark/testOnly -- -n ai.chronon.spark.test.JoinTestTag"
@JoinTestTag
class JoinTest extends AnyFunSuite {

val spark: SparkSession = SparkSessionBuilder.build("JoinTest", local = true)
private implicit val tableUtils = TableUtils(spark)
Expand All @@ -58,8 +60,6 @@ class JoinTest extends AnyFunSuite with TaggedFilterSuite {
private val namespace = "test_namespace_jointest"
tableUtils.createDatabase(namespace)

override def tagName: String = "jointest"

test("test events entities snapshot") {
val dollarTransactions = List(
Column("user", StringType, 100),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ import org.slf4j.LoggerFactory
* Left is an event source with definite ts.
* Right is an entity with snapshots and mutation values through the day.
* Join is the events and the entity value at the exact timestamp of the ts.
* To run: sbt "spark/testOnly -- -n mutationstest"
* To run: sbt "spark/testOnly -- -n ai.chronon.spark.test.MutationsTestTag"
*/
class MutationsTest extends AnyFunSuite with TaggedFilterSuite {
@transient lazy val logger: Logger = LoggerFactory.getLogger(getClass)

override def tagName: String = "mutationstest"
@MutationsTestTag
class MutationsTest extends AnyFunSuite {
@transient lazy val logger: Logger = LoggerFactory.getLogger(getClass)

val spark: SparkSession =
SparkSessionBuilder.build("MutationsTest",
Expand Down
39 changes: 0 additions & 39 deletions spark/src/test/scala/ai/chronon/spark/test/TaggedFilterSuite.scala

This file was deleted.

Loading