diff --git a/.github/image/Dockerfile b/.github/image/Dockerfile index b91193e02d..3ab8b09119 100644 --- a/.github/image/Dockerfile +++ b/.github/image/Dockerfile @@ -32,6 +32,8 @@ RUN apk add --no-cache \ autoconf \ automake \ libtool \ + libc6-compat \ + gcompat \ curl && \ curl -LSs https://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz -o thrift-${THRIFT_VERSION}.tar.gz && \ tar -xzf thrift-${THRIFT_VERSION}.tar.gz && \ diff --git a/.github/workflows/test_python.yaml b/.github/workflows/test_python.yaml index c8e64c4535..77a5a815b9 100644 --- a/.github/workflows/test_python.yaml +++ b/.github/workflows/test_python.yaml @@ -4,15 +4,9 @@ on: push: branches: - main - paths: - - 'api/py/**' - - '.github/workflows/test_python.yaml' pull_request: branches: - main - paths: - - 'api/py/**' - - '.github/workflows/test_python.yaml' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/test_scala_no_spark.yaml b/.github/workflows/test_scala_no_spark.yaml index 8258a10cac..e7a3f425ab 100644 --- a/.github/workflows/test_scala_no_spark.yaml +++ b/.github/workflows/test_scala_no_spark.yaml @@ -1,27 +1,12 @@ -name: Test non-spark Scala modules +name: Test non-spark Scala modules - 1 on: push: branches: - main - paths: - - 'flink/**' - - 'aggregator/**' - - 'online/**' - - 'api/**' - - '.github/workflows/test_scala_no_spark.yaml' - - 'build.sbt' pull_request: branches: - main - paths: - - 'flink/**' - - 'aggregator/**' - - 'online/**' - - 'api/**' - - '.github/workflows/test_scala_no_spark.yaml' - - 'build.sbt' - concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -70,20 +55,3 @@ jobs: - name: Run api tests run: | sbt "++ 2.12.18 api/test" - - - name: Run hub tests - run: | - export SBT_OPTS="-Xmx8G -Xms2G" - sbt "++ 2.12.18 hub/test" - - - name: Run service tests - run: | - export SBT_OPTS="-Xmx8G -Xms2G" - sbt "++ 2.12.18 service/test" - - - name: Run orchestrator tests - run: | - export SBT_OPTS="-Xmx8G -Xms2G" - sbt "++ 2.12.18 orchestration/test" - - diff --git a/.github/workflows/test_scala_no_spark_2.yaml b/.github/workflows/test_scala_no_spark_2.yaml new file mode 100644 index 0000000000..42911a34d5 --- /dev/null +++ b/.github/workflows/test_scala_no_spark_2.yaml @@ -0,0 +1,62 @@ +name: Test non-spark Scala modules - 2 + +on: + push: + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + no_spark_scala_tests: + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}-ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + working-directory: ${{ github.workspace }} + + steps: + - uses: actions/checkout@v4 + + - name: Set SBT resources + run: | + echo "SBT_OPTS=-Xmx8G -Xms2G" >> $GITHUB_ENV + + - name: Run hub tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 hub/test" + + - name: Run service tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 service/test" + + - name: Run service commons tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 service_commons/test" + + - name: Run orchestrator tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 orchestration/test" + + - name: Run cloud gcp tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 cloud_gcp/test" + + - name: Run cloud aws tests + run: | + export SBT_OPTS="-Xmx8G -Xms2G" + sbt "++ 2.12.18 cloud_aws/test" diff --git a/.github/workflows/test_scala_spark.yaml b/.github/workflows/test_scala_spark.yaml index 886f08f3d1..9f1512ccfc 100644 --- a/.github/workflows/test_scala_spark.yaml +++ b/.github/workflows/test_scala_spark.yaml @@ -4,17 +4,9 @@ on: push: branches: - main - paths: - - 'spark/**' - - '.github/workflows/test_scala_spark.yaml' - - 'build.sbt' pull_request: branches: - main - paths: - - 'spark/**' - - '.github/workflows/test_scala_spark.yaml' - - 'build.sbt' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreTest.scala b/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreTest.scala index 233b6adaee..d6afd2c85e 100644 --- a/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreTest.scala +++ b/cloud_gcp/src/test/scala/ai/chronon/integrations/cloud_gcp/BigTableKVStoreTest.scala @@ -15,10 +15,7 @@ import com.google.cloud.bigtable.data.v2.BigtableDataSettings import com.google.cloud.bigtable.data.v2.models.Query import com.google.cloud.bigtable.data.v2.models.Row import com.google.cloud.bigtable.data.v2.models.RowMutation -import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule -import org.junit.Rule -import org.junit.runner.RunWith -import org.junit.runners.JUnit4 +import com.google.cloud.bigtable.emulator.v2.Emulator import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.when import org.mockito.Mockito.withSettings @@ -33,16 +30,11 @@ import scala.concurrent.Await import scala.concurrent.duration.DurationInt import scala.jdk.CollectionConverters._ -@RunWith(classOf[JUnit4]) -class BigTableKVStoreTest extends AnyFlatSpec with BeforeAndAfter{ +class BigTableKVStoreTest extends AnyFlatSpec with BeforeAndAfter { import BigTableKVStore._ - // ugly hacks to wire up the emulator. See: https://stackoverflow.com/questions/32160549/using-junit-rule-with-scalatest-e-g-temporaryfolder - val _bigtableEmulator: BigtableEmulatorRule = BigtableEmulatorRule.create() - @Rule - def bigtableEmulator: BigtableEmulatorRule = _bigtableEmulator - + private var emulator: Emulator = _ private var dataClient: BigtableDataClient = _ private var adminClient: BigtableTableAdminClient = _ @@ -50,16 +42,18 @@ class BigTableKVStoreTest extends AnyFlatSpec with BeforeAndAfter{ private val instanceId = "test-instance" before { + emulator = Emulator.createBundled + emulator.start() // Configure settings to use emulator val dataSettings = BigtableDataSettings - .newBuilderForEmulator(bigtableEmulator.getPort) + .newBuilderForEmulator(emulator.getPort) .setProjectId(projectId) .setInstanceId(instanceId) .setCredentialsProvider(NoCredentialsProvider.create()) .build() val adminSettings = BigtableTableAdminSettings - .newBuilderForEmulator(bigtableEmulator.getPort) + .newBuilderForEmulator(emulator.getPort) .setProjectId(projectId) .setInstanceId(instanceId) .setCredentialsProvider(NoCredentialsProvider.create())