Drop Spark BigTable version to unlock DataProc submission

Author: piyush-zlai

Dockerfile

@@ -57,6 +57,13 @@ RUN curl https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SP
  && tar xvzf spark.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark.tgz
 
+# Add some additional custom jars for other connectors like BigTable etc
+RUN mkdir -p /opt/custom-jars && \
+    curl -L "https://repo1.maven.org/maven2/com/google/cloud/spark/bigtable/spark-bigtable_2.12/0.2.1/spark-bigtable_2.12-0.2.1.jar" \
+    -o /opt/custom-jars/spark-bigtable_2.12-0.2.1.jar && \
+    curl -L "https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-slf4j-impl/2.20.0/log4j-slf4j-impl-2.20.0.jar" \
+    -o /opt/custom-jars/log4j-slf4j-impl-2.20.0.jar
+
 # Install python deps
 COPY quickstart/requirements.txt .
 RUN pip3 install -r requirements.txt

build.sbt

@@ -217,7 +217,6 @@ lazy val cloud_gcp = project
     libraryDependencies += "com.google.cloud.bigdataoss" % "gcsio" % "3.0.3", // need it for https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystem.java
     libraryDependencies += "io.circe" %% "circe-yaml" % "1.15.0",
     libraryDependencies += "com.google.cloud.spark" %% s"spark-bigquery-with-dependencies" % "0.41.0",
-    libraryDependencies += "com.google.cloud.spark.bigtable" %% "spark-bigtable" % "0.2.1",
     libraryDependencies += "com.google.cloud.bigtable" % "bigtable-hbase-2.x" % "2.14.2",
     libraryDependencies ++= circe,
     libraryDependencies ++= avro,

quickstart/cloud_gcp/scripts/load_data.sh

@@ -33,7 +33,9 @@ echo "GroupBy upload batch jobs completed successfully!"
 
 echo "Uploading tables to KV Store"
 for dataset in purchases returns; do
-  if ! spark-submit --driver-class-path "$CLASSPATH" --class ai.chronon.integrations.cloud_gcp.Spark2BigTableLoader \
+  if ! spark-submit --driver-class-path "$CLASSPATH:/opt/custom-jars/*" \
+    --jars "/opt/custom-jars/spark-bigtable_2.12-0.2.1.jar,/opt/custom-jars/log4j-slf4j-impl-2.20.0.jar" \
+    --class ai.chronon.integrations.cloud_gcp.Spark2BigTableLoader \
     --master local[*] $CLOUD_GCP_JAR --table-name default.quickstart_${dataset}_v1_upload --dataset quickstart.${dataset}.v1 \
     --end-ds 2023-11-30 --project-id $GCP_PROJECT_ID --instance-id $GCP_INSTANCE_ID; then
     echo "Error: Failed to upload table to KV Store" >&2