Updating spark and scala version what we require at Interset

- modelled from elastic#1308
- added a script to install it locally for my mediocre test purposes

Author: tomdubiel

gradle.properties

@@ -19,12 +19,14 @@ jacksonVersion = 1.8.8
 cascadingVersion = 2.6.3
 # Spark
 spark13Version = 1.6.2
-spark20Version = 2.3.0
+spark20Version = 2.4.4
 # same as Spark's
 scala210Version = 2.10.7
 scala210MajorVersion = 2.10
 scala211Version = 2.11.12
 scala211MajorVersion = 2.11
+scala212Version = 2.12.10
+scala212MajorVersion = 2.12
 
 stormVersion = 1.0.6
 

install_local.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+PROJECT_ROOT=$(git rev-parse --show-toplevel)
+
+echo "Building es-hadoop jars with gradlew..."
+sh ${PROJECT_ROOT}/gradlew
+
+# JAR and pom file 
+JAR="${PROJECT_ROOT}/build/distributions/elasticsearch-hadoop-6.8.12-SNAPSHOT/dist/elasticsearch-spark-20_2.12-6.8.12-SNAPSHOT.jar"
+
+#   These should match the existing maven dependency
+# for the artifact you're pulling from repo
+GROUP_ID="org.elasticsearch"
+ARTIFACT_ID="elasticsearch-spark-20_2.12"
+VERSION="6.8.12-SNAPSHOT"
+
+echo "INSTALLING JAR FILE: ${JAR}"
+echo "    [group].[artifact]-[version]: ${GROUP_ID}.${ARTIFACT_ID}-${VERSION}"
+echo "INSTALLING POM FILE: ${POM}"
+
+mvn install:install-file \
+    -Dfile=${JAR} \
+    -DgroupId=${GROUP_ID} \
+    -DartifactId=${ARTIFACT_ID} \
+    -Dversion=${VERSION} \
+    -Dpackaging=jar

spark/core/main/scala/org/elasticsearch/spark/rdd/EsRDDWriter.scala

@@ -64,7 +64,10 @@ private[spark] class EsRDDWriter[T: ClassTag](val serializedSettings: String,
   def write(taskContext: TaskContext, data: Iterator[T]) {
     val writer = RestService.createWriter(settings, taskContext.partitionId.toLong, -1, log)
 
-    taskContext.addTaskCompletionListener((TaskContext) => writer.close())
+    taskContext.addTaskCompletionListener(TaskContext => {
+      writer.close()
+      Unit
+    })
 
     if (runtimeMetadata) {
       writer.repository.addRuntimeFieldExtractor(metaExtractor)

spark/sql-20/build.gradle

@@ -8,8 +8,8 @@ apply plugin: 'es.hadoop.build.integration'
 apply plugin: 'scala.variants'
 
 variants {
-    defaultVersion '2.11.12'
-    targetVersions '2.10.7', '2.11.12'
+    defaultVersion '2.12.10'
+    targetVersions '2.10.7', '2.11.12', '2.12.10'
 }
 
 println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"