elastic · neoramon · Jun 20, 2019 · Jun 20, 2019 · Nov 26, 2019 · Nov 27, 2019
diff --git a/.gitignore b/.gitignore
@@ -20,3 +20,4 @@ metastore_db
 /spark/sql-13/with_meta_*
 out/
 localRepo/
+.DS_Store
diff --git a/build.gradle b/build.gradle
@@ -3,7 +3,8 @@ import org.elasticsearch.gradle.ConcatFilesTask
 description = 'Elasticsearch for Apache Hadoop'
 
 apply plugin: 'es.hadoop.build.root'
-
+apply plugin: 'ivy-publish'
+apply plugin: 'maven-publish'
 defaultTasks 'build'
 
 allprojects {

diff --git a/dist/licenses/spark-core_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-core_2.11-2.3.0.jar.sha1
diff --git a/dist/licenses/spark-core_2.11-2.4.3.jar.sha1 b/dist/licenses/spark-core_2.11-2.4.3.jar.sha1
@@ -0,0 +1 @@
+67bf1b8c12b5f56a07a43a94736753b9851ddf0f
diff --git a/dist/licenses/spark-sql_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-sql_2.11-2.3.0.jar.sha1
diff --git a/dist/licenses/spark-sql_2.11-2.4.3.jar.sha1 b/dist/licenses/spark-sql_2.11-2.4.3.jar.sha1
@@ -0,0 +1 @@
+4a9f05701c552651cdbcf56f2ccbf5f7b752676d
diff --git a/dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-streaming_2.11-2.3.0.jar.sha1
diff --git a/dist/licenses/spark-streaming_2.11-2.4.3.jar.sha1 b/dist/licenses/spark-streaming_2.11-2.4.3.jar.sha1
@@ -0,0 +1 @@
+f501c13c1e0953be11d248c2ccda280c328b2963
diff --git a/dist/licenses/spark-yarn_2.11-2.3.0.jar.sha1 b/dist/licenses/spark-yarn_2.11-2.3.0.jar.sha1
diff --git a/dist/licenses/spark-yarn_2.11-2.4.3.jar.sha1 b/dist/licenses/spark-yarn_2.11-2.4.3.jar.sha1
@@ -0,0 +1 @@
+2935da0dcaa89d85e897b457d3b077d20d490227
diff --git a/gradle.properties b/gradle.properties
@@ -18,12 +18,14 @@ jodaVersion = 1.6
 jacksonVersion = 1.8.8
 # Spark
 spark13Version = 1.6.2
-spark20Version = 2.3.0
+spark20Version = 2.4.3
 # same as Spark's
 scala210Version = 2.10.7
 scala210MajorVersion = 2.10
 scala211Version = 2.11.12
 scala211MajorVersion = 2.11
+scala212Version = 2.12.12
+scala212MajorVersion = 2.12
 
 stormVersion = 1.0.6
 

diff --git a/licenses/scala-library-2.11.12.jar.sha1 b/licenses/scala-library-2.11.12.jar.sha1
diff --git a/licenses/scala-library-2.12.10.jar.sha1 b/licenses/scala-library-2.12.10.jar.sha1
@@ -0,0 +1 @@
+3509860bc2e5b3da001ed45aca94ffbe5694dbda
diff --git a/licenses/scala-reflect-2.11.12.jar.sha1 b/licenses/scala-reflect-2.11.12.jar.sha1
diff --git a/licenses/scala-reflect-2.12.10.jar.sha1 b/licenses/scala-reflect-2.12.10.jar.sha1
@@ -0,0 +1 @@
+14cb7beb516cd8e07716133668c427792122c926
diff --git a/licenses/spark-catalyst_2.11-2.3.0.jar.sha1 b/licenses/spark-catalyst_2.11-2.3.0.jar.sha1
diff --git a/licenses/spark-catalyst_2.12-2.4.3.jar.sha1 b/licenses/spark-catalyst_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+6ca3f78d5556b2a397901e75e54d578d0a3e78ec
diff --git a/licenses/spark-core_2.11-2.3.0.jar.sha1 b/licenses/spark-core_2.11-2.3.0.jar.sha1
diff --git a/licenses/spark-core_2.12-2.4.3.jar.sha1 b/licenses/spark-core_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+fbaab70118620dd88e702d9a3223eb0cf22c68a9
diff --git a/licenses/spark-sql_2.11-2.3.0.jar.sha1 b/licenses/spark-sql_2.11-2.3.0.jar.sha1
diff --git a/licenses/spark-sql_2.12-2.4.3.jar.sha1 b/licenses/spark-sql_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+474b70ebaf068a500ad6114d51cd0b0844e3a309
diff --git a/licenses/spark-streaming_2.11-2.3.0.jar.sha1 b/licenses/spark-streaming_2.11-2.3.0.jar.sha1
diff --git a/licenses/spark-streaming_2.12-2.4.3.jar.sha1 b/licenses/spark-streaming_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+40d63ad003f0a67be5a1896b66fa300c5f30f7c8
diff --git a/licenses/spark-yarn_2.11-2.3.0.jar.sha1 b/licenses/spark-yarn_2.11-2.3.0.jar.sha1
diff --git a/licenses/spark-yarn_2.12-2.4.3.jar.sha1 b/licenses/spark-yarn_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+2f0d4c17f507044d7c389433ab8decbe58577596
diff --git a/qa/kerberos/build.gradle b/qa/kerberos/build.gradle
@@ -66,8 +66,8 @@ dependencies {
     implementation(project(":elasticsearch-hadoop-mr"))
     implementation(project(":elasticsearch-storm"))
 
-    implementation('org.scala-lang:scala-library:2.11.12')
-    implementation('org.scala-lang:scala-reflect:2.11.12')
+    implementation('org.scala-lang:scala-library:2.12.10')    
+    implementation('org.scala-lang:scala-reflect:2.12.10')
     implementation(project(":elasticsearch-spark-20"))
 
     compileOnly("com.fasterxml.jackson.module:jackson-module-scala_2.11:2.6.7.1")

diff --git a/spark/core/main/scala/org/elasticsearch/spark/rdd/AbstractEsRDD.scala b/spark/core/main/scala/org/elasticsearch/spark/rdd/AbstractEsRDD.scala
@@ -18,8 +18,7 @@
  */
 package org.elasticsearch.spark.rdd;
 
-import scala.collection.JavaConversions.collectionAsScalaIterable
-import scala.collection.JavaConversions.mapAsJavaMap
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 import org.apache.commons.logging.LogFactory
 import org.apache.spark.Partition
@@ -45,7 +44,7 @@ private[spark] abstract class AbstractEsRDD[T: ClassTag](
   @transient protected lazy val logger = LogFactory.getLog(this.getClass())
 
   override def getPartitions: Array[Partition] = {
-    esPartitions.zipWithIndex.map { case(esPartition, idx) =>
+    esPartitions.asScala.zipWithIndex.map { case(esPartition, idx) =>
       new EsPartition(id, idx, esPartition)
     }.toArray
   }
@@ -70,7 +69,7 @@ private[spark] abstract class AbstractEsRDD[T: ClassTag](
 
   @transient private[spark] lazy val esCfg = {
     val cfg = new SparkSettingsManager().load(sc.getConf).copy();
-    cfg.merge(params)
+    cfg.merge(params.asJava)
     InitializationUtils.setUserProviderIfNotSet(cfg, classOf[HadoopUserProvider], logger)
     cfg
   }

diff --git a/spark/core/main/scala/org/elasticsearch/spark/rdd/EsRDDWriter.scala b/spark/core/main/scala/org/elasticsearch/spark/rdd/EsRDDWriter.scala
@@ -64,7 +64,10 @@ private[spark] class EsRDDWriter[T: ClassTag](val serializedSettings: String,
   def write(taskContext: TaskContext, data: Iterator[T]): Unit = {
     val writer = RestService.createWriter(settings, taskContext.partitionId.toLong, -1, log)
 
-    taskContext.addTaskCompletionListener((TaskContext) => writer.close())
+    taskContext.addTaskCompletionListener(TaskContext => {
+      writer.close()
+      Unit
+    })
 
     if (runtimeMetadata) {
       writer.repository.addRuntimeFieldExtractor(metaExtractor)

diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle
@@ -9,8 +9,8 @@ apply plugin: 'es.hadoop.build.integration'
 apply plugin: 'scala.variants'
 
 variants {
-    defaultVersion '2.11.12'
-    targetVersions '2.10.7', '2.11.12'
+    defaultVersion '2.12.10'
+    targetVersions '2.10.7', '2.11.12', '2.12.10'
 }
 
 configurations {
@@ -24,7 +24,7 @@ configurations {
     if (project.ext.scalaMajorVersion != '2.10') {
         scalaCompilerPlugin {
             defaultDependencies { dependencies ->
-                dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
+                dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.16"))
             }
         }
     }

diff --git a/spark/sql-20/licenses/scala-library-2.11.12.jar.sha1 b/spark/sql-20/licenses/scala-library-2.11.12.jar.sha1
diff --git a/spark/sql-20/licenses/scala-library-2.12.10.jar.sha1 b/spark/sql-20/licenses/scala-library-2.12.10.jar.sha1
@@ -0,0 +1 @@
+3509860bc2e5b3da001ed45aca94ffbe5694dbda
diff --git a/spark/sql-20/licenses/scala-reflect-2.11.12.jar.sha1 b/spark/sql-20/licenses/scala-reflect-2.11.12.jar.sha1
diff --git a/spark/sql-20/licenses/scala-reflect-2.12.10.jar.sha1 b/spark/sql-20/licenses/scala-reflect-2.12.10.jar.sha1
@@ -0,0 +1 @@
+14cb7beb516cd8e07716133668c427792122c926
diff --git a/spark/sql-20/licenses/spark-catalyst_2.11-2.3.0.jar.sha1 b/spark/sql-20/licenses/spark-catalyst_2.11-2.3.0.jar.sha1
diff --git a/spark/sql-20/licenses/spark-catalyst_2.12-2.4.3.jar.sha1 b/spark/sql-20/licenses/spark-catalyst_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+6ca3f78d5556b2a397901e75e54d578d0a3e78ec
diff --git a/spark/sql-20/licenses/spark-core_2.11-2.3.0.jar.sha1 b/spark/sql-20/licenses/spark-core_2.11-2.3.0.jar.sha1
diff --git a/spark/sql-20/licenses/spark-core_2.12-2.4.3.jar.sha1 b/spark/sql-20/licenses/spark-core_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+fbaab70118620dd88e702d9a3223eb0cf22c68a9
diff --git a/spark/sql-20/licenses/spark-sql_2.11-2.3.0.jar.sha1 b/spark/sql-20/licenses/spark-sql_2.11-2.3.0.jar.sha1
diff --git a/spark/sql-20/licenses/spark-sql_2.12-2.4.3.jar.sha1 b/spark/sql-20/licenses/spark-sql_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+474b70ebaf068a500ad6114d51cd0b0844e3a309
diff --git a/spark/sql-20/licenses/spark-streaming_2.11-2.3.0.jar.sha1 b/spark/sql-20/licenses/spark-streaming_2.11-2.3.0.jar.sha1
diff --git a/spark/sql-20/licenses/spark-streaming_2.12-2.4.3.jar.sha1 b/spark/sql-20/licenses/spark-streaming_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+40d63ad003f0a67be5a1896b66fa300c5f30f7c8
diff --git a/spark/sql-20/licenses/spark-yarn_2.11-2.3.0.jar.sha1 b/spark/sql-20/licenses/spark-yarn_2.11-2.3.0.jar.sha1
diff --git a/spark/sql-20/licenses/spark-yarn_2.12-2.4.3.jar.sha1 b/spark/sql-20/licenses/spark-yarn_2.12-2.4.3.jar.sha1
@@ -0,0 +1 @@
+2f0d4c17f507044d7c389433ab8decbe58577596