Add support for Spark 3.0 (#1592) (#1599)

Spark 3.0 support is added via a new source directory and subproject. Supported Scala versions are only 2.12.

Author: jbaiera

gradle.properties

@@ -26,6 +26,7 @@ spark13Version = 1.6.2
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
+spark30Version = 3.0.1
 
 # same as Spark's
 scala210Version = 2.10.7

settings.gradle

@@ -19,6 +19,10 @@ include 'sql-20'
 project(":sql-20").projectDir = new File(settingsDir, "spark/sql-20")
 project(":sql-20").name = "elasticsearch-spark-20"
 
+include 'sql-30'
+project(":sql-30").projectDir = new File(settingsDir, "spark/sql-30")
+project(":sql-30").name = "elasticsearch-spark-30"
+
 include 'storm'
 project(":storm").name = "elasticsearch-storm"
 

spark/core/build.gradle

@@ -10,6 +10,7 @@ apply plugin: 'spark.variants'
 sparkVariants {
     capabilityGroup 'org.elasticsearch.spark.variant'
     setCoreDefaultVariant "spark20scala212", spark24Version, scala212Version
+    addCoreFeatureVariant "spark30scala212", spark30Version, scala212Version
     addCoreFeatureVariant "spark20scala211", spark24Version, scala211Version
     addCoreFeatureVariant "spark20scala210", spark22Version, scala210Version
     addCoreFeatureVariant "spark13scala211", spark13Version, scala211Version

spark/sql-30/build.gradle

@@ -0,0 +1,192 @@
+import org.elasticsearch.hadoop.gradle.scala.SparkVariantPlugin
+
+description = "Elasticsearch Spark (for Spark 3.X)"
+
+apply plugin: 'java-library'
+apply plugin: 'scala'
+apply plugin: 'es.hadoop.build.integration'
+apply plugin: 'spark.variants'
+
+sparkVariants {
+    capabilityGroup 'org.elasticsearch.spark.sql.variant'
+    setDefaultVariant "spark30scala212", spark30Version, scala212Version
+
+    all { SparkVariantPlugin.SparkVariant variant ->
+        String scalaCompileTaskName = project.sourceSets
+                .getByName(variant.getSourceSetName("main"))
+                .getCompileTaskName("scala")
+
+        project.configurations {
+            create(variant.configuration('embedded')) {
+                transitive = false
+                canBeResolved = true
+            }
+            getByName(variant.configuration('implementation')) {
+                extendsFrom project.configurations.getByName(variant.configuration('embedded'))
+            }
+        }
+
+        // Configure main compile task
+        project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
+            configure(compileScala.scalaCompileOptions.forkOptions) {
+                memoryMaximumSize = '1g'
+                jvmArgs = ['-XX:MaxPermSize=512m']
+            }
+            compileScala.scalaCompileOptions.additionalParameters = [
+                    "-feature",
+                    "-unchecked",
+                    "-deprecation",
+                    "-Xfuture",
+                    "-Yno-adapted-args",
+                    "-Ywarn-dead-code",
+                    "-Ywarn-numeric-widen",
+                    "-Xfatal-warnings"
+            ]
+        }
+
+        dependencies {
+            add(variant.configuration('embedded'), project(":elasticsearch-hadoop-mr"))
+            add(variant.configuration('embedded'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+
+            add(variant.configuration('api'), "org.scala-lang:scala-library:$variant.scalaVersion")
+            add(variant.configuration('api'), "org.scala-lang:scala-reflect:$variant.scalaVersion")
+            add(variant.configuration('api'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'javax.servlet'
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('implementation'), "org.apache.spark:spark-sql_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.6") {
+                because 'spark exposes slf4j components in traits that we need to extend'
+            }
+            add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1")
+            add(variant.configuration('implementation'), "javax.xml.bind:jaxb-api:2.3.1")
+            add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:2.5.0")
+            add(variant.configuration('implementation'), "org.apache.spark:spark-catalyst_${variant.scalaMajorVersion}:$variant.sparkVersion")
+            add(variant.configuration('implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            // Scala compiler needs these for arcane reasons, but they are not used in the api nor the runtime
+            add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7")
+            add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.6.6")
+            add(variant.configuration('compileOnly'), "org.json4s:json4s-ast_${variant.scalaMajorVersion}:3.6.6")
+            add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:$variant.sparkVersion")
+
+            add(variant.configuration('test', 'implementation'), project(":test:shared"))
+            add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2")
+            add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'javax.servlet'
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-sql_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('itest', 'implementation'), project(":test:shared"))
+            add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('additionalSources'), project(":elasticsearch-hadoop-mr"))
+            add(variant.configuration('javadocSources'), project(":elasticsearch-hadoop-mr"))
+
+            add(variant.configuration('additionalSources'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+            add(variant.configuration('javadocSources'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+        }
+
+        def javaFilesOnly = { FileTreeElement spec ->
+            spec.file.name.endsWith('.java') || spec.isDirectory()
+        }
+
+        // Add java files from scala source set to javadocSourceElements.
+        project.fileTree("src/main/scala").include(javaFilesOnly).each {
+            project.artifacts.add(variant.configuration('javadocSourceElements'), it)
+        }
+
+        // Configure java source generation for javadoc purposes
+        String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
+        Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
+        scalaCompilerPlugin.defaultDependencies { dependencies ->
+            dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.13"))
+        }
+
+        ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile
+        compileScala.scalaCompileOptions.with {
+            additionalParameters = [
+                    "-Xplugin:" + configurations.getByName(variant.configuration('scalaCompilerPlugin')).asPath,
+                    "-P:genjavadoc:out=$generatedJavaDirectory".toString()
+            ]
+        }
+        // Export generated Java code from the genjavadoc compiler plugin
+        artifacts {
+            add(variant.configuration('javadocSourceElements'), project.file(generatedJavaDirectory)) {
+                builtBy compileScala
+            }
+        }
+        tasks.getByName(variant.taskName('javadoc')) {
+            dependsOn compileScala
+            source(generatedJavaDirectory)
+        }
+
+        scaladoc {
+            title = "${rootProject.description} ${version} API"
+        }
+    }
+}
+
+// deal with the messy conflicts out there
+// Ignore the scalaCompilerPlugin configurations since it is immediately resolved to configure the scala compiler tasks
+configurations.matching{ it.name.contains('CompilerPlugin') == false }.all { Configuration conf ->
+    conf.resolutionStrategy {
+        eachDependency { details ->
+            // change all javax.servlet artifacts to the one used by Spark otherwise these will lead to
+            // SecurityException (signer information wrong)
+            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
+                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
+            }
+        }
+    }
+    conf.exclude group: "org.mortbay.jetty"
+}
+
+tasks.withType(ScalaCompile) { ScalaCompile task ->
+    task.sourceCompatibility = project.ext.minimumRuntimeVersion
+    task.targetCompatibility = project.ext.minimumRuntimeVersion
+    task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
+}
+
+// Embed the embedded dependencies in the final jar after all configuration is complete
+sparkVariants {
+    all { SparkVariantPlugin.SparkVariant variant ->
+        tasks.getByName(variant.taskName('jar')) {
+            dependsOn(project.configurations.getByName(variant.configuration('embedded')))
+            // TODO: Is there a way to do this lazily? This looks like it resolves the configuration.
+            from(project.configurations.getByName(variant.configuration('embedded')).collect { it.isDirectory() ? it : zipTree(it)}) {
+                include "org/elasticsearch/**"
+                include "esh-build.properties"
+                include "META-INF/services/*"
+            }
+        }
+    }
+}

spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1

@@ -0,0 +1 @@
+5043bfebc3db072ed80fbd362e7caf00e885d8ae