Copy the Spark 2.X project for upgrade to Spark 3

Author: jbaiera

spark/sql-30/build.gradle

@@ -0,0 +1,206 @@
+import org.elasticsearch.hadoop.gradle.scala.SparkVariantPlugin
+
+description = "Elasticsearch Spark (for Spark 2.X)"
+
+apply plugin: 'java-library'
+apply plugin: 'scala'
+apply plugin: 'es.hadoop.build.integration'
+apply plugin: 'spark.variants'
+
+sparkVariants {
+    capabilityGroup 'org.elasticsearch.spark.sql.variant'
+    setDefaultVariant "spark20scala211", spark24Version, scala211Version
+    addFeatureVariant "spark20scala210", spark22Version, scala210Version
+
+    all { SparkVariantPlugin.SparkVariant variant ->
+        String scalaCompileTaskName = project.sourceSets
+                .getByName(variant.getSourceSetName("main"))
+                .getCompileTaskName("scala")
+
+        project.configurations {
+            create(variant.configuration('embedded')) {
+                transitive = false
+                canBeResolved = true
+            }
+            getByName(variant.configuration('implementation')) {
+                extendsFrom project.configurations.getByName(variant.configuration('embedded'))
+            }
+        }
+
+        // Configure main compile task
+        project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
+            configure(compileScala.scalaCompileOptions.forkOptions) {
+                memoryMaximumSize = '1g'
+                jvmArgs = ['-XX:MaxPermSize=512m']
+            }
+            compileScala.scalaCompileOptions.additionalParameters = [
+                    "-feature",
+                    "-unchecked",
+                    "-deprecation",
+                    "-Xfuture",
+                    "-Yno-adapted-args",
+                    "-Ywarn-dead-code",
+                    "-Ywarn-numeric-widen",
+                    "-Xfatal-warnings"
+            ]
+        }
+
+        dependencies {
+            add(variant.configuration('embedded'), project(":elasticsearch-hadoop-mr"))
+            add(variant.configuration('embedded'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+
+            add(variant.configuration('api'), "org.scala-lang:scala-library:$variant.scalaVersion")
+            add(variant.configuration('api'), "org.scala-lang:scala-reflect:$variant.scalaVersion")
+            add(variant.configuration('api'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'javax.servlet'
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('implementation'), "org.apache.spark:spark-sql_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('implementation'), "org.slf4j:slf4j-api:1.7.6") {
+                because 'spark exposes slf4j components in traits that we need to extend'
+            }
+            add(variant.configuration('implementation'), "commons-logging:commons-logging:1.1.1")
+            add(variant.configuration('implementation'), "javax.xml.bind:jaxb-api:2.3.1")
+            add(variant.configuration('implementation'), "com.google.protobuf:protobuf-java:2.5.0")
+            add(variant.configuration('implementation'), "org.apache.spark:spark-catalyst_${variant.scalaMajorVersion}:$variant.sparkVersion")
+            add(variant.configuration('implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            // Scala compiler needs these for arcane reasons, but they are not used in the api nor the runtime
+            add(variant.configuration('compileOnly'), "com.fasterxml.jackson.core:jackson-annotations:2.6.7")
+            add(variant.configuration('compileOnly'), "org.json4s:json4s-jackson_${variant.scalaMajorVersion}:3.2.11")
+            add(variant.configuration('compileOnly'), "org.apache.spark:spark-tags_${variant.scalaMajorVersion}:$variant.sparkVersion")
+
+            if ('2.10' == scalaMajorVersion) {
+                add(variant.configuration('implementation'), "org.apache.spark:spark-unsafe_${variant.scalaMajorVersion}:$variant.sparkVersion")
+                add(variant.configuration('implementation'), "org.apache.avro:avro:1.7.7")
+                add(variant.configuration('implementation'), "log4j:log4j:1.2.17")
+                add(variant.configuration('implementation'), "com.google.code.findbugs:jsr305:2.0.1")
+                add(variant.configuration('implementation'), "org.json4s:json4s-ast_2.10:3.2.10")
+                add(variant.configuration('implementation'), "com.esotericsoftware.kryo:kryo:2.21")
+                add(variant.configuration('compileOnly'), "org.apache.hadoop:hadoop-annotations:${project.ext.hadoopVersion}")
+                add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-core-asl:${project.ext.jacksonVersion}")
+                add(variant.configuration('compileOnly'), "org.codehaus.jackson:jackson-mapper-asl:${project.ext.jacksonVersion}")
+            }
+
+            add(variant.configuration('test', 'implementation'), project(":test:shared"))
+            add(variant.configuration('test', 'implementation'), "org.elasticsearch:securemock:1.2")
+            add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-core_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'javax.servlet'
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('test', 'implementation'), "org.apache.spark:spark-sql_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('itest', 'implementation'), project(":test:shared"))
+            add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-yarn_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+            add(variant.configuration('itest', 'implementation'), "org.apache.spark:spark-streaming_${variant.scalaMajorVersion}:$variant.sparkVersion") {
+                exclude group: 'org.apache.hadoop'
+            }
+
+            add(variant.configuration('additionalSources'), project(":elasticsearch-hadoop-mr"))
+            add(variant.configuration('javadocSources'), project(":elasticsearch-hadoop-mr"))
+
+            add(variant.configuration('additionalSources'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+            add(variant.configuration('javadocSources'), project(":elasticsearch-spark")) {
+                capabilities {
+                    requireCapability("org.elasticsearch.spark.variant:$variant.name:$project.version")
+                }
+            }
+        }
+
+        def javaFilesOnly = { FileTreeElement spec ->
+            spec.file.name.endsWith('.java') || spec.isDirectory()
+        }
+
+        // Add java files from scala source set to javadocSourceElements.
+        project.fileTree("src/main/scala").include(javaFilesOnly).each {
+            project.artifacts.add(variant.configuration('javadocSourceElements'), it)
+        }
+
+        if (variant.scalaMajorVersion != '2.10') {
+            // Configure java source generation for javadoc purposes
+            String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
+            Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
+            scalaCompilerPlugin.defaultDependencies { dependencies ->
+                dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.13"))
+            }
+
+            ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile
+            compileScala.scalaCompileOptions.with {
+                additionalParameters = [
+                        "-Xplugin:" + configurations.getByName(variant.configuration('scalaCompilerPlugin')).asPath,
+                        "-P:genjavadoc:out=$generatedJavaDirectory".toString()
+                ]
+            }
+            // Export generated Java code from the genjavadoc compiler plugin
+            artifacts {
+                add(variant.configuration('javadocSourceElements'), project.file(generatedJavaDirectory)) {
+                    builtBy compileScala
+                }
+            }
+            tasks.getByName(variant.taskName('javadoc')) {
+                dependsOn compileScala
+                source(generatedJavaDirectory)
+            }
+        }
+
+        scaladoc {
+            title = "${rootProject.description} ${version} API"
+        }
+    }
+}
+
+// deal with the messy conflicts out there
+// Ignore the scalaCompilerPlugin configurations since it is immediately resolved to configure the scala compiler tasks
+configurations.matching{ it.name.contains('CompilerPlugin') == false }.all { Configuration conf ->
+    conf.resolutionStrategy {
+        eachDependency { details ->
+            // change all javax.servlet artifacts to the one used by Spark otherwise these will lead to
+            // SecurityException (signer information wrong)
+            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
+                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
+            }
+        }
+    }
+    conf.exclude group: "org.mortbay.jetty"
+}
+
+tasks.withType(ScalaCompile) { ScalaCompile task ->
+    task.sourceCompatibility = project.ext.minimumRuntimeVersion
+    task.targetCompatibility = project.ext.minimumRuntimeVersion
+    task.options.forkOptions.executable = new File(project.ext.runtimeJavaHome, 'bin/java').absolutePath
+}
+
+// Embed the embedded dependencies in the final jar after all configuration is complete
+sparkVariants {
+    all { SparkVariantPlugin.SparkVariant variant ->
+        tasks.getByName(variant.taskName('jar')) {
+            dependsOn(project.configurations.getByName(variant.configuration('embedded')))
+            // TODO: Is there a way to do this lazily? This looks like it resolves the configuration.
+            from(project.configurations.getByName(variant.configuration('embedded')).collect { it.isDirectory() ? it : zipTree(it)}) {
+                include "org/elasticsearch/**"
+                include "esh-build.properties"
+                include "META-INF/services/*"
+            }
+        }
+    }
+}

spark/sql-30/licenses/commons-logging-1.1.1.jar.sha1

@@ -0,0 +1 @@
+5043bfebc3db072ed80fbd362e7caf00e885d8ae