Merge remote-tracking branch 'origin/main' into impl_for_wraps

Author: gf2121

.github/workflows/verify-changelog-and-set-milestone.yml

@@ -45,8 +45,7 @@ jobs:
           # Add and fetch full history of the base branch
           BASE_BRANCH=${{ github.event.pull_request.base.ref }}
           # We need this to ensure we will be able to find the merge base
-          git fetch --unshallow || true
-          git fetch upstream $BASE_BRANCH
+          git fetch upstream $BASE_BRANCH --unshallow
 
           # Find merge base commit to diff against
           BASE_COMMIT=$(git merge-base HEAD upstream/$BASE_BRANCH)
@@ -57,8 +56,7 @@ jobs:
           git log --oneline -n 3 HEAD
 
           echo "Diff:"
-          git diff --name-only $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}
-          if ! git diff --name-only $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}; then
+          if git diff --exit-code --name-only $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}; then
             echo "Change log file:${{ env.CHANGE_LOG_FILE }} does not contains an entry corresponding to changes introduced in PR. Please add a changelog entry."
             exit 0
           else

gradle/generation/extract-jdk-apis.gradle

@@ -23,18 +23,10 @@ configure(project(":lucene:core")) {
     mrjarJavaVersions = [ 24 ]
   }
 
-  configurations {
-    apiextractor
-  }
-
-  dependencies {
-    apiextractor deps.asm.core
-  }
-
   plugins.withType(JavaPlugin) {
     mrjarJavaVersions.each { jdkVersion ->
       def task = tasks.create(name: "generateJdkApiJar${jdkVersion}", type: JavaExec) {
-        description "Regenerate the API-only JAR file with public Panama Foreign & Vector API from JDK ${jdkVersion}"
+        description "Regenerate the API-only JAR file with public Panama Vector API from JDK ${jdkVersion}"
         group "generation"
 
         javaLauncher = javaToolchains.launcherFor {
@@ -46,20 +38,20 @@ configure(project(":lucene:core")) {
             javaLauncher.get()
             return true
           } catch (Exception e) {
-            logger.warn('Launcher for Java {} is not available; skipping regeneration of Panama Foreign & Vector API JAR.', jdkVersion)
+            logger.warn('Launcher for Java {} is not available; skipping regeneration of Panama Vector API JAR.', jdkVersion)
             logger.warn('Error: {}', e.cause?.message)
             logger.warn("Please make sure to point env 'JAVA{}_HOME' to exactly JDK version {} or enable Gradle toolchain auto-download.", jdkVersion, jdkVersion)
             return false
           }
         }
 
-        classpath = configurations.apiextractor
         mainClass = file("${resources}/ExtractJdkApis.java") as String
         systemProperties = [
                 'user.timezone': 'UTC',
                 'file.encoding': 'UTF-8',
         ]
         args = [
+                rootProject.minJavaVersion.toString(),
                 jdkVersion,
                 apijars.file("jdk${jdkVersion}.apijar"),
         ]

gradle/generation/extract-jdk-apis/ExtractJdkApis.java

@@ -15,6 +15,26 @@
  * limitations under the License.
  */
 import java.io.IOException;
+import java.lang.classfile.AccessFlags;
+import java.lang.classfile.Attributes;
+import java.lang.classfile.ClassFile;
+import java.lang.classfile.ClassFileBuilder;
+import java.lang.classfile.ClassFileElement;
+import java.lang.classfile.ClassFileVersion;
+import java.lang.classfile.ClassModel;
+import java.lang.classfile.ClassTransform;
+import java.lang.classfile.CodeModel;
+import java.lang.classfile.FieldModel;
+import java.lang.classfile.MethodModel;
+import java.lang.classfile.MethodTransform;
+import java.lang.classfile.attribute.InnerClassesAttribute;
+import java.lang.classfile.attribute.RuntimeInvisibleAnnotationsAttribute;
+import java.lang.classfile.attribute.SourceFileAttribute;
+import java.lang.classfile.constantpool.ClassEntry;
+import java.lang.constant.ClassDesc;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.AccessFlag;
+import java.lang.reflect.ClassFileFormatVersion;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -23,7 +43,6 @@
 import java.nio.file.attribute.FileTime;
 import java.time.Instant;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -36,42 +55,43 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-import org.objectweb.asm.AnnotationVisitor;
-import org.objectweb.asm.ClassReader;
-import org.objectweb.asm.ClassVisitor;
-import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.FieldVisitor;
-import org.objectweb.asm.MethodVisitor;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
-
 public final class ExtractJdkApis {
 
-  private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2025-04-29T00:00:00Z"));
+  private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2025-05-05T00:00:00Z"));
 
   private static final String PATTERN_VECTOR_INCUBATOR    = "jdk.incubator.vector/jdk/incubator/vector/*";
   private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";
 
   static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
       24, List.of(PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
   );
-  
+
+  private static final ClassDesc CD_PreviewFeature = ClassDesc.ofInternalName("jdk/internal/javac/PreviewFeature");
+
   public static void main(String... args) throws IOException {
-    if (args.length != 2) {
-      throw new IllegalArgumentException("Need two parameters: java version, output file");
+    if (args.length != 3) {
+      throw new IllegalArgumentException("Need two parameters: target java version, extract java version, output file");
     }
-    Integer jdk = Integer.valueOf(args[0]);
-    if (jdk.intValue() != Runtime.version().feature()) {
-      throw new IllegalStateException("Incorrect java version: " + Runtime.version().feature());
+    int targetJdk = Integer.parseInt(args[0]);
+    Integer extractJdk = Integer.valueOf(args[1]);
+    int runtimeJdk = Runtime.version().feature();
+    if (extractJdk.intValue() != runtimeJdk) {
+      throw new IllegalStateException("Incorrect runtime java version: " + runtimeJdk);
     }
-    if (!CLASSFILE_PATTERNS.containsKey(jdk)) {
-      throw new IllegalArgumentException("No support to extract stubs from java version: " + jdk);
+    if (extractJdk.intValue() < targetJdk) {
+      throw new IllegalStateException("extract java version " + extractJdk + " < target java version " + targetJdk);
     }
-    var outputPath = Paths.get(args[1]);
+    if (!CLASSFILE_PATTERNS.containsKey(extractJdk)) {
+      throw new IllegalArgumentException("No support to extract stubs from java version: " + extractJdk);
+    }
+    var outputPath = Paths.get(args[2]);
+    
+    // the output class files need to be compatible with the targetJdk of our compilation, so we need to adapt them:
+    var classFileVersion = ClassFileVersion.of(ClassFileFormatVersion.valueOf("RELEASE_" + targetJdk).major(), 0);
 
     // create JRT filesystem and build a combined FileMatcher:
     var jrtPath = Paths.get(URI.create("jrt:/")).toRealPath();
-    var patterns = CLASSFILE_PATTERNS.get(jdk).stream()
+    var patterns = CLASSFILE_PATTERNS.get(extractJdk).stream()
         .map(pattern -> jrtPath.getFileSystem().getPathMatcher("glob:" + pattern + ".class"))
         .toArray(PathMatcher[]::new);
     PathMatcher pattern = p -> Arrays.stream(patterns).anyMatch(matcher -> matcher.matches(p));
@@ -84,107 +104,94 @@ public static void main(String... args) throws IOException {
 
     // Process all class files:
     try (var out = new ZipOutputStream(Files.newOutputStream(outputPath))) {
-      process(filesToExtract, out);
+      process(filesToExtract, out, classFileVersion);
     }
   }
 
-  private static void process(List<Path> filesToExtract, ZipOutputStream out) throws IOException {
+  private static void process(List<Path> filesToExtract, ZipOutputStream out, ClassFileVersion classFileVersion) throws IOException {
+    System.out.println("Loading and analyzing " + filesToExtract.size() + " class files...");
     var classesToInclude = new HashSet<String>();
-    var references = new HashMap<String, String[]>();
-    var processed = new TreeMap<String, byte[]>();
-    System.out.println("Transforming " + filesToExtract.size() + " class files...");
+    var toProcess = new TreeMap<String, ClassModel>();
+    var cc = ClassFile.of(ClassFile.ConstantPoolSharingOption.NEW_POOL, ClassFile.DebugElementsOption.DROP_DEBUG,
+        ClassFile.LineNumbersOption.DROP_LINE_NUMBERS, ClassFile.StackMapsOption.DROP_STACK_MAPS);
     for (Path p : filesToExtract) {
-      try (var in = Files.newInputStream(p)) {
-        var reader = new ClassReader(in);
-        var cw = new ClassWriter(0);
-        var cleaner = new Cleaner(cw, classesToInclude, references);
-        reader.accept(cleaner, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
-        processed.put(reader.getClassName(), cw.toByteArray());
+      ClassModel parsed = cc.parse(p);
+      String internalName = parsed.thisClass().asInternalName();
+      toProcess.put(internalName, parsed);
+      if (isVisible(parsed.flags())) {
+        classesToInclude.add(internalName);
       }
     }
-    // recursively add all superclasses / interfaces of visible classes to classesToInclude:
+    // recursively add all superclasses / interfaces / outer classes of visible classes to classesToInclude:
     for (Set<String> a = classesToInclude; !a.isEmpty();) {
-      a = a.stream().map(references::get).filter(Objects::nonNull).flatMap(Arrays::stream).collect(Collectors.toSet());
-      classesToInclude.addAll(a);
+      classesToInclude.addAll(a = a.stream().map(toProcess::get).filter(Objects::nonNull).flatMap(ExtractJdkApis::getReferences).collect(Collectors.toSet()));
     }
     // remove all non-visible or not referenced classes:
-    processed.keySet().removeIf(Predicate.not(classesToInclude::contains));
-    System.out.println("Writing " + processed.size() + " visible classes...");
-    for (var cls : processed.entrySet()) {
-      String cn = cls.getKey();
-      System.out.println("Writing stub for class: " + cn);
-      out.putNextEntry(new ZipEntry(cn.concat(".class")).setLastModifiedTime(FIXED_FILEDATE));
-      out.write(cls.getValue());
+    toProcess.keySet().removeIf(Predicate.not(classesToInclude::contains));
+    // transformation of class files:
+    System.out.println("Writing " + toProcess.size() + " visible classes...");
+    for (var parsed : toProcess.values()) {
+      String internalName = parsed.thisClass().asInternalName();
+      System.out.println("Writing stub for class: " + internalName);
+      ClassTransform ct = ClassTransform.dropping(ce -> switch (ce) {
+        case MethodModel e -> !isVisible(e.flags());
+        case FieldModel e -> !isVisible(e.flags());
+        case SourceFileAttribute _ -> true;
+        default -> false;
+      }).andThen((builder, ce) -> {
+        switch (ce) {
+          case ClassFileVersion _ -> builder.with(classFileVersion);
+          // the PreviewFeature annotation may refer to its own inner classes and therefore we must get rid of the inner class entry:
+          case InnerClassesAttribute a -> builder.with(InnerClassesAttribute.of(a.classes().stream()
+              .filter(c -> !CD_PreviewFeature.equals(c.outerClass().map(ClassEntry::asSymbol).orElse(null)))
+              .toList()));
+          default -> builder.with(ce);
+        }
+      }).andThen(ExtractJdkApis::dropPreview)
+          .andThen(ClassTransform.transformingMethods(MethodTransform.dropping(CodeModel.class::isInstance).andThen(ExtractJdkApis::dropPreview))
+          .andThen(ClassTransform.transformingFields(ExtractJdkApis::dropPreview)));
+      out.putNextEntry(new ZipEntry(internalName.concat(".class")).setLastModifiedTime(FIXED_FILEDATE));
+      out.write(cc.transformClass(parsed, ct));
       out.closeEntry();
     }
-    classesToInclude.removeIf(processed.keySet()::contains);
-    System.out.println("Referenced classes not included: " + classesToInclude);
+    // make sure that no classes are left over except those which are in java.base module
+    classesToInclude.removeIf(toProcess.keySet()::contains);
+    var missingClasses = classesToInclude.stream().filter(internalName -> {
+      try {
+        return ClassDesc.ofInternalName(internalName).resolveConstantDesc(MethodHandles.publicLookup()).getModule() != Object.class.getModule();
+      } catch (ReflectiveOperationException _) {
+        return true;
+      }
+    }).sorted().toList();
+    if (!missingClasses.isEmpty()) {
+      throw new IllegalStateException("Some referenced classes are not publicly available in java.base module: " + missingClasses);
+    }
   }
 
-  static boolean isVisible(int access) {
-    return (access & (Opcodes.ACC_PROTECTED | Opcodes.ACC_PUBLIC)) != 0;
+  /** returns all superclasses, interfaces, and outer classes of the parsed class as stream of internal names */
+  private static Stream<String> getReferences(ClassModel parsed) {
+    var parents = Stream.concat(parsed.superclass().stream(), parsed.interfaces().stream())
+        .map(ClassEntry::asInternalName).collect(Collectors.toSet());
+    var outerClasses = parsed.findAttributes(Attributes.innerClasses()).stream()
+        .flatMap(a -> a.classes().stream())
+        .filter(i -> parents.contains(i.innerClass().asInternalName()))
+        .flatMap(i -> i.outerClass().stream())
+        .map(ClassEntry::asInternalName);
+    return Stream.concat(parents.stream(), outerClasses);
   }
 
-  static class Cleaner extends ClassVisitor {
-    private static final String PREVIEW_ANN = "jdk/internal/javac/PreviewFeature";
-    private static final String PREVIEW_ANN_DESCR = Type.getObjectType(PREVIEW_ANN).getDescriptor();
-    
-    private final Set<String> classesToInclude;
-    private final Map<String, String[]> references;
-    
-    Cleaner(ClassWriter out, Set<String> classesToInclude, Map<String, String[]> references) {
-      super(Opcodes.ASM9, out);
-      this.classesToInclude = classesToInclude;
-      this.references = references;
-    }
-
-    @Override
-    public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
-      super.visit(Opcodes.V21, access, name, signature, superName, interfaces);
-      if (isVisible(access)) {
-        classesToInclude.add(name);
-      }
-      references.put(name, Stream.concat(Stream.of(superName), Arrays.stream(interfaces)).toArray(String[]::new));
-    }
-
-    @Override
-    public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
-      return Objects.equals(descriptor, PREVIEW_ANN_DESCR) ? null : super.visitAnnotation(descriptor, visible);
-    }
-
-    @Override
-    public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) {
-      if (!isVisible(access)) {
-        return null;
-      }
-      return new FieldVisitor(Opcodes.ASM9, super.visitField(access, name, descriptor, signature, value)) {
-        @Override
-        public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
-          return Objects.equals(descriptor, PREVIEW_ANN_DESCR) ? null : super.visitAnnotation(descriptor, visible);
-        }
-      };
+  @SuppressWarnings("unchecked") // no idea how to get generics correct!?!
+  private static <E extends ClassFileElement, B extends ClassFileBuilder<E, B>> void dropPreview(ClassFileBuilder<E, B> builder, E ele) {
+    switch (ele) {
+      case RuntimeInvisibleAnnotationsAttribute att -> builder.with((E) RuntimeInvisibleAnnotationsAttribute.of(att.annotations().stream()
+          .filter(ann -> !CD_PreviewFeature.equals(ann.classSymbol()))
+          .toList()));
+      default -> builder.with(ele);
     }
-
-    @Override
-    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
-      if (!isVisible(access)) {
-        return null;
-      }
-      return new MethodVisitor(Opcodes.ASM9, super.visitMethod(access, name, descriptor, signature, exceptions)) {
-        @Override
-        public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
-          return Objects.equals(descriptor, PREVIEW_ANN_DESCR) ? null : super.visitAnnotation(descriptor, visible);
-        }
-      };
-    }
-
-    @Override
-    public void visitInnerClass(String name, String outerName, String innerName, int access) {
-      if (!Objects.equals(outerName, PREVIEW_ANN)) {
-        super.visitInnerClass(name, outerName, innerName, access);
-      }
-    }
-    
+  }
+  
+  private static boolean isVisible(AccessFlags access) {
+    return access.has(AccessFlag.PUBLIC) || access.has(AccessFlag.PROTECTED);
   }
 
 }

gradle/libs.versions.toml

@@ -2,8 +2,6 @@
 [versions]
 # expressions/ grammars
 antlr = "4.13.2"
-# expressions/ bytecode, gradle extract-jdk-apis
-asm = "9.8"
 # test assertions
 assertj = "3.27.3"
 # analysis/phonetic/ algorithms, gradle build checksums
@@ -61,12 +59,11 @@ spatial4j = "0.8"
 # benchmark/ XML parsing
 xerces = "2.12.2"
 # external datasets decompression
-zstd = "1.5.7-2"
+zstd = "1.5.7-3"
 
 [libraries]
 antlr-core = { module = "org.antlr:antlr4", version.ref = "antlr" }
 antlr-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr" }
-asm-core = { module = "org.ow2.asm:asm", version.ref = "asm" }
 assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" }
 commons-codec = { module = "commons-codec:commons-codec", version.ref = "commons-codec" }
 commons-compress = { module = "org.apache.commons:commons-compress", version.ref = "commons-compress" }

gradlew

@@ -222,7 +222,7 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
 fi
 
 GRADLE_WRAPPER_JAR="$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
-if ! ( cd $APP_HOME/gradle/wrapper && sha256sum --status -c ${GRADLE_WRAPPER_JAR}.sha256 ); then
+if ! ( cd "$APP_HOME/gradle/wrapper" && sha256sum --status -c "${GRADLE_WRAPPER_JAR}.sha256" ); then
     "$JAVACMD" $JAVA_OPTS "$APP_HOME/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "$GRADLE_WRAPPER_JAR"
     WRAPPER_STATUS=$?
     if [ "$WRAPPER_STATUS" -eq 1 ]; then

gradlew.bat

@@ -84,7 +84,7 @@ set GRADLE_WRAPPER_JAR=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 set GRADLE_WRAPPER_CHECKSUM=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar.sha256
 
 @rem Read the expected hash from .sha256 file
-for /f "tokens=1" %%A in ("%GRADLE_WRAPPER_CHECKSUM%") do (
+for /f "tokens=1 usebackq" %%A in ("%GRADLE_WRAPPER_CHECKSUM%") do (
     set "EXPECTED=%%A"
 )
 @rem Get actual SHA-256 hash using certutil