Fix the illegal lookupClass issue when using MethodHandles.Lookup on JDK 8 (#594)

* Fix MethodHandles.
* Clarify tryCatchFinally docs
* Add a test for custom `MethodHandles.lookup`
* Enhance logging

Author: zhelenskiy

bootstrap/src/sun/nio/ch/lincheck/Injections.java

@@ -11,6 +11,8 @@
 package sun.nio.ch.lincheck;
 
 import java.lang.invoke.CallSite;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Constructor;
 
 import static sun.nio.ch.lincheck.Types.convertAsmMethodType;
 
@@ -634,4 +636,46 @@ public static int getNextEventId(String type) {
     public static void setLastMethodCallEventId() {
         getEventTracker().setLastMethodCallEventId();
     }
+
+    /**
+     * Attempts to retrieve the Class object associated with the given class name.
+     * If the class is not found, it returns null instead of throwing an exception.
+     *
+     * @param name the fully qualified name of the desired class
+     * @return the Class object for the class with the specified name, 
+     *         or null if the class cannot be located
+     */
+    public static Class<?> getClassForNameOrNull(String name) {
+        try {
+            return Class.forName(name);
+        } catch (ClassNotFoundException e) {
+            return null;
+        }
+    }
+
+
+    private static Constructor<MethodHandles.Lookup> lookUpPrivateConstructor = null;
+
+    /**
+     * Provides a trusted MethodHandles.Lookup for the given class,
+     * bypassing JDK 8-specific restrictions written in the {@link MethodHandles} implementation.
+     * <p> 
+     * In JDK 8 there are additional <a href="https://github.com/frohoff/jdk8u-jdk/blob/da0da73ab82ed714dc5be94acd2f0d00fbdfe2e9/src/share/classes/java/lang/invoke/MethodHandles.java#L681">restrictions</a>,
+     * breaking {@code MethodHandles.lookup()} from being called from the Java standard library.
+     * Calling it is necessary for {@code invokedynamic} handling in the trace debugger.
+     * <p> 
+     * The function instead calls a private constructor with the TRUSTED mode via reflection.
+     *
+     * @param callerClass the class for which the trusted {@link MethodHandles.Lookup} is to be created
+     * @return a trusted {@link MethodHandles.Lookup} instance for the specified class
+     * @throws Exception if an error occurs while creating the trusted lookup instance
+     */
+    public static MethodHandles.Lookup trustedLookup(Class<?> callerClass) throws Exception {
+        if (lookUpPrivateConstructor == null) {
+            Constructor<MethodHandles.Lookup> declaredConstructor = MethodHandles.Lookup.class.getDeclaredConstructor(Class.class, int.class);
+            declaredConstructor.setAccessible(true);
+            lookUpPrivateConstructor = declaredConstructor;
+        }
+        return lookUpPrivateConstructor.newInstance(callerClass, -1);
+    }
 }

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/ManagedStrategy.kt

@@ -1477,6 +1477,10 @@ abstract class ManagedStrategy(
         params: Array<Any?>,
         result: Any?
     ) = runInsideIgnoredSection {
+        if (deterministicMethodDescriptor != null) {
+            Logger.debug { "On method return with descriptor $deterministicMethodDescriptor: $result" }
+        }
+        
         require(deterministicMethodDescriptor is DeterministicMethodDescriptor<*, *>?)
         // process intrinsic candidate methods
         if (MethodIds.isIntrinsicMethod(methodId)) {
@@ -1531,6 +1535,9 @@ abstract class ManagedStrategy(
         params: Array<Any?>,
         throwable: Throwable
     ) = runInsideIgnoredSection {
+        if (deterministicMethodDescriptor != null) {
+            Logger.debug { "On method exception with descriptor $deterministicMethodDescriptor:\n${throwable.stackTraceToString()}" }
+        }
         require(deterministicMethodDescriptor is DeterministicMethodDescriptor<*, *>?)
         if (isInTraceDebuggerMode && isFirstReplay && deterministicMethodDescriptor != null) {
             deterministicMethodDescriptor.saveFirstResult(receiver, params, KResult.failure(throwable)) {

src/jvm/main/org/jetbrains/kotlinx/lincheck/transformation/LincheckClassVisitor.kt

@@ -16,6 +16,7 @@ import org.objectweb.asm.Opcodes.*
 import org.objectweb.asm.commons.*
 import org.jetbrains.kotlinx.lincheck.transformation.InstrumentationMode.*
 import org.jetbrains.kotlinx.lincheck.transformation.transformers.*
+import org.jetbrains.kotlinx.lincheck.util.Logger
 import sun.nio.ch.lincheck.*
 
 internal class LincheckClassVisitor(
@@ -69,7 +70,10 @@ internal class LincheckClassVisitor(
         exceptions: Array<String>?
     ): MethodVisitor {
         var mv = super.visitMethod(access, methodName, desc, signature, exceptions)
-        if (access and ACC_NATIVE != 0) return mv
+        if (access and ACC_NATIVE != 0) {
+            Logger.debug { "Skipping transformation of the native method $className.$methodName" }
+            return mv
+        }
         if (instrumentationMode == STRESS) {
             return if (methodName != "<clinit>" && methodName != "<init>") {
                 CoroutineCancellabilitySupportTransformer(mv, access, className, methodName, desc)
@@ -140,7 +144,9 @@ internal class LincheckClassVisitor(
             if (isInTraceDebuggerMode) {
                 // Lincheck does not support true identity hash codes (it always uses zeroes),
                 // so there is no need for the `DeterministicInvokeDynamicTransformer` there.
-                mv = DeterministicInvokeDynamicTransformer(fileName, className, methodName, mv.newAdapter())
+                mv = DeterministicInvokeDynamicTransformer(
+                    fileName, className, methodName, classVersion, mv.newAdapter()
+                )
             }
             mv = run {
                 val st = ConstructorArgumentsSnapshotTrackerTransformer(fileName, className, methodName, mv.newAdapter(), classVisitor::isInstanceOf)
@@ -177,7 +183,7 @@ internal class LincheckClassVisitor(
         if (isInTraceDebuggerMode) {
             // Lincheck does not support true identity hash codes (it always uses zeroes),
             // so there is no need for the `DeterministicInvokeDynamicTransformer` there.
-            mv = DeterministicInvokeDynamicTransformer(fileName, className, methodName, mv.newAdapter())
+            mv = DeterministicInvokeDynamicTransformer(fileName, className, methodName, classVersion, mv.newAdapter())
         } else {
             // In trace debugger mode we record hash codes of tracked objects and substitute them on re-run,
             // otherwise, we track all hash code calls in the instrumented code

src/jvm/main/org/jetbrains/kotlinx/lincheck/transformation/TransformationUtils.kt

@@ -184,7 +184,10 @@ internal fun GeneratorAdapter.storeArguments(methodDescriptor: String): IntArray
 /**
  * Executes a try-catch-finally block within the context of the GeneratorAdapter.
  * 
- * Attention: this method does not insert `finally` blocks before inner return and throw statements.
+ * **Attention**:
+ * * This method does not insert `finally` blocks before inner return and throw statements.
+ * * It is forbidden to jump from the blocks outside and between them.
+ * * The operand stack must be empty by the beginning of the [tryCatchFinally].
  *
  * @param tryBlock The code block to be executed in the try section.
  * @param exceptionType The type of exception to be caught in the `catch` section, or null to catch all exceptions.

src/jvm/main/org/jetbrains/kotlinx/lincheck/transformation/transformers/DeterministicInvokeDynamicTransformer.kt

@@ -40,6 +40,7 @@ internal class DeterministicInvokeDynamicTransformer(
     fileName: String,
     className: String,
     methodName: String,
+    private val classVersion: Int,
     adapter: GeneratorAdapter
 ) : ManagedStrategyMethodVisitor(fileName, className, methodName, adapter) {
     override fun visitInvokeDynamicInsn(
@@ -208,7 +209,24 @@ internal class DeterministicInvokeDynamicTransformer(
         require(bootstrapMethodParameterTypes[2] == methodTypeType)
 
         // pushing predefined arguments manually
-        invokeStatic(MethodHandles::lookup)
+        if (classVersion == 52 /* Java 8 */) {
+            invokeInsideIgnoredSection {
+                push(this@DeterministicInvokeDynamicTransformer.className.replace('/', '.'))
+                invokeStatic(Injections::getClassForNameOrNull)
+                dup()
+                val endLabel = newLabel()
+                val ifNonNullLabel = newLabel()
+                ifNonNull(ifNonNullLabel)
+                pop()
+                invokeStatic(MethodHandles::lookup)
+                goTo(endLabel)
+                visitLabel(ifNonNullLabel)
+                invokeStatic(Injections::trustedLookup)
+                visitLabel(endLabel)
+            }
+        } else {
+            invokeStatic(MethodHandles::lookup)
+        }
         visitLdcInsn(name)
         visitLdcInsn(Type.getMethodType(descriptor))
         val jvmPredefinedParametersCount = 3

src/jvm/test/org/jetbrains/kotlinx/lincheck_test/trace_debugger/CollectorsTest.java

@@ -0,0 +1,40 @@
+/*
+ * Lincheck
+ *
+ * Copyright (C) 2019 - 2025 JetBrains s.r.o.
+ *
+ * This Source Code Form is subject to the terms of the
+ * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+ * with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package org.jetbrains.kotlinx.lincheck_test.trace_debugger;
+
+import org.jetbrains.kotlinx.lincheck.annotations.Operation;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * This test is intended to check that {@link MethodHandles} is successfully created even on Java 8.
+ * It is necessary for invokedynamic handling in the trace debugger.
+ * <p> 
+ * However, due to workaround for <a href="https://github.com/JetBrains/lincheck/issues/500">the issue</a>,
+ * the tests are actually run on the JDK 17.
+ * Nevertheless, when the issue is resolved,
+ * the tests will actually check the correct {@link MethodHandles.Lookup} creation.
+ */
+public class CollectorsTest extends AbstractDeterministicTest {
+    @Operation
+    public Object operation() {
+        return Arrays.stream(
+                new List[] {
+                    Arrays.stream(new String[]{"Hello ", "w"}).collect(Collectors.toList()),
+                    Arrays.stream(new String[]{"Hello ", "w"}).collect(Collectors.toList())
+                }
+        ).flatMap(Collection::stream).collect(Collectors.toList());
+    }
+}