Improves MessageBuffer support for Java 11 (#514)

* Improves MessageBuffer support of Java 11

Switches the MessageBuffer implementation to MessageBufferU for Java
versions at least 9.

Since this implementation overrides almost all MessageBuffer's method it
is safe to allow direct buffers as argument to
MessageBuffer.wrap(ByteBuffer)

* Corrects code style

* Do not switch to MessageBufferU on Java 9+

Disables the automatic switch to MessageBufferU on Java 9+, falling back
to a manual switch through Java properties.

* Run Java 11 tests on universal buffer only

Java 11 tests without the "msgpack.universal-buffer" property set where
using the universal buffer anyway:  Java 11's
"java.specification.version" does not contain a dot, so MessageBuffer
misidentified it as Java less than 7 and switched to MessageBufferU.

* Fixes DirectBufferAccess#clean on Java 11

For Java 9+ we switch from a DirectByteBuffer.cleaner().clean() call
to Unsafe.invokeCleaner(buffer).

* Corrects style

* Corrects whitespace

* Restores Java8 tests.

* Fixes IllegalAccessExceptions

Adds missing setAccessible calls.

Author: ppkarwasz

msgpack-core/src/main/java/org/msgpack/core/buffer/DirectBufferAccess.java

@@ -19,6 +19,10 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import sun.misc.Unsafe;
 
 /**
  * Wraps the difference of access methods to DirectBuffers between Android and others.
@@ -37,20 +41,24 @@ enum DirectBufferConstructorType
     }
 
     static Method mGetAddress;
+    // For Java <=8, gets a sun.misc.Cleaner
     static Method mCleaner;
     static Method mClean;
+    // For Java >=9, invokes a jdk.internal.ref.Cleaner
+    static Method mInvokeCleaner;
 
     // TODO We should use MethodHandle for efficiency, but it is not available in JDK6
-    static Constructor byteBufferConstructor;
+    static Constructor<?> byteBufferConstructor;
     static Class<?> directByteBufferClass;
     static DirectBufferConstructorType directBufferConstructorType;
     static Method memoryBlockWrapFromJni;
 
     static {
         try {
+            final ByteBuffer direct = ByteBuffer.allocateDirect(1);
             // Find the hidden constructor for DirectByteBuffer
-            directByteBufferClass = ClassLoader.getSystemClassLoader().loadClass("java.nio.DirectByteBuffer");
-            Constructor directByteBufferConstructor = null;
+            directByteBufferClass = direct.getClass();
+            Constructor<?> directByteBufferConstructor = null;
             DirectBufferConstructorType constructorType = null;
             Method mbWrap = null;
             try {
@@ -92,17 +100,139 @@ enum DirectBufferConstructorType
             mGetAddress = directByteBufferClass.getDeclaredMethod("address");
             mGetAddress.setAccessible(true);
 
-            mCleaner = directByteBufferClass.getDeclaredMethod("cleaner");
-            mCleaner.setAccessible(true);
-
-            mClean = mCleaner.getReturnType().getDeclaredMethod("clean");
-            mClean.setAccessible(true);
+            if (MessageBuffer.javaVersion <= 8) {
+                setupCleanerJava6(direct);
+            }
+            else {
+                setupCleanerJava9(direct);
+            }
         }
         catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
+    private static void setupCleanerJava6(final ByteBuffer direct)
+    {
+        Object obj;
+        obj = AccessController.doPrivileged(new PrivilegedAction<Object>()
+        {
+            @Override
+            public Object run()
+            {
+                return getCleanerMethod(direct);
+            }
+        });
+        if (obj instanceof Throwable) {
+            throw new RuntimeException((Throwable) obj);
+        }
+        mCleaner = (Method) obj;
+
+        obj = AccessController.doPrivileged(new PrivilegedAction<Object>()
+        {
+            @Override
+            public Object run()
+            {
+                return getCleanMethod(direct, mCleaner);
+            }
+        });
+        if (obj instanceof Throwable) {
+            throw new RuntimeException((Throwable) obj);
+        }
+        mClean = (Method) obj;
+    }
+
+    private static void setupCleanerJava9(final ByteBuffer direct)
+    {
+        Object obj = AccessController.doPrivileged(new PrivilegedAction<Object>()
+        {
+            @Override
+            public Object run()
+            {
+                return getInvokeCleanerMethod(direct);
+            }
+        });
+        if (obj instanceof Throwable) {
+            throw new RuntimeException((Throwable) obj);
+        }
+        mInvokeCleaner = (Method) obj;
+    }
+
+    /**
+     * Checks if we have a usable {@link DirectByteBuffer#cleaner}.
+     * @param direct a direct buffer
+     * @return the method or an error
+     */
+    private static Object getCleanerMethod(ByteBuffer direct)
+    {
+        try {
+            Method m = direct.getClass().getDeclaredMethod("cleaner");
+            m.setAccessible(true);
+            m.invoke(direct);
+            return m;
+        }
+        catch (NoSuchMethodException e) {
+            return e;
+        }
+        catch (InvocationTargetException e) {
+            return e;
+        }
+        catch (IllegalAccessException e) {
+            return e;
+        }
+    }
+
+    /**
+     * Checks if we have a usable {@link sun.misc.Cleaner#clean}.
+     * @param direct a direct buffer
+     * @param mCleaner the {@link DirectByteBuffer#cleaner} method
+     * @return the method or null
+     */
+    private static Object getCleanMethod(ByteBuffer direct, Method mCleaner)
+    {
+        try {
+            Method m = mCleaner.getReturnType().getDeclaredMethod("clean");
+            Object c = mCleaner.invoke(direct);
+            m.setAccessible(true);
+            m.invoke(c);
+            return m;
+        }
+        catch (NoSuchMethodException e) {
+            return e;
+        }
+        catch (InvocationTargetException e) {
+            return e;
+        }
+        catch (IllegalAccessException e) {
+            return e;
+        }
+    }
+
+    /**
+     * Checks if we have a usable {@link Unsafe#invokeCleaner}.
+     * @param direct a direct buffer
+     * @return the method or an error
+     */
+    private static Object getInvokeCleanerMethod(ByteBuffer direct)
+    {
+        try {
+            // See https://bugs.openjdk.java.net/browse/JDK-8171377
+            Method m = MessageBuffer.unsafe.getClass().getDeclaredMethod(
+                "invokeCleaner", ByteBuffer.class);
+            m.invoke(MessageBuffer.unsafe, direct);
+            return m;
+        }
+        catch (NoSuchMethodException e) {
+            return e;
+        }
+        catch (InvocationTargetException e) {
+            return e;
+        }
+        catch (IllegalAccessException e) {
+            return e;
+        }
+    }
+
     static long getAddress(Object base)
     {
         try {
@@ -119,8 +249,13 @@ static long getAddress(Object base)
     static void clean(Object base)
     {
         try {
-            Object cleaner = mCleaner.invoke(base);
-            mClean.invoke(cleaner);
+            if (MessageBuffer.javaVersion <= 8) {
+                Object cleaner = mCleaner.invoke(base);
+                mClean.invoke(cleaner);
+            }
+        else {
+                mInvokeCleaner.invoke(MessageBuffer.unsafe, base);
+            }
         }
         catch (Throwable e) {
             throw new RuntimeException(e);

msgpack-core/src/main/java/org/msgpack/core/buffer/MessageBuffer.java

@@ -47,6 +47,7 @@ public class MessageBuffer
 {
     static final boolean isUniversalBuffer;
     static final Unsafe unsafe;
+    static final int javaVersion = getJavaVersion();
 
     /**
      * Reference to MessageBuffer Constructors
@@ -69,21 +70,6 @@ public class MessageBuffer
         int arrayByteBaseOffset = 16;
 
         try {
-            // Check java version
-            String javaVersion = System.getProperty("java.specification.version", "");
-            int dotPos = javaVersion.indexOf('.');
-            boolean isJavaAtLeast7 = false;
-            if (dotPos != -1) {
-                try {
-                    int major = Integer.parseInt(javaVersion.substring(0, dotPos));
-                    int minor = Integer.parseInt(javaVersion.substring(dotPos + 1));
-                    isJavaAtLeast7 = major > 1 || (major == 1 && minor >= 7);
-                }
-                catch (NumberFormatException e) {
-                    e.printStackTrace(System.err);
-                }
-            }
-
             boolean hasUnsafe = false;
             try {
                 hasUnsafe = Class.forName("sun.misc.Unsafe") != null;
@@ -97,12 +83,12 @@ public class MessageBuffer
             // Is Google App Engine?
             boolean isGAE = System.getProperty("com.google.appengine.runtime.version") != null;
 
-            // For Java6, android and JVM that has no Unsafe class, use Universal MessageBuffer
+            // For Java6, android and JVM that has no Unsafe class, use Universal MessageBuffer (based on ByteBuffer).
             useUniversalBuffer =
                     Boolean.parseBoolean(System.getProperty("msgpack.universal-buffer", "false"))
                             || isAndroid
                             || isGAE
-                            || !isJavaAtLeast7
+                            || javaVersion < 7
                             || !hasUnsafe;
 
             if (!useUniversalBuffer) {
@@ -175,6 +161,31 @@ public class MessageBuffer
         }
     }
 
+    private static int getJavaVersion()
+    {
+        String javaVersion = System.getProperty("java.specification.version", "");
+        int dotPos = javaVersion.indexOf('.');
+        if (dotPos != -1) {
+            try {
+                int major = Integer.parseInt(javaVersion.substring(0, dotPos));
+                int minor = Integer.parseInt(javaVersion.substring(dotPos + 1));
+                return major > 1 ? major : minor;
+            }
+            catch (NumberFormatException e) {
+                e.printStackTrace(System.err);
+            }
+        }
+        else {
+            try {
+                return Integer.parseInt(javaVersion);
+            }
+            catch (NumberFormatException e) {
+                e.printStackTrace(System.err);
+            }
+        }
+        return 6;
+    }
+
     /**
      * Base object for resolving the relative address of the raw byte array.
      * If base == null, the address value is a raw memory address
@@ -366,7 +377,12 @@ else if (DirectBufferAccess.isDirectByteBufferInstance(buffer.reference)) {
     {
         if (bb.isDirect()) {
             if (isUniversalBuffer) {
-                throw new UnsupportedOperationException("Cannot create MessageBuffer from a DirectBuffer on this platform");
+                // MessageBufferU overrides almost all methods, only field 'size' is used.
+                this.base = null;
+                this.address = 0;
+                this.size = bb.remaining();
+                this.reference = null;
+                return;
             }
             // Direct buffer or off-heap memory
             this.base = null;

msgpack-core/src/main/java/org/msgpack/core/buffer/MessageBufferU.java

@@ -258,4 +258,16 @@ public byte[] toByteArray()
         getBytes(0, b, 0, b.length);
         return b;
     }
+
+    @Override
+    public boolean hasArray()
+    {
+        return !wrap.isDirect();
+    }
+
+    @Override
+    public byte[] array()
+    {
+        return hasArray() ? wrap.array() : null;
+    }
 }