The test still fails after JDK-8163553 was fixed:

java.lang.LinkageError
at java.lang.invoke.MethodHandleNatives.linkMethodImpl(java.base@9-internal/MethodHandleNatives.java:386)
at java.lang.invoke.MethodHandleNatives.linkMethod(java.base@9-internal/MethodHandleNatives.java:366)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.compareAndSetState(java.base@9-internal/AbstractQueuedSynchronizer.java:600)
at java.util.concurrent.locks.ReentrantLock$Sync.nonfairTryAcquire(java.base@9-internal/ReentrantLock.java:130)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.tryAcquire(java.base@9-internal/ReentrantLock.java:199)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@9-internal/AbstractQueuedSynchronizer.java:1225)
at java.util.concurrent.locks.ReentrantLock.lock(java.base@9-internal/ReentrantLock.java:267)
at ReservedStackTest$ReentrantLockTest.lockAndCall(ReservedStackTest.java:168)
at ReservedStackTest$ReentrantLockTest.run(ReservedStackTest.java:158)
at ReservedStackTest$RunWithSOEContext.recursiveCall(ReservedStackTest.java:233)
[...]
at ReservedStackTest$RunWithSOEContext.recursiveCall(ReservedStackTest.java:226)
Caused by: java.lang.StackOverflowError
at sun.invoke.util.Wrapper.forPrimitiveType(java.base@9-internal/Wrapper.java:269)
at java.lang.invoke.MethodTypeForm.<init>(java.base@9-internal/MethodTypeForm.java:176)