$ cat Test.java
class Test {
    public static void main(String[] args)
            throws InterruptedException
    {
        Thread t = new Thread(() -> {
            while (true) {
                new Thread(() -> {
                    try {
                        Thread.currentThread().join();
                    }
                    catch (InterruptedException e) {
                    }
                }).start();
            }
        });

        t.start();
        Thread.currentThread().join();
    }
}

$ javac Test.java
$ java -Xmx2048k -Xms2048k Test
Locks owned:
Mutex: [0x00007f5360013c90/0x0000000000000001] Module_lock - owner: 0x00007f5360017000
# To suppress the following error report, specify this argument
# after -XX: or in .hotspotrc: SuppressErrorAt=/mutex.cpp:1333
#
# A fatal error has been detected by the Java Runtime Environment:
#
# Internal Error (/scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutex.cpp:1333), pid=20707, tid=20708
# fatal error: acquiring lock DirtyCardQ_CBL_mon/16 out of order with lock Module_lock/6 -- possible deadlock
#
# JRE version: Java(TM) SE Runtime Environment (9.0) (slowdebug build 9-internal+0-2016-10-03-221052.shshahma.OnOutOfMemoryError)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (slowdebug 9-internal+0-2016-10-03-221052.shshahma.OnOutOfMemoryError, mixed mode, tiered, compressed oops, g1 gc, linux-amd64)
# Core dump will be written. Default location: Core dumps may be processed with "/usr/libexec/abrt-hook-ccpp %s %c %p %u %g %t e" (or dumping to /scratch/shshahma/Java/OnOutOfMemoryError/core.20707)
#
# An error report file with more information is saved as:
# /scratch/shshahma/Java/OnOutOfMemoryError/hs_err_pid20707.log
#
# If you would like to submit a bug report, please visit:
# http://bugreport.java.com/bugreport/crash.jsp
#
Current thread is 20708
Dumping core ...
Aborted (core dumped)

(gdb) bt
#0 Monitor::set_owner_implementation (this=0x7ffff0012c00, new_owner=0x7ffff0017800) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutex.cpp:1331
#1 0x00007ffff635d626 in Monitor::set_owner (this=0x7ffff0012c00, owner=0x7ffff0017800) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutex.hpp:248
#2 0x00007ffff635b4b0 in Monitor::lock_without_safepoint_check (this=0x7ffff0012c00, Self=0x7ffff0017800) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutex.cpp:954
#3 0x00007ffff635b4f8 in Monitor::lock_without_safepoint_check (this=0x7ffff0012c00) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutex.cpp:958
#4 0x00007ffff5880d34 in MutexLockerEx::MutexLockerEx (this=0x7ffff7f937c0, mutex=0x7ffff0012c00, no_safepoint_check=true)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/runtime/mutexLocker.hpp:210
#5 0x00007ffff6471393 in PtrQueueSet::enqueue_complete_buffer (this=0x7ffff71b5d80 <JavaThread::_dirty_card_queue_set>, cbn=0x7fffcc010ce0)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.cpp:256
#6 0x00007ffff647133c in PtrQueueSet::process_or_enqueue_complete_buffer (this=0x7ffff71b5d80 <JavaThread::_dirty_card_queue_set>, node=0x7fffcc010ce0)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.cpp:251
#7 0x00007ffff6471143 in PtrQueue::handle_zero_index (this=0x7ffff0017da0) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.cpp:223
#8 0x00007ffff64705e7 in PtrQueue::enqueue_known_active (this=0x7ffff0017da0, ptr=0x7ffff7fab2fc) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.cpp:64
#9 0x00007ffff5dc8709 in PtrQueue::enqueue (this=0x7ffff0017da0, ptr=0x7ffff7fab2fc) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.hpp:96
#10 0x00007ffff5e3e876 in PtrQueue::enqueue (this=0x7ffff0017da0, ptr=0x7ffff7fab2fc) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/ptrQueue.hpp:90
#11 0x00007ffff5e3e198 in G1SATBCardTableLoggingModRefBS::write_ref_field_work (this=0x7ffff0046120, field=0xffe5f8e8, new_val=0xfffef600, release=false)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/g1/g1SATBCardTableModRefBS.cpp:171
#12 0x00007ffff5ae5134 in BarrierSet::write_ref_field (this=0x7ffff0046120, field=0xffe5f8e8, new_val=0xfffef600, release=false)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/gc/shared/barrierSet.inline.hpp:57
#13 0x00007ffff5ae51e4 in update_barrier_set (p=0xffe5f8e8, v=0xfffef600, release=false) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/oops/oop.inline.hpp:46
#14 0x00007ffff5ae99ce in oop_store<unsigned int> (p=0xffe5f8e8, v=0xfffef600) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/oops/oop.inline.hpp:62
#15 0x00007ffff5ae560d in oopDesc::obj_field_put (this=0xffe5f8d0, offset=24, value=0xfffef600) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/oops/oop.inline.hpp:437
#16 0x00007ffff5f66cd8 in java_lang_Class::set_module (java_class=0xffe5f8d0, module=0xfffef600) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/javaClasses.cpp:973
#17 0x00007ffff5f6630b in java_lang_Class::fixup_module_field (k=..., module=...) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/javaClasses.cpp:892
#18 0x00007ffff6341a5e in ModuleEntryTable::patch_javabase_entries (module_handle=...) at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/moduleEntry.cpp:403
#19 0x00007ffff63417c4 in ModuleEntryTable::finalize_javabase (module_handle=..., version=0x0, location=0x7ffff0377e50)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/moduleEntry.cpp:373
#20 0x00007ffff6343edd in define_javabase_module (module=0x7ffff7f94258, version=0x0, location=0x7ffff7f94248, packages=0x7ffff7f94240, __the_thread__=0x7ffff0017800)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/modules.cpp:239
#21 0x00007ffff634426b in Modules::define_module (module=0x7ffff7f94258, version=0x0, location=0x7ffff7f94248, packages=0x7ffff7f94240, __the_thread__=0x7ffff0017800)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/classfile/modules.cpp:281
#22 0x00007ffff606c6a6 in JVM_DefineModule (env=0x7ffff0017a50, module=0x7ffff7f94258, version=0x0, location=0x7ffff7f94248, packages=0x7ffff7f94240)
    at /scratch/shshahma/Java/OnOutOfMemoryError/hotspot/src/share/vm/prims/jvm.cpp:1010
#23 0x00007ffff497dd25 in Java_java_lang_reflect_Module_defineModule0 (env=0x7ffff0017a50, cls=0x7ffff7f94230, module=0x7ffff7f94258, version=0x0, location=0x7ffff7f94248, packages=0x7ffff7f94240)
    at /scratch/shshahma/Java/OnOutOfMemoryError/jdk/src/java.base/share/native/libjava/Module.c:36
#24 0x00007fffe102ddf0 in ?? ()
#25 0x0000000000000000 in ?? ()

(gdb) l
1326 locks != NULL && locks->rank() <= this->rank() &&
1327 !SafepointSynchronize::is_at_safepoint() &&
1328 !(this == Safepoint_lock && contains(locks, Terminator_lock) &&
1329 SafepointSynchronize::is_synchronizing())) {
1330 new_owner->print_owned_locks();
1331 fatal("acquiring lock %s/%d out of order with lock %s/%d -- "
1332 "possible deadlock", this->name(), this->rank(),
1333 locks->name(), locks->rank());

(gdb) p locks
$54 = (Monitor *) 0x7ffff0013d80
(gdb) p locks->rank()
$55 = 6
(gdb) p this->rank()
$56 = 16
(gdb) p SafepointSynchronize::is_at_safepoint()
$57 = false
(gdb) p this == Safepoint_lock
$58 = false
(gdb) p contains(locks, Terminator_lock)
$59 = false
(gdb) p SafepointSynchronize::is_synchronizing()
$60 = false


Please note that the issue is reproducible with slowdebug build.

$ uname -a
Linux slc09vmu.us.oracle.com 3.8.13-55.1.6.el7uek.x86_64 #2 SMP Wed Feb 11 14:18:22 PST 2015 x86_64 x86_64 x86_64 GNU/Linux