Shenandoah: Deadlock related to OOM-during-evacuation

hotspot_gc_shenandoah occasionally times out when running TestGCBasherWithShenandoah. I reproduced with this cmdline:

java -cp $CP_TO_GCBASHER -Xlog:gc*=info,nmethod+barrier=trace -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu -XX:ShenandoahGCHeuristics=aggressive -XX:+DeoptimizeNMethodBarriersALot -XX:-Inline -XX:+ShenandoahOOMDuringEvacALot gc.stress.gcbasher.TestGCBasher 120000

Thread 13 (Thread 0x7fd98a8ed700 (LWP 180903)):
#0 0x00007fd9ba6d2260 in nanosleep () from /lib64/libpthread.so.0
#1 0x00007fd9b938b1cf in os::naked_short_nanosleep (ns=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/posix/os_posix.cpp:695
#2 os::naked_short_sleep (ms=ms@entry=1) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/posix/os_posix.cpp:701
#3 0x00007fd9b95763ba in ShenandoahEvacOOMHandler::wait_for_no_evac_threads (this=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:40
#4 ShenandoahEvacOOMHandler::handle_out_of_memory_during_evacuation (this=this@entry=0x7fd9b0072f20) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:93
#5 0x00007fd9b93127ac in ShenandoahHeap::evacuate_object (this=0x7fd9b0072480, p=..., thread=0x7fd990003b90) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp:292
#6 0x00007fd9b953a454 in ShenandoahEvacuateUpdateRootsClosure<64ul>::do_oop_work<oop> (this=<optimized out>, p=0x7fd99904748c) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/oops/oopsHierarchy.hpp:89
#7 0x00007fd9b953a706 in ShenandoahEvacuateUpdateRootsClosure<64ul>::do_oop (p=<optimized out>, this=0x7fd98a8eccf0)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp:110
#8 ShenandoahNMethod::oops_do (fix_relocations=<optimized out>, oops=<optimized out>, this=<optimized out>)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp:59
#9 ShenandoahNMethod::heal_nmethod_metadata (nmethod_data=0x7fd91c07b6e0) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp:77
#10 ShenandoahNMethodUnlinkClosure::do_nmethod (this=0x7fd9b43877f8, nm=0x7fd999046d90) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:276
#11 0x00007fd9b95c176c in ShenandoahNMethodTableSnapshot::concurrent_nmethods_do (this=0x7fd99000a940, cl=0x7fd9b43877f8)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp:34
#12 0x00007fd9b953815f in ShenandoahUnlinkTask::work (this=0x7fd9b43877e0, worker_id=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:315
#13 0x00007fd9b98766f4 in GangWorker::run_task (this=this@entry=0x7fd990003b90, data=...) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/workgroup.cpp:339
#14 0x00007fd9b9876838 in GangWorker::loop (this=0x7fd990003b90) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/workgroup.cpp:349
#15 0x00007fd9b974da26 in Thread::call_run (this=this@entry=0x7fd990003b90) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/runtime/thread.cpp:393
#16 0x00007fd9b937e636 in thread_native_entry (thread=0x7fd990003b90) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/linux/os_linux.cpp:791
#17 0x00007fd9ba6c82de in start_thread () from /lib64/libpthread.so.0
#18 0x00007fd9ba1f5e83 in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7fd9b4388700 (LWP 180895)):
#0 0x00007fd9ba6d0e66 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1 0x00007fd9ba6d0f58 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007fd9b94ce744 in PosixSemaphore::wait (this=0x7fd9b00740f8) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/posix/semaphore_posix.cpp:64
#3 0x00007fd9b98773f9 in Semaphore::wait (this=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/runtime/semaphore.hpp:55
#4 SemaphoreGangTaskDispatcher::coordinator_execute_on_workers (this=0x7fd9b0074020, task=0x7fd9b43877e0, num_workers=8, add_foreground_work=<optimized out>)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/workgroup.cpp:149
#5 0x00007fd9b9876b3d in WorkGang::run_task (this=0x7fd9b0073fa0, task=0x7fd9b43877e0, num_workers=8, add_foreground_work=<optimized out>)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/workgroup.cpp:288
#6 0x00007fd9b9537726 in ShenandoahCodeRoots::unlink (workers=0x7fd9b0073fa0, unloading_occurred=unloading_occurred@entry=false)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:332
#7 0x00007fd9b95f1fec in ShenandoahUnload::unload (this=this@entry=0x7fd9b0072e54) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp:67
#8 0x00007fd9b957eca6 in ShenandoahHeap::op_class_unloading (this=this@entry=0x7fd9b0072480) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp:1869
#9 0x00007fd9b95849fa in ShenandoahHeap::entry_class_unloading (this=this@entry=0x7fd9b0072480) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp:2910
#10 0x00007fd9b95738d8 in ShenandoahControlThread::service_concurrent_normal_cycle (this=this@entry=0x7fd9b00df010, cause=cause@entry=GCCause::_shenandoah_concurrent_gc)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp:383
#11 0x00007fd9b95750a4 in ShenandoahControlThread::run_service (this=0x7fd9b00df010) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp:198
#12 0x00007fd9b8994ddb in ConcurrentGCThread::run (this=0x7fd9b00df010) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/concurrentGCThread.cpp:50
#13 0x00007fd9b974da26 in Thread::call_run (this=this@entry=0x7fd9b00df010) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/runtime/thread.cpp:393
#14 0x00007fd9b937e636 in thread_native_entry (thread=0x7fd9b00df010) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/linux/os_linux.cpp:791
#15 0x00007fd9ba6c82de in start_thread () from /lib64/libpthread.so.0
#16 0x00007fd9ba1f5e83 in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7fd9baf12700 (LWP 180892)):
#0 0x00007fd9ba6d2260 in nanosleep () from /lib64/libpthread.so.0
#1 0x00007fd9b938b1cf in os::naked_short_nanosleep (ns=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/posix/os_posix.cpp:695
#2 os::naked_short_sleep (ms=ms@entry=1) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/os/posix/os_posix.cpp:701
#3 0x00007fd9b9575e7a in ShenandoahEvacOOMHandler::wait_for_no_evac_threads (this=this@entry=0x7fd9b0072f20)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:40
#4 0x00007fd9b95bf8a8 in ShenandoahEvacOOMHandler::enter_evacuation (thr=0x7fd9b005eed0, this=0x7fd9b0072f20)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp:38
--Type <RET> for more, q to quit, c to continue without paging--
#5 ShenandoahHeap::enter_evacuation (t=0x7fd9b005eed0, this=<optimized out>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp:89
#6 ShenandoahEvacOOMScope::ShenandoahEvacOOMScope (this=<synthetic pointer>) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp:63
#7 ShenandoahNMethod::heal_nmethod (nm=nm@entry=0x7fd9a03ca010) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:199
#8 0x00007fd9b9536b92 in ShenandoahBarrierSetNMethod::nmethod_entry_barrier (this=<optimized out>, nm=0x7fd9a03ca010)
    at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp:61
#9 0x00007fd9b862f1df in BarrierSetNMethod::nmethod_stub_entry_barrier (return_address_ptr=0x7fd9baf11158) at /home/rkennke/src/openjdk/jdk-jdk15/src/hotspot/share/gc/shared/barrierSetNMethod.cpp:64
#10 0x00007fd9988ca5f9 in ?? ()
#11 0x006f006d006f006f in ?? ()
#12 0x006f007f007f0073 in ?? ()
#13 0xff00ff00ff00ff00 in ?? ()
#14 0xff00ff00ff00ff00 in ?? ()
#15 0x2f6e75732f6d6f63 in ?? ()
#16 0x6f732f616964656d in ?? ()
#17 0x006900640065006d in ?? ()
#18 0x006f0073002f0061 in ?? ()
#19 0x40ef572000000000 in ?? ()
#20 0x0000000000000000 in ?? ()