Steps to reproduce:

$ cd test/hotspot/jtreg/gc/stress/gcbasher
$ javac -d . *.java
$ ../../../../../../build/linux-x86_64-server-fastdebug/images/jdk/bin/java -Xlog:gc -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu -XX:ShenandoahGCHeuristics=aggressive -XX:+ShenandoahOOMDuringEvacALot -cp . gc.stress.gcbasher.TestGCBasherWithShenandoah 1200000


...
[165.845s][info][gc] GC(1824) Concurrent reset 1.198ms
[165.858s][info][gc] GC(1824) Pause Init Mark (unload classes) 10.783ms
[165.887s][info][gc] GC(1824) Concurrent marking (unload classes) 29.284ms
[165.889s][info][gc] GC(1824) Pause Final Mark (unload classes) 1.048ms
[165.890s][info][gc] GC(1824) Concurrent weak roots 0.596ms
[165.890s][info][gc] GC(1824) Concurrent cleanup 36M->36M(1024M) 0.041ms
[165.890s][info][gc] GC(1824) Failed to allocate 24B for evacuation
[165.890s][info][gc] GC(1824) Cancelling GC: Allocation Failure During Evacuation
<hangs indefinitely>

gdb says these are threads that are hanging:

Thread 38 (Thread 0x7f45c9685700 (LWP 48956)):
#0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1 0x00007f45fbbb9023 in __GI___pthread_mutex_lock (mutex=0x7f4508a73e40) at ../nptl/pthread_mutex_lock.c:78
#2 0x00007f45fb024d84 in os::PlatformMutex::lock (this=0x7f4508a73e38) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/os_posix.inline.hpp:55
#3 ShenandoahSimpleLock::lock (this=0x7f4508a73e30) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp:39
#4 ShenandoahReentrantLock::lock (this=0x7f4508a73e30) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp:60
#5 0x00007f45fafb0e6b in ShenandoahReentrantLocker::ShenandoahReentrantLocker (lock=<optimized out>, this=<synthetic pointer>)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp:127
#6 ShenandoahNMethodUnlinkClosure::do_nmethod (this=0x7f45f853f818, nm=0x7f45e450d790) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:272
#7 0x00007f45fb034d55 in ShenandoahNMethodTableSnapshot::concurrent_nmethods_do (cl=0x7f45f853f818, this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:587
#8 ShenandoahConcurrentNMethodIterator::nmethods_do (this=this@entry=0x7f45f853f840, cl=cl@entry=0x7f45f853f818) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:606
#9 0x00007f45fafae8bf in ShenandoahUnlinkTask::work (this=0x7f45f853f800, worker_id=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:315
#10 0x00007f45fb2fa290 in GangWorker::run_task (data=..., this=0x7f45c400be80) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:339
#11 GangWorker::loop (this=0x7f45c400be80) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:349
#12 0x00007f45fb1c5226 in Thread::call_run (this=this@entry=0x7f45c400be80) at /home/shade/trunks/jdk-jdk/src/hotspot/share/runtime/thread.cpp:393
#13 0x00007f45fadee74e in thread_native_entry (thread=0x7f45c400be80) at /home/shade/trunks/jdk-jdk/src/hotspot/os/linux/os_linux.cpp:791
#14 0x00007f45fbbb66db in start_thread (arg=0x7f45c9685700) at pthread_create.c:463
#15 0x00007f45fc31088f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95


Thread 28 (Thread 0x7f45ca099700 (LWP 48946)):
#0 0x00007f45fc2d39d0 in __GI___nanosleep (requested_time=requested_time@entry=0x7f45ca098b20, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28
#1 0x00007f45fadfbe12 in os::naked_short_nanosleep (ns=1000000) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/os_posix.cpp:695
#2 os::naked_short_sleep (ms=ms@entry=1) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/os_posix.cpp:701
#3 0x00007f45fafec80a in ShenandoahEvacOOMHandler::wait_for_no_evac_threads (this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:40
#4 ShenandoahEvacOOMHandler::handle_out_of_memory_during_evacuation (this=this@entry=0x7f45f4085bd0) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:93
#5 0x00007f45fad7e37c in ShenandoahHeap::evacuate_object (this=0x7f45f4085130, p=..., thread=0x7f45cc014060) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp:292
#6 0x00007f45fafb0c76 in ShenandoahEvacuateUpdateRootsClosure<64ul>::do_oop_work<oop> (this=0x7f45ca098d20, p=0x7f45dd0b2a8d)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp:103
#7 0x00007f45fafb102a in ShenandoahEvacuateUpdateRootsClosure<64ul>::do_oop (p=<optimized out>, this=0x7f45ca098d20)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp:111
#8 ShenandoahNMethod::oops_do (fix_relocations=<optimized out>, oops=<optimized out>, this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp:59
#9 ShenandoahNMethod::heal_nmethod_metadata (nmethod_data=0x7f4598029a90) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp:77
#10 ShenandoahNMethodUnlinkClosure::do_nmethod (this=0x7f45f853f818, nm=0x7f45dd0b2890) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:276
#11 0x00007f45fb034d55 in ShenandoahNMethodTableSnapshot::concurrent_nmethods_do (cl=0x7f45f853f818, this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:587
#12 ShenandoahConcurrentNMethodIterator::nmethods_do (this=this@entry=0x7f45f853f840, cl=cl@entry=0x7f45f853f818) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:606
#13 0x00007f45fafae8bf in ShenandoahUnlinkTask::work (this=0x7f45f853f800, worker_id=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:315
#14 0x00007f45fb2fa290 in GangWorker::run_task (data=..., this=0x7f45cc014060) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:339
#15 GangWorker::loop (this=0x7f45cc014060) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:349
#16 0x00007f45fb1c5226 in Thread::call_run (this=this@entry=0x7f45cc014060) at /home/shade/trunks/jdk-jdk/src/hotspot/share/runtime/thread.cpp:393
#17 0x00007f45fadee74e in thread_native_entry (thread=0x7f45cc014060) at /home/shade/trunks/jdk-jdk/src/hotspot/os/linux/os_linux.cpp:791
#18 0x00007f45fbbb66db in start_thread (arg=0x7f45ca099700) at pthread_create.c:463
#19 0x00007f45fc31088f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95


Thread 5 (Thread 0x7f45f8540700 (LWP 48923)):
#0 0x00007f45fbbbf6d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f45f4086db0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
#1 do_futex_wait (sem=sem@entry=0x7f45f4086db0, abstime=0x0) at sem_waitcommon.c:111
#2 0x00007f45fbbbf7c8 in __new_sem_wait_slow (sem=sem@entry=0x7f45f4086db0, abstime=0x0) at sem_waitcommon.c:181
#3 0x00007f45fbbbf839 in __new_sem_wait (sem=sem@entry=0x7f45f4086db0) at sem_wait.c:42
#4 0x00007f45faf4a134 in PosixSemaphore::wait (this=0x7f45f4086da8) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/semaphore_posix.cpp:64
#5 0x00007f45fb2fb6d5 in Semaphore::wait (this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/runtime/semaphore.hpp:55
#6 SemaphoreGangTaskDispatcher::coordinator_execute_on_workers (this=0x7f45f4086cd0, task=<optimized out>, num_workers=<optimized out>, add_foreground_work=<optimized out>)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:149
#7 0x00007f45fb2fab5a in WorkGang::run_task (add_foreground_work=<optimized out>, num_workers=<optimized out>, task=<optimized out>, this=<optimized out>)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:288
#8 WorkGang::run_task (this=0x7f45f4086c50, task=0x7f45f853f800) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/workgroup.cpp:278
#9 0x00007f45fafadce4 in ShenandoahCodeRoots::unlink (workers=0x7f45f4086c50, unloading_occurred=unloading_occurred@entry=false)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp:332
#10 0x00007f45fb06a2fc in ShenandoahUnload::unload (this=this@entry=0x7f45f4085b04) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp:163
#11 0x00007f45faffb130 in ShenandoahHeap::op_class_unloading (this=0x7f45f4085130) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp:1864
#12 ShenandoahHeap::entry_class_unloading (this=this@entry=0x7f45f4085130) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp:2905
#13 0x00007f45fafe9d48 in ShenandoahControlThread::service_concurrent_normal_cycle (this=this@entry=0x7f45f410c2b0, cause=cause@entry=GCCause::_shenandoah_concurrent_gc)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp:391
#14 0x00007f45fafeb796 in ShenandoahControlThread::run_service (this=0x7f45f410c2b0) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp:206
#15 0x00007f45fa3e378b in ConcurrentGCThread::run (this=0x7f45f410c2b0) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/concurrentGCThread.cpp:50
#16 0x00007f45fb1c5226 in Thread::call_run (this=this@entry=0x7f45f410c2b0) at /home/shade/trunks/jdk-jdk/src/hotspot/share/runtime/thread.cpp:393
#17 0x00007f45fadee74e in thread_native_entry (thread=0x7f45f410c2b0) at /home/shade/trunks/jdk-jdk/src/hotspot/os/linux/os_linux.cpp:791
#18 0x00007f45fbbb66db in start_thread (arg=0x7f45f8540700) at pthread_create.c:463
#19 0x00007f45fc31088f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95


Thread 2 (Thread 0x7f45fc9e2700 (LWP 48920)):
#0 0x00007f45fc2d39d0 in __GI___nanosleep (requested_time=requested_time@entry=0x7f45fc9e1410, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28
#1 0x00007f45fadfbe12 in os::naked_short_nanosleep (ns=1000000) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/os_posix.cpp:695
#2 os::naked_short_sleep (ms=ms@entry=1) at /home/shade/trunks/jdk-jdk/src/hotspot/os/posix/os_posix.cpp:701
#3 0x00007f45fafec28a in ShenandoahEvacOOMHandler::wait_for_no_evac_threads (this=this@entry=0x7f45f4085bd0) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp:40
#4 0x00007f45fb0397c8 in ShenandoahEvacOOMHandler::enter_evacuation (thr=0x7f45f4072590, this=0x7f45f4085bd0)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp:38
#5 ShenandoahHeap::enter_evacuation (t=0x7f45f4072590, this=<optimized out>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp:89
#6 ShenandoahEvacOOMScope::ShenandoahEvacOOMScope (this=<synthetic pointer>) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp:63
#7 ShenandoahNMethod::heal_nmethod (nm=nm@entry=0x7f45e450d790) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp:199
#8 0x00007f45fafad182 in ShenandoahBarrierSetNMethod::nmethod_entry_barrier (this=<optimized out>, nm=0x7f45e450d790)
    at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetNMethod.cpp:61
#9 0x00007f45fa0682c9 in BarrierSetNMethod::nmethod_stub_entry_barrier (return_address_ptr=0x7f45fc9e1708) at /home/shade/trunks/jdk-jdk/src/hotspot/share/gc/shared/barrierSetNMethod.cpp:64
#10 0x00007f45dc95b5f9 in ?? ()
#11 0x0000000000000000 in ?? ()

I suppose that means no one is unlocking the lock that Thread 38 waits for?