The JVMCI compiler threads can be suspended by JVMTI because they aren't hidden from external view. If it's suspended while background compilation is disabled then it's possible to deadlock. For example vmTestbase/nsk/jdb/resume/resume002/resume002.java deadlocks because the JVMCI compiler thread is suspended while holding the MethodCompileQueue_lock and a blocking compile has been requested by the JDWP thread. This seems to affect a bunch of the debugging tests.

At a minimum I think we should make CompilerThread::is_hidden_from_external_view return true for UseJVMCINativeLibrary. We're still exposed to this problem with jargraal though but it's not quite clear how to we could fix that.

The following call stacks below show the hang. This is the call stack from the blocked JDWP agent.

  thread #19, name = 'Java: JDWP Transport Listener: dt_socket'
    frame #0: 0x00007ff817bfa0ee libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007ff817c36758 libsystem_pthread.dylib`_pthread_cond_wait + 1242
    frame #2: 0x00000001043c025b libjvm.dylib`PlatformMonitor::wait(this=0x00006000028ac198, millis=<unavailable>) at os_posix.cpp:1900:18 [opt]
    frame #3: 0x000000010433fbf0 libjvm.dylib`Monitor::wait(this=0x00006000028ac190, timeout=0) at mutex.cpp:254:25 [opt]
    frame #4: 0x00000001039d1715 libjvm.dylib`CompileBroker::wait_for_completion(CompileTask*) [inlined] MonitorLocker::wait(this=<unavailable>, timeout=0) at mutexLocker.hpp:255:21 [opt]
    frame #5: 0x00000001039d170b libjvm.dylib`CompileBroker::wait_for_completion(task=0x00006000039f0910) at compileBroker.cpp:1709:10 [opt]
    frame #6: 0x00000001039d06d8 libjvm.dylib`CompileBroker::compile_method_base(method=0x000070000971b118, osr_bci=-1, comp_level=<unavailable>, hot_method=<unavailable>, hot_count=<unavailable>, compile_reason=<unavailable>, blocking=true, thread=0x00007ff018822210) at compileBroker.cpp:1306:5 [opt]
    frame #7: 0x00000001039d1fc5 libjvm.dylib`CompileBroker::compile_method(method=0x000070000971b118, osr_bci=-1, comp_level=<unavailable>, hot_method=0x000070000971b118, hot_count=1, compile_reason=Reason_Tiered, directive=0x00007ff0181049c0, __the_thread__=0x00007ff018822210) at compileBroker.cpp:1457:5 [opt]
    frame #8: 0x00000001039d185c libjvm.dylib`CompileBroker::compile_method(method=0x000070000971b118, osr_bci=-1, comp_level=4, hot_method=<unavailable>, hot_count=<unavailable>, compile_reason=<unavailable>, __the_thread__=0x00007ff018822210) at compileBroker.cpp:1325:17 [opt]
    frame #9: 0x00000001039ac86c libjvm.dylib`CompilationPolicy::compile(mh=0x000070000971b118, bci=-1, level=<unavailable>, __the_thread__=0x00007ff018822210) at compilationPolicy.cpp:811:5 [opt]
    frame #10: 0x00000001039abf66 libjvm.dylib`CompilationPolicy::event(methodHandle const&, methodHandle const&, int, int, CompLevel, CompiledMethod*, JavaThread*) [inlined] CompilationPolicy::method_invocation_event(mh=0x000070000971b118, imh=<unavailable>, level=CompLevel_full_profile, nm=<unavailable>, __the_thread__=<unavailable>) at compilationPolicy.cpp:1147:7 [opt]
    frame #11: 0x00000001039abeea libjvm.dylib`CompilationPolicy::event(method=0x000070000971b118, inlinee=0x000070000971b1b0, branch_bci=<unavailable>, bci=<unavailable>, comp_level=CompLevel_full_profile, nm=0x000000010e286d90, __the_thread__=0x00007ff018822210) at compilationPolicy.cpp:732:5 [opt]
    frame #12: 0x00000001038243cc libjvm.dylib`Runtime1::counter_overflow(JavaThread*, int, Method*) [inlined] counter_overflow_helper(current=0x00007ff018822210, branch_bci=-1, m=<unavailable>) at c1_Runtime1.cpp:481:12 [opt]
    frame #13: 0x0000000103824181 libjvm.dylib`Runtime1::counter_overflow(current=0x00007ff018822210, bci=-1, method=<unavailable>) at c1_Runtime1.cpp:488:14 [opt]
    frame #14: 0x0000000115176f31
    frame #15: 0x000000010e28763b
    frame #16: 0x0000000103dadce8 libjvm.dylib`JavaCalls::call_helper(result=<unavailable>, method=<unavailable>, args=<unavailable>, __the_thread__=<unavailable>) at javaCalls.cpp:415:7 [opt]
    frame #17: 0x0000000103dac813 libjvm.dylib`JavaCalls::call_virtual(JavaValue*, Klass*, Symbol*, Symbol*, JavaCallArguments*, JavaThread*) [inlined] JavaCalls::call(result=0x000070000971ba18, method=0x000070000971b908, args=0x000070000971b970, __the_thread__=0x00007ff018822210) at javaCalls.cpp:329:3 [opt]
    frame #18: 0x0000000103dac7f8 libjvm.dylib`JavaCalls::call_virtual(result=<unavailable>, spec_klass=0x000000080002e1f8, name=0x000000080052d168, signature=0x0000000800512c88, args=<unavailable>, __the_thread__=0x00007ff018822210) at javaCalls.cpp:185:3 [opt]
    frame #19: 0x0000000103dac930 libjvm.dylib`JavaCalls::call_virtual(result=<unavailable>, receiver=<unavailable>, spec_klass=<unavailable>, name=<unavailable>, signature=<unavailable>, __the_thread__=0x00007ff018822210) at javaCalls.cpp:191:3 [opt]
    frame #20: 0x00000001040b5105 libjvm.dylib`JvmtiEnvBase::get_subgroups(current_thread=0x00007ff018822210, group_hdl=Handle @ r13, count_ptr=0x000070000971bb54, group_objs_p=0x000070000971bb30) at jvmtiEnvBase.cpp:815:3 [opt]
    frame #21: 0x00000001040a4b3a libjvm.dylib`JvmtiEnv::GetThreadGroupChildren(this=0x00007ff018104600, group=<unavailable>, thread_count_ptr=0x000070000971bc34, threads_ptr=0x000070000971bc28, group_count_ptr=0x000070000971bc30, groups_ptr=0x000070000971bc20) at jvmtiEnv.cpp:1684:9 [opt]
    frame #22: 0x00000001040494fc libjvm.dylib`jvmti_GetThreadGroupChildren(env=0x00007ff018104600, group=<unavailable>, thread_count_ptr=0x000070000971bc34, threads_ptr=0x000070000971bc28, group_count_ptr=<unavailable>, groups_ptr=<unavailable>) at jvmtiEnter.cpp:1174:20 [opt]
    frame #23: 0x0000000100cc86bc libjdwp.dylib`children(in=0x000070000971bc70, out=0x000070000971bcc8) at ThreadGroupReferenceImpl.c:108:18 [opt]
    frame #24: 0x0000000100cd0e33 libjdwp.dylib`debugLoop_run at debugLoop.c:162:33 [opt]

This is the suspended JVMCI native compiler thread:

  thread #16, name = 'Java: JVMCI-native CompilerThread0'
    frame #0: 0x00007ff817bfa0ee libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007ff817c36758 libsystem_pthread.dylib`_pthread_cond_wait + 1242
    frame #2: 0x00000001043c025b libjvm.dylib`PlatformMonitor::wait(this=0x00007ff00f80d3e0, millis=<unavailable>) at os_posix.cpp:1900:18 [opt]
    frame #3: 0x000000010433f94b libjvm.dylib`Monitor::wait_without_safepoint_check(this=0x00007ff00f80d3d8, timeout=0) at mutex.cpp:226:27 [opt]
    frame #4: 0x0000000103cf2c6a libjvm.dylib`HandshakeState::do_self_suspend(this=0x00007ff00f80d3c8) at handshake.cpp:684:11 [opt]
    frame #5: 0x0000000103cf3da8 libjvm.dylib`ThreadSelfSuspensionHandshake::do_thread(this=<unavailable>, thr=0x00007ff00f80ce10) at handshake.cpp:700:33 [opt]
    frame #6: 0x0000000103cf0767 libjvm.dylib`HandshakeOperation::do_handshake(this=0x0000600002dc1c80, thread=0x00007ff00f80ce10) at handshake.cpp:326:20 [opt]
    frame #7: 0x0000000103cf27df libjvm.dylib`HandshakeState::process_by_self(this=0x00007ff00f80d3c8, allow_suspend=<unavailable>, check_async_exception=<unavailable>) at handshake.cpp:569:13 [opt]
    frame #8: 0x00000001044d28cf libjvm.dylib`SafepointMechanism::process(thread=0x00007ff00f80ce10, allow_suspend=<unavailable>, check_async_exception=<unavailable>) at safepointMechanism.cpp:159:96 [opt]
    frame #9: 0x0000000103780c72 libjvm.dylib`ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState, bool) [inlined] SafepointMechanism::process_if_requested(thread=0x00007ff00f80ce10, allow_suspend=true, check_async_exception=<unavailable>) at safepointMechanism.inline.hpp:83:5 [opt]
    frame #10: 0x0000000103780c62 libjvm.dylib`ThreadStateTransition::transition_from_native(JavaThread*, JavaThreadState, bool) [inlined] SafepointMechanism::process_if_requested_with_exit_check(thread=0x00007ff00f80ce10, check_async_exception=<unavailable>) at safepointMechanism.inline.hpp:88:3 [opt]
    frame #11: 0x0000000103780c62 libjvm.dylib`ThreadStateTransition::transition_from_native(thread=0x00007ff00f80ce10, to=_thread_in_vm, check_asyncs=true) at interfaceSupport.inline.hpp:106:5 [opt]
    frame #12: 0x0000000103fbe14d libjvm.dylib`JVMCIEnv::init_env_mode_runtime(JavaThread*, JNIEnv_*, bool) [inlined] ThreadToNativeFromVM::~ThreadToNativeFromVM(this=0x00007000093125b8) at interfaceSupport.inline.hpp:191:5 [opt]
    frame #13: 0x0000000103fbe13a libjvm.dylib`JVMCIEnv::init_env_mode_runtime(JavaThread*, JNIEnv_*, bool) [inlined] ThreadToNativeFromVM::~ThreadToNativeFromVM(this=0x00007000093125b8) at interfaceSupport.inline.hpp:190:27 [opt]
    frame #14: 0x0000000103fbe13a libjvm.dylib`JVMCIEnv::init_env_mode_runtime(JavaThread*, JNIEnv_*, bool) [inlined] JNIAccessMark::~JNIAccessMark(this=0x00007000093125b0) at jniAccessMark.inline.hpp:38:7 [opt]
    frame #15: 0x0000000103fbe131 libjvm.dylib`JVMCIEnv::init_env_mode_runtime(JavaThread*, JNIEnv_*, bool) [inlined] JNIAccessMark::~JNIAccessMark(this=0x00007000093125b0) at jniAccessMark.inline.hpp:38:7 [opt]
    frame #16: 0x0000000103fbe131 libjvm.dylib`JVMCIEnv::init_env_mode_runtime(this=0x0000700009312710, thread=0x00007ff00f80ce10, parent_env=<unavailable>, jni_enomem_is_fatal=<unavailable>) at jvmciEnv.cpp:201:1 [opt]
    frame #17: 0x0000000103fbe57a libjvm.dylib`JVMCIEnv::JVMCIEnv(JavaThread*, JVMCICompileState*, char const*, int) at jvmciEnv.cpp:208:3 [opt]
    frame #18: 0x0000000103fbe55d libjvm.dylib`JVMCIEnv::JVMCIEnv(this=0x0000700009312710, thread=<unavailable>, compile_state=0x0000700009312668, file=<unavailable>, line=<unavailable>) at jvmciEnv.cpp:204:106 [opt]
    frame #19: 0x00000001039d452c libjvm.dylib`CompileBroker::invoke_compiler_on_method(task=0x00006000039f0910) at compileBroker.cpp:2196:16 [opt]
    frame #20: 0x00000001039d36d7 libjvm.dylib`CompileBroker::compiler_thread_loop() at compileBroker.cpp:1944:11 [opt]
    frame #21: 0x0000000103de0d56 libjvm.dylib`JavaThread::thread_main_inner(this=0x00007ff00f80ce10) at javaThread.cpp:719:5 [opt]