In project Lilliput I encountered a crash that might be an upstream bug. The crash is caused by a bad Klass* (which in Lilliput is in the object header) which in turn is caused by a forwarded object in young/eden space that the linear scan exposes to the heap inspection code. The heap inspection code is entered via PSParallelCompact calling pre_full_gc_dump(), that is after heap has been made parsable but before marking. It is not totally clear how we can get a forwarded object in eden in the first place. Maybe a young-GC got cancelled and/or this is in-fact a dead object?

The problem can easily be reproduced. Apply the patch:

diff --git a/src/hotspot/share/memory/heapInspection.cpp b/src/hotspot/share/memory/heapInspection.cpp
index 262dc62d977..4ef11f4bcd0 100644
--- a/src/hotspot/share/memory/heapInspection.cpp
+++ b/src/hotspot/share/memory/heapInspection.cpp
@@ -213,6 +213,7 @@ KlassInfoEntry* KlassInfoTable::lookup(Klass* k) {
 // Return false if the entry could not be recorded on account
 // of running out of space required to create a new entry.
 bool KlassInfoTable::record_instance(const oop obj) {
+ assert(!obj->is_forwarded(), "object should not be forwarded");
   Klass* k = obj->klass();
   KlassInfoEntry* elt = lookup(k);
   // elt may be null if it's a new klass for which we

and run:

make run-test TEST=gc/logging/TestUnifiedLoggingSwitchStress.java TEST_VM_OPTS="-XX:+UseParallelGC"

I have reproduced the problem with ParallelGC, but it may not be GC-specific.

It seems like this may be benign without Lilliput, but we would be double-counting the forwarded object.

Relevant stack-traces (from fastdebug build, may miss some frames):

Thread 6 (Thread 0xffffc83381f0 (LWP 9138)):
#0 0x0000fffff7f47ae0 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1 0x0000fffff7f47c00 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x0000fffff705978c in PosixSemaphore::wait (this=this@entry=0xfffff0043ea8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/os/posix/semaphore_posix.cpp:66
#3 0x0000fffff73fde8c in Semaphore::wait (this=0xfffff0043ea8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/semaphore.hpp:51
#4 WorkerTaskDispatcher::coordinator_distribute_task (num_workers=<optimized out>, task=0xffffc8336c28, this=0xfffff0043e78) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/workerThread.cpp:51
#5 WorkerThreads::run_task (this=this@entry=0xfffff0043e50, task=task@entry=0xffffc8336c28) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/workerThread.cpp:176
#6 0x0000fffff686df54 in HeapInspection::populate_table (this=this@entry=0xffffc8336e78, cit=cit@entry=0xffffc8336de8, filter=filter@entry=0x0, workers=workers@entry=0xfffff0043e50) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp:575
#7 0x0000fffff686e21c in HeapInspection::heap_inspection (this=0xffffc8336e78, this@entry=0xffffc8336e98, st=0xffffc8336f90, workers=workers@entry=0xfffff0043e50) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp:594
#8 0x0000fffff6809590 in VM_GC_HeapInspection::doit (this=this@entry=0xffffc8336e88) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/gcVMOperations.cpp:191
#9 0x0000fffff6493424 in CollectedHeap::full_gc_dump (this=this@entry=0xfffff0043da0, timer=timer@entry=0xfffff7d65a00 <PSParallelCompact::_gc_timer>, before=before@entry=true) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/collectedHeap.cpp:580
#10 0x0000fffff6493bac in CollectedHeap::pre_full_gc_dump (this=this@entry=0xfffff0043da0, timer=timer@entry=0xfffff7d65a00 <PSParallelCompact::_gc_timer>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/collectedHeap.cpp:585
#11 0x0000fffff6fc4fdc in PSParallelCompact::invoke_no_policy (maximum_heap_compaction=false) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/psParallelCompact.cpp:1749
#12 0x0000fffff6fc5708 in PSParallelCompact::invoke_no_policy (maximum_heap_compaction=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/psParallelCompact.cpp:1907
#13 0x0000fffff6fd7558 in PSScavenge::invoke () at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/psScavenge.cpp:258
#14 0x0000fffff6f38ad4 in ParallelScavengeHeap::failed_mem_allocate (this=this@entry=0xfffff0043da0, size=88505) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp:465
#15 0x0000fffff6fdd084 in VM_ParallelGCFailedAllocation::doit (this=0xffffc1c220a8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/psVMOperations.cpp:45
#16 0x0000fffff73aaeb0 in VM_Operation::evaluate (this=this@entry=0xffffc1c220a8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmOperations.cpp:71
#17 0x0000fffff73d426c in VMThread::evaluate_operation (this=this@entry=0xfffff0159f20, op=0xffffc1c220a8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:281
#18 0x0000fffff73d51a0 in VMThread::inner_execute (this=this@entry=0xfffff0159f20, op=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:435
#19 0x0000fffff73d53ac in VMThread::loop (this=0xfffff0159f20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:502
#20 0x0000fffff73d5504 in VMThread::run (this=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:175
#21 0x0000fffff72c77c0 in Thread::call_run (this=this@entry=0xfffff0159f20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/thread.cpp:217
#22 0x0000fffff6efb5fc in thread_native_entry (thread=0xfffff0159f20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/os/linux/os_linux.cpp:783
#23 0x0000fffff7f3e230 in start_thread () from /lib64/libpthread.so.0
#24 0x0000fffff7e6b7dc in thread_start () from /lib64/libc.so.6

Thread 25 (Thread 0xffffc1c231f0 (LWP 9157)):
#0 0x0000fffff7f44e38 in pthread_cond_wait@@GLIBC_2.17 () from /lib64/libpthread.so.0
#1 0x0000fffff6f0c900 in PlatformMonitor::wait (this=this@entry=0xfffff7d60cf8 <mutex_init()::VMOperation_lock_storage+8>, millis=millis@entry=0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/os/posix/os_posix.cpp:1900
#2 0x0000fffff6e59944 in Monitor::wait (this=0xfffff7d60cf0 <mutex_init()::VMOperation_lock_storage>, timeout=timeout@entry=0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/mutex.cpp:254
#3 0x0000fffff73d40f8 in MonitorLocker::wait (timeout=0, this=0xffffc1c21fe8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/mutexLocker.hpp:254
#4 VMThread::wait_until_executed (op=op@entry=0xffffc1c220a8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:377
#5 0x0000fffff73d5758 in VMThread::execute (op=op@entry=0xffffc1c220a8) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/vmThread.cpp:555
#6 0x0000fffff6f39ddc in ParallelScavengeHeap::mem_allocate (this=0xfffff0043da0, size=88505, gc_overhead_limit_was_exceeded=0xfffff7c8e000) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp:340
#7 0x0000fffff6daef98 in MemAllocator::mem_allocate_outside_tlab (allocation=..., this=0xffffc1c22228) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/memAllocator.cpp:240
#8 MemAllocator::mem_allocate_slow (this=0xffffc1c22228, allocation=...) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/memAllocator.cpp:348
#9 0x0000fffff6daf2a8 in MemAllocator::mem_allocate (allocation=..., this=0xffffc1c22228) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/memAllocator.cpp:360
#10 MemAllocator::allocate (this=0xffffc1c22228, this@entry=0xffffc1c22288) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/memAllocator.cpp:367
#11 0x0000fffff7305530 in CollectedHeap::array_allocate (__the_thread__=0xfffff03f8e20, do_zero=true, length=708028, size=<optimized out>, klass=0x800002520, this=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/collectedHeap.inline.hpp:41
#12 TypeArrayKlass::allocate_common (this=this@entry=0x800002520, length=length@entry=708028, do_zero=do_zero@entry=true, __the_thread__=__the_thread__@entry=0xfffff03f8e20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/oops/typeArrayKlass.cpp:93
#13 0x0000fffff6ed1148 in TypeArrayKlass::allocate (__the_thread__=0xfffff03f8e20, length=708028, this=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/oops/typeArrayKlass.hpp:68
#14 oopFactory::new_typeArray (type=<optimized out>, length=length@entry=708028, __the_thread__=__the_thread__@entry=0xfffff03f8e20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/oopFactory.cpp:93
#15 0x0000fffff703f774 in OptoRuntime::new_array_C (array_type=<optimized out>, len=708028, current=0xfffff03f8e20) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/opto/runtime.cpp:267
#16 0x0000ffffdfe14314 in ?? ()
Backtrace stopped: previous frame inner to this frame (corrupt stack?)


Thread 33 (Thread 0xffffc0c2e1f0 (LWP 9165)):
#0 0x0000fffff7dc2834 in raise () from /lib64/libc.so.6
#1 0x0000fffff7dc4140 in abort () from /lib64/libc.so.6
#2 0x0000fffff6f0cdf4 in os::abort (dump_core=true, siginfo=<optimized out>, context=<optimized out>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/os/posix/os_posix.cpp:2016
#3 0x0000fffff73a40a8 in VMError::report_and_die (id=0, id@entry=-536870912, message=0xffff4c02fc70 "@", message@entry=0xfffff765fad0 "assert(!obj->is_forwarded()) failed", detail_fmt=0x5 <error: Cannot access memory at address 0x5>, detail_fmt@entry=0xfffff766e688 "should not be forwarded",
    detail_args=<error reading variable: Cannot access memory at address 0x8>, thread=0xff8, pc=0xff8 <error: Cannot access memory at address 0xff8>, pc@entry=0x0, siginfo=0xfffff6eef4b8 <os::malloc(unsigned long, MEMFLAGS, NativeCallStack const&)+536>, siginfo@entry=0x0, context=0xffffc0c2d420, context@entry=0xfffff7d113b0 <g_stored_assertion_context>,
    filename=filename@entry=0xfffff766e500 "/home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp", lineno=lineno@entry=216, size=size@entry=0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/utilities/vmError.cpp:1913
#4 0x0000fffff73a4754 in VMError::report_and_die (thread=<optimized out>, context=context@entry=0xfffff7d113b0 <g_stored_assertion_context>, filename=filename@entry=0xfffff766e500 "/home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp", lineno=lineno@entry=216, message=message@entry=0xfffff765fad0 "assert(!obj->is_forwarded()) failed",
    detail_fmt=detail_fmt@entry=0xfffff766e688 "should not be forwarded", detail_args=<error reading variable: Cannot access memory at address 0x87>) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/utilities/vmError.cpp:1578
#5 0x0000fffff656c7b4 in report_vm_error (file=file@entry=0xfffff766e500 "/home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp", line=line@entry=216, error_msg=error_msg@entry=0xfffff765fad0 "assert(!obj->is_forwarded()) failed", detail_fmt=detail_fmt@entry=0xfffff766e688 "should not be forwarded")
    at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/utilities/debug.cpp:191
#6 0x0000fffff686cf78 in KlassInfoTable::record_instance (this=this@entry=0xffffc0c2d8c8, obj=...) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp:216
#7 0x0000fffff686f904 in RecordInstanceClosure::do_object (this=this@entry=0xffffc0c2d8e0, obj=...) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp:524
#8 0x0000fffff6e558a0 in MutableSpace::object_iterate (this=0xfffff004a700, cl=cl@entry=0xffffc0c2d8e0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/mutableSpace.cpp:238
#9 0x0000fffff6f38e5c in ParallelScavengeHeap::object_iterate_parallel (this=<optimized out>, cl=0xffffc0c2d8e0, claimer=0xfffff038c560) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp:607
#10 0x0000fffff686cc78 in ParHeapInspectTask::work (this=0xffffc8336c28, worker_id=0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/memory/heapInspection.cpp:555
#11 0x0000fffff73fd8fc in WorkerTaskDispatcher::worker_run_task (this=0xfffff0043e78) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/workerThread.cpp:69
#12 WorkerThread::run (this=0xffffbc0053a0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/gc/shared/workerThread.cpp:196
#13 0x0000fffff72c77c0 in Thread::call_run (this=this@entry=0xffffbc0053a0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/share/runtime/thread.cpp:217
#14 0x0000fffff6efb5fc in thread_native_entry (thread=0xffffbc0053a0) at /home/rkennke/src/OpenJDK21Src/src/hotspot/os/linux/os_linux.cpp:783
#15 0x0000fffff7f3e230 in start_thread () from /lib64/libpthread.so.0
#16 0x0000fffff7e6b7dc in thread_start () from /lib64/libc.so.6