data (core,libs,hs_err for pid 27452) is in
cores2-da-x64-1.central
/cores_data/pool-1/data2/72998388/may28-27452/
#
# An unexpected error has been detected by HotSpot Virtual Machine:
#
# SIGBUS (0xa) at pc=0xfffffd7ff7c8270d, pid=27452, tid=2038
#
# Java VM: Java HotSpot(TM) 64-Bit Server VM (1.5.0_20-b02 mixed mode)
# Problematic frame:
# v ~C2IAdapter
#
--------------- T H R E A D ---------------
Current thread (0x0000000009291730): JavaThread "RMI TCP Connection(814)-10.11.70.102" daemon [_thread_in_Java, id=2038]
siginfo:si_signo=10, si_errno=12, si_code=3, si_addr=0xfffffd7ead9e9ff8
Registers:
RAX=0xfffffd7febd21cc0, RBX=0x0000000000000000, RCX=0xfffffd7f28c26288, RDX=0xfffffd7f28c26288
RSP=0xfffffd7ead9fdff8, RBP=0xfffffd7f2850e3c8, RSI=0x0000000000000000, RDI=0x000000000000000a
, R8=0x007ffffebf945e36, R9=0xff7ffd80276c4000, R10=0x0000000000000001, R11=0x000000000000000c, R12=0x0000000000000000, R13=0xfffffd7f2850e3c8, R14=0x000000000000000a, R15=0x0000000009291730
RIP=0xfffffd7ff7c8270d, RFLAGS=0x0000000000010246
....
Instructions: (pc=0xfffffd7ff7c8270d)
0xfffffd7ff7c826fd: 90 90 90 48 8b 58 50 48 85 db 0f 85 13 7f fe ff
0xfffffd7ff7c8270d: 89 84 24 00 c0 fe ff 48 83 ec 28 4c 89 74 24 18
Stack: [0xfffffd7ead800000,0xfffffd7eada00000), sp=0xfffffd7ead9fdff8, free space=2039k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
v ~C2IAdapter
core:
current thread: t@2038
=>[1] libc.so.1:__lwp_kill(0x7f6, 0x6, 0xffffffffa87e9180, 0xfffffd7fff2ed4be, 0xfffffd7ead9fd640, 0x6), at 0xfffffd7fff2ecb8a
[2] libc.so.1:_thr_kill(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2e5483
[3] libc.so.1:raise(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff291dc9
[4] libc.so.1:abort(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff271340
[5] libjvm.so:os::abort(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffee2eb93
[6] libjvm.so:VMError::report_and_die(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffee99ada
[7] libjvm.so:JVM_handle_solaris_signal(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffe9fbcb8
[8] libjvm.so:signalHandler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffe9fbe40
[9] libc.so.1:__sighndlr(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2e7386
[10] libc.so.1:call_user_handler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2dbc32
[11] libc.so.1:sigacthandler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2dbe4e ---- called from signal handler with signal 10 (SIGBUS) ------
[12] 0xfffffd7ff7c8270d(0xa, 0x0, 0xfffffd7f28c26288, 0xfffffd7f28c26288, 0x7ffffebf945e36, 0xff7ffd80276c4000, 0x0, 0x0, 0x0, 0x0, 0xfffffd7f28c26288, 0x0, 0x0, 0xfffffd7f2850e3c8, 0xa, 0x0, 0x0, 0x0, 0x0, 0xfffffd7ff8656000), at 0xfffffd7ff7c8270d
[13] 0xfffffd7ff864d788(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ff864d788
....
0xfffffd7ff7c826c0: orb $0x0000000000000068,%al
0xfffffd7ff7c826c2: sbbl 0x000000000000002c(%rax),%ebp
0xfffffd7ff7c826c5: pushq $0x000000000000681e
0xfffffd7ff7c826ca: addb %al,(%rax)
0xfffffd7ff7c826cc: addb %al,(%rax)
0xfffffd7ff7c826ce: addb %al,(%rax)
0xfffffd7ff7c826d0: addb %al,(%rax)
0xfffffd7ff7c826d2: addb %al,(%rax)
0xfffffd7ff7c826d4: addb %al,(%rax)
0xfffffd7ff7c826d6: addb %al,(%rax)
0xfffffd7ff7c826d8: addb %al,(%rax)
0xfffffd7ff7c826da: addb %al,(%rax)
0xfffffd7ff7c826dc: addb %al,(%rax)
0xfffffd7ff7c826de: addb %al,(%rax)
0xfffffd7ff7c826e0: movq 0x0000000000000008(%rcx),%rbx
0xfffffd7ff7c826e4: cmpq 0x0000000000000018(%rax),%rbx
0xfffffd7ff7c826e8: movq 0x0000000000000010(%rax),%rax
0xfffffd7ff7c826ec: jne 0xfffffd7ff7c6a160 [ 0xfffffd7ff7c6a160, .-0x1858c
]
0xfffffd7ff7c826f2: nop
0xfffffd7ff7c826f3: nop
0xfffffd7ff7c826f4: nop
0xfffffd7ff7c826f5: nop
0xfffffd7ff7c826f6: nop
0xfffffd7ff7c826f7: nop
0xfffffd7ff7c826f8: nop
0xfffffd7ff7c826f9: nop
0xfffffd7ff7c826fa: nop
0xfffffd7ff7c826fb: nop
0xfffffd7ff7c826fc: nop
0xfffffd7ff7c826fd: nop
0xfffffd7ff7c826fe: nop
0xfffffd7ff7c826ff: nop
0xfffffd7ff7c82700: movq 0x0000000000000050(%rax),%rbx
0xfffffd7ff7c82704: testq %rbx,%rbx
0xfffffd7ff7c82707: jne 0xfffffd7ff7c6a620 [ 0xfffffd7ff7c6a620, .-0x180e7
]
0xfffffd7ff7c8270d: movl %eax,0xfffffffffffec000(%rsp)
There is another core file under
/cores_data/pool-1/data2/72998388/core-lib-data
for pid 2553, with simliar kind of crash but different
stack trace.
Both crashed with Bus Err with ENOMEM, but there seems to be more than
enough free space left, and customer/on-site SE do not believe this
is a case of memory resource.
cores2-da-x64-1.central
/cores_data/pool-1/data2/72998388/may28-27452/
#
# An unexpected error has been detected by HotSpot Virtual Machine:
#
# SIGBUS (0xa) at pc=0xfffffd7ff7c8270d, pid=27452, tid=2038
#
# Java VM: Java HotSpot(TM) 64-Bit Server VM (1.5.0_20-b02 mixed mode)
# Problematic frame:
# v ~C2IAdapter
#
--------------- T H R E A D ---------------
Current thread (0x0000000009291730): JavaThread "RMI TCP Connection(814)-10.11.70.102" daemon [_thread_in_Java, id=2038]
siginfo:si_signo=10, si_errno=12, si_code=3, si_addr=0xfffffd7ead9e9ff8
Registers:
RAX=0xfffffd7febd21cc0, RBX=0x0000000000000000, RCX=0xfffffd7f28c26288, RDX=0xfffffd7f28c26288
RSP=0xfffffd7ead9fdff8, RBP=0xfffffd7f2850e3c8, RSI=0x0000000000000000, RDI=0x000000000000000a
, R8=0x007ffffebf945e36, R9=0xff7ffd80276c4000, R10=0x0000000000000001, R11=0x000000000000000c, R12=0x0000000000000000, R13=0xfffffd7f2850e3c8, R14=0x000000000000000a, R15=0x0000000009291730
RIP=0xfffffd7ff7c8270d, RFLAGS=0x0000000000010246
....
Instructions: (pc=0xfffffd7ff7c8270d)
0xfffffd7ff7c826fd: 90 90 90 48 8b 58 50 48 85 db 0f 85 13 7f fe ff
0xfffffd7ff7c8270d: 89 84 24 00 c0 fe ff 48 83 ec 28 4c 89 74 24 18
Stack: [0xfffffd7ead800000,0xfffffd7eada00000), sp=0xfffffd7ead9fdff8, free space=2039k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
v ~C2IAdapter
core:
current thread: t@2038
=>[1] libc.so.1:__lwp_kill(0x7f6, 0x6, 0xffffffffa87e9180, 0xfffffd7fff2ed4be, 0xfffffd7ead9fd640, 0x6), at 0xfffffd7fff2ecb8a
[2] libc.so.1:_thr_kill(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2e5483
[3] libc.so.1:raise(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff291dc9
[4] libc.so.1:abort(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff271340
[5] libjvm.so:os::abort(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffee2eb93
[6] libjvm.so:VMError::report_and_die(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffee99ada
[7] libjvm.so:JVM_handle_solaris_signal(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffe9fbcb8
[8] libjvm.so:signalHandler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ffe9fbe40
[9] libc.so.1:__sighndlr(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2e7386
[10] libc.so.1:call_user_handler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2dbc32
[11] libc.so.1:sigacthandler(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7fff2dbe4e ---- called from signal handler with signal 10 (SIGBUS) ------
[12] 0xfffffd7ff7c8270d(0xa, 0x0, 0xfffffd7f28c26288, 0xfffffd7f28c26288, 0x7ffffebf945e36, 0xff7ffd80276c4000, 0x0, 0x0, 0x0, 0x0, 0xfffffd7f28c26288, 0x0, 0x0, 0xfffffd7f2850e3c8, 0xa, 0x0, 0x0, 0x0, 0x0, 0xfffffd7ff8656000), at 0xfffffd7ff7c8270d
[13] 0xfffffd7ff864d788(0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfffffd7ff864d788
....
0xfffffd7ff7c826c0: orb $0x0000000000000068,%al
0xfffffd7ff7c826c2: sbbl 0x000000000000002c(%rax),%ebp
0xfffffd7ff7c826c5: pushq $0x000000000000681e
0xfffffd7ff7c826ca: addb %al,(%rax)
0xfffffd7ff7c826cc: addb %al,(%rax)
0xfffffd7ff7c826ce: addb %al,(%rax)
0xfffffd7ff7c826d0: addb %al,(%rax)
0xfffffd7ff7c826d2: addb %al,(%rax)
0xfffffd7ff7c826d4: addb %al,(%rax)
0xfffffd7ff7c826d6: addb %al,(%rax)
0xfffffd7ff7c826d8: addb %al,(%rax)
0xfffffd7ff7c826da: addb %al,(%rax)
0xfffffd7ff7c826dc: addb %al,(%rax)
0xfffffd7ff7c826de: addb %al,(%rax)
0xfffffd7ff7c826e0: movq 0x0000000000000008(%rcx),%rbx
0xfffffd7ff7c826e4: cmpq 0x0000000000000018(%rax),%rbx
0xfffffd7ff7c826e8: movq 0x0000000000000010(%rax),%rax
0xfffffd7ff7c826ec: jne 0xfffffd7ff7c6a160 [ 0xfffffd7ff7c6a160, .-0x1858c
]
0xfffffd7ff7c826f2: nop
0xfffffd7ff7c826f3: nop
0xfffffd7ff7c826f4: nop
0xfffffd7ff7c826f5: nop
0xfffffd7ff7c826f6: nop
0xfffffd7ff7c826f7: nop
0xfffffd7ff7c826f8: nop
0xfffffd7ff7c826f9: nop
0xfffffd7ff7c826fa: nop
0xfffffd7ff7c826fb: nop
0xfffffd7ff7c826fc: nop
0xfffffd7ff7c826fd: nop
0xfffffd7ff7c826fe: nop
0xfffffd7ff7c826ff: nop
0xfffffd7ff7c82700: movq 0x0000000000000050(%rax),%rbx
0xfffffd7ff7c82704: testq %rbx,%rbx
0xfffffd7ff7c82707: jne 0xfffffd7ff7c6a620 [ 0xfffffd7ff7c6a620, .-0x180e7
]
0xfffffd7ff7c8270d: movl %eax,0xfffffffffffec000(%rsp)
There is another core file under
/cores_data/pool-1/data2/72998388/core-lib-data
for pid 2553, with simliar kind of crash but different
stack trace.
Both crashed with Bus Err with ENOMEM, but there seems to be more than
enough free space left, and customer/on-site SE do not believe this
is a case of memory resource.