The server vm crashed. I have core and an environment to analyze.

Here is result that I investigated. Please try to find out root
cause from the core. If you can not conclud in this time, what
information should we get further? please suggest me.
So far, I can not find similar known bug.

All information are under following directory.
(Sorry, we can not directory under cores.central because this is DTS call.)


[env]
host : scone.japan.sun.com (root/netscape)
localtion: /home/naoki/work/fujitsu/0020
JDK : jdk1.5.0_04
OS : RHEL ES3.0 update3
SA : /root/tools/sa/


[result]
# cd /home/naoki/work/fujitsu/0020
# /root/tools/gdb-6.4/gdb/gdb /usr/local/jdk1.5.0_04/jre/bin/java core.28761
----------------------------------------------------------------

(gdb) where
#0 0xb74b4cdf in raise () from /lib/tls/libc.so.6
#1 0xb74b640d in abort () from /lib/tls/libc.so.6
#2 0xb6efb445 in os::abort () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#3 0xb6fc4f04 in VMError::report_and_die () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#4 0xb6effb3a in JVM_handle_linux_signal () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#5 0xb6efd0c4 in signalHandler () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#6 <signal handler called>
#7 0xb6f92cf2 in Threads::remove () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#8 0xb6f8f011 in JavaThread::exit () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#9 0xb6f8e72c in JavaThread::run () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#10 0xb6efeb68 in _start () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
#11 0xb75ccdec in start_thread () from /lib/tls/libpthread.so.0
#12 0xb756919a in clone () from /lib/tls/libc.so.6
----------------------------------------------------------------

(gdb) frame 7
#7 0xb6f92cf2 in Threads::remove () from /usr/local/jdk1.5.0_04/jre/lib/i386/server/libjvm.so
(gdb) info frame
Stack level 7, frame at 0x586ef5b0:
 eip = 0xb6f92cf2 in Threads::remove(JavaThread*); saved eip 0xb6f8f011
 called by frame at 0x586ef680, caller of frame at 0x586ef208
 Arglist at 0x586ef5a8, args:
 Locals at 0x586ef5a8, Previous frame's sp is 0x586ef5b0
 Saved registers:
  ebx at 0x586ef59c, ebp at 0x586ef5a8, esi at 0x586ef5a0, edi at 0x586ef5a4, eip at 0x586ef5ac
----------------------------------------------------------------

(gdb) x/20x 0x586ef580
0x586ef580: 0x0805cb30 0x000005dc 0x00001000 0x00001000
0x586ef590: 0xad3ec010 0x000005dc 0xb6f92cbb 0xb7071dcc
0x586ef5a0: 0x62dce86c 0x5fdf76d8 0x586ef678 0xb6f8f011
0x586ef5b0: 0x5fdf76d8 0x00000000 0x5fdf76d8 0x5fdf76d8
0x586ef5c0: 0x00000000 0x586ef600 0xb7487634 0xb74876fc
...
----------------------------------------------------------------

Dump of assembler code for function Threads::remove:
0xb6f92cb0 <Threads::remove+0>: push %ebp ### %ebp == 0x586ef678
0xb6f92cb1 <Threads::remove+1>: mov %esp,%ebp ### %ebp == 0x586ef5a8
0xb6f92cb3 <Threads::remove+3>: push %edi ### %edi == 0x5fdf76d8
0xb6f92cb4 <Threads::remove+4>: push %esi ### %esi == 0x62dce86c
0xb6f92cb5 <Threads::remove+5>: push %ebx ### %ebx == 0xb7071dcc
0xb6f92cb6 <Threads::remove+6>: call 0xb6f92cbb <Threads::remove+11>
0xb6f92cbb <Threads::remove+11>: pop %ebx ### %ebx == 0xb6f92cbb
0xb6f92cbc <Threads::remove+12>: add $0xdf111,%ebx ### %ebx == 0xb7071dcc <__JCR_LIST__+4>: xor $0x4c,%al
0xb6f92cc2 <Threads::remove+18>: sub $0x28,%esp ### %esp == 0x586ef59c-0x28 = 0x586ef574
0xb6f92cc5 <Threads::remove+21>: mov 0x20d4(%ebx),%eax ### %eax == 0xb70647b4
0xb6f92ccb <Threads::remove+27>: mov 0x8(%ebp),%esi ### %esi == 0x5fdf76d8 !!! <-- p
0xb6f92cce <Threads::remove+30>: mov (%eax),%eax ### %eax == 0x0805cb30 (0xb70647b4 <Threads_lock>:)
0xb6f92cd0 <Threads::remove+32>: push %eax ### %esp == 0x586ef598(?)
0xb6f92cd1 <Threads::remove+33>: mov %eax,0xffffffd8(%ebp) ### (0x586ef580) = 0x0805cb30?
0xb6f92cd4 <Threads::remove+36>: call 0xb6ee4240 <_ZN5Mutex4lockEv>
0xb6f92cd9 <Threads::remove+41>: xor %edx,%edx
0xb6f92cdb <Threads::remove+43>: mov 0xcd4(%ebx),%ecx
0xb6f92ce1 <Threads::remove+49>: add $0x10,%esp
0xb6f92ce4 <Threads::remove+52>: mov (%ecx),%eax
0xb6f92ce6 <Threads::remove+54>: cmp %esi,%eax
0xb6f92ce8 <Threads::remove+56>: je 0xb6f92cfc <Threads::remove+76>
0xb6f92cea <Threads::remove+58>: lea 0x0(%esi),%esi

0xb6f92cf0 <Threads::remove+64>: mov %eax,%edx ### prev = current;
0xb6f92cf2 <Threads::remove+66>: mov 0x9c(%eax),%eax ### current = current->next();
                                              ^^^^^^^^^^^^^^^<-- SEGV
0xb6f92cf8 <Threads::remove+72>: cmp %esi,%eax
0xb6f92cfa <Threads::remove+74>: jne 0xb6f92cf0 <Threads::remove+64>
...
----------------------------------------------------------------

3235 void Threads::remove(JavaThread* p) {
3236 // Extra scope needed for Thread_lock, so we can check
3237 // that we do not remove thread without safepoint code notice
3238 { MutexLocker ml(Threads_lock);
3239
3240 assert(includes(p), "p must be present");
3241
3242 JavaThread* current = _thread_list;
3243 JavaThread* prev = NULL;
3244
3245 while (current != p) { ### p == 0x5fdf76d8
3246 prev = current;
3247 current = current->next();
           ^^^^^^^^^^^^^^^^^^^^^^^^^<-- SEGV
3248 }
...
----------------------------------------------------------------
###
----------------------------------------------------------------
735 JavaThread* next() const { return _next; }
----------------------------------------------------------------

(gdb) info frame
Stack level 6, frame at 0x586ef208:
 eip = 0xb75d2df0 in __restore_rt; saved eip 0xb6f92cf2
 called by frame at 0x586ef5b0, caller of frame at 0x586ef204
 Arglist at unknown address.
 Locals at unknown address, Previous frame's sp at 0x586ef2c0
 Saved registers:
  eax at 0x586ef2d0, ecx at 0x586ef2cc, edx at 0x586ef2c8, ebx at 0x586ef2c4, ebp at 0x586ef2bc, esi at 0x586ef2b8,
  edi at 0x586ef2b4, eip at 0x586ef2dc, cs at 0x586ef2e0, ss at 0x586ef2ec, ds at 0x586ef2b0, es at 0x586ef2ac,
  fs at 0x586ef2a8, gs at 0x586ef2a4
(gdb) x/x 0x586ef2d0
0x586ef2d0: 0x00000000
----------------------------------------------------------------
### At the time of SEGV, %eax == 0x0.
###
### Walk through _thread_list.
----------------------------------------------------------------
(gdb) x/x 0xb7068c28
0xb7068c28 <_thread_list>: 0x55800480
(gdb) x/x 0x55800480
0x55800480: 0xb7068c48
(gdb) x/x 0x55800480+0x9c
0x5580051c: 0x55804a58
(gdb) x/x 0x55804a58+0x9c
0x55804af4: 0x55804850
(gdb) x/x 0x55804850+0x9c
0x558048ec: 0x58f037e8
(gdb) x/x 0x58f037e8+0x9c
0x58f03884: 0x55809ca0
(gdb) x/x 0x55809ca0+0x9c
0x55809d3c: 0x08343108
(gdb) x/x 0x08343108+0x9c
0x83431a4: 0x5fdf67e0
(gdb) x/x 0x5fdf67e0+0x9c
0x5fdf687c: 0x55821e28
(gdb) x/x 0x55821e28+0x9c
0x55821ec4: 0x58f18f28
(gdb) x/x 0x58f18f28+0x9c
0x58f18fc4: 0x62dc6968
(gdb) x/x 0x62dc6968+0x9c
0x62dc6a04: 0x5ead7010
(gdb) x/x 0x5ead7010+0x9c
0x5ead70ac: 0x62dc7fb0
(gdb) x/x 0x62dc7fb0+0x9c
0x62dc804c: 0x62dc4f28
(gdb) x/x 0x62dc4f28+0x9c
0x62dc4fc4: 0x5fdcf508
(gdb) x/x 0x5fdcf508+0x9c
0x5fdcf5a4: 0x5ead7ce8
(gdb) x/x 0x5ead7ce8+0x9c
0x5ead7d84: 0x62d817f8
(gdb) x/x 0x62d817f8+0x9c
0x62d81894: 0x5d3e3680
(gdb) x/x 0x5d3e3680+0x9c
0x5d3e371c: 0x5fddfe08
(gdb) x/x 0x5fddfe08+0x9c
0x5fddfea4: 0x62d07070
(gdb) x/x 0x62d07070+0x9c
0x62d0710c: 0x62dc5ca0
(gdb) x/x 0x62dc5ca0+0x9c
0x62dc5d3c: 0x5d3e0cd8
(gdb) x/x 0x5d3e0cd8+0x9c
0x5d3e0d74: 0x5fdd28c0
(gdb) x/x 0x5fdd28c0+0x9c
0x5fdd295c: 0x62d81210
(gdb) x/x 0x62d81210+0x9c
0x62d812ac: 0x62d0ae90
(gdb) x/x 0x62d0ae90+0x9c
0x62d0af2c: 0x5ea539a8
(gdb) x/x 0x5ea539a8+0x9c
0x5ea53a44: 0x62d5f418
(gdb) x/x 0x62d5f418+0x9c
0x62d5f4b4: 0x558224c8
(gdb) x/x 0x558224c8+0x9c
0x55822564: 0x558219e0
(gdb) x/x 0x558219e0+0x9c
0x55821a7c: 0x5fdf76d8 ### p !!!
----------------------------------------------------------------
###
### At the time of SEGV was received, the '_thread_list'
### was incorrect list. But after the thread handler completed
### and the process dumped a core, the list got correct.
### I guess race condition leads this symptom.
###
### As far as I looked thorough the sunsolve, I've not found
### such bug yet.


Now, we don't have java thread dump information, so I tried
to get it from SA, but it failed for me.