Problem with weblogic server 6.0 using jdk1.3. running sol 2.7
After running (WLS) around one houer plus when more users access DB (oracle) the WL server hangs create core file.
Here is the stack trace information from dbx.
These are all the threads
dbx) threads
t@1 ?() sleep on 0x3b9d0 in _cond_wait_cancel()
t@2 b l@2 ?() running in __signotifywait()
t@3 ?() sleep on 0xff391340 in _reap_wait()
t@4 b l@5 _start() running in ___lwp_cond_wait()
> t@5 b l@6 _co_timerset() running in ___lwp_mutex_lock()
t@6 a l@10 _start() sleep on 0x14c460 in __lwp_sema_wait()
t@7 a l@22 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@8 a l@3 _start() sleep on 0x28240 in __lwp_sema_wait()
t@9 _start() sleep on 0x3a0d0 in _cond_wait_cancel()
t@10 _start() sleep on 0x399c0 in _cond_wait_cancel()
t@11 a l@24 _start() sleep on 0x39a90 in __lwp_sema_wait()
t@12 a l@4 _start() sleep on 0x28240 in __lwp_sema_wait()
t@13 a l@7 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@14 a l@13 _start() sleep on 0xff394598 in __lwp_sema_wait()
t@15 a l@9 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@16 a l@23 _start() sleep on 0x28240 in __lwp_sema_wait()
t@17 a l@8 _start() running in _so_recvfrom()
t@18 a l@16 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@19 a l@19 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@20 a l@18 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@21 a l@12 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@22 a l@20 _start() sleep on 0x28240 in __lwp_sema_wait()
t@23 a l@15 _start() sleep on 0x28240 in __lwp_sema_wait()
t@24 a l@21 _start() sleep on 0x28240 in __lwp_sema_wait()
t@25 a l@25 _start() running in ___lwp_mutex_lock()
t@26 a l@11 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@27 a l@14 _start() sleep on 0x28240 in __lwp_sema_wait()
t@28 _start() sleep on 0x79c560 in _cond_wait_cancel()
t@29 _start() sleep on 0x2fd4e8 in _cond_timedwait_cancel()
t@30 a l@17 _start() sleep on 0x29de48 in __lwp_sema_wait()
t@31 _start() sleep on 0x4e47c0 in _cond_timedwait_cancel()
t@33 _start() sleep on 0x13a848 in _cond_wait_cancel()
t@34 _start() sleep on 0x139fd8 in _cond_wait_cancel()
t@37 a l@1 _start() sleep on 0x28240 in __lwp_sema_wait()
This is the stack trace for the timer thread (getting the SIGALRM)
(dbx) thread t@5
t@5 (l@6) stopped in ___lwp_mutex_lock at 0xff317c70
0xff317c70: ___lwp_mutex_lock+0x0008: ta 0x8
(dbx) where
current thread: t@5
=>[1] ___lwp_mutex_lock(0xff394250, 0xe, 0x0, 0x0, 0x0, 0x0), at 0xff317c70
[2] sigacthandler(0xe, 0x0, 0xfed09a10, 0xff38c984, 0x0, 0xfed09d28), at 0xff37823c
---- called from signal handler with signal 14 (SIGALRM) ------
[3] __lwp_sema_wait(0x5b, 0x70114a58, 0x0, 0xfed09d34, 0x0, 0xfed09d28), at 0xff317d38
[4] _co_timerset(0xff38e358, 0xff38d698, 0xff38c984, 0xff38dfd8, 0x3, 0xff38c984), at 0xff36a570
This is the stack trace for the other thread blocked in sigacthandler (getting the SIGLWP)
In this example, it is running a JNI call that is in Oracle code.
(dbx) thread t@25
t@25 (l@25) stopped in ___lwp_mutex_lock at 0xff317c70
0xff317c70: ___lwp_mutex_lock+0x0008: ta 0x8
(dbx) where
current thread: t@25
=>[1] ___lwp_mutex_lock(0xff394250, 0x21, 0x0, 0x0, 0x0, 0x0), at 0xff317c70
[2] sigacthandler(0x21, 0xd4f7f910, 0xd4f7f658, 0xff38c984, 0x32, 0x62b0bf), at 0xff37823c
---- called from signal handler with signal 33 (SIGLWP) ------
[3] ttcs2c(0x0, 0x62b0bf, 0x33, 0x5, 0xd6749dd8, 0x8bb0a4), at 0xd43a7098
[4] ttcacr(0xd6749dd8, 0x5, 0x4bb89a, 0x0, 0x1, 0x2b3c3c), at 0xd43b3d60
[5] ttcdrv(0x8bad28, 0xd6749eb4, 0x0, 0x0, 0x8bb00e, 0x8bb00c), at 0xd437ecf8
[6] nioqwa(0xd6749e7c, 0x0, 0xd437d67c, 0x8bad28, 0xd6749e00, 0x0), at 0xd4290714
[7] upirtrc(0x8b9f44, 0x0, 0x0, 0xd6749e7c, 0x0, 0x0), at 0xd4128980
[8] kpuexe(0x8bc064, 0x8bac74, 0x2b3c3c, 0x8b9f44, 0x8bac64, 0x8bad28), at 0xd40be0d0
[9] upiefn(0x3ff, 0x8b9f44, 0xd6749eb4, 0x2000, 0x1, 0xd6749dd8), at 0xd4127a50
[10] ociefn(0x51b300, 0x64, 0x0, 0x0, 0x20, 0xd883b63a), at 0xd417aa6c
[11] Java_weblogic_db_oci_OciCursor_execAndFetch(0x6c6388, 0xd4f80d84, 0x64, 0x5dd, 0x5, 0x0), at 0xd64a5e78
[12] 0xca8cc(0xd4f80d84, 0xd4f80dec, 0xd4f80df0, 0xffffffff, 0x0, 0x109a0), at 0xca8cb
[13] 0xc5c88(0xd4f80df4, 0xd883b63a, 0xd4, 0xe0858, 0x8, 0xd4f80d18), at 0xc5c87
[14] 0xfa4306c8(0xd8837170, 0x64, 0xb, 0xf8038d70, 0x8, 0xd4f80db0), at 0xfa4306c7
[15] 0xfa5017f8(0xd8836f00, 0x0, 0x6c62f0, 0xe05f0, 0xda522d48, 0x0), at 0xfa5017f7
[16] 0xfa51316c(0xd8839098, 0xf88112e8, 0xd4f80f8c, 0xd82e0, 0x18, 0x0), at 0xfa51316b
[17] 0xfa41cec4(0xd8839098, 0xf88112e8, 0x0, 0x0, 0xbcc00, 0x109a0), at 0xfa41cec3
[18] 0xc6138(0xd4f8100c, 0xf88112e8, 0x0, 0xe0d18, 0x4, 0xd4f80f28), at 0xc6137
[19] 0xc6138(0xd4f810cc, 0xf88112e8, 0x0, 0xe0d18, 0x4, 0xd4f80fb0), at 0xc6137
[20] 0xc6138(0xd4f81174, 0xd4f8120c, 0xd4f81210, 0xe0d18, 0x4, 0xd4f81038), at 0xc6137
[21] 0xc5c88(0xd4f81214, 0xda3c7158, 0xd4f81288, 0xe0858, 0x8, 0xd4f81110), at 0xc5c87
[22] 0xc5c88(0xd4f8128c, 0x1, 0xd4f8129c, 0xe0328, 0x8, 0xd4f811a8), at 0xc5c87
[23] 0xfa403d08(0xda394038, 0xd8831888, 0xd4f812f0, 0xe771c81, 0x8, 0xd4f81248), at 0xfa403d07
[24] 0xfa50898c(0xda394038, 0xda377020, 0x4, 0xda752a10, 0xda9fd128, 0x0), at0xfa50898b
[25] 0xfa507d30(0xda3b3e78, 0x0, 0xd4f81364, 0xd4f81384, 0xbcc00, 0x109a0), at 0xfa507d2f
[26] 0xc5c88(0xd4f81434, 0xf847f2c0, 0xd4f814b8, 0xe0328, 0x14, 0xd4f81310), at 0xc5c87
[27] 0xc5c88(0xd4f814c4, 0xf83f18e0, 0xd4f81520, 0xe0858, 0x10, 0xd4f813c8), at 0xc5c87
[28] 0xc5c88(0xd4f81528, 0xd4f8152c, 0xd4f815d8, 0xe0d18, 0x10, 0xd4f81458), at 0xc5c87
[29] 0xfa40b98c(0xda377020, 0x4, 0xda752a10, 0xe0328, 0xc, 0xd4f814e0), at 0xfa40b98b
[30] 0xfa4cf3a4(0xda377020, 0xda099138, 0xda010240, 0xf8416f70, 0xda08e968, 0x6c62f0), at 0xfa4cf3a3
[31] 0xfa416e50(0xda752a10, 0xda377020, 0xd4f816b0, 0xda0964a0, 0xbcc00, 0x109a0), at 0xfa416e4f
[32] 0xc6318(0xd4f816b4, 0xf80eaea8, 0xd4f81740, 0xe0d18, 0x8, 0xd4f815d0), at 0xc6317
[33] 0xc6318(0xd4f81744, 0x1, 0xd4f81748, 0xe0d18, 0x8, 0xd4f81650), at 0xc6317
[34] 0xc5e68(0xd4f817cc, 0x6c62f0, 0x8, 0xe05f0, 0x8, 0xd4f816d8), at 0xc5e67
[35] 0xfe53a734(0xd4f8185c, 0xd4f81bb8, 0xa, 0xf83e2290, 0x4, 0xd4f81770), at0xfe53a733
[36] JavaCalls::call_helper(0xd4f81afc, 0xfe53a6c0, 0xd4f81af4, 0x6c62f0, 0xd4f818e4, 0x5), at 0xfdb203dc
[37] os::os_exception_wrapper(0xfdb1fc90, 0xd4f81bb0, 0xd4f81a38, 0xd4f81af4,0x6c62f0, 0x1), at 0xfdd04844
[38] JavaCalls::call(0xd4f81bb0, 0xd4f81a38, 0xd4f81af4, 0x6c62f0, 0xd4f81a44, 0xd4f81a40), at 0xfdb1fc24
[39] JavaCalls::call_virtual(0xd4f81a40, 0xd4f81a3c, 0xd4f81ae8, 0xd4f81ae4, 0xd4f81af4, 0x6c62f0), at 0xfdb1ece0
[40] JavaCalls::call_virtual(0xd4f81bb0, 0xd4f81ba0, 0xd4f81b9c, 0xd4f81b98, 0xd4f81b94, 0x6c62f0), at 0xfdb1eddc
[41] thread_entry(0x6c62f0, 0x6c62f0, 0x1, 0x0, 0x0, 0x0), at 0xfdb97c80
[42] JavaThread::thread_main_inner(0x6c62f0, 0x31da10, 0x6, 0x0, 0x0, 0x0), at 0xfde170f0
[43] JavaThread::run(0x6c62f0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfde16f18
[44] _start(0x6c62f0, 0xff245d60, 0x0, 0xd5081e50, 0x1, 0xfe401000), at 0xfdcfe9c0
This is the stack trace for the thread calling sigaction (waiting on semaphore)
In this example, it is running a JNI call that is in Oracle code doing
a login.
(dbx) where
current thread: t@14
=>[1] __lwp_sema_wait(0xd5a81e78, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xff317d34
[2] _park(0xd5a81dc0, 0xd5a81e78, 0x0, 0x8, 0xff38d710, 0xd5b81dc0), at 0xff36b1b8
[3] _swtch(0x5, 0xff38c984, 0xd5a81e54, 0xd5a81e50, 0xd5a81e4c, 0xd5a81e48), at 0xff36aeac
[4] _mutex_adaptive_lock(0xff394598, 0x4c00, 0xff38c984, 0x1, 0x4d58, 0xfffeffff), at 0xff36c744
[5] _cmutex_lock(0xff394598, 0xff, 0x1000, 0xff37974c, 0x0, 0x0), at 0xff36c4f4
[6] _discard_ignored_sig(0x25, 0xd, 0xff394278, 0xff394590, 0x320, 0x19), at 0xff37974c
[7] _setsighandler(0xd, 0xd5a7c094, 0xd5a7c074, 0xff38c984, 0xd5a7bf4c, 0xff393c84), at 0xff3791b8
[8] _sigaction(0xd, 0xd5a7c094, 0xd5a7c074, 0x0, 0x354348, 0x0), at 0xff378bbc
[9] sslsigreghndlr(0xd, 0x1, 0x0, 0x0, 0x0, 0xd5a7c131), at 0xd451ac58
[10] snssgign(0xd, 0x0, 0xfffffffc, 0x10, 0x6c56d0, 0xd5a7d711), at 0xd42213d8
[11] nssgign(0x46aa80, 0x2, 0x0, 0x0, 0x0, 0x1), at 0xd422129c
[12] nsopen(0x7f8a90, 0xd461e970, 0x0, 0x0, 0x0, 0x4000), at 0xd421a01c
[13] nscall1(0xffffffff, 0x7f8a90, 0xd5a7d95c, 0xd5a7d950, 0xd5a7d684, 0x7f8aec), at 0xd4203ea8
[14] nscall(0x354348, 0x7f8a90, 0x0, 0x7f8aec, 0x13e208, 0x0), at 0xd42031f4
[15] niotns(0x1, 0x4bddd8, 0xd45f4444, 0x1414, 0x4bddcc, 0x0), at 0xd428cc14
[16] nigcall(0xd5a7dba0, 0x82, 0xd461162c, 0x1803, 0x4bddcc, 0xd6749fbc), at 0xd4286ac4
[17] osncon(0x4, 0x1803, 0x0, 0x1, 0x1, 0x0), at 0xd41fc800
[18] upiini(0x4bddcc, 0x1400, 0x4bbcac, 0xd6749fbc, 0xd6749f18, 0x4bc9cc), at 0xd4129344
[19] upiah0(0xd6749f18, 0xd5a804ec, 0x6b, 0x0, 0x0, 0xff00), at 0xd411a3c4
[20] upilgn(0x0, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff, 0x0), at 0xd41bd54c
[21] upilog(0xd6749f18, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff, 0xd5a804ec), at 0xd41bea10
[22] ocilog(0xd6749ed8, 0xd6749f18, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff), at 0xd417b1d4
[23] Java_weblogic_db_oci_OciConnection_native_1loginwithblock(0xffffffff, 0xd5a808f4, 0xd5a808f0, 0xd5a808ec, 0xd5a808e8, 0x1), at 0xd649d2e0
[24] 0xca8cc(0xd5a808f4, 0xd5a808f8, 0xd5a808f8, 0xe0ac0, 0x0, 0x109a0), at 0xca8cb
[25] 0xc5c88(0xd5a8098c, 0xda1b5968, 0xe3, 0xe0858, 0x14, 0xd5a80888), at 0xc5c87
[26] 0xc5c88(0xd5a80a4c, 0xf87ecfb8, 0xd5a80ad4, 0xe05f0, 0x14, 0xd5a80920), at 0xc5c87
[27] 0xc6138(0xd5a80ad8, 0xf8804878, 0x0, 0xe0d18, 0xc, 0xd5a809e0), at 0xc6137
[28] 0xc5c88(0xd5a80b6c, 0xf87e85b0, 0xd5a80b70, 0xe0858, 0x8, 0xd5a80a78), at 0xc5c87
[29] 0xc6138(0xd5a80bec, 0xd5a810e0, 0xd5a810e4, 0xe0d18, 0x4, 0xd5a80b00), at 0xc6137
[30] 0xfa41acc8(0xda24c568, 0x1, 0xda3c85d0, 0xda3c85d0, 0x8, 0xd5a80ba8), at 0xfa41acc7
[31] 0xfa4e11b4(0xf8038d70, 0xda1faff0, 0x0, 0x0, 0x1, 0x0), at 0xfa4e11b3
[32] 0xfa4df5b0(0xda24c568, 0x0, 0xd5a80d64, 0xd5a80d7c, 0xbcc00, 0x109a0), at 0xfa4df5af
[33] 0xc5c88(0xd5a80e04, 0xd5a80e84, 0xd5a80e88, 0xe0858, 0x14, 0xd5a80d10), at 0xc5c87
[34] 0xc5c88(0xd5a80e8c, 0xd5a80f14, 0xd5a80f18, 0xe0858, 0x10, 0xd5a80d98), at 0xc5c87
[35] 0xc5c88(0xd5a80f24, 0xd5a80fa4, 0xd5a80fa8, 0xe0328, 0x8, 0xd5a80e20), at 0xc5c87
[36] 0xc5c88(0xd5a80fac, 0xd5a81050, 0xd5a81054, 0xe0ac0, 0x10, 0xd5a80eb8), at 0xc5c87
[37] 0xc5c88(0xd5a81054, 0xd5a810e0, 0xd5a810e4, 0xe0ac0, 0x8, 0xd5a80f40), at 0xc5c87
[38] 0xc5c88(0xd5a810e4, 0xf88581a0, 0x0, 0xe0328, 0x4, 0xd5a80fe8), at 0xc5c87
[39] 0xc6138(0xd5a81184, 0xac, 0xd5a8121c, 0xe0d18, 0x4, 0xd5a81080), at 0xc6137
[40] 0xc5c88(0xd5a81224, 0xd5a812b8, 0xd5a812bc, 0xe0858, 0x4, 0xd5a81118), at 0xc5c87
[41] 0xc5c88(0xd5a812c4, 0xd87401c0, 0xd5a8135c, 0xe0798, 0xc, 0xd5a811c0), at 0xc5c87
[42] 0xc5c88(0xd5a81364, 0xf8a14da0, 0x230, 0xe0268, 0xc, 0xd5a81258), at 0xc5c87
[43] 0xc6138(0xd5a81414, 0xf850d208, 0x1a2, 0xe0c58, 0xc, 0xd5a812f8), at 0xc6137
[44] 0xc5c88(0xd5a814c4, 0xf83f18e0, 0xd5a81520, 0xe0328, 0x14, 0xd5a813a0), at 0xc5c87
[45] 0xc5c88(0xd5a81528, 0xd5a8152c, 0xd5a815d8, 0xe0d18, 0x10, 0xd5a81458), at 0xc5c87
[46] 0xfa40b98c(0xdac86b70, 0x4, 0xda7c8490, 0xe0328, 0xc, 0xd5a814e0), at 0xfa40b98b
[47] 0xfa4cf3a4(0xdac86b70, 0xda096e88, 0xda010240, 0xf8416f70, 0xda08e968, 0x2849b8), at 0xfa4cf3a3
[48] 0xfa416e50(0xda7c8490, 0xdac86b70, 0xd5a816b0, 0xda0964a0, 0xbcc00, 0x109a0), at 0xfa416e4f
[49] 0xc6318(0xd5a816b4, 0xf80eaea8, 0xd5a81740, 0xe0d18, 0x8, 0xd5a815d0), at 0xc6317
[50] 0xc6318(0xd5a81744, 0x1, 0xd5a81748, 0xe0d18, 0x8, 0xd5a81650), at 0xc6317
[51] 0xc5e68(0xd5a817cc, 0x2849b8, 0x8, 0xe05f0, 0x8, 0xd5a816d8), at 0xc5e67
[52] 0xfe53a734(0xd5a8185c, 0xd5a81bb8, 0xa, 0xf83e2290, 0x4, 0xd5a81770), at 0xfe53a733
[53] JavaCalls::call_helper(0xd5a81afc, 0xfe53a6c0, 0xd5a81af4, 0x2849b8, 0xd5a818e4, 0x5), at 0xfdb203dc
[54] os::os_exception_wrapper(0xfdb1fc90, 0xd5a81bb0, 0xd5a81a38, 0xd5a81af4, 0x2849b8, 0x1), at 0xfdd04844
[55] JavaCalls::call(0xd5a81bb0, 0xd5a81a38, 0xd5a81af4, 0x2849b8, 0xd5a81a44, 0xd5a81a40), at 0xfdb1fc24
[56] JavaCalls::call_virtual(0xd5a81a40, 0xd5a81a3c, 0xd5a81ae8, 0xd5a81ae4, 0xd5a81af4, 0x2849b8), at 0xfdb1ece0
[57] JavaCalls::call_virtual(0xd5a81bb0, 0xd5a81ba0, 0xd5a81b9c, 0xd5a81b98, 0xd5a81b94, 0x2849b8), at 0xfdb1eddc
[58] thread_entry(0x2849b8, 0x2849b8, 0x1, 0x0, 0x0, 0x0), at 0xfdb97c80
[59] JavaThread::thread_main_inner(0x2849b8, 0x1e8838, 0x6, 0x0, 0x0, 0x0), at 0xfde170f0
[60] JavaThread::run(0x2849b8, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfde16f18
[61] _start(0x2849b8, 0xff245d60, 0x0, 0xd5b81e50, 0x1, 0xfe401000), at 0xfdcfe9c0
These three threads hang forever.
After running (WLS) around one houer plus when more users access DB (oracle) the WL server hangs create core file.
Here is the stack trace information from dbx.
These are all the threads
dbx) threads
t@1 ?() sleep on 0x3b9d0 in _cond_wait_cancel()
t@2 b l@2 ?() running in __signotifywait()
t@3 ?() sleep on 0xff391340 in _reap_wait()
t@4 b l@5 _start() running in ___lwp_cond_wait()
> t@5 b l@6 _co_timerset() running in ___lwp_mutex_lock()
t@6 a l@10 _start() sleep on 0x14c460 in __lwp_sema_wait()
t@7 a l@22 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@8 a l@3 _start() sleep on 0x28240 in __lwp_sema_wait()
t@9 _start() sleep on 0x3a0d0 in _cond_wait_cancel()
t@10 _start() sleep on 0x399c0 in _cond_wait_cancel()
t@11 a l@24 _start() sleep on 0x39a90 in __lwp_sema_wait()
t@12 a l@4 _start() sleep on 0x28240 in __lwp_sema_wait()
t@13 a l@7 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@14 a l@13 _start() sleep on 0xff394598 in __lwp_sema_wait()
t@15 a l@9 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@16 a l@23 _start() sleep on 0x28240 in __lwp_sema_wait()
t@17 a l@8 _start() running in _so_recvfrom()
t@18 a l@16 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@19 a l@19 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@20 a l@18 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@21 a l@12 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@22 a l@20 _start() sleep on 0x28240 in __lwp_sema_wait()
t@23 a l@15 _start() sleep on 0x28240 in __lwp_sema_wait()
t@24 a l@21 _start() sleep on 0x28240 in __lwp_sema_wait()
t@25 a l@25 _start() running in ___lwp_mutex_lock()
t@26 a l@11 _start() sleep on 0x3a928 in __lwp_sema_wait()
t@27 a l@14 _start() sleep on 0x28240 in __lwp_sema_wait()
t@28 _start() sleep on 0x79c560 in _cond_wait_cancel()
t@29 _start() sleep on 0x2fd4e8 in _cond_timedwait_cancel()
t@30 a l@17 _start() sleep on 0x29de48 in __lwp_sema_wait()
t@31 _start() sleep on 0x4e47c0 in _cond_timedwait_cancel()
t@33 _start() sleep on 0x13a848 in _cond_wait_cancel()
t@34 _start() sleep on 0x139fd8 in _cond_wait_cancel()
t@37 a l@1 _start() sleep on 0x28240 in __lwp_sema_wait()
This is the stack trace for the timer thread (getting the SIGALRM)
(dbx) thread t@5
t@5 (l@6) stopped in ___lwp_mutex_lock at 0xff317c70
0xff317c70: ___lwp_mutex_lock+0x0008: ta 0x8
(dbx) where
current thread: t@5
=>[1] ___lwp_mutex_lock(0xff394250, 0xe, 0x0, 0x0, 0x0, 0x0), at 0xff317c70
[2] sigacthandler(0xe, 0x0, 0xfed09a10, 0xff38c984, 0x0, 0xfed09d28), at 0xff37823c
---- called from signal handler with signal 14 (SIGALRM) ------
[3] __lwp_sema_wait(0x5b, 0x70114a58, 0x0, 0xfed09d34, 0x0, 0xfed09d28), at 0xff317d38
[4] _co_timerset(0xff38e358, 0xff38d698, 0xff38c984, 0xff38dfd8, 0x3, 0xff38c984), at 0xff36a570
This is the stack trace for the other thread blocked in sigacthandler (getting the SIGLWP)
In this example, it is running a JNI call that is in Oracle code.
(dbx) thread t@25
t@25 (l@25) stopped in ___lwp_mutex_lock at 0xff317c70
0xff317c70: ___lwp_mutex_lock+0x0008: ta 0x8
(dbx) where
current thread: t@25
=>[1] ___lwp_mutex_lock(0xff394250, 0x21, 0x0, 0x0, 0x0, 0x0), at 0xff317c70
[2] sigacthandler(0x21, 0xd4f7f910, 0xd4f7f658, 0xff38c984, 0x32, 0x62b0bf), at 0xff37823c
---- called from signal handler with signal 33 (SIGLWP) ------
[3] ttcs2c(0x0, 0x62b0bf, 0x33, 0x5, 0xd6749dd8, 0x8bb0a4), at 0xd43a7098
[4] ttcacr(0xd6749dd8, 0x5, 0x4bb89a, 0x0, 0x1, 0x2b3c3c), at 0xd43b3d60
[5] ttcdrv(0x8bad28, 0xd6749eb4, 0x0, 0x0, 0x8bb00e, 0x8bb00c), at 0xd437ecf8
[6] nioqwa(0xd6749e7c, 0x0, 0xd437d67c, 0x8bad28, 0xd6749e00, 0x0), at 0xd4290714
[7] upirtrc(0x8b9f44, 0x0, 0x0, 0xd6749e7c, 0x0, 0x0), at 0xd4128980
[8] kpuexe(0x8bc064, 0x8bac74, 0x2b3c3c, 0x8b9f44, 0x8bac64, 0x8bad28), at 0xd40be0d0
[9] upiefn(0x3ff, 0x8b9f44, 0xd6749eb4, 0x2000, 0x1, 0xd6749dd8), at 0xd4127a50
[10] ociefn(0x51b300, 0x64, 0x0, 0x0, 0x20, 0xd883b63a), at 0xd417aa6c
[11] Java_weblogic_db_oci_OciCursor_execAndFetch(0x6c6388, 0xd4f80d84, 0x64, 0x5dd, 0x5, 0x0), at 0xd64a5e78
[12] 0xca8cc(0xd4f80d84, 0xd4f80dec, 0xd4f80df0, 0xffffffff, 0x0, 0x109a0), at 0xca8cb
[13] 0xc5c88(0xd4f80df4, 0xd883b63a, 0xd4, 0xe0858, 0x8, 0xd4f80d18), at 0xc5c87
[14] 0xfa4306c8(0xd8837170, 0x64, 0xb, 0xf8038d70, 0x8, 0xd4f80db0), at 0xfa4306c7
[15] 0xfa5017f8(0xd8836f00, 0x0, 0x6c62f0, 0xe05f0, 0xda522d48, 0x0), at 0xfa5017f7
[16] 0xfa51316c(0xd8839098, 0xf88112e8, 0xd4f80f8c, 0xd82e0, 0x18, 0x0), at 0xfa51316b
[17] 0xfa41cec4(0xd8839098, 0xf88112e8, 0x0, 0x0, 0xbcc00, 0x109a0), at 0xfa41cec3
[18] 0xc6138(0xd4f8100c, 0xf88112e8, 0x0, 0xe0d18, 0x4, 0xd4f80f28), at 0xc6137
[19] 0xc6138(0xd4f810cc, 0xf88112e8, 0x0, 0xe0d18, 0x4, 0xd4f80fb0), at 0xc6137
[20] 0xc6138(0xd4f81174, 0xd4f8120c, 0xd4f81210, 0xe0d18, 0x4, 0xd4f81038), at 0xc6137
[21] 0xc5c88(0xd4f81214, 0xda3c7158, 0xd4f81288, 0xe0858, 0x8, 0xd4f81110), at 0xc5c87
[22] 0xc5c88(0xd4f8128c, 0x1, 0xd4f8129c, 0xe0328, 0x8, 0xd4f811a8), at 0xc5c87
[23] 0xfa403d08(0xda394038, 0xd8831888, 0xd4f812f0, 0xe771c81, 0x8, 0xd4f81248), at 0xfa403d07
[24] 0xfa50898c(0xda394038, 0xda377020, 0x4, 0xda752a10, 0xda9fd128, 0x0), at0xfa50898b
[25] 0xfa507d30(0xda3b3e78, 0x0, 0xd4f81364, 0xd4f81384, 0xbcc00, 0x109a0), at 0xfa507d2f
[26] 0xc5c88(0xd4f81434, 0xf847f2c0, 0xd4f814b8, 0xe0328, 0x14, 0xd4f81310), at 0xc5c87
[27] 0xc5c88(0xd4f814c4, 0xf83f18e0, 0xd4f81520, 0xe0858, 0x10, 0xd4f813c8), at 0xc5c87
[28] 0xc5c88(0xd4f81528, 0xd4f8152c, 0xd4f815d8, 0xe0d18, 0x10, 0xd4f81458), at 0xc5c87
[29] 0xfa40b98c(0xda377020, 0x4, 0xda752a10, 0xe0328, 0xc, 0xd4f814e0), at 0xfa40b98b
[30] 0xfa4cf3a4(0xda377020, 0xda099138, 0xda010240, 0xf8416f70, 0xda08e968, 0x6c62f0), at 0xfa4cf3a3
[31] 0xfa416e50(0xda752a10, 0xda377020, 0xd4f816b0, 0xda0964a0, 0xbcc00, 0x109a0), at 0xfa416e4f
[32] 0xc6318(0xd4f816b4, 0xf80eaea8, 0xd4f81740, 0xe0d18, 0x8, 0xd4f815d0), at 0xc6317
[33] 0xc6318(0xd4f81744, 0x1, 0xd4f81748, 0xe0d18, 0x8, 0xd4f81650), at 0xc6317
[34] 0xc5e68(0xd4f817cc, 0x6c62f0, 0x8, 0xe05f0, 0x8, 0xd4f816d8), at 0xc5e67
[35] 0xfe53a734(0xd4f8185c, 0xd4f81bb8, 0xa, 0xf83e2290, 0x4, 0xd4f81770), at0xfe53a733
[36] JavaCalls::call_helper(0xd4f81afc, 0xfe53a6c0, 0xd4f81af4, 0x6c62f0, 0xd4f818e4, 0x5), at 0xfdb203dc
[37] os::os_exception_wrapper(0xfdb1fc90, 0xd4f81bb0, 0xd4f81a38, 0xd4f81af4,0x6c62f0, 0x1), at 0xfdd04844
[38] JavaCalls::call(0xd4f81bb0, 0xd4f81a38, 0xd4f81af4, 0x6c62f0, 0xd4f81a44, 0xd4f81a40), at 0xfdb1fc24
[39] JavaCalls::call_virtual(0xd4f81a40, 0xd4f81a3c, 0xd4f81ae8, 0xd4f81ae4, 0xd4f81af4, 0x6c62f0), at 0xfdb1ece0
[40] JavaCalls::call_virtual(0xd4f81bb0, 0xd4f81ba0, 0xd4f81b9c, 0xd4f81b98, 0xd4f81b94, 0x6c62f0), at 0xfdb1eddc
[41] thread_entry(0x6c62f0, 0x6c62f0, 0x1, 0x0, 0x0, 0x0), at 0xfdb97c80
[42] JavaThread::thread_main_inner(0x6c62f0, 0x31da10, 0x6, 0x0, 0x0, 0x0), at 0xfde170f0
[43] JavaThread::run(0x6c62f0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfde16f18
[44] _start(0x6c62f0, 0xff245d60, 0x0, 0xd5081e50, 0x1, 0xfe401000), at 0xfdcfe9c0
This is the stack trace for the thread calling sigaction (waiting on semaphore)
In this example, it is running a JNI call that is in Oracle code doing
a login.
(dbx) where
current thread: t@14
=>[1] __lwp_sema_wait(0xd5a81e78, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xff317d34
[2] _park(0xd5a81dc0, 0xd5a81e78, 0x0, 0x8, 0xff38d710, 0xd5b81dc0), at 0xff36b1b8
[3] _swtch(0x5, 0xff38c984, 0xd5a81e54, 0xd5a81e50, 0xd5a81e4c, 0xd5a81e48), at 0xff36aeac
[4] _mutex_adaptive_lock(0xff394598, 0x4c00, 0xff38c984, 0x1, 0x4d58, 0xfffeffff), at 0xff36c744
[5] _cmutex_lock(0xff394598, 0xff, 0x1000, 0xff37974c, 0x0, 0x0), at 0xff36c4f4
[6] _discard_ignored_sig(0x25, 0xd, 0xff394278, 0xff394590, 0x320, 0x19), at 0xff37974c
[7] _setsighandler(0xd, 0xd5a7c094, 0xd5a7c074, 0xff38c984, 0xd5a7bf4c, 0xff393c84), at 0xff3791b8
[8] _sigaction(0xd, 0xd5a7c094, 0xd5a7c074, 0x0, 0x354348, 0x0), at 0xff378bbc
[9] sslsigreghndlr(0xd, 0x1, 0x0, 0x0, 0x0, 0xd5a7c131), at 0xd451ac58
[10] snssgign(0xd, 0x0, 0xfffffffc, 0x10, 0x6c56d0, 0xd5a7d711), at 0xd42213d8
[11] nssgign(0x46aa80, 0x2, 0x0, 0x0, 0x0, 0x1), at 0xd422129c
[12] nsopen(0x7f8a90, 0xd461e970, 0x0, 0x0, 0x0, 0x4000), at 0xd421a01c
[13] nscall1(0xffffffff, 0x7f8a90, 0xd5a7d95c, 0xd5a7d950, 0xd5a7d684, 0x7f8aec), at 0xd4203ea8
[14] nscall(0x354348, 0x7f8a90, 0x0, 0x7f8aec, 0x13e208, 0x0), at 0xd42031f4
[15] niotns(0x1, 0x4bddd8, 0xd45f4444, 0x1414, 0x4bddcc, 0x0), at 0xd428cc14
[16] nigcall(0xd5a7dba0, 0x82, 0xd461162c, 0x1803, 0x4bddcc, 0xd6749fbc), at 0xd4286ac4
[17] osncon(0x4, 0x1803, 0x0, 0x1, 0x1, 0x0), at 0xd41fc800
[18] upiini(0x4bddcc, 0x1400, 0x4bbcac, 0xd6749fbc, 0xd6749f18, 0x4bc9cc), at 0xd4129344
[19] upiah0(0xd6749f18, 0xd5a804ec, 0x6b, 0x0, 0x0, 0xff00), at 0xd411a3c4
[20] upilgn(0x0, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff, 0x0), at 0xd41bd54c
[21] upilog(0xd6749f18, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff, 0xd5a804ec), at 0xd41bea10
[22] ocilog(0xd6749ed8, 0xd6749f18, 0xd5a806ec, 0xffffffff, 0xd5a805ec, 0xffffffff), at 0xd417b1d4
[23] Java_weblogic_db_oci_OciConnection_native_1loginwithblock(0xffffffff, 0xd5a808f4, 0xd5a808f0, 0xd5a808ec, 0xd5a808e8, 0x1), at 0xd649d2e0
[24] 0xca8cc(0xd5a808f4, 0xd5a808f8, 0xd5a808f8, 0xe0ac0, 0x0, 0x109a0), at 0xca8cb
[25] 0xc5c88(0xd5a8098c, 0xda1b5968, 0xe3, 0xe0858, 0x14, 0xd5a80888), at 0xc5c87
[26] 0xc5c88(0xd5a80a4c, 0xf87ecfb8, 0xd5a80ad4, 0xe05f0, 0x14, 0xd5a80920), at 0xc5c87
[27] 0xc6138(0xd5a80ad8, 0xf8804878, 0x0, 0xe0d18, 0xc, 0xd5a809e0), at 0xc6137
[28] 0xc5c88(0xd5a80b6c, 0xf87e85b0, 0xd5a80b70, 0xe0858, 0x8, 0xd5a80a78), at 0xc5c87
[29] 0xc6138(0xd5a80bec, 0xd5a810e0, 0xd5a810e4, 0xe0d18, 0x4, 0xd5a80b00), at 0xc6137
[30] 0xfa41acc8(0xda24c568, 0x1, 0xda3c85d0, 0xda3c85d0, 0x8, 0xd5a80ba8), at 0xfa41acc7
[31] 0xfa4e11b4(0xf8038d70, 0xda1faff0, 0x0, 0x0, 0x1, 0x0), at 0xfa4e11b3
[32] 0xfa4df5b0(0xda24c568, 0x0, 0xd5a80d64, 0xd5a80d7c, 0xbcc00, 0x109a0), at 0xfa4df5af
[33] 0xc5c88(0xd5a80e04, 0xd5a80e84, 0xd5a80e88, 0xe0858, 0x14, 0xd5a80d10), at 0xc5c87
[34] 0xc5c88(0xd5a80e8c, 0xd5a80f14, 0xd5a80f18, 0xe0858, 0x10, 0xd5a80d98), at 0xc5c87
[35] 0xc5c88(0xd5a80f24, 0xd5a80fa4, 0xd5a80fa8, 0xe0328, 0x8, 0xd5a80e20), at 0xc5c87
[36] 0xc5c88(0xd5a80fac, 0xd5a81050, 0xd5a81054, 0xe0ac0, 0x10, 0xd5a80eb8), at 0xc5c87
[37] 0xc5c88(0xd5a81054, 0xd5a810e0, 0xd5a810e4, 0xe0ac0, 0x8, 0xd5a80f40), at 0xc5c87
[38] 0xc5c88(0xd5a810e4, 0xf88581a0, 0x0, 0xe0328, 0x4, 0xd5a80fe8), at 0xc5c87
[39] 0xc6138(0xd5a81184, 0xac, 0xd5a8121c, 0xe0d18, 0x4, 0xd5a81080), at 0xc6137
[40] 0xc5c88(0xd5a81224, 0xd5a812b8, 0xd5a812bc, 0xe0858, 0x4, 0xd5a81118), at 0xc5c87
[41] 0xc5c88(0xd5a812c4, 0xd87401c0, 0xd5a8135c, 0xe0798, 0xc, 0xd5a811c0), at 0xc5c87
[42] 0xc5c88(0xd5a81364, 0xf8a14da0, 0x230, 0xe0268, 0xc, 0xd5a81258), at 0xc5c87
[43] 0xc6138(0xd5a81414, 0xf850d208, 0x1a2, 0xe0c58, 0xc, 0xd5a812f8), at 0xc6137
[44] 0xc5c88(0xd5a814c4, 0xf83f18e0, 0xd5a81520, 0xe0328, 0x14, 0xd5a813a0), at 0xc5c87
[45] 0xc5c88(0xd5a81528, 0xd5a8152c, 0xd5a815d8, 0xe0d18, 0x10, 0xd5a81458), at 0xc5c87
[46] 0xfa40b98c(0xdac86b70, 0x4, 0xda7c8490, 0xe0328, 0xc, 0xd5a814e0), at 0xfa40b98b
[47] 0xfa4cf3a4(0xdac86b70, 0xda096e88, 0xda010240, 0xf8416f70, 0xda08e968, 0x2849b8), at 0xfa4cf3a3
[48] 0xfa416e50(0xda7c8490, 0xdac86b70, 0xd5a816b0, 0xda0964a0, 0xbcc00, 0x109a0), at 0xfa416e4f
[49] 0xc6318(0xd5a816b4, 0xf80eaea8, 0xd5a81740, 0xe0d18, 0x8, 0xd5a815d0), at 0xc6317
[50] 0xc6318(0xd5a81744, 0x1, 0xd5a81748, 0xe0d18, 0x8, 0xd5a81650), at 0xc6317
[51] 0xc5e68(0xd5a817cc, 0x2849b8, 0x8, 0xe05f0, 0x8, 0xd5a816d8), at 0xc5e67
[52] 0xfe53a734(0xd5a8185c, 0xd5a81bb8, 0xa, 0xf83e2290, 0x4, 0xd5a81770), at 0xfe53a733
[53] JavaCalls::call_helper(0xd5a81afc, 0xfe53a6c0, 0xd5a81af4, 0x2849b8, 0xd5a818e4, 0x5), at 0xfdb203dc
[54] os::os_exception_wrapper(0xfdb1fc90, 0xd5a81bb0, 0xd5a81a38, 0xd5a81af4, 0x2849b8, 0x1), at 0xfdd04844
[55] JavaCalls::call(0xd5a81bb0, 0xd5a81a38, 0xd5a81af4, 0x2849b8, 0xd5a81a44, 0xd5a81a40), at 0xfdb1fc24
[56] JavaCalls::call_virtual(0xd5a81a40, 0xd5a81a3c, 0xd5a81ae8, 0xd5a81ae4, 0xd5a81af4, 0x2849b8), at 0xfdb1ece0
[57] JavaCalls::call_virtual(0xd5a81bb0, 0xd5a81ba0, 0xd5a81b9c, 0xd5a81b98, 0xd5a81b94, 0x2849b8), at 0xfdb1eddc
[58] thread_entry(0x2849b8, 0x2849b8, 0x1, 0x0, 0x0, 0x0), at 0xfdb97c80
[59] JavaThread::thread_main_inner(0x2849b8, 0x1e8838, 0x6, 0x0, 0x0, 0x0), at 0xfde170f0
[60] JavaThread::run(0x2849b8, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfde16f18
[61] _start(0x2849b8, 0xff245d60, 0x0, 0xd5b81e50, 0x1, 0xfe401000), at 0xfdcfe9c0
These three threads hang forever.
- duplicates
-
JDK-4363638 Program hangs during creation of Java VM, if used with native DB interface.
-
- Closed
-