Uploaded image for project: 'JDK'
  1. JDK
  2. JDK-4546307

libjvm 1.3.1 Scavenge func SIGSEGV causes Weblogics Apps Server 6 SP2 to Abort

XMLWordPrintable

    • sparc
    • solaris_8

      The Weblogics Apps server was upgraded to SP2 with a new JVM (1.3.1_01).
      After the upgrade, the Weblogics Apps server begins to coredump in JVM with
      SIGSEGV followed by SIGABORT.

      This has happened several times within a short period of 5-6 days.
      Reported Occurances:
      Dec 03, 2001, 13:56:21 SGT
      Nov 29, 2001, 14:20:21 SGT
      Nov 23, 2001, 11:17:45 SGT
      Nov 01, 2001, 14:58:42 SGT


      Loading modules: [ ]
      > $C
      fee0a410 libthread.so.1`__sigprocmask+8(ff36c340, 0, 0, fee0bd78, ff37e000, 0)
      fee0a4b0 libthread.so.1`_sigon+0xd0(fee0bd78, ff385980, 6, fee0a574, fee0bd78, fee0a5b8)
      fee0a510 libthread.so.1`_thrp_kill+0xf8(0, 4, 6, ff37e000, 4, ff2bc4a0)
      fee0a578 libc.so.1`raise+0x40(6, 0, 0, ffffffff, ff2bc40c, fee0a6c8)
      fee0a5d8 libc.so.1`abort+0x100(ff2b801c, fee0a6c8, 0, fffffff8, 4, fee0a6e9)
      fee0a668 libjvm.so`__1cCosFabort6Fl_v_+0xb8(1, fe49a000, 1, fee0a748, 0, fe0de390)
      fee0a6e8 libjvm.so`__1cCosbBhandle_unexpected_exception6FpnGThread_ipCpv_v_+0x254 (ff2bc2fc,
                    fe5087cc, fe4acc2c, fe441aac, fe49a000, fee0a768)
      fee0af98 libjvm.so`JVM_handle_solaris_signal+0x8ec(0, 0, fee0b270, fe49a000, b, fee0b528)
      fee0b058 libthread.so.1`__libthread_segvhdlr+0xe4(b, fee0b528, fee0b270, ff37e000, b, 0)
      fee0b0d8 libthread.so.1`__sighndlr+0xc(b, fee0b528, fee0b270, ff368f78, fee0be10, fee0be00)
      fee0b138 libthread.so.1`sigacthandler+0x704(b, fee0bd78, 0, 0, 0, ff37e000)
      fee0b5a8 libjvm.so`__1cIScavengeXscavenge_oop_with_check6FppnHoopDesc__v_+0x4c(d4a80390, d4a80390, fee0b7f0, 0, 0, 0)
      fee0b608 libjvm.so`__1cJOopMapSetGall_do6FpknFframe_pnICodeBlob_pknLRegisterMap_pFppnHoopDesc__vpF9C9C_vpF9C_vp9H_v_+0x5cc
                    (fee0b7e0, fe41b3ec, fee0b7f0, fe0de344, ffe0, 1)
      fee0b6a0 libjvm.so`__1cJOopMapSetHoops_do6FpknFframe_pnICodeBlob_pknLRegisterMap_pFppnHoopDesc__v_v_+0x2c(fee0b7e0,
                    fad55410, fee0b7f0, fe0de344, 0, 332e2c)
      fee0b700 libjvm.so`__1cFframeHoops_do6MpFppnHoopDesc__vpnLRegisterMap__v_+0xd8(fad55410, fe0de344, fee0b7f0, fe49a000,
                    fee0b7e0, fe0de344)
      fee0b770 libjvm.so`__1cKJavaThreadHoops_do6MpFppnHoopDesc__v_v_+0x13c(0, fe0de344, fe4a4e74, fe49a000, e1db6338, fe0de344)
      fee0b9b0 libjvm.so`__1cHThreadsHoops_do6FpFppnHoopDesc__v_v_+0x34(fe49a000, 31c9e8, fe0de344, 20, 27c620, fe0de344)
      fee0ba10 libjvm.so`__1cIScavengeTinvoke_at_safepoint6Filrl_v_+0x660(fe50a7e4, fe4af294, fe4af288, 0, fe510ef8, fe50a7dc)
      fee0bad0 libjvm.so`__1cMVM_OperationIevaluate6M_v_+0x34(d4a7f918, 164, fee0b, fe49a000, 104218, fee0bb3c)
      fee0bb40 libjvm.so`__1cIVMThreadSevaluate_operation6MpnMVM_Operation__v_+0x3c(104218, d4a7f918, fe49a000, fe4aed24,
                    fe4ab46c, fe49a000)
      fee0bbb0 libjvm.so`__1cIVMThreadEloop6M_v_+0x2e8(104218, d4a7f918, 8, fe4b621c, fe4b6210, fe4b6220)
      fee0bc58 libjvm.so`__1cIVMThreadDrun6M_v_+0x7c(fe49a000, 104218, 0, 0, 0, 0)
      fee0bcb8 libjvm.so`_start+0x20(fe49a000, ff37f690, 1, 1, ff37e000, 0)
      fee0bd18 libthread.so.1`_thread_start+0x40(104218, 0, 0, 0, 0, 0)

      > ::status
      debugging core file of java (32-bit) from CMS2PAPP
      executable file: /usr/j2re1_3_1/bin/sparc/native_threads/java
      initial argv:
      /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java -server -XX:NewSize=128m -X
      status: SIGABRT (Abort)



      Here is another look at the application corefile using the pstack command:

      orchard:/tybong/10151114/03-dec-core/tmp $ more pstack.out
      core '03deccore' of 413: /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java -server -XX:NewSiz
      ----------------- lwp# 4 / thread# 4 --------------------
       ff369ab8 __sigprocmask (ff36c340, 0, 0, fee0bd78, ff37e000, 0) + 8
       ff35dcac _sigon (fee0bd78, ff385980, 6, fee0a574, fee0bd78, fee0a5b8) + d0
       ff360cc0 _thrp_kill (0, 4, 6, ff37e000, 4, ff2bc4a0) + f8
       ff24b190 raise (6, 0, 0, ffffffff, ff2bc40c, fee0a6c8) + 40
       ff2357bc abort (ff2b801c, fee0a6c8, 0, fffffff8, 4, fee0a6e9) + 100
       fe3a4644 ???????? (1, fe49a000, 1, fee0a748, 0, fe0de390)
       fe3a3768 ???????? (ff2bc2fc, fe5087cc, fe4acc2c, fe441aac, fe49a000, fee0a768)
       fe207304 ???????? (0, 0, fee0b270, fe49a000, b, fee0b528)
       ff36905c __libthread_segvhdlr (b, fee0b528, fee0b270, ff37e000, b, 0) + e4
       ff36bbcc __sighndlr (b, fee0b528, fee0b270, ff368f78, fee0be10, fee0be00) + c
       ff3683b8 sigacthandler (b, fee0bd78, 0, 0, 0, ff37e000) + 704
       --- called from signal handler with signal 11 (SIGSEGV) ---
       fe0de390 ???????? (d4a80390, d4a80390, fee0b7f0, 0, 0, 0) <----------This function is the culprit
       fe167008 ???????? (fee0b7e0, fe41b3ec, fee0b7f0, fe0de344, ffe0, 1)
       fe1671ac ???????? (fee0b7e0, fad55410, fee0b7f0, fe0de344, 0, 332e2c)
       fe137038 ???????? (fad55410, fe0de344, fee0b7f0, fe49a000, fee0b7e0, fe0de344)
       fe17b258 ???????? (0, fe0de344, fe4a4e74, fe49a000, e1db6338, fe0de344)
       fe1f7ee0 ???????? (fe49a000, 31c9e8, fe0de344, 20, 27c620, fe0de344)
       fe20d698 ???????? (fe50a7e4, fe4af294, fe4af288, 0, fe510ef8, fe50a7dc)
       fe1934ac ???????? (d4a7f918, 164, fee0b, fe49a000, 104218, fee0bb3c)
       fe193550 ???????? (104218, d4a7f918, fe49a000, fe4aed24, fe4ab46c, fe49a000)
       fe2cf130 ???????? (104218, d4a7f918, 8, fe4b621c, fe4b6210, fe4b6220)
       fe2ceabc ???????? (fe49a000, 104218, 0, 0, 0, 0)
       fe2106fc ???????? (fe49a000, ff37f690, 1, 1, ff37e000, 0)
       ff36bad0 _thread_start (104218, 0, 0, 0, 0, 0) + 40


      comparing the 2 outputs, we know that

      fee0b5a8 libjvm.so`__1cIScavengeXscavenge_oop_with_check6FppnHoopDesc__v_+0x4c(d4a80390, d4a80390, fee0b7f0, 0, 0, 0)
           and
      fe0de390 ???????? (d4a80390, d4a80390, fee0b7f0, 0, 0, 0)

      are the same function.

      All 4 JVM core suffered Segmentation fault in the same module in libjvm.so, the Scavenge function called
      __1cIScavengeXscavenge_oop_with_check6FppnHoopDesc__v_.

      The 4 error logs also points to the same PC (program counter) of 0xfe0de390.
      Furthermore, the error ID is also the same.

      -----------Set 1--------

      orchard:/tybong/10151114/03-dec-core/tmp $ more hs_err_pid310.log

      Unexpected Signal : 11 occurred at PC=0xfe0de390
      Function name=JVM_ReleaseUTF
      Library=/usr/j2re1_3_1/lib/sparc/server/libjvm.so


      Dynamic libraries:
      0x10000 /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java
      0xff350000 /usr/lib/libthread.so.1
      0xff390000 /usr/lib/libdl.so.1
      0xff200000 /usr/lib/libc.so.1
      0xff330000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
      0xfe000000 /usr/j2re1_3_1/lib/sparc/server/libjvm.so
      0xff2e0000 /usr/lib/libCrun.so.1
      0xff1e0000 /usr/lib/libsocket.so.1
      0xff100000 /usr/lib/libnsl.so.1
      0xff0d0000 /usr/lib/libm.so.1
      0xff310000 /usr/lib/libw.so.1
      0xff0b0000 /usr/lib/libmp.so.2
      0xff080000 /usr/j2re1_3_1/lib/sparc/native_threads/libhpi.so
      0xff050000 /usr/j2re1_3_1/lib/sparc/libverify.so
      0xfe7c0000 /usr/j2re1_3_1/lib/sparc/libjava.so
      0xff020000 /usr/j2re1_3_1/lib/sparc/libzip.so
      0xfdfe0000 /usr/j2re1_3_1/lib/sparc/libnet.so
      0xfabc0000 /app/oracle8i/lib/libocijdbc8.so
      0xd1400000 /app/oracle8i/lib/libclntsh.so.8.0
      0xfaba0000 /app/oracle8i/lib/libwtc8.so
      0xfaae0000 /usr/lib/libsched.so.1
      0xfaac0000 /usr/lib/libgen.so.1
      0xfaaa0000 /usr/lib/libaio.so.1
      0xfa9e0000 /usr/lib/nss_files.so.1
      0xfa9c0000 /weblogic/weblogic/bea/wlserver6.0/lib/solaris/libmuxer.so

      Local Time = Fri Nov 23 11:17:45 2001
      Elapsed Time = 18911
      #
      # HotSpot Virtual Machine Error : 11
      # Error ID : 4F530E43505002C4 01
      # Please report this error at
      # http://java.sun.com/cgi-bin/bugreport.cgi
      #
      # Java VM: Java HotSpot(TM) Server VM (1.3.1_01 mixed mode)
      #

      ------- Set 2 -----------

      orchard:/tybong/10151114/03-dec-core/tmp $ more hs_err_pid413.log

      Unexpected Signal : 11 occurred at PC=0xfe0de390
      Function name=JVM_ReleaseUTF
      Library=/usr/j2re1_3_1/lib/sparc/server/libjvm.so


      Dynamic libraries:
      0x10000 /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java
      0xff350000 /usr/lib/libthread.so.1
      0xff390000 /usr/lib/libdl.so.1
      0xff200000 /usr/lib/libc.so.1
      0xff330000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
      0xfe000000 /usr/j2re1_3_1/lib/sparc/server/libjvm.so
      0xff2e0000 /usr/lib/libCrun.so.1
      0xff1e0000 /usr/lib/libsocket.so.1
      0xff100000 /usr/lib/libnsl.so.1
      0xff0d0000 /usr/lib/libm.so.1
      0xff310000 /usr/lib/libw.so.1
      0xff0b0000 /usr/lib/libmp.so.2
      0xff080000 /usr/j2re1_3_1/lib/sparc/native_threads/libhpi.so
      0xff050000 /usr/j2re1_3_1/lib/sparc/libverify.so
      0xfe7c0000 /usr/j2re1_3_1/lib/sparc/libjava.so
      0xff020000 /usr/j2re1_3_1/lib/sparc/libzip.so
      0xfdfe0000 /usr/j2re1_3_1/lib/sparc/libnet.so
      0xfabd0000 /app/oracle8i/lib/libocijdbc8.so
      0xd1400000 /app/oracle8i/lib/libclntsh.so.8.0
      0xfabb0000 /app/oracle8i/lib/libwtc8.so
      0xfab90000 /usr/lib/libsched.so.1
      0xfaae0000 /usr/lib/libgen.so.1
      0xfaac0000 /usr/lib/libaio.so.1
      0xfaaa0000 /usr/lib/nss_files.so.1
      0xfa9e0000 /weblogic/weblogic/bea/wlserver6.0/lib/solaris/libmuxer.so

      Local Time = Mon Dec 3 13:56:21 2001
      Elapsed Time = 232658
      #
      # HotSpot Virtual Machine Error : 11
      # Error ID : 4F530E43505002C4 01
      # Please report this error at
      # http://java.sun.com/cgi-bin/bugreport.cgi
      #
      # Java VM: Java HotSpot(TM) Server VM (1.3.1_01 mixed mode)
      #

      -----------Set 3-------------

      orchard:/tybong/10151114/03-dec-core/tmp $ more hs_err_pid5418.log

      Unexpected Signal : 11 occurred at PC=0xfe0de390
      Function name=JVM_ReleaseUTF
      Library=/usr/j2re1_3_1/lib/sparc/server/libjvm.so


      Dynamic libraries:
      0x10000 /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java
      0xff350000 /usr/lib/libthread.so.1
      0xff390000 /usr/lib/libdl.so.1
      0xff200000 /usr/lib/libc.so.1
      0xff330000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
      0xfe000000 /usr/j2re1_3_1/lib/sparc/server/libjvm.so
      0xff2e0000 /usr/lib/libCrun.so.1
      0xff1e0000 /usr/lib/libsocket.so.1
      0xff100000 /usr/lib/libnsl.so.1
      0xff0d0000 /usr/lib/libm.so.1
      0xff310000 /usr/lib/libw.so.1
      0xff0b0000 /usr/lib/libmp.so.2
      0xff080000 /usr/j2re1_3_1/lib/sparc/native_threads/libhpi.so
      0xff050000 /usr/j2re1_3_1/lib/sparc/libverify.so
      0xfe7c0000 /usr/j2re1_3_1/lib/sparc/libjava.so
      0xff020000 /usr/j2re1_3_1/lib/sparc/libzip.so
      0xfdfe0000 /usr/j2re1_3_1/lib/sparc/libnet.so
      0xfabd0000 /app/oracle8i/lib/libocijdbc8.so
      0xd1400000 /app/oracle8i/lib/libclntsh.so.8.0
      0xfabb0000 /app/oracle8i/lib/libwtc8.so
      0xfab90000 /usr/lib/libsched.so.1
      0xfaae0000 /usr/lib/libgen.so.1
      0xfaac0000 /usr/lib/libaio.so.1
      0xfaaa0000 /usr/lib/nss_files.so.1
      0xfa9e0000 /weblogic/weblogic/bea/wlserver6.0/lib/solaris/libmuxer.so

      Local Time = Thu Nov 29 14:20:21 2001
      Elapsed Time = 75983
      #
      # HotSpot Virtual Machine Error : 11
      # Error ID : 4F530E43505002C4 01
      # Please report this error at
      # http://java.sun.com/cgi-bin/bugreport.cgi
      #
      # Java VM: Java HotSpot(TM) Server VM (1.3.1_01 mixed mode)
      #

      -----Set 4------------

      orchard:/tybong/10151114/03-dec-core/tmp $ more hs_err_pid715.log

      Unexpected Signal : 11 occurred at PC=0xfe0de390
      Function name=JVM_ReleaseUTF
      Library=/usr/j2re1_3_1/lib/sparc/server/libjvm.so


      Dynamic libraries:
      0x10000 /usr/j2re1_3_1/bin/../bin/sparc/native_threads/java
      0xff350000 /usr/lib/libthread.so.1
      0xff390000 /usr/lib/libdl.so.1
      0xff200000 /usr/lib/libc.so.1
      0xff330000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
      0xfe000000 /usr/j2re1_3_1/lib/sparc/server/libjvm.so
      0xff2d0000 /usr/lib/libCrun.so.1
      0xff1e0000 /usr/lib/libsocket.so.1
      0xff100000 /usr/lib/libnsl.so.1
      0xff0d0000 /usr/lib/libm.so.1
      0xff300000 /usr/lib/libw.so.1
      0xff0b0000 /usr/lib/libmp.so.2
      0xff080000 /usr/j2re1_3_1/lib/sparc/native_threads/libhpi.so
      0xff050000 /usr/j2re1_3_1/lib/sparc/libverify.so
      0xfe7c0000 /usr/j2re1_3_1/lib/sparc/libjava.so
      0xff020000 /usr/j2re1_3_1/lib/sparc/libzip.so
      0xfdfe0000 /usr/j2re1_3_1/lib/sparc/libnet.so
      0xfabd0000 /app/oracle8i/lib/libocijdbc8.so
      0xd1800000 /app/oracle8i/lib/libclntsh.so.8.0
      0xfabb0000 /app/oracle8i/lib/libwtc8.so
      0xfab90000 /usr/lib/libsched.so.1
      0xfaae0000 /usr/lib/libgen.so.1
      0xfaac0000 /usr/lib/libaio.so.1
      0xfaaa0000 /usr/lib/nss_files.so.1
      0xfa9e0000 /weblogic/weblogic/bea/wlserver6.0/lib/solaris/libmuxer.so

      Local Time = Thu Nov 1 14:58:42 2001
      Elapsed Time = 19734
      #
      # HotSpot Virtual Machine Error : 11
      # Error ID : 4F530E43505002C4 01
      # Please report this error at
      # http://java.sun.com/cgi-bin/bugreport.cgi
      #
      # Java VM: Java HotSpot(TM) Server VM (1.3.1_01 mixed mode)
      #







            mearagolsunw Murali Earagolla (Inactive)
            duke J. Duke
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Created:
              Updated:
              Resolved:
              Imported:
              Indexed: