0000, 0xd863f2d0, 0xd84e3200, 0xf1400000)
 compacting perm gen total 11520K, used 11344K [0xf1400000, 0xf1f40000, 0xf9400000)
   the space 11520K, 98% used [0xf1400000, 0xf1f14260, 0xf1f14400, 0xf1f40000)

Local Time = Tue Nov 4 01:55:49 2003
Elapsed Time = 54653
#
# HotSpot Virtual Machine Error : 10
# Error ID : 4F530E43505002EF 01
# Please report this error at
# http://java.sun.com/cgi-bin/bugreport.cgi
#
# Java VM: Java HotSpot(TM) Client VM (1.4.2_01-b06 mixed mode)
#
%


Customer experiences crashes in his production environment.
There is no testcase, unfortunately.
Meanwhile the crashes appear on a daily basis.

The stacktraces of the crash look consistent and identical every time, even
with different Java versions. Customer tried client VMs of Java 1.4.1_01,
1.4.1_05, and 1.4.2_01.

The latest stacktrace with Java 1.4.2_01 looks as follows:

% more pstack_core_0211-00
core '/su1/save_core/core_0211-00' of 10822: java -Xms512m -Xmx512m -DJM2DIR=/usr/jm2 -DHOSTNAME=bangkok -classpath
----------------- lwp# 5 / thread# 4 --------------------
 ff3791f0 __sigprocmask (ff37b7f4, 0, 0, fe501d98, ff38c000, 0) + 8
 ff36dd0c _sigon (fe501d98, ff3938a8, 6, fe500684, fe501d98, ff39321c) + d0
 ff370d4c _thrp_kill (0, 4, 6, ff38c000, 4, ff340450) + f8
 ff2cba74 raise (6, 0, 0, ffffffff, ff3403bc, ff3931fc) + 40
 ff2b595c abort (ff33c000, fe5007d8, 0, fffffff8, 4, fe5007f9) + 100
 fe3394ec __1cCosFabort6Fi_v_ (1, fe3ce589, fe500888, fe3f6000, fe40b840, 35aa24) + 80
 fe3375e4 __1cCosbBhandle_unexpected_exception6FpnGThread_ipCpv_v_ (0, b, fe1a8e18, fe5015c0, b, 0) + 2cc
 fe33cf64 JVM_handle_solaris_signal (fe1a8e18, fe5015c0, fe501308, 3400, 3518, 0) + 8a4
 ff37b118 __sighndlr (b, fe5015c0, fe501308, fe33af64, fe501e40, fe501e30) + c
 ff37811c sigacthandler (b, fe501d98, 0, 0, 0, ff38c000) + 708
 --- called from signal handler with signal 11 (SIGSEGV) ---
 fe1a8e18 __1cPContiguousSpaceWprepare_for_compaction6MpnMCompactPoint__v_ (d6b5c6b0, d5400000, 0, 0, 0, 0) + 1e4
 fe1a8bf0 __1cKGenerationWprepare_for_compaction6MpnMCompactPoint__v_ (95de8, fe50176c, f5510978, 95c1c, fe41d7f8, 138a8) +
 2c
 fe1a8b88 __1cQGenCollectedHeapWprepare_for_compaction6M_v_ (95cc8, fe3bccb5, 0, 1, 2a770, 0) + 3c
 fe1a598c __1cMGenMarkSweepTinvoke_at_safepoint6FipnSReferenceProcessor_i_v_ (fe422a10, 50d49f0, 0, 4000, 4c00, 4e6c) + 28c
 fe1a56d0 __1cbCOneContigSpaceCardGenerationHcollect6MiiIii_v_ (976f0, 1, 0, 0, 0, 0) + 34
 fe18bab8 __1cQGenCollectedHeapNdo_collection6MiiIiiiri_v_ (0, fe3f6000, 0, fe407188, 0, 1) + 4f0
 fe1b2eb0 __1cQGenCollectedHeapSdo_full_collection6Miiri_v_ (95cc8, 0, 1, d3e015f4, fe3f6000, fe501ac8) + 20
 fe16d524 __1cMVM_OperationIevaluate6M_v_ (d3e015d8, 288cdc, fe3f6000, 2cc28, 3a3404, fe16a3dc) + 8c
 fe16d3a4 __1cIVMThreadSevaluate_operation6MpnMVM_Operation__v_ (bca50, d3e015d8, 49b0, 4800, 4ac8, 0) + 84
 fe0c6a18 __1cIVMThreadEloop6M_v_ (4000, 3c00, 3ed8, 3c00, 3e80, 3800) + 3e0
 fe0c63ec __1cIVMThreadDrun6M_v_ (bca50, 0, fe4152d8, ffff8000, 0, ff38c000) + 8c
 fe0c62d8 _start (bca50, ff38d658, 1, 1, ff38c000, 0) + 134
 ff37b01c _thread_start (bca50, 0, 0, 0, 0, 0) + 40
----------------- lwp# 6 / thread# 5 --------------------


 Here are some more data. Please find it all attached to this bugreport:

% ls -l hs*
-rwxr-xr-x 1 tl15687 sun 3160 Nov 3 12:55 hs_err_pid13312.log.1.4.1_05
-rwxr-xr-x 1 tl15687 sun 2581 Nov 3 12:55 hs_err_pid1379.log.1.4.2_01
-rw-r--r-- 1 tl15687 sun 2480 Nov 3 12:55 hs_err_pid1394.log.1.4.2_01
-rwxr-xr-x 1 tl15687 sun 1819 Nov 3 12:55 hs_err_pid16244.log.1.4.1_01
-rwxr-xr-x 1 tl15687 sun 1816 Nov 3 12:55 hs_err_pid1700.log.1.4.1_01
-rwxr-xr-x 1 tl15687 sun 1818 Nov 3 12:55 hs_err_pid23086.log.1.4.1_01
-rwxr-xr-x 1 tl15687 sun 1959 Nov 3 12:55 hs_err_pid24560.log.1.4.1_05
-rwxr-xr-x 1 tl15687 sun 2655 Nov 3 12:55 hs_err_pid29642.log.1.4.2_01
-rwxr-xr-x 1 tl15687 sun 3208 Nov 3 12:55 hs_err_pid5056.log.1.4.1_01
-rwxr-xr-x 1 tl15687 sun 2478 Nov 3 12:55 hs_err_pid5707.log.1.4.2_01
-rwxr-xr-x 1 tl15687 sun 3210 Nov 3 12:55 hs_err_pid6623.log.1.4.1_01
-rw-r--r-- 1 tl15687 sun 2585 Nov 3 12:55 hs_err_pid7748.log.1.4.2_01
-rwxr-xr-x 1 tl15687 sun 3211 Nov 3 12:55 hs_err_pid9759.log.1.4.1_01
%

% ls -l pstack*
-rwxr-xr-x 1 tl15687 sun 144510 Nov 3 12:56 pstack_core
-rw-r--r-- 1 tl15687 sun 170505 Nov 3 12:07 pstack_core3010-21
-rw-r--r-- 1 tl15687 sun 271517 Nov 3 12:07 pstack_core_0111-07
-rw-r--r-- 1 tl15687 sun 196642 Nov 3 12:07 pstack_core_0111-20
-rw-r--r-- 1 tl15687 sun 179376 Nov 3 12:07 pstack_core_0111-22
-rw-r--r-- 1 tl15687 sun 131104 Nov 3 12:07 pstack_core_0211-00
-rw-r--r-- 1 tl15687 sun 240218 Nov 3 12:07 pstack_core_3110-20
%

% grep Function hs*
hs_err_pid16244.log.1.4.1_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4B30]
hs_err_pid1700.log.1.4.1_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4A84]
hs_err_pid23086.log.1.4.1_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4B30]
hs_err_pid5056.log.1.4.1_01:Function=[Unknown. Nearest: JVM_IsSupportedJNIVersion+0xAD8]
hs_err_pid6623.log.1.4.1_01:Function=[Unknown. Nearest: JVM_IsSupportedJNIVersion+0xAD8]
hs_err_pid9759.log.1.4.1_01:Function=[Unknown. Nearest: JVM_IsSupportedJNIVersion+0xAD8]
hs_err_pid13312.log.1.4.1_05:Function=[Unknown. Nearest: JVM_IsSupportedJNIVersion+0xAD8]
hs_err_pid24560.log.1.4.1_05:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4B30]
hs_err_pid1379.log.1.4.2_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4C68]
hs_err_pid1394.log.1.4.2_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4C68]
hs_err_pid5707.log.1.4.2_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4C68]
hs_err_pid7748.log.1.4.2_01:Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4C68]
hs_err_pid29642.log.1.4.2_01:Function=[Unknown.]
%


% more hs_err_pid7748.log.1.4.2_01

Unexpected Signal : 10 occurred at PC=0xFE1A8E18
Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4C68]
Library=/usr/java1.4.2_01/jre/lib/sparc/client/libjvm.so


Dynamic libraries:
0x10000 java
0xff360000 /usr/lib/libthread.so.1
0xff3a0000 /usr/lib/libdl.so.1
0xff280000 /usr/lib/libc.so.1
0xff260000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
0xfe000000 /usr/java1.4.2_01/jre/lib/sparc/client/libjvm.so
0xff210000 /usr/lib/libCrun.so.1
0xff1f0000 /usr/lib/libsocket.so.1
0xff100000 /usr/lib/libnsl.so.1
0xff1c0000 /usr/lib/libm.so.1
0xff0e0000 /usr/lib/libsched.so.1
0xff240000 /usr/lib/libw.so.1
0xff0a0000 /usr/lib/libmp.so.2
0xff070000 /usr/java1.4.2_01/jre/lib/sparc/native_threads/libhpi.so
0xff020000 /usr/java1.4.2_01/jre/lib/sparc/libverify.so
0xfe7b0000 /usr/java1.4.2_01/jre/lib/sparc/libjava.so
0xfe790000 /usr/java1.4.2_01/jre/lib/sparc/libzip.so
0xfe610000 /usr/lib/locale/en_US.ISO8859-1/en_US.ISO8859-1.so.2
0xd4780000 /usr/java1.4.2_01/jre/lib/sparc/libawt.so
0xf9480000 /usr/java1.4.2_01/jre/lib/sparc/libmlib_image.so
0xf9410000 /usr/java1.4.2_01/jre/lib/sparc/motif21/libmawt.so
0xd4500000 /usr/lib/libXm.so.4
0xd4490000 /usr/lib/libXt.so.4
0xf9790000 /usr/lib/libXext.so.0
0xfb810000 /usr/openwin/lib/libXtst.so.1
0xd4380000 /usr/lib/libX11.so.4
0xd4820000 /usr/openwin/lib/libdps.so.5
0xf96a0000 /usr/lib/libSM.so.6
0xf95d0000 /usr/lib/libICE.so.6
0xd5150000 /usr/openwin/lib/libdga.so.1
0xd4fd0000 /usr/java1.4.2_01/jre/lib/sparc/libnet.so
0xf95b0000 /usr/java1.4.2_01/jre/lib/sparc/librmi.so
0xd4fb0000 /usr/lib/nss_dns.so.1
0xd4730000 /usr/lib/libresolv.so.2
0xd4e60000 /usr/lib/nss_files.so.1

Heap at VM Abort:
Heap
 def new generation total 56576K, used 51473K [0xd5400000, 0xd8ce0000, 0xd8ce0000)
  eden space 54912K, 93% used [0xd5400000, 0xd8644578, 0xd89a0000)
  from space 1664K, 0% used [0xd8b40000, 0xd8b40000, 0xd8ce0000)
  to space 1664K, 0% used [0xd89a0000, 0xd89a0000, 0xd8b40000)
 tenured generation total 466048K, used 56125K [0xd8ce0000, 0xf5400000, 0xf5400000)
   the space 466048K, 12% used [0xd8ce0000, 0xdc3af5e8, 0xdc285200, 0xf5400000)
 compacting perm gen total 11008K, used 10778K [0xf5400000, 0xf5ec0000, 0xf9400000)
   the space 11008K, 97% used [0xf5400000, 0xf5e86940, 0xf5e86a00, 0xf5ec0000)

Local Time = Fri Oct 24 08:32:05 2003
Elapsed Time = 22842
#
# HotSpot Virtual Machine Error : 10
# Error ID : 4F530E43505002EF 01
# Please report this error at
# http://java.sun.com/cgi-bin/bugreport.cgi
#
# Java VM: Java HotSpot(TM) Client VM (1.4.2_01-b06 mixed mode)
#
%

 "Perm generation space" is pretty much filled up (97%).
  Running with "-XX:MaxPermSize=128m" did not improve the situation.


 The earliest crash, we have data from, is July 13:

% more hs_err_pid16244.log.1.4.1_01

Unexpected Signal : 11 occurred at PC=0xFE1CE4BC
Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4B30]
Library=/usr/java1.4.1/jre/lib/sparc/client/libjvm.so


Dynamic libraries:
0x10000 java
0xff350000 /usr/lib/libthread.so.1
0xff390000 /usr/lib/libdl.so.1
0xff200000 /usr/lib/libc.so.1
0xff330000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
0xfe000000 /usr/java1.4.1/jre/lib/sparc/client/libjvm.so
0xff2e0000 /usr/lib/libCrun.so.1
0xff1e0000 /usr/lib/libsocket.so.1
0xff100000 /usr/lib/libnsl.so.1
0xff0d0000 /usr/lib/libm.so.1
0xff310000 /usr/lib/libw.so.1
0xff0b0000 /usr/lib/libmp.so.2
0xff080000 /usr/java1.4.1/jre/lib/sparc/native_threads/libhpi.so
0xff050000 /usr/java1.4.1/jre/lib/sparc/libverify.so
0xfe7c0000 /usr/java1.4.1/jre/lib/sparc/libjava.so
0xff030000 /usr/java1.4.1/jre/lib/sparc/libzip.so
0xfe6b0000 /usr/lib/locale/en_US.ISO8859-1/en_US.ISO8859-1.so.2
0xd5580000 /usr/java1.4.1/jre/lib/sparc/libawt.so
0xfa080000 /usr/java1.4.1/jre/lib/sparc/libmlib_image.so
0xfa010000 /usr/java1.4.1/jre/lib/sparc/motif21/libmawt.so
0xd5300000 /usr/lib/libXm.so.4
0xd5290000 /usr/lib/libXt.so.4
0xfa290000 /usr/lib/libXext.so.0
0xfc7d0000 /usr/openwin/lib/libXtst.so.1
0xd5180000 /usr/lib/libX11.so.4
0xd5fa0000 /usr/openwin/lib/libdps.so.5
0xfa1a0000 /usr/lib/libSM.so.6
0xd5ed0000 /usr/lib/libICE.so.6
0xd5ea0000 /usr/openwin/lib/libdga.so.1
0xd5dd0000 /usr/java1.4.1/jre/lib/sparc/libnet.so
0xd5db0000 /usr/java1.4.1/jre/lib/sparc/librmi.so

Local Time = Sun Jul 13 07:20:02 2003
Elapsed Time = 1017899
#
# HotSpot Virtual Machine Error : 11
# Error ID : 4F530E43505002E6 01
# Please report this error at
# http://java.sun.com/cgi-bin/bugreport.cgi
#
# Java VM: Java HotSpot(TM) Client VM (1.4.1_01-b01 mixed mode)
#
%



###@###.### 2003-11-04
------------------------------
 Here is the latest data with "-XX:MaxPermSize=128m". Please find the files also attached
 to this bug report:

% more pstack_core_0411-01
core '/su1/save_core/core_0411-01' of 12392: java -Xms512m -Xmx512m -DJM2DIR=/usr/jm2 -XX:MaxPermSize=128m -DHOSTNA
----------------- lwp# 5 / thread# 4 --------------------
 ff3791f0 __sigprocmask (ff37b7f4, 0, 0, fe501d98, ff38c000, 0) + 8
 ff36dd0c _sigon (fe501d98, ff3938a8, 6, fe500684, fe501d98, ff39321c) + d0
 ff370d4c _thrp_kill (0, 4, 6, ff38c000, 4, ff340450) + f8
 ff2cba74 raise (6, 0, 0, ffffffff, ff3403bc, ff3931fc) + 40
 ff2b595c abort (ff33c000, fe5007d8, 0, fffffff8, 4, fe5007f9) + 100
 fe3394ec __1cCosFabort6Fi_v_ (1, fe3ce589, fe500888, fe3f6000, fe40b840, 35aa24) + 80
 fe3375e4 __1cCosbBhandle_unexpected_exception6FpnGThread_ipCpv_v_ (0, a, fe1a8d78, fe5015c0, a, 0) + 2cc
 fe33cf64 JVM_handle_solaris_signal (fe1a8d78, fe5015c0, fe501308, 3400, 3518, 0) + 8a4
 ff37b118 __sighndlr (a, fe5015c0, fe501308, fe33af64, fe501e40, fe501e30) + c
 ff37811c sigacthandler (a, fe501d98, 0, 0, 0, ff38c000) + 708
 --- called from signal handler with signal 10 (SIGBUS) ---
 fe1a8d78 __1cPContiguousSpaceWprepare_for_compaction6MpnMCompactPoint__v_ (d19a42a0, d1400000, 0, 0, 0, 0) + 144
 fe1a8bf0 __1cKGenerationWprepare_for_compaction6MpnMCompactPoint__v_ (95e60, fe50176c, f1510978, 95c94, fe41d7f8, 138a8) + 2c
 fe1a8b88 __1cQGenCollectedHeapWprepare_for_compaction6M_v_ (95d40, fe3bccb5, 0, 1, 2a560, 0) + 3c
 fe1a598c __1cMGenMarkSweepTinvoke_at_safepoint6FipnSReferenceProcessor_i_v_ (fe422a10, 5478098, 0, 4000, 4c00, 4e6c) + 28c
 fe1a56d0 __1cbCOneContigSpaceCardGenerationHcollect6MiiIii_v_ (97768, 1, 0, 0, 0, 0) + 34
 fe18bab8 __1cQGenCollectedHeapNdo_collection6MiiIiiiri_v_ (0, fe3f6000, 0, fe407188, 0, 1) + 4f0
 fe1b2eb0 __1cQGenCollectedHeapSdo_full_collection6Miiri_v_ (95d40, 0, 1, cfe015f4, fe3f6000, fe501ac8) + 20
 fe16d524 __1cMVM_OperationIevaluate6M_v_ (cfe015d8, 288cdc, fe3f6000, 2cca0, 3a3404, fe16a3dc) + 8c
 fe16d3a4 __1cIVMThreadSevaluate_operation6MpnMVM_Operation__v_ (bcac8, cfe015d8, 49b0, 4800, 4ac8, 0) + 84
 fe0c6a18 __1cIVMThreadEloop6M_v_ (4000, 3c00, 3ed8, 3c00, 3e80, 3800) + 3e0
 fe0c63ec __1cIVMThreadDrun6M_v_ (bcac8, 0, fe4152d8, ffff8000, 0, ff38c000) + 8c
 fe0c62d8 _start (bcac8, ff38d658, 1, 1, ff38c000, 0) + 134
 ff37b01c _thread_start (bcac8, 0, 0, 0, 0, 0) + 40
----------------- lwp# 6 / thread# 5 --------------------
[ ... ]


% more hs_err_pid12392.log

Unexpected Signal : 10 occurred at PC=0xFE1A8D78
Function=[Unknown. Nearest: JVM_GetCPFieldClassNameUTF+0x4BC8]
Library=/usr/java1.4.2_01/jre/lib/sparc/client/libjvm.so


Dynamic libraries:
0x10000 java
0xff360000 /usr/lib/libthread.so.1
0xff3a0000 /usr/lib/libdl.so.1
0xff280000 /usr/lib/libc.so.1
0xff260000 /usr/platform/SUNW,Ultra-250/lib/libc_psr.so.1
0xfe000000 /usr/java1.4.2_01/jre/lib/sparc/client/libjvm.so
0xff210000 /usr/lib/libCrun.so.1
0xff1f0000 /usr/lib/libsocket.so.1
0xff100000 /usr/lib/libnsl.so.1
0xff1c0000 /usr/lib/libm.so.1
0xff0e0000 /usr/lib/libsched.so.1
0xff240000 /usr/lib/libw.so.1
0xff0a0000 /usr/lib/libmp.so.2
0xff070000 /usr/java1.4.2_01/jre/lib/sparc/native_threads/libhpi.so
0xff020000 /usr/java1.4.2_01/jre/lib/sparc/libverify.so
0xfe7b0000 /usr/java1.4.2_01/jre/lib/sparc/libjava.so
0xfe790000 /usr/java1.4.2_01/jre/lib/sparc/libzip.so
0xfe560000 /usr/lib/locale/en_US.ISO8859-1/en_US.ISO8859-1.so.2
0xd0780000 /usr/java1.4.2_01/jre/lib/sparc/libawt.so
0xf9480000 /usr/java1.4.2_01/jre/lib/sparc/libmlib_image.so
0xf9410000 /usr/java1.4.2_01/jre/lib/sparc/motif21/libmawt.so
0xd0500000 /usr/lib/libXm.so.4
0xd0490000 /usr/lib/libXt.so.4
0xf9790000 /usr/lib/libXext.so.0
0xfb810000 /usr/openwin/lib/libXtst.so.1
0xd0380000 /usr/lib/libX11.so.4
0xd0820000 /usr/openwin/lib/libdps.so.5
0xf96a0000 /usr/lib/libSM.so.6
0xd1150000 /usr/lib/libICE.so.6
0xd0fd0000 /usr/openwin/lib/libdga.so.1
0xd0e50000 /usr/java1.4.2_01/jre/lib/sparc/libnet.so
0xd0fb0000 /usr/java1.4.2_01/jre/lib/sparc/librmi.so

Heap at VM Abort:
Heap
 def new generation total 56576K, used 27747K [0xd1400000, 0xd4ce0000, 0xd4ce0000)
  eden space 54912K, 50% used [0xd1400000, 0xd2f18dc8, 0xd49a0000)
  from space 1664K, 0% used [0xd4b40000, 0xd4b40000, 0xd4ce0000)
  to space 1664K, 0% used [0xd49a0000, 0xd49a0000, 0xd4b40000)
 tenured generation total 466048K, used 58748K [0xd4ce0000, 0xf1400000, 0xf1400000)
   the space 466048K, 12% used [0xd4ce