symptom is a deep recursion in the call stack, hotspot dies with:
      An irrecoverable stack overflow has occurred.
message. This stack depth is causing grief for dbx (it only displays
the top 100 frames.) I used my "dump_stki" dumb stack dump
script and managed to dump ~800 frames but then dbx dies again.
the stack frames all apear to be legitimate but there are only 2
return addresses:
                          591 0x000d3cdc: jmp %g3
                          237 0x000d3f84: jmp %g3
We have tried -Xss2m and it died immediately with above error.
We had a marginally longer run at -Xss1m on the STE machine.

though we have been unable to look at
             the bottom of the stack with dbx, pstack will dump it:

 core 'acpapp2.stack256.core' of 2583: /BA/opt/jdk131/bin/../bin/sparc/native_threads/java_g -Xss256k -XX:+Pr
----------------- lwp# 19 / thread# 19 --------------------
 ff31bdc4 _lwp_kill (6, 0, 0, ffffffff, ff33e3b4, 0) + 8
 ff2b5a7c abort (ff33a000, d0d34e90, 0, 4, 0, d0d34eb1) + 100
 fe8a95f8 void os::abort(long) (1, fedcd5c4, d0d34f8c, b, 0, d0d35800) + e0
 fe89b688 void os::handle_unexpected_exception(Thread*,int,unsigned char*,void*) (3d8408, b, fb01378c, d0d35d78, 0, 0) + 3d8
 fe8bb0c0 JVM_handle_solaris_signal (b, d0d35d78, d0d35ac0, 1, 0, 0) + ff8
 fe8ad52c signalHandler (b, d0d35d78, d0d35ac0, 0, 0, 0) + 4c
 ff374cc8 __sighndlr (b, d0d35d78, d0d35ac0, fe8ad4e0, 0, 0) + c
 ff36fb00 call_user_handler (fefb2200, 13, ff3878e0, d0d35ac0, d0d35d78, b) + 254
 ff36fccc sigacthandler (fefb2200, d0d35d78, d0d35ac0, ff386000, d0d35d78, b) + 64
 --- called from signal handler with signal -17096192 (SIG Unknown) ---
 fb01378c ???????? (f31f6db0, f316d6f0, d0d35f60, edb20, f316d6f0, 0)
 000d3cc4 ???????? (f31f6db0, d0d35f68, d0d35f68, edf20, 0, 0)
 000d3cc4 ???????? (f31f6db0, d0d36088, d0d3608c, edf20, f2c7aa10, 0)
 000d3cc4 ???????? (f31f6db0, f2c5f650, d0d3610c, edf20, f2c5f650, 0)
 000d3cc4 ???????? (f31f6db0, d0d36190, d0d36194, edf20, 52, 0)
 000d3cc4 ???????? (d31bc460, d0d3621c, d0d36220, edf20, f3943be0, 0)
 000d3dd4 ???????? (d31bc460, f2c06fe8, 0, edd20, d49c0000, 0)
 000d3dd4 ???????? (d51a74d8, 3, d0d36350, edd20, d31bc0e8, 0)
 fb13d3ac ???????? (d51a74d8, f2f929e0, d0d36414, e709c, 9, 0)
 000d3f6c ???????? (d2d1e1b0, d0d36498, d0d3649c, ee110, d31bc130, 0)
 000d3dd4 ???????? (d2d1e1b0, d66a4, d0d36560, ed90c, f2c7aa10, 0)
 000d407c ???????? (d2cf8ff0, d0d365e8, d0d365ec, ee110, a, 0)
 000d3dd4 ???????? (d2cf8ff0, f31f6db0, 0, edb20, d49c0000, 0)
 000d3dd4 ???????? (d31bbed0, d66a4, d0d3679c, edb20, 9, 0)
 
... representing about 2800 stack frames with the 2 addresses above

 000d407c ???????? (d2c49a60, b6, d0d6f670, ee110, d50fdc90, 0)
 000d3dd4 ???????? (d5113c40, b6, d0d6f7bc, e8338, 0, 0)
 000d3dd4 ???????? (d5113c40, b6, d4c198a0, e8338, f2cfd940, 0)
 000d3dd4 ???????? (d50fdc90, b6, d0d6f8f0, e8338, f2c03010, 0)
 000d3dd4 ???????? (d51a74d8, d66a4, d0d6f980, e8338, d4e690e8, 0)
 000d407c ???????? (d4e6be98, 1, d0d6f988, ee110, d4459890, 0)
 000d3dd4 ???????? (d4459890, 3d8408, 8, edb20, d0d6fd4c, 3d89d0)
 fef48284 StubRoutines::_code1 (d0d6fa9c, d0d6fdf8, a, f30da7f8, d66a4, d0d6fd40) + 4e4
fe6bf454 void JavaCalls::call_helper(JavaValue*,methodHandle*,JavaCallArguments*,Thread*) (d0d6fd3c, fef48200, d0d6fd34 , 3d8408, d0d6fb24, 5) + 74c
 fe8ad41c void os::os_exception_wrapper(void(*)(JavaValue*,methodHandle*,JavaCallArguments*,Thread*),JavaValue*,methodHandle*,JavaCallArguments*,Thread*) (fe6bed08, d0d6fdf0, d0d6fc78, d0d6fd34, 3d8408, 1) + 5c
 fe6bec9c void JavaCalls::call(JavaValue*,methodHandle,JavaCallArguments*,Thread*) (d0d6fdf0, d0d6fc78, d0d6fd34, 3d8408, d0d6fc84, d0d6fc80) + 11c
 fe6bdd58 void JavaCalls::call_virtual(JavaValue*,KlassHandle,symbolHandle,symbolHandle,JavaCallArguments*,Thread*) (d0d6fc80, d0d6fc7c, d0d6fd28, d0d6fd24, d0d6fd34, 3d8408) + 258
 fe6bde54 void JavaCalls::call_virtual(JavaValue*,Handle,KlassHandle,symbolHandle,symbolHandle,Thread*) (d0d6fdf0, d0d6f
 000d3dd4 ???????? (d5113c40, b6, d0d6f7bc, e8338, 0, 0)
 000d3dd4 ???????? (d5113c40, b6, d4c198a0, e8338, f2cfd940, 0)
 000d3dd4 ???????? (d50fdc90, b6, d0d6f8f0, e8338, f2c03010, 0)
 000d3dd4 ???????? (d51a74d8, d66a4, d0d6f980, e8338, d4e690e8, 0)
 000d407c ???????? (d4e6be98, 1, d0d6f988, ee110, d4459890, 0)
 000d3dd4 ???????? (d4459890, 3d8408, 8, edb20, d0d6fd4c, 3d89d0)
 fef48284 StubRoutines::_code1 (d0d6fa9c, d0d6fdf8, a, f30da7f8, d66a4, d0d6fd40) + 4e4
 fe6bf454 void JavaCalls::call_helper(JavaValue*,methodHandle*,JavaCallArguments*,Thread*) (d0d6fd3c, fef48200, d0d6fd34
, 3d8408, d0d6fb24, 5) + 74c
 fe8ad41c void os::os_exception_wrapper(void(*)(JavaValue*,methodHandle*,JavaCallArguments*,Thread*),JavaValue*,methodHa
ndle*,JavaCallArguments*,Thread*) (fe6bed08, d0d6fdf0, d0d6fc78, d0d6fd34, 3d8408, 1) + 5c
 fe6bec9c void JavaCalls::call(JavaValue*,methodHandle,JavaCallArguments*,Thread*) (d0d6fdf0, d0d6fc78, d0d6fd34, 3d8408
, d0d6fc84, d0d6fc80) + 11c
 fe6bdd58 void JavaCalls::call_virtual(JavaValue*,KlassHandle,symbolHandle,symbolHandle,JavaCallArguments*,Thread*) (d0d
6fc80, d0d6fc7c, d0d6fd28, d0d6fd24, d0d6fd34, 3d8408) + 258
 fe6bde54 void JavaCalls::call_virtual(JavaValue*,Handle,KlassHandle,symbolHandle,symbolHandle,Thread*) (d0d6fdf0, d0d6f
de0, d0d6fddc, d0d6fdd8, d0d6fdd4, 3d8408) + bc
 fe737fb4 void thread_entry(JavaThread*,Thread*) (3d8408, 3d8408, 1, 0, 0, 0) + 104
 fe9d4700 void JavaThread::thread_main_inner() (3d8408, 3d8ff0, 6, 0, 0, 0) + 1b8
 fe9d4524 void JavaThread::run() (3d8408, 0, 0, 0, 0, 0) + 22c
 fe8a7588 _start (3d8408, fefb2200, 0, 0, 0, 0) + e0
 ff374970 _lwp_start (0, 0, 0, 0, 0, 0)