Running an invokedynamic demo on the client compiler (i386) in -Xcomp mode runs into an assertion error in SharedRuntime::resolve_sub_helper at this code:

  // JSR 292
  // If the resolved method is a MethodHandle invoke target the call
  // site must be a MethodHandle call site.
  if (callee_method->is_method_handle_invoke()) {
    assert(caller_nm->is_method_handle_return(caller_frame.pc()), "must be MH call site");
  }

The call is to MH.invokeExact(Object,Object,Object,Object)void.

The caller is sun.dyn.MethodHandleImpl$AllocateObject::invoke_L3. The call site does not apparently preserve %esp. See the dump below.

To reproduce, unzip the attached file indy-bsm-test.zip and run according to the instructions in the README.

Sample command lines:

$ (cd make; gmake jvmg1)
$ cd build/solaris/solaris_i486_compiler1/jvmg/
$ ln -s indy-bsm-test/*.jar .
$ ./gamma -XX:+ShowMessageBoxOnError -XX:CIStart=646 -Xcomp -XX:CICompilerCount=1 -Xbatch -XX:-PrintCompilation -XX:+UnlockExperimentalVMOptions -XX:+EnableInvokeDynamic -Xbootclasspath/p:IndyLocalBootstrap.jar:InvokeDynamicDemo.jar:JRS292.jar meth.IndyLocalBootstrap




(dbx) p caller_nm.print()
Compiled (c1) 646 nmethod sun.dyn.MethodHandleImpl$AllocateObject::invoke_L3 (21 bytes)
 total in heap [0xfbef2088,0xfbef23b0] = 808
 relocation [0xfbef2158,0xfbef21a4] = 76
 main code [0xfbef21b0,0xfbef2260] = 176
 stub code [0xfbef2260,0xfbef22b6] = 86
 constants [0xfbef22b6,0xfbef22b8] = 2
 oops [0xfbef22b8,0xfbef22bc] = 4
 scopes data [0xfbef22bc,0xfbef2310] = 84
 scopes pcs [0xfbef2310,0xfbef23a0] = 144
 dependencies [0xfbef23a0,0xfbef23a4] = 4
 nul chk table [0xfbef23a4,0xfbef23b0] = 12
 oops [0xfbef22b8,0xfbef22bc] = 4
((class nmethod *) caller_nm)->print() = (void)
(dbx) p caller_frame
caller_frame = {
    _sp = 0x80465a0
    _pc = 0xfbef220c "\x8bD$ \x83\xc4(]\x85^E"
    _cb = 0xfbef2088
    _deopt_state = not_deoptimized
    _fp = 0x8046600
    _unextended_sp = 0x80465a0
}
(dbx) dis 0xfbef21b0,0xfbef2260
0xfbef21b0: nop
0xfbef21b1: nop
0xfbef21b2: nop
0xfbef21b3: nop
0xfbef21b4: nop
0xfbef21b5: nop
0xfbef21b6: nop
0xfbef21b7: cmpl 0x00000004(%ecx),%eax
0xfbef21ba: jne 0xfbeaad90 [ 0xfbeaad90, .-0x4742a ]
0xfbef21c0: movl %eax,0xffffc000(%esp)
0xfbef21c7: pushl %ebp
0xfbef21c8: subl $0x00000028,%esp
0xfbef21cb: movl %edx,0x0000001c(%esp)
0xfbef21cf: movl %ecx,%ebx
0xfbef21d1: movl %ebx,%ecx
0xfbef21d3: movl %ebx,0x00000018(%esp)
0xfbef21d7: call 0xfbef7b20 [ 0xfbef7b20, .+0x5949 ]
0xfbef21dc: movl 0x00000018(%esp),%ecx
0xfbef21e0: movl 0x00000020(%ecx),%ecx
0xfbef21e3: cmpl (%ecx),%eax
0xfbef21e5: movl %eax,%edx
0xfbef21e7: movl 0x0000001c(%esp),%esi
0xfbef21eb: movl %esi,(%esp)
0xfbef21ee: movl 0x00000030(%esp),%esi
0xfbef21f2: movl %esi,0x00000004(%esp)
0xfbef21f6: movl 0x00000034(%esp),%esi
0xfbef21fa: movl %esi,0x00000008(%esp)
0xfbef21fe: movl %eax,0x00000020(%esp)
0xfbef2202: movl $0xffffffff,%eax
0xfbef2207: call 0xfbeab210 [ 0xfbeab210, .-0x46ff7 ]
0xfbef220c: movl 0x00000020(%esp),%eax <<<<<<<<
0xfbef2210: addl $0x00000028,%esp
0xfbef2213: popl %ebp
0xfbef2214: testl %eax,0xfef20100 [ 0xfef20100 ]
0xfbef221a: ret
0xfbef221b: call 0xfbeef060 [ 0xfbeef060, .-0x31bb ]
0xfbef2220: pushl $__RTTI__1nRAbstractAssembler_+0x867e
0xfbef2225: call 0xfbef222a [ 0xfbef222a, .+5 ]
0xfbef222a: pushal
0xfbef222b: call debug32 [ 0xfe31a890, .+0x2428665 ]
0xfbef2230: hlt
0xfbef2231: nop
0xfbef2232: nop
0xfbef2233: nop
0xfbef2234: movl %gs:0x000000e0,%esi
0xfbef223c: movl 0x0000018c(%esi),%eax
0xfbef2242: movl $0x00000000,0x0000018c(%esi)
0xfbef224c: movl $0x00000000,0x00000190(%esi)
0xfbef2256: addl $0x00000028,%esp
0xfbef2259: popl %ebp
0xfbef225a: jmp 0xfbeee890 [ 0xfbeee890, .-0x39ca ]
0xfbef225f: hlt
0xfbef2260: nop
(dbx) call find(0xfbef7b20)

"Executing find"
Compiled (c1) 663 nmethod sun.dyn.MethodHandleImpl$AllocateObject::allocate (11 bytes)
 total in heap [0xfbef7a08,0xfbef7c50] = 584
 relocation [0xfbef7ad8,0xfbef7b0c] = 52
 main code [0xfbef7b10,0xfbef7b80] = 112
 stub code [0xfbef7b80,0xfbef7bca] = 74
 constants [0xfbef7bca,0xfbef7bcc] = 2
 oops [0xfbef7bcc,0xfbef7bd0] = 4
 scopes data [0xfbef7bd0,0xfbef7bec] = 28
 scopes pcs [0xfbef7bec,0xfbef7c4c] = 96
 dependencies [0xfbef7c4c,0xfbef7c50] = 4
 oops [0xfbef7bcc,0xfbef7bd0] = 4
t@1 (l@1) stopped in _waitid at 0xfe17a4d7
0xfe17a4d7: _waitid+0x0007: jae _waitid+0x13 [ 0xfe17a4e3, .+0xc ]
Current function is SharedRuntime::resolve_sub_helper
  942 assert(caller_nm->is_method_handle_return(caller_frame.pc()), "must be MH call site");
(dbx) call find(0xfbeab210)

"Executing find"
[CodeBlob (0xfbeab1c8)]
Framesize: 71
Runtime Stub (0xfbeab1c8): resolve_virtual_call
Could not load hsdis-i386.so; library not loadable; PrintAssembly is disabled
t@1 (l@1) stopped in _waitid at 0xfe17a4d7
0xfe17a4d7: _waitid+0x0007: jae _waitid+0x13 [ 0xfe17a4e3, .+0xc ]
Current function is SharedRuntime::resolve_sub_helper
  942 assert(caller_nm->is_method_handle_return(caller_frame.pc()), "must be MH call site");