The server compiler cannot gracefully handle calls to MH.invoke[Exact] when one of the types present is not on the BCP. The problem is in SystemDictionary::find_method_handle_invoke, when run in the compile thread, when it uses the path involving non_cached_result. We should be caching the required results on a class loader object associated with the referencing class.

/*
  Run:
  $ javac -d . TooManyTraps.java
  $ java -client -cp . -XX:+PrintCompilation TooManyTraps
  $ java -server -cp . -XX:+PrintCompilation TooManyTraps
 */
import java.lang.invoke.*;
import static java.lang.invoke.MethodType.*;
import static java.lang.invoke.MethodHandles.*;

class TooManyTraps {
    static void callme(Object x) { }
    static void callme(TooManyTraps x) { }
    public static void main(String... av) throws Throwable {
        MethodHandle mh1 = lookup().findStatic(TooManyTraps.class, "callme", methodType(void.class, Object.class));
        MethodHandle mh2 = lookup().findStatic(TooManyTraps.class, "callme", methodType(void.class, TooManyTraps.class));
        System.out.println("starting...");
        long t0 = System.currentTimeMillis();
        for (int i = 0; i < 1000; i++) {
            test1(1000, mh1);
        }
        long t1 = System.currentTimeMillis();
        System.out.println("fast way took "+(t1-t0)+"ms");
        t0 = System.currentTimeMillis();
        for (int i = 0; i < 1000; i++) {
            test2(1000, mh2);
        }
        t1 = System.currentTimeMillis();
        System.out.println("slow way took "+(t1-t0)+"ms");
    }
    static void test1(int reps, MethodHandle mh) throws Throwable {
        for (int i = 0; i < reps; i++) {
            mh.invokeExact((Object)null);
        }
    }
    static void test2(int reps, MethodHandle mh) throws Throwable {
        for (int i = 0; i < reps; i++) {
            mh.invokeExact((TooManyTraps)null);
        }
    }
}

/*
--------
$JAVA_HOME/bin/javac -d . TooManyTraps.java
--------
$JAVA_HOME/bin/java -client -XX:+PrintCompilation -cp . TooManyTraps
starting...
     70 1 java.lang.String::equals (88 bytes)
     72 2 java.lang.String::hashCode (67 bytes)
     72 3 java.lang.String::charAt (33 bytes)
     72 4 java.lang.String::indexOf (87 bytes)
     72 5 sun.nio.cs.US_ASCII$Encoder::encode (107 bytes)
     73 6 java.lang.String::lastIndexOf (68 bytes)
     73 7 TooManyTraps::callme (1 bytes)
     73 8 TooManyTraps::test1 (22 bytes)
fast way took 19ms
     88 9 TooManyTraps::callme (1 bytes)
     88 10 TooManyTraps::test2 (22 bytes)
slow way took 23ms
--------
$JAVA_HOME/bin/java -server -XX:+PrintCompilation -cp . TooManyTraps
starting...
     78 1 TooManyTraps::callme (1 bytes)
     85 2 TooManyTraps::test1 (22 bytes)
     87 1 % TooManyTraps::test1 @ 2 (22 bytes)
fast way took 21ms
    100 3 TooManyTraps::callme (1 bytes)
    100 4 TooManyTraps::test2 (22 bytes)
    100 4 TooManyTraps::test2 (22 bytes) made not entrant
    101 2 % TooManyTraps::test2 @ 2 (22 bytes)
    102 2 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    102 5 TooManyTraps::test2 (22 bytes)
    102 3 % TooManyTraps::test2 @ 2 (22 bytes)
    103 3 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    103 5 TooManyTraps::test2 (22 bytes) made not entrant
    103 4 % TooManyTraps::test2 @ 2 (22 bytes)
    103 4 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    104 6 TooManyTraps::test2 (22 bytes)
    104 5 % TooManyTraps::test2 @ 2 (22 bytes)
    130 6 TooManyTraps::test2 (22 bytes) made not entrant
    131 5 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    131 7 TooManyTraps::test2 (22 bytes)
    131 6 % TooManyTraps::test2 @ 2 (22 bytes)
    132 6 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    132 7 TooManyTraps::test2 (22 bytes) made not entrant
    132 7 % TooManyTraps::test2 @ 2 (22 bytes)
    133 7 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    133 8 TooManyTraps::test2 (22 bytes)
    133 8 % TooManyTraps::test2 @ 2 (22 bytes)
    134 8 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    134 8 TooManyTraps::test2 (22 bytes) made not entrant
    135 9 % TooManyTraps::test2 @ 2 (22 bytes)
    135 9 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    135 9 TooManyTraps::test2 (22 bytes)
    179 9 TooManyTraps::test2 (22 bytes) made not entrant
    180 10 % TooManyTraps::test2 @ 2 (22 bytes)
    181 10 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    181 10 TooManyTraps::test2 (22 bytes)
    181 11 % TooManyTraps::test2 @ 2 (22 bytes)
    182 11 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    182 10 TooManyTraps::test2 (22 bytes) made not entrant
    183 12 % TooManyTraps::test2 @ 2 (22 bytes)
    185 12 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    185 11 TooManyTraps::test2 (22 bytes)
    185 13 % TooManyTraps::test2 @ 2 (22 bytes)
    186 13 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    186 11 TooManyTraps::test2 (22 bytes) made not entrant
    186 14 % TooManyTraps::test2 @ 2 (22 bytes)
    187 14 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    208 12 TooManyTraps::test2 (22 bytes)
    209 15 % TooManyTraps::test2 @ 2 (22 bytes)
    210 15 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    210 12 TooManyTraps::test2 (22 bytes) made not entrant
    210 16 % TooManyTraps::test2 @ 2 (22 bytes)
    211 16 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    211 13 TooManyTraps::test2 (22 bytes)
    211 17 % TooManyTraps::test2 @ 2 (22 bytes)
    212 17 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    212 13 TooManyTraps::test2 (22 bytes) made not entrant
    212 18 % TooManyTraps::test2 @ 2 (22 bytes)
    213 18 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    213 14 TooManyTraps::test2 (22 bytes)
    213 19 % TooManyTraps::test2 @ 2 (22 bytes)
    214 19 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    245 14 TooManyTraps::test2 (22 bytes) made not entrant
    245 20 % TooManyTraps::test2 @ 2 (22 bytes)
    246 20 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    246 15 TooManyTraps::test2 (22 bytes)
    246 21 % TooManyTraps::test2 @ 2 (22 bytes)
    247 21 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    247 15 TooManyTraps::test2 (22 bytes) made not entrant
    248 22 % TooManyTraps::test2 @ 2 (22 bytes)
    249 22 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    249 16 TooManyTraps::test2 (22 bytes)
    251 23 % TooManyTraps::test2 @ 2 (22 bytes)
    251 23 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    251 16 TooManyTraps::test2 (22 bytes) made not entrant
    252 24 % TooManyTraps::test2 @ 2 (22 bytes)
    252 24 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    293 17 TooManyTraps::test2 (22 bytes)
    294 25 % TooManyTraps::test2 @ 2 (22 bytes)
    294 25 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    295 17 TooManyTraps::test2 (22 bytes) made not entrant
    295 26 % TooManyTraps::test2 @ 2 (22 bytes)
    296 26 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    296 18 TooManyTraps::test2 (22 bytes)
    296 27 % TooManyTraps::test2 @ 2 (22 bytes)
    297 27 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    297 18 TooManyTraps::test2 (22 bytes) made not entrant
    297 28 % TooManyTraps::test2 @ 2 (22 bytes)
    298 28 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    298 19 TooManyTraps::test2 (22 bytes)
    298 29 % TooManyTraps::test2 @ 2 (22 bytes)
    299 29 % TooManyTraps::test2 @ -2 (22 bytes) made not entrant
    343 19 TooManyTraps::test2 (22 bytes) made not entrant
slow way took 246ms
--------
 */