A crash in ToolkitErrorHandler() in XlibWrapper.c

http://mail.openjdk.java.net/pipermail/awt-dev/2012-November/003928.html
(from Andrew Haley aph@redhat.com)

This one was reported by the LibreOffice folks.

We don't check the return argument of JNU_GetEnv() in ToolkitErrorHandler:


static int ToolkitErrorHandler(Display * dpy, XErrorEvent * event) {
    if (jvm != NULL) {
        JNIEnv * env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
        return JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XToolkit", "globalErrorHandler", "(JJ)I",
                                          ptr_to_jlong(dpy), ptr_to_jlong(event)).i;
    } else {
        return 0;
    }
}


JNU_GetEnv() will return NULL if this thread is not a Java thread, so
we crash. This will happen if SWT is loaded in an application that
uses X itself in some threads.

The patch is pretty trivial, we just have to check env before using it:


--- jdk/src/solaris/native/sun/xawt/XlibWrapper.c~ 2012-10-11 17:20:54.000000000 +0100
+++ jdk/src/solaris/native/sun/xawt/XlibWrapper.c 2012-11-30 10:52:19.980613972 +0000
@@ -1260,13 +1260,15 @@

 JavaVM* jvm = NULL;
 static int ToolkitErrorHandler(Display * dpy, XErrorEvent * event) {
+ JNIEnv * env;
     if (jvm != NULL) {
- JNIEnv * env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
- return JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XToolkit", "globalErrorHandler", "(JJ)I",
- ptr_to_jlong(dpy), ptr_to_jlong(event)).i;
- } else {
- return 0;
+ env = (JNIEnv *)JNU_GetEnv(jvm, JNI_VERSION_1_2);
+ if (env) {
+ return JNU_CallStaticMethodByName(env, NULL, "sun/awt/X11/XToolkit", "globalErrorHandler", "(JJ)I",
+ ptr_to_jlong(dpy), ptr_to_jlong(event)).i;
+ }
     }
+ return 0;
 }

 /*