* reproducer
** tree
|-- debugcontroltest.properties
|-- jmx-test-cert.pkcs12
|-- jmxremote.password
|-- pom.xml
`-- src
    `-- main
        |-- java
        |   `-- debugcontrol
        |       |-- DebugController.java
        |       |-- client
        |       |   `-- JMXSSLClient.java
        |       `-- server
        |           `-- JMXSSLServer.java
        `-- resources
            `-- jmxremote.password

** debugcontroltest.properties
debugcontroltest.host = localhost
debugcontroltest.port = 9876
debugcontroltest.stop_time = 120000

debugcontroltest.jmxremote.password.filename = jmxremote.password
debugcontroltest.cert.filename = jmx-test-cert.pkcs12
debugcontroltest.cert.type = pkcs12
debugcontroltest.cert.pass = changeit

sun.rmi.transport.tcp.responseTimeout = 1000
sun.rmi.transport.tcp.handshakeTimeout = 1000

** jmx-test-cert.pkcs12
*** binary file. Create pkcs12 certificates by openssl or other commands.

** jmxremote.password (top directory)
monitorRole adminadmin
controlRole adminadmin

** jmxremote.password (resources directory)
*** empty file. just try `touch src\main\resources\jmxremote.password`

** pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>debugcontrol</groupId>
    <artifactId>debugcontrol</artifactId>
    <version>1.0-SNAPSHOT</version>

</project>

** DebugController.java
package debugcontrol;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.AclEntry;
import java.nio.file.attribute.AclEntryPermission;
import java.nio.file.attribute.AclEntryType;
import java.nio.file.attribute.AclFileAttributeView;
import java.nio.file.attribute.UserPrincipal;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;

/**
 * RMI connection timeout test. This program starts a simple sleep server
 * program (JMXSSLServer) on external jdb process with a breakpoint at
 * sun.security.ssl.ServerHandshaker.clientHello set. It then starts a client
 * process (JMXSSLCient) which tries to connect the sleep/jdb process.
 * ServerHandshaker.clientHello responds to the client hello message and sends
 * SSL record back. By setting breakpont in that function, we can emulate the
 * error mode in which client keeps waiting SSL record from server.
 *
 * JMXConnectorFactory.connect() ignores sun.rmi.transport.tcp.responseTimeout,
 * so wait the response from server infinitely. Once a fixes was added, then
 * the client return "0" when the connection timeout happen.
 * This DebugControlTest returns the client's return code.
 */
public class DebugController {

    public static final String PROP_FILE_NAME = "debugcontroltest.properties";
    private static Properties dctProp = getDctProp();

    private static final String HOST =
dctProp.getProperty("debugcontroltest.host", "localhost");
    private static final String PORT =
dctProp.getProperty("debugcontroltest.port", "9876");
    private static final int STOP_TIME =
Integer.parseInt(dctProp.getProperty("debugcontroltest.stop_time",
"60000"));

    private static FileSystem fs = FileSystems.getDefault();
    private static Path jmxremotePasswordPath
            = fs.getPath(getResourceDirString(),

dctProp.getProperty("debugcontroltest.jmxremote.password.filename",
"jmxremote.password"));
    private static Path certPath = fs.getPath(getResourceDirString(),
            dctProp.getProperty("debugcontroltest.cert.filename",
"jmx-test-cert.pkcs12"));
    private static String keyStoreType =
dctProp.getProperty("debugcontroltest.cert.type");
    private static String keyStorePass =
dctProp.getProperty("debugcontroltest.cert.pass");

    private static String responseTimeout =
dctProp.getProperty("sun.rmi.transport.tcp.responseTimeout", "100");
    private static String handshakeTimeout =
dctProp.getProperty("sun.rmi.transport.tcp.handshakeTimeout", "100");

    public static void main(String[] args) throws Exception {
        runJMXServerWithDebugger();
        runJMXClientOnSeparateProcess();
    }

    /**
     * Load properties from PROP_FILE_NAME and returns Properties instance.
     * If PROP_FILE_NAME was not found, return an empty Properties instance.
     *
     * @return Properties instance
     */
    static Properties getDctProp() {
        File f = new File(PROP_FILE_NAME);
        Properties props = new Properties();
        if (!f.exists()) {
            return props;
        }

        FileReader fr = null;
        try {
            fr = new FileReader(f);
            props.load(fr);
            fr.close();
        } catch (FileNotFoundException fnfe) {
        } catch (IOException ioe) {
            System.err.println("[WARN] " + ioe.toString());
        }

        return props;
    }

    /**
     * Prepare to run jdb process.
     *
     */
    static void runJMXServerWithDebugger() throws FileNotFoundException {
        adjustRemotePasswordPermission();
        final Path jdbPath = getJdbPath();
        if (jdbPath == null) {
            throw new FileNotFoundException("jdb executable was not
found. Check JDK_HOME, JAVA_HOME or TESTJAVA");
        }
        new Thread("jdb-run-thread") {
            public void run() {
                runJMXServerWithDebuggerBody(jdbPath);
            }
        }.start();
    }

    /**
     * Run server.JMXSSLServer with jdb process.
     *
     * @param jdbPath
     */
    static void runJMXServerWithDebuggerBody(Path jdbPath) {
        final String[] args = new String[]{
            jdbPath.toString(),
            "-classpath", getTargetClassPath(),
            "-J-Duser.language=en",
            "-Dcom.sun.management.jmxremote.port=" + PORT,
            "-Dcom.sun.management.jmxremote.password.file=" +
jmxremotePasswordPath.toString(),
            "-Djavax.net.ssl.keyStore=" + certPath.toString(),
            "-Djavax.net.ssl.keyStoreType=" + keyStoreType,
            "-Djavax.net.ssl.keyStorePassword=" + keyStorePass,
            "debugcontrol.server.JMXSSLServer"
        };
        System.out.println("[INFO] Server process args");
        for (int i = 0; i < args.length; i++) {
            System.out.println(" args[" + i + "] " + args[i]);
        }
        ProcessBuilder pb = new ProcessBuilder(args);
        try {
            Process proc = pb.start();
            OutputStream os = proc.getOutputStream();
            PrintWriter pw = new PrintWriter(os);
            InputStream is = proc.getInputStream();
            final BufferedReader br = new BufferedReader(new
InputStreamReader(is));
            InputStream es = proc.getErrorStream();
            final BufferedReader bre = new BufferedReader(new
InputStreamReader(es));

            new Thread("server-stdout") {
                public void run() {
                    String s = null;
                    try {
                        while ((s = br.readLine()) != null) {
                            System.out.println("ser-out: " + s);
                        }
                    } catch (IOException ioe) {
                        ioe.printStackTrace();
                    }
                }
            }.start();

            new Thread("server-stderr") {
                public void run() {
                    String s = null;
                    try {
                        while ((s = br.readLine()) != null) {
                            System.out.println("ser-err: " + s);
                        }
                    } catch (IOException ioe) {
                        ioe.printStackTrace();
                    }
                }
            }.start();

            // jdb commands
            String[] commands = new String[]{
                "stop in sun.security.ssl.ServerHandshaker.clientHello",
                "run"
            };
            for (int i = 0; i < commands.length; i++) {
                pw.println(commands[i]);
                pw.flush();
                try {
                    Thread.sleep(2000);
                } catch (InterruptedException ie) {
                }
            }

            try {
                Thread.sleep(STOP_TIME);
            } catch (InterruptedException ie) { }
            System.out.println("[INFO] sending quit to jdb");
            pw.println("quit");
            pw.flush();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * Run client.JMXSSLClient on separate process.
     *
     */
    static void runJMXClientOnSeparateProcess() throws FileNotFoundException {
        // Wait 10 sec to launch server.
        try {
            Thread.sleep(10 * 1000L);
        } catch (InterruptedException ie) {  }

        Path javaPath = getJavaPath();
        if (javaPath == null) {
            throw new FileNotFoundException("java executable was not
found. Check JDK_HOME, JAVA_HOME or TESTJAVA");
        }

        final String[] args = new String[]{
            javaPath.toString(),
            "-classpath", getTargetClassPath(),
            "-Duser.language=en",
            "-Djavax.net.ssl.trustStore=" + certPath.toString(),
            "-Djavax.net.ssl.trustStoreType=" + keyStoreType,
            "-Djavax.net.ssl.trustStorePassword=" + keyStorePass,
            "-Dsun.rmi.transport.tcp.responseTimeout=" + responseTimeout,
            "-Dsun.rmi.transport.tcp.handshakeTimeout=" + handshakeTimeout,
            "debugcontrol.client.JMXSSLClient",
            HOST,PORT
        };
        System.out.println("[INFO] Client process args:");
        for (int i = 0; i < args.length; i++) {
            System.out.println(" args[" + i + "] " + args[i]);
        }
        ProcessBuilder pb = new ProcessBuilder(args);
        try {
            Process proc = pb.start();
            final InputStream is = proc.getInputStream();
            final BufferedReader br = new BufferedReader(new
InputStreamReader(is));
            final InputStream es = proc.getErrorStream();
            final BufferedReader bre = new BufferedReader(new
InputStreamReader(es));
            long t0 = System.currentTimeMillis();
            new Thread("client-stdout") {
                public void run() {
                    String s = null;
                    try {
                        while ((s = br.readLine()) != null) {
                            System.out.println("cli-out: " + s);
                        }
                    } catch (IOException ioe) {
                        ioe.printStackTrace();
                    }
                }
            }.start();

            new Thread("client-stderr") {
                public void run() {
                    String s = null;
                    try {
                        while ((s = bre.readLine()) != null) {
                            System.out.println("cli-err: " + s);
                        }
                    } catch (IOException ioe) {
                        ioe.printStackTrace();
                    }
                }
            }.start();

            int rc = proc.waitFor();
            long t1 = System.currentTimeMillis();
            System.out.println("[INFO] "+ (new Date()).toString() + "
Client done.  Result code: " + rc);
            System.out.println("[INFO] Client took " + (t1 - t0) + " msec.");
            if (System.getenv("DEBUG_CONTROL_TEST_STAY") != null) {
                System.out.println("[INFO] Press return to exit.");
                BufferedReader br1 = new BufferedReader(new
InputStreamReader(System.in));
                br1.readLine();
            }
            // exit with client's return code.
            System.exit(rc);
        } catch (IOException ioe) {
            ioe.printStackTrace();
        } catch (InterruptedException ie) {
            ie.printStackTrace();
        }
    }

    /**
     * Set jmxremote.password to readable/writable only to owner. JMX server
     * side does not start if access to the file is not limited to owner.
     * setReadable/setWritable calls set file permission to 0600 on u*ix system.
     * For windows, the calls have not effect. Need to set ACL using
     * java.nio.file API Files.setFileAttributeView(Path,
     * AclFileAttributeView.class).
     */
    static void adjustRemotePasswordPermission() {
        if (onWindows()) {
            limitAclToOwnerRead(jmxremotePasswordPath);
        } else {
            File f = jmxremotePasswordPath.toFile();
            f.setReadable(false, false); // chmod a-r
            f.setReadable(true, true);    // chmod u+r
            f.setWritable(false, false); // chmod a-w
            f.setWritable(true, true);    // chmod u+w
        }
    }

    static Path getJavaPath() {
        String jdbBasename = "java";
        if (isOnWindows()) {
            jdbBasename = "java.exe";
        }
        String jdkhome = System.getenv("JDK_HOME");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        jdkhome = System.getenv("JAVA_HOME");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        // http://openjdk.java.net/jtreg/tag-spec.html
        jdkhome = System.getenv("TESTJAVA");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        // try java.home property
        jdkhome = System.getProperty("java.home");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        return null;
    }

    /**
     * Search jdb around the directory set in JDK_HOME, JAVA_HOME or TESTJAVA if
     * set and return as Path instance if found. Otherwise, return null.
     *
     * @return jdbPath
     */
    static Path getJdbPath() {
        String jdbBasename = "jdb";
        if (isOnWindows()) {
            jdbBasename = "jdb.exe";
        }
        String jdkhome = System.getenv("JDK_HOME");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        jdkhome = System.getenv("JAVA_HOME");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        // http://openjdk.java.net/jtreg/tag-spec.html
        jdkhome = System.getenv("TESTJAVA");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        // try java.home property
        jdkhome = System.getProperty("java.home");
        if (jdkhome != null) {
            Path jdkPath = getFileAroundJdkHome(jdkhome, jdbBasename);
            if (jdkPath != null) {
                return jdkPath;
            }
        }
        return null;
    }

    /**
     * Find and return filename around home which might be jre or jdk home
     *
     * @param home jdk home or jre home
     * @param filename to find
     * @return
     */
    static Path getFileAroundJdkHome(String home, String filename) {
        // try as jdk home
        Path fpath = fs.getPath(home, "bin", filename);
        if (Files.exists(fpath)) {
            return fpath;
        }
        // then try as jre dir under jdk
        fpath = fs.getPath(home).getParent().resolve("bin").resolve(filename);
        if (Files.exists(fpath)) {
            return fpath;
        }
        return null;
    }

    /**
     * Returns true if os.name property starts with windows ignoring case.
     *
     * @return true if os.name starts with windows
     */
    static boolean isOnWindows() {
        String osname = System.getProperty("os.name");
        if (osname.toLowerCase().trim().toLowerCase().startsWith("windows")) {
            return true;
        } else {
            return false;
        }
    }

    static boolean onWindows() {
        String osName = System.getProperty("os.name", "generic").toLowerCase();
        return osName.indexOf("windows") >= 0;
    }

    static String getResourceDirString() {
        if (System.getenv("TESTSRC") != null) {
            return System.getenv("TESTSRC");
        }
        return "";
    }

    static String getTargetClassPath() {
        if (System.getenv("TESTCLASSES") != null) {
            return System.getenv("TESTCLASSES");
        }
        return "target/classes";
    }

    /**
     * Perform acl change equivalent to "cacls path /P <user.name>:R.
     *
     * @param path
     */
    static void limitAclToOwnerRead(Path path) {
        FileSystem fs = FileSystems.getDefault();
        String userName = System.getProperty("user.name");
        try {
            UserPrincipal me =
fs.getUserPrincipalLookupService().lookupPrincipalByName(userName);
            AclEntry entry = AclEntry.newBuilder()
                    .setType(AclEntryType.ALLOW)
                    .setPrincipal(me)
                    .setPermissions(AclEntryPermission.READ_DATA,
                            AclEntryPermission.READ_ATTRIBUTES,
                            AclEntryPermission.READ_NAMED_ATTRS,
                            AclEntryPermission.READ_ACL)
                    .build();
            AclFileAttributeView view =
Files.getFileAttributeView(path, AclFileAttributeView.class);
            List<AclEntry> owerReadOnlyAcl = Arrays.asList(entry);
            view.setAcl(owerReadOnlyAcl);
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }

}

** JMXSSLClient.java
package debugcontrol.client;

import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import javax.management.MBeanServerConnection;
import javax.management.ObjectInstance;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
import javax.rmi.ssl.SslRMIClientSocketFactory;
import java.net.MalformedURLException;
import java.io.IOException;
import java.net.SocketTimeoutException;

/**
 * Connect JMX server at HOST:PORT with SSL, password authetication.
 * If the connection was made, then run simple remote JMX operations.
 * This JMXSSLClient returns 0 when a connection timeout happen for test.
 */
public class JMXSSLClient {

    private static String HOST = "localhost";
    private static int PORT = 9876;

    public static void main(String[] args) throws Exception {
        if (args.length == 2) {
            HOST = args[0];
            PORT = Integer.parseInt(args[1]);
        }
        execute();
    }

    static void execute() {
        try {
            Thread.sleep(5000);
        } catch (InterruptedException ie) { }

        try {
            JMXServiceURL url = new
JMXServiceURL(String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi",
HOST, PORT));
            System.out.println("[INFO] Service URL: " + url.toString());
            String[] credentials = new String[]{"controlRole", "adminadmin"};
            SslRMIClientSocketFactory csf = new SslRMIClientSocketFactory();
            Map<String, Object> env = new HashMap<String, Object>();
            env.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE,
csf);
            env.put("jmx.remote.credentials", credentials);

            JMXConnector jmxConnector = JMXConnectorFactory.connect(url, env);
            MBeanServerConnection mbeanServerConnection =
jmxConnector.getMBeanServerConnection();

            String[] domains = mbeanServerConnection.getDomains();
            System.out.print("Domains:");
            for (int i = 0; i < domains.length; i++) {
                System.out.print(" " + domains[i]);
            }

            mbeanList(mbeanServerConnection);
            System.out.println("[INFO] Client done.");
            System.exit(0);
        } catch (MalformedURLException me) {
            me.printStackTrace();
            System.exit(1);
        } catch (java.rmi.RemoteException re) {
            // connection refused
            if (re.getCause() instanceof SocketTimeoutException) {
                System.out.println("[INFO] Conglaturation. We got a timeout.");
                System.exit(0);
            }
            re.printStackTrace();
            System.exit(2);
        } catch (IOException e) {
            e.printStackTrace();
            System.exit(3);
        }
    }

    /**
     * Simple mbean server connection operation.
     */
    static void mbeanList(MBeanServerConnection conn) {
        try {
            Set<ObjectInstance> mbeans = conn.queryMBeans(null, null);
            for (ObjectInstance oi : mbeans) {
                System.out.println(String.format("name=%s,class=%s",
oi.getObjectName(), oi.getClassName()));
            }
        } catch (IOException ioe) {
            ioe.printStackTrace();
        }
    }

    static String getResourceDirString() {
        if (System.getenv("TESTSRC") != null) {
            return System.getenv("TESTSRC");
        }
        return (String) System.getProperty("user.dir");
    }
}

** JMXSSLServer.java
package debugcontrol.server;

/**
 * Simple sleep server.
 * @author KUBOTA Yuji
 */
public class JMXSSLServer {
    public static void main(String[] args) {
        System.out.println("[INFO] Server launched then sleep...");
        try {
            while (true) { Thread.sleep(1000 * 1000L); }
        } catch (InterruptedException ie) { }
    }
}