A DESCRIPTION OF THE PROBLEM :
Client authentication in SSLServerSocket is not working, when configured with endpoint identification algorithm "HTTPS" for certificates issued to a hostname without an IP address as SAN.
There is a VM option (jdk.tls.trustNameService ) provided to explicitly trust the name service, but it is only effective if the socket is in client mode (see sun.security.ssl.SSLSocketImpl::doneConnect line 1208).

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
- set VM option jdk.tls.trustNameService to true
- create a SSLServerSocket
- activate client authentication on SSLServerSocket (setNeedClientAuth(true))
- set endpoint identification algorithm to HTTPS
- connect to the SSLServerSocket with a certificate issued to your hostname

EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
Connection is established
ACTUAL -
Connection is refused with following exception:
java.security.cert.CertificateException: No subject alternative names present
at java.base/sun.security.util.HostnameChecker.matchIP(HostnameChecker.java:137)
at java.base/sun.security.util.HostnameChecker.match(HostnameChecker.java:96)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkIdentity(X509TrustManagerImpl.java:459)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkIdentity(X509TrustManagerImpl.java:434)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:233)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkClientTrusted(X509TrustManagerImpl.java:123)
at java.base/sun.security.ssl.CertificateMessage$T12CertificateConsumer.checkClientCerts(CertificateMessage.java:676)
... 12 more

---------- BEGIN SOURCE ----------
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * This Test tries do demonstrate that Client Authentication with Endpoint Identification is not working in SSLServerSocket.<br/>
 * <br/>
 * There are some requirements to the test environment:
 * <ul>
 * <li>A name service must be present in your network.</li>
 * <li>The name service must have an A- and PTR-record for your machine.</li>
 * <li>Key- and trust-stores for your machine must be created and added to the resources.</li>
 * </ul>
 * <br/>
 * The certificates are created as follows (replace 'hostname' with your machine's actual hostname):
 * <pre>
 * keytool -genkeypair -alias server -dname "CN=hostname" -ext EKU:false=serverAuth -keystore server_keys.pfx -storetype PKCS12 -storepass certtest
 * keytool -genkeypair -alias client -dname "CN=hostname" -ext EKU:false=clientAuth -keystore client_keys.pfx -storetype PKCS12 -storepass certtest
 * keytool -exportcert -keystore server_keys.pfx -storepass certtest -file server.crt -alias server
 * keytool -exportcert -keystore client_keys.pfx -storepass certtest -file client.crt -alias client
 * keytool -noprompt -importcert -trustcacerts -file client.crt -alias client -keystore server_trust.pfx -storetype PKCS12 -storepass certtest
 * keytool -noprompt -importcert -trustcacerts -file server.crt -alias server -keystore client_trust.pfx -storetype PKCS12 -storepass certtest
 * </pre>
 */
public class Test {

    private static final int PORT = 1111;
    private static final String KEY_MANAGER_PW = "certtest";

    public static void main(String[] args) throws GeneralSecurityException, IOException {
        // define the name service as trustworthy for endpoint identification
        System.setProperty("jdk.tls.trustNameService", "true");

        // get the hostname of the current machine
        String myHostName = InetAddress.getLocalHost().getHostName();

        // create server socket, that listens for incoming connections and starts a handshake
        SSLServerSocket serverSocket = createServerSocket();
        Thread acceptThread = new Thread(() -> {
            boolean running = true;
            while (running) {
                try {
                    Socket socket = serverSocket.accept();
                    if (socket instanceof SSLSocket) {
                        ((SSLSocket) socket).startHandshake();
                        Logger.getLogger(Test.class.getName()).log(Level.INFO, "Handshake successful.");
                        running = false;
                    }
                } catch (SSLHandshakeException e) {
                    Logger.getLogger(Test.class.getName()).log(Level.INFO, "Handshake failed.", e);
                    running = false;
                } catch (IOException e) {
                    Logger.getLogger(Test.class.getName()).log(Level.INFO, "Some different problem occurred.", e);
                    running = false;
                }
            }
        });
        acceptThread.start();

        // connect to server socket and start the handshake
        SSLSocket socket = createSocket(myHostName);
        socket.startHandshake();
    }

    private static SSLServerSocket createServerSocket() throws IOException, GeneralSecurityException {
        final File keyStore = new File(Test.class.getResource("server_keys.pfx").getFile());
        final File trustStore = new File(Test.class.getResource("server_trust.pfx").getFile());
        final SSLContext sslContext = createSSLContext(keyStore, trustStore);

        final SSLServerSocket serverSocket = (SSLServerSocket) sslContext.getServerSocketFactory().createServerSocket(PORT);

        // enable client auth with endpoint identification via hostname
        SSLParameters sslParams = new SSLParameters();
        sslParams.setEndpointIdentificationAlgorithm("HTTPS");
        serverSocket.setSSLParameters(sslParams);
        serverSocket.setNeedClientAuth(true);

        return serverSocket;
    }

    private static SSLSocket createSocket(String hostname) throws IOException, GeneralSecurityException {
        final File keyStore = new File(Test.class.getResource("client_keys.pfx").getFile());
        final File trustStore = new File(Test.class.getResource("client_trust.pfx").getFile());
        final SSLContext sslContext = createSSLContext(keyStore, trustStore);

        final SSLSocket socket = (SSLSocket) sslContext.getSocketFactory().createSocket(hostname, PORT);

        // enable endpoint identification via hostname
        SSLParameters sslParams = new SSLParameters();
        sslParams.setEndpointIdentificationAlgorithm("HTTPS");
        socket.setSSLParameters(sslParams);

        return socket;
    }

    private static SSLContext createSSLContext(File keyManagerKS, File trustManagerKS)
            throws GeneralSecurityException, IOException {
        // create Key Manager
        KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
        KeyStore ksKeys = KeyStore.getInstance("pkcs12");
        ksKeys.load(new FileInputStream(keyManagerKS), KEY_MANAGER_PW.toCharArray());
        keyManagerFactory.init(ksKeys, KEY_MANAGER_PW.toCharArray());
        final KeyManager[] keyManagers = keyManagerFactory.getKeyManagers();

        // create Trust Manager
        TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
        KeyStore ksTrust = KeyStore.getInstance("pkcs12");
        ksTrust.load(new FileInputStream(trustManagerKS), KEY_MANAGER_PW.toCharArray());
        trustManagerFactory.init(ksTrust);
        final TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();

        // create SSL Context
        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
        sslContext.init(keyManagers, trustManagers, null);
        return sslContext;
    }
}

---------- END SOURCE ----------

FREQUENCY : always