ADDITIONAL SYSTEM INFORMATION :
Mac OS 10.11, but also CentOS 7.
java version "11.0.1" 2018-10-16 LTS
Java(TM) SE Runtime Environment 18.9 (build 11.0.1+13-LTS)
Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.1+13-LTS, mixed mode)

A DESCRIPTION OF THE PROBLEM :
Using NSS or SoftHSM as PKCS11 module containing an EC server key, client cannot connect with any protocol.



REGRESSION : Last worked in version 8u192

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
I used a modified version of HTTPSServer from https://www.pixelstech.net/article/1445603357-A-HTTPS-client-and-HTTPS-server-demo-in-Java, to use PKCS11 instead of JKS.

run the server with
java -Djava.security.properties=file:////tmp/java-nss.security -Djava.security.debug=sunpkcs11 -cp . TLSServer

contents of java-nss.security:
security.provider.14=SunPKCS11 /Users/fabiop/src/tmp/nss.conf

contents of nss.conf
name = NSScrypto
nssLibraryDirectory = /tmp/nss-3.41/dist/Darwin15.6.0_cc_64_OPT.OBJ/lib
nssSecmodDirectory = sql:/tmp/nss-3.41/dist/Darwin15.6.0_cc_64_OPT.OBJ


EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
Connection is established successfully
ACTUAL -
Server throws:

java.lang.ClassCastException: class sun.security.pkcs11.P11Key$P11PrivateKey cannot be cast to class java.security.interfaces.ECPrivateKey (sun.security.pkcs11.P11Key$P11PrivateKey is in module jdk.crypto.cryptoki of loader 'platform'; java.security.interfaces.ECPrivateKey is in module java.base of loader 'bootstrap')
at java.base/sun.security.ssl.SignatureScheme.getPreferableAlgorithm(SignatureScheme.java:436)
at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyMessage.<init>(CertificateVerify.java:867)
at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyProducer.onProduceCertificateVerify(CertificateVerify.java:1077)
at java.base/sun.security.ssl.CertificateVerify$T13CertificateVerifyProducer.produce(CertificateVerify.java:1070)
at java.base/sun.security.ssl.SSLHandshake.produce(SSLHandshake.java:436)
at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.goServerHello(ClientHello.java:1224)
at java.base/sun.security.ssl.ClientHello$T13ClientHelloConsumer.consume(ClientHello.java:1160)
at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.onClientHello(ClientHello.java:849)
at java.base/sun.security.ssl.ClientHello$ClientHelloConsumer.consume(ClientHello.java:810)

---------- BEGIN SOURCE ----------
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.KeyStore;

import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;

class TLSServer {
    private int port = 8080;
    private boolean isServerDone = false;

    public static void main(String[] args){
        TLSServer server = new TLSServer();
        server.run();
    }

    TLSServer(){
    }

    // Create the and initialize the SSLContext
    private SSLContext createSSLContext(){
        try{
            KeyStore keyStore = KeyStore.getInstance("PKCS11");
            keyStore.load(null,"password".toCharArray());
            System.out.println("Keystore provider : " + keyStore.getProvider());
            // Create key manager
            KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("SunX509");
            keyManagerFactory.init(keyStore, "password".toCharArray());
            KeyManager[] km = keyManagerFactory.getKeyManagers();

            // Create trust manager
            TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("SunX509");
            trustManagerFactory.init(keyStore);
            TrustManager[] tm = trustManagerFactory.getTrustManagers();

            // Initialize SSLContext
            SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
            sslContext.init(km, tm, null);

            return sslContext;
        } catch (Exception ex){
            ex.printStackTrace();
        }

        return null;
    }

    // Start to run the server
    public void run(){
        SSLContext sslContext = this.createSSLContext();

        try{
            // Create server socket factory
            SSLServerSocketFactory sslServerSocketFactory = sslContext.getServerSocketFactory();

            // Create server socket
            SSLServerSocket sslServerSocket = (SSLServerSocket) sslServerSocketFactory.createServerSocket(this.port);

            System.out.println("SSL server started");
            while(!isServerDone){
                SSLSocket sslSocket = (SSLSocket) sslServerSocket.accept();

                // Start the server thread
                new ServerThread(sslSocket).start();
            }
        } catch (Exception ex){
            ex.printStackTrace();
        }
    }

    // Thread handling the socket from client
    static class ServerThread extends Thread {
        private SSLSocket sslSocket = null;

        ServerThread(SSLSocket sslSocket){
            this.sslSocket = sslSocket;
        }

        public void run(){
            sslSocket.setEnabledCipherSuites(sslSocket.getSupportedCipherSuites());

            try{
                // Start handshake
                sslSocket.startHandshake();

                // Get session after the connection is established
                SSLSession sslSession = sslSocket.getSession();

                System.out.println("SSLSession :");
                System.out.println("\tProtocol : "+sslSession.getProtocol());
                System.out.println("\tCipher suite : "+sslSession.getCipherSuite());

                // Start handling application content
                InputStream inputStream = sslSocket.getInputStream();

                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

                String line = null;
                while((line = bufferedReader.readLine()) != null){
                    System.out.println("Input : "+line);

                    if(line.trim().isEmpty()){
                        break;
                    }
                }

                sslSocket.close();
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
    }
}
---------- END SOURCE ----------

FREQUENCY : always