A simple TLS connections case uses a SunX509 trust manager, which contains a RSA and a DSA certificates, and specifies cipher suite TLS_RSA_WITH_AES_256_GCM_SHA384 to make sure the RSA certificate is selected.
But the trust manager may use DSA algorithm to validate the RSA certificate.

Please use the attached test to reproduce this issue. The followings are some debug logs:
...
ssl: Ignoring alias cert-dsa: key algorithm does not match
ssl: KeyMgr: choosing key: cert-rsa (verified: OK)
Standard ciphersuite chosen: TLS_RSA_WITH_AES_256_GCM_SHA384
...
%% Invalidated: [Session-3, TLS_RSA_WITH_AES_256_GCM_SHA384]
Thread-0, SEND TLSv1.2 ALERT: fatal, description = certificate_unknown
Thread-0, WRITE: TLSv1.2 Alert, length = 2
main, READ: TLSv1.2 Alert, length = 2
main, RECV TLSv1.2 ALERT: fatal, certificate_unknown
%% Invalidated: [Session-2, TLS_RSA_WITH_AES_256_GCM_SHA384]
Thread-0, called closeSocket()
main, called closeSocket()
main, handling exception: javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown
main, called close()
main, called closeInternal(true)
main, called closeSocket()
Server failed: javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown
Thread-0, handling exception: javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: Certificate signature validation failed
javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown
at java.base/sun.security.ssl.Alerts.getSSLException(Alerts.java:198)
at java.base/sun.security.ssl.Alerts.getSSLException(Alerts.java:159)
at java.base/sun.security.ssl.SSLSocketImpl.recvAlert(SSLSocketImpl.java:2046)
at java.base/sun.security.ssl.SSLSocketImpl.processInputRecord(SSLSocketImpl.java:1207)
at java.base/sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1074)
at java.base/sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
at java.base/sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1402)
at java.base/sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(SSLSocketImpl.java:907)
at java.base/sun.security.ssl.AppInputStream.read(AppInputStream.java:144)
at java.base/sun.security.ssl.AppInputStream.read(AppInputStream.java:84)
at SSLSocketTemplate.runServerApplication(SSLSocketTemplate.java:94)
at SSLSocketTemplate.doServerSide(SSLSocketTemplate.java:270)
at SSLSocketTemplate.startServer(SSLSocketTemplate.java:755)
at SSLSocketTemplate.bootup(SSLSocketTemplate.java:669)
at SSLSocketTemplate.run(SSLSocketTemplate.java:83)
at SSLSocketTemplate.main(SSLSocketTemplate.java:76)
Thread-0, called close()
Thread-0, called closeInternal(true)
Thread-0, called closeSocket()
Client died: javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: Certificate signature validation failed
javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: Certificate signature validation failed
at java.base/sun.security.ssl.Alerts.getSSLException(Alerts.java:198)
at java.base/sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1974)
at java.base/sun.security.ssl.Handshaker.fatalSE(Handshaker.java:345)
at java.base/sun.security.ssl.Handshaker.fatalSE(Handshaker.java:339)
at java.base/sun.security.ssl.ClientHandshaker.checkServerCerts(ClientHandshaker.java:1968)
at java.base/sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1777)
at java.base/sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:264)
at java.base/sun.security.ssl.Handshaker.processLoop(Handshaker.java:1098)
at java.base/sun.security.ssl.Handshaker.processRecord(Handshaker.java:1026)
at java.base/sun.security.ssl.SSLSocketImpl.processInputRecord(SSLSocketImpl.java:1137)
at java.base/sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1074)
at java.base/sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
at java.base/sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1402)
at java.base/sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:733)
at java.base/sun.security.ssl.AppOutputStream.write(AppOutputStream.java:67)
at java.base/sun.security.ssl.AppOutputStream.write(AppOutputStream.java:81)
at SSLSocketTemplate.runClientApplication(SSLSocketTemplate.java:112)
at SSLSocketTemplate.doClientSide(SSLSocketTemplate.java:345)
at SSLSocketTemplate.access$3(SSLSocketTemplate.java:289)
at SSLSocketTemplate$2.run(SSLSocketTemplate.java:769)
Caused by: sun.security.validator.ValidatorException: Certificate signature validation failed
at java.base/sun.security.validator.SimpleValidator.engineValidate(SimpleValidator.java:216)
at java.base/sun.security.validator.Validator.validate(Validator.java:264)
at java.base/sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:343)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:226)
at java.base/sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:133)
at java.base/sun.security.ssl.ClientHandshaker.checkServerCerts(ClientHandshaker.java:1947)
... 15 more
Caused by: java.security.InvalidKeyException: No installed provider supports this key: sun.security.provider.DSAPublicKeyImpl
at java.base/java.security.Signature$Delegate.chooseProvider(Signature.java:1161)
at java.base/java.security.Signature$Delegate.engineInitVerify(Signature.java:1193)
at java.base/java.security.Signature.initVerify(Signature.java:475)
at java.base/sun.security.x509.X509CertImpl.verify(X509CertImpl.java:438)
at java.base/sun.security.x509.X509CertImpl.verify(X509CertImpl.java:392)
at java.base/sun.security.validator.SimpleValidator.engineValidate(SimpleValidator.java:214)
... 20 more