-
Enhancement
-
Resolution: Not an Issue
-
P4
-
None
-
11
-
generic
-
generic
A DESCRIPTION OF THE PROBLEM :
Connecting to a remote kafka service fails during SSL Handshake, due to expired certificates.
For 11, the returned exception provides no helpful information to mitigate the problem, only an obscure Unexpected server message: server_hello.
javax.net.ssl.SSLProtocolException: Unexpected handshake message: server_hello
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:129)
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:340)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:296)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:287)
at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:437)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1074)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1061)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1008)
at org.apache.kafka.common.network.SslTransportLayer.runDelegatedTasks(SslTransportLayer.java:432)
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:516)
at org.apache.kafka.common.network.SslTransportLayer.doHandshake(SslTransportLayer.java:370)
at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:293)
at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:178)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543)
at org.apache.kafka.common.network.Selector.poll(Selector.java:481)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:265)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:215)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:246)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnready(ConsumerCoordinator.java:459)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:487)
at org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1262)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1231)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1211)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollConsumer(KafkaMessageListenerContainer.java:1522)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doPoll(KafkaMessageListenerContainer.java:1512)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1340)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1252)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:829)}
)
After a lot of debugging, we decided to downgrade to Java 8 as we have some services running with the same config.
The 8's version of the exception provided useful information, for us to be able to mitigate the problem.
org.apache.kafka.common.errors.SslAuthenticationException: SSL handshake failed
Caused by: javax.net.ssl.SSLProtocolException: Handshake message sequence violation, 2
at sun.security.ssl.Handshaker.checkThrown(Handshaker.java:1530) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.checkTaskThrown(SSLEngineImpl.java:528) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:802) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:766) ~[na:1.8.0_232]
at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624) ~[na:1.8.0_232]
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:511) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.doHandshake(SslTransportLayer.java:370) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:293) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:178) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.Selector.poll(Selector.java:481) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:265) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:215) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:246) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnready(ConsumerCoordinator.java:459) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:487) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1262) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1231) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1211) ~[kafka-clients-3.1.1.jar!/:na]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollConsumer(KafkaMessageListenerContainer.java:1522) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doPoll(KafkaMessageListenerContainer.java:1512) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1340) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1252) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_232]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_232]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_232]
Caused by: javax.net.ssl.SSLProtocolException: Handshake message sequence violation, 2
at sun.security.ssl.HandshakeStateManager.check(HandshakeStateManager.java:362) ~[na:1.8.0_232]
at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:196) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1037) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:970) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:967) ~[na:1.8.0_232]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1459) ~[na:1.8.0_232]
at org.apache.kafka.common.network.SslTransportLayer.runDelegatedTasks(SslTransportLayer.java:432) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:516) ~[kafka-clients-3.1.1.jar!/:na]
... 22 common frames omitted
Connecting to a remote kafka service fails during SSL Handshake, due to expired certificates.
For 11, the returned exception provides no helpful information to mitigate the problem, only an obscure Unexpected server message: server_hello.
javax.net.ssl.SSLProtocolException: Unexpected handshake message: server_hello
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:129)
at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:340)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:296)
at java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:287)
at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:437)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1074)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1061)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1008)
at org.apache.kafka.common.network.SslTransportLayer.runDelegatedTasks(SslTransportLayer.java:432)
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:516)
at org.apache.kafka.common.network.SslTransportLayer.doHandshake(SslTransportLayer.java:370)
at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:293)
at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:178)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543)
at org.apache.kafka.common.network.Selector.poll(Selector.java:481)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:265)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:215)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:246)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnready(ConsumerCoordinator.java:459)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:487)
at org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1262)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1231)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1211)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollConsumer(KafkaMessageListenerContainer.java:1522)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doPoll(KafkaMessageListenerContainer.java:1512)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1340)
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1252)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:829)}
)
After a lot of debugging, we decided to downgrade to Java 8 as we have some services running with the same config.
The 8's version of the exception provided useful information, for us to be able to mitigate the problem.
org.apache.kafka.common.errors.SslAuthenticationException: SSL handshake failed
Caused by: javax.net.ssl.SSLProtocolException: Handshake message sequence violation, 2
at sun.security.ssl.Handshaker.checkThrown(Handshaker.java:1530) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.checkTaskThrown(SSLEngineImpl.java:528) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:802) ~[na:1.8.0_232]
at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:766) ~[na:1.8.0_232]
at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624) ~[na:1.8.0_232]
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:511) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.doHandshake(SslTransportLayer.java:370) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:293) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:178) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.Selector.poll(Selector.java:481) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:265) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:215) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:246) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnready(ConsumerCoordinator.java:459) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:487) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.updateAssignmentMetadataIfNeeded(KafkaConsumer.java:1262) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1231) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1211) ~[kafka-clients-3.1.1.jar!/:na]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollConsumer(KafkaMessageListenerContainer.java:1522) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doPoll(KafkaMessageListenerContainer.java:1512) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1340) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1252) ~[spring-kafka-2.8.7.jar!/:2.8.7]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_232]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_232]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_232]
Caused by: javax.net.ssl.SSLProtocolException: Handshake message sequence violation, 2
at sun.security.ssl.HandshakeStateManager.check(HandshakeStateManager.java:362) ~[na:1.8.0_232]
at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:196) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1037) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:970) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$1.run(Handshaker.java:967) ~[na:1.8.0_232]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_232]
at sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1459) ~[na:1.8.0_232]
at org.apache.kafka.common.network.SslTransportLayer.runDelegatedTasks(SslTransportLayer.java:432) ~[kafka-clients-3.1.1.jar!/:na]
at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:516) ~[kafka-clients-3.1.1.jar!/:na]
... 22 common frames omitted