We are detecting the following deadlock when we do thread dumps (attached at the end). It has happened numerous times and seems to always involve the class loader.
The first thread varies but the second thread always has the same stack involving the sun.security package.
We are running Solaris 8 and Java 1.3.1_03
FOUND A JAVA LEVEL DEADLOCK:
----------------------------
"tcpConnection-6802-157":
waiting to lock monitor 0x12be90 (object 0x9ac0fe78, a sun.misc.Launcher$ExtClassLoader),
which is locked by "tcpConnection-6802-105"
"tcpConnection-6802-105":
waiting to lock monitor 0x12bec8 (object 0x9ac12348, a sun.misc.Launcher$AppClassLoader),
which is locked by "tcpConnection-6802-157"
JAVA STACK INFORMATION FOR THREADS LISTED ABOVE:
------------------------------------------------
Java Stack for "tcpConnection-6802-157":
==========
at java.lang.ClassLoader.loadClass(ClassLoader.java:283)
at java.lang.ClassLoader.loadClass(ClassLoader.java:287)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:281)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:310)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:115)
at oracle.jdbc.ttc7.TTC7Protocol.safelyGetClassForName(TTC7Protocol.java:3564)
at oracle.jdbc.driver.OracleConnection.safelyGetClassForName(OracleConnection.java:5059)
at oracle.jdbc.driver.OracleConnection.addClassMapEntry(OracleConnection.java:2837)
at oracle.jdbc.driver.OracleConnection.addDefaultClassMapEntriesTo(OracleConnection.java:2828)
at oracle.jdbc.driver.OracleConnection.initializeClassMap(OracleConnection.java:2514)
at oracle.jdbc.driver.OracleConnection.ensureClassMapExists(OracleConnection.java:2508)
at oracle.jdbc.driver.OracleConnection.getTypeMap(OracleConnection.java:2814)
at oracle.jdbc.driver.OracleStatement.getObjectValue(OracleStatement.java:5833)
at oracle.jdbc.driver.OracleResultSetImpl.getObject(OracleResultSetImpl.java:765)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.getObject(Unknown Source)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.fetchRow(Unknown Source)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.executeCall(Unknown Source)
at oracle.toplink.threetier.ServerSession.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.selectOneRow(Unknown Source)
at oracle.toplink.internal.queryframework.ExpressionQueryMechanism.selectOneRowFromTable(Unknown Source)
at oracle.toplink.internal.queryframework.ExpressionQueryMechanism.selectOneRow(Unknown Source)
at oracle.toplink.queryframework.ReadObjectQuery.execute(Unknown Source)
at oracle.toplink.queryframework.DatabaseQuery.execute(Unknown Source)
at oracle.toplink.queryframework.ReadQuery.execute(Unknown Source)
at oracle.toplink.publicinterface.Session.internalExecuteQuery(Unknown Source)
at oracle.toplink.threetier.ServerSession.internalExecuteQuery(Unknown Source)
at oracle.toplink.threetier.ClientSession.internalExecuteQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.executeQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.executeQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.readObject(Unknown Source)
at upoc.persistence.manager.DeviceManager.getWirelessDeviceForDeviceNumber(DeviceManager.java:176)
at upoc.operation.CreateDeviceOperation._operate(CreateDeviceOperation.java:192)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.CreateSubscriberWithDeviceOperation._operate(CreateSubscriberWithDeviceOperation.java:142)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.Operation.chainExecute(Operation.java:115)
at upoc.operation.Operation.execute(Operation.java:81)
at upoc.jsp.action.SimpleSignup.performAction(SimpleSignup.java:127)
at upoc.jsp.action.JSPAction.execute(JSPAction.java:174)
at upoc.jsp.action.JSPActionManager.handleAction(JSPActionManager.java:69)
at upoc.jsp.tag.WebPageTag._processJspPage(WebPageTag.java:303)
at upoc.jsp.tag.JspPageTag.doStartTag(JspPageTag.java:118)
at _simpleSignup__jsp._jspService(_simpleSignup__jsp.java:41)
at com.caucho.jsp.JavaPage.service(JavaPage.java:75)
at com.caucho.jsp.Page.subservice(Page.java:506)
at com.caucho.server.http.FilterChainPage.doFilter(FilterChainPage.java:182)
at upoc.servlet.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:55)
at com.caucho.server.http.FilterChainFilter.doFilter(FilterChainFilter.java:88)
at com.caucho.server.http.Invocation.service(Invocation.java:315)
at com.caucho.server.http.CacheInvocation.service(CacheInvocation.java:135)
at com.caucho.server.http.RunnerRequest.handleRequest(RunnerRequest.java:344)
at com.caucho.server.http.RunnerRequest.handleConnection(RunnerRequest.java:274)
at com.caucho.server.TcpConnection.run(TcpConnection.java:139)
at java.lang.Thread.run(Thread.java:479)
Java Stack for "tcpConnection-6802-105":
==========
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:274)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at sun.security.x509.OIDMap$1.run(OIDMap.java:254)
at java.security.AccessController.doPrivileged(Native Method)
at sun.security.x509.OIDMap.loadOidClass(OIDMap.java:251)
at sun.security.x509.OIDMap.getClass(OIDMap.java:234)
at sun.security.x509.CertificateExtensions.parseExtension(CertificateExtensions.java:83)
at sun.security.x509.CertificateExtensions.init(CertificateExtensions.java:76)
at sun.security.x509.CertificateExtensions.(CertificateExtensions.java:55)
at sun.security.x509.X509CertInfo.parse(X509CertInfo.java:714)
at sun.security.x509.X509CertInfo.(X509CertInfo.java:150)
at sun.security.x509.X509CertImpl.parse(X509CertImpl.java:1039)
at sun.security.x509.X509CertImpl.(X509CertImpl.java:144)
at sun.security.provider.X509Factory.engineGenerateCertificate(X509Factory.java:84)
at java.security.cert.CertificateFactory.generateCertificate(CertificateFactory.java:281)
at sun.security.pkcs.PKCS7.parseSignedData(PKCS7.java:252)
at sun.security.pkcs.PKCS7.parse(PKCS7.java:128)
at sun.security.pkcs.PKCS7.parse(PKCS7.java:97)
at sun.security.pkcs.PKCS7.(PKCS7.java:85)
at sun.security.util.SignatureFileVerifier.(SignatureFileVerifier.java:62)
at java.util.jar.JarVerifier.processEntry(JarVerifier.java:258)
at java.util.jar.JarVerifier.update(JarVerifier.java:192)
at java.util.jar.JarFile.initializeVerifier(JarFile.java:243)
at java.util.jar.JarFile.getInputStream(JarFile.java:305)
at sun.misc.URLClassPath$4.getInputStream(URLClassPath.java:532)
at sun.misc.Resource.getBytes(Resource.java:55)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:240)
at java.net.URLClassLoader.access$100(URLClassLoader.java:51)
at java.net.URLClassLoader$1.run(URLClassLoader.java:190)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:183)
at java.lang.ClassLoader.loadClass(ClassLoader.java:294)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:310)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.(DashoA6275)
at com.sun.net.ssl.internal.ssl.SSLSocketFactoryImpl.createSocket(DashoA6275)
at org.apache.axis.components.net.JSSESocketFactory.create(JSSESocketFactory.java:119)
at org.apache.axis.transport.http.HTTPSender.getSocket(HTTPSender.java:157)
at org.apache.axis.transport.http.HTTPSender.invoke(HTTPSender.java:114)
at org.apache.axis.strategies.InvocationStrategy.visit(InvocationStrategy.java:71)
at org.apache.axis.SimpleChain.doVisiting(SimpleChain.java:150)
at org.apache.axis.SimpleChain.invoke(SimpleChain.java:120)
at org.apache.axis.client.AxisClient.invoke(AxisClient.java:180)
at org.apache.axis.client.Call.invokeEngine(Call.java:2526)
at org.apache.axis.client.Call.invoke(Call.java:2515)
at org.apache.axis.client.Call.invoke(Call.java:2210)
at org.apache.axis.client.Call.invoke(Call.java:2133)
at org.apache.axis.client.Call.invoke(Call.java:1656)
at upoc.billing.qpass.PurchasePortSoapBindingStub.makePurchase(PurchasePortSoapBindingStub.java:200)
at upoc.billing.provider.QpassBillingHandler.preauthorizeMin(QpassBillingHandler.java:190)
at upoc.util.ServiceProviderUtil.getAttPreauthCodeForDeviceNumber(ServiceProviderUtil.java:302)
at upoc.operation.CreateDeviceOperation._operate(CreateDeviceOperation.java:211)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.CreateSubscriberWithDeviceOperation._operate(CreateSubscriberWithDeviceOperation.java:142)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.Operation.chainExecute(Operation.java:115)
at upoc.operation.Operation.execute(Operation.java:81)
at upoc.jsp.action.SimpleSignup.performAction(SimpleSignup.java:127)
at upoc.jsp.action.JSPAction.execute(JSPAction.java:174)
at upoc.jsp.action.JSPActionManager.handleAction(JSPActionManager.java:69)
at upoc.jsp.tag.WebPageTag._processJspPage(WebPageTag.java:303)
at upoc.jsp.tag.JspPageTag.doStartTag(JspPageTag.java:118)
at _simpleSignup__jsp._jspService(_simpleSignup__jsp.java:41)
at com.caucho.jsp.JavaPage.service(JavaPage.java:75)
at com.caucho.jsp.Page.subservice(Page.java:506)
at com.caucho.server.http.FilterChainPage.doFilter(FilterChainPage.java:182)
at upoc.servlet.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:55)
at com.caucho.server.http.FilterChainFilter.doFilter(FilterChainFilter.java:88)
at com.caucho.server.http.Invocation.service(Invocation.java:315)
at com.caucho.server.http.CacheInvocation.service(CacheInvocation.java:135)
at com.caucho.server.http.RunnerRequest.handleRequest(RunnerRequest.java:344)
at com.caucho.server.http.RunnerRequest.handleConnection(RunnerRequest.java:274)
at com.caucho.server.TcpConnection.run(TcpConnection.java:139)
at java.lang.Thread.run(Thread.java:479)
Found 1 deadlock.
The first thread varies but the second thread always has the same stack involving the sun.security package.
We are running Solaris 8 and Java 1.3.1_03
FOUND A JAVA LEVEL DEADLOCK:
----------------------------
"tcpConnection-6802-157":
waiting to lock monitor 0x12be90 (object 0x9ac0fe78, a sun.misc.Launcher$ExtClassLoader),
which is locked by "tcpConnection-6802-105"
"tcpConnection-6802-105":
waiting to lock monitor 0x12bec8 (object 0x9ac12348, a sun.misc.Launcher$AppClassLoader),
which is locked by "tcpConnection-6802-157"
JAVA STACK INFORMATION FOR THREADS LISTED ABOVE:
------------------------------------------------
Java Stack for "tcpConnection-6802-157":
==========
at java.lang.ClassLoader.loadClass(ClassLoader.java:283)
at java.lang.ClassLoader.loadClass(ClassLoader.java:287)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:281)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:310)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:115)
at oracle.jdbc.ttc7.TTC7Protocol.safelyGetClassForName(TTC7Protocol.java:3564)
at oracle.jdbc.driver.OracleConnection.safelyGetClassForName(OracleConnection.java:5059)
at oracle.jdbc.driver.OracleConnection.addClassMapEntry(OracleConnection.java:2837)
at oracle.jdbc.driver.OracleConnection.addDefaultClassMapEntriesTo(OracleConnection.java:2828)
at oracle.jdbc.driver.OracleConnection.initializeClassMap(OracleConnection.java:2514)
at oracle.jdbc.driver.OracleConnection.ensureClassMapExists(OracleConnection.java:2508)
at oracle.jdbc.driver.OracleConnection.getTypeMap(OracleConnection.java:2814)
at oracle.jdbc.driver.OracleStatement.getObjectValue(OracleStatement.java:5833)
at oracle.jdbc.driver.OracleResultSetImpl.getObject(OracleResultSetImpl.java:765)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.getObject(Unknown Source)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.fetchRow(Unknown Source)
at oracle.toplink.internal.databaseaccess.DatabaseAccessor.executeCall(Unknown Source)
at oracle.toplink.threetier.ServerSession.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.executeCall(Unknown Source)
at oracle.toplink.internal.queryframework.CallQueryMechanism.selectOneRow(Unknown Source)
at oracle.toplink.internal.queryframework.ExpressionQueryMechanism.selectOneRowFromTable(Unknown Source)
at oracle.toplink.internal.queryframework.ExpressionQueryMechanism.selectOneRow(Unknown Source)
at oracle.toplink.queryframework.ReadObjectQuery.execute(Unknown Source)
at oracle.toplink.queryframework.DatabaseQuery.execute(Unknown Source)
at oracle.toplink.queryframework.ReadQuery.execute(Unknown Source)
at oracle.toplink.publicinterface.Session.internalExecuteQuery(Unknown Source)
at oracle.toplink.threetier.ServerSession.internalExecuteQuery(Unknown Source)
at oracle.toplink.threetier.ClientSession.internalExecuteQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.executeQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.executeQuery(Unknown Source)
at oracle.toplink.publicinterface.Session.readObject(Unknown Source)
at upoc.persistence.manager.DeviceManager.getWirelessDeviceForDeviceNumber(DeviceManager.java:176)
at upoc.operation.CreateDeviceOperation._operate(CreateDeviceOperation.java:192)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.CreateSubscriberWithDeviceOperation._operate(CreateSubscriberWithDeviceOperation.java:142)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.Operation.chainExecute(Operation.java:115)
at upoc.operation.Operation.execute(Operation.java:81)
at upoc.jsp.action.SimpleSignup.performAction(SimpleSignup.java:127)
at upoc.jsp.action.JSPAction.execute(JSPAction.java:174)
at upoc.jsp.action.JSPActionManager.handleAction(JSPActionManager.java:69)
at upoc.jsp.tag.WebPageTag._processJspPage(WebPageTag.java:303)
at upoc.jsp.tag.JspPageTag.doStartTag(JspPageTag.java:118)
at _simpleSignup__jsp._jspService(_simpleSignup__jsp.java:41)
at com.caucho.jsp.JavaPage.service(JavaPage.java:75)
at com.caucho.jsp.Page.subservice(Page.java:506)
at com.caucho.server.http.FilterChainPage.doFilter(FilterChainPage.java:182)
at upoc.servlet.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:55)
at com.caucho.server.http.FilterChainFilter.doFilter(FilterChainFilter.java:88)
at com.caucho.server.http.Invocation.service(Invocation.java:315)
at com.caucho.server.http.CacheInvocation.service(CacheInvocation.java:135)
at com.caucho.server.http.RunnerRequest.handleRequest(RunnerRequest.java:344)
at com.caucho.server.http.RunnerRequest.handleConnection(RunnerRequest.java:274)
at com.caucho.server.TcpConnection.run(TcpConnection.java:139)
at java.lang.Thread.run(Thread.java:479)
Java Stack for "tcpConnection-6802-105":
==========
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:274)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at sun.security.x509.OIDMap$1.run(OIDMap.java:254)
at java.security.AccessController.doPrivileged(Native Method)
at sun.security.x509.OIDMap.loadOidClass(OIDMap.java:251)
at sun.security.x509.OIDMap.getClass(OIDMap.java:234)
at sun.security.x509.CertificateExtensions.parseExtension(CertificateExtensions.java:83)
at sun.security.x509.CertificateExtensions.init(CertificateExtensions.java:76)
at sun.security.x509.CertificateExtensions.(CertificateExtensions.java:55)
at sun.security.x509.X509CertInfo.parse(X509CertInfo.java:714)
at sun.security.x509.X509CertInfo.(X509CertInfo.java:150)
at sun.security.x509.X509CertImpl.parse(X509CertImpl.java:1039)
at sun.security.x509.X509CertImpl.(X509CertImpl.java:144)
at sun.security.provider.X509Factory.engineGenerateCertificate(X509Factory.java:84)
at java.security.cert.CertificateFactory.generateCertificate(CertificateFactory.java:281)
at sun.security.pkcs.PKCS7.parseSignedData(PKCS7.java:252)
at sun.security.pkcs.PKCS7.parse(PKCS7.java:128)
at sun.security.pkcs.PKCS7.parse(PKCS7.java:97)
at sun.security.pkcs.PKCS7.(PKCS7.java:85)
at sun.security.util.SignatureFileVerifier.(SignatureFileVerifier.java:62)
at java.util.jar.JarVerifier.processEntry(JarVerifier.java:258)
at java.util.jar.JarVerifier.update(JarVerifier.java:192)
at java.util.jar.JarFile.initializeVerifier(JarFile.java:243)
at java.util.jar.JarFile.getInputStream(JarFile.java:305)
at sun.misc.URLClassPath$4.getInputStream(URLClassPath.java:532)
at sun.misc.Resource.getBytes(Resource.java:55)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:240)
at java.net.URLClassLoader.access$100(URLClassLoader.java:51)
at java.net.URLClassLoader$1.run(URLClassLoader.java:190)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:183)
at java.lang.ClassLoader.loadClass(ClassLoader.java:294)
at java.lang.ClassLoader.loadClass(ClassLoader.java:250)
at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:310)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.(DashoA6275)
at com.sun.net.ssl.internal.ssl.SSLSocketFactoryImpl.createSocket(DashoA6275)
at org.apache.axis.components.net.JSSESocketFactory.create(JSSESocketFactory.java:119)
at org.apache.axis.transport.http.HTTPSender.getSocket(HTTPSender.java:157)
at org.apache.axis.transport.http.HTTPSender.invoke(HTTPSender.java:114)
at org.apache.axis.strategies.InvocationStrategy.visit(InvocationStrategy.java:71)
at org.apache.axis.SimpleChain.doVisiting(SimpleChain.java:150)
at org.apache.axis.SimpleChain.invoke(SimpleChain.java:120)
at org.apache.axis.client.AxisClient.invoke(AxisClient.java:180)
at org.apache.axis.client.Call.invokeEngine(Call.java:2526)
at org.apache.axis.client.Call.invoke(Call.java:2515)
at org.apache.axis.client.Call.invoke(Call.java:2210)
at org.apache.axis.client.Call.invoke(Call.java:2133)
at org.apache.axis.client.Call.invoke(Call.java:1656)
at upoc.billing.qpass.PurchasePortSoapBindingStub.makePurchase(PurchasePortSoapBindingStub.java:200)
at upoc.billing.provider.QpassBillingHandler.preauthorizeMin(QpassBillingHandler.java:190)
at upoc.util.ServiceProviderUtil.getAttPreauthCodeForDeviceNumber(ServiceProviderUtil.java:302)
at upoc.operation.CreateDeviceOperation._operate(CreateDeviceOperation.java:211)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.CreateSubscriberWithDeviceOperation._operate(CreateSubscriberWithDeviceOperation.java:142)
at upoc.operation.Operation.chainExecute(Operation.java:251)
at upoc.operation.Operation.chainExecute(Operation.java:181)
at upoc.operation.Operation.chainExecute(Operation.java:115)
at upoc.operation.Operation.execute(Operation.java:81)
at upoc.jsp.action.SimpleSignup.performAction(SimpleSignup.java:127)
at upoc.jsp.action.JSPAction.execute(JSPAction.java:174)
at upoc.jsp.action.JSPActionManager.handleAction(JSPActionManager.java:69)
at upoc.jsp.tag.WebPageTag._processJspPage(WebPageTag.java:303)
at upoc.jsp.tag.JspPageTag.doStartTag(JspPageTag.java:118)
at _simpleSignup__jsp._jspService(_simpleSignup__jsp.java:41)
at com.caucho.jsp.JavaPage.service(JavaPage.java:75)
at com.caucho.jsp.Page.subservice(Page.java:506)
at com.caucho.server.http.FilterChainPage.doFilter(FilterChainPage.java:182)
at upoc.servlet.RequestMeteringFilter.doFilter(RequestMeteringFilter.java:55)
at com.caucho.server.http.FilterChainFilter.doFilter(FilterChainFilter.java:88)
at com.caucho.server.http.Invocation.service(Invocation.java:315)
at com.caucho.server.http.CacheInvocation.service(CacheInvocation.java:135)
at com.caucho.server.http.RunnerRequest.handleRequest(RunnerRequest.java:344)
at com.caucho.server.http.RunnerRequest.handleConnection(RunnerRequest.java:274)
at com.caucho.server.TcpConnection.run(TcpConnection.java:139)
at java.lang.Thread.run(Thread.java:479)
Found 1 deadlock.
- duplicates
-
JDK-4710268 ClassLoader deadlock in OIDMap.java
-
- Resolved
-