Submitter reporting high contention in security libraries. These are being observed during 100 concurrent threads load test for OAM Authentication. Sample thread dumps below.


Bottleneck 1 :
============

"Grizzly-worker(61)" daemon prio=10 tid=0x00007f2b4400d000 nid=0x64ef waiting
for monitor entry [0x00007f2c466e5000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at java.security.Provider.getService(Provider.java:680)
- waiting to lock <0x000000074045d918> (a sun.security.provider.Sun)
at sun.security.jca.ProviderList$ServiceList.tryGet(ProviderList.java:436)
at sun.security.jca.ProviderList$ServiceList.access$200(ProviderList.java:375)
at sun.security.jca.ProviderList$ServiceList$1.hasNext(ProviderList.java:485)
at java.security.Signature.getInstance(Signature.java:223)
at oracle.idm.gateway.grs.token.impl.TokenUtils.verify(TokenUtils.java:174)
at oracle.idm.gateway.grs.token.impl.AToken.verify(AToken.java:583)
at oracle.idm.gateway.grs.token.impl.TokenUtils.verifyAToken(TokenUtils.java:671)


"Grizzly-worker(62)" daemon prio=10 tid=0x00007f2b5c012800 nid=0x64f0
runnable [0x00007f2c465e4000]
   java.lang.Thread.State: RUNNABLE
at java.security.Provider.getService(Provider.java:680)
- locked <0x000000074045d918> (a sun.security.provider.Sun)
at sun.security.jca.ProviderList$ServiceList.tryGet(ProviderList.java:436)
at sun.security.jca.ProviderList$ServiceList.access$200(ProviderList.java:375)
at sun.security.jca.ProviderList$ServiceList$1.hasNext(ProviderList.java:485)
at java.security.Signature.getInstance(Signature.java:223)
at oracle.idm.gateway.grs.token.impl.TokenUtils.sign(TokenUtils.java:152)
at oracle.idm.gateway.grs.token.impl.AToken.sign(AToken.java:476)
at oracle.idm.gateway.grs.token.impl.TokenUtils.createAToken(TokenUtils.java:639)

==========


Bottleneck 2 :
============

"Grizzly-worker(92)" daemon prio=10 tid=0x00007f2b1c012000 nid=0x6902 waiting
for monitor entry [0x00007f2c45cda000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at sun.security.rsa.RSACore$BlindingParameters.getBlindingRandomPair(RSACore.java :388)
- waiting to lock <0x0000000742057aa0> (a sun.security.rsa.RSACore$BlindingParameters)
at sun.security.rsa.RSACore.getBlindingRandomPair(RSACore.java:429)
at sun.security.rsa.RSACore.crtCrypt(RSACore.java:165)
at sun.security.rsa.RSACore.rsa(RSACore.java:110)
at sun.security.rsa.RSASignature.engineSign(RSASignature.java:177)
at java.security.Signature$Delegate.engineSign(Signature.java:1180)
at java.security.Signature.sign(Signature.java:553)
at oracle.idm.gateway.grs.token.impl.TokenUtils.sign(TokenUtils.java:155)
at oracle.idm.gateway.grs.token.impl.AToken.sign(AToken.java:476)
at oracle.idm.gateway.grs.token.impl.TokenUtils.createAToken(TokenUtils.java:639)


"Grizzly-worker(90)" daemon prio=10 tid=0x00007f2b7000c800 nid=0x68ff
runnable [0x00007f2c461e0000]
   java.lang.Thread.State: RUNNABLE
at sun.security.rsa.RSACore$BlindingParameters.getBlindingRandomPair(RSACore.java :388)
- locked <0x0000000742057aa0> (a sun.security.rsa.RSACore$BlindingParameters)
at sun.security.rsa.RSACore.getBlindingRandomPair(RSACore.java:429)
at sun.security.rsa.RSACore.crtCrypt(RSACore.java:165)
at sun.security.rsa.RSACore.rsa(RSACore.java:110)
at sun.security.rsa.RSASignature.engineSign(RSASignature.java:177)
at java.security.Signature$Delegate.engineSign(Signature.java:1180)
at java.security.Signature.sign(Signature.java:553)
at oracle.idm.gateway.grs.token.impl.TokenUtils.sign(TokenUtils.java:155)
at oracle.idm.gateway.grs.token.impl.AToken.sign(AToken.java:476)
at oracle.idm.gateway.grs.token.impl.TokenUtils.createAToken(TokenUtils.java:639)