-
Bug
-
Resolution: Duplicate
-
P4
-
None
-
11.0.10, 19.0.1
-
generic
-
generic
ADDITIONAL SYSTEM INFORMATION :
Oracle Linux, the system is about scheduler run and then sleep for 60 seconds and so on.
I'm using Spring boot and native JDBC, and @EnableAsync and Java exectuor
A DESCRIPTION OF THE PROBLEM :
java.lang.OutOfMemoryError: Java heap space
Dumping heap to java_pid28044.hprof ...
Heap dump file created [176113675 bytes in 4.795 secs]
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Keep-Alive-Timer"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-5"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-4"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Catalina-utility-2"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-BlockPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "pool-1-thread-1"
STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run the below 5 Classes, with the application.properties and you will find that graph is getting higher forever.
to reproduce it quickly assign few memory in the maximum java size Xmx
like
-Xms50M -Xmx100M -XX:+UseG1GC -XX:MinHeapFreeRatio=2 -XX:MaxHeapFreeRatio=10 -XX:+HeapDumpOnOutOfMemoryError
ACTUAL -
java.lang.OutOfMemoryError: Java heap space
Dumping heap to java_pid28044.hprof ...
Heap dump file created [176113675 bytes in 4.795 secs]
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Keep-Alive-Timer"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-5"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-4"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Catalina-utility-2"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-BlockPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "pool-1-thread-1"
---------- BEGIN SOURCE ----------
package com.sherif.concurrentcalls;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class Class1 {
@Autowired
FinancialServiceCaller slowServiceCaller;
@Autowired
private Connection con;
/*@Autowired
ExecutorService executor;
*/
@Value("${dbReal_Schema}")
private String dbRealSchema;
@Value("${batch_Size}")
private int batchSize;
@Value("${result_Set_Fetch_Size}")
private int resultSetFetchSize;
@Value("${run_Every}")
private int runEvery;
@Value("${run_Every_Unit}")
private String runEveryUnit;
@Value("${sleep_Every}")
private int sleepEvery;
@Value("${accelerate_Run_Every}")
private int accelerateRunEvery;
/*
public void execute() throws InterruptedException{
System.out.println("Test Class 1"); Thread.sleep(10000); }
*/
public void execute() throws Exception {
/* System.out.println("numberOfRounds " + ++numOfRounds); */
Instant start = Instant.now();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 1");
System.out.println("Test Class 1");
doProcess(start);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 2");
// Thread.sleep(sleepEvery);
/*
if (numOfRounds % 20 == 0) { numOfRounds = 0; System.gc(); }
*/
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 3");
}
private void doProcess(Instant start) throws Exception {
PreparedStatement pStatement = null;
PreparedStatement pStatement2 = null;
ResultSet rs = null;
String query = null;
String response = null;
int i = 0;
int period = runEvery;
CompletableFuture<String> completedObj = null;
List<CompletableFuture<String>> allFutures = new ArrayList<>();
Map<String, CompletableFuture<String>> requests = new HashMap<>();
try {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 4");
Timestamp startDateTimestamp = null;
Timestamp endDateTimestamp = null;
Timestamp accDateTimestamp = null;
Timestamp maxTimestamp = null;
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 41");
//
pStatement = con.prepareStatement(
"SELECT max (finalizedtime) from " + dbRealSchema + ".RDS$FINANCIALRECEIPT",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 42");
while (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 43");
maxTimestamp = rs.getTimestamp(1);
System.out.println("maxTimestamp= " + maxTimestamp);
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 44");
rs.close();
pStatement.close();
pStatement = con.prepareStatement(
"SELECT max (finalizedtime) start_time, max (finalizedtime) + interval '" + runEvery + "' "
+ runEveryUnit + " end_time ," + " max (finalizedtime) + interval '" + accelerateRunEvery
+ "' " + runEveryUnit + " acc_time from FINANCIAL_API_FINALIZEDTIME",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 51");
while (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 52");
startDateTimestamp = rs.getTimestamp(1);
System.out.println("startDateTimestamp= " + startDateTimestamp);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 53");
endDateTimestamp = rs.getTimestamp(2);
System.out.println("endDateTimestamp= " + endDateTimestamp);
accDateTimestamp = rs.getTimestamp(3);
// System.out.println("accDateTimestamp= " + accDateTimestamp);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 54");
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 55");
long diff = (maxTimestamp.getTime() - startDateTimestamp.getTime()) / (1000 * 60);
if (diff >= 5) {
period = accelerateRunEvery;
endDateTimestamp = accDateTimestamp;
}
System.out.println(maxTimestamp);
System.out.println(startDateTimestamp);
System.out.println(diff);
rs.close();
pStatement.close();
pStatement = con.prepareStatement(
"select count(1) from " + dbRealSchema + ".RDS$FINANCIALRECEIPT where finalizedtime >= ?",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.setTimestamp(1, startDateTimestamp);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 56");
if (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 57");
if (rs.getInt(1) <= 0) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 58");
rs.close();
pStatement.close();
return;
}
rs.close();
pStatement.close();
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 6");
pStatement = con
.prepareStatement("Insert into FINANCIAL_API_FINALIZEDTIME Select max(finalizedTime) + Interval '"
+ period + "' " + runEveryUnit + " from FINANCIAL_API_FINALIZEDTIME",java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.executeUpdate();
con.commit();
pStatement.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 60");
pStatement = con.prepareStatement("SELECT /* + PARALLEL (20) */ FINANCIALTRANSACTIONID,TRANSFERTYPE FROM "
+ dbRealSchema + ".RDS$FINANCIALRECEIPT where finalizedtime >= ?" + " and finalizedtime < ?"
+ " and transfertype in " + "( "
+ "'BATCH_TRANSFER', 'CASH_IN', 'CASH_OUT', 'DEBIT', 'FLOAT_TRANSFER', 'PAYMENT', 'TRANSFER', 'TRANSFER_FROM_ANY_BANK', 'WITHDRAWAL', "
+ "'ADJUSTMENT','REVERSAL','COMMISSIONING','RESOLVE_DEPOSIT' " + ") ",java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.setTimestamp(1, startDateTimestamp);
pStatement.setTimestamp(2, endDateTimestamp);
rs = pStatement.executeQuery();
rs.setFetchSize(resultSetFetchSize);
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 7");
query = "insert into FRAUDAPISTATUS(TRANSACTIONID,TRANSFERTYPE,REQUEST_SENT) values (?,?,1)";
pStatement2 = con.prepareStatement(query,java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
int counter = 0;
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 8");
BigDecimal tranId;
String tranType = null;
while (rs.next()) {
++counter;
tranId = rs.getBigDecimal(1);
tranType = rs.getString(2);
completedObj = slowServiceCaller.callOtherService(tranId, tranType);
pStatement2.setBigDecimal(1, tranId);
pStatement2.setString(2, tranType);
pStatement2.addBatch();
if (counter % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}
rs.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 9");
System.out.println(new Timestamp(System.currentTimeMillis()) + " Counter is " + counter);
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
pStatement.close();
pStatement2.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 90");
//CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[0]));
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 91");
query = "insert into FRAUDAPISTATUS2(response,TRANSACTIONID,GET_response) values (?,?,1)";
pStatement2 = con.prepareStatement(query,java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
/*System.out.println(new Timestamp(System.currentTimeMillis()) + " Count of Responses: "+allFutures.size());*/
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 10");
/*Iterator<Map.Entry<String, CompletableFuture<String>>> itr = requests.entrySet().iterator();
i = 0;
while(itr.hasNext())
{
++i;
Map.Entry<String, CompletableFuture<String>> entry = itr.next();
pStatement2.setString(2, entry.getKey());
try {
//response = entry.getValue().get(10, TimeUnit.MILLISECONDS).toString();
response = entry.getValue().get().toString();
//System.out.println(response);
}
catch (Exception ex)
{
//ex.printStackTrace();
if (ex instanceof TimeoutException)
response = "java.util.concurrent.TimeoutException";
else
response = ex.getMessage();
//System.out.println(response);
pStatement2.setString(1,response);
pStatement2.addBatch();
if (i % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
continue;
}
pStatement2.setString(1,response);
pStatement2.addBatch();
if (i % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}*/
/*for (i = 0; i < allFutures.size(); i++) {
//if (allFutures.get(i).isCompletedExceptionally())
// response = allFutures.get(i).toString();
//else
// response = allFutures.get(i).get().toString();
pStatement2.setLong(2, (i+1));
try {
response = allFutures.get(i).get().toString();
System.out.println(response);
}
catch (Exception ex)
{
//ex.printStackTrace();
response = ex.getMessage();
System.out.println(response);
pStatement2.setString(1,response);
pStatement2.addBatch();
if ((i+1) % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
continue;
}
pStatement2.setString(1,response);
pStatement2.addBatch();
if ((i+1) % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}*/
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 11");
rs.close();
pStatement.close();
pStatement2.close();
System.out.println(new Timestamp(System.currentTimeMillis()) + " Total time: "
+ Duration.between(start, Instant.now()).getSeconds());
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 12");
//executor.shutdown();
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
===========================================
package com.sherif.concurrentcalls;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.EventListener;
@Configuration
public class ThreadManager {
@Autowired
private /*FinancialTransactionRunner*/ Class1 class1;
@Autowired
private /*NonFinancialTransactionRunner*/ Class2 class2;
Thread RunningThread1 = null;
Thread RunningThread2 = null;
Thread GCThread = null;
int numOfRounds=0;
@EventListener(ApplicationReadyEvent.class)
public void run() throws Exception {
if (RunningThread1 == null) {
RunningThread1 = new Thread() {
public void run() {
try {
startProcessing();
} catch (Exception e) {
e.printStackTrace();
}
}
};
RunningThread1.start();
} else {
System.out.println("I'm alive inside Thread1");
}
if (RunningThread2 == null) {
RunningThread2 = new Thread() {
public void run() {
try {
startProcessing2();
} catch (Exception e) {
e.printStackTrace();
}
}
};
RunningThread2.start();
}
else {
System.out.println("I'm alive inside Thread2");
}
/*if (GCThread == null) {
GCThread = new Thread() {
public void run() {
try {
GC();
} catch (Exception e) {
e.printStackTrace();
}
}
};
GCThread.start();
}
else
{
System.out.println("I've alive inside GC");
}*/
}
private void startProcessing() throws Exception {
while (true) {
class1.execute();
Thread.sleep(60000);
}
}
private void startProcessing2() throws Exception {
while (true) {
class2.execute();
}
}
private void GC() throws Exception {
while (true) {
System.gc();
Thread.sleep(30000);
}
}
}
========================
package com.sherif.concurrentcalls;
import java.sql.Connection;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
@Component
public class Class2 {
@Autowired
private RestTemplate restTemplate;
@Autowired
private Connection con;
public void execute() throws InterruptedException{
System.out.println("Test Class 2");
Thread.sleep(5000);
}
}
====================
package com.sherif.concurrentcalls;
import java.math.BigDecimal;
import java.util.concurrent.CompletableFuture;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@Service
public class FinancialServiceCaller {
@Autowired
private RestTemplate restTemplate;
@Value("${CSBF1}")
private String CSBF1;
@Value("${CSBF2}")
private String CSBF2;
@Value("${CSBF3}")
private String CSBF3;
@Value("${CSBF4}")
private String CSBF4;
@Async
public /*void*/ CompletableFuture<String> callOtherService(BigDecimal finTranId, String tranType) throws Exception {
//System.out.println("Before API Calling");
StringBuilder localSlowServiceEndpoint = new StringBuilder();
// String localSlowServiceEndpoint = null;
if (tranType.equals("BATCH_TRANSFER") || tranType.equals("CASH_IN") || tranType.equals("CASH_OUT") || tranType.equals("DEBIT")
|| tranType.equals("FLOAT_TRANSFER") || tranType.equals("PAYMENT") || tranType.equals("TRANSFER")
|| tranType.equals("TRANSFER_FROM_ANY_BANK") || tranType.equals("WITHDRAWAL")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/financialtransaction?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF1 + "?transactionId=" + finTranId);
} else if (tranType.equals("ADJUSTMENT") || tranType.equals("REVERSAL")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/adjustmentreversal?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF2 + "?transactionId=" + finTranId);
} else if (tranType.equals("COMMISSIONING")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/agentcommission?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF3 + "?transactionId=" + finTranId);
} else if (tranType.equals("RESOLVE_DEPOSIT")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/resolvedeposit?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF4 + "?transactionId=" + finTranId);
} else
return null;
//return ;
String responseObj =
restTemplate.getForObject(localSlowServiceEndpoint.toString(), String.class);
return CompletableFuture.completedFuture(responseObj);
}
}
============================
package com.sherif.concurrentcalls;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Timestamp;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.web.client.RestTemplate;
@SpringBootApplication
@EnableAsync
public class ConcurrentcallsApplication {
@Value("${dbConnection_String}")
private String dbConnectionString;
@Value("${dbBridge_Schema}")
private String dbBridgeSchema;
@Value("${dbBridge_Password}")
private String dbBridgePassword;
@Value("${core_Pool_Size}")
private int corePoolSize;
@Value("${max_Core_Pool_Size}")
private int maxCorePoolSize;
@Value("${queue_Capacity}")
private int queueCapacity;
public static void main(String[] args) throws Exception {
System.out.println("Starting");
SpringApplication.run(ConcurrentcallsApplication.class, args);
}
/*@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}*/
@Bean
public RestTemplate restTemplate() {
SimpleClientHttpRequestFactory rf = new SimpleClientHttpRequestFactory();
rf.setBufferRequestBody(false);
return new RestTemplate(rf);
}
/*@Bean
public Executor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(corePoolSize);
executor.setMaxPoolSize(maxCorePoolSize);
//executor.setKeepAliveSeconds(30);
executor.setThreadNamePrefix("worker-exec-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
//executor.setAllowCoreThreadTimeOut(true);
//executor.setQueueCapacity(queueCapacity);
executor.initialize();
return executor;
}*/
@Bean(name="taskExecutor")
public Executor executor() {
//Executor executor = Executors.newSingleThreadExecutor();
ExecutorService executor = Executors.newFixedThreadPool(1);
//ExecutorService executor = Executors.newWorkStealingPool(5);
//ExecutorService executor = Executors.newWorkStealingPool();
//ExecutorService executor = Executors.newCachedThreadPool();
/*
ThreadFactory ThreadFactory = Executors.defaultThreadFactory();
ExecutorService executor = Executors.newCachedThreadPool(ThreadFactory);
*/
return executor;
}
/*@Bean
public NewExecuter executor() {
return new NewExecuter();
}*/
@Bean
public Connection JDBCInit() throws Exception {
System.out.println(new Timestamp(System.currentTimeMillis()) + " Before JDBC Open");
Class.forName("oracle.jdbc.driver.OracleDriver");
Connection con = DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:orcl", "RDS_UG", "RDS_UG");
/*
Connection con=DriverManager.getConnection(
dbConnectionString,dbBridgeSchema,dbBridgePassword);
*/
con.setAutoCommit(false);
System.out.println(new Timestamp(System.currentTimeMillis()) + " After JDBC Open");
return con;
}
}
=============================
server.port=9090
# DB Info
dbReal_Schema=RDS_UG
dbBridge_Schema=SASAML_API
dbBridge_Password=Sasap#9867
dbConnection_String=jdbc:oracle:thin:@10.156.195.67:1522/RDSASAML
# Result Set Info
batch_Size=1000
result_Set_Fetch_Size=2000
# Thread Info
core_Pool_Size=1
max_Core_Pool_Size=1
queue_Capacity=20000
#Scheduler Info
run_Every=1
run_Every_Unit=MINUTE
sleep_Every=60000
accelerate_Run_Every=2
# End Points
# Financial APIs
CSBF1=https://ugfraudprod.mtn.co.ug/financial/financialtransaction
CSBF2=https://ugfraudprod.mtn.co.ug/financial/adjustmentreversal
CSBF3=https://ugfraudprod.mtn.co.ug/financial/agentcommission
CSBF4=https://ugfraudprod.mtn.co.ug/financial/resolvedeposit
# Non-Financial APIs
CSNM1=https://ugfraudprod.mtn.co.ug/nonfinancial/activations
CSNM3=https://ugfraudprod.mtn.co.ug/nonfinancial/aauditlogactivateeretree
CSNM5=https://ugfraudprod.mtn.co.ug/nonfinancial/auditlogprofileprivilegechanges
CSNM6=https://ugfraudprod.mtn.co.ug/nonfinancial/pinchangereset
CSNM7=https://ugfraudprod.mtn.co.ug/nonfinancial/registration
CSNM8=https://ugfraudprod.mtn.co.ug/nonfinancial/walletaccountprofilechange
CSNM9=https://ugfraudprod.mtn.co.ug/nonfinancial/mtninternaluserprofilechange
CSNM10=https://ugfraudprod.mtn.co.ug/nonfinancial/imsichange
CSNM11=https://ugfraudprod.mtn.co.ug/nonfinancial/msisdnchange
---------- END SOURCE ----------
FREQUENCY : always
Oracle Linux, the system is about scheduler run and then sleep for 60 seconds and so on.
I'm using Spring boot and native JDBC, and @EnableAsync and Java exectuor
A DESCRIPTION OF THE PROBLEM :
java.lang.OutOfMemoryError: Java heap space
Dumping heap to java_pid28044.hprof ...
Heap dump file created [176113675 bytes in 4.795 secs]
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Keep-Alive-Timer"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-5"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-4"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Catalina-utility-2"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-BlockPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "pool-1-thread-1"
STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run the below 5 Classes, with the application.properties and you will find that graph is getting higher forever.
to reproduce it quickly assign few memory in the maximum java size Xmx
like
-Xms50M -Xmx100M -XX:+UseG1GC -XX:MinHeapFreeRatio=2 -XX:MaxHeapFreeRatio=10 -XX:+HeapDumpOnOutOfMemoryError
ACTUAL -
java.lang.OutOfMemoryError: Java heap space
Dumping heap to java_pid28044.hprof ...
Heap dump file created [176113675 bytes in 4.795 secs]
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Keep-Alive-Timer"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-5"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-4"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-ClientPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Catalina-utility-2"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "http-nio-9090-BlockPoller"
Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "pool-1-thread-1"
---------- BEGIN SOURCE ----------
package com.sherif.concurrentcalls;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class Class1 {
@Autowired
FinancialServiceCaller slowServiceCaller;
@Autowired
private Connection con;
/*@Autowired
ExecutorService executor;
*/
@Value("${dbReal_Schema}")
private String dbRealSchema;
@Value("${batch_Size}")
private int batchSize;
@Value("${result_Set_Fetch_Size}")
private int resultSetFetchSize;
@Value("${run_Every}")
private int runEvery;
@Value("${run_Every_Unit}")
private String runEveryUnit;
@Value("${sleep_Every}")
private int sleepEvery;
@Value("${accelerate_Run_Every}")
private int accelerateRunEvery;
/*
public void execute() throws InterruptedException{
System.out.println("Test Class 1"); Thread.sleep(10000); }
*/
public void execute() throws Exception {
/* System.out.println("numberOfRounds " + ++numOfRounds); */
Instant start = Instant.now();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 1");
System.out.println("Test Class 1");
doProcess(start);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 2");
// Thread.sleep(sleepEvery);
/*
if (numOfRounds % 20 == 0) { numOfRounds = 0; System.gc(); }
*/
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 3");
}
private void doProcess(Instant start) throws Exception {
PreparedStatement pStatement = null;
PreparedStatement pStatement2 = null;
ResultSet rs = null;
String query = null;
String response = null;
int i = 0;
int period = runEvery;
CompletableFuture<String> completedObj = null;
List<CompletableFuture<String>> allFutures = new ArrayList<>();
Map<String, CompletableFuture<String>> requests = new HashMap<>();
try {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 4");
Timestamp startDateTimestamp = null;
Timestamp endDateTimestamp = null;
Timestamp accDateTimestamp = null;
Timestamp maxTimestamp = null;
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 41");
//
pStatement = con.prepareStatement(
"SELECT max (finalizedtime) from " + dbRealSchema + ".RDS$FINANCIALRECEIPT",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 42");
while (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 43");
maxTimestamp = rs.getTimestamp(1);
System.out.println("maxTimestamp= " + maxTimestamp);
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 44");
rs.close();
pStatement.close();
pStatement = con.prepareStatement(
"SELECT max (finalizedtime) start_time, max (finalizedtime) + interval '" + runEvery + "' "
+ runEveryUnit + " end_time ," + " max (finalizedtime) + interval '" + accelerateRunEvery
+ "' " + runEveryUnit + " acc_time from FINANCIAL_API_FINALIZEDTIME",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 51");
while (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 52");
startDateTimestamp = rs.getTimestamp(1);
System.out.println("startDateTimestamp= " + startDateTimestamp);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 53");
endDateTimestamp = rs.getTimestamp(2);
System.out.println("endDateTimestamp= " + endDateTimestamp);
accDateTimestamp = rs.getTimestamp(3);
// System.out.println("accDateTimestamp= " + accDateTimestamp);
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 54");
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 55");
long diff = (maxTimestamp.getTime() - startDateTimestamp.getTime()) / (1000 * 60);
if (diff >= 5) {
period = accelerateRunEvery;
endDateTimestamp = accDateTimestamp;
}
System.out.println(maxTimestamp);
System.out.println(startDateTimestamp);
System.out.println(diff);
rs.close();
pStatement.close();
pStatement = con.prepareStatement(
"select count(1) from " + dbRealSchema + ".RDS$FINANCIALRECEIPT where finalizedtime >= ?",
java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.setTimestamp(1, startDateTimestamp);
rs = pStatement.executeQuery();
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 56");
if (rs.next()) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 57");
if (rs.getInt(1) <= 0) {
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 58");
rs.close();
pStatement.close();
return;
}
rs.close();
pStatement.close();
}
// System.out.println(new Timestamp(System.currentTimeMillis()) + " 6");
pStatement = con
.prepareStatement("Insert into FINANCIAL_API_FINALIZEDTIME Select max(finalizedTime) + Interval '"
+ period + "' " + runEveryUnit + " from FINANCIAL_API_FINALIZEDTIME",java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.executeUpdate();
con.commit();
pStatement.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 60");
pStatement = con.prepareStatement("SELECT /* + PARALLEL (20) */ FINANCIALTRANSACTIONID,TRANSFERTYPE FROM "
+ dbRealSchema + ".RDS$FINANCIALRECEIPT where finalizedtime >= ?" + " and finalizedtime < ?"
+ " and transfertype in " + "( "
+ "'BATCH_TRANSFER', 'CASH_IN', 'CASH_OUT', 'DEBIT', 'FLOAT_TRANSFER', 'PAYMENT', 'TRANSFER', 'TRANSFER_FROM_ANY_BANK', 'WITHDRAWAL', "
+ "'ADJUSTMENT','REVERSAL','COMMISSIONING','RESOLVE_DEPOSIT' " + ") ",java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
pStatement.setTimestamp(1, startDateTimestamp);
pStatement.setTimestamp(2, endDateTimestamp);
rs = pStatement.executeQuery();
rs.setFetchSize(resultSetFetchSize);
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 7");
query = "insert into FRAUDAPISTATUS(TRANSACTIONID,TRANSFERTYPE,REQUEST_SENT) values (?,?,1)";
pStatement2 = con.prepareStatement(query,java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
int counter = 0;
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 8");
BigDecimal tranId;
String tranType = null;
while (rs.next()) {
++counter;
tranId = rs.getBigDecimal(1);
tranType = rs.getString(2);
completedObj = slowServiceCaller.callOtherService(tranId, tranType);
pStatement2.setBigDecimal(1, tranId);
pStatement2.setString(2, tranType);
pStatement2.addBatch();
if (counter % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}
rs.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 9");
System.out.println(new Timestamp(System.currentTimeMillis()) + " Counter is " + counter);
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
pStatement.close();
pStatement2.close();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 90");
//CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[0]));
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 91");
query = "insert into FRAUDAPISTATUS2(response,TRANSACTIONID,GET_response) values (?,?,1)";
pStatement2 = con.prepareStatement(query,java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
/*System.out.println(new Timestamp(System.currentTimeMillis()) + " Count of Responses: "+allFutures.size());*/
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 10");
/*Iterator<Map.Entry<String, CompletableFuture<String>>> itr = requests.entrySet().iterator();
i = 0;
while(itr.hasNext())
{
++i;
Map.Entry<String, CompletableFuture<String>> entry = itr.next();
pStatement2.setString(2, entry.getKey());
try {
//response = entry.getValue().get(10, TimeUnit.MILLISECONDS).toString();
response = entry.getValue().get().toString();
//System.out.println(response);
}
catch (Exception ex)
{
//ex.printStackTrace();
if (ex instanceof TimeoutException)
response = "java.util.concurrent.TimeoutException";
else
response = ex.getMessage();
//System.out.println(response);
pStatement2.setString(1,response);
pStatement2.addBatch();
if (i % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
continue;
}
pStatement2.setString(1,response);
pStatement2.addBatch();
if (i % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}*/
/*for (i = 0; i < allFutures.size(); i++) {
//if (allFutures.get(i).isCompletedExceptionally())
// response = allFutures.get(i).toString();
//else
// response = allFutures.get(i).get().toString();
pStatement2.setLong(2, (i+1));
try {
response = allFutures.get(i).get().toString();
System.out.println(response);
}
catch (Exception ex)
{
//ex.printStackTrace();
response = ex.getMessage();
System.out.println(response);
pStatement2.setString(1,response);
pStatement2.addBatch();
if ((i+1) % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
continue;
}
pStatement2.setString(1,response);
pStatement2.addBatch();
if ((i+1) % batchSize == 0) {
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
}
}*/
pStatement2.executeBatch();
pStatement2.clearBatch();
con.commit();
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 11");
rs.close();
pStatement.close();
pStatement2.close();
System.out.println(new Timestamp(System.currentTimeMillis()) + " Total time: "
+ Duration.between(start, Instant.now()).getSeconds());
//System.out.println(new Timestamp(System.currentTimeMillis()) + " 12");
//executor.shutdown();
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
===========================================
package com.sherif.concurrentcalls;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.EventListener;
@Configuration
public class ThreadManager {
@Autowired
private /*FinancialTransactionRunner*/ Class1 class1;
@Autowired
private /*NonFinancialTransactionRunner*/ Class2 class2;
Thread RunningThread1 = null;
Thread RunningThread2 = null;
Thread GCThread = null;
int numOfRounds=0;
@EventListener(ApplicationReadyEvent.class)
public void run() throws Exception {
if (RunningThread1 == null) {
RunningThread1 = new Thread() {
public void run() {
try {
startProcessing();
} catch (Exception e) {
e.printStackTrace();
}
}
};
RunningThread1.start();
} else {
System.out.println("I'm alive inside Thread1");
}
if (RunningThread2 == null) {
RunningThread2 = new Thread() {
public void run() {
try {
startProcessing2();
} catch (Exception e) {
e.printStackTrace();
}
}
};
RunningThread2.start();
}
else {
System.out.println("I'm alive inside Thread2");
}
/*if (GCThread == null) {
GCThread = new Thread() {
public void run() {
try {
GC();
} catch (Exception e) {
e.printStackTrace();
}
}
};
GCThread.start();
}
else
{
System.out.println("I've alive inside GC");
}*/
}
private void startProcessing() throws Exception {
while (true) {
class1.execute();
Thread.sleep(60000);
}
}
private void startProcessing2() throws Exception {
while (true) {
class2.execute();
}
}
private void GC() throws Exception {
while (true) {
System.gc();
Thread.sleep(30000);
}
}
}
========================
package com.sherif.concurrentcalls;
import java.sql.Connection;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
@Component
public class Class2 {
@Autowired
private RestTemplate restTemplate;
@Autowired
private Connection con;
public void execute() throws InterruptedException{
System.out.println("Test Class 2");
Thread.sleep(5000);
}
}
====================
package com.sherif.concurrentcalls;
import java.math.BigDecimal;
import java.util.concurrent.CompletableFuture;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@Service
public class FinancialServiceCaller {
@Autowired
private RestTemplate restTemplate;
@Value("${CSBF1}")
private String CSBF1;
@Value("${CSBF2}")
private String CSBF2;
@Value("${CSBF3}")
private String CSBF3;
@Value("${CSBF4}")
private String CSBF4;
@Async
public /*void*/ CompletableFuture<String> callOtherService(BigDecimal finTranId, String tranType) throws Exception {
//System.out.println("Before API Calling");
StringBuilder localSlowServiceEndpoint = new StringBuilder();
// String localSlowServiceEndpoint = null;
if (tranType.equals("BATCH_TRANSFER") || tranType.equals("CASH_IN") || tranType.equals("CASH_OUT") || tranType.equals("DEBIT")
|| tranType.equals("FLOAT_TRANSFER") || tranType.equals("PAYMENT") || tranType.equals("TRANSFER")
|| tranType.equals("TRANSFER_FROM_ANY_BANK") || tranType.equals("WITHDRAWAL")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/financialtransaction?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF1 + "?transactionId=" + finTranId);
} else if (tranType.equals("ADJUSTMENT") || tranType.equals("REVERSAL")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/adjustmentreversal?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF2 + "?transactionId=" + finTranId);
} else if (tranType.equals("COMMISSIONING")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/agentcommission?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF3 + "?transactionId=" + finTranId);
} else if (tranType.equals("RESOLVE_DEPOSIT")) {
//localSlowServiceEndpoint.append("http://localhost:9091/rest/micSerTest1?transactionId=" + finTranId);
//localSlowServiceEndpoint.append("http://10.156.198.42:8080/financial/resolvedeposit?transactionId=" + finTranId);
localSlowServiceEndpoint.append(CSBF4 + "?transactionId=" + finTranId);
} else
return null;
//return ;
String responseObj =
restTemplate.getForObject(localSlowServiceEndpoint.toString(), String.class);
return CompletableFuture.completedFuture(responseObj);
}
}
============================
package com.sherif.concurrentcalls;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Timestamp;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.web.client.RestTemplate;
@SpringBootApplication
@EnableAsync
public class ConcurrentcallsApplication {
@Value("${dbConnection_String}")
private String dbConnectionString;
@Value("${dbBridge_Schema}")
private String dbBridgeSchema;
@Value("${dbBridge_Password}")
private String dbBridgePassword;
@Value("${core_Pool_Size}")
private int corePoolSize;
@Value("${max_Core_Pool_Size}")
private int maxCorePoolSize;
@Value("${queue_Capacity}")
private int queueCapacity;
public static void main(String[] args) throws Exception {
System.out.println("Starting");
SpringApplication.run(ConcurrentcallsApplication.class, args);
}
/*@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}*/
@Bean
public RestTemplate restTemplate() {
SimpleClientHttpRequestFactory rf = new SimpleClientHttpRequestFactory();
rf.setBufferRequestBody(false);
return new RestTemplate(rf);
}
/*@Bean
public Executor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(corePoolSize);
executor.setMaxPoolSize(maxCorePoolSize);
//executor.setKeepAliveSeconds(30);
executor.setThreadNamePrefix("worker-exec-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
//executor.setAllowCoreThreadTimeOut(true);
//executor.setQueueCapacity(queueCapacity);
executor.initialize();
return executor;
}*/
@Bean(name="taskExecutor")
public Executor executor() {
//Executor executor = Executors.newSingleThreadExecutor();
ExecutorService executor = Executors.newFixedThreadPool(1);
//ExecutorService executor = Executors.newWorkStealingPool(5);
//ExecutorService executor = Executors.newWorkStealingPool();
//ExecutorService executor = Executors.newCachedThreadPool();
/*
ThreadFactory ThreadFactory = Executors.defaultThreadFactory();
ExecutorService executor = Executors.newCachedThreadPool(ThreadFactory);
*/
return executor;
}
/*@Bean
public NewExecuter executor() {
return new NewExecuter();
}*/
@Bean
public Connection JDBCInit() throws Exception {
System.out.println(new Timestamp(System.currentTimeMillis()) + " Before JDBC Open");
Class.forName("oracle.jdbc.driver.OracleDriver");
Connection con = DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:orcl", "RDS_UG", "RDS_UG");
/*
Connection con=DriverManager.getConnection(
dbConnectionString,dbBridgeSchema,dbBridgePassword);
*/
con.setAutoCommit(false);
System.out.println(new Timestamp(System.currentTimeMillis()) + " After JDBC Open");
return con;
}
}
=============================
server.port=9090
# DB Info
dbReal_Schema=RDS_UG
dbBridge_Schema=SASAML_API
dbBridge_Password=Sasap#9867
dbConnection_String=jdbc:oracle:thin:@10.156.195.67:1522/RDSASAML
# Result Set Info
batch_Size=1000
result_Set_Fetch_Size=2000
# Thread Info
core_Pool_Size=1
max_Core_Pool_Size=1
queue_Capacity=20000
#Scheduler Info
run_Every=1
run_Every_Unit=MINUTE
sleep_Every=60000
accelerate_Run_Every=2
# End Points
# Financial APIs
CSBF1=https://ugfraudprod.mtn.co.ug/financial/financialtransaction
CSBF2=https://ugfraudprod.mtn.co.ug/financial/adjustmentreversal
CSBF3=https://ugfraudprod.mtn.co.ug/financial/agentcommission
CSBF4=https://ugfraudprod.mtn.co.ug/financial/resolvedeposit
# Non-Financial APIs
CSNM1=https://ugfraudprod.mtn.co.ug/nonfinancial/activations
CSNM3=https://ugfraudprod.mtn.co.ug/nonfinancial/aauditlogactivateeretree
CSNM5=https://ugfraudprod.mtn.co.ug/nonfinancial/auditlogprofileprivilegechanges
CSNM6=https://ugfraudprod.mtn.co.ug/nonfinancial/pinchangereset
CSNM7=https://ugfraudprod.mtn.co.ug/nonfinancial/registration
CSNM8=https://ugfraudprod.mtn.co.ug/nonfinancial/walletaccountprofilechange
CSNM9=https://ugfraudprod.mtn.co.ug/nonfinancial/mtninternaluserprofilechange
CSNM10=https://ugfraudprod.mtn.co.ug/nonfinancial/imsichange
CSNM11=https://ugfraudprod.mtn.co.ug/nonfinancial/msisdnchange
---------- END SOURCE ----------
FREQUENCY : always