As reported here:
https://bugzilla.redhat.com/show_bug.cgi?id=2010221

We observe a native memory leak when repeating JDI operations from Eclipse in a debuggee JVM.

jemalloc reports the biggest memory leak as:

[76800 bytes leaked]
je_prof_backtrace (/home/sandreev/git/misc/jemalloc/src/prof.c:636 (discriminator 2))
je_malloc_default (/home/sandreev/git/misc/jemalloc/src/jemalloc.c:2289)
os::malloc (/data/git/jdk11/src/hotspot/share/runtime/os.cpp:697)
os::malloc (/data/git/jdk11/src/hotspot/share/runtime/os.cpp:660 (discriminator 3))
JvmtiEnvBase::allocate (/data/git/jdk11/src/hotspot/share/prims/jvmtiEnvBase.hpp:195)
JvmtiEnvBase::jvmtiMalloc (/data/git/jdk11/src/hotspot/share/prims/jvmtiEnvBase.cpp:501)
JvmtiEnv::GetMethodName (/data/git/jdk11/src/hotspot/share/prims/jvmtiEnv.cpp:2954)
jvmti_GetMethodName (/data/git/jdk11/build/linux-x86_64-normal-server-slowdebug/hotspot/variant-server/gensrc/jvmtifiles/jvmtiEnter.cpp:4366)
methodSignature (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/util.c:728)
fillInvokeRequest (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/invoker.c:284)
invoker_requestInvoke (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/invoker.c:371)
sharedInvoke (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/util.c:609)
invokeStatic (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/ClassTypeImpl.c:175)
debugLoop_run (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/debugLoop.c:159)
connectionInitiated (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/transport.c:296)
attachThread (/data/git/jdk11/src/jdk.jdwp.agent/share/native/libjdwp/transport.c:370)
JvmtiAgentThread::call_start_function (/data/git/jdk11/src/hotspot/share/prims/jvmtiImpl.cpp:85)
JvmtiAgentThread::start_function_wrapper (/data/git/jdk11/src/hotspot/share/prims/jvmtiImpl.cpp:79)
JavaThread::thread_main_inner (/data/git/jdk11/src/hotspot/share/runtime/thread.cpp:1752)
JavaThread::run (/data/git/jdk11/src/hotspot/share/runtime/thread.cpp:1732)
thread_native_entry (/data/git/jdk11/src/hotspot/os/linux/os_linux.cpp:698)
start_thread (/usr/src/debug/glibc-2.17-c758a686/nptl/pthread_create.c:308)
?? (/usr/src/debug////////glibc-2.17-c758a686/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:113)

How reproducible:

Run JDI operations with Eclipse in any Java program. E.g. can be done with a command handler:

public class SampleHandler extends AbstractHandler {

@Override
public Object execute(ExecutionEvent event) throws ExecutionException {
Job job = new Job("JDI operation") {
@Override
protected IStatus run(IProgressMonitor monitor) {
int n = 20 * 60;
for (int i = 0; i < n; ++i) {
try {
IJavaDebugTarget javaTarget = getFirstJavaDebugTarget();
if (javaTarget != null) {
IJavaThread javaThread = getFirstSuspendedJavaThread(javaTarget);
if (javaThread != null) {
IJavaType[] javaClass = javaTarget.getJavaTypes("java.lang.String");
if (javaClass[0] instanceof IJavaClassType) {
IJavaClassType jdiClassType = (IJavaClassType) javaClass[0];
IJavaObject instance = jdiClassType.newInstance("()V", null, javaThread);
try {
instance.disableCollection();
if (!monitor.isCanceled()) {
instance.sendMessage("isEmpty", "()Z", null, javaThread, false);
}
} finally {
instance.enableCollection();
}
}
}
}
} catch (DebugException e) {
e.printStackTrace();
}
}
return Status.OK_STATUS;
}
};
job.schedule();

return null;
}

private IJavaDebugTarget getFirstJavaDebugTarget() {
for (IDebugTarget debugTarget : DebugPlugin.getDefault().getLaunchManager().getDebugTargets()) {
if (debugTarget instanceof IJavaDebugTarget) {
return (IJavaDebugTarget) debugTarget;
}
}
return null;
}

private IJavaThread getFirstSuspendedJavaThread(IJavaDebugTarget javaDebugTarget) throws DebugException {
for (IThread thread : javaDebugTarget.getThreads()) {
if (thread.isSuspended() && (thread instanceof IJavaThread)) {
return (IJavaThread) thread;
}
}
return null;
}
}

Actual results:

Memory consumption of debuggee JVM increases as JDI operations are repeated.

Expected results:

No memory is leaked by fillInvokeRequest() in invoker.c.