FULL PRODUCT VERSION :


A DESCRIPTION OF THE PROBLEM :
A JVMTI agent returning a new bytecode array from a ClassFileLoadHook callback results in a memory leak in JVM.

Please find below a small Java application and JVMTI agent reproducing the problem.

The Java class Test creates classes with Unsafe.

Observations:

1. The problem is reproducible with Java 7, but is not reproducible in Java 8.

2. The problem is reproducible for classes generated with Unsafe.defineAnonymousClass(), but is not reproducible for "normal" unloaded classes created in custom loaders which are created and then garbage collected.

3. For simplicity, the example agent creates a new copy of the original bytecode (i.e. unmodified). The problem was originally detected which a real agent which modifies the bytecode.


THE PROBLEM WAS REPRODUCIBLE WITH -Xint FLAG: Did not try

THE PROBLEM WAS REPRODUCIBLE WITH -server FLAG: Yes

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
1. Compile the agent and the Java class.
2. The example uses ASM bytecode modification library. We used asm-5.0.3.jar
3. Run with Java 7:
java -agentlib:agent -cp .;asm-5.0.3.jar Test
4. Monitor the process virtual memory usage. In Windows, it's visible in Task Manager in the column "Commit size", as well as in "Memory". It will constantly grow.

Running the same on Java 8 does not show a leak - used memory slightly fluctuates but does not grow.


REPRODUCIBILITY :
This bug can be reproduced always.

---------- BEGIN SOURCE ----------
-------------------------------------------------
Java code
-------------------------------------------------
import java.lang.reflect.Field;
import java.lang.reflect.Method;

import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.MethodVisitor;
import org.objectweb.asm.Opcodes;

public class Test implements Opcodes {
  static Object theUnsafe;
  static Method defineAnonymousClass;
  static byte[] bytecode;

  static {
    Object unsafe;
    try {
      Class<?> unsafeClass = ClassLoader.getSystemClassLoader().loadClass("sun.misc.Unsafe");
      Field theUnsafeField = unsafeClass.getDeclaredField("theUnsafe");
      theUnsafeField.setAccessible(true);
      unsafe = theUnsafeField.get(null);
      try {
        defineAnonymousClass = unsafeClass.getDeclaredMethod("defineAnonymousClass", Class.class, byte[].class, Object[].class);
      }
      catch (NoSuchMethodException e) {
        throw new AssertionError("Must use at least JDK7");
      }
    }
    catch (Exception e) {
      e.printStackTrace();
      throw new AssertionError("Something went wrong!");
    }
    theUnsafe = unsafe;
    bytecode = generateBytecode();
  }

  static byte[] generateBytecode() {
    ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS | ClassWriter.COMPUTE_FRAMES);
    cw.visit(V1_1, ACC_PUBLIC | ACC_SUPER, "GenerateClass", null, "java/lang/Object", null);
    MethodVisitor ctor = cw.visitMethod(ACC_PUBLIC, "<init>", "()V", null, null);
    ctor.visitCode();
    ctor.visitVarInsn(ALOAD, 0);
    ctor.visitMethodInsn(INVOKESPECIAL, "java/lang/Object", "<init>", "()V", false);
    ctor.visitInsn(RETURN);
    ctor.visitMaxs(-1, -1);
    ctor.visitEnd();
    cw.visitEnd();
    return cw.toByteArray();
  }

  static Class<?> defineAnonymousClass(Class<?> hostClass, byte[] bytecode) throws Exception {
    return (Class<?>)defineAnonymousClass.invoke(theUnsafe, hostClass, bytecode, null);
  }

  public static void main(String[] args) throws Exception {
    int count = 99999999;
    for (int i = 0; i < count; i++) {
      defineAnonymousClass(Test.class, bytecode);
    }
  }
}


-------------------------------------------------
JVMTI agent code
-------------------------------------------------

#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

#include <jni.h>
#include <jvmti.h>

static jvmtiEnv* ourJVMTI;

#define ZERO(X) memset(&X, 0, sizeof(X))

static void JNICALL class_file_load_hook_handler(
  jvmtiEnv* jvmti,
  JNIEnv* jni,
  jclass class_being_redefined,
  jobject loader,
  const char* name,
  jobject protection_domain,
  jint class_data_len,
  const unsigned char* class_data,
  jint* new_class_data_len,
  unsigned char** new_class_data
) {
  *new_class_data_len = 0;
  *new_class_data = NULL;

  // return exact copy of the original bytecode

  unsigned char* newData = NULL;

  jint rc = ourJVMTI->Allocate(class_data_len, &newData);
  assert(rc == JVMTI_ERROR_NONE);
  assert(newData);

  for (jint i=0; i < class_data_len; i++){
    newData[i] = class_data[i];
  }

  *new_class_data_len = class_data_len;
  *new_class_data = newData;
}

extern "C" JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM* vm, char* options, void* /*reserved*/) {
  // create environment
  {
    jint rc = vm->GetEnv((void **)&ourJVMTI, JVMTI_VERSION);
    assert(rc == JNI_OK);
    assert(ourJVMTI);
  }

  // set capabilities
  {
    jvmtiCapabilities capabilities;
    ZERO(capabilities);
    capabilities.can_retransform_classes = 1; // IMPORTANT

    jint rc = ourJVMTI->AddCapabilities(&capabilities);
    assert(rc == JVMTI_ERROR_NONE);
  }

  // set callbacks
  {
    jvmtiEventCallbacks callbacks;
    ZERO(callbacks);

    callbacks.ClassFileLoadHook = &class_file_load_hook_handler;
    jint rc = ourJVMTI->SetEventCallbacks(&callbacks, sizeof(callbacks));
    assert(rc == JVMTI_ERROR_NONE);
  }

  // turn notification on
  {
    jint rc = ourJVMTI->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, NULL);
    assert(rc == JVMTI_ERROR_NONE);
  }

  fprintf(stderr,"Agent loaded\n");
  fflush(stderr);

  return 0;
}

---------- END SOURCE ----------