This issue was reported by Michael Rasmussen. Following are copied from the original email:

I have some questions regarding Class Data Sharing and class file transforming agents:

1. Is it possible to detect during Agent_OnLoad if CDS is used or not?
Checking for "sharing" in the "java.vm.info" system property, which seems to be how the tests are detecting it, is not an option, as this property contains bogus information during Agent_OnLoad.

2. Should the registration of a ClassFileHook, especially with can_generate_all_class_hook_events and can_generate_early_class_hook_events capabilities set, disable CDS?

3. Is it possible to explicitly disable CDS from a JVM-TI agent during Agent_OnLoad?


The reason for these questions is that we (JRebel) recently became aware of an issue if CDS is used, where transforming some of the boot classes causes the JVM to crash.
The simplest example of this is replacing Object's byte in a ClassFileLoadHook with a copy of themselves, see inlined example agent code below.
I don't know if there are more issues than that, as that's where we are currently stuck.

Running on a 11+28 fastdebug (with classes.jsa file present) it produces the following assert:
# Internal Error (.../src/hotspot/share/oops/constantPool.cpp:325), pid=589, tid=590
# assert(on_stack()) failed: should always be set for shared constant pools

if running with -Xshare:off (or no classes.jsa file), it works without any issue.

Kind regards
Michael Rasmussen

/* --- snip --- */
#include <string.h>
#include <jni.h>
#include <jvmti.h>

void JNICALL callback_ClassFileLoadHook(jvmtiEnv *jvmti, JNIEnv* env,
jclass class_being_redefined, jobject loader, const char* name, jobject protection_domain,
jint class_data_len, const unsigned char* class_data,
jint* new_class_data_len, unsigned char** new_class_data) {

if (strcmp("java/lang/Object", name) == 0) {
*new_class_data_len = class_data_len;
(*jvmti)->Allocate(jvmti, *new_class_data_len, new_class_data);
memcpy(*new_class_data, class_data, *new_class_data_len);
}
}

JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
jvmtiEnv *jvmti = NULL;
jint rc = (*vm)->GetEnv(vm, (void **)&jvmti, JVMTI_VERSION_9);

jvmtiCapabilities caps;
memset(&caps, 0, sizeof(caps));

caps.can_redefine_classes = 1;
caps.can_generate_all_class_hook_events = 1;
caps.can_generate_early_vmstart = 1;
caps.can_generate_early_class_hook_events = 1;

(*jvmti)->AddCapabilities(jvmti, &caps);

jvmtiEventCallbacks callbacks;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.ClassFileLoadHook = &callback_ClassFileLoadHook;
(*jvmti)->SetEventCallbacks(jvmti, &callbacks, sizeof(callbacks));
(*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, NULL);

return 0;
}
/* --- snap --- */

Following is another issue reported by Michael Rasmussen. If needed, we can split that into a separate bug.

I did a small test (attached below) with adding a default method to Serializable, and it crashes as well, though differently -- don't know if what Volker mentioned fixes this though.

Running the below with CDS off, I get:
$ ../jdk/bin/java -Xshare:off -agentpath:./libagent.so Test.java
public default java.lang.Object java.io.Serializable._$_$_blah()

With it on, I get the following crash:
$ ../jdk/bin/java -Xshare:on -agentpath:./libagent.so Test.java
# Internal Error (.../src/hotspot/share/classfile/javaClasses.cpp:667), pid=277, tid=278
# assert(java_string->klass() == SystemDictionary::String_klass()) failed: must be java_string

Stack trace:
V [libjvm.so+0xe80ea5] java_lang_String::equals(oop, unsigned short*, int)+0x85
V [libjvm.so+0x9e6e44] CompactHashtable<oop, char>::decode_entry(CompactHashtable<oop, char>*, unsigned int, char const*, int)+0x54
V [libjvm.so+0x167fae6] StringTable::lookup_shared(unsigned short*, int, unsigned int)+0x166
V [libjvm.so+0x16801df] StringTable::intern(Handle, unsigned short*, int, Thread*)+0x6f
V [libjvm.so+0x16802e3] StringTable::intern(Symbol*, Thread*)+0x53
V [libjvm.so+0xae9921] ConstantPool::uncached_string_at(int, Thread*)+0x31
V [libjvm.so+0xc4426c] fieldDescriptor::string_initial_value(Thread*) const+0x9c
V [libjvm.so+0xe88056] initialize_static_field(fieldDescriptor*, Handle, Thread*)+0x1c6
V [libjvm.so+0xe23819] InstanceKlass::do_local_static_fields(void (*)(fieldDescriptor*, Handle, Thread*), Handle, Thread*)+0xd9
V [libjvm.so+0xe70157] java_lang_Class::initialize_mirror_fields(Klass*, Handle, Handle, Thread*)+0x117
V [libjvm.so+0xe723c3] java_lang_Class::create_mirror(Klass*, Handle, Handle, Handle, Thread*)+0x723
V [libjvm.so+0x95b8fd] ClassFileParser::fill_instance_klass(InstanceKlass*, bool, Thread*)+0x1c8d
V [libjvm.so+0x95c12f] ClassFileParser::create_instance_klass(bool, Thread*)+0x5f
V [libjvm.so+0x118e5f5] KlassFactory::create_from_stream(ClassFileStream*, Symbol*, ClassLoaderData*, Handle, InstanceKlass const*, GrowableArray<Handle>*, Thread*)+0x615
V [libjvm.so+0x96b76c] ClassLoader::load_class(Symbol*, bool, Thread*)+0x1bc
V [libjvm.so+0x171a3e6] SystemDictionary::load_instance_class(Symbol*, Handle, Thread*)+0x5f6
V [libjvm.so+0x1718139] SystemDictionary::resolve_instance_class_or_null(Symbol*, Handle, Handle, Thread*)+0xa69
V [libjvm.so+0x1718662] SystemDictionary::resolve_or_null(Symbol*, Handle, Handle, Thread*)+0x42
V [libjvm.so+0x17187de] SystemDictionary::resolve_or_fail(Symbol*, Handle, Handle, bool, Thread*)+0x1e
V [libjvm.so+0x1718991] SystemDictionary::initialize_wk_klass(SystemDictionary::WKID, int, Thread*)+0x151
V [libjvm.so+0x1718af1] SystemDictionary::initialize_wk_klasses_until(SystemDictionary::WKID, SystemDictionary::WKID&, Thread*)+0x61
V [libjvm.so+0x1718d33] SystemDictionary::initialize_preloaded_classes(Thread*)+0x153
V [libjvm.so+0x17190ea] SystemDictionary::initialize(Thread*)+0x21a
V [libjvm.so+0x17a8140] Universe::genesis(Thread*)+0x5f0
V [libjvm.so+0x17a8b5c] universe2_init()+0x2c
V [libjvm.so+0xe1e558] init_globals()+0xa8
V [libjvm.so+0x1766d17] Threads::create_vm(JavaVMInitArgs*, bool*)+0x2d7


Sample code:
/* --- snip ---- */
#include <string.h>
#include <jni.h>
#include <jvmti.h>

static char *serializable_class_name = "java/io/Serializable";
/*
  public interface java.io.Serializable {
    default Object _$_$_blah() { return this; }
  }
*/
static jbyte serializable_class_bytes[] = {
202, 254, 186, 190, 0, 0, 0, 55, 0, 10, 1, 0, 20, 106, 97, 118,
97, 47, 105, 111, 47, 83, 101, 114, 105, 97, 108, 105, 122, 97, 98, 108,
101, 7, 0, 1, 1, 0, 16, 106, 97, 118, 97, 47, 108, 97, 110, 103,
47, 79, 98, 106, 101, 99, 116, 7, 0, 3, 1, 0, 17, 83, 101, 114,
105, 97, 108, 105, 122, 97, 98, 108, 101, 46, 106, 97, 118, 97, 1, 0,
9, 95, 36, 95, 36, 95, 98, 108, 97, 104, 1, 0, 20, 40, 41, 76,
106, 97, 118, 97, 47, 108, 97, 110, 103, 47, 79, 98, 106, 101, 99, 116,
59, 1, 0, 4, 67, 111, 100, 101, 1, 0, 10, 83, 111, 117, 114, 99,
101, 70, 105, 108, 101, 6, 1, 0, 2, 0, 4, 0, 0, 0, 0, 0,
1, 0, 1, 0, 6, 0, 7, 0, 1, 0, 8, 0, 0, 0, 14, 0,
1, 0, 1, 0, 0, 0, 2, 42, 176, 0, 0, 0, 0, 0, 1, 0,
9, 0, 0, 0, 2, 0, 5
};
static jsize serializable_class_bytes_len = 183;

void JNICALL callback_ClassFileLoadHook(jvmtiEnv *jvmti, JNIEnv* env,
jclass class_being_redefined, jobject loader, const char* name, jobject protection_domain,
jint class_data_len, const unsigned char* class_data,
jint* new_class_data_len, unsigned char** new_class_data) {

if (strcmp(serializable_class_name, name) == 0) {
*new_class_data_len = serializable_class_bytes_len;
(*jvmti)->Allocate(jvmti, *new_class_data_len, new_class_data);
memcpy(*new_class_data, serializable_class_bytes, *new_class_data_len);
}
}

JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
jvmtiEnv *jvmti;
jint rc = (*vm)->GetEnv(vm, (void **)&jvmti, JVMTI_VERSION_9);

jvmtiCapabilities caps;
memset(&caps, 0, sizeof(caps));

caps.can_redefine_classes = 1;
caps.can_generate_all_class_hook_events = 1;
caps.can_generate_early_vmstart = 1;
caps.can_generate_early_class_hook_events = 1;

(*jvmti)->AddCapabilities(jvmti, &caps);

jvmtiEventCallbacks callbacks;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.ClassFileLoadHook = &callback_ClassFileLoadHook;
(*jvmti)->SetEventCallbacks(jvmti, &callbacks, sizeof(callbacks));
(*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, NULL);

return 0;
}
/* ---- snap --- */
import java.io.Serializable;
import java.util.stream.Stream;

public class Test {
  public static void main(String[] args) {
    Stream.of(Serializable.class.getDeclaredMethods()).forEach(System.out::println);
  }
}
/* ---- snude ---- */