I'm not sure about the API Instrumentation, but one possible way is to use the JVMTI function GetClassStatus.
, JVM, JVMTI_CLASS_STATUS_INITIALIZED
JavaVM *jvm;
jvmtiEnv *jvmti;
jvmtiError err;
env->GetJavaVM(&jvm);
jvm->GetEnv((void **) &jvmti, JVMTI_VERSION_1_2);
jint classCount = 0;
jclass * classes;
jvmti->GetLoadedClasses(&classCount, &classes);
for (int i = 0; i < classCount; i++) {
jint classStatus = 0;
jvmti->GetClassStatus(classes[i], &classStatus);
if (classStatus != JVMTI_CLASS_STATUS_PRIMITIVE
&& classStatus != JVMTI_CLASS_STATUS_ARRAY
&& classStatus != JVMTI_CLASS_STATUS_ERROR
&& !(classStatus & JVMTI_CLASS_STATUS_INITIALIZED)) {
}
}