8341767: [lworld] JIT support for nullable, atomic, flat arrays

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -2333,6 +2333,8 @@ void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeS
   }
   if (is_dest) {
     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
+    // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
+    __ test_flat_array_oop(obj, tmp, *slow_path->entry());
   } else {
     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
   }
@@ -2418,7 +2420,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
   }
-
   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
   }

src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad

@@ -77,7 +77,7 @@ static void write_barrier_post(MacroAssembler* masm,
 
 %}
 
-// TODO 8341767 (same applies to g1StoreLSpecialTwoOops)
+// TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
 // - Can we use an unbound register for src?
 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
 // - Is the zero-extend really required in all the places?

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -1953,6 +1953,7 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
   // If this array is a flat array, take the slow path.
   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
   if (!op->value()->is_illegal()) {
+    // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
     // The array is not a flat array, but it might be null-free. If we are storing
     // a null into a null-free array, take the slow path (which will throw NPE).
     Label skip;
@@ -3188,6 +3189,8 @@ void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeS
   }
   if (is_dest) {
     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
+    // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
+    __ test_flat_array_oop(obj, tmp, *slow_path->entry());
   } else {
     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
   }

src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad

@@ -105,7 +105,7 @@ instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3,
   ins_pipe(ialu_mem_reg);
 %}
 
-// TODO 8341767 (same applies to g1StoreLSpecialTwoOops)
+// TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
 // - Can we use an unbound register for src?
 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
 // - Is the zero-extend really required in all the places?

src/hotspot/share/c1/c1_GraphBuilder.cpp

@@ -1085,6 +1085,7 @@ void GraphBuilder::load_indexed(BasicType type) {
   LoadIndexed* load_indexed = nullptr;
   Instruction* result = nullptr;
   if (array->is_loaded_flat_array()) {
+    // TODO 8350865 This is currently dead code
     ciType* array_type = array->declared_type();
     ciInlineKlass* elem_klass = array_type->as_flat_array_klass()->element_klass()->as_inline_klass();
 

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -1674,7 +1674,7 @@ void LIRGenerator::do_StoreField(StoreField* x) {
 #endif
 
     // Zero the payload
-    BasicType bt = vk->payload_size_to_basic_type();
+    BasicType bt = vk->atomic_size_to_basic_type(field->is_null_free());
     LIR_Opr payload = new_register((bt == T_LONG) ? bt : T_INT);
     LIR_Opr zero = (bt == T_LONG) ? LIR_OprFact::longConst(0) : LIR_OprFact::intConst(0);
     __ move(zero, payload);
@@ -1936,7 +1936,7 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
   }
 
   if (x->should_profile()) {
-    if (x->array()->is_loaded_flat_array()) {
+    if (is_loaded_flat_array) {
       // No need to profile a store to a flat array of known type. This can happen if
       // the type only became known after optimizations (for example, after the PhiSimplifier).
       x->set_should_profile(false);
@@ -1961,6 +1961,7 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
   }
 
   if (is_loaded_flat_array) {
+    // TODO 8350865 This is currently dead code
     if (!x->value()->is_null_free()) {
       __ null_check(value.result(), new CodeEmitInfo(range_check_info));
     }
@@ -2145,7 +2146,7 @@ void LIRGenerator::do_LoadField(LoadField* x) {
     LIRItem dest(buffer, this);
 
     // Copy the payload to the buffer
-    BasicType bt = vk->payload_size_to_basic_type();
+    BasicType bt = vk->atomic_size_to_basic_type(field->is_null_free());
     LIR_Opr payload = new_register((bt == T_LONG) ? bt : T_INT);
     access_load_at(decorators, bt, object, LIR_OprFact::intConst(field->offset_in_bytes()), payload,
                    // Make sure to emit an implicit null check

src/hotspot/share/c1/c1_Runtime1.cpp

@@ -433,6 +433,7 @@ JRT_END
 
 JRT_ENTRY(void, Runtime1::new_null_free_array(JavaThread* current, Klass* array_klass, jint length))
   NOT_PRODUCT(_new_null_free_array_slowcase_cnt++;)
+  // TODO 8350865 This is dead code since 8325660 because null-free arrays can only be created via the factory methods that are not yet implemented in C1. Should probably be fixed by 8265122.
 
   // Note: no handle for klass needed since they are not used
   //       anymore after new_objArray() and no GC can happen before.
@@ -445,7 +446,6 @@ JRT_ENTRY(void, Runtime1::new_null_free_array(JavaThread* current, Klass* array_
   // Logically creates elements, ensure klass init
   elem_klass->initialize(CHECK);
   arrayOop obj= nullptr;
-  //  Limitation here, only non-atomic layouts are supported
   if (UseArrayFlattening && vk->has_non_atomic_layout()) {
     obj = oopFactory::new_flatArray(elem_klass, length, LayoutKind::NON_ATOMIC_FLAT, CHECK);
   } else {
@@ -474,7 +474,7 @@ JRT_ENTRY(void, Runtime1::new_multi_array(JavaThread* current, Klass* klass, int
 JRT_END
 
 
-static void profile_flat_array(JavaThread* current, bool load) {
+static void profile_flat_array(JavaThread* current, bool load, bool null_free) {
   ResourceMark rm(current);
   vframeStream vfst(current, true);
   assert(!vfst.at_end(), "Java frame must exist");
@@ -495,18 +495,24 @@ static void profile_flat_array(JavaThread* current, bool load) {
       assert(load, "should be an array load");
       ArrayLoadData* load_data = (ArrayLoadData*) data;
       load_data->set_flat_array();
+      if (null_free) {
+        load_data->set_null_free_array();
+      }
     } else {
       assert(data->is_ArrayStoreData(), "");
       assert(!load, "should be an array store");
       ArrayStoreData* store_data = (ArrayStoreData*) data;
       store_data->set_flat_array();
+      if (null_free) {
+        store_data->set_null_free_array();
+      }
     }
   }
 }
 
 JRT_ENTRY(void, Runtime1::load_flat_array(JavaThread* current, flatArrayOopDesc* array, int index))
   assert(array->klass()->is_flatArray_klass(), "should not be called");
-  profile_flat_array(current, true);
+  profile_flat_array(current, true, array->is_null_free_array());
 
   NOT_PRODUCT(_load_flat_array_slowcase_cnt++;)
   assert(array->length() > 0 && index < array->length(), "already checked");
@@ -515,23 +521,21 @@ JRT_ENTRY(void, Runtime1::load_flat_array(JavaThread* current, flatArrayOopDesc*
   current->set_vm_result(obj);
 JRT_END
 
-
 JRT_ENTRY(void, Runtime1::store_flat_array(JavaThread* current, flatArrayOopDesc* array, int index, oopDesc* value))
+  // TOOD 8350865 We can call here with a non-flat array because of LIR_Assembler::emit_opFlattenedArrayCheck
   if (array->klass()->is_flatArray_klass()) {
-    profile_flat_array(current, false);
+    profile_flat_array(current, false, array->is_null_free_array());
   }
 
   NOT_PRODUCT(_store_flat_array_slowcase_cnt++;)
-  if (value == nullptr) {
-    assert(array->klass()->is_flatArray_klass() || array->klass()->is_null_free_array_klass(), "should not be called");
+  if (value == nullptr && array->is_null_free_array()) {
     SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException());
   } else {
     assert(array->klass()->is_flatArray_klass(), "should not be called");
     array->write_value_to_flat_array(value, index, CHECK);
   }
 JRT_END
 
-
 JRT_ENTRY(int, Runtime1::substitutability_check(JavaThread* current, oopDesc* left, oopDesc* right))
   NOT_PRODUCT(_substitutability_check_slowcase_cnt++;)
   JavaCallArguments args;

src/hotspot/share/ci/ciArray.cpp

@@ -119,6 +119,11 @@ ciConstant ciArray::element_value_by_offset(intptr_t element_offset) {
   return element_value((jint) index);
 }
 
+bool ciArray::is_null_free() {
+  VM_ENTRY_MARK;
+  return get_oop()->is_null_free_array();
+}
+
 // ------------------------------------------------------------------
 // ciArray::print_impl
 //

src/hotspot/share/ci/ciArray.hpp

@@ -74,7 +74,7 @@ class ciArray : public ciObject {
   bool is_array()        { return true; }
 
   virtual bool is_flat()      { return false; }
-  virtual bool is_null_free() { return false; }
+  virtual bool is_null_free();
 };
 
 #endif // SHARE_CI_CIARRAY_HPP

src/hotspot/share/ci/ciArrayKlass.cpp

@@ -101,24 +101,44 @@ bool ciArrayKlass::is_leaf_type() {
 // ciArrayKlass::make
 //
 // Make an array klass of the specified element type.
-ciArrayKlass* ciArrayKlass::make(ciType* element_type, bool null_free) {
+ciArrayKlass* ciArrayKlass::make(ciType* element_type, bool flat, bool null_free, bool atomic) {
   if (element_type->is_primitive_type()) {
     return ciTypeArrayKlass::make(element_type->basic_type());
   }
 
   ciKlass* klass = element_type->as_klass();
   assert(!null_free || !klass->is_loaded() || klass->is_inlinetype() || klass->is_abstract() ||
          klass->is_java_lang_Object(), "only value classes are null free");
-  if (null_free && klass->is_loaded() && klass->is_inlinetype()) {
+  if (klass->is_loaded() && klass->is_inlinetype()) {
     GUARDED_VM_ENTRY(
       EXCEPTION_CONTEXT;
       Klass* ak = nullptr;
       InlineKlass* vk = InlineKlass::cast(klass->get_Klass());
-      if (vk->flat_array()) {
-        // Current limitation: returns only non-atomic flat arrays, atomic layout not supported here
-        ak = vk->flat_array_klass(LayoutKind::NON_ATOMIC_FLAT, THREAD);
-      } else {
+      if (flat && vk->flat_array()) {
+        LayoutKind lk;
+        if (null_free) {
+          if (vk->is_naturally_atomic()) {
+            atomic = vk->has_atomic_layout();
+          }
+          if (!atomic && !vk->has_non_atomic_layout()) {
+            // TODO 8350865 Impossible type
+            lk = vk->has_atomic_layout() ? LayoutKind::ATOMIC_FLAT : LayoutKind::NULLABLE_ATOMIC_FLAT;
+          } else {
+            lk = atomic ? LayoutKind::ATOMIC_FLAT : LayoutKind::NON_ATOMIC_FLAT;
+          }
+        } else {
+          if (!vk->has_nullable_atomic_layout()) {
+            // TODO 8350865 Impossible type, null-able flat is always atomic.
+            lk = vk->has_atomic_layout() ? LayoutKind::ATOMIC_FLAT : LayoutKind::NON_ATOMIC_FLAT;
+          } else {
+            lk = LayoutKind::NULLABLE_ATOMIC_FLAT;
+          }
+        }
+        ak = vk->flat_array_klass(lk, THREAD);
+      } else if (null_free) {
         ak = vk->null_free_reference_array(THREAD);
+      } else {
+        return ciObjArrayKlass::make(klass);
       }
       if (HAS_PENDING_EXCEPTION) {
         CLEAR_PENDING_EXCEPTION;
@@ -143,3 +163,6 @@ ciInstance* ciArrayKlass::component_mirror_instance() const {
   )
 }
 
+bool ciArrayKlass::is_elem_null_free() const {
+  GUARDED_VM_ENTRY(return !is_type_array_klass() && get_Klass()->is_null_free_array_klass();)
+}

src/hotspot/share/ci/ciArrayKlass.hpp

@@ -58,12 +58,12 @@ class ciArrayKlass : public ciKlass {
   // The one-level type of the array elements.
   virtual ciKlass* element_klass() { return nullptr; }
 
-  static ciArrayKlass* make(ciType* klass, bool null_free = false);
+  static ciArrayKlass* make(ciType* klass, bool flat = false, bool null_free = false, bool atomic = false);
 
   int array_header_in_bytes();
   ciInstance* component_mirror_instance() const;
 
-  virtual bool is_elem_null_free() const { return false; }
+  bool is_elem_null_free() const;
 };
 
 #endif // SHARE_CI_CIARRAYKLASS_HPP

src/hotspot/share/ci/ciFlatArray.hpp

@@ -42,7 +42,6 @@ class ciFlatArray : public ciArray {
 
 public:
   bool is_flat()      { return true; }
-  bool is_null_free() { return true; }
 };
 
 #endif // SHARE_VM_CI_CIFLATARRAY_HPP

src/hotspot/share/ci/ciFlatArrayKlass.hpp

@@ -71,8 +71,6 @@ class ciFlatArrayKlass : public ciArrayKlass {
   virtual bool can_be_inline_array_klass() {
     return true;
   }
-
-  virtual bool is_elem_null_free() const { return true; }
 };
 
 

src/hotspot/share/ci/ciInlineKlass.cpp

@@ -141,10 +141,29 @@ InlineKlass* ciInlineKlass::get_InlineKlass() const {
   GUARDED_VM_ENTRY(return to_InlineKlass();)
 }
 
-// Convert payload size in bytes to corresponding BasicType
-BasicType ciInlineKlass::payload_size_to_basic_type() const {
+bool ciInlineKlass::has_non_atomic_layout() const {
+  GUARDED_VM_ENTRY(return get_InlineKlass()->has_non_atomic_layout();)
+}
+
+bool ciInlineKlass::has_atomic_layout() const {
+  GUARDED_VM_ENTRY(return get_InlineKlass()->has_atomic_layout();)
+}
+
+bool ciInlineKlass::has_nullable_atomic_layout() const {
+  GUARDED_VM_ENTRY(return get_InlineKlass()->has_nullable_atomic_layout();)
+}
+
+int ciInlineKlass::null_marker_offset_in_payload() const {
+  GUARDED_VM_ENTRY(return get_InlineKlass()->null_marker_offset_in_payload();)
+}
+
+// Convert size of atomic layout in bytes to corresponding BasicType
+BasicType ciInlineKlass::atomic_size_to_basic_type(bool null_free) const {
   VM_ENTRY_MARK
-  int size = get_InlineKlass()->payload_size_in_bytes();
+  InlineKlass* vk = get_InlineKlass();
+  assert(!null_free || vk->has_atomic_layout(), "No null-free atomic layout available");
+  assert( null_free || vk->has_nullable_atomic_layout(), "No nullable atomic layout available");
+  int size = null_free ? vk->atomic_size_in_bytes() : vk->nullable_atomic_size_in_bytes();
   BasicType bt;
   if (size == sizeof(jlong)) {
     bt = T_LONG;

src/hotspot/share/ci/ciInlineKlass.hpp

@@ -89,7 +89,11 @@ class ciInlineKlass : public ciInstanceKlass {
   address unpack_handler() const;
   InlineKlass* get_InlineKlass() const;
   int nullable_size_in_bytes() const;
-  BasicType payload_size_to_basic_type() const;
+  bool has_non_atomic_layout() const;
+  bool has_atomic_layout() const;
+  bool has_nullable_atomic_layout() const;
+  int null_marker_offset_in_payload() const;
+  BasicType atomic_size_to_basic_type(bool null_free) const;
 };
 
 #endif // SHARE_VM_CI_CIINLINEKLASS_HPP

src/hotspot/share/ci/ciObjArray.cpp

@@ -44,8 +44,3 @@ ciObject* ciObjArray::obj_at(int index) {
     return CURRENT_ENV->get_object(o);
   }
 }
-
-bool ciObjArray::is_null_free() {
-  VM_ENTRY_MARK;
-  return get_objArrayOop()->is_null_free_array();
-}

src/hotspot/share/ci/ciObjArray.hpp

@@ -52,7 +52,6 @@ class ciObjArray : public ciArray {
   ciObject* obj_at(int index);
 
   bool is_flat()      { return false; }
-  bool is_null_free();
 };
 
 #endif // SHARE_CI_CIOBJARRAY_HPP

src/hotspot/share/ci/ciObjArrayKlass.cpp

@@ -193,7 +193,3 @@ ciKlass* ciObjArrayKlass::exact_klass() {
   }
   return nullptr;
 }
-
-bool ciObjArrayKlass::is_elem_null_free() const {
-  GUARDED_VM_ENTRY(return get_Klass()->is_null_free_array_klass();)
-}

src/hotspot/share/ci/ciObjArrayKlass.hpp

@@ -76,8 +76,6 @@ class ciObjArrayKlass : public ciArrayKlass {
   virtual bool can_be_inline_array_klass() {
     return element_klass()->can_be_inline_klass();
   }
-
-  virtual bool is_elem_null_free() const;
 };
 
 #endif // SHARE_CI_CIOBJARRAYKLASS_HPP

src/hotspot/share/ci/ciReplay.cpp

@@ -512,7 +512,7 @@ class CompileReplay : public StackObj {
         return k;
       }
       obj = ciReplay::obj_field(obj, field);
-      // array
+      // TODO 8350865 I think we need to handle null-free/flat arrays here
       if (obj != nullptr && obj->is_objArray()) {
         objArrayOop arr = (objArrayOop)obj;
         int index = parse_int("index");
@@ -1131,6 +1131,7 @@ class CompileReplay : public StackObj {
           } else if (field_signature[0] == JVM_SIGNATURE_ARRAY &&
                      field_signature[1] == JVM_SIGNATURE_CLASS) {
             Klass* actual_array_klass = parse_klass(CHECK_(true));
+            // TODO 8350865 I think we need to handle null-free/flat arrays here
             Klass* kelem = ObjArrayKlass::cast(actual_array_klass)->element_klass();
             value = oopFactory::new_objArray(kelem, length, CHECK_(true));
           } else {

src/hotspot/share/classfile/vmIntrinsics.cpp

@@ -262,6 +262,8 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
   case vmIntrinsics::_getLength:
   case vmIntrinsics::_newArray:
   case vmIntrinsics::_newNullRestrictedArray:
+  case vmIntrinsics::_newNullRestrictedAtomicArray:
+  case vmIntrinsics::_newNullableAtomicArray:
   case vmIntrinsics::_getClass:
     if (!InlineClassNatives) return true;
     break;