8382713: [VectorAPI] Perform late inlining of failed vector intrinsics

src/hotspot/share/opto/callGenerator.cpp

@@ -437,6 +437,28 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal
   return cg;
 }
 
+class LateInlineVectorCallGenerator : public LateInlineCallGenerator {
+ private:
+  CallGenerator* _fallback_cg;
+
+ public:
+  LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) :
+    LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {}
+
+  virtual bool is_vector_late_inline() const { return true; }
+  virtual CallGenerator* fallback_inline_cg() const { return _fallback_cg; }
+
+  virtual JVMState* generate(JVMState* jvms) {
+    Compile::current()->vector_late_inlines()->append(this);
+    return LateInlineCallGenerator::generate(jvms);
+  }
+};
+
+CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) {
+  return new LateInlineVectorCallGenerator(m, intrinsic_cg, fallback_cg);
+}
+
+
 // Allow inlining decisions to be delayed
 class LateInlineVirtualCallGenerator : public VirtualCallGenerator {
  private:

src/hotspot/share/opto/callGenerator.hpp

@@ -75,6 +75,8 @@ class CallGenerator : public ArenaObj {
   // same but for method handle calls
   virtual bool      is_mh_late_inline() const      { return false; }
   virtual bool      is_string_late_inline() const  { return false; }
+  virtual bool      is_vector_late_inline() const  { return false; }
+  virtual CallGenerator* fallback_inline_cg() const { return nullptr; }
   virtual bool      is_boxing_late_inline() const  { return false; }
   virtual bool      is_vector_reboxing_late_inline() const  { return false; }
   virtual bool      is_virtual_late_inline() const { return false; }
@@ -142,6 +144,7 @@ class CallGenerator : public ArenaObj {
   static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg);
   static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const);
   static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg);
+  static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg);
   static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
   static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
   static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses);

src/hotspot/share/opto/compile.cpp

@@ -416,6 +416,7 @@ void Compile::remove_useless_node(Node* dead) {
     remove_useless_late_inlines(                &_late_inlines, dead);
     remove_useless_late_inlines(         &_string_late_inlines, dead);
     remove_useless_late_inlines(         &_boxing_late_inlines, dead);
+    remove_useless_late_inlines(         &_vector_late_inlines, dead);
     remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
 
     if (dead->is_CallStaticJava()) {
@@ -480,6 +481,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
   remove_useless_late_inlines(                &_late_inlines, useful);
   remove_useless_late_inlines(         &_string_late_inlines, useful);
   remove_useless_late_inlines(         &_boxing_late_inlines, useful);
+  remove_useless_late_inlines(         &_vector_late_inlines, useful);
   remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
   DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
 }
@@ -693,6 +695,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
       _string_late_inlines(comp_arena(), 2, 0, nullptr),
       _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
       _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr),
+      _vector_late_inlines(comp_arena(), 2, 0, nullptr),
       _late_inlines_pos(0),
       _has_mh_late_inlines(false),
       _oom(false),
@@ -2229,6 +2232,26 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) {
     inline_incrementally_cleanup(igvn);
   }
 
+  if (_vector_late_inlines.length() > 0) {
+    while (_vector_late_inlines.length() > 0) {
+      CallGenerator* cg = _vector_late_inlines.pop();
+      CallGenerator* fallback = CallGenerator::for_late_inline(cg->method(), cg->fallback_inline_cg());
+      fallback = fallback->with_call_node(cg->call_node());
+      add_late_inline(fallback);
+    }
+
+    while (_late_inlines.length() > 0) {
+      igvn_worklist()->ensure_empty();
+
+      while (inline_incrementally_one()) {
+        assert(!failing_internal() || failure_is_artificial(), "inconsistent");
+      }
+      if (failing())  return;
+
+      inline_incrementally_cleanup(igvn);
+    }
+  }
+
   set_inlining_incrementally(false);
 }
 

src/hotspot/share/opto/compile.hpp

@@ -480,6 +480,7 @@ class Compile : public Phase {
   GrowableArray<CallGenerator*> _boxing_late_inlines; // same but for boxing operations
 
   GrowableArray<CallGenerator*> _vector_reboxing_late_inlines; // same but for vector reboxing operations
+  GrowableArray<CallGenerator*> _vector_late_inlines; // inline fallback implementation for failed intrinsics
 
   int                           _late_inlines_pos;    // Where in the queue should the next late inlining candidate go (emulate depth first inlining)
   bool                          _has_mh_late_inlines; // Can there still be a method handle late inlining pending?
@@ -508,6 +509,11 @@ class Compile : public Phase {
   InlinePrinter _inline_printer;
 
 public:
+
+  GrowableArray<CallGenerator*>* vector_late_inlines() {
+    return &_vector_late_inlines;
+  }
+
   void* barrier_set_state() const { return _barrier_set_state; }
 
   InlinePrinter* inline_printer() { return &_inline_printer; }

src/hotspot/share/opto/doCall.cpp

@@ -166,7 +166,9 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
         cg_intrinsic = cg;
         cg = nullptr;
       } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) {
-        return CallGenerator::for_late_inline(callee, cg);
+        float expected_uses = jvms->method()->scale_count(site_count, prof_factor);
+        CallGenerator* inline_cg = CallGenerator::for_inline(callee, expected_uses);
+        return CallGenerator::for_vector_late_inline(callee, cg, inline_cg);
       } else {
         return cg;
       }