Skip to content

8382713: [VectorAPI] Perform late inlining of failed vector intrinsics #30876

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jatin-bhateja wants to merge 10 commits into
base: master
Choose a base branch
from
Open

8382713: [VectorAPI] Perform late inlining of failed vector intrinsics #30876

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c7e6fce
8382713: [VectorAPI] Perform late inlining of failed vector intrinsics
jatin-bhateja Apr 22, 2026
931d45e
Review comments resolutions
jatin-bhateja Apr 27, 2026
e779a2f
Review comments resolutions
jatin-bhateja Apr 28, 2026
5f46f5b
Review comments resolutions
jatin-bhateja May 5, 2026
d18bd2a
Review comments resolution
jatin-bhateja May 6, 2026
8171911
Review comment resolution
jatin-bhateja May 8, 2026
679e444
Review comments resolution
jatin-bhateja May 12, 2026
dc3ffe8
Review comments resolution
jatin-bhateja May 18, 2026
77150d9
Review comments resolutions
jatin-bhateja May 19, 2026
ae4a373
Review comments resolutions
jatin-bhateja May 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions src/hotspot/share/opto/callGenerator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,65 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal
return cg;
}

class LateInlineVectorCallGenerator : public LateInlineCallGenerator {
protected:
bool _use_fallback_generator;
CallGenerator* _inline_cg2;

public:
LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) :
LateInlineCallGenerator(method, intrinsic_cg) , _inline_cg2(inline_cg) {
_use_fallback_generator = false;
}

virtual CallGenerator* inline_cg() const {
return _use_fallback_generator ? _inline_cg2 : _inline_cg;
}
virtual bool inline_fallback() const;
virtual bool is_vector_late_inline() const { return true; }
virtual void enable_fallback_generation() {
_use_fallback_generator = true;
}
};

bool LateInlineVectorCallGenerator::inline_fallback() const {
Copy link
Copy Markdown
Contributor

@iwanowww iwanowww Apr 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the purpose of this method? All vector intrinsics do have fallback implementation. If there are any cases added later, then they don't have to rely on LateInlineVectorCallGenerator.

switch (method()->intrinsic_id()) {
case vmIntrinsics::_VectorUnaryOp:
case vmIntrinsics::_VectorBinaryOp:
case vmIntrinsics::_VectorUnaryLibOp:
case vmIntrinsics::_VectorBinaryLibOp:
case vmIntrinsics::_VectorTernaryOp:
case vmIntrinsics::_VectorSelectFromTwoVectorOp:
case vmIntrinsics::_VectorFromBitsCoerced:
case vmIntrinsics::_VectorLoadOp:
case vmIntrinsics::_VectorLoadMaskedOp:
case vmIntrinsics::_VectorStoreOp:
case vmIntrinsics::_VectorStoreMaskedOp:
case vmIntrinsics::_VectorGatherOp:
case vmIntrinsics::_VectorScatterOp:
case vmIntrinsics::_VectorReductionCoerced:
case vmIntrinsics::_VectorTest:
case vmIntrinsics::_VectorBlend:
case vmIntrinsics::_VectorCompare:
case vmIntrinsics::_VectorRearrange:
case vmIntrinsics::_VectorSelectFrom:
case vmIntrinsics::_VectorExtract:
case vmIntrinsics::_VectorInsert:
case vmIntrinsics::_VectorBroadcastInt:
case vmIntrinsics::_VectorConvert:
case vmIntrinsics::_VectorMaskOp:
case vmIntrinsics::_VectorCompressExpand:
return true;
default:
return false;
}
}

CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) {
return new LateInlineVectorCallGenerator(m, intrinsic_cg, inline_cg);
}


// Allow inlining decisions to be delayed
class LateInlineVirtualCallGenerator : public VirtualCallGenerator {
private:
Expand Down
4 changes: 4 additions & 0 deletions src/hotspot/share/opto/callGenerator.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ class CallGenerator : public ArenaObj {
// same but for method handle calls
virtual bool is_mh_late_inline() const { return false; }
virtual bool is_string_late_inline() const { return false; }
virtual bool is_vector_late_inline() const { return false; }
virtual bool inline_fallback() const { return false; }
virtual void enable_fallback_generation() { return; }
virtual bool is_boxing_late_inline() const { return false; }
virtual bool is_vector_reboxing_late_inline() const { return false; }
virtual bool is_virtual_late_inline() const { return false; }
Expand Down Expand Up @@ -142,6 +145,7 @@ class CallGenerator : public ArenaObj {
static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg);
static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const);
static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg);
static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg);
static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses);
Expand Down
45 changes: 45 additions & 0 deletions src/hotspot/share/opto/compile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,7 @@ void Compile::remove_useless_node(Node* dead) {
remove_useless_late_inlines( &_late_inlines, dead);
remove_useless_late_inlines( &_string_late_inlines, dead);
remove_useless_late_inlines( &_boxing_late_inlines, dead);
remove_useless_late_inlines( &_vector_late_inlines, dead);
remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);

if (dead->is_CallStaticJava()) {
Expand Down Expand Up @@ -480,6 +481,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
remove_useless_late_inlines( &_late_inlines, useful);
remove_useless_late_inlines( &_string_late_inlines, useful);
remove_useless_late_inlines( &_boxing_late_inlines, useful);
remove_useless_late_inlines( &_vector_late_inlines, useful);
remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
}
Expand Down Expand Up @@ -693,6 +695,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
_string_late_inlines(comp_arena(), 2, 0, nullptr),
_boxing_late_inlines(comp_arena(), 2, 0, nullptr),
_vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr),
_vector_late_inlines(comp_arena(), 2, 0, nullptr),
_late_inlines_pos(0),
_has_mh_late_inlines(false),
_oom(false),
Expand Down Expand Up @@ -2016,6 +2019,32 @@ void Compile::process_for_unstable_if_traps(PhaseIterGVN& igvn) {
igvn.optimize();
}

void Compile::inline_vector_fallback(PhaseIterGVN& igvn) {
Comment thread
jatin-bhateja marked this conversation as resolved.
Outdated
while (_vector_late_inlines.length() > 0) {
CallGenerator* cg = _vector_late_inlines.pop();
assert(cg->is_vector_late_inline(), "");
if (cg->inline_fallback()) {
cg->enable_fallback_generation();
cg->do_late_inline();
}
if (failing()) return;
}
_vector_late_inlines.trunc_to(0);

inline_incrementally_cleanup(igvn);

while (_late_inlines.length() > 0) {
igvn_worklist()->ensure_empty(); // should be done with igvn

while (inline_incrementally_one()) {
assert(!failing_internal() || failure_is_artificial(), "inconsistent");
}
if (failing()) return;

inline_incrementally_cleanup(igvn);
}
}

// StringOpts and late inlining of string methods
void Compile::inline_string_calls(bool parse_time) {
{
Expand Down Expand Up @@ -2096,6 +2125,9 @@ bool Compile::inline_incrementally_one() {
} else if (inlining_progress()) {
_late_inlines_pos = i+1; // restore the position in case new elements were inserted
print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node());
if (cg->is_vector_late_inline()) {
C->vector_late_inlines()->remove_if_existing(cg);
Comment thread
jatin-bhateja marked this conversation as resolved.
Outdated
}
break; // process one call site at a time
} else {
bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node());
Expand All @@ -2106,6 +2138,9 @@ bool Compile::inline_incrementally_one() {
// Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt
assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass");
cg->call_node()->set_generator(cg);
if (cg->is_vector_late_inline()) {
C->vector_late_inlines()->append_if_missing(cg);
Comment thread
jatin-bhateja marked this conversation as resolved.
Outdated
}
}
}
} else {
Expand Down Expand Up @@ -2229,6 +2264,16 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) {
inline_incrementally_cleanup(igvn);
}

if (_vector_late_inlines.length() > 0) {
Comment thread
jatin-bhateja marked this conversation as resolved.
Outdated
inline_vector_fallback(igvn);

if (failing()) {
return;
}

inline_incrementally_cleanup(igvn);
}

set_inlining_incrementally(false);
}

Expand Down
7 changes: 7 additions & 0 deletions src/hotspot/share/opto/compile.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,7 @@ class Compile : public Phase {
GrowableArray<CallGenerator*> _boxing_late_inlines; // same but for boxing operations

GrowableArray<CallGenerator*> _vector_reboxing_late_inlines; // same but for vector reboxing operations
GrowableArray<CallGenerator*> _vector_late_inlines; // inline fallback implementation for failed intrinsics

int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining)
bool _has_mh_late_inlines; // Can there still be a method handle late inlining pending?
Expand Down Expand Up @@ -508,6 +509,11 @@ class Compile : public Phase {
InlinePrinter _inline_printer;

public:

GrowableArray<CallGenerator*>* vector_late_inlines() {
return &_vector_late_inlines;
}

void* barrier_set_state() const { return _barrier_set_state; }

InlinePrinter* inline_printer() { return &_inline_printer; }
Expand Down Expand Up @@ -1130,6 +1136,7 @@ class Compile : public Phase {
bool should_stress_inlining() { return StressIncrementalInlining && (random() % 2) == 0; }
bool should_delay_inlining() { return AlwaysIncrementalInline || should_stress_inlining(); }
void inline_string_calls(bool parse_time);
void inline_vector_fallback(PhaseIterGVN& igvn);
void inline_boxing_calls(PhaseIterGVN& igvn);
bool optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode);
void remove_root_to_sfpts_edges(PhaseIterGVN& igvn);
Expand Down
4 changes: 3 additions & 1 deletion src/hotspot/share/opto/doCall.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
cg_intrinsic = cg;
cg = nullptr;
} else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) {
return CallGenerator::for_late_inline(callee, cg);
float expected_uses = jvms->method()->scale_count(site_count, prof_factor);
CallGenerator* inline_cg = CallGenerator::for_inline(callee, expected_uses);
Comment thread
jatin-bhateja marked this conversation as resolved.
Outdated
return CallGenerator::for_vector_late_inline(callee, cg, inline_cg);
} else {
return cg;
}
Expand Down