Skip to content

Commit

Permalink
8334060: Implementation of Late Barrier Expansion for G1
Browse files Browse the repository at this point in the history 
Co-authored-by: Roberto Castañeda Lozano <[email protected]>
Co-authored-by: Erik Österlund <[email protected]>
Co-authored-by: Siyao Liu <[email protected]>
Co-authored-by: Kim Barrett <[email protected]>
Co-authored-by: Amit Kumar <[email protected]>
Co-authored-by: Martin Doerr <[email protected]>
Co-authored-by: Feilong Jiang <[email protected]>
Co-authored-by: Sergey Nazarkin <[email protected]>
Reviewed-by: kvn, tschatzl, fyang, ayang, kbarrett
  • Loading branch information
@robcasloz @fisk
@641bill @offamitkumar @TheRealMDoerr @feilongjiang
8 people committed Oct 3, 2024
1 parent c6e7e55 commit 0b467e9
Show file tree
Hide file tree
Showing 58 changed files with 6,387 additions and 1,448 deletions.
7 changes: 7 additions & 0 deletions make/hotspot/gensrc/GensrcAdlc.gmk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,13 @@ ifeq ($(call check-jvm-feature, compiler2), true)
)))
endif

ifeq ($(call check-jvm-feature, g1gc), true)
AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU).ad \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU_ARCH).ad \
)))
endif

SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad

INSERT_FILENAME_AWK_SCRIPT := \
Expand Down
24 changes: 16 additions & 8 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2620,7 +2620,8 @@ static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
if (is_vshift_con_pattern(n, m) ||
is_vector_bitwise_not_pattern(n, m) ||
is_valid_sve_arith_imm_pattern(n, m)) {
is_valid_sve_arith_imm_pattern(n, m) ||
is_encode_and_store_pattern(n, m)) {
mstack.push(m, Visit);
return true;
}
Expand Down Expand Up @@ -6410,7 +6411,7 @@ instruct loadP(iRegPNoSp dst, memory mem)
instruct loadN(iRegNNoSp dst, memory mem)
%{
match(Set dst (LoadN mem));
predicate(!needs_acquiring_load(n));
predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);

ins_cost(4 * INSN_COST);
format %{ "ldrw $dst, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -6839,7 +6840,7 @@ instruct storeimmP0(immP0 zero, memory mem)
instruct storeN(iRegN src, memory mem)
%{
match(Set mem (StoreN mem src));
predicate(!needs_releasing_store(n));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);

ins_cost(INSN_COST);
format %{ "strw $src, $mem\t# compressed ptr" %}
Expand All @@ -6852,7 +6853,7 @@ instruct storeN(iRegN src, memory mem)
instruct storeImmN0(immN0 zero, memory mem)
%{
match(Set mem (StoreN mem zero));
predicate(!needs_releasing_store(n));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);

ins_cost(INSN_COST);
format %{ "strw zr, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -7086,6 +7087,7 @@ instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
%{
match(Set dst (LoadN mem));
predicate(n->as_Load()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "ldarw $dst, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -7253,6 +7255,7 @@ instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem src));
predicate(n->as_Store()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "stlrw $src, $mem\t# compressed ptr" %}
Expand All @@ -7265,6 +7268,7 @@ instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem zero));
predicate(n->as_Store()->barrier_data() == 0);

ins_cost(VOLATILE_REF_COST);
format %{ "stlrw zr, $mem\t# compressed ptr" %}
Expand Down Expand Up @@ -8061,6 +8065,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{

match(Set res (CompareAndSwapN mem (Binary oldval newval)));
predicate(n->as_LoadStore()->barrier_data() == 0);
ins_cost(2 * VOLATILE_REF_COST);

effect(KILL cr);
Expand Down Expand Up @@ -8175,7 +8180,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new

instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{

predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);

Expand Down Expand Up @@ -8280,6 +8285,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
Expand Down Expand Up @@ -8389,7 +8395,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
Expand Down Expand Up @@ -8501,6 +8507,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
Expand Down Expand Up @@ -8620,7 +8627,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(KILL cr);
Expand Down Expand Up @@ -8681,6 +8688,7 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}

instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
Expand Down Expand Up @@ -8724,7 +8732,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}

instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
predicate(needs_acquiring_load_exclusive(n));
predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(VOLATILE_REF_COST);
format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
Expand Down
4 changes: 4 additions & 0 deletions src/hotspot/cpu/aarch64/cas.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ define(`CAS_INSN',
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchange$1$6(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{
ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));),
$1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);),
$1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),
`dnl')
match(Set res (CompareAndExchange$1 mem (Binary oldval newval)));
Expand Down Expand Up @@ -122,7 +124,9 @@ define(`CAS_INSN3',
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{
ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));),
$1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);),
$1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),
`dnl')
match(Set res (WeakCompareAndSwap$1 mem (Binary oldval newval)));
Expand Down
Loading

0 comments on commit 0b467e9

Please sign in to comment.