From 0f8518fa76e07f2629fd87e0b9cbed0f573bcbe4 Mon Sep 17 00:00:00 2001 From: loongson-jvm Date: Thu, 9 Jan 2025 11:25:41 +0800 Subject: [PATCH] Update (2025.01.09, 2nd) 35121: LA port of 8334060: Implementation of Late Barrier Expansion for G1 35409: LA port of 8320318: ObjectMonitor Responsible thread 35408: LA port of 8340181: Shenandoah: Cleanup ShenandoahRuntime stubs 35407: LA port of 8340576: Some JVMCI flags are inconsistent --- .../loongarch/c2_MacroAssembler_loongarch.cpp | 59 +- .../gc/g1/g1BarrierSetAssembler_loongarch.cpp | 276 +++++++--- .../gc/g1/g1BarrierSetAssembler_loongarch.hpp | 25 +- .../cpu/loongarch/gc/g1/g1_loongarch.ad | 509 ++++++++++++++++++ .../shared/barrierSetAssembler_loongarch.cpp | 4 +- ...henandoahBarrierSetAssembler_loongarch.cpp | 12 +- src/hotspot/cpu/loongarch/loongarch_64.ad | 70 +-- .../loongarch/sharedRuntime_loongarch_64.cpp | 4 +- 8 files changed, 823 insertions(+), 136 deletions(-) create mode 100644 src/hotspot/cpu/loongarch/gc/g1/g1_loongarch.ad diff --git a/src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.cpp b/src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.cpp index 8e3a1e18ca0c9..4204a75009d95 100644 --- a/src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.cpp +++ b/src/hotspot/cpu/loongarch/c2_MacroAssembler_loongarch.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2021, 2024, Loongson Technology. All rights reserved. + * Copyright (c) 2021, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -143,6 +143,7 @@ void C2_MacroAssembler::fast_unlock_c2(Register oop, Register box, Register flag Label cont; Label object_has_monitor; Label count, no_count; + Label unlocked; assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight"); assert_different_registers(oop, box, tmp, disp_hdr, flag); @@ -192,17 +193,38 @@ void C2_MacroAssembler::fast_unlock_c2(Register oop, Register box, Register flag b(cont); bind(notRecursive); + + // Compute owner address. + // Set owner to null. + // Release to satisfy the JMM + membar(Assembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::StoreStore)); + st_d(R0, tmp, in_bytes(ObjectMonitor::owner_offset())); + // We need a full fence after clearing owner to avoid stranding. + // StoreLoad achieves this. + membar(MacroAssembler::StoreLoad); + + // Check if the entry lists are empty. ld_d(flag, Address(tmp, ObjectMonitor::EntryList_offset())); ld_d(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset())); orr(AT, flag, disp_hdr); + beqz(AT, unlocked); // If so we are done. + + // Check if there is a successor. + ld_d(AT, Address(tmp, ObjectMonitor::succ_offset())); + bnez(AT, unlocked); // If so we are done. + + // Save the monitor pointer in the current thread, so we can try to + // reacquire the lock in SharedRuntime::monitor_exit_helper(). + st_d(tmp, Address(TREG, JavaThread::unlocked_inflated_monitor_offset())); move(flag, R0); - bnez(AT, cont); + b(cont); - addi_d(AT, tmp, in_bytes(ObjectMonitor::owner_offset())); - amswap_db_d(tmp, R0, AT); + bind(unlocked); li(flag, 1); + // Intentional fall-through + bind(cont); // flag == 1 indicates success // flag == 0 indicates failure @@ -479,28 +501,35 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi bind(not_recursive); - Label release; const Register tmp2_owner_addr = tmp2; // Compute owner address. lea(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset())); + // Set owner to null. + // Release to satisfy the JMM + membar(Assembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::StoreStore)); + st_d(R0, tmp2_owner_addr, 0); + // We need a full fence after clearing owner to avoid stranding. + // StoreLoad achieves this. + membar(MacroAssembler::StoreLoad); + // Check if the entry lists are empty. ld_d(AT, Address(tmp1_monitor, ObjectMonitor::EntryList_offset())); ld_d(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset())); orr(AT, AT, tmp3_t); - beqz(AT, release); + beqz(AT, unlocked); // If so we are done. - // The owner may be anonymous and we removed the last obj entry in - // the lock-stack. This loses the information about the owner. - // Write the thread to the owner field so the runtime knows the owner. - st_d(TREG, tmp2_owner_addr, 0); - b(slow_path); + // Check if there is a successor. + ld_d(AT, Address(tmp1_monitor, ObjectMonitor::succ_offset())); + bnez(AT, unlocked); // If so we are done. - bind(release); - // Set owner to null. - membar(Assembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::StoreStore)); - st_d(R0, tmp2_owner_addr, 0); + // Save the monitor pointer in the current thread, so we can try + // to reacquire the lock in SharedRuntime::monitor_exit_helper(). + st_d(tmp1_monitor, Address(TREG, JavaThread::unlocked_inflated_monitor_offset())); + + move(flag, R0); + b(slow_path); } bind(unlocked); diff --git a/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.cpp b/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.cpp index 04e50bead073e..ea24d1c53ca72 100644 --- a/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.cpp +++ b/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2021, 2024, Loongson Technology. All rights reserved. + * Copyright (c) 2021, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,7 +38,10 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/g1/c1/g1BarrierSetC1.hpp" -#endif +#endif // COMPILER1 +#ifdef COMPILER2 +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#endif // COMPILER2 #define __ masm-> @@ -124,6 +127,54 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator } } +static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, + const Register thread, const Register value, const Register tmp1, const Register tmp2) { + // Can we store a value in the given thread's buffer? + // (The index field is typed as size_t.) + __ ld_d(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address) + __ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer) + // The buffer is not full, store value into it. + __ addi_d(tmp1, tmp1, -wordSize); // tmp1 := next index + __ st_d(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index + __ ld_d(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address + __ stx_d(value, tmp2, tmp1); // *(buffer address + next index) := value +} + +static void generate_pre_barrier_fast_path(MacroAssembler* masm, + const Register thread, + const Register tmp1) { + Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ ld_wu(tmp1, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ ld_bu(tmp1, in_progress); + } +} + +static void generate_pre_barrier_slow_path(MacroAssembler* masm, + const Register obj, + const Register pre_val, + const Register thread, + const Register tmp1, + const Register tmp2, + Label& done, + Label& runtime) { + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); + } + // Is the previous value null? + __ beqz(pre_val, done); + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::satb_mark_queue_index_offset(), + G1ThreadLocalData::satb_mark_queue_buffer_offset(), + runtime, + thread, pre_val, tmp1, tmp2); + __ b(done); +} + void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -144,41 +195,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, assert_different_registers(obj, pre_val, tmp1, tmp2); assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); - Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); - - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { - __ ld_w(tmp1, in_progress); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ ld_b(tmp1, in_progress); - } + generate_pre_barrier_fast_path(masm, thread, tmp1); + // If marking is not active (*(mark queue active address) == 0), jump to done __ beqz(tmp1, done); - - // Do we need to load the previous value? - if (obj != noreg) { - __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); - } - - // Is the previous value null? - __ beqz(pre_val, done); - - // Can we store original value in the thread's buffer? - // Is index == 0? - // (The index field is typed as size_t.) - - __ ld_d(tmp1, index); - __ beqz(tmp1, runtime); - - __ addi_d(tmp1, tmp1, -1 * wordSize); - __ st_d(tmp1, index); - __ ld_d(tmp2, buffer); - - // Record the previous value - __ stx_d(pre_val, tmp1, tmp2); - __ b(done); + generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime); __ bind(runtime); @@ -208,6 +228,49 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ bind(done); } +static void generate_post_barrier_fast_path(MacroAssembler* masm, + const Register store_addr, + const Register new_val, + const Register tmp1, + const Register tmp2, + Label& done, + bool new_val_may_be_null) { + // Does store cross heap regions? + __ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value + __ srli_d(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) + __ beqz(tmp1, done); + // Crosses regions, storing null? + if (new_val_may_be_null) { + __ beqz(new_val, done); + } + // Storing region crossing non-null, is card young? + __ srli_d(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base + __ load_byte_map_base(tmp2); // tmp2 := card table base address + __ add_d(tmp1, tmp1, tmp2); // tmp1 := card address + __ ld_bu(tmp2, Address(tmp1)); // tmp2 := card +} + +static void generate_post_barrier_slow_path(MacroAssembler* masm, + const Register thread, + const Register tmp1, + const Register tmp2, + Label& done, + Label& runtime) { + __ membar(MacroAssembler::StoreLoad); // StoreLoad membar + __ ld_bu(tmp2, Address(tmp1)); // tmp2 := card + __ beqz(tmp2, done); + // Storing a region crossing, non-null oop, card is clean. + // Dirty card and log. + STATIC_ASSERT(CardTable::dirty_card_val() == 0); + __ st_b(R0, Address(tmp1)); // *(card address) := dirty_card_val + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::dirty_card_queue_index_offset(), + G1ThreadLocalData::dirty_card_queue_buffer_offset(), + runtime, + thread, tmp1, tmp2, AT); + __ b(done); +} + void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, @@ -216,67 +279,120 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register tmp2) { assert(thread == TREG, "must be"); assert_different_registers(store_addr, thread, tmp1, tmp2, SCR1); - assert(store_addr != noreg && new_val != noreg && tmp1 != noreg - && tmp2 != noreg, "expecting a register"); - - Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); - - CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set()); - assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code"); + assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, + "expecting a register"); Label done; Label runtime; - // Does store cross heap regions? - __ xorr(tmp1, store_addr, new_val); - __ srli_d(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); - __ beqz(tmp1, done); + generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */); + // If card is young, jump to done (tmp2 holds the card value) + __ li(AT, (int)G1CardTable::g1_young_card_val()); + __ beq(tmp2, AT, done); // card == young_card_val? + generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime); - // crosses regions, storing null? - __ beqz(new_val, done); + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(store_addr); + __ push(saved); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread); + __ pop(saved); - // storing region crossing non-null, is card already dirty? - const Register card_addr = tmp1; + __ bind(done); +} - __ srli_d(card_addr, store_addr, CardTable::card_shift()); - // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT - // a valid address and therefore is not properly handled by the relocation code. - __ li(tmp2, (intptr_t)ct->card_table()->byte_map_base()); - __ add_d(card_addr, card_addr, tmp2); +#if defined(COMPILER2) - __ ld_bu(tmp2, card_addr, 0); - __ addi_d(tmp2, tmp2, -1 * (int)G1CardTable::g1_young_card_val()); - __ beqz(tmp2, done); +static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { + SaveLiveRegisters save_registers(masm, stub); + if (c_rarg0 != arg) { + __ move(c_rarg0, arg); + } + __ move(c_rarg1, TREG); + __ li(AT, runtime_path); + __ jalr(AT); +} - assert((int)CardTable::dirty_card_val() == 0, "must be 0"); +void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* stub) { + assert(thread == TREG, "must be"); + assert_different_registers(obj, pre_val, tmp1, tmp2); + assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - __ membar(__ StoreLoad); - __ ld_bu(tmp2, card_addr, 0); - __ beqz(tmp2, done); + stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2); - // storing a region crossing, non-null oop, card is clean. - // dirty card and log. - __ st_b(R0, card_addr, 0); + generate_pre_barrier_fast_path(masm, thread, tmp1); + // If marking is active (*(mark queue active address) != 0), jump to stub (slow path) + __ bnez(tmp1, *stub->entry()); - __ ld_d(SCR1, queue_index); - __ beqz(SCR1, runtime); - __ addi_d(SCR1, SCR1, -1 * wordSize); - __ st_d(SCR1, queue_index); - __ ld_d(tmp2, buffer); - __ ld_d(SCR1, queue_index); - __ stx_d(card_addr, tmp2, SCR1); - __ b(done); + __ bind(*stub->continuation()); +} + +void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register obj = stub->obj(); + Register pre_val = stub->pre_val(); + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); + Register tmp2 = stub->tmp2(); + + __ bind(*stub->entry()); + generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime); __ bind(runtime); - // save the live input values - __ push(store_addr); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, TREG); - __ pop(store_addr); + generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); + __ b(*stub->continuation()); +} - __ bind(done); +void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* stub) { + assert(thread == TREG, "must be"); + assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, AT); + assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, + "expecting a register"); + + stub->initialize_registers(thread, tmp1, tmp2); + + bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0; + generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null); + // If card is not young, jump to stub (slow path) (tmp2 holds the card value) + __ li(AT, (int)G1CardTable::g1_young_card_val()); + __ sub_d(AT, tmp2, AT); + __ bnez(AT, *stub->entry()); + + __ bind(*stub->continuation()); +} + +void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); // tmp1 holds the card address. + Register tmp2 = stub->tmp2(); + + __ bind(*stub->entry()); + generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime); + + __ bind(runtime); + generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); + __ b(*stub->continuation()); } +#endif // COMPILER2 + void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { bool in_heap = (decorators & IN_HEAP) != 0; diff --git a/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.hpp b/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.hpp index b3725301da27b..0097cf5cbe378 100644 --- a/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.hpp +++ b/src/hotspot/cpu/loongarch/gc/g1/g1BarrierSetAssembler_loongarch.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2022, 2023, Loongson Technology. All rights reserved. + * Copyright (c) 2022, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,8 @@ class LIR_Assembler; class StubAssembler; class G1PreBarrierStub; class G1PostBarrierStub; +class G1PreBarrierStubC2; +class G1PostBarrierStubC2; class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { protected: @@ -65,6 +67,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm); +#ifdef COMPILER2 + void g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* c2_stub); + void generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const; + void g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* c2_stub); + void generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const; +#endif + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2); }; diff --git a/src/hotspot/cpu/loongarch/gc/g1/g1_loongarch.ad b/src/hotspot/cpu/loongarch/gc/g1/g1_loongarch.ad new file mode 100644 index 0000000000000..f353cef5a9cf7 --- /dev/null +++ b/src/hotspot/cpu/loongarch/gc/g1/g1_loongarch.ad @@ -0,0 +1,509 @@ +// +// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2025, Loongson Technology. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#include "gc/shared/gc_globals.hpp" + +%} + +source %{ + +#include "gc/g1/g1BarrierSetAssembler_loongarch.hpp" +#include "gc/g1/g1BarrierSetRuntime.hpp" + +static void write_barrier_pre(MacroAssembler* masm, + const MachNode* node, + Register obj, + Register pre_val, + Register tmp1, + Register tmp2, + RegSet preserve = RegSet(), + RegSet no_preserve = RegSet()) { + if (!G1PreBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node); + for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) { + stub->preserve(*reg); + } + for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) { + stub->dont_preserve(*reg); + } + g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, TREG, tmp1, tmp2, stub); +} + +static void write_barrier_post(MacroAssembler* masm, + const MachNode* node, + Register store_addr, + Register new_val, + Register tmp1, + Register tmp2) { + if (!G1PostBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node); + g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, TREG, tmp1, tmp2, stub); +} + +%} + +instruct g1StoreP(indirect mem, mRegP src, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreP mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(125); + format %{ "st_d $src, [$mem]\t# TEMP($tmp1, $tmp2, $tmp3) @g1StoreP" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ loadstore_enc($src$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::STORE_LONG); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1StoreN(indirect mem, mRegN src, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(125); + format %{ "st_w $src, [$mem]\t# TEMP($tmp1, $tmp2, $tmp3) @g1StoreN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ loadstore_enc($src$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::STORE_INT); + if ((barrier_data() & G1C2BarrierPost) != 0) { + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ decode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + } + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1EncodePAndStoreN(indirect mem, mRegP src, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem (EncodeP src))); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(125); + format %{ "encodeP_and_storeN $src, [$mem]\t# TEMP($tmp1, $tmp2, $tmp3) @g1EncodePAndStoreN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert(as_Register($mem$$base) == $mem$$Register, "mem base = register"); + assert($mem$$index == -1 && $mem$$scale == 0, "assert index or scale"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ encode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + __ loadstore_enc($tmp1$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::STORE_INT); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeP(mRegP res, indirect mem, mRegP oldval, mRegP newval, mRegP tmp1, mRegP tmp2) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2); + ins_cost(2 * MEMORY_REF_COST); + format %{ "cmpxchg $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2) @g1CompareAndExchangeP" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + // Pass $oldval to the pre-barrier (instead of loading from $mem), because + // $oldval is the only value that can be overwritten. + // The same holds for g1CompareAndSwapP and its Acq variant. + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* retold */, false /* acquire */, false /* weak */, true /* exchange */); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangePAcq(mRegP res, indirect mem, mRegP oldval, mRegP newval, mRegP tmp1, mRegP tmp2) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 && + (((CompareAndExchangeNode*)n)->order() == MemNode::acquire || ((CompareAndExchangeNode*)n)->order() == MemNode::seqcst)); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2); + ins_cost(MEMORY_REF_COST); + format %{ "cmpxchg $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2) @g1CompareAndExchangePAcq" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + // Pass $oldval to the pre-barrier (instead of loading from $mem), because + // $oldval is the only value that can be overwritten. + // The same holds for g1CompareAndSwapP and its Acq variant. + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* retold */, true /* acquire */, false /* weak */, true /* exchange */); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeN(mRegN res, indirect mem, mRegN oldval, mRegN newval, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * MEMORY_REF_COST); + format %{ "cmpxchg32 $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2, $tmp3) @g1CompareAndExchangeN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg32(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* sign */, false /* retold */, false /* acquire */, false /* weak */, true /* exchange */); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeNAcq(mRegN res, indirect mem, mRegN oldval, mRegN newval, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 && + (((CompareAndExchangeNode*)n)->order() == MemNode::acquire || ((CompareAndExchangeNode*)n)->order() == MemNode::seqcst)); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(MEMORY_REF_COST); + format %{ "cmpxchg32 $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2, $tmp3) @g1CompareAndExchangeNAcq" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg32(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* sign */, false /* retold */, true /* acquire */, false /* weak */, true /* exchange */); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapP(mRegI res, indirect mem, mRegP newval, mRegP tmp1, mRegP tmp2, mRegP oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2); + ins_cost(2 * MEMORY_REF_COST); + format %{ "cmpxchg $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2) @g1CompareAndSwapP" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* retold */, false /* acquire */); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapPAcq(mRegI res, indirect mem, mRegP newval, mRegP tmp1, mRegP tmp2, mRegP oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 && + (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst)); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2); + ins_cost(MEMORY_REF_COST); + format %{ "cmpxchg $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2) @g1CompareAndSwapPAcq" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* retold */, true /* acquire */); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapN(mRegI res, indirect mem, mRegN newval, mRegP tmp1, mRegP tmp2, mRegP tmp3, mRegN oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * MEMORY_REF_COST); + format %{ "cmpxchg32 $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2, $tmp3) @g1CompareAndSwapN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg32(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* sign */, false /* retold */, false /* acquire */); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapNAcq(mRegI res, indirect mem, mRegN newval, mRegP tmp1, mRegP tmp2, mRegP tmp3, mRegN oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 && + (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst)); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(MEMORY_REF_COST); + format %{ "cmpxchg32 $res = [$mem], $oldval, $newval\t# TEMP($tmp1, $tmp2, $tmp3) @g1CompareAndSwapNAcq" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ cmpxchg32(Address(as_Register($mem$$base), 0), $oldval$$Register, $newval$$Register, $res$$Register, + false /* sign */, false /* retold */, true /* acquire */); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1GetAndSetP(indirect mem, mRegP newval, mRegP tmp1, mRegP tmp2, mRegP preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetP mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2); + ins_cost(2 * MEMORY_REF_COST); + format %{ "amswap_db_d $preval, $newval, [$mem]\t# TEMP($tmp1, $tmp2) @g1GetAndSetP" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $preval$$Register /* pre_val (as a temporary register) */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ amswap_db_d($preval$$Register, $newval$$Register, as_Register($mem$$base)); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1GetAndSetN(indirect mem, mRegN newval, mRegP tmp1, mRegP tmp2, mRegP tmp3, mRegN preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetN mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * MEMORY_REF_COST); + format %{ "amswap_db_w $preval, $newval, [$mem]\t# TEMP($tmp1, $tmp2) @g1GetAndSetN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ amswap_db_w(AT, $newval$$Register, as_Register($mem$$base)); + __ bstrpick_d($preval$$Register, AT, 31, 0); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1LoadP(mRegP dst, indirect mem, mRegP tmp1, mRegP tmp2) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadP mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2); + ins_cost(125); + format %{ "ld_d $dst, [$mem]\t# TEMP($tmp1, $tmp2) @g1LoadP" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_LONG); + write_barrier_pre(masm, this, + noreg /* obj */, + $dst$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1LoadN(mRegN dst, indirect mem, mRegP tmp1, mRegP tmp2, mRegP tmp3) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadN mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(125); + format %{ "ld_wu $dst, [$mem]\t# TEMP($tmp1, $tmp2, $tmp3) @g1LoadN" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_U_INT); + if ((barrier_data() & G1C2BarrierPre) != 0) { + __ decode_heap_oop($tmp1$$Register, $dst$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + } + %} + ins_pipe(pipe_slow); +%} diff --git a/src/hotspot/cpu/loongarch/gc/shared/barrierSetAssembler_loongarch.cpp b/src/hotspot/cpu/loongarch/gc/shared/barrierSetAssembler_loongarch.cpp index 3213bf9c056fa..fcade74221156 100644 --- a/src/hotspot/cpu/loongarch/gc/shared/barrierSetAssembler_loongarch.cpp +++ b/src/hotspot/cpu/loongarch/gc/shared/barrierSetAssembler_loongarch.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2018, 2024, Loongson Technology. All rights reserved. + * Copyright (c) 2018, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -482,7 +482,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) { } // Remove C-ABI SOE registers and scratch regs - _gp_regs -= RegSet::range(S0, S7) + RegSet::of(SP, SCR1, SCR2); + _gp_regs -= RegSet::range(S0, S7) + RegSet::of(SP, SCR1); } SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub) diff --git a/src/hotspot/cpu/loongarch/gc/shenandoah/shenandoahBarrierSetAssembler_loongarch.cpp b/src/hotspot/cpu/loongarch/gc/shenandoah/shenandoahBarrierSetAssembler_loongarch.cpp index a8651697d236b..a88bae3e25b81 100644 --- a/src/hotspot/cpu/loongarch/gc/shenandoah/shenandoahBarrierSetAssembler_loongarch.cpp +++ b/src/hotspot/cpu/loongarch/gc/shenandoah/shenandoahBarrierSetAssembler_loongarch.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright (c) 2022, 2024, Loongson Technology. All rights reserved. + * Copyright (c) 2022, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,9 +67,9 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec __ push(saved_regs); if (UseCompressedOops) { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count); } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count); } __ pop(saved_regs); __ bind(done); @@ -165,9 +165,9 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, if (expand_call) { assert(pre_val != A1, "smashed arg"); - __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread); } else { - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread); } __ pop(saved); @@ -717,7 +717,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss __ bind(runtime); __ push_call_clobbered_registers(); __ load_parameter(0, pre_val); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre), pre_val, thread); __ pop_call_clobbered_registers(); __ bind(done); diff --git a/src/hotspot/cpu/loongarch/loongarch_64.ad b/src/hotspot/cpu/loongarch/loongarch_64.ad index f3e051ebfb867..98ad07c43ab58 100644 --- a/src/hotspot/cpu/loongarch/loongarch_64.ad +++ b/src/hotspot/cpu/loongarch/loongarch_64.ad @@ -1,6 +1,6 @@ // // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. -// Copyright (c) 2015, 2024, Loongson Technology. All rights reserved. +// Copyright (c) 2015, 2025, Loongson Technology. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1170,7 +1170,9 @@ uint Matcher::vector_ideal_reg(int size) { // Should the matcher clone input 'm' of node 'n'? bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) { - if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con) + if (is_vshift_con_pattern(n, m) || // ShiftV src (ShiftCntV con) + is_encode_and_store_pattern(n, m) + ) { mstack.push(m, Visit); // m = ShiftCntV return true; } @@ -4217,31 +4219,32 @@ instruct storeL_immL_0_volatile(indirect mem, immL_0 zero) %{ // Load Compressed Pointer instruct loadN(mRegN dst, memory mem) %{ - match(Set dst (LoadN mem)); + predicate(n->as_Load()->barrier_data() == 0); + match(Set dst (LoadN mem)); - ins_cost(125); - format %{ "ld_wu $dst, $mem\t# compressed ptr @ loadN" %} - ins_encode %{ - relocInfo::relocType disp_reloc = $mem->disp_reloc(); - assert(disp_reloc == relocInfo::none, "cannot have disp"); - __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_U_INT); - %} - ins_pipe( ialu_load ); + ins_cost(125); + format %{ "ld_wu $dst, $mem\t# compressed ptr @ loadN" %} + ins_encode %{ + relocInfo::relocType disp_reloc = $mem->disp_reloc(); + assert(disp_reloc == relocInfo::none, "cannot have disp"); + __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_U_INT); + %} + ins_pipe( ialu_load ); %} instruct loadN2P(mRegP dst, memory mem) %{ - match(Set dst (DecodeN (LoadN mem))); - predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0); + predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0 && n->in(1)->as_Load()->barrier_data() == 0); + match(Set dst (DecodeN (LoadN mem))); - ins_cost(125); - format %{ "ld_wu $dst, $mem\t# @ loadN2P" %} - ins_encode %{ - relocInfo::relocType disp_reloc = $mem->disp_reloc(); - assert(disp_reloc == relocInfo::none, "cannot have disp"); - __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_U_INT); - %} - ins_pipe( ialu_load ); + ins_cost(125); + format %{ "ld_wu $dst, $mem\t# @ loadN2P" %} + ins_encode %{ + relocInfo::relocType disp_reloc = $mem->disp_reloc(); + assert(disp_reloc == relocInfo::none, "cannot have disp"); + __ loadstore_enc($dst$$Register, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp, C2_MacroAssembler::LOAD_U_INT); + %} + ins_pipe( ialu_load ); %} // Load Pointer @@ -4343,8 +4346,8 @@ instruct loadRange(mRegI dst, memory_loadRange mem) %{ instruct storeP(memory mem, mRegP src ) %{ + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem src)); - predicate(!needs_releasing_store(n)); ins_cost(125); format %{ "st_d $src, $mem #@storeP" %} @@ -4355,6 +4358,7 @@ instruct storeP(memory mem, mRegP src ) %{ %} instruct storeP_volatile(indirect mem, mRegP src ) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem src)); ins_cost(130); @@ -4367,8 +4371,8 @@ instruct storeP_volatile(indirect mem, mRegP src ) %{ // Store null Pointer, mark word, or other simple pointer constant. instruct storeImmP_immP_0(memory mem, immP_0 zero) %{ + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem zero)); - predicate(!needs_releasing_store(n)); ins_cost(125); format %{ "mov $mem, $zero #@storeImmP_0" %} @@ -4379,6 +4383,7 @@ instruct storeImmP_immP_0(memory mem, immP_0 zero) %{ %} instruct storeImmP_immP_0_volatile(indirect mem, immP_0 zero) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem zero)); ins_cost(130); @@ -4392,8 +4397,8 @@ instruct storeImmP_immP_0_volatile(indirect mem, immP_0 zero) %{ // Store Compressed Pointer instruct storeN(memory mem, mRegN src) %{ + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem src)); - predicate(!needs_releasing_store(n)); ins_cost(125); format %{ "st_w $mem, $src\t# compressed ptr @ storeN" %} @@ -4405,6 +4410,7 @@ instruct storeN(memory mem, mRegN src) instruct storeN_volatile(indirect mem, mRegN src) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem src)); ins_cost(130); @@ -4417,8 +4423,8 @@ instruct storeN_volatile(indirect mem, mRegN src) instruct storeP2N(memory mem, mRegP src) %{ + predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0 && !needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem (EncodeP src))); - predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0 && !needs_releasing_store(n)); ins_cost(125); format %{ "st_w $mem, $src\t# @ storeP2N" %} @@ -4430,8 +4436,8 @@ instruct storeP2N(memory mem, mRegP src) instruct storeP2N_volatile(indirect mem, mRegP src) %{ + predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0 && n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem (EncodeP src))); - predicate(CompressedOops::base() == nullptr && CompressedOops::shift() == 0); ins_cost(130); format %{ "amswap_db_w R0, $src, $mem # @ storeP2N" %} @@ -4494,8 +4500,8 @@ instruct storeP2NKlass_volatile(indirect mem, mRegP src) instruct storeImmN_immN_0(memory mem, immN_0 zero) %{ + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem zero)); - predicate(!needs_releasing_store(n)); ins_cost(125); format %{ "storeN0 zero, $mem\t# compressed ptr" %} @@ -4507,6 +4513,7 @@ instruct storeImmN_immN_0(memory mem, immN_0 zero) instruct storeImmN_immN_0_volatile(indirect mem, immN_0 zero) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem zero)); ins_cost(130); @@ -11355,9 +11362,9 @@ instruct compareAndSwapL(mRegI res, memory_exclusive mem_ptr, mRegL oldval, mReg %} instruct compareAndSwapP(mRegI res, memory_exclusive mem_ptr, mRegP oldval, mRegP newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval))); effect(TEMP_DEF res); - predicate(n->as_LoadStore()->barrier_data() == 0); ins_cost(3 * MEMORY_REF_COST); format %{ "CMPXCHG $newval, [$mem_ptr], $oldval @ compareAndSwapP" %} ins_encode %{ @@ -11372,6 +11379,7 @@ instruct compareAndSwapP(mRegI res, memory_exclusive mem_ptr, mRegP oldval, mRe %} instruct compareAndSwapN(mRegI res, indirect mem_ptr, mRegN oldval, mRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval))); effect(TEMP_DEF res); ins_cost(3 * MEMORY_REF_COST); @@ -11450,6 +11458,7 @@ instruct get_and_setL(indirect mem, mRegL newv, mRegL prev) %{ %} instruct get_and_setN(indirect mem, mRegN newv, mRegN prev) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set prev (GetAndSetN mem newv)); ins_cost(2 * MEMORY_REF_COST); format %{ "amswap_db_w $prev, $newv, [$mem] @get_and_setN" %} @@ -11465,6 +11474,7 @@ instruct get_and_setN(indirect mem, mRegN newv, mRegN prev) %{ %} instruct get_and_setP(indirect mem, mRegP newv, mRegP prev) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set prev (GetAndSetP mem newv)); effect(TEMP_DEF prev); ins_cost(2 * MEMORY_REF_COST); @@ -11683,7 +11693,7 @@ instruct compareAndExchangeP(mRegP res, memory_exclusive mem, mRegP oldval, mReg %} instruct compareAndExchangeN(mRegN res, memory_exclusive mem, mRegN oldval, mRegN newval) %{ - + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(2 * MEMORY_REF_COST); effect(TEMP_DEF res); @@ -11816,7 +11826,7 @@ instruct weakCompareAndSwapP_acq(mRegI res, memory_exclusive mem, mRegP oldval, %} instruct weakCompareAndSwapN(mRegI res, memory_exclusive mem, mRegN oldval, mRegN newval) %{ - + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); effect(TEMP_DEF res); ins_cost(2 * MEMORY_REF_COST); diff --git a/src/hotspot/cpu/loongarch/sharedRuntime_loongarch_64.cpp b/src/hotspot/cpu/loongarch/sharedRuntime_loongarch_64.cpp index 93181830228fd..57105eb78d02a 100644 --- a/src/hotspot/cpu/loongarch/sharedRuntime_loongarch_64.cpp +++ b/src/hotspot/cpu/loongarch/sharedRuntime_loongarch_64.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2015, 2024, Loongson Technology. All rights reserved. + * Copyright (c) 2015, 2025, Loongson Technology. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2200,7 +2200,7 @@ void SharedRuntime::generate_deopt_blob() { int reexecute_offset = __ pc() - start; #if INCLUDE_JVMCI && !defined(COMPILER1) - if (EnableJVMCI && UseJVMCICompiler) { + if (UseJVMCICompiler) { // JVMCI does not use this kind of deoptimization __ should_not_reach_here(); }