From cd36a75911738ec348b8973885b3145a330af0ad Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Fri, 28 Jun 2024 15:41:48 +0200 Subject: [PATCH] make IPI barrier a real function Signed-off-by: Axel Heider --- include/arch/arm/arch/machine.h | 11 +++++++++++ include/arch/arm/arch/machine/hardware.h | 10 ---------- include/arch/riscv/arch/machine.h | 4 ++++ include/arch/riscv/arch/machine/hardware.h | 6 ------ include/arch/x86/arch/kernel/x2apic.h | 12 ++++++++++-- include/arch/x86/arch/kernel/xapic.h | 13 ++++++++++--- src/arch/x86/smp/ipi.c | 2 +- src/smp/ipi.c | 2 +- 8 files changed, 37 insertions(+), 23 deletions(-) diff --git a/include/arch/arm/arch/machine.h b/include/arch/arm/arch/machine.h index 5df9279355..4db5ea6038 100644 --- a/include/arch/arm/arch/machine.h +++ b/include/arch/arm/arch/machine.h @@ -63,6 +63,17 @@ static inline void arch_pause(void) { /* TODO */ } + +static inline void ipi_mem_barrier(void) +{ + /* For GICv2 systems a dmb() is sufficient, but it's not enough with GICv3 + * due to the way IPIs is triggered (memory-mapped or MSR inst.). A dmb() + * does not prevent re-ordering to happen between memory accesses and + * instructions, this guarantee requires a dsb(). + */ + dsb_ishst(); +} + #endif /* ENABLE_SMP_SUPPORT */ /* Update the value of the actual regsiter to hold the expected value */ diff --git a/include/arch/arm/arch/machine/hardware.h b/include/arch/arm/arch/machine/hardware.h index 029ea51c0e..931e2d591b 100644 --- a/include/arch/arm/arch/machine/hardware.h +++ b/include/arch/arm/arch/machine/hardware.h @@ -19,17 +19,7 @@ typedef word_t vm_fault_type_t; #define PAGE_BASE(_p, _s) ((_p) & ~MASK(pageBitsForSize((_s)))) #define PAGE_OFFSET(_p, _s) ((_p) & MASK(pageBitsForSize((_s)))) -#define IPI_MEM_BARRIER \ - do { \ - /* This can be relaxed for GICv2 but for GICv3 dmb() no longer works */ \ - /* since the way IPI is triggered is different (memory-mapped or MSR inst.) */ \ - /* and dmb() is not able to avoid re-ordering between memory accesses and */ \ - /* instructions. In order to support both GICv2 and v3 dsb() is required. */ \ - dsb_ishst(); \ - } while (0) - #endif /* __ASSEMBLER__ */ #define L1_CACHE_LINE_SIZE_BITS CONFIG_L1_CACHE_LINE_SIZE_BITS #define L1_CACHE_LINE_SIZE BIT(L1_CACHE_LINE_SIZE_BITS) - diff --git a/include/arch/riscv/arch/machine.h b/include/arch/riscv/arch/machine.h index 568d35e36a..1ec27c4fa7 100644 --- a/include/arch/riscv/arch/machine.h +++ b/include/arch/riscv/arch/machine.h @@ -71,6 +71,10 @@ static inline void fence_rw_rw(void) asm volatile("fence rw, rw" ::: "memory"); } +static inline void ipi_mem_barrier(void) { + fence_rw_rw(); +} + static inline void fence_w_rw(void) { asm volatile("fence w, rw" ::: "memory"); diff --git a/include/arch/riscv/arch/machine/hardware.h b/include/arch/riscv/arch/machine/hardware.h index 1c8506fb93..1e8759094d 100644 --- a/include/arch/riscv/arch/machine/hardware.h +++ b/include/arch/riscv/arch/machine/hardware.h @@ -122,9 +122,3 @@ static inline void arch_clean_invalidate_caches(void) #define LOAD_S STRINGIFY(LOAD) #define STORE_S STRINGIFY(STORE) - -#define IPI_MEM_BARRIER \ - do { \ - asm volatile("fence rw,rw" ::: "memory"); \ - } while (0) - diff --git a/include/arch/x86/arch/kernel/x2apic.h b/include/arch/x86/arch/kernel/x2apic.h index 3d517c2e73..1a7a41b1fd 100644 --- a/include/arch/x86/arch/kernel/x2apic.h +++ b/include/arch/x86/arch/kernel/x2apic.h @@ -65,6 +65,14 @@ static inline void apic_write_icr(word_t high, word_t low) x86_wrmsr(APIC_ICR, icr); } -#define IPI_ICR_BARRIER asm volatile("mfence" ::: "memory") -#define IPI_MEM_BARRIER IPI_ICR_BARRIER +static inline void ipi_icr_barrier(void) +{ + asm volatile("mfence" ::: "memory"); +} + +static inline void ipi_mem_barrier(void) +{ + IPI_ICR_BARRIER; +} + #endif /* CONFIG_X2APIC */ diff --git a/include/arch/x86/arch/kernel/xapic.h b/include/arch/x86/arch/kernel/xapic.h index 2be88896e6..b3fbc0f9c0 100644 --- a/include/arch/x86/arch/kernel/xapic.h +++ b/include/arch/x86/arch/kernel/xapic.h @@ -66,7 +66,14 @@ static inline void apic_write_icr(word_t high, word_t low) apic_write_reg(APIC_ICR1, low); } -#define IPI_ICR_BARRIER asm volatile("" ::: "memory") -#define IPI_MEM_BARRIER IPI_ICR_BARRIER -#endif /* CONFIG_XAPIC */ +static inline void ipi_icr_barrier(void) +{ + asm volatile("" ::: "memory"); +} +static inline void ipi_mem_barrier(void) +{ + ipi_icr_barrier(); +} + +#endif /* CONFIG_XAPIC */ diff --git a/src/arch/x86/smp/ipi.c b/src/arch/x86/smp/ipi.c index 198148594b..5c0dfeefd7 100644 --- a/src/arch/x86/smp/ipi.c +++ b/src/arch/x86/smp/ipi.c @@ -110,7 +110,7 @@ static void x86_ipi_send_mask(interrupt_t ipi, word_t mask, bool_t isBlocking) } while (mask != 0); /* broadcast IPIs to clusters... */ - IPI_ICR_BARRIER; + ipi_icr_barrier(); for (int i = 0; i < nr_target_clusters; i++) { apic_send_ipi_cluster(ipi, target_clusters[i]); } diff --git a/src/smp/ipi.c b/src/smp/ipi.c index d96638d97d..3bbe6fbbf3 100644 --- a/src/smp/ipi.c +++ b/src/smp/ipi.c @@ -139,7 +139,7 @@ void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking) if (nr_target_cores > 0) { /* sending IPIs... */ - IPI_MEM_BARRIER; + ipi_mem_barrier(); for (int i = 0; i < nr_target_cores; i++) { ipi_send_target(ipi, cpuIndexToID(target_cores[i])); }