aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAtish Patra <atish.patra@wdc.com>2019-03-09 12:57:20 -0800
committerAnup Patel <anup@brainfault.org>2019-03-12 22:07:25 +0530
commit90cb4917b584b07df3e4037a3b660e9ac28e3384 (patch)
tree4b4905f9c70c5909d2b642255d542b71fb700710
parent508a27204cbbca0a9430236e56681e5e0d343fb9 (diff)
lib: Implement sfence.vma correctly.
Currently, OpenSBI doesn't distinguish between sfence.vma and sfence.vm asid calls. Moreover, it ignores the page ranges and just flush entire TLB everytime. Fix the sfence implementation by keeping all the tlb flush info in scratch area. The relevant Linux kernel code was added by https://patchwork.kernel.org/project/linux-riscv/list/?series=89695 However, this patch is backward compatible with older version kernel that doesn't have the above patches as well. Fixes #87 Signed-off-by: Atish Patra <atish.patra@wdc.com>
Diffstat
-rw-r--r--include/sbi/riscv_asm.h5
-rw-r--r--include/sbi/sbi_ipi.h15
-rw-r--r--include/sbi/sbi_scratch.h10
-rw-r--r--lib/sbi_ecall.c17
-rw-r--r--lib/sbi_ipi.c71
-rw-r--r--lib/sbi_system.c2
6 files changed, 105 insertions, 15 deletions
diff --git a/include/sbi/riscv_asm.h b/include/sbi/riscv_asm.h
index a88c086..9648629 100644
--- a/include/sbi/riscv_asm.h
+++ b/include/sbi/riscv_asm.h
@@ -26,6 +26,11 @@
#error "Unexpected __riscv_xlen"
#endif
+#define PAGE_SHIFT (12)
+#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
+#define PAGE_MASK (~(PAGE_SIZE - 1))
+#define SBI_TLB_FLUSH_ALL ((unsigned long)-1)
+
#define REG_L __REG_SEL(ld, lw)
#define REG_S __REG_SEL(sd, sw)
#define SZREG __REG_SEL(8, 4)
diff --git a/include/sbi/sbi_ipi.h b/include/sbi/sbi_ipi.h
index 708dd8c..1aafe35 100644
--- a/include/sbi/sbi_ipi.h
+++ b/include/sbi/sbi_ipi.h
@@ -15,12 +15,23 @@
#define SBI_IPI_EVENT_SOFT 0x1
#define SBI_IPI_EVENT_FENCE_I 0x2
#define SBI_IPI_EVENT_SFENCE_VMA 0x4
-#define SBI_IPI_EVENT_HALT 0x8
+#define SBI_IPI_EVENT_SFENCE_VMA_ASID 0x8
+#define SBI_IPI_EVENT_HALT 0x10
struct sbi_scratch;
+struct sbi_ipi_data {
+ unsigned long ipi_type;
+};
+
+struct sbi_tlb_info {
+ unsigned long start;
+ unsigned long size;
+ unsigned long asid;
+};
+
int sbi_ipi_send_many(struct sbi_scratch *scratch,
- ulong *pmask, u32 event);
+ ulong *pmask, u32 event, void *data);
void sbi_ipi_clear_smode(struct sbi_scratch *scratch);
diff --git a/include/sbi/sbi_scratch.h b/include/sbi/sbi_scratch.h
index 70ab384..1534893 100644
--- a/include/sbi/sbi_scratch.h
+++ b/include/sbi/sbi_scratch.h
@@ -35,16 +35,14 @@
/** Offset of ipi_type in sbi_ipi_data */
#define SBI_IPI_DATA_IPI_TYPE_OFFSET (15 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_TLB_INFO_OFFSET (16 * __SIZEOF_POINTER__)
/** Maximum size of sbi_scratch and sbi_ipi_data */
#define SBI_SCRATCH_SIZE (32 * __SIZEOF_POINTER__)
#ifndef __ASSEMBLY__
#include <sbi/sbi_types.h>
-
-struct sbi_ipi_data {
- unsigned long ipi_type;
-};
+#include <sbi/sbi_ipi.h>
/** Representation of per-HART scratch space */
struct sbi_scratch {
@@ -80,6 +78,10 @@ struct sbi_scratch {
#define sbi_ipi_data_ptr(scratch) \
((struct sbi_ipi_data *)(void*)scratch + SBI_IPI_DATA_IPI_TYPE_OFFSET)
+/** Get pointer to tlb flush info from sbi_scratch */
+#define sbi_tlb_info_ptr(scratch) \
+((struct sbi_tlb_info *)(void*)scratch + SBI_SCRATCH_TLB_INFO_OFFSET)
+
#endif
#endif
diff --git a/lib/sbi_ecall.c b/lib/sbi_ecall.c
index 42123ba..6312469 100644
--- a/lib/sbi_ecall.c
+++ b/lib/sbi_ecall.c
@@ -34,6 +34,7 @@ int sbi_ecall_handler(u32 hartid, ulong mcause,
struct sbi_scratch *scratch)
{
int ret = SBI_ENOTSUPP;
+ struct sbi_tlb_info tlb_info;
switch (regs->a7) {
case SBI_ECALL_SET_TIMER:
@@ -59,16 +60,26 @@ int sbi_ecall_handler(u32 hartid, ulong mcause,
break;
case SBI_ECALL_SEND_IPI:
ret = sbi_ipi_send_many(scratch, (ulong *)regs->a0,
- SBI_IPI_EVENT_SOFT);
+ SBI_IPI_EVENT_SOFT, NULL);
break;
case SBI_ECALL_REMOTE_FENCE_I:
ret = sbi_ipi_send_many(scratch, (ulong *)regs->a0,
- SBI_IPI_EVENT_FENCE_I);
+ SBI_IPI_EVENT_FENCE_I, NULL);
break;
case SBI_ECALL_REMOTE_SFENCE_VMA:
+ tlb_info.start = (unsigned long)regs->a1;
+ tlb_info.size = (unsigned long)regs->a2;
+
+ ret = sbi_ipi_send_many(scratch, (ulong *)regs->a0,
+ SBI_IPI_EVENT_SFENCE_VMA, &tlb_info);
+ break;
case SBI_ECALL_REMOTE_SFENCE_VMA_ASID:
+ tlb_info.start = (unsigned long)regs->a1;
+ tlb_info.size = (unsigned long)regs->a2;
+ tlb_info.asid = (unsigned long)regs->a3;
+
ret = sbi_ipi_send_many(scratch, (ulong *)regs->a0,
- SBI_IPI_EVENT_SFENCE_VMA);
+ SBI_IPI_EVENT_SFENCE_VMA_ASID, &tlb_info);
break;
case SBI_ECALL_SHUTDOWN:
sbi_system_shutdown(scratch, 0);
diff --git a/lib/sbi_ipi.c b/lib/sbi_ipi.c
index e0f2a19..468db9a 100644
--- a/lib/sbi_ipi.c
+++ b/lib/sbi_ipi.c
@@ -19,10 +19,13 @@
#include <sbi/sbi_timer.h>
#include <sbi/sbi_unpriv.h>
-static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event)
+static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid,
+ u32 event, void *data)
{
struct sbi_scratch *remote_scratch = NULL;
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+ struct sbi_tlb_info *tlb_info = data;
+ struct sbi_tlb_info *ipi_tlb_data;
if (sbi_platform_hart_disabled(plat, hartid))
return -1;
@@ -31,6 +34,13 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event)
* trigger the interrupt
*/
remote_scratch = sbi_hart_id_to_scratch(scratch, hartid);
+ if (event == SBI_IPI_EVENT_SFENCE_VMA ||
+ event == SBI_IPI_EVENT_SFENCE_VMA_ASID) {
+ ipi_tlb_data = sbi_tlb_info_ptr(remote_scratch);
+ ipi_tlb_data->start = tlb_info->start;
+ ipi_tlb_data->size = tlb_info->size;
+ ipi_tlb_data->asid = tlb_info->asid;
+ }
atomic_raw_set_bit(event, &sbi_ipi_data_ptr(remote_scratch)->ipi_type);
mb();
sbi_platform_ipi_send(plat, hartid);
@@ -41,7 +51,7 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event)
}
int sbi_ipi_send_many(struct sbi_scratch *scratch,
- ulong *pmask, u32 event)
+ ulong *pmask, u32 event, void *data)
{
ulong i, m;
ulong mask = sbi_hart_available_mask();
@@ -53,13 +63,13 @@ int sbi_ipi_send_many(struct sbi_scratch *scratch,
/* send IPIs to every other hart on the set */
for (i = 0, m = mask; m; i++, m >>= 1)
if ((m & 1UL) && (i != hartid))
- sbi_ipi_send(scratch, i, event);
+ sbi_ipi_send(scratch, i, event, data);
/* If the current hart is on the set, send an IPI
* to it as well
*/
if (mask & (1UL << hartid))
- sbi_ipi_send(scratch, hartid, event);
+ sbi_ipi_send(scratch, hartid, event, data);
return 0;
@@ -70,6 +80,51 @@ void sbi_ipi_clear_smode(struct sbi_scratch *scratch)
csr_clear(CSR_MIP, MIP_SSIP);
}
+static void sbi_ipi_tlb_flush_all(unsigned long asid)
+{
+ __asm__ __volatile__ ("sfence.vma x0, %0"
+ : : "r" (asid)
+ : "memory");
+}
+
+static void sbi_ipi_sfence_vma(struct sbi_tlb_info *tinfo)
+{
+ unsigned long start = tinfo->start;
+ unsigned long size = tinfo->size;
+ unsigned long i;
+
+ if (start == 0 && size == 0)
+ sbi_ipi_tlb_flush_all(0);
+
+ for (i = 0; i < size; i += PAGE_SIZE) {
+ __asm__ __volatile__ ("sfence.vma %0"
+ : : "r" (start + i)
+ : "memory");
+ }
+}
+
+static void sbi_ipi_sfence_vma_asid(struct sbi_tlb_info *tinfo)
+{
+ unsigned long start = tinfo->start;
+ unsigned long size = tinfo->size;
+ unsigned long asid = tinfo->asid;
+ unsigned long i;
+
+ if (start == 0 && size == 0)
+ sbi_ipi_tlb_flush_all(0);
+ /* Flush entire MM context */
+ if (size == SBI_TLB_FLUSH_ALL) {
+ sbi_ipi_tlb_flush_all(asid);
+ return;
+ }
+
+ for (i = 0; i < size; i += PAGE_SIZE) {
+ __asm__ __volatile__ ("sfence.vma %0, %1"
+ : : "r" (start + i), "r" (asid)
+ : "memory");
+ }
+}
+
void sbi_ipi_process(struct sbi_scratch *scratch)
{
const struct sbi_platform *plat = sbi_platform_ptr(scratch);
@@ -91,7 +146,10 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
__asm__ __volatile("fence.i");
break;
case SBI_IPI_EVENT_SFENCE_VMA:
- __asm__ __volatile("sfence.vma");
+ sbi_ipi_sfence_vma(sbi_tlb_info_ptr(scratch));
+ break;
+ case SBI_IPI_EVENT_SFENCE_VMA_ASID:
+ sbi_ipi_sfence_vma_asid(sbi_tlb_info_ptr(scratch));
break;
case SBI_IPI_EVENT_HALT:
sbi_hart_hang();
@@ -104,6 +162,9 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)
{
sbi_ipi_data_ptr(scratch)->ipi_type = 0x00;
+ sbi_tlb_info_ptr(scratch)->start = 0x00;
+ sbi_tlb_info_ptr(scratch)->size = 0x00;
+ sbi_tlb_info_ptr(scratch)->asid = 0x00;
/* Enable software interrupts */
csr_set(CSR_MIE, MIP_MSIP);
diff --git a/lib/sbi_system.c b/lib/sbi_system.c
index f23d046..b373828 100644
--- a/lib/sbi_system.c
+++ b/lib/sbi_system.c
@@ -41,7 +41,7 @@ void __attribute__((noreturn)) sbi_system_shutdown(struct sbi_scratch *scratch,
/* If that fails (or is not implemented) send an IPI on every
* hart to hang and then hang the current hart */
- sbi_ipi_send_many(scratch, NULL, SBI_IPI_EVENT_HALT);
+ sbi_ipi_send_many(scratch, NULL, SBI_IPI_EVENT_HALT, NULL);
sbi_hart_hang();
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-18 03:20:02 +0000