diff options
author | Anup Patel <anup.patel@wdc.com> | 2019-05-24 05:28:05 +0300 |
---|---|---|
committer | Anup Patel <anup@brainfault.org> | 2019-05-24 05:52:47 +0300 |
commit | 95b7480ab4b438cf545afae200b40012e83f1dda (patch) | |
tree | 44e30d69d4222833e36d3092005ac0a09b45f391 | |
parent | 2dfed32c463eef80a0eea7108117ce18dc80c527 (diff) | |
download | opensbi-95b7480ab4b438cf545afae200b40012e83f1dda.tar.xz |
lib: Factor-out TLB management from IPI management
This patch factor-out TLB management from IPI management to separate
sources sbi_tlb.c and sbi_tlb.h.
Signed-off-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Atish Patra <atish.patra@wdc.com>
-rw-r--r-- | include/sbi/riscv_asm.h | 3 | ||||
-rw-r--r-- | include/sbi/sbi_ipi.h | 16 | ||||
-rw-r--r-- | include/sbi/sbi_tlb.h | 48 | ||||
-rw-r--r-- | lib/objects.mk | 1 | ||||
-rw-r--r-- | lib/sbi_ecall.c | 1 | ||||
-rw-r--r-- | lib/sbi_ipi.c | 203 | ||||
-rw-r--r-- | lib/sbi_tlb.c | 227 |
7 files changed, 292 insertions, 207 deletions
diff --git a/include/sbi/riscv_asm.h b/include/sbi/riscv_asm.h index 704c53c..4f12c03 100644 --- a/include/sbi/riscv_asm.h +++ b/include/sbi/riscv_asm.h @@ -32,9 +32,6 @@ #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE - 1)) -#define SBI_TLB_FLUSH_ALL ((unsigned long)-1) -#define SBI_TLB_FLUSH_MAX_SIZE (1UL << 30) - #define REG_L __REG_SEL(ld, lw) #define REG_S __REG_SEL(sd, sw) #define SZREG __REG_SEL(8, 4) diff --git a/include/sbi/sbi_ipi.h b/include/sbi/sbi_ipi.h index 108e5aa..b8afb5b 100644 --- a/include/sbi/sbi_ipi.h +++ b/include/sbi/sbi_ipi.h @@ -22,28 +22,12 @@ /* clang-format on */ -#define SBI_TLB_FIFO_NUM_ENTRIES 4 -enum sbi_tlb_info_types { - SBI_TLB_FLUSH_VMA, - SBI_TLB_FLUSH_VMA_ASID, - SBI_TLB_FLUSH_VMA_VMID -}; - struct sbi_scratch; struct sbi_ipi_data { unsigned long ipi_type; }; -struct sbi_tlb_info { - unsigned long start; - unsigned long size; - unsigned long asid; - unsigned long type; -}; - -#define SBI_TLB_INFO_SIZE sizeof(struct sbi_tlb_info) - int sbi_ipi_send_many(struct sbi_scratch *scratch, ulong *pmask, u32 event, void *data); diff --git a/include/sbi/sbi_tlb.h b/include/sbi/sbi_tlb.h new file mode 100644 index 0000000..f3d93d4 --- /dev/null +++ b/include/sbi/sbi_tlb.h @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + * + * Authors: + * Atish Patra <atish.patra@wdc.com> + * Anup Patel <anup.patel@wdc.com> + */ + +#ifndef __SBI_TLB_H__ +#define __SBI_TLB_H__ + +#include <sbi/sbi_types.h> + +/* clang-format off */ + +#define SBI_TLB_FLUSH_ALL ((unsigned long)-1) +#define SBI_TLB_FLUSH_MAX_SIZE (1UL << 30) + +/* clang-format on */ + +#define SBI_TLB_FIFO_NUM_ENTRIES 4 + +enum sbi_tlb_info_types { + SBI_TLB_FLUSH_VMA, + SBI_TLB_FLUSH_VMA_ASID, + SBI_TLB_FLUSH_VMA_VMID +}; + +struct sbi_scratch; + +struct sbi_tlb_info { + unsigned long start; + unsigned long size; + unsigned long asid; + unsigned long type; +}; + +#define SBI_TLB_INFO_SIZE sizeof(struct sbi_tlb_info) + +int sbi_tlb_fifo_update(struct sbi_scratch *scratch, u32 event, void *data); + +void sbi_tlb_fifo_process(struct sbi_scratch *scratch, u32 event); + +int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot); + +#endif diff --git a/lib/objects.mk b/lib/objects.mk index bfa5a3c..8eb860f 100644 --- a/lib/objects.mk +++ b/lib/objects.mk @@ -24,6 +24,7 @@ lib-objs-y += sbi_misaligned_ldst.o lib-objs-y += sbi_scratch.o lib-objs-y += sbi_system.o lib-objs-y += sbi_timer.o +lib-objs-y += sbi_tlb.o lib-objs-y += sbi_trap.o # External Libraries to include diff --git a/lib/sbi_ecall.c b/lib/sbi_ecall.c index d82766a..a8ad40c 100644 --- a/lib/sbi_ecall.c +++ b/lib/sbi_ecall.c @@ -14,6 +14,7 @@ #include <sbi/sbi_ipi.h> #include <sbi/sbi_system.h> #include <sbi/sbi_timer.h> +#include <sbi/sbi_tlb.h> #include <sbi/sbi_trap.h> #define SBI_ECALL_VERSION_MAJOR 0 diff --git a/lib/sbi_ipi.c b/lib/sbi_ipi.c index 09d2f7d..da6b06d 100644 --- a/lib/sbi_ipi.c +++ b/lib/sbi_ipi.c @@ -13,92 +13,23 @@ #include <sbi/riscv_atomic.h> #include <sbi/riscv_unpriv.h> #include <sbi/sbi_error.h> -#include <sbi/sbi_fifo.h> -#include <sbi/sbi_hart.h> #include <sbi/sbi_bitops.h> +#include <sbi/sbi_hart.h> #include <sbi/sbi_ipi.h> #include <sbi/sbi_platform.h> #include <sbi/sbi_timer.h> +#include <sbi/sbi_tlb.h> #include <plat/string.h> static unsigned long ipi_data_off; -static unsigned long ipi_tlb_fifo_off; -static unsigned long ipi_tlb_fifo_mem_off; - -static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr, - struct sbi_tlb_info *next) -{ - unsigned long curr_end; - unsigned long next_end; - int ret = SBI_FIFO_UNCHANGED; - - if (!curr || !next) - return ret; - - next_end = next->start + next->size; - curr_end = curr->start + curr->size; - if (next->start <= curr->start && next_end > curr_end) { - curr->start = next->start; - curr->size = next->size; - ret = SBI_FIFO_UPDATED; - } else if (next->start >= curr->start && next_end <= curr_end) { - ret = SBI_FIFO_SKIP; - } - - return ret; -} - -/** - * Call back to decide if an inplace fifo update is required or next entry can - * can be skipped. Here are the different cases that are being handled. - * - * Case1: - * if next flush request range lies within one of the existing entry, skip - * the next entry. - * Case2: - * if flush request range in current fifo entry lies within next flush - * request, update the current entry. - * Case3: - if a complete vma flush is requested, then all entries can be deleted - and new request can be enqueued. This will not be done for ASID case - as that means we have to iterate again in the fifo to figure out which - entries belong to that ASID. - */ -int sbi_tlb_fifo_update_cb(void *in, void *data) -{ - struct sbi_tlb_info *curr; - struct sbi_tlb_info *next; - int ret = SBI_FIFO_UNCHANGED; - - if (!in && !!data) - return ret; - - curr = (struct sbi_tlb_info *)data; - next = (struct sbi_tlb_info *)in; - if (next->type == SBI_TLB_FLUSH_VMA_ASID && - curr->type == SBI_TLB_FLUSH_VMA_ASID) { - if (next->asid == curr->asid) - ret = __sbi_tlb_fifo_range_check(curr, next); - } else if (next->type == SBI_TLB_FLUSH_VMA && - curr->type == SBI_TLB_FLUSH_VMA) { - if (next->size == SBI_TLB_FLUSH_ALL) - ret = SBI_FIFO_RESET; - else - ret = __sbi_tlb_fifo_range_check(curr, next); - } - - return ret; -} static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event, void *data) { + int ret; struct sbi_scratch *remote_scratch = NULL; const struct sbi_platform *plat = sbi_platform_ptr(scratch); struct sbi_ipi_data *ipi_data; - struct sbi_fifo *ipi_tlb_fifo; - struct sbi_tlb_info *tinfo = data; - int ret = SBI_FIFO_UNCHANGED; if (sbi_platform_hart_disabled(plat, hartid)) return -1; @@ -109,45 +40,21 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event, */ remote_scratch = sbi_hart_id_to_scratch(scratch, hartid); ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off); - ipi_tlb_fifo = sbi_scratch_offset_ptr(remote_scratch, - ipi_tlb_fifo_off); if (event == SBI_IPI_EVENT_SFENCE_VMA || event == SBI_IPI_EVENT_SFENCE_VMA_ASID) { - /* - * If address range to flush is too big then simply - * upgrade it to flush all because we can only flush - * 4KB at a time. - */ - if (tinfo->size >= SBI_TLB_FLUSH_MAX_SIZE) { - tinfo->start = 0; - tinfo->size = SBI_TLB_FLUSH_ALL; - } - - ret = sbi_fifo_inplace_update(ipi_tlb_fifo, data, - sbi_tlb_fifo_update_cb); - if (ret == SBI_FIFO_SKIP || ret == SBI_FIFO_UPDATED) { + ret = sbi_tlb_fifo_update(remote_scratch, event, data); + if (ret > 0) goto done; - } - while (sbi_fifo_enqueue(ipi_tlb_fifo, data) < 0) { - /** - * For now, Busy loop until there is space in the fifo. - * There may be case where target hart is also - * enqueue in source hart's fifo. Both hart may busy - * loop leading to a deadlock. - * TODO: Introduce a wait/wakeup event mechansim to handle - * this properly. - */ - __asm__ __volatile("nop"); - __asm__ __volatile("nop"); - } + else if (ret < 0) + return ret; } atomic_raw_set_bit(event, &ipi_data->ipi_type); mb(); sbi_platform_ipi_send(plat, hartid); if (event != SBI_IPI_EVENT_SOFT) sbi_platform_ipi_sync(plat, hartid); -done: +done: return 0; } @@ -180,69 +87,13 @@ void sbi_ipi_clear_smode(struct sbi_scratch *scratch) csr_clear(CSR_MIP, MIP_SSIP); } -static void sbi_ipi_tlb_flush_all() -{ - __asm__ __volatile("sfence.vma"); -} - -static void sbi_ipi_sfence_vma(struct sbi_tlb_info *tinfo) -{ - unsigned long start = tinfo->start; - unsigned long size = tinfo->size; - unsigned long i; - - if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { - sbi_ipi_tlb_flush_all(); - return; - } - - for (i = 0; i < size; i += PAGE_SIZE) { - __asm__ __volatile__("sfence.vma %0" - : - : "r"(start + i) - : "memory"); - } -} - -static void sbi_ipi_sfence_vma_asid(struct sbi_tlb_info *tinfo) -{ - unsigned long start = tinfo->start; - unsigned long size = tinfo->size; - unsigned long asid = tinfo->asid; - unsigned long i; - - if (start == 0 && size == 0) { - sbi_ipi_tlb_flush_all(); - return; - } - - /* Flush entire MM context for a given ASID */ - if (size == SBI_TLB_FLUSH_ALL) { - __asm__ __volatile__("sfence.vma x0, %0" - : - : "r"(asid) - : "memory"); - return; - } - - for (i = 0; i < size; i += PAGE_SIZE) { - __asm__ __volatile__("sfence.vma %0, %1" - : - : "r"(start + i), "r"(asid) - : "memory"); - } -} - void sbi_ipi_process(struct sbi_scratch *scratch) { volatile unsigned long ipi_type; - struct sbi_tlb_info tinfo; unsigned int ipi_event; const struct sbi_platform *plat = sbi_platform_ptr(scratch); struct sbi_ipi_data *ipi_data = sbi_scratch_offset_ptr(scratch, ipi_data_off); - struct sbi_fifo *ipi_tlb_fifo = - sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off); u32 hartid = sbi_current_hartid(); sbi_platform_ipi_clear(plat, hartid); @@ -260,13 +111,7 @@ void sbi_ipi_process(struct sbi_scratch *scratch) break; case SBI_IPI_EVENT_SFENCE_VMA: case SBI_IPI_EVENT_SFENCE_VMA_ASID: - while (!sbi_fifo_dequeue(ipi_tlb_fifo, &tinfo)) { - if (tinfo.type == SBI_TLB_FLUSH_VMA) - sbi_ipi_sfence_vma(&tinfo); - else if (tinfo.type == SBI_TLB_FLUSH_VMA_ASID) - sbi_ipi_sfence_vma_asid(&tinfo); - memset(&tinfo, 0, SBI_TLB_INFO_SIZE); - } + sbi_tlb_fifo_process(scratch, ipi_event); break; case SBI_IPI_EVENT_HALT: sbi_hart_hang(); @@ -278,8 +123,7 @@ void sbi_ipi_process(struct sbi_scratch *scratch) int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot) { - void *ipi_tlb_mem; - struct sbi_fifo *ipi_tlb_q; + int ret; struct sbi_ipi_data *ipi_data; if (cold_boot) { @@ -287,34 +131,17 @@ int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot) "IPI_DATA"); if (!ipi_data_off) return SBI_ENOMEM; - ipi_tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*ipi_tlb_q), - "IPI_TLB_FIFO"); - if (!ipi_tlb_fifo_off) { - sbi_scratch_free_offset(ipi_data_off); - return SBI_ENOMEM; - } - ipi_tlb_fifo_mem_off = sbi_scratch_alloc_offset( - SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE, - "IPI_TLB_FIFO_MEM"); - if (!ipi_tlb_fifo_mem_off) { - sbi_scratch_free_offset(ipi_tlb_fifo_off); - sbi_scratch_free_offset(ipi_data_off); - return SBI_ENOMEM; - } } else { - if (!ipi_data_off || - !ipi_tlb_fifo_off || - !ipi_tlb_fifo_mem_off) + if (!ipi_data_off) return SBI_ENOMEM; } ipi_data = sbi_scratch_offset_ptr(scratch, ipi_data_off); - ipi_tlb_q = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off); - ipi_tlb_mem = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_mem_off); - ipi_data->ipi_type = 0x00; - sbi_fifo_init(ipi_tlb_q, ipi_tlb_mem, - SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE); + + ret = sbi_tlb_fifo_init(scratch, cold_boot); + if (ret) + return ret; /* Enable software interrupts */ csr_set(CSR_MIE, MIP_MSIP); diff --git a/lib/sbi_tlb.c b/lib/sbi_tlb.c new file mode 100644 index 0000000..814d402 --- /dev/null +++ b/lib/sbi_tlb.c @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + * + * Authors: + * Atish Patra <atish.patra@wdc.com> + * Anup Patel <anup.patel@wdc.com> + */ + +#include <sbi/riscv_asm.h> +#include <sbi/riscv_barrier.h> +#include <sbi/sbi_error.h> +#include <sbi/sbi_fifo.h> +#include <sbi/sbi_hart.h> +#include <sbi/sbi_bitops.h> +#include <sbi/sbi_scratch.h> +#include <sbi/sbi_tlb.h> +#include <plat/string.h> + +static unsigned long ipi_tlb_fifo_off; +static unsigned long ipi_tlb_fifo_mem_off; + +static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr, + struct sbi_tlb_info *next) +{ + unsigned long curr_end; + unsigned long next_end; + int ret = SBI_FIFO_UNCHANGED; + + if (!curr || !next) + return ret; + + next_end = next->start + next->size; + curr_end = curr->start + curr->size; + if (next->start <= curr->start && next_end > curr_end) { + curr->start = next->start; + curr->size = next->size; + ret = SBI_FIFO_UPDATED; + } else if (next->start >= curr->start && next_end <= curr_end) { + ret = SBI_FIFO_SKIP; + } + + return ret; +} + +/** + * Call back to decide if an inplace fifo update is required or next entry can + * can be skipped. Here are the different cases that are being handled. + * + * Case1: + * if next flush request range lies within one of the existing entry, skip + * the next entry. + * Case2: + * if flush request range in current fifo entry lies within next flush + * request, update the current entry. + * Case3: + if a complete vma flush is requested, then all entries can be deleted + and new request can be enqueued. This will not be done for ASID case + as that means we have to iterate again in the fifo to figure out which + entries belong to that ASID. + */ +static int sbi_tlb_fifo_update_cb(void *in, void *data) +{ + struct sbi_tlb_info *curr; + struct sbi_tlb_info *next; + int ret = SBI_FIFO_UNCHANGED; + + if (!in && !!data) + return ret; + + curr = (struct sbi_tlb_info *)data; + next = (struct sbi_tlb_info *)in; + if (next->type == SBI_TLB_FLUSH_VMA_ASID && + curr->type == SBI_TLB_FLUSH_VMA_ASID) { + if (next->asid == curr->asid) + ret = __sbi_tlb_fifo_range_check(curr, next); + } else if (next->type == SBI_TLB_FLUSH_VMA && + curr->type == SBI_TLB_FLUSH_VMA) { + if (next->size == SBI_TLB_FLUSH_ALL) + ret = SBI_FIFO_RESET; + else + ret = __sbi_tlb_fifo_range_check(curr, next); + } + + return ret; +} + +int sbi_tlb_fifo_update(struct sbi_scratch *scratch, u32 event, void *data) +{ + int ret; + struct sbi_fifo *ipi_tlb_fifo; + struct sbi_tlb_info *tinfo = data; + + ipi_tlb_fifo = sbi_scratch_offset_ptr(scratch, + ipi_tlb_fifo_off); + /* + * If address range to flush is too big then simply + * upgrade it to flush all because we can only flush + * 4KB at a time. + */ + if (tinfo->size >= SBI_TLB_FLUSH_MAX_SIZE) { + tinfo->start = 0; + tinfo->size = SBI_TLB_FLUSH_ALL; + } + + ret = sbi_fifo_inplace_update(ipi_tlb_fifo, data, + sbi_tlb_fifo_update_cb); + if (ret == SBI_FIFO_SKIP || ret == SBI_FIFO_UPDATED) { + return 1; + } + + while (sbi_fifo_enqueue(ipi_tlb_fifo, data) < 0) { + /** + * For now, Busy loop until there is space in the fifo. + * There may be case where target hart is also + * enqueue in source hart's fifo. Both hart may busy + * loop leading to a deadlock. + * TODO: Introduce a wait/wakeup event mechansim to handle + * this properly. + */ + __asm__ __volatile("nop"); + __asm__ __volatile("nop"); + } + + return 0; +} + +static void sbi_tlb_flush_all(void) +{ + __asm__ __volatile("sfence.vma"); +} + +static void sbi_tlb_fifo_sfence_vma(struct sbi_tlb_info *tinfo) +{ + unsigned long start = tinfo->start; + unsigned long size = tinfo->size; + unsigned long i; + + if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) { + sbi_tlb_flush_all(); + return; + } + + for (i = 0; i < size; i += PAGE_SIZE) { + __asm__ __volatile__("sfence.vma %0" + : + : "r"(start + i) + : "memory"); + } +} + +static void sbi_tlb_fifo_sfence_vma_asid(struct sbi_tlb_info *tinfo) +{ + unsigned long start = tinfo->start; + unsigned long size = tinfo->size; + unsigned long asid = tinfo->asid; + unsigned long i; + + if (start == 0 && size == 0) { + sbi_tlb_flush_all(); + return; + } + + /* Flush entire MM context for a given ASID */ + if (size == SBI_TLB_FLUSH_ALL) { + __asm__ __volatile__("sfence.vma x0, %0" + : + : "r"(asid) + : "memory"); + return; + } + + for (i = 0; i < size; i += PAGE_SIZE) { + __asm__ __volatile__("sfence.vma %0, %1" + : + : "r"(start + i), "r"(asid) + : "memory"); + } +} + +void sbi_tlb_fifo_process(struct sbi_scratch *scratch, u32 event) +{ + struct sbi_tlb_info tinfo; + struct sbi_fifo *ipi_tlb_fifo = + sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off); + + while (!sbi_fifo_dequeue(ipi_tlb_fifo, &tinfo)) { + if (tinfo.type == SBI_TLB_FLUSH_VMA) + sbi_tlb_fifo_sfence_vma(&tinfo); + else if (tinfo.type == SBI_TLB_FLUSH_VMA_ASID) + sbi_tlb_fifo_sfence_vma_asid(&tinfo); + memset(&tinfo, 0, SBI_TLB_INFO_SIZE); + } +} + +int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot) +{ + void *ipi_tlb_mem; + struct sbi_fifo *ipi_tlb_q; + + if (cold_boot) { + ipi_tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*ipi_tlb_q), + "IPI_TLB_FIFO"); + if (!ipi_tlb_fifo_off) + return SBI_ENOMEM; + ipi_tlb_fifo_mem_off = sbi_scratch_alloc_offset( + SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE, + "IPI_TLB_FIFO_MEM"); + if (!ipi_tlb_fifo_mem_off) { + sbi_scratch_free_offset(ipi_tlb_fifo_off); + return SBI_ENOMEM; + } + } else { + if (!ipi_tlb_fifo_off || + !ipi_tlb_fifo_mem_off) + return SBI_ENOMEM; + } + + ipi_tlb_q = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off); + ipi_tlb_mem = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_mem_off); + + sbi_fifo_init(ipi_tlb_q, ipi_tlb_mem, + SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE); + + return 0; +} |