lib: Fix race conditions in tlb fifo access.

Linux kernel expects tlb flush SBI call to be completely synchronous i.e.
the SBI call should only return once corresponding *fence* instruction is
executed.

OpenSBI manages the outstanding TLB flush requests by keeping them in a
per hart based fifo. However, there are few corner cases that may lead to
race conditions while updating the fifo.

Currently, the caller hart waits for IPI acknowledgement via clint
address which is not a very good method as synchronization on MMIO may not
be supported in every platform. Moreover, the waiter doesn't have any way of
identifying if the IPI is received for specific tlb flush request or any
other IPI. This may lead to unpredictable behavior in supervisor/user space.

Fix this by waiting on individual fifo entries rather than MMIO address.
Currently, a relaxed loop is being used because wfi again involves MMIO write
which would be slower compared to relaxed loop. To avoid deadlock, fifo
is processed every time a hart loops for fifo enqueue or fifo sync to consume
the tlb flush requests sent by other harts.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
Signed-off-by: Atish Patra <atish.patra@wdc.com>

1 files changed, 30 insertions, 24 deletions

diff --git a/lib/sbi/sbi_ipi.c b/lib/sbi/sbi_ipi.c
index b1077b5..1c84f9e 100644
--- a/lib/sbi/sbi_ipi.c
+++ b/lib/sbi/sbi_ipi.c
@@ -9,15 +9,13 @@
  */
 
 #include <sbi/riscv_asm.h>
-#include <sbi/riscv_barrier.h>
 #include <sbi/riscv_atomic.h>
+#include <sbi/riscv_barrier.h>
 #include <sbi/riscv_unpriv.h>
 #include <sbi/sbi_error.h>
-#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_platform.h>
-#include <sbi/sbi_timer.h>
 #include <sbi/sbi_tlb.h>
 
 static unsigned long ipi_data_off;
@@ -40,20 +38,22 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event,
 	remote_scratch = sbi_hart_id_to_scratch(scratch, hartid);
 	ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off);
 	if (event == SBI_IPI_EVENT_SFENCE_VMA ||
-	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID) {
-		ret = sbi_tlb_fifo_update(remote_scratch, event, data);
-		if (ret > 0)
-			goto done;
-		else if (ret < 0)
+	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID ||
+	    event == SBI_IPI_EVENT_FENCE_I ) {
+		ret = sbi_tlb_fifo_update(remote_scratch, hartid, data);
+		if (ret < 0)
 			return ret;
 	}
 	atomic_raw_set_bit(event, &ipi_data->ipi_type);
-	mb();
+	smp_wmb();
 	sbi_platform_ipi_send(plat, hartid);
-	if (event != SBI_IPI_EVENT_SOFT)
-		sbi_platform_ipi_sync(plat, hartid);
 
-done:
+	if (event == SBI_IPI_EVENT_SFENCE_VMA ||
+	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID ||
+	    event == SBI_IPI_EVENT_FENCE_I ) {
+		sbi_tlb_fifo_sync(scratch);
+	}
+
 	return 0;
 }
 
@@ -70,12 +70,13 @@ int sbi_ipi_send_many(struct sbi_scratch *scratch, struct unpriv_trap *uptrap,
 			return SBI_ETRAP;
 	}
 
-	/* send IPIs to every other hart on the set */
+	/* Send IPIs to every other hart on the set */
 	for (i = 0, m = mask; m; i++, m >>= 1)
 		if ((m & 1UL) && (i != hartid))
 			sbi_ipi_send(scratch, i, event, data);
 
-	/* If the current hart is on the set, send an IPI
+	/*
+	 * If the current hart is on the set, send an IPI
 	 * to it as well
 	 */
 	if (mask & (1UL << hartid))
@@ -91,7 +92,7 @@ void sbi_ipi_clear_smode(struct sbi_scratch *scratch)
 
 void sbi_ipi_process(struct sbi_scratch *scratch)
 {
-	volatile unsigned long ipi_type;
+	unsigned long ipi_type;
 	unsigned int ipi_event;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 	struct sbi_ipi_data *ipi_data =
@@ -100,27 +101,32 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
 	u32 hartid = sbi_current_hartid();
 	sbi_platform_ipi_clear(plat, hartid);
 
-	do {
-		ipi_type = ipi_data->ipi_type;
-		rmb();
-		ipi_event = __ffs(ipi_type);
+	ipi_type = atomic_raw_xchg_ulong(&ipi_data->ipi_type, 0);
+	ipi_event = 0;
+	while (ipi_type) {
+		if (!(ipi_type & 1UL))
+			goto skip;
+
 		switch (ipi_event) {
 		case SBI_IPI_EVENT_SOFT:
 			csr_set(CSR_MIP, MIP_SSIP);
 			break;
 		case SBI_IPI_EVENT_FENCE_I:
-			__asm__ __volatile("fence.i");
-			break;
 		case SBI_IPI_EVENT_SFENCE_VMA:
 		case SBI_IPI_EVENT_SFENCE_VMA_ASID:
-			sbi_tlb_fifo_process(scratch, ipi_event);
+			sbi_tlb_fifo_process(scratch);
 			break;
 		case SBI_IPI_EVENT_HALT:
 			sbi_hart_hang();
 			break;
+		default:
+			break;
 		};
-		ipi_type = atomic_raw_clear_bit(ipi_event, &ipi_data->ipi_type);
-	} while (ipi_type > 0);
+
+skip:
+		ipi_type = ipi_type >> 1;
+		ipi_event++;
+	};
 }
 
 int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)