summaryrefslogtreecommitdiff
path: root/kernel/rcu
diff options
context:
space:
mode:
authorThomas Gleixner <tglx@linutronix.de>2020-05-03 16:08:52 +0300
committerThomas Gleixner <tglx@linutronix.de>2020-05-19 16:51:21 +0300
commit8ae0ae6737ad449c8ae21e2bb01d9736f360a933 (patch)
tree1143878ecb15a2eeb6790b837da31d14a96c0776 /kernel/rcu
parent9ea366f669ded353ae49754216c042e7d2f72ba6 (diff)
downloadlinux-8ae0ae6737ad449c8ae21e2bb01d9736f360a933.tar.xz
rcu: Provide rcu_irq_exit_preempt()
Interrupts and exceptions invoke rcu_irq_enter() on entry and need to invoke rcu_irq_exit() before they either return to the interrupted code or invoke the scheduler due to preemption. The general assumption is that RCU idle code has to have preemption disabled so that a return from interrupt cannot schedule. So the return from interrupt code invokes rcu_irq_exit() and preempt_schedule_irq(). If there is any imbalance in the rcu_irq/nmi* invocations or RCU idle code had preemption enabled then this goes unnoticed until the CPU goes idle or some other RCU check is executed. Provide rcu_irq_exit_preempt() which can be invoked from the interrupt/exception return code in case that preemption is enabled. It invokes rcu_irq_exit() and contains a few sanity checks in case that CONFIG_PROVE_RCU is enabled to catch such issues directly. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Paul E. McKenney <paulmck@kernel.org> Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Link: https://lkml.kernel.org/r/20200505134904.364456424@linutronix.de
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/tree.c22
1 files changed, 22 insertions, 0 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 945401674b9d..62ee01299386 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -743,6 +743,28 @@ void noinstr rcu_irq_exit(void)
rcu_nmi_exit();
}
+/**
+ * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
+ * towards in kernel preemption
+ *
+ * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
+ * from RCU point of view. Invoked from return from interrupt before kernel
+ * preemption.
+ */
+void rcu_irq_exit_preempt(void)
+{
+ lockdep_assert_irqs_disabled();
+ rcu_nmi_exit();
+
+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
+ "RCU dynticks_nesting counter underflow/zero!");
+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
+ DYNTICK_IRQ_NONIDLE,
+ "Bad RCU dynticks_nmi_nesting counter\n");
+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
+ "RCU in extended quiescent state!");
+}
+
/*
* Wrapper for rcu_irq_exit() where interrupts are enabled.
*