summaryrefslogtreecommitdiff
path: root/kernel/rcu
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/tree.c15
-rw-r--r--kernel/rcu/tree_plugin.h13
2 files changed, 24 insertions, 4 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index fb6b80aa34f6..a6ddfae6978d 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1335,6 +1335,19 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
}
/*
+ * Move and classify callbacks, but only if doing so won't require
+ * that the RCU grace-period kthread be awakened.
+ */
+static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
+ struct rcu_data *rdp)
+{
+ raw_lockdep_assert_held_rcu_node(rnp);
+ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
+ return;
+ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
+}
+
+/*
* Update CPU-local rcu_data state to record the beginnings and ends of
* grace periods. The caller must hold the ->lock of the leaf rcu_node
* structure corresponding to the current CPU, and must have irqs disabled.
@@ -2118,6 +2131,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
rcu_segcblist_n_lazy_cbs(&rdp->cblist),
rcu_segcblist_n_cbs(&rdp->cblist), bl);
rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
+ if (offloaded)
+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
rcu_nocb_unlock_irqrestore(rdp, flags);
/* Invoke callbacks. */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 5cbc0848647c..c10afe778430 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1643,10 +1643,15 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
} else if (len > rdp->qlen_last_fqs_check + qhimark) {
/* ... or if many callbacks queued. */
rdp->qlen_last_fqs_check = len;
- if (!irqs_disabled_flags(flags)) {
- wake_nocb_gp(rdp, true, flags);
- trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
- TPS("WakeOvf"));
+ if (!rdp->nocb_cb_sleep &&
+ rcu_segcblist_ready_cbs(&rdp->cblist)) {
+ // Already going full tilt, so don't try to rewake.
+ rcu_nocb_unlock_irqrestore(rdp, flags);
+ } else if (rcu_segcblist_pend_cbs(&rdp->cblist) &&
+ raw_spin_trylock_rcu_node(rdp->mynode)) {
+ rcu_advance_cbs_nowake(rdp->mynode, rdp);
+ raw_spin_unlock_rcu_node(rdp->mynode);
+ rcu_nocb_unlock_irqrestore(rdp, flags);
} else {
wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
TPS("WakeOvfIsDeferred"));