summaryrefslogtreecommitdiff
path: root/include/linux/rcupdate_trace.h
diff options
context:
space:
mode:
authorAlexei Starovoitov <ast@kernel.org>2020-09-24 05:32:09 +0300
committerAlexei Starovoitov <ast@kernel.org>2020-09-24 05:32:09 +0300
commit182bf3f3ddb607a0715b73a8396af1fce0945c14 (patch)
treed882c33f47f1aff70a14ffd4ce9a997e0d710d40 /include/linux/rcupdate_trace.h
parentf00f2f7fe86036a586e26bd55ad86310477199a3 (diff)
parentf747c7e15d7bc71a967a94ceda686cf2460b69e8 (diff)
downloadlinux-182bf3f3ddb607a0715b73a8396af1fce0945c14.tar.xz
Merge branch 'rtt-speedup.2020.09.16a' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into bpf-next
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Diffstat (limited to 'include/linux/rcupdate_trace.h')
-rw-r--r--include/linux/rcupdate_trace.h4
1 files changed, 4 insertions, 0 deletions
diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index aaaac8ac927c..3e7919fc5f34 100644
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h
@@ -50,6 +50,7 @@ static inline void rcu_read_lock_trace(void)
struct task_struct *t = current;
WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
+ barrier();
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
t->trc_reader_special.b.need_mb)
smp_mb(); // Pairs with update-side barriers
@@ -72,6 +73,9 @@ static inline void rcu_read_unlock_trace(void)
rcu_lock_release(&rcu_trace_lock_map);
nesting = READ_ONCE(t->trc_reader_nesting) - 1;
+ barrier(); // Critical section before disabling.
+ // Disable IPI-based setting of .need_qs.
+ WRITE_ONCE(t->trc_reader_nesting, INT_MIN);
if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
WRITE_ONCE(t->trc_reader_nesting, nesting);
return; // We assume shallow reader nesting.