summaryrefslogtreecommitdiff
path: root/kernel/rcu
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2018-08-15 00:41:49 +0300
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2018-08-31 02:10:49 +0300
commit4e6ea4ef56f9425cd239ffdb6be45b3aeeb347fd (patch)
tree81702573011d79ef39d2934e020372976256b9de /kernel/rcu
parent55cda2290bf9d8510fbe7c1939a36680476c69c4 (diff)
downloadlinux-4e6ea4ef56f9425cd239ffdb6be45b3aeeb347fd.tar.xz
srcu: Make early-boot call_srcu() reuse workqueue lists
Allocating a list_head structure that is almost never used, and, when used, is used only during early boot (rcu_init() and earlier), is a bit wasteful. This commit therefore eliminates that list_head in favor of the one in the work_struct structure. This is safe because the work_struct structure cannot be used until after rcu_init() returns. Reported-by: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Tejun Heo <tj@kernel.org> Cc: Lai Jiangshan <jiangshanlai@gmail.com> Tested-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/srcutiny.c10
-rw-r--r--kernel/rcu/srcutree.c11
2 files changed, 10 insertions, 11 deletions
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index d233f0c63f6f..b46e6683f8c9 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -48,7 +48,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
sp->srcu_gp_waiting = false;
sp->srcu_idx = 0;
INIT_WORK(&sp->srcu_work, srcu_drive_gp);
- INIT_LIST_HEAD(&sp->srcu_boot_entry);
+ INIT_LIST_HEAD(&sp->srcu_work.entry);
return 0;
}
@@ -185,8 +185,8 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
if (!READ_ONCE(sp->srcu_gp_running)) {
if (likely(srcu_init_done))
schedule_work(&sp->srcu_work);
- else if (list_empty(&sp->srcu_boot_entry))
- list_add(&sp->srcu_boot_entry, &srcu_boot_list);
+ else if (list_empty(&sp->srcu_work.entry))
+ list_add(&sp->srcu_work.entry, &srcu_boot_list);
}
}
EXPORT_SYMBOL_GPL(call_srcu);
@@ -224,8 +224,8 @@ void __init srcu_init(void)
srcu_init_done = true;
while (!list_empty(&srcu_boot_list)) {
sp = list_first_entry(&srcu_boot_list,
- struct srcu_struct, srcu_boot_entry);
- list_del_init(&sp->srcu_boot_entry);
+ struct srcu_struct, srcu_work.entry);
+ list_del_init(&sp->srcu_work.entry);
schedule_work(&sp->srcu_work);
}
}
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 2e7f6b460150..86c7fd0a1bfe 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -186,7 +186,6 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static)
mutex_init(&sp->srcu_barrier_mutex);
atomic_set(&sp->srcu_barrier_cpu_cnt, 0);
INIT_DELAYED_WORK(&sp->work, process_srcu);
- INIT_LIST_HEAD(&sp->srcu_boot_entry);
if (!is_static)
sp->sda = alloc_percpu(struct srcu_data);
init_srcu_struct_nodes(sp, is_static);
@@ -708,8 +707,8 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,
if (likely(srcu_init_done))
queue_delayed_work(rcu_gp_wq, &sp->work,
srcu_get_delay(sp));
- else if (list_empty(&sp->srcu_boot_entry))
- list_add(&sp->srcu_boot_entry, &srcu_boot_list);
+ else if (list_empty(&sp->work.work.entry))
+ list_add(&sp->work.work.entry, &srcu_boot_list);
}
spin_unlock_irqrestore_rcu_node(sp, flags);
}
@@ -1323,10 +1322,10 @@ void __init srcu_init(void)
srcu_init_done = true;
while (!list_empty(&srcu_boot_list)) {
- sp = list_first_entry(&srcu_boot_list,
- struct srcu_struct, srcu_boot_entry);
+ sp = list_first_entry(&srcu_boot_list, struct srcu_struct,
+ work.work.entry);
check_init_srcu_struct(sp);
- list_del_init(&sp->srcu_boot_entry);
+ list_del_init(&sp->work.work.entry);
queue_work(rcu_gp_wq, &sp->work.work);
}
}