From e52b1db37b89b69ceb08b521a808bd2cf4724481 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 8 Aug 2012 11:10:25 -0700 Subject: timer: Generalize timer->base flags handling To prepare for addition of another flag, generalize timer->base flags handling. * Rename from TBASE_*_FLAG to TIMER_* and make them LU constants. * Define and use TIMER_FLAG_MASK for flags masking so that multiple flags can be handled correctly. * Don't dereference timer->base directly even if !tbase_get_deferrable(). All two such places are already passed in @base, so use it instead. * Make sure tvec_base's alignment is large enough for timer->base flags using BUILD_BUG_ON(). Signed-off-by: Tejun Heo Cc: torvalds@linux-foundation.org Cc: peterz@infradead.org Link: http://lkml.kernel.org/r/1344449428-24962-2-git-send-email-tj@kernel.org Signed-off-by: Thomas Gleixner --- include/linux/timer.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/timer.h b/include/linux/timer.h index 6abd9138beda..cbd32ec4dd15 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -58,7 +58,9 @@ extern struct tvec_base boot_tvec_bases; * the timer will be serviced when the CPU eventually wakes up with a * subsequent non-deferrable timer. */ -#define TBASE_DEFERRABLE_FLAG (0x1) +#define TIMER_DEFERRABLE 0x1LU + +#define TIMER_FLAG_MASK 0x1LU #define TIMER_INITIALIZER(_function, _expires, _data) { \ .entry = { .prev = TIMER_ENTRY_STATIC }, \ @@ -72,7 +74,7 @@ extern struct tvec_base boot_tvec_bases; } #define TBASE_MAKE_DEFERRED(ptr) ((struct tvec_base *) \ - ((unsigned char *)(ptr) + TBASE_DEFERRABLE_FLAG)) + ((unsigned char *)(ptr) + TIMER_DEFERRABLE)) #define TIMER_DEFERRED_INITIALIZER(_function, _expires, _data) {\ .entry = { .prev = TIMER_ENTRY_STATIC }, \ -- cgit v1.2.3 From 5a9af38d05f6a1bd0d3f1f69a074cdbe9c87e977 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 8 Aug 2012 11:10:26 -0700 Subject: timer: Relocate declarations of init_timer_on_stack_key() init_timer_on_stack_key() is used by init macro definitions. Move init_timer_on_stack_key() and destroy_timer_on_stack() declarations above init macro defs. This will make the next init cleanup patch easier to read. Signed-off-by: Tejun Heo Cc: torvalds@linux-foundation.org Cc: peterz@infradead.org Link: http://lkml.kernel.org/r/1344449428-24962-3-git-send-email-tj@kernel.org Signed-off-by: Thomas Gleixner --- include/linux/timer.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/include/linux/timer.h b/include/linux/timer.h index cbd32ec4dd15..1d364aed1802 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -97,6 +97,21 @@ void init_timer_deferrable_key(struct timer_list *timer, const char *name, struct lock_class_key *key); +#ifdef CONFIG_DEBUG_OBJECTS_TIMERS +extern void init_timer_on_stack_key(struct timer_list *timer, + const char *name, + struct lock_class_key *key); +extern void destroy_timer_on_stack(struct timer_list *timer); +#else +static inline void destroy_timer_on_stack(struct timer_list *timer) { } +static inline void init_timer_on_stack_key(struct timer_list *timer, + const char *name, + struct lock_class_key *key) +{ + init_timer_key(timer, name, key); +} +#endif + #ifdef CONFIG_LOCKDEP #define init_timer(timer) \ do { \ @@ -150,21 +165,6 @@ void init_timer_deferrable_key(struct timer_list *timer, setup_deferrable_timer_on_stack_key((timer), NULL, NULL, (fn), (data)) #endif -#ifdef CONFIG_DEBUG_OBJECTS_TIMERS -extern void init_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key); -extern void destroy_timer_on_stack(struct timer_list *timer); -#else -static inline void destroy_timer_on_stack(struct timer_list *timer) { } -static inline void init_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key) -{ - init_timer_key(timer, name, key); -} -#endif - static inline void setup_timer_key(struct timer_list * timer, const char *name, struct lock_class_key *key, -- cgit v1.2.3 From fc683995a6c4e604d62ab9a488ac2c1ba94fa868 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 8 Aug 2012 11:10:27 -0700 Subject: timer: Clean up timer initializers Over time, timer initializers became messy with unnecessarily duplicated code which are inconsistently spread across timer.h and timer.c. This patch cleans up timer initializers. * timer.c::__init_timer() is renamed to do_init_timer(). * __TIMER_INITIALIZER() added. It takes @flags and all initializers are wrappers around it. * init_timer[_on_stack]_key() now take @flags. * __init_timer[_on_stack]() added. They take @flags and all init macros are wrappers around them. * __setup_timer[_on_stack]() added. It uses __init_timer() and takes @flags. All setup macros are wrappers around the two. Note that this patch doesn't add missing init/setup combinations - e.g. init_timer_deferrable_on_stack(). Adding missing ones is trivial. Signed-off-by: Tejun Heo Cc: torvalds@linux-foundation.org Cc: peterz@infradead.org Link: http://lkml.kernel.org/r/1344449428-24962-4-git-send-email-tj@kernel.org Signed-off-by: Thomas Gleixner --- include/linux/timer.h | 123 ++++++++++++++++---------------------------------- kernel/timer.c | 56 ++++++----------------- 2 files changed, 53 insertions(+), 126 deletions(-) (limited to 'include') diff --git a/include/linux/timer.h b/include/linux/timer.h index 1d364aed1802..3f95c1fa615e 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -62,136 +62,91 @@ extern struct tvec_base boot_tvec_bases; #define TIMER_FLAG_MASK 0x1LU -#define TIMER_INITIALIZER(_function, _expires, _data) { \ +#define __TIMER_INITIALIZER(_function, _expires, _data, _flags) { \ .entry = { .prev = TIMER_ENTRY_STATIC }, \ .function = (_function), \ .expires = (_expires), \ .data = (_data), \ - .base = &boot_tvec_bases, \ + .base = (void *)((unsigned long)&boot_tvec_bases + (_flags)), \ .slack = -1, \ __TIMER_LOCKDEP_MAP_INITIALIZER( \ __FILE__ ":" __stringify(__LINE__)) \ } -#define TBASE_MAKE_DEFERRED(ptr) ((struct tvec_base *) \ - ((unsigned char *)(ptr) + TIMER_DEFERRABLE)) +#define TIMER_INITIALIZER(_function, _expires, _data) \ + __TIMER_INITIALIZER((_function), (_expires), (_data), 0) -#define TIMER_DEFERRED_INITIALIZER(_function, _expires, _data) {\ - .entry = { .prev = TIMER_ENTRY_STATIC }, \ - .function = (_function), \ - .expires = (_expires), \ - .data = (_data), \ - .base = TBASE_MAKE_DEFERRED(&boot_tvec_bases), \ - __TIMER_LOCKDEP_MAP_INITIALIZER( \ - __FILE__ ":" __stringify(__LINE__)) \ - } +#define TIMER_DEFERRED_INITIALIZER(_function, _expires, _data) \ + __TIMER_INITIALIZER((_function), (_expires), (_data), TIMER_DEFERRABLE) #define DEFINE_TIMER(_name, _function, _expires, _data) \ struct timer_list _name = \ TIMER_INITIALIZER(_function, _expires, _data) -void init_timer_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key); -void init_timer_deferrable_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key); +void init_timer_key(struct timer_list *timer, unsigned int flags, + const char *name, struct lock_class_key *key); #ifdef CONFIG_DEBUG_OBJECTS_TIMERS extern void init_timer_on_stack_key(struct timer_list *timer, - const char *name, + unsigned int flags, const char *name, struct lock_class_key *key); extern void destroy_timer_on_stack(struct timer_list *timer); #else static inline void destroy_timer_on_stack(struct timer_list *timer) { } static inline void init_timer_on_stack_key(struct timer_list *timer, - const char *name, + unsigned int flags, const char *name, struct lock_class_key *key) { - init_timer_key(timer, name, key); + init_timer_key(timer, flags, name, key); } #endif #ifdef CONFIG_LOCKDEP -#define init_timer(timer) \ +#define __init_timer(_timer, _flags) \ do { \ static struct lock_class_key __key; \ - init_timer_key((timer), #timer, &__key); \ + init_timer_key((_timer), (_flags), #_timer, &__key); \ } while (0) -#define init_timer_deferrable(timer) \ +#define __init_timer_on_stack(_timer, _flags) \ do { \ static struct lock_class_key __key; \ - init_timer_deferrable_key((timer), #timer, &__key); \ + init_timer_on_stack_key((_timer), (_flags), #_timer, &__key); \ } while (0) +#else +#define __init_timer(_timer, _flags) \ + init_timer_key((_timer), (_flags), NULL, NULL) +#define __init_timer_on_stack(_timer, _flags) \ + init_timer_on_stack_key((_timer), (_flags), NULL, NULL) +#endif +#define init_timer(timer) \ + __init_timer((timer), 0) +#define init_timer_deferrable(timer) \ + __init_timer((timer), TIMER_DEFERRABLE) #define init_timer_on_stack(timer) \ + __init_timer_on_stack((timer), 0) + +#define __setup_timer(_timer, _fn, _data, _flags) \ do { \ - static struct lock_class_key __key; \ - init_timer_on_stack_key((timer), #timer, &__key); \ + __init_timer((_timer), (_flags)); \ + (_timer)->function = (_fn); \ + (_timer)->data = (_data); \ } while (0) -#define setup_timer(timer, fn, data) \ +#define __setup_timer_on_stack(_timer, _fn, _data, _flags) \ do { \ - static struct lock_class_key __key; \ - setup_timer_key((timer), #timer, &__key, (fn), (data));\ + __init_timer_on_stack((_timer), (_flags)); \ + (_timer)->function = (_fn); \ + (_timer)->data = (_data); \ } while (0) +#define setup_timer(timer, fn, data) \ + __setup_timer((timer), (fn), (data), 0) #define setup_timer_on_stack(timer, fn, data) \ - do { \ - static struct lock_class_key __key; \ - setup_timer_on_stack_key((timer), #timer, &__key, \ - (fn), (data)); \ - } while (0) + __setup_timer_on_stack((timer), (fn), (data), 0) #define setup_deferrable_timer_on_stack(timer, fn, data) \ - do { \ - static struct lock_class_key __key; \ - setup_deferrable_timer_on_stack_key((timer), #timer, \ - &__key, (fn), \ - (data)); \ - } while (0) -#else -#define init_timer(timer)\ - init_timer_key((timer), NULL, NULL) -#define init_timer_deferrable(timer)\ - init_timer_deferrable_key((timer), NULL, NULL) -#define init_timer_on_stack(timer)\ - init_timer_on_stack_key((timer), NULL, NULL) -#define setup_timer(timer, fn, data)\ - setup_timer_key((timer), NULL, NULL, (fn), (data)) -#define setup_timer_on_stack(timer, fn, data)\ - setup_timer_on_stack_key((timer), NULL, NULL, (fn), (data)) -#define setup_deferrable_timer_on_stack(timer, fn, data)\ - setup_deferrable_timer_on_stack_key((timer), NULL, NULL, (fn), (data)) -#endif - -static inline void setup_timer_key(struct timer_list * timer, - const char *name, - struct lock_class_key *key, - void (*function)(unsigned long), - unsigned long data) -{ - timer->function = function; - timer->data = data; - init_timer_key(timer, name, key); -} - -static inline void setup_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key, - void (*function)(unsigned long), - unsigned long data) -{ - timer->function = function; - timer->data = data; - init_timer_on_stack_key(timer, name, key); -} - -extern void setup_deferrable_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key, - void (*function)(unsigned long), - unsigned long data); + __setup_timer_on_stack((timer), (fn), (data), TIMER_DEFERRABLE) /** * timer_pending - is a timer pending? diff --git a/kernel/timer.c b/kernel/timer.c index cf7af56940b7..8d185a1677cc 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -100,11 +100,6 @@ static inline struct tvec_base *tbase_get_base(struct tvec_base *base) return ((struct tvec_base *)((unsigned long)base & ~TIMER_FLAG_MASK)); } -static inline void timer_set_deferrable(struct timer_list *timer) -{ - timer->base = TBASE_MAKE_DEFERRED(timer->base); -} - static inline void timer_set_base(struct timer_list *timer, struct tvec_base *new_base) { @@ -564,16 +559,14 @@ static inline void debug_timer_assert_init(struct timer_list *timer) debug_object_assert_init(timer, &timer_debug_descr); } -static void __init_timer(struct timer_list *timer, - const char *name, - struct lock_class_key *key); +static void do_init_timer(struct timer_list *timer, unsigned int flags, + const char *name, struct lock_class_key *key); -void init_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key) +void init_timer_on_stack_key(struct timer_list *timer, unsigned int flags, + const char *name, struct lock_class_key *key) { debug_object_init_on_stack(timer, &timer_debug_descr); - __init_timer(timer, name, key); + do_init_timer(timer, flags, name, key); } EXPORT_SYMBOL_GPL(init_timer_on_stack_key); @@ -614,12 +607,13 @@ static inline void debug_assert_init(struct timer_list *timer) debug_timer_assert_init(timer); } -static void __init_timer(struct timer_list *timer, - const char *name, - struct lock_class_key *key) +static void do_init_timer(struct timer_list *timer, unsigned int flags, + const char *name, struct lock_class_key *key) { + struct tvec_base *base = __raw_get_cpu_var(tvec_bases); + timer->entry.next = NULL; - timer->base = __raw_get_cpu_var(tvec_bases); + timer->base = (void *)((unsigned long)base | flags); timer->slack = -1; #ifdef CONFIG_TIMER_STATS timer->start_site = NULL; @@ -629,22 +623,10 @@ static void __init_timer(struct timer_list *timer, lockdep_init_map(&timer->lockdep_map, name, key, 0); } -void setup_deferrable_timer_on_stack_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key, - void (*function)(unsigned long), - unsigned long data) -{ - timer->function = function; - timer->data = data; - init_timer_on_stack_key(timer, name, key); - timer_set_deferrable(timer); -} -EXPORT_SYMBOL_GPL(setup_deferrable_timer_on_stack_key); - /** * init_timer_key - initialize a timer * @timer: the timer to be initialized + * @flags: timer flags * @name: name of the timer * @key: lockdep class key of the fake lock used for tracking timer * sync lock dependencies @@ -652,24 +634,14 @@ EXPORT_SYMBOL_GPL(setup_deferrable_timer_on_stack_key); * init_timer_key() must be done to a timer prior calling *any* of the * other timer functions. */ -void init_timer_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key) +void init_timer_key(struct timer_list *timer, unsigned int flags, + const char *name, struct lock_class_key *key) { debug_init(timer); - __init_timer(timer, name, key); + do_init_timer(timer, flags, name, key); } EXPORT_SYMBOL(init_timer_key); -void init_timer_deferrable_key(struct timer_list *timer, - const char *name, - struct lock_class_key *key) -{ - init_timer_key(timer, name, key); - timer_set_deferrable(timer); -} -EXPORT_SYMBOL(init_timer_deferrable_key); - static inline void detach_timer(struct timer_list *timer, bool clear_pending) { struct list_head *entry = &timer->entry; -- cgit v1.2.3 From c5f66e99b7cb091e3d51ae8e8156892e8feb7fa3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 8 Aug 2012 11:10:28 -0700 Subject: timer: Implement TIMER_IRQSAFE Timer internals are protected with irq-safe locks but timer execution isn't, so a timer being dequeued for execution and its execution aren't atomic against IRQs. This makes it impossible to wait for its completion from IRQ handlers and difficult to shoot down a timer from IRQ handlers. This issue caused some issues for delayed_work interface. Because there's no way to reliably shoot down delayed_work->timer from IRQ handlers, __cancel_delayed_work() can't share the logic to steal the target delayed_work with cancel_delayed_work_sync(), and can only steal delayed_works which are on queued on timer. Similarly, the pending mod_delayed_work() can't be used from IRQ handlers. This patch adds a new timer flag TIMER_IRQSAFE, which makes the timer to be executed without enabling IRQ after dequeueing such that its dequeueing and execution are atomic against IRQ handlers. This makes it safe to wait for the timer's completion from IRQ handlers, for example, using del_timer_sync(). It can never be executing on the local CPU and if executing on other CPUs it won't be interrupted until done. This will enable simplifying delayed_work cancel/mod interface. Signed-off-by: Tejun Heo Cc: torvalds@linux-foundation.org Cc: peterz@infradead.org Link: http://lkml.kernel.org/r/1344449428-24962-5-git-send-email-tj@kernel.org Signed-off-by: Thomas Gleixner --- include/linux/timer.h | 16 ++++++++++++---- kernel/timer.c | 35 ++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/include/linux/timer.h b/include/linux/timer.h index 3f95c1fa615e..8c5a197e1587 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -49,18 +49,26 @@ extern struct tvec_base boot_tvec_bases; #endif /* - * Note that all tvec_bases are 2 byte aligned and lower bit of - * base in timer_list is guaranteed to be zero. Use the LSB to - * indicate whether the timer is deferrable. + * Note that all tvec_bases are at least 4 byte aligned and lower two bits + * of base in timer_list is guaranteed to be zero. Use them for flags. * * A deferrable timer will work normally when the system is busy, but * will not cause a CPU to come out of idle just to service it; instead, * the timer will be serviced when the CPU eventually wakes up with a * subsequent non-deferrable timer. + * + * An irqsafe timer is executed with IRQ disabled and it's safe to wait for + * the completion of the running instance from IRQ handlers, for example, + * by calling del_timer_sync(). + * + * Note: The irq disabled callback execution is a special case for + * workqueue locking issues. It's not meant for executing random crap + * with interrupts disabled. Abuse is monitored! */ #define TIMER_DEFERRABLE 0x1LU +#define TIMER_IRQSAFE 0x2LU -#define TIMER_FLAG_MASK 0x1LU +#define TIMER_FLAG_MASK 0x3LU #define __TIMER_INITIALIZER(_function, _expires, _data, _flags) { \ .entry = { .prev = TIMER_ENTRY_STATIC }, \ diff --git a/kernel/timer.c b/kernel/timer.c index 8d185a1677cc..706fe4c53e82 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -95,6 +95,11 @@ static inline unsigned int tbase_get_deferrable(struct tvec_base *base) return ((unsigned int)(unsigned long)base & TIMER_DEFERRABLE); } +static inline unsigned int tbase_get_irqsafe(struct tvec_base *base) +{ + return ((unsigned int)(unsigned long)base & TIMER_IRQSAFE); +} + static inline struct tvec_base *tbase_get_base(struct tvec_base *base) { return ((struct tvec_base *)((unsigned long)base & ~TIMER_FLAG_MASK)); @@ -1002,14 +1007,14 @@ EXPORT_SYMBOL(try_to_del_timer_sync); * * Synchronization rules: Callers must prevent restarting of the timer, * otherwise this function is meaningless. It must not be called from - * interrupt contexts. The caller must not hold locks which would prevent - * completion of the timer's handler. The timer's handler must not call - * add_timer_on(). Upon exit the timer is not queued and the handler is - * not running on any CPU. + * interrupt contexts unless the timer is an irqsafe one. The caller must + * not hold locks which would prevent completion of the timer's + * handler. The timer's handler must not call add_timer_on(). Upon exit the + * timer is not queued and the handler is not running on any CPU. * - * Note: You must not hold locks that are held in interrupt context - * while calling this function. Even if the lock has nothing to do - * with the timer in question. Here's why: + * Note: For !irqsafe timers, you must not hold locks that are held in + * interrupt context while calling this function. Even if the lock has + * nothing to do with the timer in question. Here's why: * * CPU0 CPU1 * ---- ---- @@ -1046,7 +1051,7 @@ int del_timer_sync(struct timer_list *timer) * don't use it in hardirq context, because it * could lead to deadlock. */ - WARN_ON(in_irq()); + WARN_ON(in_irq() && !tbase_get_irqsafe(timer->base)); for (;;) { int ret = try_to_del_timer_sync(timer); if (ret >= 0) @@ -1153,19 +1158,27 @@ static inline void __run_timers(struct tvec_base *base) while (!list_empty(head)) { void (*fn)(unsigned long); unsigned long data; + bool irqsafe; timer = list_first_entry(head, struct timer_list,entry); fn = timer->function; data = timer->data; + irqsafe = tbase_get_irqsafe(timer->base); timer_stats_account_timer(timer); base->running_timer = timer; detach_expired_timer(timer, base); - spin_unlock_irq(&base->lock); - call_timer_fn(timer, fn, data); - spin_lock_irq(&base->lock); + if (irqsafe) { + spin_unlock(&base->lock); + call_timer_fn(timer, fn, data); + spin_lock(&base->lock); + } else { + spin_unlock_irq(&base->lock); + call_timer_fn(timer, fn, data); + spin_lock_irq(&base->lock); + } } } base->running_timer = NULL; -- cgit v1.2.3