Skip to content

Commit

Permalink
rcu: Don't redundantly disable irqs in rcu_irq_{enter,exit}()
Browse files Browse the repository at this point in the history
This commit replaces a local_irq_save()/local_irq_restore() pair with
a lockdep assertion that interrupts are already disabled.  This should
remove the corresponding overhead from the interrupt entry/exit fastpaths.

This change was inspired by the fact that Iftekhar Ahmed's mutation
testing showed that removing rcu_irq_enter()'s call to local_ird_restore()
had no effect, which might indicate that interrupts were always enabled
anyway.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  • Loading branch information
Paul E. McKenney committed Dec 8, 2015
1 parent d117c8a commit 7c9906c
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 10 deletions.
4 changes: 2 additions & 2 deletions include/linux/rcupdate.h
Original file line number Diff line number Diff line change
Expand Up @@ -379,9 +379,9 @@ static inline void rcu_init_nohz(void)
*/
#define RCU_NONIDLE(a) \
do { \
rcu_irq_enter(); \
rcu_irq_enter_irqson(); \
do { a; } while (0); \
rcu_irq_exit(); \
rcu_irq_exit_irqson(); \
} while (0)

/*
Expand Down
8 changes: 8 additions & 0 deletions include/linux/rcutiny.h
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ static inline void rcu_irq_enter(void)
{
}

static inline void rcu_irq_exit_irqson(void)
{
}

static inline void rcu_irq_enter_irqson(void)
{
}

static inline void rcu_irq_exit(void)
{
}
Expand Down
2 changes: 2 additions & 0 deletions include/linux/rcutree.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ void rcu_idle_enter(void);
void rcu_idle_exit(void);
void rcu_irq_enter(void);
void rcu_irq_exit(void);
void rcu_irq_enter_irqson(void);
void rcu_irq_exit_irqson(void);

void exit_rcu(void);

Expand Down
4 changes: 2 additions & 2 deletions include/linux/tracepoint.h
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,8 @@ extern void syscall_unregfunc(void);
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), \
rcu_irq_enter(), \
rcu_irq_exit()); \
rcu_irq_enter_irqson(), \
rcu_irq_exit_irqson()); \
}
#else
#define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args)
Expand Down
32 changes: 26 additions & 6 deletions kernel/rcu/tree.c
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,7 @@ void rcu_user_enter(void)
*
* Exit from an interrupt handler, which might possibly result in entering
* idle mode, in other words, leaving the mode in which read-side critical
* sections can occur.
* sections can occur. The caller must have disabled interrupts.
*
* This code assumes that the idle loop never does anything that might
* result in unbalanced calls to irq_enter() and irq_exit(). If your
Expand All @@ -745,11 +745,10 @@ void rcu_user_enter(void)
*/
void rcu_irq_exit(void)
{
unsigned long flags;
long long oldval;
struct rcu_dynticks *rdtp;

local_irq_save(flags);
RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_exit() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting--;
Expand All @@ -760,6 +759,17 @@ void rcu_irq_exit(void)
else
rcu_eqs_enter_common(oldval, true);
rcu_sysidle_enter(1);
}

/*
* Wrapper for rcu_irq_exit() where interrupts are enabled.
*/
void rcu_irq_exit_irqson(void)
{
unsigned long flags;

local_irq_save(flags);
rcu_irq_exit();
local_irq_restore(flags);
}

Expand Down Expand Up @@ -857,7 +867,7 @@ void rcu_user_exit(void)
*
* Enter an interrupt handler, which might possibly result in exiting
* idle mode, in other words, entering the mode in which read-side critical
* sections can occur.
* sections can occur. The caller must have disabled interrupts.
*
* Note that the Linux kernel is fully capable of entering an interrupt
* handler that it never exits, for example when doing upcalls to
Expand All @@ -873,11 +883,10 @@ void rcu_user_exit(void)
*/
void rcu_irq_enter(void)
{
unsigned long flags;
struct rcu_dynticks *rdtp;
long long oldval;

local_irq_save(flags);
RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_enter() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting++;
Expand All @@ -888,6 +897,17 @@ void rcu_irq_enter(void)
else
rcu_eqs_exit_common(oldval, true);
rcu_sysidle_exit(1);
}

/*
* Wrapper for rcu_irq_enter() where interrupts are enabled.
*/
void rcu_irq_enter_irqson(void)
{
unsigned long flags;

local_irq_save(flags);
rcu_irq_enter();
local_irq_restore(flags);
}

Expand Down

0 comments on commit 7c9906c

Please sign in to comment.