Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 245758
b: refs/heads/master
c: 7e8b4c7
h: refs/heads/master
v: v3
  • Loading branch information
Paul E. McKenney authored and Paul E. McKenney committed May 6, 2011
1 parent 019e55e commit 9c1d410
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 75 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 203373c81b83e98da82836c4b8b5dd1e6fc9011f
refs/heads/master: 7e8b4c72344e0d904b0e3fa9fd2eb116f04b3d41
163 changes: 89 additions & 74 deletions trunk/kernel/rcutiny_plugin.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,15 +106,22 @@ struct rcu_preempt_ctrlblk {
unsigned long n_grace_periods;
#ifdef CONFIG_RCU_BOOST
unsigned long n_tasks_boosted;
/* Total number of tasks boosted. */
unsigned long n_exp_boosts;
/* Number of tasks boosted for expedited GP. */
unsigned long n_normal_boosts;
unsigned long n_normal_balk_blkd_tasks;
unsigned long n_normal_balk_gp_tasks;
unsigned long n_normal_balk_boost_tasks;
unsigned long n_normal_balk_notyet;
unsigned long n_normal_balk_nos;
unsigned long n_exp_balk_blkd_tasks;
unsigned long n_exp_balk_nos;
/* Number of tasks boosted for normal GP. */
unsigned long n_balk_blkd_tasks;
/* Refused to boost: no blocked tasks. */
unsigned long n_balk_exp_gp_tasks;
/* Refused to boost: nothing blocking GP. */
unsigned long n_balk_boost_tasks;
/* Refused to boost: already boosting. */
unsigned long n_balk_notyet;
/* Refused to boost: not yet time. */
unsigned long n_balk_nos;
/* Refused to boost: not sure why, though. */
/* This can happen due to race conditions. */
#endif /* #ifdef CONFIG_RCU_BOOST */
#endif /* #ifdef CONFIG_RCU_TRACE */
};
Expand Down Expand Up @@ -199,7 +206,6 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t)

#ifdef CONFIG_RCU_BOOST
static void rcu_initiate_boost_trace(void);
static void rcu_initiate_exp_boost_trace(void);
#endif /* #ifdef CONFIG_RCU_BOOST */

/*
Expand All @@ -225,16 +231,13 @@ static void show_tiny_preempt_stats(struct seq_file *m)
rcu_preempt_ctrlblk.n_normal_boosts,
(int)(jiffies & 0xffff),
(int)(rcu_preempt_ctrlblk.boost_time & 0xffff));
seq_printf(m, " %s: nt=%lu gt=%lu bt=%lu ny=%lu nos=%lu\n",
"normal balk",
rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks,
rcu_preempt_ctrlblk.n_normal_balk_gp_tasks,
rcu_preempt_ctrlblk.n_normal_balk_boost_tasks,
rcu_preempt_ctrlblk.n_normal_balk_notyet,
rcu_preempt_ctrlblk.n_normal_balk_nos);
seq_printf(m, " exp balk: bt=%lu nos=%lu\n",
rcu_preempt_ctrlblk.n_exp_balk_blkd_tasks,
rcu_preempt_ctrlblk.n_exp_balk_nos);
seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu ny=%lu nos=%lu\n",
" balk",
rcu_preempt_ctrlblk.n_balk_blkd_tasks,
rcu_preempt_ctrlblk.n_balk_exp_gp_tasks,
rcu_preempt_ctrlblk.n_balk_boost_tasks,
rcu_preempt_ctrlblk.n_balk_notyet,
rcu_preempt_ctrlblk.n_balk_nos);
#endif /* #ifdef CONFIG_RCU_BOOST */
}

Expand All @@ -252,23 +255,59 @@ static int rcu_boost(void)
{
unsigned long flags;
struct rt_mutex mtx;
struct list_head *np;
struct task_struct *t;
struct list_head *tb;

if (rcu_preempt_ctrlblk.boost_tasks == NULL)
if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
rcu_preempt_ctrlblk.exp_tasks == NULL)
return 0; /* Nothing to boost. */

raw_local_irq_save(flags);
t = container_of(rcu_preempt_ctrlblk.boost_tasks, struct task_struct,
rcu_node_entry);
np = rcu_next_node_entry(t);

/*
* Recheck with irqs disabled: all tasks in need of boosting
* might exit their RCU read-side critical sections on their own
* if we are preempted just before disabling irqs.
*/
if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
rcu_preempt_ctrlblk.exp_tasks == NULL) {
raw_local_irq_restore(flags);
return 0;
}

/*
* Preferentially boost tasks blocking expedited grace periods.
* This cannot starve the normal grace periods because a second
* expedited grace period must boost all blocked tasks, including
* those blocking the pre-existing normal grace period.
*/
if (rcu_preempt_ctrlblk.exp_tasks != NULL) {
tb = rcu_preempt_ctrlblk.exp_tasks;
RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++);
} else {
tb = rcu_preempt_ctrlblk.boost_tasks;
RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++);
}
RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++);

/*
* We boost task t by manufacturing an rt_mutex that appears to
* be held by task t. We leave a pointer to that rt_mutex where
* task t can find it, and task t will release the mutex when it
* exits its outermost RCU read-side critical section. Then
* simply acquiring this artificial rt_mutex will boost task
* t's priority. (Thanks to tglx for suggesting this approach!)
*/
t = container_of(tb, struct task_struct, rcu_node_entry);
rt_mutex_init_proxy_locked(&mtx, t);
t->rcu_boost_mutex = &mtx;
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
raw_local_irq_restore(flags);
rt_mutex_lock(&mtx);
RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++);
rt_mutex_unlock(&mtx);
return rcu_preempt_ctrlblk.boost_tasks != NULL;
rt_mutex_unlock(&mtx); /* Keep lockdep happy. */

return rcu_preempt_ctrlblk.boost_tasks != NULL ||
rcu_preempt_ctrlblk.exp_tasks != NULL;
}

/*
Expand All @@ -283,39 +322,24 @@ static int rcu_boost(void)
*/
static int rcu_initiate_boost(void)
{
if (!rcu_preempt_blocked_readers_cgp()) {
RCU_TRACE(rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks++);
if (!rcu_preempt_blocked_readers_cgp() &&
rcu_preempt_ctrlblk.exp_tasks == NULL) {
RCU_TRACE(rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++);
return 0;
}
if (rcu_preempt_ctrlblk.gp_tasks != NULL &&
rcu_preempt_ctrlblk.boost_tasks == NULL &&
ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time)) {
rcu_preempt_ctrlblk.boost_tasks = rcu_preempt_ctrlblk.gp_tasks;
if (rcu_preempt_ctrlblk.exp_tasks != NULL ||
(rcu_preempt_ctrlblk.gp_tasks != NULL &&
rcu_preempt_ctrlblk.boost_tasks == NULL &&
ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time))) {
if (rcu_preempt_ctrlblk.exp_tasks == NULL)
rcu_preempt_ctrlblk.boost_tasks =
rcu_preempt_ctrlblk.gp_tasks;
invoke_rcu_kthread();
RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++);
} else
RCU_TRACE(rcu_initiate_boost_trace());
return 1;
}

/*
* Initiate boosting for an expedited grace period.
*/
static void rcu_initiate_expedited_boost(void)
{
unsigned long flags;

raw_local_irq_save(flags);
if (!list_empty(&rcu_preempt_ctrlblk.blkd_tasks)) {
rcu_preempt_ctrlblk.boost_tasks =
rcu_preempt_ctrlblk.blkd_tasks.next;
invoke_rcu_kthread();
RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++);
} else
RCU_TRACE(rcu_initiate_exp_boost_trace());
raw_local_irq_restore(flags);
}

#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)

/*
Expand Down Expand Up @@ -346,13 +370,6 @@ static int rcu_initiate_boost(void)
return rcu_preempt_blocked_readers_cgp();
}

/*
* If there is no RCU priority boosting, we don't initiate expedited boosting.
*/
static void rcu_initiate_expedited_boost(void)
{
}

/*
* If there is no RCU priority boosting, nothing to do at grace-period start.
*/
Expand Down Expand Up @@ -786,13 +803,16 @@ void synchronize_rcu_expedited(void)
rpcp->exp_tasks = rpcp->blkd_tasks.next;
if (rpcp->exp_tasks == &rpcp->blkd_tasks)
rpcp->exp_tasks = NULL;
local_irq_restore(flags);

/* Wait for tail of ->blkd_tasks list to drain. */
if (rcu_preempted_readers_exp())
rcu_initiate_expedited_boost();
if (!rcu_preempted_readers_exp())
local_irq_restore(flags);
else {
rcu_initiate_boost();
local_irq_restore(flags);
wait_event(sync_rcu_preempt_exp_wq,
!rcu_preempted_readers_exp());
}

/* Clean up and exit. */
barrier(); /* ensure expedited GP seen before counter increment. */
Expand Down Expand Up @@ -905,22 +925,17 @@ void __init rcu_scheduler_starting(void)

static void rcu_initiate_boost_trace(void)
{
if (rcu_preempt_ctrlblk.gp_tasks == NULL)
rcu_preempt_ctrlblk.n_normal_balk_gp_tasks++;
if (list_empty(&rcu_preempt_ctrlblk.blkd_tasks))
rcu_preempt_ctrlblk.n_balk_blkd_tasks++;
else if (rcu_preempt_ctrlblk.gp_tasks == NULL &&
rcu_preempt_ctrlblk.exp_tasks == NULL)
rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++;
else if (rcu_preempt_ctrlblk.boost_tasks != NULL)
rcu_preempt_ctrlblk.n_normal_balk_boost_tasks++;
rcu_preempt_ctrlblk.n_balk_boost_tasks++;
else if (!ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time))
rcu_preempt_ctrlblk.n_normal_balk_notyet++;
else
rcu_preempt_ctrlblk.n_normal_balk_nos++;
}

static void rcu_initiate_exp_boost_trace(void)
{
if (list_empty(&rcu_preempt_ctrlblk.blkd_tasks))
rcu_preempt_ctrlblk.n_exp_balk_blkd_tasks++;
rcu_preempt_ctrlblk.n_balk_notyet++;
else
rcu_preempt_ctrlblk.n_exp_balk_nos++;
rcu_preempt_ctrlblk.n_balk_nos++;
}

#endif /* #ifdef CONFIG_RCU_BOOST */
Expand Down

0 comments on commit 9c1d410

Please sign in to comment.