Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history
yaml
---
r: 269173
b: refs/heads/master
c: ca38062
h: refs/heads/master
i:
  269171: 257e0e6
v: v3
  • Loading branch information
Suresh Siddha authored and Ingo Molnar committed Oct 6, 2011
1 parent 239ef24 commit 91a071e
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 23 deletions.
2 changes: 1 addition & 1 deletion [refs]
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 9243a169acb9df9c63632fb7d5464359a107877a
refs/heads/master: ca38062e57e97791c2f62e3dbd06caf3ebb5721c
21 changes: 19 additions & 2 deletions trunk/kernel/sched.c
Original file line number Diff line number Diff line change
Expand Up @@ -1404,6 +1404,18 @@ void wake_up_idle_cpu(int cpu)
smp_send_reschedule(cpu);
}

static inline bool got_nohz_idle_kick(void)
{
return idle_cpu(smp_processor_id()) && this_rq()->nohz_balance_kick;
}

#else /* CONFIG_NO_HZ */

static inline bool got_nohz_idle_kick(void)
{
return false;
}

#endif /* CONFIG_NO_HZ */

static u64 sched_avg_period(void)
Expand Down Expand Up @@ -2717,7 +2729,7 @@ static void sched_ttwu_pending(void)

void scheduler_ipi(void)
{
if (llist_empty(&this_rq()->wake_list))
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
return;

/*
Expand All @@ -2735,6 +2747,12 @@ void scheduler_ipi(void)
*/
irq_enter();
sched_ttwu_pending();

/*
* Check if someone kicked us for doing the nohz idle load balance.
*/
if (unlikely(got_nohz_idle_kick() && !need_resched()))
raise_softirq_irqoff(SCHED_SOFTIRQ);
irq_exit();
}

Expand Down Expand Up @@ -8288,7 +8306,6 @@ void __init sched_init(void)
rq_attach_root(rq, &def_root_domain);
#ifdef CONFIG_NO_HZ
rq->nohz_balance_kick = 0;
init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
#endif
#endif
init_rq_hrtick(rq);
Expand Down
29 changes: 9 additions & 20 deletions trunk/kernel/sched_fair.c
Original file line number Diff line number Diff line change
Expand Up @@ -4269,22 +4269,6 @@ static int active_load_balance_cpu_stop(void *data)
}

#ifdef CONFIG_NO_HZ

static DEFINE_PER_CPU(struct call_single_data, remote_sched_softirq_cb);

static void trigger_sched_softirq(void *data)
{
raise_softirq_irqoff(SCHED_SOFTIRQ);
}

static inline void init_sched_softirq_csd(struct call_single_data *csd)
{
csd->func = trigger_sched_softirq;
csd->info = NULL;
csd->flags = 0;
csd->priv = 0;
}

/*
* idle load balancing details
* - One of the idle CPUs nominates itself as idle load_balancer, while
Expand Down Expand Up @@ -4450,11 +4434,16 @@ static void nohz_balancer_kick(int cpu)
}

if (!cpu_rq(ilb_cpu)->nohz_balance_kick) {
struct call_single_data *cp;

cpu_rq(ilb_cpu)->nohz_balance_kick = 1;
cp = &per_cpu(remote_sched_softirq_cb, cpu);
__smp_call_function_single(ilb_cpu, cp, 0);

smp_mb();
/*
* Use smp_send_reschedule() instead of resched_cpu().
* This way we generate a sched IPI on the target cpu which
* is idle. And the softirq performing nohz idle load balance
* will be run before returning from the IPI.
*/
smp_send_reschedule(ilb_cpu);
}
return;
}
Expand Down

0 comments on commit 91a071e

Please sign in to comment.