Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 147172
b: refs/heads/master
c: aa9c4c0
h: refs/heads/master
v: v3
  • Loading branch information
Ingo Molnar committed Dec 23, 2008
1 parent 1a055cc commit 8e37368
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 25 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 7671581f1666ef4b54a1c1e598c51ac44c060a9b
refs/heads/master: aa9c4c0f967fdb482ea95e8473ec3d201e6e0781
8 changes: 8 additions & 0 deletions trunk/include/linux/kernel_stat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,15 @@ static inline unsigned int kstat_irqs(unsigned int irq)
return sum;
}


/*
* Lock/unlock the current runqueue - to extract task statistics:
*/
extern void curr_rq_lock_irq_save(unsigned long *flags);
extern void curr_rq_unlock_irq_restore(unsigned long *flags);
extern unsigned long long __task_delta_exec(struct task_struct *tsk, int update);
extern unsigned long long task_delta_exec(struct task_struct *);

extern void account_user_time(struct task_struct *, cputime_t);
extern void account_user_time_scaled(struct task_struct *, cputime_t);
extern void account_system_time(struct task_struct *, int, cputime_t);
Expand Down
17 changes: 12 additions & 5 deletions trunk/kernel/exit.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,12 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
forget_original_parent(tsk);
exit_task_namespaces(tsk);

/*
* Flush inherited counters to the parent - before the parent
* gets woken up by child-exit notifications.
*/
perf_counter_exit_task(tsk);

write_lock_irq(&tasklist_lock);
if (group_dead)
kill_orphaned_pgrp(tsk->group_leader, NULL);
Expand Down Expand Up @@ -1093,11 +1099,6 @@ NORET_TYPE void do_exit(long code)
mpol_put(tsk->mempolicy);
tsk->mempolicy = NULL;
#endif
/*
* These must happen late, after the PID is not
* hashed anymore, but still at a point that may sleep:
*/
perf_counter_exit_task(tsk);
#ifdef CONFIG_FUTEX
if (unlikely(!list_empty(&tsk->pi_state_list)))
exit_pi_state_list(tsk);
Expand All @@ -1121,6 +1122,12 @@ NORET_TYPE void do_exit(long code)
if (tsk->splice_pipe)
__free_pipe_info(tsk->splice_pipe);

/*
* These must happen late, after the PID is not
* hashed anymore, but still at a point that may sleep:
*/
perf_counter_exit_task(tsk);

preempt_disable();
/* causes final put_task_struct in finish_task_switch(). */
tsk->state = TASK_DEAD;
Expand Down
70 changes: 54 additions & 16 deletions trunk/kernel/perf_counter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
#include <linux/kernel_stat.h>
#include <linux/perf_counter.h>

/*
Expand Down Expand Up @@ -106,7 +107,8 @@ static void __perf_counter_remove_from_context(void *info)
if (ctx->task && cpuctx->task_ctx != ctx)
return;

spin_lock_irqsave(&ctx->lock, flags);
curr_rq_lock_irq_save(&flags);
spin_lock(&ctx->lock);

if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
counter->hw_ops->disable(counter);
Expand Down Expand Up @@ -135,7 +137,8 @@ static void __perf_counter_remove_from_context(void *info)
perf_max_counters - perf_reserved_percpu);
}

spin_unlock_irqrestore(&ctx->lock, flags);
spin_unlock(&ctx->lock);
curr_rq_unlock_irq_restore(&flags);
}


Expand Down Expand Up @@ -209,7 +212,8 @@ static void __perf_install_in_context(void *info)
if (ctx->task && cpuctx->task_ctx != ctx)
return;

spin_lock_irqsave(&ctx->lock, flags);
curr_rq_lock_irq_save(&flags);
spin_lock(&ctx->lock);

/*
* Protect the list operation against NMI by disabling the
Expand All @@ -232,7 +236,8 @@ static void __perf_install_in_context(void *info)
if (!ctx->task && cpuctx->max_pertask)
cpuctx->max_pertask--;

spin_unlock_irqrestore(&ctx->lock, flags);
spin_unlock(&ctx->lock);
curr_rq_unlock_irq_restore(&flags);
}

/*
Expand Down Expand Up @@ -438,15 +443,19 @@ int perf_counter_task_disable(void)
struct task_struct *curr = current;
struct perf_counter_context *ctx = &curr->perf_counter_ctx;
struct perf_counter *counter;
unsigned long flags;
u64 perf_flags;
int cpu;

if (likely(!ctx->nr_counters))
return 0;

local_irq_disable();
curr_rq_lock_irq_save(&flags);
cpu = smp_processor_id();

/* force the update of the task clock: */
__task_delta_exec(curr, 1);

perf_counter_task_sched_out(curr, cpu);

spin_lock(&ctx->lock);
Expand All @@ -463,7 +472,7 @@ int perf_counter_task_disable(void)

spin_unlock(&ctx->lock);

local_irq_enable();
curr_rq_unlock_irq_restore(&flags);

return 0;
}
Expand All @@ -473,15 +482,19 @@ int perf_counter_task_enable(void)
struct task_struct *curr = current;
struct perf_counter_context *ctx = &curr->perf_counter_ctx;
struct perf_counter *counter;
unsigned long flags;
u64 perf_flags;
int cpu;

if (likely(!ctx->nr_counters))
return 0;

local_irq_disable();
curr_rq_lock_irq_save(&flags);
cpu = smp_processor_id();

/* force the update of the task clock: */
__task_delta_exec(curr, 1);

spin_lock(&ctx->lock);

/*
Expand All @@ -493,14 +506,15 @@ int perf_counter_task_enable(void)
if (counter->state != PERF_COUNTER_STATE_OFF)
continue;
counter->state = PERF_COUNTER_STATE_INACTIVE;
counter->hw_event.disabled = 0;
}
hw_perf_restore(perf_flags);

spin_unlock(&ctx->lock);

perf_counter_task_sched_in(curr, cpu);

local_irq_enable();
curr_rq_unlock_irq_restore(&flags);

return 0;
}
Expand Down Expand Up @@ -540,8 +554,11 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
static void __read(void *info)
{
struct perf_counter *counter = info;
unsigned long flags;

curr_rq_lock_irq_save(&flags);
counter->hw_ops->read(counter);
curr_rq_unlock_irq_restore(&flags);
}

static u64 perf_counter_read(struct perf_counter *counter)
Expand Down Expand Up @@ -860,13 +877,27 @@ static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
.read = cpu_clock_perf_counter_read,
};

static void task_clock_perf_counter_update(struct perf_counter *counter)
/*
* Called from within the scheduler:
*/
static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
{
u64 prev, now;
struct task_struct *curr = counter->task;
u64 delta;

WARN_ON_ONCE(counter->task != current);

delta = __task_delta_exec(curr, update);

return curr->se.sum_exec_runtime + delta;
}

static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
{
u64 prev;
s64 delta;

prev = atomic64_read(&counter->hw.prev_count);
now = current->se.sum_exec_runtime;

atomic64_set(&counter->hw.prev_count, now);

Expand All @@ -877,17 +908,23 @@ static void task_clock_perf_counter_update(struct perf_counter *counter)

static void task_clock_perf_counter_read(struct perf_counter *counter)
{
task_clock_perf_counter_update(counter);
u64 now = task_clock_perf_counter_val(counter, 1);

task_clock_perf_counter_update(counter, now);
}

static void task_clock_perf_counter_enable(struct perf_counter *counter)
{
atomic64_set(&counter->hw.prev_count, current->se.sum_exec_runtime);
u64 now = task_clock_perf_counter_val(counter, 0);

atomic64_set(&counter->hw.prev_count, now);
}

static void task_clock_perf_counter_disable(struct perf_counter *counter)
{
task_clock_perf_counter_update(counter);
u64 now = task_clock_perf_counter_val(counter, 0);

task_clock_perf_counter_update(counter, now);
}

static const struct hw_perf_counter_ops perf_ops_task_clock = {
Expand Down Expand Up @@ -1267,6 +1304,7 @@ __perf_counter_exit_task(struct task_struct *child,
{
struct perf_counter *parent_counter;
u64 parent_val, child_val;
unsigned long flags;
u64 perf_flags;

/*
Expand All @@ -1275,7 +1313,7 @@ __perf_counter_exit_task(struct task_struct *child,
* Be careful about zapping the list - IRQ/NMI context
* could still be processing it:
*/
local_irq_disable();
curr_rq_lock_irq_save(&flags);
perf_flags = hw_perf_save_disable();

if (child_counter->state == PERF_COUNTER_STATE_ACTIVE) {
Expand All @@ -1294,7 +1332,7 @@ __perf_counter_exit_task(struct task_struct *child,
list_del_init(&child_counter->list_entry);

hw_perf_restore(perf_flags);
local_irq_enable();
curr_rq_unlock_irq_restore(&flags);

parent_counter = child_counter->parent;
/*
Expand Down
49 changes: 46 additions & 3 deletions trunk/kernel/sched.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ static inline int cpu_of(struct rq *rq)
#define task_rq(p) cpu_rq(task_cpu(p))
#define cpu_curr(cpu) (cpu_rq(cpu)->curr)

static inline void update_rq_clock(struct rq *rq)
inline void update_rq_clock(struct rq *rq)
{
rq->clock = sched_clock_cpu(cpu_of(rq));
}
Expand Down Expand Up @@ -969,6 +969,26 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
}
}

void curr_rq_lock_irq_save(unsigned long *flags)
__acquires(rq->lock)
{
struct rq *rq;

local_irq_save(*flags);
rq = cpu_rq(smp_processor_id());
spin_lock(&rq->lock);
}

void curr_rq_unlock_irq_restore(unsigned long *flags)
__releases(rq->lock)
{
struct rq *rq;

rq = cpu_rq(smp_processor_id());
spin_unlock(&rq->lock);
local_irq_restore(*flags);
}

void task_rq_unlock_wait(struct task_struct *p)
{
struct rq *rq = task_rq(p);
Expand Down Expand Up @@ -2558,7 +2578,6 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
struct task_struct *next)
{
fire_sched_out_preempt_notifiers(prev, next);
perf_counter_task_sched_out(prev, cpu_of(rq));
prepare_lock_switch(rq, next);
prepare_arch_switch(next);
}
Expand Down Expand Up @@ -4089,6 +4108,29 @@ DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
* Return any ns on the sched_clock that have not yet been banked in
* @p in case that task is currently running.
*/
unsigned long long __task_delta_exec(struct task_struct *p, int update)
{
s64 delta_exec;
struct rq *rq;

rq = task_rq(p);
WARN_ON_ONCE(!runqueue_is_locked());
WARN_ON_ONCE(!task_current(rq, p));

if (update)
update_rq_clock(rq);

delta_exec = rq->clock - p->se.exec_start;

WARN_ON_ONCE(delta_exec < 0);

return delta_exec;
}

/*
* Return any ns on the sched_clock that have not yet been banked in
* @p in case that task is currently running.
Expand Down Expand Up @@ -4316,13 +4358,13 @@ void scheduler_tick(void)
update_rq_clock(rq);
update_cpu_load(rq);
curr->sched_class->task_tick(rq, curr, 0);
perf_counter_task_tick(curr, cpu);
spin_unlock(&rq->lock);

#ifdef CONFIG_SMP
rq->idle_at_tick = idle_cpu(cpu);
trigger_load_balance(rq, cpu);
#endif
perf_counter_task_tick(curr, cpu);
}

#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
Expand Down Expand Up @@ -4512,6 +4554,7 @@ asmlinkage void __sched schedule(void)

if (likely(prev != next)) {
sched_info_switch(prev, next);
perf_counter_task_sched_out(prev, cpu);

rq->nr_switches++;
rq->curr = next;
Expand Down

0 comments on commit 8e37368

Please sign in to comment.