Skip to content

Commit

Permalink
perf, trace: Optimize tracepoints by using per-tracepoint-per-cpu hli…
Browse files Browse the repository at this point in the history 
…st to track events

Avoid the swevent hash-table by using per-tracepoint
hlists.

Also, avoid conditionals on the fast path by ordering
with probe unregister so that we should never get on
the callback path without the data being there.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
LKML-Reference: <20100521090710.473188012@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
  • Loading branch information
Peter Zijlstra authored and Ingo Molnar committed May 21, 2010
1 parent b7e2ece commit 1c024ec
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 124 deletions.
16 changes: 8 additions & 8 deletions include/linux/ftrace_event.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ struct ftrace_event_call {
void *data;

int perf_refcount;
void *perf_data;
struct hlist_head *perf_events;
int (*perf_event_enable)(struct ftrace_event_call *);
void (*perf_event_disable)(struct ftrace_event_call *);
};
Expand Down Expand Up @@ -192,21 +192,21 @@ struct perf_event;

DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);

extern int perf_trace_enable(int event_id, void *data);
extern void perf_trace_disable(int event_id);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
extern int perf_trace_init(struct perf_event *event);
extern void perf_trace_destroy(struct perf_event *event);
extern int perf_trace_enable(struct perf_event *event);
extern void perf_trace_disable(struct perf_event *event);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
char *filter_str);
extern void ftrace_profile_free_filter(struct perf_event *event);
extern void *perf_trace_buf_prepare(int size, unsigned short type,
struct pt_regs *regs, int *rctxp);

static inline void
perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
u64 count, struct pt_regs *regs, void *event)
u64 count, struct pt_regs *regs, void *head)
{
struct trace_entry *entry = raw_data;

perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
perf_tp_event(addr, count, raw_data, size, regs, head);
perf_swevent_put_recursion_context(rctx);
}
#endif
Expand Down
6 changes: 4 additions & 2 deletions include/linux/perf_event.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,7 @@ struct perf_event {
perf_overflow_handler_t overflow_handler;

#ifdef CONFIG_EVENT_TRACING
struct ftrace_event_call *tp_event;
struct event_filter *filter;
#endif

Expand Down Expand Up @@ -992,8 +993,9 @@ static inline bool perf_paranoid_kernel(void)
}

extern void perf_event_init(void);
extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
int entry_size, struct pt_regs *regs, void *event);
extern void perf_tp_event(u64 addr, u64 count, void *record,
int entry_size, struct pt_regs *regs,
struct hlist_head *head);
extern void perf_bp_event(struct perf_event *event, void *data);

#ifndef perf_misc_flags
Expand Down
4 changes: 3 additions & 1 deletion include/trace/ftrace.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,7 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \
struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
struct ftrace_raw_##call *entry; \
u64 __addr = 0, __count = 1; \
struct hlist_head *head; \
int __entry_size; \
int __data_size; \
int rctx; \
Expand All @@ -790,8 +791,9 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \
\
{ assign; } \
\
head = per_cpu_ptr(event_call->perf_events, smp_processor_id());\
perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \
__count, __regs, event_call->perf_data); \
__count, __regs, head); \
}

#undef DEFINE_EVENT
Expand Down
94 changes: 48 additions & 46 deletions kernel/perf_event.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4005,9 +4005,6 @@ static void perf_swevent_add(struct perf_event *event, u64 nr,
perf_swevent_overflow(event, 0, nmi, data, regs);
}

static int perf_tp_event_match(struct perf_event *event,
struct perf_sample_data *data);

static int perf_exclude_event(struct perf_event *event,
struct pt_regs *regs)
{
Expand Down Expand Up @@ -4037,10 +4034,6 @@ static int perf_swevent_match(struct perf_event *event,
if (perf_exclude_event(event, regs))
return 0;

if (event->attr.type == PERF_TYPE_TRACEPOINT &&
!perf_tp_event_match(event, data))
return 0;

return 1;
}

Expand Down Expand Up @@ -4122,7 +4115,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,

int perf_swevent_get_recursion_context(void)
{
struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
int rctx;

if (in_nmi())
Expand All @@ -4134,10 +4127,8 @@ int perf_swevent_get_recursion_context(void)
else
rctx = 0;

if (cpuctx->recursion[rctx]) {
put_cpu_var(perf_cpu_context);
if (cpuctx->recursion[rctx])
return -1;
}

cpuctx->recursion[rctx]++;
barrier();
Expand All @@ -4151,7 +4142,6 @@ void perf_swevent_put_recursion_context(int rctx)
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
barrier();
cpuctx->recursion[rctx]--;
put_cpu_var(perf_cpu_context);
}
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);

Expand All @@ -4162,6 +4152,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi,
struct perf_sample_data data;
int rctx;

preempt_disable_notrace();
rctx = perf_swevent_get_recursion_context();
if (rctx < 0)
return;
Expand All @@ -4171,6 +4162,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi,
do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);

perf_swevent_put_recursion_context(rctx);
preempt_enable_notrace();
}

static void perf_swevent_read(struct perf_event *event)
Expand Down Expand Up @@ -4486,11 +4478,43 @@ static int swevent_hlist_get(struct perf_event *event)

#ifdef CONFIG_EVENT_TRACING

void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
int entry_size, struct pt_regs *regs, void *event)
static const struct pmu perf_ops_tracepoint = {
.enable = perf_trace_enable,
.disable = perf_trace_disable,
.read = perf_swevent_read,
.unthrottle = perf_swevent_unthrottle,
};

static int perf_tp_filter_match(struct perf_event *event,
struct perf_sample_data *data)
{
void *record = data->raw->data;

if (likely(!event->filter) || filter_match_preds(event->filter, record))
return 1;
return 0;
}

static int perf_tp_event_match(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs)
{
if (perf_exclude_event(event, regs))
return 0;

if (!perf_tp_filter_match(event, data))
return 0;

return 1;
}

void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
struct pt_regs *regs, struct hlist_head *head)
{
const int type = PERF_TYPE_TRACEPOINT;
struct perf_sample_data data;
struct perf_event *event;
struct hlist_node *node;

struct perf_raw_record raw = {
.size = entry_size,
.data = record,
Expand All @@ -4499,30 +4523,18 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
perf_sample_data_init(&data, addr);
data.raw = &raw;

if (!event) {
do_perf_sw_event(type, event_id, count, 1, &data, regs);
return;
rcu_read_lock();
hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
if (perf_tp_event_match(event, &data, regs))
perf_swevent_add(event, count, 1, &data, regs);
}

if (perf_swevent_match(event, type, event_id, &data, regs))
perf_swevent_add(event, count, 1, &data, regs);
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(perf_tp_event);

static int perf_tp_event_match(struct perf_event *event,
struct perf_sample_data *data)
{
void *record = data->raw->data;

if (likely(!event->filter) || filter_match_preds(event->filter, record))
return 1;
return 0;
}

static void tp_perf_event_destroy(struct perf_event *event)
{
perf_trace_disable(event->attr.config);
swevent_hlist_put(event);
perf_trace_destroy(event);
}

static const struct pmu *tp_perf_event_init(struct perf_event *event)
Expand All @@ -4538,17 +4550,13 @@ static const struct pmu *tp_perf_event_init(struct perf_event *event)
!capable(CAP_SYS_ADMIN))
return ERR_PTR(-EPERM);

if (perf_trace_enable(event->attr.config, event))
err = perf_trace_init(event);
if (err)
return NULL;

event->destroy = tp_perf_event_destroy;
err = swevent_hlist_get(event);
if (err) {
perf_trace_disable(event->attr.config);
return ERR_PTR(err);
}

return &perf_ops_generic;
return &perf_ops_tracepoint;
}

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
Expand Down Expand Up @@ -4576,12 +4584,6 @@ static void perf_event_free_filter(struct perf_event *event)

#else

static int perf_tp_event_match(struct perf_event *event,
struct perf_sample_data *data)
{
return 1;
}

static const struct pmu *tp_perf_event_init(struct perf_event *event)
{
return NULL;
Expand Down
Loading

0 comments on commit 1c024ec

Please sign in to comment.