From 8bc6c5b3620970fd37251ec8b1862757ace16ce8 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 14 Nov 2008 17:47:34 -0500 Subject: [PATCH] --- yaml --- r: 121132 b: refs/heads/master c: 954e100d2275cb2f150f2b18d5cddcdf67b956ac h: refs/heads/master v: v3 --- [refs] | 2 +- trunk/arch/x86/include/asm/ftrace.h | 8 - trunk/arch/x86/kernel/entry_32.S | 18 +- trunk/arch/x86/kernel/ftrace.c | 286 +++++++++----------- trunk/include/linux/ftrace.h | 69 +---- trunk/include/linux/rcupdate.h | 2 + trunk/kernel/module.c | 2 +- trunk/kernel/trace/Kconfig | 1 + trunk/kernel/trace/ftrace.c | 278 ++++++++----------- trunk/kernel/trace/trace.c | 23 +- trunk/kernel/trace/trace.h | 3 +- trunk/kernel/trace/trace_boot.c | 3 +- trunk/kernel/trace/trace_branch.c | 3 +- trunk/kernel/trace/trace_functions.c | 3 +- trunk/kernel/trace/trace_functions_return.c | 16 +- trunk/kernel/trace/trace_irqsoff.c | 9 +- trunk/kernel/trace/trace_mmiotrace.c | 3 +- trunk/kernel/trace/trace_nop.c | 3 +- trunk/kernel/trace/trace_sched_switch.c | 3 +- trunk/kernel/trace/trace_sched_wakeup.c | 3 +- trunk/kernel/trace/trace_selftest.c | 70 +---- trunk/kernel/trace/trace_sysprof.c | 3 +- 22 files changed, 318 insertions(+), 493 deletions(-) diff --git a/[refs] b/[refs] index bd5b6b0f6f17..74b21c1c9bf2 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: e7d3737ea1b102030f44e96c97754101e41515f0 +refs/heads/master: 954e100d2275cb2f150f2b18d5cddcdf67b956ac diff --git a/trunk/arch/x86/include/asm/ftrace.h b/trunk/arch/x86/include/asm/ftrace.h index 2bb43b433e07..9b6a1fa19e70 100644 --- a/trunk/arch/x86/include/asm/ftrace.h +++ b/trunk/arch/x86/include/asm/ftrace.h @@ -17,14 +17,6 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) */ return addr - 1; } - -#ifdef CONFIG_DYNAMIC_FTRACE - -struct dyn_arch_ftrace { - /* No extra data needed for x86 */ -}; - -#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* __ASSEMBLY__ */ #endif /* CONFIG_FUNCTION_TRACER */ diff --git a/trunk/arch/x86/kernel/entry_32.S b/trunk/arch/x86/kernel/entry_32.S index 74defe21ba42..f97621149839 100644 --- a/trunk/arch/x86/kernel/entry_32.S +++ b/trunk/arch/x86/kernel/entry_32.S @@ -1190,7 +1190,7 @@ ENTRY(mcount) jnz trace #ifdef CONFIG_FUNCTION_RET_TRACER cmpl $ftrace_stub, ftrace_function_return - jnz ftrace_return_caller + jnz trace_return #endif .globl ftrace_stub ftrace_stub: @@ -1211,15 +1211,9 @@ trace: popl %ecx popl %eax jmp ftrace_stub -END(mcount) -#endif /* CONFIG_DYNAMIC_FTRACE */ -#endif /* CONFIG_FUNCTION_TRACER */ #ifdef CONFIG_FUNCTION_RET_TRACER -ENTRY(ftrace_return_caller) - cmpl $0, function_trace_stop - jne ftrace_stub - +trace_return: pushl %eax pushl %ecx pushl %edx @@ -1229,8 +1223,7 @@ ENTRY(ftrace_return_caller) popl %edx popl %ecx popl %eax - ret -END(ftrace_return_caller) + jmp ftrace_stub .globl return_to_handler return_to_handler: @@ -1244,7 +1237,10 @@ return_to_handler: popl %ecx popl %eax ret -#endif +#endif /* CONFIG_FUNCTION_RET_TRACER */ +END(mcount) +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* CONFIG_FUNCTION_TRACER */ .section .rodata,"a" #include "syscall_table_32.S" diff --git a/trunk/arch/x86/kernel/ftrace.c b/trunk/arch/x86/kernel/ftrace.c index 924153edd973..fe832738e1e2 100644 --- a/trunk/arch/x86/kernel/ftrace.c +++ b/trunk/arch/x86/kernel/ftrace.c @@ -24,6 +24,133 @@ #include + +#ifdef CONFIG_FUNCTION_RET_TRACER + +/* + * These functions are picked from those used on + * this page for dynamic ftrace. They have been + * simplified to ignore all traces in NMI context. + */ +static atomic_t in_nmi; + +void ftrace_nmi_enter(void) +{ + atomic_inc(&in_nmi); +} + +void ftrace_nmi_exit(void) +{ + atomic_dec(&in_nmi); +} + +/* Add a function return address to the trace stack on thread info.*/ +static int push_return_trace(unsigned long ret, unsigned long long time, + unsigned long func) +{ + int index; + struct thread_info *ti = current_thread_info(); + + /* The return trace stack is full */ + if (ti->curr_ret_stack == FTRACE_RET_STACK_SIZE - 1) + return -EBUSY; + + index = ++ti->curr_ret_stack; + ti->ret_stack[index].ret = ret; + ti->ret_stack[index].func = func; + ti->ret_stack[index].calltime = time; + + return 0; +} + +/* Retrieve a function return address to the trace stack on thread info.*/ +static void pop_return_trace(unsigned long *ret, unsigned long long *time, + unsigned long *func) +{ + int index; + + struct thread_info *ti = current_thread_info(); + index = ti->curr_ret_stack; + *ret = ti->ret_stack[index].ret; + *func = ti->ret_stack[index].func; + *time = ti->ret_stack[index].calltime; + ti->curr_ret_stack--; +} + +/* + * Send the trace to the ring-buffer. + * @return the original return address. + */ +unsigned long ftrace_return_to_handler(void) +{ + struct ftrace_retfunc trace; + pop_return_trace(&trace.ret, &trace.calltime, &trace.func); + trace.rettime = cpu_clock(raw_smp_processor_id()); + ftrace_function_return(&trace); + + return trace.ret; +} + +/* + * Hook the return address and push it in the stack of return addrs + * in current thread info. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) +{ + unsigned long old; + unsigned long long calltime; + int faulted; + unsigned long return_hooker = (unsigned long) + &return_to_handler; + + /* Nmi's are currently unsupported */ + if (atomic_read(&in_nmi)) + return; + + /* + * Protect against fault, even if it shouldn't + * happen. This tool is too much intrusive to + * ignore such a protection. + */ + asm volatile( + "1: movl (%[parent_old]), %[old]\n" + "2: movl %[return_hooker], (%[parent_replaced])\n" + " movl $0, %[faulted]\n" + + ".section .fixup, \"ax\"\n" + "3: movl $1, %[faulted]\n" + ".previous\n" + + ".section __ex_table, \"a\"\n" + " .long 1b, 3b\n" + " .long 2b, 3b\n" + ".previous\n" + + : [parent_replaced] "=r" (parent), [old] "=r" (old), + [faulted] "=r" (faulted) + : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker) + : "memory" + ); + + if (WARN_ON(faulted)) { + unregister_ftrace_return(); + return; + } + + if (WARN_ON(!__kernel_text_address(old))) { + unregister_ftrace_return(); + *parent = old; + return; + } + + calltime = cpu_clock(raw_smp_processor_id()); + + if (push_return_trace(old, calltime, self_addr) == -EBUSY) + *parent = old; +} + +#endif + #ifdef CONFIG_DYNAMIC_FTRACE union ftrace_code_union { @@ -39,7 +166,7 @@ static int ftrace_calc_offset(long ip, long addr) return (int)(addr - ip); } -static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) +unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) { static union ftrace_code_union calc; @@ -184,12 +311,12 @@ do_ftrace_mod_code(unsigned long ip, void *new_code) static unsigned char ftrace_nop[MCOUNT_INSN_SIZE]; -static unsigned char *ftrace_nop_replace(void) +unsigned char *ftrace_nop_replace(void) { return ftrace_nop; } -static int +int ftrace_modify_code(unsigned long ip, unsigned char *old_code, unsigned char *new_code) { @@ -222,29 +349,6 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code, return 0; } -int ftrace_make_nop(struct module *mod, - struct dyn_ftrace *rec, unsigned long addr) -{ - unsigned char *new, *old; - unsigned long ip = rec->ip; - - old = ftrace_call_replace(ip, addr); - new = ftrace_nop_replace(); - - return ftrace_modify_code(rec->ip, old, new); -} - -int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) -{ - unsigned char *new, *old; - unsigned long ip = rec->ip; - - old = ftrace_nop_replace(); - new = ftrace_call_replace(ip, addr); - - return ftrace_modify_code(rec->ip, old, new); -} - int ftrace_update_ftrace_func(ftrace_func_t func) { unsigned long ip = (unsigned long)(&ftrace_call); @@ -322,133 +426,3 @@ int __init ftrace_dyn_arch_init(void *data) return 0; } #endif - -#ifdef CONFIG_FUNCTION_RET_TRACER - -#ifndef CONFIG_DYNAMIC_FTRACE - -/* - * These functions are picked from those used on - * this page for dynamic ftrace. They have been - * simplified to ignore all traces in NMI context. - */ -static atomic_t in_nmi; - -void ftrace_nmi_enter(void) -{ - atomic_inc(&in_nmi); -} - -void ftrace_nmi_exit(void) -{ - atomic_dec(&in_nmi); -} -#endif /* !CONFIG_DYNAMIC_FTRACE */ - -/* Add a function return address to the trace stack on thread info.*/ -static int push_return_trace(unsigned long ret, unsigned long long time, - unsigned long func) -{ - int index; - struct thread_info *ti = current_thread_info(); - - /* The return trace stack is full */ - if (ti->curr_ret_stack == FTRACE_RET_STACK_SIZE - 1) - return -EBUSY; - - index = ++ti->curr_ret_stack; - barrier(); - ti->ret_stack[index].ret = ret; - ti->ret_stack[index].func = func; - ti->ret_stack[index].calltime = time; - - return 0; -} - -/* Retrieve a function return address to the trace stack on thread info.*/ -static void pop_return_trace(unsigned long *ret, unsigned long long *time, - unsigned long *func) -{ - int index; - - struct thread_info *ti = current_thread_info(); - index = ti->curr_ret_stack; - *ret = ti->ret_stack[index].ret; - *func = ti->ret_stack[index].func; - *time = ti->ret_stack[index].calltime; - ti->curr_ret_stack--; -} - -/* - * Send the trace to the ring-buffer. - * @return the original return address. - */ -unsigned long ftrace_return_to_handler(void) -{ - struct ftrace_retfunc trace; - pop_return_trace(&trace.ret, &trace.calltime, &trace.func); - trace.rettime = cpu_clock(raw_smp_processor_id()); - ftrace_function_return(&trace); - - return trace.ret; -} - -/* - * Hook the return address and push it in the stack of return addrs - * in current thread info. - */ -void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) -{ - unsigned long old; - unsigned long long calltime; - int faulted; - unsigned long return_hooker = (unsigned long) - &return_to_handler; - - /* Nmi's are currently unsupported */ - if (atomic_read(&in_nmi)) - return; - - /* - * Protect against fault, even if it shouldn't - * happen. This tool is too much intrusive to - * ignore such a protection. - */ - asm volatile( - "1: movl (%[parent_old]), %[old]\n" - "2: movl %[return_hooker], (%[parent_replaced])\n" - " movl $0, %[faulted]\n" - - ".section .fixup, \"ax\"\n" - "3: movl $1, %[faulted]\n" - ".previous\n" - - ".section __ex_table, \"a\"\n" - " .long 1b, 3b\n" - " .long 2b, 3b\n" - ".previous\n" - - : [parent_replaced] "=r" (parent), [old] "=r" (old), - [faulted] "=r" (faulted) - : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker) - : "memory" - ); - - if (WARN_ON(faulted)) { - unregister_ftrace_return(); - return; - } - - if (WARN_ON(!__kernel_text_address(old))) { - unregister_ftrace_return(); - *parent = old; - return; - } - - calltime = cpu_clock(raw_smp_processor_id()); - - if (push_return_trace(old, calltime, self_addr) == -EBUSY) - *parent = old; -} - -#endif /* CONFIG_FUNCTION_RET_TRACER */ diff --git a/trunk/include/linux/ftrace.h b/trunk/include/linux/ftrace.h index f1af1aab00e6..4fbc4a8b86a5 100644 --- a/trunk/include/linux/ftrace.h +++ b/trunk/include/linux/ftrace.h @@ -25,17 +25,6 @@ struct ftrace_ops { extern int function_trace_stop; -/* - * Type of the current tracing. - */ -enum ftrace_tracing_type_t { - FTRACE_TYPE_ENTER = 0, /* Hook the call of the function */ - FTRACE_TYPE_RETURN, /* Hook the return of the function */ -}; - -/* Current tracing type, default is FTRACE_TYPE_ENTER */ -extern enum ftrace_tracing_type_t ftrace_tracing_type; - /** * ftrace_stop - stop function tracer. * @@ -85,9 +74,6 @@ static inline void ftrace_start(void) { } #endif /* CONFIG_FUNCTION_TRACER */ #ifdef CONFIG_DYNAMIC_FTRACE -/* asm/ftrace.h must be defined for archs supporting dynamic ftrace */ -#include - enum { FTRACE_FL_FREE = (1 << 0), FTRACE_FL_FAILED = (1 << 1), @@ -102,7 +88,6 @@ struct dyn_ftrace { struct list_head list; unsigned long ip; /* address of mcount call-site */ unsigned long flags; - struct dyn_arch_ftrace arch; }; int ftrace_force_update(void); @@ -110,43 +95,22 @@ void ftrace_set_filter(unsigned char *buf, int len, int reset); /* defined in arch */ extern int ftrace_ip_converted(unsigned long ip); +extern unsigned char *ftrace_nop_replace(void); +extern unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr); extern int ftrace_dyn_arch_init(void *data); extern int ftrace_update_ftrace_func(ftrace_func_t func); extern void ftrace_caller(void); extern void ftrace_call(void); extern void mcount_call(void); -#ifdef CONFIG_FUNCTION_RET_TRACER -extern void ftrace_return_caller(void); -#endif -/** - * ftrace_make_nop - convert code into top - * @mod: module structure if called by module load initialization - * @rec: the mcount call site record - * @addr: the address that the call site should be calling - * - * This is a very sensitive operation and great care needs - * to be taken by the arch. The operation should carefully - * read the location, check to see if what is read is indeed - * what we expect it to be, and then on success of the compare, - * it should write to the location. - * - * The code segment at @rec->ip should be a caller to @addr - * - * Return must be: - * 0 on success - * -EFAULT on error reading the location - * -EINVAL on a failed compare of the contents - * -EPERM on error writing to the location - * Any other value will be considered a failure. - */ -extern int ftrace_make_nop(struct module *mod, - struct dyn_ftrace *rec, unsigned long addr); +/* May be defined in arch */ +extern int ftrace_arch_read_dyn_info(char *buf, int size); /** - * ftrace_make_call - convert a nop call site into a call to addr - * @rec: the mcount call site record - * @addr: the address that the call site should call + * ftrace_modify_code - modify code segment + * @ip: the address of the code segment + * @old_code: the contents of what is expected to be there + * @new_code: the code to patch in * * This is a very sensitive operation and great care needs * to be taken by the arch. The operation should carefully @@ -154,8 +118,6 @@ extern int ftrace_make_nop(struct module *mod, * what we expect it to be, and then on success of the compare, * it should write to the location. * - * The code segment at @rec->ip should be a nop - * * Return must be: * 0 on success * -EFAULT on error reading the location @@ -163,11 +125,8 @@ extern int ftrace_make_nop(struct module *mod, * -EPERM on error writing to the location * Any other value will be considered a failure. */ -extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr); - - -/* May be defined in arch */ -extern int ftrace_arch_read_dyn_info(char *buf, int size); +extern int ftrace_modify_code(unsigned long ip, unsigned char *old_code, + unsigned char *new_code); extern int skip_trace(unsigned long ip); @@ -300,13 +259,11 @@ static inline void ftrace_dump(void) { } #ifdef CONFIG_FTRACE_MCOUNT_RECORD extern void ftrace_init(void); -extern void ftrace_init_module(struct module *mod, - unsigned long *start, unsigned long *end); +extern void ftrace_init_module(unsigned long *start, unsigned long *end); #else static inline void ftrace_init(void) { } static inline void -ftrace_init_module(struct module *mod, - unsigned long *start, unsigned long *end) { } +ftrace_init_module(unsigned long *start, unsigned long *end) { } #endif @@ -324,7 +281,7 @@ struct ftrace_retfunc { /* Type of a callback handler of tracing return function */ typedef void (*trace_function_return_t)(struct ftrace_retfunc *); -extern int register_ftrace_return(trace_function_return_t func); +extern void register_ftrace_return(trace_function_return_t func); /* The current handler in use */ extern trace_function_return_t ftrace_function_return; extern void unregister_ftrace_return(void); diff --git a/trunk/include/linux/rcupdate.h b/trunk/include/linux/rcupdate.h index 86f1f5e43e33..895dc9c1088c 100644 --- a/trunk/include/linux/rcupdate.h +++ b/trunk/include/linux/rcupdate.h @@ -142,6 +142,7 @@ struct rcu_head { * on the write-side to insure proper synchronization. */ #define rcu_read_lock_sched() preempt_disable() +#define rcu_read_lock_sched_notrace() preempt_disable_notrace() /* * rcu_read_unlock_sched - marks the end of a RCU-classic critical section @@ -149,6 +150,7 @@ struct rcu_head { * See rcu_read_lock_sched for more information. */ #define rcu_read_unlock_sched() preempt_enable() +#define rcu_read_unlock_sched_notrace() preempt_enable_notrace() diff --git a/trunk/kernel/module.c b/trunk/kernel/module.c index 69791274e899..1f4cc00e0c20 100644 --- a/trunk/kernel/module.c +++ b/trunk/kernel/module.c @@ -2201,7 +2201,7 @@ static noinline struct module *load_module(void __user *umod, /* sechdrs[0].sh_size is always zero */ mseg = section_objs(hdr, sechdrs, secstrings, "__mcount_loc", sizeof(*mseg), &num_mcount); - ftrace_init_module(mod, mseg, mseg + num_mcount); + ftrace_init_module(mseg, mseg + num_mcount); err = module_finalize(hdr, sechdrs, mod); if (err < 0) diff --git a/trunk/kernel/trace/Kconfig b/trunk/kernel/trace/Kconfig index b8378fad29a3..9c89526b6b7c 100644 --- a/trunk/kernel/trace/Kconfig +++ b/trunk/kernel/trace/Kconfig @@ -59,6 +59,7 @@ config FUNCTION_TRACER config FUNCTION_RET_TRACER bool "Kernel Function return Tracer" + depends on !DYNAMIC_FTRACE depends on HAVE_FUNCTION_RET_TRACER depends on FUNCTION_TRACER help diff --git a/trunk/kernel/trace/ftrace.c b/trunk/kernel/trace/ftrace.c index 2f78a45aac14..54cb9a7d15e5 100644 --- a/trunk/kernel/trace/ftrace.c +++ b/trunk/kernel/trace/ftrace.c @@ -50,9 +50,6 @@ static int last_ftrace_enabled; /* Quick disabling of function tracer. */ int function_trace_stop; -/* By default, current tracing type is normal tracing. */ -enum ftrace_tracing_type_t ftrace_tracing_type = FTRACE_TYPE_ENTER; - /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. @@ -337,7 +334,7 @@ ftrace_record_ip(unsigned long ip) { struct dyn_ftrace *rec; - if (ftrace_disabled) + if (!ftrace_enabled || ftrace_disabled) return NULL; rec = ftrace_alloc_dyn_node(ip); @@ -351,138 +348,107 @@ ftrace_record_ip(unsigned long ip) return rec; } -static void print_ip_ins(const char *fmt, unsigned char *p) -{ - int i; - - printk(KERN_CONT "%s", fmt); - - for (i = 0; i < MCOUNT_INSN_SIZE; i++) - printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); -} - -static void ftrace_bug(int failed, unsigned long ip) -{ - switch (failed) { - case -EFAULT: - FTRACE_WARN_ON_ONCE(1); - pr_info("ftrace faulted on modifying "); - print_ip_sym(ip); - break; - case -EINVAL: - FTRACE_WARN_ON_ONCE(1); - pr_info("ftrace failed to modify "); - print_ip_sym(ip); - print_ip_ins(" actual: ", (unsigned char *)ip); - printk(KERN_CONT "\n"); - break; - case -EPERM: - FTRACE_WARN_ON_ONCE(1); - pr_info("ftrace faulted on writing "); - print_ip_sym(ip); - break; - default: - FTRACE_WARN_ON_ONCE(1); - pr_info("ftrace faulted on unknown error "); - print_ip_sym(ip); - } -} - +#define FTRACE_ADDR ((long)(ftrace_caller)) static int -__ftrace_replace_code(struct dyn_ftrace *rec, int enable) +__ftrace_replace_code(struct dyn_ftrace *rec, + unsigned char *old, unsigned char *new, int enable) { unsigned long ip, fl; - unsigned long ftrace_addr; - -#ifdef CONFIG_FUNCTION_RET_TRACER - if (ftrace_tracing_type == FTRACE_TYPE_ENTER) - ftrace_addr = (unsigned long)ftrace_caller; - else - ftrace_addr = (unsigned long)ftrace_return_caller; -#else - ftrace_addr = (unsigned long)ftrace_caller; -#endif ip = rec->ip; - /* - * If this record is not to be traced and - * it is not enabled then do nothing. - * - * If this record is not to be traced and - * it is enabled then disabled it. - * - */ - if (rec->flags & FTRACE_FL_NOTRACE) { - if (rec->flags & FTRACE_FL_ENABLED) - rec->flags &= ~FTRACE_FL_ENABLED; - else - return 0; - - } else if (ftrace_filtered && enable) { + if (ftrace_filtered && enable) { /* - * Filtering is on: + * If filtering is on: + * + * If this record is set to be filtered and + * is enabled then do nothing. + * + * If this record is set to be filtered and + * it is not enabled, enable it. + * + * If this record is not set to be filtered + * and it is not enabled do nothing. + * + * If this record is set not to trace then + * do nothing. + * + * If this record is set not to trace and + * it is enabled then disable it. + * + * If this record is not set to be filtered and + * it is enabled, disable it. */ - fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED); - - /* Record is filtered and enabled, do nothing */ - if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) - return 0; + fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE | + FTRACE_FL_ENABLED); - /* Record is not filtered and is not enabled do nothing */ - if (!fl) + if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) || + (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) || + !fl || (fl == FTRACE_FL_NOTRACE)) return 0; - /* Record is not filtered but enabled, disable it */ - if (fl == FTRACE_FL_ENABLED) + /* + * If it is enabled disable it, + * otherwise enable it! + */ + if (fl & FTRACE_FL_ENABLED) { + /* swap new and old */ + new = old; + old = ftrace_call_replace(ip, FTRACE_ADDR); rec->flags &= ~FTRACE_FL_ENABLED; - else - /* Otherwise record is filtered but not enabled, enable it */ + } else { + new = ftrace_call_replace(ip, FTRACE_ADDR); rec->flags |= FTRACE_FL_ENABLED; + } } else { - /* Disable or not filtered */ if (enable) { - /* if record is enabled, do nothing */ - if (rec->flags & FTRACE_FL_ENABLED) + /* + * If this record is set not to trace and is + * not enabled, do nothing. + */ + fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED); + if (fl == FTRACE_FL_NOTRACE) return 0; - rec->flags |= FTRACE_FL_ENABLED; + new = ftrace_call_replace(ip, FTRACE_ADDR); + } else + old = ftrace_call_replace(ip, FTRACE_ADDR); + if (enable) { + if (rec->flags & FTRACE_FL_ENABLED) + return 0; + rec->flags |= FTRACE_FL_ENABLED; } else { - - /* if record is not enabled do nothing */ if (!(rec->flags & FTRACE_FL_ENABLED)) return 0; - rec->flags &= ~FTRACE_FL_ENABLED; } } - if (rec->flags & FTRACE_FL_ENABLED) - return ftrace_make_call(rec, ftrace_addr); - else - return ftrace_make_nop(NULL, rec, ftrace_addr); + return ftrace_modify_code(ip, old, new); } static void ftrace_replace_code(int enable) { int i, failed; + unsigned char *new = NULL, *old = NULL; struct dyn_ftrace *rec; struct ftrace_page *pg; + if (enable) + old = ftrace_nop_replace(); + else + new = ftrace_nop_replace(); + for (pg = ftrace_pages_start; pg; pg = pg->next) { for (i = 0; i < pg->index; i++) { rec = &pg->records[i]; - /* - * Skip over free records and records that have - * failed. - */ - if (rec->flags & FTRACE_FL_FREE || - rec->flags & FTRACE_FL_FAILED) + /* don't modify code that has already faulted */ + if (rec->flags & FTRACE_FL_FAILED) continue; /* ignore updates to this record's mcount site */ @@ -493,30 +459,68 @@ static void ftrace_replace_code(int enable) unfreeze_record(rec); } - failed = __ftrace_replace_code(rec, enable); + failed = __ftrace_replace_code(rec, old, new, enable); if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { rec->flags |= FTRACE_FL_FAILED; if ((system_state == SYSTEM_BOOTING) || !core_kernel_text(rec->ip)) { ftrace_free_rec(rec); - } else - ftrace_bug(failed, rec->ip); + } } } } } +static void print_ip_ins(const char *fmt, unsigned char *p) +{ + int i; + + printk(KERN_CONT "%s", fmt); + + for (i = 0; i < MCOUNT_INSN_SIZE; i++) + printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); +} + static int -ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) +ftrace_code_disable(struct dyn_ftrace *rec) { unsigned long ip; + unsigned char *nop, *call; int ret; ip = rec->ip; - ret = ftrace_make_nop(mod, rec, mcount_addr); + nop = ftrace_nop_replace(); + call = ftrace_call_replace(ip, mcount_addr); + + ret = ftrace_modify_code(ip, call, nop); if (ret) { - ftrace_bug(ret, ip); + switch (ret) { + case -EFAULT: + FTRACE_WARN_ON_ONCE(1); + pr_info("ftrace faulted on modifying "); + print_ip_sym(ip); + break; + case -EINVAL: + FTRACE_WARN_ON_ONCE(1); + pr_info("ftrace failed to modify "); + print_ip_sym(ip); + print_ip_ins(" expected: ", call); + print_ip_ins(" actual: ", (unsigned char *)ip); + print_ip_ins(" replace: ", nop); + printk(KERN_CONT "\n"); + break; + case -EPERM: + FTRACE_WARN_ON_ONCE(1); + pr_info("ftrace faulted on writing "); + print_ip_sym(ip); + break; + default: + FTRACE_WARN_ON_ONCE(1); + pr_info("ftrace faulted on unknown error "); + print_ip_sym(ip); + } + rec->flags |= FTRACE_FL_FAILED; return 0; } @@ -556,7 +560,8 @@ static void ftrace_startup(void) mutex_lock(&ftrace_start_lock); ftrace_start_up++; - command |= FTRACE_ENABLE_CALLS; + if (ftrace_start_up == 1) + command |= FTRACE_ENABLE_CALLS; if (saved_ftrace_func != ftrace_trace_function) { saved_ftrace_func = ftrace_trace_function; @@ -634,7 +639,7 @@ static cycle_t ftrace_update_time; static unsigned long ftrace_update_cnt; unsigned long ftrace_update_tot_cnt; -static int ftrace_update_code(struct module *mod) +static int ftrace_update_code(void) { struct dyn_ftrace *p, *t; cycle_t start, stop; @@ -651,7 +656,7 @@ static int ftrace_update_code(struct module *mod) list_del_init(&p->list); /* convert record (i.e, patch mcount-call with NOP) */ - if (ftrace_code_disable(mod, p)) { + if (ftrace_code_disable(p)) { p->flags |= FTRACE_FL_CONVERTED; ftrace_update_cnt++; } else @@ -1206,7 +1211,7 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable) mutex_lock(&ftrace_sysctl_lock); mutex_lock(&ftrace_start_lock); - if (ftrace_start_up && ftrace_enabled) + if (iter->filtered && ftrace_start_up && ftrace_enabled) ftrace_run_update_code(FTRACE_ENABLE_CALLS); mutex_unlock(&ftrace_start_lock); mutex_unlock(&ftrace_sysctl_lock); @@ -1293,8 +1298,7 @@ static __init int ftrace_init_debugfs(void) fs_initcall(ftrace_init_debugfs); -static int ftrace_convert_nops(struct module *mod, - unsigned long *start, +static int ftrace_convert_nops(unsigned long *start, unsigned long *end) { unsigned long *p; @@ -1305,32 +1309,23 @@ static int ftrace_convert_nops(struct module *mod, p = start; while (p < end) { addr = ftrace_call_adjust(*p++); - /* - * Some architecture linkers will pad between - * the different mcount_loc sections of different - * object files to satisfy alignments. - * Skip any NULL pointers. - */ - if (!addr) - continue; ftrace_record_ip(addr); } /* disable interrupts to prevent kstop machine */ local_irq_save(flags); - ftrace_update_code(mod); + ftrace_update_code(); local_irq_restore(flags); mutex_unlock(&ftrace_start_lock); return 0; } -void ftrace_init_module(struct module *mod, - unsigned long *start, unsigned long *end) +void ftrace_init_module(unsigned long *start, unsigned long *end) { if (ftrace_disabled || start == end) return; - ftrace_convert_nops(mod, start, end); + ftrace_convert_nops(start, end); } extern unsigned long __start_mcount_loc[]; @@ -1360,8 +1355,7 @@ void __init ftrace_init(void) last_ftrace_enabled = ftrace_enabled = 1; - ret = ftrace_convert_nops(NULL, - __start_mcount_loc, + ret = ftrace_convert_nops(__start_mcount_loc, __stop_mcount_loc); return; @@ -1417,17 +1411,10 @@ int register_ftrace_function(struct ftrace_ops *ops) return -1; mutex_lock(&ftrace_sysctl_lock); - - if (ftrace_tracing_type == FTRACE_TYPE_RETURN) { - ret = -EBUSY; - goto out; - } - ret = __register_ftrace_function(ops); ftrace_startup(); - -out: mutex_unlock(&ftrace_sysctl_lock); + return ret; } @@ -1493,45 +1480,16 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, } #ifdef CONFIG_FUNCTION_RET_TRACER - -/* The callback that hooks the return of a function */ trace_function_return_t ftrace_function_return = (trace_function_return_t)ftrace_stub; - -int register_ftrace_return(trace_function_return_t func) +void register_ftrace_return(trace_function_return_t func) { - int ret = 0; - - mutex_lock(&ftrace_sysctl_lock); - - /* - * Don't launch return tracing if normal function - * tracing is already running. - */ - if (ftrace_trace_function != ftrace_stub) { - ret = -EBUSY; - goto out; - } - - ftrace_tracing_type = FTRACE_TYPE_RETURN; ftrace_function_return = func; - ftrace_startup(); - -out: - mutex_unlock(&ftrace_sysctl_lock); - return ret; } void unregister_ftrace_return(void) { - mutex_lock(&ftrace_sysctl_lock); - ftrace_function_return = (trace_function_return_t)ftrace_stub; - ftrace_shutdown(); - /* Restore normal tracing type */ - ftrace_tracing_type = FTRACE_TYPE_ENTER; - - mutex_unlock(&ftrace_sysctl_lock); } #endif diff --git a/trunk/kernel/trace/trace.c b/trunk/kernel/trace/trace.c index 396fda034e3f..4a904623e05d 100644 --- a/trunk/kernel/trace/trace.c +++ b/trunk/kernel/trace/trace.c @@ -1051,7 +1051,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip) * Need to use raw, since this must be called before the * recursive protection is performed. */ - local_irq_save(flags); + raw_local_irq_save(flags); cpu = raw_smp_processor_id(); data = tr->data[cpu]; disabled = atomic_inc_return(&data->disabled); @@ -1062,7 +1062,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip) } atomic_dec(&data->disabled); - local_irq_restore(flags); + raw_local_irq_restore(flags); } #ifdef CONFIG_FUNCTION_RET_TRACER @@ -2638,11 +2638,8 @@ static int tracing_set_tracer(char *buf) current_trace->reset(tr); current_trace = t; - if (t->init) { - ret = t->init(tr); - if (ret) - goto out; - } + if (t->init) + t->init(tr); trace_branch_enable(tr); out: @@ -2658,9 +2655,6 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf, char buf[max_tracer_type_len+1]; int i; size_t ret; - int err; - - ret = cnt; if (cnt > max_tracer_type_len) cnt = max_tracer_type_len; @@ -2674,11 +2668,12 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf, for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) buf[i] = 0; - err = tracing_set_tracer(buf); - if (err) - return err; + ret = tracing_set_tracer(buf); + if (!ret) + ret = cnt; - filp->f_pos += ret; + if (ret > 0) + filp->f_pos += ret; return ret; } diff --git a/trunk/kernel/trace/trace.h b/trunk/kernel/trace/trace.h index cdbd5cc22be8..790ea8c0e1f3 100644 --- a/trunk/kernel/trace/trace.h +++ b/trunk/kernel/trace/trace.h @@ -264,8 +264,7 @@ enum print_line_t { */ struct tracer { const char *name; - /* Your tracer should raise a warning if init fails */ - int (*init)(struct trace_array *tr); + void (*init)(struct trace_array *tr); void (*reset)(struct trace_array *tr); void (*start)(struct trace_array *tr); void (*stop)(struct trace_array *tr); diff --git a/trunk/kernel/trace/trace_boot.c b/trunk/kernel/trace/trace_boot.c index a4fa2c57e34e..cb333b7fd113 100644 --- a/trunk/kernel/trace/trace_boot.c +++ b/trunk/kernel/trace/trace_boot.c @@ -47,7 +47,7 @@ static void reset_boot_trace(struct trace_array *tr) tracing_reset(tr, cpu); } -static int boot_trace_init(struct trace_array *tr) +static void boot_trace_init(struct trace_array *tr) { int cpu; boot_trace = tr; @@ -56,7 +56,6 @@ static int boot_trace_init(struct trace_array *tr) tracing_reset(tr, cpu); tracing_sched_switch_assign_trace(tr); - return 0; } static enum print_line_t diff --git a/trunk/kernel/trace/trace_branch.c b/trunk/kernel/trace/trace_branch.c index 44bd39539d61..85265553918f 100644 --- a/trunk/kernel/trace/trace_branch.c +++ b/trunk/kernel/trace/trace_branch.c @@ -125,7 +125,7 @@ static void stop_branch_trace(struct trace_array *tr) disable_branch_tracing(); } -static int branch_trace_init(struct trace_array *tr) +static void branch_trace_init(struct trace_array *tr) { int cpu; @@ -133,7 +133,6 @@ static int branch_trace_init(struct trace_array *tr) tracing_reset(tr, cpu); start_branch_trace(tr); - return 0; } static void branch_trace_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_functions.c b/trunk/kernel/trace/trace_functions.c index e74f6d0a3216..8693b7a0a5b2 100644 --- a/trunk/kernel/trace/trace_functions.c +++ b/trunk/kernel/trace/trace_functions.c @@ -42,10 +42,9 @@ static void stop_function_trace(struct trace_array *tr) tracing_stop_cmdline_record(); } -static int function_trace_init(struct trace_array *tr) +static void function_trace_init(struct trace_array *tr) { start_function_trace(tr); - return 0; } static void function_trace_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_functions_return.c b/trunk/kernel/trace/trace_functions_return.c index a68564af022b..7680b21537dd 100644 --- a/trunk/kernel/trace/trace_functions_return.c +++ b/trunk/kernel/trace/trace_functions_return.c @@ -14,18 +14,28 @@ #include "trace.h" -static int return_trace_init(struct trace_array *tr) +static void start_return_trace(struct trace_array *tr) +{ + register_ftrace_return(&trace_function_return); +} + +static void stop_return_trace(struct trace_array *tr) +{ + unregister_ftrace_return(); +} + +static void return_trace_init(struct trace_array *tr) { int cpu; for_each_online_cpu(cpu) tracing_reset(tr, cpu); - return register_ftrace_return(&trace_function_return); + start_return_trace(tr); } static void return_trace_reset(struct trace_array *tr) { - unregister_ftrace_return(); + stop_return_trace(tr); } diff --git a/trunk/kernel/trace/trace_irqsoff.c b/trunk/kernel/trace/trace_irqsoff.c index 7c2e326bbc8b..d919d4eaa7cc 100644 --- a/trunk/kernel/trace/trace_irqsoff.c +++ b/trunk/kernel/trace/trace_irqsoff.c @@ -416,12 +416,11 @@ static void irqsoff_tracer_close(struct trace_iterator *iter) } #ifdef CONFIG_IRQSOFF_TRACER -static int irqsoff_tracer_init(struct trace_array *tr) +static void irqsoff_tracer_init(struct trace_array *tr) { trace_type = TRACER_IRQS_OFF; __irqsoff_tracer_init(tr); - return 0; } static struct tracer irqsoff_tracer __read_mostly = { @@ -443,12 +442,11 @@ static struct tracer irqsoff_tracer __read_mostly = #endif #ifdef CONFIG_PREEMPT_TRACER -static int preemptoff_tracer_init(struct trace_array *tr) +static void preemptoff_tracer_init(struct trace_array *tr) { trace_type = TRACER_PREEMPT_OFF; __irqsoff_tracer_init(tr); - return 0; } static struct tracer preemptoff_tracer __read_mostly = @@ -473,12 +471,11 @@ static struct tracer preemptoff_tracer __read_mostly = #if defined(CONFIG_IRQSOFF_TRACER) && \ defined(CONFIG_PREEMPT_TRACER) -static int preemptirqsoff_tracer_init(struct trace_array *tr) +static void preemptirqsoff_tracer_init(struct trace_array *tr) { trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; __irqsoff_tracer_init(tr); - return 0; } static struct tracer preemptirqsoff_tracer __read_mostly = diff --git a/trunk/kernel/trace/trace_mmiotrace.c b/trunk/kernel/trace/trace_mmiotrace.c index 433d650eda9f..51bcf370215e 100644 --- a/trunk/kernel/trace/trace_mmiotrace.c +++ b/trunk/kernel/trace/trace_mmiotrace.c @@ -30,14 +30,13 @@ static void mmio_reset_data(struct trace_array *tr) tracing_reset(tr, cpu); } -static int mmio_trace_init(struct trace_array *tr) +static void mmio_trace_init(struct trace_array *tr) { pr_debug("in %s\n", __func__); mmio_trace_array = tr; mmio_reset_data(tr); enable_mmiotrace(); - return 0; } static void mmio_trace_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_nop.c b/trunk/kernel/trace/trace_nop.c index 0e77415caed3..2ef1d227e7d8 100644 --- a/trunk/kernel/trace/trace_nop.c +++ b/trunk/kernel/trace/trace_nop.c @@ -24,7 +24,7 @@ static void stop_nop_trace(struct trace_array *tr) /* Nothing to do! */ } -static int nop_trace_init(struct trace_array *tr) +static void nop_trace_init(struct trace_array *tr) { int cpu; ctx_trace = tr; @@ -33,7 +33,6 @@ static int nop_trace_init(struct trace_array *tr) tracing_reset(tr, cpu); start_nop_trace(tr); - return 0; } static void nop_trace_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_sched_switch.c b/trunk/kernel/trace/trace_sched_switch.c index 863390557b44..be35bdfe2e38 100644 --- a/trunk/kernel/trace/trace_sched_switch.c +++ b/trunk/kernel/trace/trace_sched_switch.c @@ -206,11 +206,10 @@ static void stop_sched_trace(struct trace_array *tr) tracing_stop_sched_switch_record(); } -static int sched_switch_trace_init(struct trace_array *tr) +static void sched_switch_trace_init(struct trace_array *tr) { ctx_trace = tr; start_sched_trace(tr); - return 0; } static void sched_switch_trace_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_sched_wakeup.c b/trunk/kernel/trace/trace_sched_wakeup.c index 0067b49746c1..983f2b1478c9 100644 --- a/trunk/kernel/trace/trace_sched_wakeup.c +++ b/trunk/kernel/trace/trace_sched_wakeup.c @@ -331,11 +331,10 @@ static void stop_wakeup_tracer(struct trace_array *tr) unregister_trace_sched_wakeup(probe_wakeup); } -static int wakeup_tracer_init(struct trace_array *tr) +static void wakeup_tracer_init(struct trace_array *tr) { wakeup_trace = tr; start_wakeup_tracer(tr); - return 0; } static void wakeup_tracer_reset(struct trace_array *tr) diff --git a/trunk/kernel/trace/trace_selftest.c b/trunk/kernel/trace/trace_selftest.c index 88c8eb70f54a..24e6e075e6d6 100644 --- a/trunk/kernel/trace/trace_selftest.c +++ b/trunk/kernel/trace/trace_selftest.c @@ -52,7 +52,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count) int cpu, ret = 0; /* Don't allow flipping of max traces now */ - local_irq_save(flags); + raw_local_irq_save(flags); __raw_spin_lock(&ftrace_max_lock); cnt = ring_buffer_entries(tr->buffer); @@ -63,7 +63,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count) break; } __raw_spin_unlock(&ftrace_max_lock); - local_irq_restore(flags); + raw_local_irq_restore(flags); if (count) *count = cnt; @@ -71,11 +71,6 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count) return ret; } -static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret) -{ - printk(KERN_WARNING "Failed to init %s tracer, init returned %d\n", - trace->name, init_ret); -} #ifdef CONFIG_FUNCTION_TRACER #ifdef CONFIG_DYNAMIC_FTRACE @@ -116,11 +111,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace, ftrace_set_filter(func_name, strlen(func_name), 1); /* enable tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - goto out; - } + trace->init(tr); /* Sleep for a 1/10 of a second */ msleep(100); @@ -190,12 +181,7 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr) ftrace_enabled = 1; tracer_enabled = 1; - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - goto out; - } - + trace->init(tr); /* Sleep for a 1/10 of a second */ msleep(100); /* stop the tracing. */ @@ -237,12 +223,7 @@ trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr) int ret; /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return ret; - } - + trace->init(tr); /* reset the max latency */ tracing_max_latency = 0; /* disable interrupts for a bit */ @@ -291,12 +272,7 @@ trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr) } /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return ret; - } - + trace->init(tr); /* reset the max latency */ tracing_max_latency = 0; /* disable preemption for a bit */ @@ -345,11 +321,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array * } /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - goto out; - } + trace->init(tr); /* reset the max latency */ tracing_max_latency = 0; @@ -477,12 +449,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr) wait_for_completion(&isrt); /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return ret; - } - + trace->init(tr); /* reset the max latency */ tracing_max_latency = 0; @@ -538,12 +505,7 @@ trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr int ret; /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return ret; - } - + trace->init(tr); /* Sleep for a 1/10 of a second */ msleep(100); /* stop the tracing. */ @@ -570,12 +532,7 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr) int ret; /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return 0; - } - + trace->init(tr); /* Sleep for a 1/10 of a second */ msleep(100); /* stop the tracing. */ @@ -597,12 +554,7 @@ trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr) int ret; /* start the tracing */ - ret = trace->init(tr); - if (ret) { - warn_failed_init_tracer(trace, ret); - return ret; - } - + trace->init(tr); /* Sleep for a 1/10 of a second */ msleep(100); /* stop the tracing. */ diff --git a/trunk/kernel/trace/trace_sysprof.c b/trunk/kernel/trace/trace_sysprof.c index 54960edb96d0..05f753422aea 100644 --- a/trunk/kernel/trace/trace_sysprof.c +++ b/trunk/kernel/trace/trace_sysprof.c @@ -261,12 +261,11 @@ static void stop_stack_trace(struct trace_array *tr) mutex_unlock(&sample_timer_lock); } -static int stack_trace_init(struct trace_array *tr) +static void stack_trace_init(struct trace_array *tr) { sysprof_trace = tr; start_stack_trace(tr); - return 0; } static void stack_trace_reset(struct trace_array *tr)