From e3b24b716f6ba5bba13297fbc17cb1a7dea9d363 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 26 Apr 2011 19:36:14 +0200 Subject: [PATCH] --- yaml --- r: 245461 b: refs/heads/master c: 7bd5fafeb414cf00deee32c82834f8bf1426b9ac h: refs/heads/master i: 245459: a16d60b09a7491c6e18dde086c4fe25a4ea4954a v: v3 --- [refs] | 2 +- trunk/arch/mips/include/asm/jump_label.h | 22 +- trunk/arch/s390/Kconfig | 1 - trunk/arch/s390/include/asm/jump_label.h | 37 -- trunk/arch/s390/kernel/Makefile | 2 +- trunk/arch/s390/kernel/jump_label.c | 59 -- trunk/arch/sparc/include/asm/jump_label.h | 25 +- trunk/arch/x86/include/asm/alternative.h | 3 +- trunk/arch/x86/include/asm/jump_label.h | 27 +- trunk/arch/x86/kernel/alternative.c | 2 +- trunk/arch/x86/kernel/cpu/perf_event.c | 12 +- trunk/arch/x86/kernel/cpu/perf_event_intel.c | 16 - trunk/arch/x86/kernel/cpu/perf_event_p4.c | 17 +- trunk/arch/x86/kernel/module.c | 1 - trunk/include/asm-generic/vmlinux.lds.h | 14 +- trunk/include/linux/dynamic_debug.h | 2 + trunk/include/linux/ftrace.h | 8 +- trunk/include/linux/jump_label.h | 89 ++- trunk/include/linux/jump_label_ref.h | 44 ++ trunk/include/linux/perf_event.h | 119 ++-- trunk/include/linux/tracepoint.h | 22 +- trunk/kernel/Makefile | 6 +- trunk/kernel/events/Makefile | 6 - trunk/kernel/{events => }/hw_breakpoint.c | 0 trunk/kernel/jump_label.c | 539 +++++++++++-------- trunk/kernel/{events/core.c => perf_event.c} | 44 +- trunk/kernel/trace/Kconfig | 2 +- trunk/kernel/trace/ftrace.c | 359 +++++++----- trunk/kernel/trace/trace.c | 15 +- trunk/kernel/trace/trace_output.c | 3 - trunk/kernel/trace/trace_printk.c | 120 +---- trunk/kernel/tracepoint.c | 23 +- 32 files changed, 803 insertions(+), 838 deletions(-) delete mode 100644 trunk/arch/s390/include/asm/jump_label.h delete mode 100644 trunk/arch/s390/kernel/jump_label.c create mode 100644 trunk/include/linux/jump_label_ref.h delete mode 100644 trunk/kernel/events/Makefile rename trunk/kernel/{events => }/hw_breakpoint.c (100%) rename trunk/kernel/{events/core.c => perf_event.c} (99%) diff --git a/[refs] b/[refs] index 5787ddd36f43..3e8430753bc7 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: e04d1b23f9706186187dcb0be1a752e48dcc540b +refs/heads/master: 7bd5fafeb414cf00deee32c82834f8bf1426b9ac diff --git a/trunk/arch/mips/include/asm/jump_label.h b/trunk/arch/mips/include/asm/jump_label.h index 1881b316ca45..7622ccf75076 100644 --- a/trunk/arch/mips/include/asm/jump_label.h +++ b/trunk/arch/mips/include/asm/jump_label.h @@ -20,18 +20,16 @@ #define WORD_INSN ".word" #endif -static __always_inline bool arch_static_branch(struct jump_label_key *key) -{ - asm goto("1:\tnop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - WORD_INSN " 1b, %l[l_yes], %0\n\t" - ".popsection\n\t" - : : "i" (key) : : l_yes); - return false; -l_yes: - return true; -} +#define JUMP_LABEL(key, label) \ + do { \ + asm goto("1:\tnop\n\t" \ + "nop\n\t" \ + ".pushsection __jump_table, \"a\"\n\t" \ + WORD_INSN " 1b, %l[" #label "], %0\n\t" \ + ".popsection\n\t" \ + : : "i" (key) : : label); \ + } while (0) + #endif /* __KERNEL__ */ diff --git a/trunk/arch/s390/Kconfig b/trunk/arch/s390/Kconfig index 4a7f14079e03..2508a6f31588 100644 --- a/trunk/arch/s390/Kconfig +++ b/trunk/arch/s390/Kconfig @@ -88,7 +88,6 @@ config S390 select HAVE_KERNEL_XZ select HAVE_GET_USER_PAGES_FAST select HAVE_ARCH_MUTEX_CPU_RELAX - select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 select ARCH_INLINE_SPIN_TRYLOCK select ARCH_INLINE_SPIN_TRYLOCK_BH select ARCH_INLINE_SPIN_LOCK diff --git a/trunk/arch/s390/include/asm/jump_label.h b/trunk/arch/s390/include/asm/jump_label.h deleted file mode 100644 index 95a6cf2b5b67..000000000000 --- a/trunk/arch/s390/include/asm/jump_label.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef _ASM_S390_JUMP_LABEL_H -#define _ASM_S390_JUMP_LABEL_H - -#include - -#define JUMP_LABEL_NOP_SIZE 6 - -#ifdef CONFIG_64BIT -#define ASM_PTR ".quad" -#define ASM_ALIGN ".balign 8" -#else -#define ASM_PTR ".long" -#define ASM_ALIGN ".balign 4" -#endif - -static __always_inline bool arch_static_branch(struct jump_label_key *key) -{ - asm goto("0: brcl 0,0\n" - ".pushsection __jump_table, \"aw\"\n" - ASM_ALIGN "\n" - ASM_PTR " 0b, %l[label], %0\n" - ".popsection\n" - : : "X" (key) : : label); - return false; -label: - return true; -} - -typedef unsigned long jump_label_t; - -struct jump_entry { - jump_label_t code; - jump_label_t target; - jump_label_t key; -}; - -#endif diff --git a/trunk/arch/s390/kernel/Makefile b/trunk/arch/s390/kernel/Makefile index 5ff15dacb571..64230bc392fa 100644 --- a/trunk/arch/s390/kernel/Makefile +++ b/trunk/arch/s390/kernel/Makefile @@ -23,7 +23,7 @@ CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \ processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \ s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \ - vdso.o vtime.o sysinfo.o nmi.o sclp.o jump_label.o + vdso.o vtime.o sysinfo.o nmi.o sclp.o obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) diff --git a/trunk/arch/s390/kernel/jump_label.c b/trunk/arch/s390/kernel/jump_label.c deleted file mode 100644 index 44cc06bedf77..000000000000 --- a/trunk/arch/s390/kernel/jump_label.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Jump label s390 support - * - * Copyright IBM Corp. 2011 - * Author(s): Jan Glauber - */ -#include -#include -#include -#include -#include - -#ifdef HAVE_JUMP_LABEL - -struct insn { - u16 opcode; - s32 offset; -} __packed; - -struct insn_args { - unsigned long *target; - struct insn *insn; - ssize_t size; -}; - -static int __arch_jump_label_transform(void *data) -{ - struct insn_args *args = data; - int rc; - - rc = probe_kernel_write(args->target, args->insn, args->size); - WARN_ON_ONCE(rc < 0); - return 0; -} - -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) -{ - struct insn_args args; - struct insn insn; - - if (type == JUMP_LABEL_ENABLE) { - /* brcl 15,offset */ - insn.opcode = 0xc0f4; - insn.offset = (entry->target - entry->code) >> 1; - } else { - /* brcl 0,0 */ - insn.opcode = 0xc004; - insn.offset = 0; - } - - args.target = (void *) entry->code; - args.insn = &insn; - args.size = JUMP_LABEL_NOP_SIZE; - - stop_machine(__arch_jump_label_transform, &args, NULL); -} - -#endif diff --git a/trunk/arch/sparc/include/asm/jump_label.h b/trunk/arch/sparc/include/asm/jump_label.h index fc73a82366f8..427d4684e0d2 100644 --- a/trunk/arch/sparc/include/asm/jump_label.h +++ b/trunk/arch/sparc/include/asm/jump_label.h @@ -7,20 +7,17 @@ #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct jump_label_key *key) -{ - asm goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); - return false; -l_yes: - return true; -} +#define JUMP_LABEL(key, label) \ + do { \ + asm goto("1:\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".pushsection __jump_table, \"a\"\n\t"\ + ".align 4\n\t" \ + ".word 1b, %l[" #label "], %c0\n\t" \ + ".popsection \n\t" \ + : : "i" (key) : : label);\ + } while (0) #endif /* __KERNEL__ */ diff --git a/trunk/arch/x86/include/asm/alternative.h b/trunk/arch/x86/include/asm/alternative.h index 8cdd1e247975..13009d1af99a 100644 --- a/trunk/arch/x86/include/asm/alternative.h +++ b/trunk/arch/x86/include/asm/alternative.h @@ -4,6 +4,7 @@ #include #include #include +#include #include /* @@ -190,7 +191,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len); extern void *text_poke_smp(void *addr, const void *opcode, size_t len); extern void text_poke_smp_batch(struct text_poke_param *params, int n); -#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL) +#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) #define IDEAL_NOP_SIZE_5 5 extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; extern void arch_init_ideal_nop5(void); diff --git a/trunk/arch/x86/include/asm/jump_label.h b/trunk/arch/x86/include/asm/jump_label.h index a32b18ce6ead..574dbc22893a 100644 --- a/trunk/arch/x86/include/asm/jump_label.h +++ b/trunk/arch/x86/include/asm/jump_label.h @@ -5,25 +5,20 @@ #include #include -#include #define JUMP_LABEL_NOP_SIZE 5 -#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" - -static __always_inline bool arch_static_branch(struct jump_label_key *key) -{ - asm goto("1:" - JUMP_LABEL_INITIAL_NOP - ".pushsection __jump_table, \"aw\" \n\t" - _ASM_ALIGN "\n\t" - _ASM_PTR "1b, %l[l_yes], %c0 \n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); - return false; -l_yes: - return true; -} +# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" + +# define JUMP_LABEL(key, label) \ + do { \ + asm goto("1:" \ + JUMP_LABEL_INITIAL_NOP \ + ".pushsection __jump_table, \"aw\" \n\t"\ + _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \ + ".popsection \n\t" \ + : : "i" (key) : : label); \ + } while (0) #endif /* __KERNEL__ */ diff --git a/trunk/arch/x86/kernel/alternative.c b/trunk/arch/x86/kernel/alternative.c index 651454b0c811..4a234677e213 100644 --- a/trunk/arch/x86/kernel/alternative.c +++ b/trunk/arch/x86/kernel/alternative.c @@ -679,7 +679,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); } -#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL) +#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) #ifdef CONFIG_X86_64 unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 }; diff --git a/trunk/arch/x86/kernel/cpu/perf_event.c b/trunk/arch/x86/kernel/cpu/perf_event.c index 0de6b2b31f61..dc9e212e31f6 100644 --- a/trunk/arch/x86/kernel/cpu/perf_event.c +++ b/trunk/arch/x86/kernel/cpu/perf_event.c @@ -1295,16 +1295,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs) cpuc = &__get_cpu_var(cpu_hw_events); - /* - * Some chipsets need to unmask the LVTPC in a particular spot - * inside the nmi handler. As a result, the unmasking was pushed - * into all the nmi handlers. - * - * This generic handler doesn't seem to have any issues where the - * unmasking occurs so it was left at the top. - */ - apic_write(APIC_LVTPC, APIC_DM_NMI); - for (idx = 0; idx < x86_pmu.num_counters; idx++) { if (!test_bit(idx, cpuc->active_mask)) { /* @@ -1391,6 +1381,8 @@ perf_event_nmi_handler(struct notifier_block *self, return NOTIFY_DONE; } + apic_write(APIC_LVTPC, APIC_DM_NMI); + handled = x86_pmu.handle_irq(args->regs); if (!handled) return NOTIFY_DONE; diff --git a/trunk/arch/x86/kernel/cpu/perf_event_intel.c b/trunk/arch/x86/kernel/cpu/perf_event_intel.c index 7cf2ec59c813..9ae4a2aa7398 100644 --- a/trunk/arch/x86/kernel/cpu/perf_event_intel.c +++ b/trunk/arch/x86/kernel/cpu/perf_event_intel.c @@ -933,16 +933,6 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) cpuc = &__get_cpu_var(cpu_hw_events); - /* - * Some chipsets need to unmask the LVTPC in a particular spot - * inside the nmi handler. As a result, the unmasking was pushed - * into all the nmi handlers. - * - * This handler doesn't seem to have any issues with the unmasking - * so it was left at the top. - */ - apic_write(APIC_LVTPC, APIC_DM_NMI); - intel_pmu_disable_all(); handled = intel_pmu_drain_bts_buffer(); status = intel_pmu_get_status(); @@ -1474,12 +1464,6 @@ static __init int intel_pmu_init(void) x86_pmu.event_constraints = intel_snb_event_constraints; x86_pmu.pebs_constraints = intel_snb_pebs_events; - - /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; - /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/ - intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1; - pr_cont("SandyBridge events, "); break; diff --git a/trunk/arch/x86/kernel/cpu/perf_event_p4.c b/trunk/arch/x86/kernel/cpu/perf_event_p4.c index ead584fb6a7d..44d4383f2b3e 100644 --- a/trunk/arch/x86/kernel/cpu/perf_event_p4.c +++ b/trunk/arch/x86/kernel/cpu/perf_event_p4.c @@ -949,20 +949,11 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) x86_pmu_stop(event, 0); } - if (handled) + if (handled) { + /* p4 quirk: unmask it again */ + apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); inc_irq_stat(apic_perf_irqs); - - /* - * When dealing with the unmasking of the LVTPC on P4 perf hw, it has - * been observed that the OVF bit flag has to be cleared first _before_ - * the LVTPC can be unmasked. - * - * The reason is the NMI line will continue to be asserted while the OVF - * bit is set. This causes a second NMI to generate if the LVTPC is - * unmasked before the OVF bit is cleared, leading to unknown NMI - * messages. - */ - apic_write(APIC_LVTPC, APIC_DM_NMI); + } return handled; } diff --git a/trunk/arch/x86/kernel/module.c b/trunk/arch/x86/kernel/module.c index 52f256f2cc81..ab23f1ad4bf1 100644 --- a/trunk/arch/x86/kernel/module.c +++ b/trunk/arch/x86/kernel/module.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include diff --git a/trunk/include/asm-generic/vmlinux.lds.h b/trunk/include/asm-generic/vmlinux.lds.h index 75a8692d144f..bd297a20ab98 100644 --- a/trunk/include/asm-generic/vmlinux.lds.h +++ b/trunk/include/asm-generic/vmlinux.lds.h @@ -170,10 +170,6 @@ STRUCT_ALIGN(); \ *(__tracepoints) \ /* implement dynamic printk debug */ \ - . = ALIGN(8); \ - VMLINUX_SYMBOL(__start___jump_table) = .; \ - *(__jump_table) \ - VMLINUX_SYMBOL(__stop___jump_table) = .; \ . = ALIGN(8); \ VMLINUX_SYMBOL(__start___verbose) = .; \ *(__verbose) \ @@ -232,6 +228,8 @@ \ BUG_TABLE \ \ + JUMP_TABLE \ + \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ @@ -591,6 +589,14 @@ #define BUG_TABLE #endif +#define JUMP_TABLE \ + . = ALIGN(8); \ + __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \ + VMLINUX_SYMBOL(__start___jump_table) = .; \ + *(__jump_table) \ + VMLINUX_SYMBOL(__stop___jump_table) = .; \ + } + #ifdef CONFIG_PM_TRACE #define TRACEDATA \ . = ALIGN(4); \ diff --git a/trunk/include/linux/dynamic_debug.h b/trunk/include/linux/dynamic_debug.h index e747ecd48e1c..0c9653f11c18 100644 --- a/trunk/include/linux/dynamic_debug.h +++ b/trunk/include/linux/dynamic_debug.h @@ -1,6 +1,8 @@ #ifndef _DYNAMIC_DEBUG_H #define _DYNAMIC_DEBUG_H +#include + /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They * use independent hash functions, to reduce the chance of false positives. diff --git a/trunk/include/linux/ftrace.h b/trunk/include/linux/ftrace.h index 32047449b309..ca29e03c1fac 100644 --- a/trunk/include/linux/ftrace.h +++ b/trunk/include/linux/ftrace.h @@ -147,9 +147,11 @@ extern int ftrace_text_reserved(void *start, void *end); enum { FTRACE_FL_FREE = (1 << 0), - FTRACE_FL_FILTER = (1 << 1), - FTRACE_FL_ENABLED = (1 << 2), - FTRACE_FL_NOTRACE = (1 << 3), + FTRACE_FL_FAILED = (1 << 1), + FTRACE_FL_FILTER = (1 << 2), + FTRACE_FL_ENABLED = (1 << 3), + FTRACE_FL_NOTRACE = (1 << 4), + FTRACE_FL_CONVERTED = (1 << 5), }; struct dyn_ftrace { diff --git a/trunk/include/linux/jump_label.h b/trunk/include/linux/jump_label.h index 83e745f3ead7..7880f18e4b86 100644 --- a/trunk/include/linux/jump_label.h +++ b/trunk/include/linux/jump_label.h @@ -1,43 +1,20 @@ #ifndef _LINUX_JUMP_LABEL_H #define _LINUX_JUMP_LABEL_H -#include -#include - #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) - -struct jump_label_key { - atomic_t enabled; - struct jump_entry *entries; -#ifdef CONFIG_MODULES - struct jump_label_mod *next; -#endif -}; - # include # define HAVE_JUMP_LABEL #endif enum jump_label_type { - JUMP_LABEL_DISABLE = 0, JUMP_LABEL_ENABLE, + JUMP_LABEL_DISABLE }; struct module; #ifdef HAVE_JUMP_LABEL -#ifdef CONFIG_MODULES -#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL} -#else -#define JUMP_LABEL_INIT {{ 0 }, NULL} -#endif - -static __always_inline bool static_branch(struct jump_label_key *key) -{ - return arch_static_branch(key); -} - extern struct jump_entry __start___jump_table[]; extern struct jump_entry __stop___jump_table[]; @@ -46,37 +23,37 @@ extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type); extern void arch_jump_label_text_poke_early(jump_label_t addr); -extern int jump_label_text_reserved(void *start, void *end); -extern void jump_label_inc(struct jump_label_key *key); -extern void jump_label_dec(struct jump_label_key *key); -extern bool jump_label_enabled(struct jump_label_key *key); +extern void jump_label_update(unsigned long key, enum jump_label_type type); extern void jump_label_apply_nops(struct module *mod); +extern int jump_label_text_reserved(void *start, void *end); -#else +#define jump_label_enable(key) \ + jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE); -#include +#define jump_label_disable(key) \ + jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE); -#define JUMP_LABEL_INIT {ATOMIC_INIT(0)} +#else -struct jump_label_key { - atomic_t enabled; -}; +#define JUMP_LABEL(key, label) \ +do { \ + if (unlikely(*key)) \ + goto label; \ +} while (0) -static __always_inline bool static_branch(struct jump_label_key *key) -{ - if (unlikely(atomic_read(&key->enabled))) - return true; - return false; -} +#define jump_label_enable(cond_var) \ +do { \ + *(cond_var) = 1; \ +} while (0) -static inline void jump_label_inc(struct jump_label_key *key) -{ - atomic_inc(&key->enabled); -} +#define jump_label_disable(cond_var) \ +do { \ + *(cond_var) = 0; \ +} while (0) -static inline void jump_label_dec(struct jump_label_key *key) +static inline int jump_label_apply_nops(struct module *mod) { - atomic_dec(&key->enabled); + return 0; } static inline int jump_label_text_reserved(void *start, void *end) @@ -87,16 +64,16 @@ static inline int jump_label_text_reserved(void *start, void *end) static inline void jump_label_lock(void) {} static inline void jump_label_unlock(void) {} -static inline bool jump_label_enabled(struct jump_label_key *key) -{ - return !!atomic_read(&key->enabled); -} - -static inline int jump_label_apply_nops(struct module *mod) -{ - return 0; -} - #endif +#define COND_STMT(key, stmt) \ +do { \ + __label__ jl_enabled; \ + JUMP_LABEL(key, jl_enabled); \ + if (0) { \ +jl_enabled: \ + stmt; \ + } \ +} while (0) + #endif diff --git a/trunk/include/linux/jump_label_ref.h b/trunk/include/linux/jump_label_ref.h new file mode 100644 index 000000000000..e5d012ad92c6 --- /dev/null +++ b/trunk/include/linux/jump_label_ref.h @@ -0,0 +1,44 @@ +#ifndef _LINUX_JUMP_LABEL_REF_H +#define _LINUX_JUMP_LABEL_REF_H + +#include +#include + +#ifdef HAVE_JUMP_LABEL + +static inline void jump_label_inc(atomic_t *key) +{ + if (atomic_add_return(1, key) == 1) + jump_label_enable(key); +} + +static inline void jump_label_dec(atomic_t *key) +{ + if (atomic_dec_and_test(key)) + jump_label_disable(key); +} + +#else /* !HAVE_JUMP_LABEL */ + +static inline void jump_label_inc(atomic_t *key) +{ + atomic_inc(key); +} + +static inline void jump_label_dec(atomic_t *key) +{ + atomic_dec(key); +} + +#undef JUMP_LABEL +#define JUMP_LABEL(key, label) \ +do { \ + if (unlikely(__builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(key), atomic_t *), \ + atomic_read((atomic_t *)(key)), *(key)))) \ + goto label; \ +} while (0) + +#endif /* HAVE_JUMP_LABEL */ + +#endif /* _LINUX_JUMP_LABEL_REF_H */ diff --git a/trunk/include/linux/perf_event.h b/trunk/include/linux/perf_event.h index 207c16976a17..ee9f1e782800 100644 --- a/trunk/include/linux/perf_event.h +++ b/trunk/include/linux/perf_event.h @@ -2,8 +2,8 @@ * Performance events: * * Copyright (C) 2008-2009, Thomas Gleixner - * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar - * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra + * Copyright (C) 2008-2009, Red Hat, Inc., Ingo Molnar + * Copyright (C) 2008-2009, Red Hat, Inc., Peter Zijlstra * * Data type definitions, declarations, prototypes. * @@ -468,9 +468,9 @@ enum perf_callchain_context { PERF_CONTEXT_MAX = (__u64)-4095, }; -#define PERF_FLAG_FD_NO_GROUP (1U << 0) -#define PERF_FLAG_FD_OUTPUT (1U << 1) -#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ +#define PERF_FLAG_FD_NO_GROUP (1U << 0) +#define PERF_FLAG_FD_OUTPUT (1U << 1) +#define PERF_FLAG_PID_CGROUP (1U << 2) /* pid=cgroup id, per-cpu mode only */ #ifdef __KERNEL__ /* @@ -484,9 +484,9 @@ enum perf_callchain_context { #endif struct perf_guest_info_callbacks { - int (*is_in_guest)(void); - int (*is_user_mode)(void); - unsigned long (*get_guest_ip)(void); + int (*is_in_guest) (void); + int (*is_user_mode) (void); + unsigned long (*get_guest_ip) (void); }; #ifdef CONFIG_HAVE_HW_BREAKPOINT @@ -505,7 +505,7 @@ struct perf_guest_info_callbacks { #include #include #include -#include +#include #include #include @@ -652,19 +652,19 @@ struct pmu { * Start the transaction, after this ->add() doesn't need to * do schedulability tests. */ - void (*start_txn) (struct pmu *pmu); /* optional */ + void (*start_txn) (struct pmu *pmu); /* optional */ /* * If ->start_txn() disabled the ->add() schedulability test * then ->commit_txn() is required to perform one. On success * the transaction is closed. On error the transaction is kept * open until ->cancel_txn() is called. */ - int (*commit_txn) (struct pmu *pmu); /* optional */ + int (*commit_txn) (struct pmu *pmu); /* optional */ /* * Will cancel the transaction, assumes ->del() is called * for each successful ->add() during the transaction. */ - void (*cancel_txn) (struct pmu *pmu); /* optional */ + void (*cancel_txn) (struct pmu *pmu); /* optional */ }; /** @@ -712,15 +712,15 @@ typedef void (*perf_overflow_handler_t)(struct perf_event *, int, struct pt_regs *regs); enum perf_group_flag { - PERF_GROUP_SOFTWARE = 0x1, + PERF_GROUP_SOFTWARE = 0x1, }; -#define SWEVENT_HLIST_BITS 8 -#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) +#define SWEVENT_HLIST_BITS 8 +#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) struct swevent_hlist { - struct hlist_head heads[SWEVENT_HLIST_SIZE]; - struct rcu_head rcu_head; + struct hlist_head heads[SWEVENT_HLIST_SIZE]; + struct rcu_head rcu_head; }; #define PERF_ATTACH_CONTEXT 0x01 @@ -733,13 +733,13 @@ struct swevent_hlist { * This is a per-cpu dynamically allocated data structure. */ struct perf_cgroup_info { - u64 time; - u64 timestamp; + u64 time; + u64 timestamp; }; struct perf_cgroup { - struct cgroup_subsys_state css; - struct perf_cgroup_info *info; /* timing info, one per cpu */ + struct cgroup_subsys_state css; + struct perf_cgroup_info *info; /* timing info, one per cpu */ }; #endif @@ -923,7 +923,7 @@ struct perf_event_context { /* * Number of contexts where an event can trigger: - * task, softirq, hardirq, nmi. + * task, softirq, hardirq, nmi. */ #define PERF_NR_CONTEXTS 4 @@ -1001,7 +1001,8 @@ struct perf_sample_data { struct perf_raw_record *raw; }; -static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr) +static inline +void perf_sample_data_init(struct perf_sample_data *data, u64 addr) { data->addr = addr; data->raw = NULL; @@ -1033,12 +1034,13 @@ static inline int is_software_event(struct perf_event *event) return event->pmu->task_ctx_nr == perf_sw_context; } -extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; +extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); #ifndef perf_arch_fetch_caller_regs -static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } +static inline void +perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } #endif /* @@ -1061,24 +1063,26 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) { struct pt_regs hot_regs; - if (static_branch(&perf_swevent_enabled[event_id])) { - if (!regs) { - perf_fetch_caller_regs(&hot_regs); - regs = &hot_regs; - } - __perf_sw_event(event_id, nr, nmi, regs, addr); + JUMP_LABEL(&perf_swevent_enabled[event_id], have_event); + return; + +have_event: + if (!regs) { + perf_fetch_caller_regs(&hot_regs); + regs = &hot_regs; } + __perf_sw_event(event_id, nr, nmi, regs, addr); } -extern struct jump_label_key perf_sched_events; +extern atomic_t perf_sched_events; static inline void perf_event_task_sched_in(struct task_struct *task) { - if (static_branch(&perf_sched_events)) - __perf_event_task_sched_in(task); + COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task)); } -static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next) +static inline +void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next) { perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); @@ -1096,10 +1100,14 @@ extern void perf_event_fork(struct task_struct *tsk); /* Callchains */ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); -extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs); -extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs); +extern void perf_callchain_user(struct perf_callchain_entry *entry, + struct pt_regs *regs); +extern void perf_callchain_kernel(struct perf_callchain_entry *entry, + struct pt_regs *regs); + -static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) +static inline void +perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) { if (entry->nr < PERF_MAX_STACK_DEPTH) entry->ip[entry->nr++] = ip; @@ -1135,9 +1143,9 @@ extern void perf_tp_event(u64 addr, u64 count, void *record, extern void perf_bp_event(struct perf_event *event, void *data); #ifndef perf_misc_flags -# define perf_misc_flags(regs) \ - (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL) -# define perf_instruction_pointer(regs) instruction_pointer(regs) +#define perf_misc_flags(regs) (user_mode(regs) ? PERF_RECORD_MISC_USER : \ + PERF_RECORD_MISC_KERNEL) +#define perf_instruction_pointer(regs) instruction_pointer(regs) #endif extern int perf_output_begin(struct perf_output_handle *handle, @@ -1172,9 +1180,9 @@ static inline void perf_bp_event(struct perf_event *event, void *data) { } static inline int perf_register_guest_info_callbacks -(struct perf_guest_info_callbacks *callbacks) { return 0; } +(struct perf_guest_info_callbacks *callbacks) { return 0; } static inline int perf_unregister_guest_info_callbacks -(struct perf_guest_info_callbacks *callbacks) { return 0; } +(struct perf_guest_info_callbacks *callbacks) { return 0; } static inline void perf_event_mmap(struct vm_area_struct *vma) { } static inline void perf_event_comm(struct task_struct *tsk) { } @@ -1187,22 +1195,23 @@ static inline void perf_event_disable(struct perf_event *event) { } static inline void perf_event_task_tick(void) { } #endif -#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) +#define perf_output_put(handle, x) \ + perf_output_copy((handle), &(x), sizeof(x)) /* * This has to have a higher priority than migration_notifier in sched.c. */ -#define perf_cpu_notifier(fn) \ -do { \ - static struct notifier_block fn##_nb __cpuinitdata = \ - { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ - fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ - (void *)(unsigned long)smp_processor_id()); \ - fn(&fn##_nb, (unsigned long)CPU_STARTING, \ - (void *)(unsigned long)smp_processor_id()); \ - fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ - (void *)(unsigned long)smp_processor_id()); \ - register_cpu_notifier(&fn##_nb); \ +#define perf_cpu_notifier(fn) \ +do { \ + static struct notifier_block fn##_nb __cpuinitdata = \ + { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ + fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ + (void *)(unsigned long)smp_processor_id()); \ + fn(&fn##_nb, (unsigned long)CPU_STARTING, \ + (void *)(unsigned long)smp_processor_id()); \ + fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ + (void *)(unsigned long)smp_processor_id()); \ + register_cpu_notifier(&fn##_nb); \ } while (0) #endif /* __KERNEL__ */ diff --git a/trunk/include/linux/tracepoint.h b/trunk/include/linux/tracepoint.h index d530a4460a0b..97c84a58efb8 100644 --- a/trunk/include/linux/tracepoint.h +++ b/trunk/include/linux/tracepoint.h @@ -29,7 +29,7 @@ struct tracepoint_func { struct tracepoint { const char *name; /* Tracepoint name */ - struct jump_label_key key; + int state; /* State. */ void (*regfunc)(void); void (*unregfunc)(void); struct tracepoint_func __rcu *funcs; @@ -146,7 +146,9 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin, extern struct tracepoint __tracepoint_##name; \ static inline void trace_##name(proto) \ { \ - if (static_branch(&__tracepoint_##name.key)) \ + JUMP_LABEL(&__tracepoint_##name.state, do_trace); \ + return; \ +do_trace: \ __DO_TRACE(&__tracepoint_##name, \ TP_PROTO(data_proto), \ TP_ARGS(data_args), \ @@ -174,14 +176,14 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin, * structures, so we create an array of pointers that will be used for iteration * on the tracepoints. */ -#define DEFINE_TRACE_FN(name, reg, unreg) \ - static const char __tpstrtab_##name[] \ - __attribute__((section("__tracepoints_strings"))) = #name; \ - struct tracepoint __tracepoint_##name \ - __attribute__((section("__tracepoints"))) = \ - { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\ - static struct tracepoint * const __tracepoint_ptr_##name __used \ - __attribute__((section("__tracepoints_ptrs"))) = \ +#define DEFINE_TRACE_FN(name, reg, unreg) \ + static const char __tpstrtab_##name[] \ + __attribute__((section("__tracepoints_strings"))) = #name; \ + struct tracepoint __tracepoint_##name \ + __attribute__((section("__tracepoints"))) = \ + { __tpstrtab_##name, 0, reg, unreg, NULL }; \ + static struct tracepoint * const __tracepoint_ptr_##name __used \ + __attribute__((section("__tracepoints_ptrs"))) = \ &__tracepoint_##name; #define DEFINE_TRACE(name) \ diff --git a/trunk/kernel/Makefile b/trunk/kernel/Makefile index e9cf19155b46..85cbfb31e73e 100644 --- a/trunk/kernel/Makefile +++ b/trunk/kernel/Makefile @@ -21,6 +21,7 @@ CFLAGS_REMOVE_mutex-debug.o = -pg CFLAGS_REMOVE_rtmutex-debug.o = -pg CFLAGS_REMOVE_cgroup-debug.o = -pg CFLAGS_REMOVE_sched_clock.o = -pg +CFLAGS_REMOVE_perf_event.o = -pg CFLAGS_REMOVE_irq_work.o = -pg endif @@ -102,9 +103,8 @@ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o obj-$(CONFIG_IRQ_WORK) += irq_work.o - -obj-$(CONFIG_PERF_EVENTS) += events/ - +obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o obj-$(CONFIG_PADATA) += padata.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/trunk/kernel/events/Makefile b/trunk/kernel/events/Makefile deleted file mode 100644 index 1ce23d3d8394..000000000000 --- a/trunk/kernel/events/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_core.o = -pg -endif - -obj-y := core.o -obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o diff --git a/trunk/kernel/events/hw_breakpoint.c b/trunk/kernel/hw_breakpoint.c similarity index 100% rename from trunk/kernel/events/hw_breakpoint.c rename to trunk/kernel/hw_breakpoint.c diff --git a/trunk/kernel/jump_label.c b/trunk/kernel/jump_label.c index 74d1c099fbd1..3b79bd938330 100644 --- a/trunk/kernel/jump_label.c +++ b/trunk/kernel/jump_label.c @@ -2,23 +2,43 @@ * jump label support * * Copyright (C) 2009 Jason Baron - * Copyright (C) 2011 Peter Zijlstra * */ +#include #include #include #include #include +#include #include #include #include -#include #ifdef HAVE_JUMP_LABEL +#define JUMP_LABEL_HASH_BITS 6 +#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS) +static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; + /* mutex to protect coming/going of the the jump_label table */ static DEFINE_MUTEX(jump_label_mutex); +struct jump_label_entry { + struct hlist_node hlist; + struct jump_entry *table; + int nr_entries; + /* hang modules off here */ + struct hlist_head modules; + unsigned long key; +}; + +struct jump_label_module_entry { + struct hlist_node hlist; + struct jump_entry *table; + int nr_entries; + struct module *mod; +}; + void jump_label_lock(void) { mutex_lock(&jump_label_mutex); @@ -29,11 +49,6 @@ void jump_label_unlock(void) mutex_unlock(&jump_label_mutex); } -bool jump_label_enabled(struct jump_label_key *key) -{ - return !!atomic_read(&key->enabled); -} - static int jump_label_cmp(const void *a, const void *b) { const struct jump_entry *jea = a; @@ -49,7 +64,7 @@ static int jump_label_cmp(const void *a, const void *b) } static void -jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) +sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) { unsigned long size; @@ -58,25 +73,118 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); } -static void jump_label_update(struct jump_label_key *key, int enable); +static struct jump_label_entry *get_jump_label_entry(jump_label_t key) +{ + struct hlist_head *head; + struct hlist_node *node; + struct jump_label_entry *e; + u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); + + head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; + hlist_for_each_entry(e, node, head, hlist) { + if (key == e->key) + return e; + } + return NULL; +} -void jump_label_inc(struct jump_label_key *key) +static struct jump_label_entry * +add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table) { - if (atomic_inc_not_zero(&key->enabled)) - return; + struct hlist_head *head; + struct jump_label_entry *e; + u32 hash; + + e = get_jump_label_entry(key); + if (e) + return ERR_PTR(-EEXIST); + + e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL); + if (!e) + return ERR_PTR(-ENOMEM); + + hash = jhash((void *)&key, sizeof(jump_label_t), 0); + head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; + e->key = key; + e->table = table; + e->nr_entries = nr_entries; + INIT_HLIST_HEAD(&(e->modules)); + hlist_add_head(&e->hlist, head); + return e; +} - jump_label_lock(); - if (atomic_add_return(1, &key->enabled) == 1) - jump_label_update(key, JUMP_LABEL_ENABLE); - jump_label_unlock(); +static int +build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) +{ + struct jump_entry *iter, *iter_begin; + struct jump_label_entry *entry; + int count; + + sort_jump_label_entries(start, stop); + iter = start; + while (iter < stop) { + entry = get_jump_label_entry(iter->key); + if (!entry) { + iter_begin = iter; + count = 0; + while ((iter < stop) && + (iter->key == iter_begin->key)) { + iter++; + count++; + } + entry = add_jump_label_entry(iter_begin->key, + count, iter_begin); + if (IS_ERR(entry)) + return PTR_ERR(entry); + } else { + WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n"); + return -1; + } + } + return 0; } -void jump_label_dec(struct jump_label_key *key) +/*** + * jump_label_update - update jump label text + * @key - key value associated with a a jump label + * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE + * + * Will enable/disable the jump for jump label @key, depending on the + * value of @type. + * + */ + +void jump_label_update(unsigned long key, enum jump_label_type type) { - if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) - return; + struct jump_entry *iter; + struct jump_label_entry *entry; + struct hlist_node *module_node; + struct jump_label_module_entry *e_module; + int count; - jump_label_update(key, JUMP_LABEL_DISABLE); + jump_label_lock(); + entry = get_jump_label_entry((jump_label_t)key); + if (entry) { + count = entry->nr_entries; + iter = entry->table; + while (count--) { + if (kernel_text_address(iter->code)) + arch_jump_label_transform(iter, type); + iter++; + } + /* eanble/disable jump labels in modules */ + hlist_for_each_entry(e_module, module_node, &(entry->modules), + hlist) { + count = e_module->nr_entries; + iter = e_module->table; + while (count--) { + if (iter->key && + kernel_text_address(iter->code)) + arch_jump_label_transform(iter, type); + iter++; + } + } + } jump_label_unlock(); } @@ -89,33 +197,77 @@ static int addr_conflict(struct jump_entry *entry, void *start, void *end) return 0; } -static int __jump_label_text_reserved(struct jump_entry *iter_start, - struct jump_entry *iter_stop, void *start, void *end) +#ifdef CONFIG_MODULES + +static int module_conflict(void *start, void *end) { + struct hlist_head *head; + struct hlist_node *node, *node_next, *module_node, *module_node_next; + struct jump_label_entry *e; + struct jump_label_module_entry *e_module; struct jump_entry *iter; + int i, count; + int conflict = 0; + + for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { + head = &jump_label_table[i]; + hlist_for_each_entry_safe(e, node, node_next, head, hlist) { + hlist_for_each_entry_safe(e_module, module_node, + module_node_next, + &(e->modules), hlist) { + count = e_module->nr_entries; + iter = e_module->table; + while (count--) { + if (addr_conflict(iter, start, end)) { + conflict = 1; + goto out; + } + iter++; + } + } + } + } +out: + return conflict; +} + +#endif + +/*** + * jump_label_text_reserved - check if addr range is reserved + * @start: start text addr + * @end: end text addr + * + * checks if the text addr located between @start and @end + * overlaps with any of the jump label patch addresses. Code + * that wants to modify kernel text should first verify that + * it does not overlap with any of the jump label addresses. + * Caller must hold jump_label_mutex. + * + * returns 1 if there is an overlap, 0 otherwise + */ +int jump_label_text_reserved(void *start, void *end) +{ + struct jump_entry *iter; + struct jump_entry *iter_start = __start___jump_table; + struct jump_entry *iter_stop = __start___jump_table; + int conflict = 0; iter = iter_start; while (iter < iter_stop) { - if (addr_conflict(iter, start, end)) - return 1; + if (addr_conflict(iter, start, end)) { + conflict = 1; + goto out; + } iter++; } - return 0; -} - -static void __jump_label_update(struct jump_label_key *key, - struct jump_entry *entry, int enable) -{ - for (; entry->key == (jump_label_t)(unsigned long)key; entry++) { - /* - * entry->code set to 0 invalidates module init text sections - * kernel_text_address() verifies we are not in core kernel - * init code, see jump_label_invalidate_module_init(). - */ - if (entry->code && kernel_text_address(entry->code)) - arch_jump_label_transform(entry, enable); - } + /* now check modules */ +#ifdef CONFIG_MODULES + conflict = module_conflict(start, end); +#endif +out: + return conflict; } /* @@ -125,173 +277,142 @@ void __weak arch_jump_label_text_poke_early(jump_label_t addr) { } -static __init int jump_label_init(void) +static __init int init_jump_label(void) { + int ret; struct jump_entry *iter_start = __start___jump_table; struct jump_entry *iter_stop = __stop___jump_table; - struct jump_label_key *key = NULL; struct jump_entry *iter; jump_label_lock(); - jump_label_sort_entries(iter_start, iter_stop); - - for (iter = iter_start; iter < iter_stop; iter++) { + ret = build_jump_label_hashtable(__start___jump_table, + __stop___jump_table); + iter = iter_start; + while (iter < iter_stop) { arch_jump_label_text_poke_early(iter->code); - if (iter->key == (jump_label_t)(unsigned long)key) - continue; - - key = (struct jump_label_key *)(unsigned long)iter->key; - atomic_set(&key->enabled, 0); - key->entries = iter; -#ifdef CONFIG_MODULES - key->next = NULL; -#endif + iter++; } jump_label_unlock(); - - return 0; + return ret; } -early_initcall(jump_label_init); +early_initcall(init_jump_label); #ifdef CONFIG_MODULES -struct jump_label_mod { - struct jump_label_mod *next; - struct jump_entry *entries; - struct module *mod; -}; - -static int __jump_label_mod_text_reserved(void *start, void *end) -{ - struct module *mod; - - mod = __module_text_address((unsigned long)start); - if (!mod) - return 0; - - WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); - - return __jump_label_text_reserved(mod->jump_entries, - mod->jump_entries + mod->num_jump_entries, - start, end); -} - -static void __jump_label_mod_update(struct jump_label_key *key, int enable) -{ - struct jump_label_mod *mod = key->next; - - while (mod) { - __jump_label_update(key, mod->entries, enable); - mod = mod->next; - } -} - -/*** - * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() - * @mod: module to patch - * - * Allow for run-time selection of the optimal nops. Before the module - * loads patch these with arch_get_jump_label_nop(), which is specified by - * the arch specific jump label code. - */ -void jump_label_apply_nops(struct module *mod) +static struct jump_label_module_entry * +add_jump_label_module_entry(struct jump_label_entry *entry, + struct jump_entry *iter_begin, + int count, struct module *mod) { - struct jump_entry *iter_start = mod->jump_entries; - struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; - struct jump_entry *iter; - - /* if the module doesn't have jump label entries, just return */ - if (iter_start == iter_stop) - return; - - for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_text_poke_early(iter->code); + struct jump_label_module_entry *e; + + e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); + if (!e) + return ERR_PTR(-ENOMEM); + e->mod = mod; + e->nr_entries = count; + e->table = iter_begin; + hlist_add_head(&e->hlist, &entry->modules); + return e; } -static int jump_label_add_module(struct module *mod) +static int add_jump_label_module(struct module *mod) { - struct jump_entry *iter_start = mod->jump_entries; - struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; - struct jump_entry *iter; - struct jump_label_key *key = NULL; - struct jump_label_mod *jlm; + struct jump_entry *iter, *iter_begin; + struct jump_label_entry *entry; + struct jump_label_module_entry *module_entry; + int count; /* if the module doesn't have jump label entries, just return */ - if (iter_start == iter_stop) + if (!mod->num_jump_entries) return 0; - jump_label_sort_entries(iter_start, iter_stop); - - for (iter = iter_start; iter < iter_stop; iter++) { - if (iter->key == (jump_label_t)(unsigned long)key) - continue; - - key = (struct jump_label_key *)(unsigned long)iter->key; - - if (__module_address(iter->key) == mod) { - atomic_set(&key->enabled, 0); - key->entries = iter; - key->next = NULL; - continue; + sort_jump_label_entries(mod->jump_entries, + mod->jump_entries + mod->num_jump_entries); + iter = mod->jump_entries; + while (iter < mod->jump_entries + mod->num_jump_entries) { + entry = get_jump_label_entry(iter->key); + iter_begin = iter; + count = 0; + while ((iter < mod->jump_entries + mod->num_jump_entries) && + (iter->key == iter_begin->key)) { + iter++; + count++; } - - jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); - if (!jlm) - return -ENOMEM; - - jlm->mod = mod; - jlm->entries = iter; - jlm->next = key->next; - key->next = jlm; - - if (jump_label_enabled(key)) - __jump_label_update(key, iter, JUMP_LABEL_ENABLE); + if (!entry) { + entry = add_jump_label_entry(iter_begin->key, 0, NULL); + if (IS_ERR(entry)) + return PTR_ERR(entry); + } + module_entry = add_jump_label_module_entry(entry, iter_begin, + count, mod); + if (IS_ERR(module_entry)) + return PTR_ERR(module_entry); } - return 0; } -static void jump_label_del_module(struct module *mod) +static void remove_jump_label_module(struct module *mod) { - struct jump_entry *iter_start = mod->jump_entries; - struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; - struct jump_entry *iter; - struct jump_label_key *key = NULL; - struct jump_label_mod *jlm, **prev; + struct hlist_head *head; + struct hlist_node *node, *node_next, *module_node, *module_node_next; + struct jump_label_entry *e; + struct jump_label_module_entry *e_module; + int i; - for (iter = iter_start; iter < iter_stop; iter++) { - if (iter->key == (jump_label_t)(unsigned long)key) - continue; - - key = (struct jump_label_key *)(unsigned long)iter->key; - - if (__module_address(iter->key) == mod) - continue; - - prev = &key->next; - jlm = key->next; - - while (jlm && jlm->mod != mod) { - prev = &jlm->next; - jlm = jlm->next; - } + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return; - if (jlm) { - *prev = jlm->next; - kfree(jlm); + for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { + head = &jump_label_table[i]; + hlist_for_each_entry_safe(e, node, node_next, head, hlist) { + hlist_for_each_entry_safe(e_module, module_node, + module_node_next, + &(e->modules), hlist) { + if (e_module->mod == mod) { + hlist_del(&e_module->hlist); + kfree(e_module); + } + } + if (hlist_empty(&e->modules) && (e->nr_entries == 0)) { + hlist_del(&e->hlist); + kfree(e); + } } } } -static void jump_label_invalidate_module_init(struct module *mod) +static void remove_jump_label_module_init(struct module *mod) { - struct jump_entry *iter_start = mod->jump_entries; - struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; + struct hlist_head *head; + struct hlist_node *node, *node_next, *module_node, *module_node_next; + struct jump_label_entry *e; + struct jump_label_module_entry *e_module; struct jump_entry *iter; + int i, count; + + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return; - for (iter = iter_start; iter < iter_stop; iter++) { - if (within_module_init(iter->code, mod)) - iter->code = 0; + for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { + head = &jump_label_table[i]; + hlist_for_each_entry_safe(e, node, node_next, head, hlist) { + hlist_for_each_entry_safe(e_module, module_node, + module_node_next, + &(e->modules), hlist) { + if (e_module->mod != mod) + continue; + count = e_module->nr_entries; + iter = e_module->table; + while (count--) { + if (within_module_init(iter->code, mod)) + iter->key = 0; + iter++; + } + } + } } } @@ -305,77 +426,59 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val, switch (val) { case MODULE_STATE_COMING: jump_label_lock(); - ret = jump_label_add_module(mod); + ret = add_jump_label_module(mod); if (ret) - jump_label_del_module(mod); + remove_jump_label_module(mod); jump_label_unlock(); break; case MODULE_STATE_GOING: jump_label_lock(); - jump_label_del_module(mod); + remove_jump_label_module(mod); jump_label_unlock(); break; case MODULE_STATE_LIVE: jump_label_lock(); - jump_label_invalidate_module_init(mod); + remove_jump_label_module_init(mod); jump_label_unlock(); break; } - - return notifier_from_errno(ret); -} - -struct notifier_block jump_label_module_nb = { - .notifier_call = jump_label_module_notify, - .priority = 1, /* higher than tracepoints */ -}; - -static __init int jump_label_init_module(void) -{ - return register_module_notifier(&jump_label_module_nb); + return ret; } -early_initcall(jump_label_init_module); - -#endif /* CONFIG_MODULES */ /*** - * jump_label_text_reserved - check if addr range is reserved - * @start: start text addr - * @end: end text addr - * - * checks if the text addr located between @start and @end - * overlaps with any of the jump label patch addresses. Code - * that wants to modify kernel text should first verify that - * it does not overlap with any of the jump label addresses. - * Caller must hold jump_label_mutex. + * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() + * @mod: module to patch * - * returns 1 if there is an overlap, 0 otherwise + * Allow for run-time selection of the optimal nops. Before the module + * loads patch these with arch_get_jump_label_nop(), which is specified by + * the arch specific jump label code. */ -int jump_label_text_reserved(void *start, void *end) +void jump_label_apply_nops(struct module *mod) { - int ret = __jump_label_text_reserved(__start___jump_table, - __stop___jump_table, start, end); + struct jump_entry *iter; - if (ret) - return ret; + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return; -#ifdef CONFIG_MODULES - ret = __jump_label_mod_text_reserved(start, end); -#endif - return ret; + iter = mod->jump_entries; + while (iter < mod->jump_entries + mod->num_jump_entries) { + arch_jump_label_text_poke_early(iter->code); + iter++; + } } -static void jump_label_update(struct jump_label_key *key, int enable) -{ - struct jump_entry *entry = key->entries; - - /* if there are no users, entry can be NULL */ - if (entry) - __jump_label_update(key, entry, enable); +struct notifier_block jump_label_module_nb = { + .notifier_call = jump_label_module_notify, + .priority = 0, +}; -#ifdef CONFIG_MODULES - __jump_label_mod_update(key, enable); -#endif +static __init int init_jump_label_module(void) +{ + return register_module_notifier(&jump_label_module_nb); } +early_initcall(init_jump_label_module); + +#endif /* CONFIG_MODULES */ #endif diff --git a/trunk/kernel/events/core.c b/trunk/kernel/perf_event.c similarity index 99% rename from trunk/kernel/events/core.c rename to trunk/kernel/perf_event.c index 0fc34a370ba4..8e81a9860a0d 100644 --- a/trunk/kernel/events/core.c +++ b/trunk/kernel/perf_event.c @@ -2,8 +2,8 @@ * Performance events core code: * * Copyright (C) 2008 Thomas Gleixner - * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar - * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra + * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar + * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra * Copyright © 2009 Paul Mackerras, IBM Corp. * * For licensing details see kernel-base/COPYING @@ -39,10 +39,10 @@ #include struct remote_function_call { - struct task_struct *p; - int (*func)(void *info); - void *info; - int ret; + struct task_struct *p; + int (*func)(void *info); + void *info; + int ret; }; static void remote_function(void *data) @@ -76,10 +76,10 @@ static int task_function_call(struct task_struct *p, int (*func) (void *info), void *info) { struct remote_function_call data = { - .p = p, - .func = func, - .info = info, - .ret = -ESRCH, /* No such (running) process */ + .p = p, + .func = func, + .info = info, + .ret = -ESRCH, /* No such (running) process */ }; if (task_curr(p)) @@ -100,10 +100,10 @@ task_function_call(struct task_struct *p, int (*func) (void *info), void *info) static int cpu_function_call(int cpu, int (*func) (void *info), void *info) { struct remote_function_call data = { - .p = NULL, - .func = func, - .info = info, - .ret = -ENXIO, /* No such CPU */ + .p = NULL, + .func = func, + .info = info, + .ret = -ENXIO, /* No such CPU */ }; smp_call_function_single(cpu, remote_function, &data, 1); @@ -125,7 +125,7 @@ enum event_type_t { * perf_sched_events : >0 events exist * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu */ -struct jump_label_key perf_sched_events __read_mostly; +atomic_t perf_sched_events __read_mostly; static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); static atomic_t nr_mmap_events __read_mostly; @@ -5429,7 +5429,7 @@ static int swevent_hlist_get(struct perf_event *event) return err; } -struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; +atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; static void sw_perf_event_destroy(struct perf_event *event) { @@ -7445,11 +7445,11 @@ static void perf_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp, } struct cgroup_subsys perf_subsys = { - .name = "perf_event", - .subsys_id = perf_subsys_id, - .create = perf_cgroup_create, - .destroy = perf_cgroup_destroy, - .exit = perf_cgroup_exit, - .attach = perf_cgroup_attach, + .name = "perf_event", + .subsys_id = perf_subsys_id, + .create = perf_cgroup_create, + .destroy = perf_cgroup_destroy, + .exit = perf_cgroup_exit, + .attach = perf_cgroup_attach, }; #endif /* CONFIG_CGROUP_PERF */ diff --git a/trunk/kernel/trace/Kconfig b/trunk/kernel/trace/Kconfig index 2ad39e556cb4..61d7d59f4a1a 100644 --- a/trunk/kernel/trace/Kconfig +++ b/trunk/kernel/trace/Kconfig @@ -141,7 +141,7 @@ if FTRACE config FUNCTION_TRACER bool "Kernel Function Tracer" depends on HAVE_FUNCTION_TRACER - select FRAME_POINTER if !ARM_UNWIND && !S390 && !MICROBLAZE + select FRAME_POINTER if !ARM_UNWIND && !S390 select KALLSYMS select GENERIC_TRACER select CONTEXT_SWITCH_TRACER diff --git a/trunk/kernel/trace/ftrace.c b/trunk/kernel/trace/ftrace.c index d3406346ced6..ee24fa1935ac 100644 --- a/trunk/kernel/trace/ftrace.c +++ b/trunk/kernel/trace/ftrace.c @@ -39,20 +39,16 @@ #include "trace_stat.h" #define FTRACE_WARN_ON(cond) \ - ({ \ - int ___r = cond; \ - if (WARN_ON(___r)) \ + do { \ + if (WARN_ON(cond)) \ ftrace_kill(); \ - ___r; \ - }) + } while (0) #define FTRACE_WARN_ON_ONCE(cond) \ - ({ \ - int ___r = cond; \ - if (WARN_ON_ONCE(___r)) \ + do { \ + if (WARN_ON_ONCE(cond)) \ ftrace_kill(); \ - ___r; \ - }) + } while (0) /* hash bits for specific function selection */ #define FTRACE_HASH_BITS 7 @@ -151,34 +147,6 @@ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) } #endif -static void update_ftrace_function(void) -{ - ftrace_func_t func; - - /* - * If there's only one function registered, then call that - * function directly. Otherwise, we need to iterate over the - * registered callers. - */ - if (ftrace_list == &ftrace_list_end || - ftrace_list->next == &ftrace_list_end) - func = ftrace_list->func; - else - func = ftrace_list_func; - - /* If we filter on pids, update to use the pid function */ - if (!list_empty(&ftrace_pids)) { - set_ftrace_pid_function(func); - func = ftrace_pid_func; - } -#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST - ftrace_trace_function = func; -#else - __ftrace_trace_function = func; - ftrace_trace_function = ftrace_test_stop_func; -#endif -} - static int __register_ftrace_function(struct ftrace_ops *ops) { ops->next = ftrace_list; @@ -190,8 +158,30 @@ static int __register_ftrace_function(struct ftrace_ops *ops) */ rcu_assign_pointer(ftrace_list, ops); - if (ftrace_enabled) - update_ftrace_function(); + if (ftrace_enabled) { + ftrace_func_t func; + + if (ops->next == &ftrace_list_end) + func = ops->func; + else + func = ftrace_list_func; + + if (!list_empty(&ftrace_pids)) { + set_ftrace_pid_function(func); + func = ftrace_pid_func; + } + + /* + * For one func, simply call it directly. + * For more than one func, call the chain. + */ +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ftrace_trace_function = func; +#else + __ftrace_trace_function = func; + ftrace_trace_function = ftrace_test_stop_func; +#endif + } return 0; } @@ -219,19 +209,52 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops) *p = (*p)->next; - if (ftrace_enabled) - update_ftrace_function(); + if (ftrace_enabled) { + /* If we only have one func left, then call that directly */ + if (ftrace_list->next == &ftrace_list_end) { + ftrace_func_t func = ftrace_list->func; + + if (!list_empty(&ftrace_pids)) { + set_ftrace_pid_function(func); + func = ftrace_pid_func; + } +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ftrace_trace_function = func; +#else + __ftrace_trace_function = func; +#endif + } + } return 0; } static void ftrace_update_pid_func(void) { - /* Only do something if we are tracing something */ + ftrace_func_t func; + if (ftrace_trace_function == ftrace_stub) return; - update_ftrace_function(); +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + func = ftrace_trace_function; +#else + func = __ftrace_trace_function; +#endif + + if (!list_empty(&ftrace_pids)) { + set_ftrace_pid_function(func); + func = ftrace_pid_func; + } else { + if (func == ftrace_pid_func) + func = ftrace_pid_function; + } + +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ftrace_trace_function = func; +#else + __ftrace_trace_function = func; +#endif } #ifdef CONFIG_FUNCTION_PROFILER @@ -1056,16 +1079,19 @@ static void ftrace_replace_code(int enable) struct ftrace_page *pg; int failed; - if (unlikely(ftrace_disabled)) - return; - do_for_each_ftrace_rec(pg, rec) { - /* Skip over free records */ - if (rec->flags & FTRACE_FL_FREE) + /* + * Skip over free records, records that have + * failed and not converted. + */ + if (rec->flags & FTRACE_FL_FREE || + rec->flags & FTRACE_FL_FAILED || + !(rec->flags & FTRACE_FL_CONVERTED)) continue; failed = __ftrace_replace_code(rec, enable); if (failed) { + rec->flags |= FTRACE_FL_FAILED; ftrace_bug(failed, rec->ip); /* Stop processing */ return; @@ -1081,12 +1107,10 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) ip = rec->ip; - if (unlikely(ftrace_disabled)) - return 0; - ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); if (ret) { ftrace_bug(ret, ip); + rec->flags |= FTRACE_FL_FAILED; return 0; } return 1; @@ -1249,10 +1273,10 @@ static int ftrace_update_code(struct module *mod) */ if (!ftrace_code_disable(mod, p)) { ftrace_free_rec(p); - /* Game over */ - break; + continue; } + p->flags |= FTRACE_FL_CONVERTED; ftrace_update_cnt++; /* @@ -1327,8 +1351,9 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) enum { FTRACE_ITER_FILTER = (1 << 0), FTRACE_ITER_NOTRACE = (1 << 1), - FTRACE_ITER_PRINTALL = (1 << 2), - FTRACE_ITER_HASH = (1 << 3), + FTRACE_ITER_FAILURES = (1 << 2), + FTRACE_ITER_PRINTALL = (1 << 3), + FTRACE_ITER_HASH = (1 << 4), }; #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ @@ -1438,9 +1463,6 @@ t_next(struct seq_file *m, void *v, loff_t *pos) struct ftrace_iterator *iter = m->private; struct dyn_ftrace *rec = NULL; - if (unlikely(ftrace_disabled)) - return NULL; - if (iter->flags & FTRACE_ITER_HASH) return t_hash_next(m, pos); @@ -1461,6 +1483,12 @@ t_next(struct seq_file *m, void *v, loff_t *pos) rec = &iter->pg->records[iter->idx++]; if ((rec->flags & FTRACE_FL_FREE) || + (!(iter->flags & FTRACE_ITER_FAILURES) && + (rec->flags & FTRACE_FL_FAILED)) || + + ((iter->flags & FTRACE_ITER_FAILURES) && + !(rec->flags & FTRACE_FL_FAILED)) || + ((iter->flags & FTRACE_ITER_FILTER) && !(rec->flags & FTRACE_FL_FILTER)) || @@ -1493,10 +1521,6 @@ static void *t_start(struct seq_file *m, loff_t *pos) loff_t l; mutex_lock(&ftrace_lock); - - if (unlikely(ftrace_disabled)) - return NULL; - /* * If an lseek was done, then reset and start from beginning. */ @@ -1605,6 +1629,24 @@ ftrace_avail_open(struct inode *inode, struct file *file) return ret; } +static int +ftrace_failures_open(struct inode *inode, struct file *file) +{ + int ret; + struct seq_file *m; + struct ftrace_iterator *iter; + + ret = ftrace_avail_open(inode, file); + if (!ret) { + m = file->private_data; + iter = m->private; + iter->flags = FTRACE_ITER_FAILURES; + } + + return ret; +} + + static void ftrace_filter_reset(int enable) { struct ftrace_page *pg; @@ -1615,6 +1657,8 @@ static void ftrace_filter_reset(int enable) if (enable) ftrace_filtered = 0; do_for_each_ftrace_rec(pg, rec) { + if (rec->flags & FTRACE_FL_FAILED) + continue; rec->flags &= ~type; } while_for_each_ftrace_rec(); mutex_unlock(&ftrace_lock); @@ -1716,63 +1760,42 @@ static int ftrace_match(char *str, char *regex, int len, int type) return matched; } -static void -update_record(struct dyn_ftrace *rec, unsigned long flag, int not) -{ - if (not) - rec->flags &= ~flag; - else - rec->flags |= flag; -} - static int -ftrace_match_record(struct dyn_ftrace *rec, char *mod, - char *regex, int len, int type) +ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) { char str[KSYM_SYMBOL_LEN]; - char *modname; - - kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); - - if (mod) { - /* module lookup requires matching the module */ - if (!modname || strcmp(modname, mod)) - return 0; - - /* blank search means to match all funcs in the mod */ - if (!len) - return 1; - } + kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); return ftrace_match(str, regex, len, type); } -static int match_records(char *buff, int len, char *mod, int enable, int not) +static int ftrace_match_records(char *buff, int len, int enable) { - unsigned search_len = 0; + unsigned int search_len; struct ftrace_page *pg; struct dyn_ftrace *rec; - int type = MATCH_FULL; - char *search = buff; unsigned long flag; + char *search; + int type; + int not; int found = 0; - if (len) { - type = filter_parse_regex(buff, len, &search, ¬); - search_len = strlen(search); - } - flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; + type = filter_parse_regex(buff, len, &search, ¬); - mutex_lock(&ftrace_lock); - - if (unlikely(ftrace_disabled)) - goto out_unlock; + search_len = strlen(search); + mutex_lock(&ftrace_lock); do_for_each_ftrace_rec(pg, rec) { - if (ftrace_match_record(rec, mod, search, search_len, type)) { - update_record(rec, flag, not); + if (rec->flags & FTRACE_FL_FAILED) + continue; + + if (ftrace_match_record(rec, search, search_len, type)) { + if (not) + rec->flags &= ~flag; + else + rec->flags |= flag; found = 1; } /* @@ -1781,23 +1804,43 @@ static int match_records(char *buff, int len, char *mod, int enable, int not) */ if (enable && (rec->flags & FTRACE_FL_FILTER)) ftrace_filtered = 1; - } while_for_each_ftrace_rec(); - out_unlock: mutex_unlock(&ftrace_lock); return found; } static int -ftrace_match_records(char *buff, int len, int enable) +ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, + char *regex, int len, int type) { - return match_records(buff, len, NULL, enable, 0); + char str[KSYM_SYMBOL_LEN]; + char *modname; + + kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); + + if (!modname || strcmp(modname, mod)) + return 0; + + /* blank search means to match all funcs in the mod */ + if (len) + return ftrace_match(str, regex, len, type); + else + return 1; } static int ftrace_match_module_records(char *buff, char *mod, int enable) { + unsigned search_len = 0; + struct ftrace_page *pg; + struct dyn_ftrace *rec; + int type = MATCH_FULL; + char *search = buff; + unsigned long flag; int not = 0; + int found = 0; + + flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; /* blank or '*' mean the same */ if (strcmp(buff, "*") == 0) @@ -1809,7 +1852,32 @@ static int ftrace_match_module_records(char *buff, char *mod, int enable) not = 1; } - return match_records(buff, strlen(buff), mod, enable, not); + if (strlen(buff)) { + type = filter_parse_regex(buff, strlen(buff), &search, ¬); + search_len = strlen(search); + } + + mutex_lock(&ftrace_lock); + do_for_each_ftrace_rec(pg, rec) { + + if (rec->flags & FTRACE_FL_FAILED) + continue; + + if (ftrace_match_module_record(rec, mod, + search, search_len, type)) { + if (not) + rec->flags &= ~flag; + else + rec->flags |= flag; + found = 1; + } + if (enable && (rec->flags & FTRACE_FL_FILTER)) + ftrace_filtered = 1; + + } while_for_each_ftrace_rec(); + mutex_unlock(&ftrace_lock); + + return found; } /* @@ -1961,13 +2029,12 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, return -EINVAL; mutex_lock(&ftrace_lock); - - if (unlikely(ftrace_disabled)) - goto out_unlock; - do_for_each_ftrace_rec(pg, rec) { - if (!ftrace_match_record(rec, NULL, search, len, type)) + if (rec->flags & FTRACE_FL_FAILED) + continue; + + if (!ftrace_match_record(rec, search, len, type)) continue; entry = kmalloc(sizeof(*entry), GFP_KERNEL); @@ -2172,10 +2239,6 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, mutex_lock(&ftrace_regex_lock); - ret = -ENODEV; - if (unlikely(ftrace_disabled)) - goto out_unlock; - if (file->f_mode & FMODE_READ) { struct seq_file *m = file->private_data; iter = m->private; @@ -2350,16 +2413,14 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable) ftrace_match_records(parser->buffer, parser->idx, enable); } + mutex_lock(&ftrace_lock); + if (ftrace_start_up && ftrace_enabled) + ftrace_run_update_code(FTRACE_ENABLE_CALLS); + mutex_unlock(&ftrace_lock); + trace_parser_put(parser); kfree(iter); - if (file->f_mode & FMODE_WRITE) { - mutex_lock(&ftrace_lock); - if (ftrace_start_up && ftrace_enabled) - ftrace_run_update_code(FTRACE_ENABLE_CALLS); - mutex_unlock(&ftrace_lock); - } - mutex_unlock(&ftrace_regex_lock); return 0; } @@ -2383,6 +2444,13 @@ static const struct file_operations ftrace_avail_fops = { .release = seq_release_private, }; +static const struct file_operations ftrace_failures_fops = { + .open = ftrace_failures_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + static const struct file_operations ftrace_filter_fops = { .open = ftrace_filter_open, .read = seq_read, @@ -2505,6 +2573,9 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) bool exists; int i; + if (ftrace_disabled) + return -ENODEV; + /* decode regex */ type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) @@ -2513,18 +2584,12 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) search_len = strlen(search); mutex_lock(&ftrace_lock); - - if (unlikely(ftrace_disabled)) { - mutex_unlock(&ftrace_lock); - return -ENODEV; - } - do_for_each_ftrace_rec(pg, rec) { - if (rec->flags & FTRACE_FL_FREE) + if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) continue; - if (ftrace_match_record(rec, NULL, search, search_len, type)) { + if (ftrace_match_record(rec, search, search_len, type)) { /* if it is in the array */ exists = false; for (i = 0; i < *idx; i++) { @@ -2614,6 +2679,9 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) trace_create_file("available_filter_functions", 0444, d_tracer, NULL, &ftrace_avail_fops); + trace_create_file("failures", 0444, + d_tracer, NULL, &ftrace_failures_fops); + trace_create_file("set_ftrace_filter", 0644, d_tracer, NULL, &ftrace_filter_fops); @@ -2635,6 +2703,7 @@ static int ftrace_process_locs(struct module *mod, { unsigned long *p; unsigned long addr; + unsigned long flags; mutex_lock(&ftrace_lock); p = start; @@ -2651,7 +2720,10 @@ static int ftrace_process_locs(struct module *mod, ftrace_record_ip(addr); } + /* disable interrupts to prevent kstop machine */ + local_irq_save(flags); ftrace_update_code(mod); + local_irq_restore(flags); mutex_unlock(&ftrace_lock); return 0; @@ -2663,11 +2735,10 @@ void ftrace_release_mod(struct module *mod) struct dyn_ftrace *rec; struct ftrace_page *pg; - mutex_lock(&ftrace_lock); - if (ftrace_disabled) - goto out_unlock; + return; + mutex_lock(&ftrace_lock); do_for_each_ftrace_rec(pg, rec) { if (within_module_core(rec->ip, mod)) { /* @@ -2678,7 +2749,6 @@ void ftrace_release_mod(struct module *mod) ftrace_free_rec(rec); } } while_for_each_ftrace_rec(); - out_unlock: mutex_unlock(&ftrace_lock); } @@ -3073,17 +3143,16 @@ void ftrace_kill(void) */ int register_ftrace_function(struct ftrace_ops *ops) { - int ret = -1; - - mutex_lock(&ftrace_lock); + int ret; if (unlikely(ftrace_disabled)) - goto out_unlock; + return -1; + + mutex_lock(&ftrace_lock); ret = __register_ftrace_function(ops); ftrace_startup(0); - out_unlock: mutex_unlock(&ftrace_lock); return ret; } @@ -3111,14 +3180,14 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - int ret = -ENODEV; - - mutex_lock(&ftrace_lock); + int ret; if (unlikely(ftrace_disabled)) - goto out; + return -ENODEV; + + mutex_lock(&ftrace_lock); - ret = proc_dointvec(table, write, buffer, lenp, ppos); + ret = proc_dointvec(table, write, buffer, lenp, ppos); if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) goto out; diff --git a/trunk/kernel/trace/trace.c b/trunk/kernel/trace/trace.c index e0e14ce0caab..d38c16a06a6f 100644 --- a/trunk/kernel/trace/trace.c +++ b/trunk/kernel/trace/trace.c @@ -2013,10 +2013,9 @@ enum print_line_t print_trace_line(struct trace_iterator *iter) { enum print_line_t ret; - if (iter->lost_events && - !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", - iter->cpu, iter->lost_events)) - return TRACE_TYPE_PARTIAL_LINE; + if (iter->lost_events) + trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", + iter->cpu, iter->lost_events); if (iter->trace && iter->trace->print_line) { ret = iter->trace->print_line(iter); @@ -3230,14 +3229,6 @@ tracing_read_pipe(struct file *filp, char __user *ubuf, if (iter->seq.len >= cnt) break; - - /* - * Setting the full flag means we reached the trace_seq buffer - * size and we should leave by partial output condition above. - * One of the trace_seq_* functions is not used properly. - */ - WARN_ONCE(iter->seq.full, "full flag set for trace type %d", - iter->ent->type); } trace_access_unlock(iter->cpu_file); trace_event_read_unlock(); diff --git a/trunk/kernel/trace/trace_output.c b/trunk/kernel/trace/trace_output.c index cf535ccedc86..456be9063c2d 100644 --- a/trunk/kernel/trace/trace_output.c +++ b/trunk/kernel/trace/trace_output.c @@ -830,9 +830,6 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event); enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags, struct trace_event *event) { - if (!trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type)) - return TRACE_TYPE_PARTIAL_LINE; - return TRACE_TYPE_HANDLED; } diff --git a/trunk/kernel/trace/trace_printk.c b/trunk/kernel/trace/trace_printk.c index dff763b7baf1..2547d8813cf0 100644 --- a/trunk/kernel/trace/trace_printk.c +++ b/trunk/kernel/trace/trace_printk.c @@ -32,7 +32,7 @@ static DEFINE_MUTEX(btrace_mutex); struct trace_bprintk_fmt { struct list_head list; - const char *fmt; + char fmt[0]; }; static inline struct trace_bprintk_fmt *lookup_format(const char *fmt) @@ -49,7 +49,6 @@ static void hold_module_trace_bprintk_format(const char **start, const char **end) { const char **iter; - char *fmt; mutex_lock(&btrace_mutex); for (iter = start; iter < end; iter++) { @@ -59,18 +58,14 @@ void hold_module_trace_bprintk_format(const char **start, const char **end) continue; } - tb_fmt = kmalloc(sizeof(*tb_fmt), GFP_KERNEL); - if (tb_fmt) - fmt = kmalloc(strlen(*iter) + 1, GFP_KERNEL); - if (tb_fmt && fmt) { + tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt) + + strlen(*iter) + 1, GFP_KERNEL); + if (tb_fmt) { list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list); - strcpy(fmt, *iter); - tb_fmt->fmt = fmt; + strcpy(tb_fmt->fmt, *iter); *iter = tb_fmt->fmt; - } else { - kfree(tb_fmt); + } else *iter = NULL; - } } mutex_unlock(&btrace_mutex); } @@ -89,76 +84,6 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self, return 0; } -/* - * The debugfs/tracing/printk_formats file maps the addresses with - * the ASCII formats that are used in the bprintk events in the - * buffer. For userspace tools to be able to decode the events from - * the buffer, they need to be able to map the address with the format. - * - * The addresses of the bprintk formats are in their own section - * __trace_printk_fmt. But for modules we copy them into a link list. - * The code to print the formats and their addresses passes around the - * address of the fmt string. If the fmt address passed into the seq - * functions is within the kernel core __trace_printk_fmt section, then - * it simply uses the next pointer in the list. - * - * When the fmt pointer is outside the kernel core __trace_printk_fmt - * section, then we need to read the link list pointers. The trick is - * we pass the address of the string to the seq function just like - * we do for the kernel core formats. To get back the structure that - * holds the format, we simply use containerof() and then go to the - * next format in the list. - */ -static const char ** -find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos) -{ - struct trace_bprintk_fmt *mod_fmt; - - if (list_empty(&trace_bprintk_fmt_list)) - return NULL; - - /* - * v will point to the address of the fmt record from t_next - * v will be NULL from t_start. - * If this is the first pointer or called from start - * then we need to walk the list. - */ - if (!v || start_index == *pos) { - struct trace_bprintk_fmt *p; - - /* search the module list */ - list_for_each_entry(p, &trace_bprintk_fmt_list, list) { - if (start_index == *pos) - return &p->fmt; - start_index++; - } - /* pos > index */ - return NULL; - } - - /* - * v points to the address of the fmt field in the mod list - * structure that holds the module print format. - */ - mod_fmt = container_of(v, typeof(*mod_fmt), fmt); - if (mod_fmt->list.next == &trace_bprintk_fmt_list) - return NULL; - - mod_fmt = container_of(mod_fmt->list.next, typeof(*mod_fmt), list); - - return &mod_fmt->fmt; -} - -static void format_mod_start(void) -{ - mutex_lock(&btrace_mutex); -} - -static void format_mod_stop(void) -{ - mutex_unlock(&btrace_mutex); -} - #else /* !CONFIG_MODULES */ __init static int module_trace_bprintk_format_notify(struct notifier_block *self, @@ -166,13 +91,6 @@ module_trace_bprintk_format_notify(struct notifier_block *self, { return 0; } -static inline const char ** -find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos) -{ - return NULL; -} -static inline void format_mod_start(void) { } -static inline void format_mod_stop(void) { } #endif /* CONFIG_MODULES */ @@ -235,33 +153,20 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap) } EXPORT_SYMBOL_GPL(__ftrace_vprintk); -static const char **find_next(void *v, loff_t *pos) -{ - const char **fmt = v; - int start_index; - - if (!fmt) - fmt = __start___trace_bprintk_fmt + *pos; - - start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt; - - if (*pos < start_index) - return fmt; - - return find_next_mod_format(start_index, v, fmt, pos); -} - static void * t_start(struct seq_file *m, loff_t *pos) { - format_mod_start(); - return find_next(NULL, pos); + const char **fmt = __start___trace_bprintk_fmt + *pos; + + if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt) + return NULL; + return fmt; } static void *t_next(struct seq_file *m, void * v, loff_t *pos) { (*pos)++; - return find_next(v, pos); + return t_start(m, pos); } static int t_show(struct seq_file *m, void *v) @@ -300,7 +205,6 @@ static int t_show(struct seq_file *m, void *v) static void t_stop(struct seq_file *m, void *p) { - format_mod_stop(); } static const struct seq_operations show_format_seq_ops = { diff --git a/trunk/kernel/tracepoint.c b/trunk/kernel/tracepoint.c index b219f1449c54..68187af4889e 100644 --- a/trunk/kernel/tracepoint.c +++ b/trunk/kernel/tracepoint.c @@ -251,9 +251,9 @@ static void set_tracepoint(struct tracepoint_entry **entry, { WARN_ON(strcmp((*entry)->name, elem->name) != 0); - if (elem->regfunc && !jump_label_enabled(&elem->key) && active) + if (elem->regfunc && !elem->state && active) elem->regfunc(); - else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active) + else if (elem->unregfunc && elem->state && !active) elem->unregfunc(); /* @@ -264,10 +264,13 @@ static void set_tracepoint(struct tracepoint_entry **entry, * is used. */ rcu_assign_pointer(elem->funcs, (*entry)->funcs); - if (active && !jump_label_enabled(&elem->key)) - jump_label_inc(&elem->key); - else if (!active && jump_label_enabled(&elem->key)) - jump_label_dec(&elem->key); + if (!elem->state && active) { + jump_label_enable(&elem->state); + elem->state = active; + } else if (elem->state && !active) { + jump_label_disable(&elem->state); + elem->state = active; + } } /* @@ -278,11 +281,13 @@ static void set_tracepoint(struct tracepoint_entry **entry, */ static void disable_tracepoint(struct tracepoint *elem) { - if (elem->unregfunc && jump_label_enabled(&elem->key)) + if (elem->unregfunc && elem->state) elem->unregfunc(); - if (jump_label_enabled(&elem->key)) - jump_label_dec(&elem->key); + if (elem->state) { + jump_label_disable(&elem->state); + elem->state = 0; + } rcu_assign_pointer(elem->funcs, NULL); }