From b993e02ac869b3f571800dcda21c5c8495891ba1 Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Fri, 18 Dec 2009 16:12:56 +0000 Subject: [PATCH] --- yaml --- r: 182527 b: refs/heads/master c: 1b1d9258181bae199dc940f4bd0298126b9a73d9 h: refs/heads/master i: 182525: 95b357bf9c50e00ecaa4d54aec6742d0b6bb3b6c 182523: f3a1a76957b38ded1cd993e7e23e59cb87633fe9 182519: 7be69615b4e6c721bdeeadfede771546ef4d44d8 182511: ce7ff1d7b70989e4914f6f25aca6f3644a1ee26a 182495: a6dc8dadbbc5edd5028d5be57dc24f34a0199d55 182463: 1ed8cb27c9329a3643352acaa63752e66aee54f6 182399: c01428dffdb69ffd3584cc653409c6275fcb2173 182271: f85754414df4c22b85b3b5c1259241c9a270428f v: v3 --- [refs] | 2 +- trunk/arch/x86/include/asm/alternative.h | 7 ++++++- trunk/arch/x86/include/asm/uaccess_64.h | 21 ++++++++++++++++++++- trunk/arch/x86/kernel/alternative.c | 4 +++- trunk/arch/x86/kernel/x8664_ksyms_64.c | 3 ++- trunk/arch/x86/lib/copy_user_64.S | 6 ------ 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/[refs] b/[refs] index ba30d03c0db5..f6874f1877ea 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: 499a5f1efa0b0ac56ec5d060412aed84ae68e63e +refs/heads/master: 1b1d9258181bae199dc940f4bd0298126b9a73d9 diff --git a/trunk/arch/x86/include/asm/alternative.h b/trunk/arch/x86/include/asm/alternative.h index 69b74a7b877f..3b5b828767b6 100644 --- a/trunk/arch/x86/include/asm/alternative.h +++ b/trunk/arch/x86/include/asm/alternative.h @@ -125,11 +125,16 @@ static inline void alternatives_smp_switch(int smp) {} asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ : output : "i" (0), ## input) +/* Like alternative_io, but for replacing a direct call with another one. */ +#define alternative_call(oldfunc, newfunc, feature, output, input...) \ + asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ + : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) + /* * use this macro(s) if you need more than one output parameter * in alternative_io */ -#define ASM_OUTPUT2(a, b) a, b +#define ASM_OUTPUT2(a...) a struct paravirt_patch_site; #ifdef CONFIG_PARAVIRT diff --git a/trunk/arch/x86/include/asm/uaccess_64.h b/trunk/arch/x86/include/asm/uaccess_64.h index 46324c6a4f6e..a78c40305447 100644 --- a/trunk/arch/x86/include/asm/uaccess_64.h +++ b/trunk/arch/x86/include/asm/uaccess_64.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include /* @@ -16,7 +18,24 @@ /* Handles exceptions in both to and from, but doesn't do access_ok */ __must_check unsigned long -copy_user_generic(void *to, const void *from, unsigned len); +copy_user_generic_string(void *to, const void *from, unsigned len); +__must_check unsigned long +copy_user_generic_unrolled(void *to, const void *from, unsigned len); + +static __always_inline __must_check unsigned long +copy_user_generic(void *to, const void *from, unsigned len) +{ + unsigned ret; + + alternative_call(copy_user_generic_unrolled, + copy_user_generic_string, + X86_FEATURE_REP_GOOD, + ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from), + "=d" (len)), + "1" (to), "2" (from), "3" (len) + : "memory", "rcx", "r8", "r9", "r10", "r11"); + return ret; +} __must_check unsigned long _copy_to_user(void __user *to, const void *from, unsigned len); diff --git a/trunk/arch/x86/kernel/alternative.c b/trunk/arch/x86/kernel/alternative.c index de7353c0ce9c..2589ea4c60ce 100644 --- a/trunk/arch/x86/kernel/alternative.c +++ b/trunk/arch/x86/kernel/alternative.c @@ -205,7 +205,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) { struct alt_instr *a; - char insnbuf[MAX_PATCH_LEN]; + u8 insnbuf[MAX_PATCH_LEN]; DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); for (a = start; a < end; a++) { @@ -223,6 +223,8 @@ void __init_or_module apply_alternatives(struct alt_instr *start, } #endif memcpy(insnbuf, a->replacement, a->replacementlen); + if (*insnbuf == 0xe8 && a->replacementlen == 5) + *(s32 *)(insnbuf + 1) += a->replacement - a->instr; add_nops(insnbuf + a->replacementlen, a->instrlen - a->replacementlen); text_poke_early(instr, insnbuf, a->instrlen); diff --git a/trunk/arch/x86/kernel/x8664_ksyms_64.c b/trunk/arch/x86/kernel/x8664_ksyms_64.c index 619f7f88b8cc..693920b22496 100644 --- a/trunk/arch/x86/kernel/x8664_ksyms_64.c +++ b/trunk/arch/x86/kernel/x8664_ksyms_64.c @@ -26,7 +26,8 @@ EXPORT_SYMBOL(__put_user_2); EXPORT_SYMBOL(__put_user_4); EXPORT_SYMBOL(__put_user_8); -EXPORT_SYMBOL(copy_user_generic); +EXPORT_SYMBOL(copy_user_generic_string); +EXPORT_SYMBOL(copy_user_generic_unrolled); EXPORT_SYMBOL(__copy_user_nocache); EXPORT_SYMBOL(_copy_from_user); EXPORT_SYMBOL(_copy_to_user); diff --git a/trunk/arch/x86/lib/copy_user_64.S b/trunk/arch/x86/lib/copy_user_64.S index cf889d4e076a..71100c98e337 100644 --- a/trunk/arch/x86/lib/copy_user_64.S +++ b/trunk/arch/x86/lib/copy_user_64.S @@ -90,12 +90,6 @@ ENTRY(_copy_from_user) CFI_ENDPROC ENDPROC(_copy_from_user) -ENTRY(copy_user_generic) - CFI_STARTPROC - ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string - CFI_ENDPROC -ENDPROC(copy_user_generic) - .section .fixup,"ax" /* must zero dest */ ENTRY(bad_from_user)