diff --git a/[refs] b/[refs] index fe8ed3f0e612..dc29702a10c0 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: 39ec58f3fea47c242724109cc1da999f74810bbc +refs/heads/master: cb9dc92c0a1b76165c8c334402e27191084b2047 diff --git a/trunk/arch/arm/lib/uaccess_with_memcpy.c b/trunk/arch/arm/lib/uaccess_with_memcpy.c index bf987b4a2571..92838e79654d 100644 --- a/trunk/arch/arm/lib/uaccess_with_memcpy.c +++ b/trunk/arch/arm/lib/uaccess_with_memcpy.c @@ -49,14 +49,11 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) return 1; } -unsigned long -__copy_to_user(void __user *to, const void *from, unsigned long n) +static unsigned long noinline +__copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) { int atomic; - if (n < 1024) - return __copy_to_user_std(to, from, n); - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { memcpy((void *)to, from, n); return 0; @@ -99,11 +96,24 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) return n; } -unsigned long __clear_user(void __user *addr, unsigned long n) +unsigned long +__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + /* + * This test is stubbed out of the main function above to keep + * the overhead for small copies low by avoiding a large + * register dump on the stack just to reload them right away. + * With frame pointer disabled, tail call optimization kicks in + * as well making this test almost invisible. + */ + if (n < 1024) + return __copy_to_user_std(to, from, n); + return __copy_to_user_memcpy(to, from, n); +} + +static unsigned long noinline +__clear_user_memset(void __user *addr, unsigned long n) { - if (n < 256) - return __clear_user_std(addr, n); - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { memset((void *)addr, 0, n); return 0; @@ -137,3 +147,11 @@ unsigned long __clear_user(void __user *addr, unsigned long n) out: return n; } + +unsigned long __clear_user(void __user *addr, unsigned long n) +{ + /* See rational for this in __copy_to_user() above. */ + if (n < 256) + return __clear_user_std(addr, n); + return __clear_user_memset(addr, n); +}