diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
index 9b0e58f9f970c..b7d0bd6b14063 100644
--- a/arch/sparc/lib/GENcopy_from_user.S
+++ b/arch/sparc/lib/GENcopy_from_user.S
@@ -23,7 +23,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
index ce79b495c65d0..780550e1afc74 100644
--- a/arch/sparc/lib/GENcopy_to_user.S
+++ b/arch/sparc/lib/GENcopy_to_user.S
@@ -27,7 +27,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
index 01ac546a86f4f..119ccb9a54f49 100644
--- a/arch/sparc/lib/NG2copy_from_user.S
+++ b/arch/sparc/lib/NG2copy_from_user.S
@@ -28,7 +28,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
index c477e1594a72e..7fe1ccefd9d05 100644
--- a/arch/sparc/lib/NG2copy_to_user.S
+++ b/arch/sparc/lib/NG2copy_to_user.S
@@ -37,7 +37,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
index 39bb8912c6e7d..5d1e4d1ac21ed 100644
--- a/arch/sparc/lib/NGcopy_from_user.S
+++ b/arch/sparc/lib/NGcopy_from_user.S
@@ -25,7 +25,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
index bf62fc1a26c72..ff630dcb273c9 100644
--- a/arch/sparc/lib/NGcopy_to_user.S
+++ b/arch/sparc/lib/NGcopy_to_user.S
@@ -28,7 +28,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop
#endif
diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
index f14efdd6d4a93..a6ae2ea04bf5d 100644
--- a/arch/sparc/lib/U1copy_from_user.S
+++ b/arch/sparc/lib/U1copy_from_user.S
@@ -23,7 +23,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop; \
#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
index 59d531b4a1a5f..f4b970eeb485f 100644
--- a/arch/sparc/lib/U1copy_to_user.S
+++ b/arch/sparc/lib/U1copy_to_user.S
@@ -23,7 +23,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop; \
#include "U1memcpy.S"
diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
index b4cbfe5a97baf..ef1e493afdfa9 100644
--- a/arch/sparc/lib/U3copy_to_user.S
+++ b/arch/sparc/lib/U3copy_to_user.S
@@ -23,7 +23,7 @@
#define PREAMBLE \
rd %asi, %g1; \
cmp %g1, ASI_AIUS; \
- bne,pn %icc, memcpy_user_stub; \
+ bne,pn %icc, ___copy_in_user; \
nop; \
#include "U3memcpy.S"
diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
index 884f46499d4e0..302c0e60dc2ce 100644
--- a/arch/sparc/lib/copy_in_user.S
+++ b/arch/sparc/lib/copy_in_user.S
@@ -3,6 +3,7 @@
* Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
*/
+#include <linux/linkage.h>
#include <asm/asi.h>
#define XCC xcc
@@ -27,18 +28,7 @@
* to copy register windows around during thread cloning.
*/
- .globl ___copy_in_user
- .type ___copy_in_user,#function
-___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
- /* Writing to %asi is _expensive_ so we hardcode it.
- * Reading %asi to check for KERNEL_DS is comparatively
- * cheap.
- */
- rd %asi, %g1
- cmp %g1, ASI_AIUS
- bne,pn %icc, memcpy_user_stub
- nop
-
+ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
cmp %o2, 0
be,pn %XCC, 85f
or %o0, %o1, %o3
@@ -49,22 +39,24 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
/* 16 < len <= 64 */
andcc %o3, 0x7, %g0
bne,pn %XCC, 90f
- sub %o0, %o1, %o3
+ nop
andn %o2, 0x7, %o4
and %o2, 0x7, %o2
1: subcc %o4, 0x8, %o4
EX(ldxa [%o1] %asi, %o5)
- EX(stxa %o5, [%o1 + %o3] ASI_AIUS)
+ EX(stxa %o5, [%o0] %asi)
+ add %o1, 0x8, %o1
bgu,pt %XCC, 1b
- add %o1, 0x8, %o1
+ add %o0, 0x8, %o0
andcc %o2, 0x4, %g0
be,pt %XCC, 1f
nop
sub %o2, 0x4, %o2
EX(lduwa [%o1] %asi, %o5)
- EX(stwa %o5, [%o1 + %o3] ASI_AIUS)
+ EX(stwa %o5, [%o0] %asi)
add %o1, 0x4, %o1
+ add %o0, 0x4, %o0
1: cmp %o2, 0
be,pt %XCC, 85f
nop
@@ -74,14 +66,15 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
80: /* 0 < len <= 16 */
andcc %o3, 0x3, %g0
bne,pn %XCC, 90f
- sub %o0, %o1, %o3
+ nop
82:
subcc %o2, 4, %o2
EX(lduwa [%o1] %asi, %g1)
- EX(stwa %g1, [%o1 + %o3] ASI_AIUS)
+ EX(stwa %g1, [%o0] %asi)
+ add %o1, 4, %o1
bgu,pt %XCC, 82b
- add %o1, 4, %o1
+ add %o0, 4, %o0
85: retl
clr %o0
@@ -90,26 +83,10 @@ ___copy_in_user: /* %o0=dst, %o1=src, %o2=len */
90:
subcc %o2, 1, %o2
EX(lduba [%o1] %asi, %g1)
- EX(stba %g1, [%o1 + %o3] ASI_AIUS)
+ EX(stba %g1, [%o0] %asi)
+ add %o1, 1, %o1
bgu,pt %XCC, 90b
- add %o1, 1, %o1
+ add %o0, 1, %o0
retl
clr %o0
-
- .size ___copy_in_user, .-___copy_in_user
-
- /* Act like copy_{to,in}_user(), ie. return zero instead
- * of original destination pointer. This is invoked when
- * copy_{to,in}_user() finds that %asi is kernel space.
- */
- .globl memcpy_user_stub
- .type memcpy_user_stub,#function
-memcpy_user_stub:
- save %sp, -192, %sp
- mov %i0, %o0
- mov %i1, %o1
- call memcpy
- mov %i2, %o2
- ret
- restore %g0, %g0, %o0
- .size memcpy_user_stub, .-memcpy_user_stub
+ENDPROC(___copy_in_user)