Permalink
Switch branches/tags
aj/revert-sunrpc aj/shared-linux-fcntl allan/config-files andros/avx512f-mem andros/libmvec andros/pr19654 archlinux/2.18/master azanella/bz12683 carlos/dlmopen carlos/master cmetcalf/tile-2.15 cvs/fedora-2_3-branch cvs/fedora-2_5-branch cvs/fedora-branch cvs/glibc-2-1-branch cvs/glibc-2-2-branch cvs/glibc-2_0_x cvs/glibc-2_3-branch cvs/glibc-2_5-branch cvs/glibc-2_6-branch cvs/glibc-2_7-branch cvs/glibc-2_8-branch cvs/glibc-2_9-branch cvs/glibc-2_10-branch cvs/master cvs/sparc-2_0_x-branch cvs/thomas-posix1996 davem/sparc dj/malloc fedora/master fedora/2.10/master fedora/2.11/master fedora/2.12/master fedora/2.13/master fedora/2.14/master fedora/2.22/master fw/extend_alloca gentoo/2.18 gentoo/2.19 gentoo/2.20 gentoo/2.21 gentoo/2.22 gentoo/2.23 hjl/avx128 hjl/avx512f-mem/master hjl/avx512f-mem/old hjl/avx512f-mem/prefetcht1 hjl/cpuid/master hjl/cpuid/2.21 hjl/cpuid/2.22 hjl/ctor/release/2.11 hjl/ctor/release/2.12 hjl/erms/i386 hjl/erms/ifunc hjl/erms/master hjl/erms/nt hjl/erms/2.22 hjl/erms/2.23 hjl/fma/master hjl/global hjl/gold hjl/i386/master hjl/i486/multiarch-old hjl/i486/multiarch hjl/implies hjl/init hjl/ld.so/master hjl/memcpy/dpdk/master hjl/mempcpy hjl/plt/master hjl/plt/2.21 hjl/plt/2.22 hjl/pr13862 hjl/pr14370 hjl/pr14562/master hjl/pr14562/2.16 hjl/pr14654 hjl/pr14716 hjl/pr14831 hjl/pr14937 hjl/pr14941 hjl/pr14955 hjl/pr14995 hjl/pr17711/2.18 hjl/pr17711/2.19 hjl/pr17711/2.20 hjl/pr17711/2.21 hjl/pr17841/master hjl/pr17841/2.21 hjl/pr18078 hjl/pr18422 hjl/pr18661 hjl/pr18696 hjl/pr18858/master hjl/pr19122 hjl/pr19178/master hjl/pr19363/clobber hjl/pr19363/master hjl/pr19363/2.22 hjl/pr19371/master hjl/pr19463 hjl/pr19583 hjl/pr19590 hjl/pr19679/2.23 hjl/pr19776/master hjl/pthread/2.21 hjl/pthread/2.22 hjl/release/2.20/master hjl/secondary/master hjl/secondary/2.21 hjl/size/master hjl/tst-plt hjl/unaligned hjl/x32/master hjl/x32/release/2.12 hjl/x32/release/2.15 hjl/x86 hjl/32bit/master hjl/32bit/2.22 ibm/master ibm/2.8/master ibm/2.10/master ibm/2.11/master ibm/2.12/master ibm/2.13/master ibm/2.16/master ibm/2.18/master ibm/2.19/master ibm/2.20/master ibm/2.22/master ldmitrie/intel_mpx linaro/master lxoliva/getaddrinfo master neleai/string-x64 neleai/strlen origin/ibm/2.8/master pasky/fixes-overdue pasky/fixes release/2.10/master release/2.11/master release/2.12/master release/2.13/master release/2.14/master release/2.15/master release/2.16/master release/2.17/master release/2.18/master release/2.19/master release/2.20/master release/2.21/master release/2.22/master release/2.23/master roland/Wshadow roland/add-on-abi-tags roland/backtrace-syms roland/cancelhandling roland/disable-nis roland/getpid roland/gold-vs-libc roland/hwcap_mask roland/manual-check roland/nacl-debug-hack roland/nacl-exit-stacks roland/nacl-port/master roland/nptl_db roland/pthread_attr_getstack roland/stat64 roland/sysconf-clocks roland/tempname roland/x86_64-crt1-cfi rsa/hwcap2_v3 rsa/hwcap2_v4 rsa/hwcap2_v5 rsa/hwcap2_v6 rsa/power8_partial rsa/power8 rsa/stdint_headers rsa/stdint_noheaders rsa/stdint rsa/2.17_backports_v2 rsa/2.17_backports_v3 rsa/2.17_backports rth/aa-memset rth/aa-opt rth/execl rth/tramp siddhesh/is_in_module siddhesh/mmap-fallback siddhesh/posix-wundef siddhesh/sem_timedwait siddhesh/tunables tuliom/multilib
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
375 lines (348 sloc) 6.86 KB
/* strstr with unaligned loads
Copyright (C) 2009-2016 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
ENTRY(__strstr_sse2_unaligned)
movzbl (%rsi), %eax
testb %al, %al
je L(empty)
movzbl 1(%rsi), %edx
testb %dl, %dl
je L(strchr)
movd %eax, %xmm1
movd %edx, %xmm2
movq %rdi, %rax
andl $4095, %eax
punpcklbw %xmm1, %xmm1
cmpq $4031, %rax
punpcklbw %xmm2, %xmm2
punpcklwd %xmm1, %xmm1
punpcklwd %xmm2, %xmm2
pshufd $0, %xmm1, %xmm1
pshufd $0, %xmm2, %xmm2
ja L(cross_page)
movdqu (%rdi), %xmm3
pxor %xmm5, %xmm5
movdqu 1(%rdi), %xmm4
movdqa %xmm3, %xmm6
pcmpeqb %xmm1, %xmm3
pcmpeqb %xmm2, %xmm4
movdqu 16(%rdi), %xmm0
pcmpeqb %xmm5, %xmm6
pminub %xmm4, %xmm3
movdqa %xmm3, %xmm4
movdqu 17(%rdi), %xmm3
pcmpeqb %xmm0, %xmm5
pcmpeqb %xmm2, %xmm3
por %xmm6, %xmm4
pcmpeqb %xmm1, %xmm0
pminub %xmm3, %xmm0
por %xmm5, %xmm0
pmovmskb %xmm4, %r8d
pmovmskb %xmm0, %eax
salq $16, %rax
orq %rax, %r8
je L(next_32_bytes)
L(next_pair_index):
bsf %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero1)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found1)
cmpb 2(%rax), %dl
jne L(next_pair)
xorl %edx, %edx
jmp L(pair_loop_start)
.p2align 4
L(strchr):
movzbl %al, %esi
jmp __strchr_sse2
.p2align 4
L(pair_loop):
addq $1, %rdx
cmpb 2(%rax,%rdx), %cl
jne L(next_pair)
L(pair_loop_start):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop)
L(found1):
ret
L(zero1):
xorl %eax, %eax
ret
.p2align 4
L(next_pair):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index)
.p2align 4
L(next_32_bytes):
movdqu 32(%rdi), %xmm3
pxor %xmm5, %xmm5
movdqu 33(%rdi), %xmm4
movdqa %xmm3, %xmm6
pcmpeqb %xmm1, %xmm3
pcmpeqb %xmm2, %xmm4
movdqu 48(%rdi), %xmm0
pcmpeqb %xmm5, %xmm6
pminub %xmm4, %xmm3
movdqa %xmm3, %xmm4
movdqu 49(%rdi), %xmm3
pcmpeqb %xmm0, %xmm5
pcmpeqb %xmm2, %xmm3
por %xmm6, %xmm4
pcmpeqb %xmm1, %xmm0
pminub %xmm3, %xmm0
por %xmm5, %xmm0
pmovmskb %xmm4, %eax
salq $32, %rax
pmovmskb %xmm0, %r8d
salq $48, %r8
orq %rax, %r8
je L(loop_header)
L(next_pair2_index):
bsfq %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero2)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found2)
cmpb 2(%rax), %dl
jne L(next_pair2)
xorl %edx, %edx
jmp L(pair_loop2_start)
.p2align 4
L(pair_loop2):
addq $1, %rdx
cmpb 2(%rax,%rdx), %cl
jne L(next_pair2)
L(pair_loop2_start):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop2)
L(found2):
ret
L(zero2):
xorl %eax, %eax
ret
L(empty):
mov %rdi, %rax
ret
.p2align 4
L(next_pair2):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair2_index)
L(loop_header):
movq $-512, %r11
movq %rdi, %r9
pxor %xmm7, %xmm7
andq $-64, %rdi
.p2align 4
L(loop):
movdqa 64(%rdi), %xmm3
movdqu 63(%rdi), %xmm6
movdqa %xmm3, %xmm0
pxor %xmm2, %xmm3
pxor %xmm1, %xmm6
movdqa 80(%rdi), %xmm10
por %xmm3, %xmm6
pminub %xmm10, %xmm0
movdqu 79(%rdi), %xmm3
pxor %xmm2, %xmm10
pxor %xmm1, %xmm3
movdqa 96(%rdi), %xmm9
por %xmm10, %xmm3
pminub %xmm9, %xmm0
pxor %xmm2, %xmm9
movdqa 112(%rdi), %xmm8
addq $64, %rdi
pminub %xmm6, %xmm3
movdqu 31(%rdi), %xmm4
pminub %xmm8, %xmm0
pxor %xmm2, %xmm8
pxor %xmm1, %xmm4
por %xmm9, %xmm4
pminub %xmm4, %xmm3
movdqu 47(%rdi), %xmm5
pxor %xmm1, %xmm5
por %xmm8, %xmm5
pminub %xmm5, %xmm3
pminub %xmm3, %xmm0
pcmpeqb %xmm7, %xmm0
pmovmskb %xmm0, %eax
testl %eax, %eax
je L(loop)
pminub (%rdi), %xmm6
pminub 32(%rdi),%xmm4
pminub 48(%rdi),%xmm5
pcmpeqb %xmm7, %xmm6
pcmpeqb %xmm7, %xmm5
pmovmskb %xmm6, %edx
movdqa 16(%rdi), %xmm8
pcmpeqb %xmm7, %xmm4
movdqu 15(%rdi), %xmm0
pmovmskb %xmm5, %r8d
movdqa %xmm8, %xmm3
pmovmskb %xmm4, %ecx
pcmpeqb %xmm1,%xmm0
pcmpeqb %xmm2,%xmm3
salq $32, %rcx
pcmpeqb %xmm7,%xmm8
salq $48, %r8
pminub %xmm0,%xmm3
orq %rcx, %rdx
por %xmm3,%xmm8
orq %rdx, %r8
pmovmskb %xmm8, %eax
salq $16, %rax
orq %rax, %r8
je L(loop)
L(next_pair_index3):
bsfq %r8, %rcx
addq %rdi, %rcx
cmpb $0, (%rcx)
je L(zero)
xorl %eax, %eax
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(success3)
cmpb 1(%rcx), %dl
jne L(next_pair3)
jmp L(pair_loop_start3)
.p2align 4
L(pair_loop3):
addq $1, %rax
cmpb 1(%rcx,%rax), %dl
jne L(next_pair3)
L(pair_loop_start3):
movzbl 3(%rsi,%rax), %edx
testb %dl, %dl
jne L(pair_loop3)
L(success3):
lea -1(%rcx), %rax
ret
.p2align 4
L(next_pair3):
addq %rax, %r11
movq %rdi, %rax
subq %r9, %rax
cmpq %r11, %rax
jl L(switch_strstr)
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index3)
jmp L(loop)
.p2align 4
L(switch_strstr):
movq %rdi, %rdi
jmp __strstr_sse2
.p2align 4
L(cross_page):
movq %rdi, %rax
pxor %xmm0, %xmm0
andq $-64, %rax
movdqa (%rax), %xmm3
movdqu -1(%rax), %xmm4
movdqa %xmm3, %xmm8
movdqa 16(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm0, %xmm8
pcmpeqb %xmm2, %xmm3
movdqa %xmm5, %xmm7
pminub %xmm4, %xmm3
movdqu 15(%rax), %xmm4
pcmpeqb %xmm0, %xmm7
por %xmm3, %xmm8
movdqa %xmm5, %xmm3
movdqa 32(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm2, %xmm3
movdqa %xmm5, %xmm6
pmovmskb %xmm8, %ecx
pminub %xmm4, %xmm3
movdqu 31(%rax), %xmm4
por %xmm3, %xmm7
movdqa %xmm5, %xmm3
pcmpeqb %xmm0, %xmm6
movdqa 48(%rax), %xmm5
pcmpeqb %xmm1, %xmm4
pmovmskb %xmm7, %r8d
pcmpeqb %xmm2, %xmm3
pcmpeqb %xmm5, %xmm0
pminub %xmm4, %xmm3
movdqu 47(%rax), %xmm4
por %xmm3, %xmm6
movdqa %xmm5, %xmm3
salq $16, %r8
pcmpeqb %xmm1, %xmm4
pcmpeqb %xmm2, %xmm3
pmovmskb %xmm6, %r10d
pminub %xmm4, %xmm3
por %xmm3, %xmm0
salq $32, %r10
orq %r10, %r8
orq %rcx, %r8
movl %edi, %ecx
pmovmskb %xmm0, %edx
subl %eax, %ecx
salq $48, %rdx
orq %rdx, %r8
shrq %cl, %r8
je L(loop_header)
L(next_pair_index4):
bsfq %r8, %rax
addq %rdi, %rax
cmpb $0, (%rax)
je L(zero)
cmpq %rax,%rdi
je L(next_pair4)
movzbl 2(%rsi), %edx
testb %dl, %dl
je L(found3)
cmpb 1(%rax), %dl
jne L(next_pair4)
xorl %edx, %edx
jmp L(pair_loop_start4)
.p2align 4
L(pair_loop4):
addq $1, %rdx
cmpb 1(%rax,%rdx), %cl
jne L(next_pair4)
L(pair_loop_start4):
movzbl 3(%rsi,%rdx), %ecx
testb %cl, %cl
jne L(pair_loop4)
L(found3):
subq $1, %rax
ret
.p2align 4
L(next_pair4):
leaq -1(%r8), %rax
andq %rax, %r8
jne L(next_pair_index4)
jmp L(loop_header)
.p2align 4
L(found):
rep
ret
.p2align 4
L(zero):
xorl %eax, %eax
ret
END(__strstr_sse2_unaligned)