[thirdparty/glibc.git] / sysdeps / x86_64 / memchr.S

/* Copyright (C) 2011-2019 Free Software Foundation, Inc.
   Contributed by Intel Corporation.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>

#ifdef USE_AS_WMEMCHR
# define MEMCHR		wmemchr
# define PCMPEQ		pcmpeqd
#else
# define MEMCHR		memchr
# define PCMPEQ		pcmpeqb
#endif

/* fast SSE2 version with using pmaxub and 64 byte loop */

	.text
ENTRY(MEMCHR)
	movd	%esi, %xmm1
	mov	%edi, %ecx

#ifdef USE_AS_WMEMCHR
	test	%rdx, %rdx
	jz	L(return_null)
	shl	$2, %rdx
#else
	punpcklbw %xmm1, %xmm1
	test	%rdx, %rdx
	jz	L(return_null)
	punpcklbw %xmm1, %xmm1
#endif

	and	$63, %ecx
	pshufd	$0, %xmm1, %xmm1

	cmp	$48, %ecx
	ja	L(crosscache)

	movdqu	(%rdi), %xmm0
	PCMPEQ	%xmm1, %xmm0
	pmovmskb %xmm0, %eax
	test	%eax, %eax

	jnz	L(matches_1)
	sub	$16, %rdx
	jbe	L(return_null)
	add	$16, %rdi
	and	$15, %ecx
	and	$-16, %rdi
	add	%rcx, %rdx
	sub	$64, %rdx
	jbe	L(exit_loop)
	jmp	L(loop_prolog)

	.p2align 4
L(crosscache):
	and	$15, %ecx
	and	$-16, %rdi
	movdqa	(%rdi), %xmm0

	PCMPEQ	%xmm1, %xmm0
/* Check if there is a match.  */
	pmovmskb %xmm0, %eax
/* Remove the leading bytes.  */
	sar	%cl, %eax
	test	%eax, %eax
	je	L(unaligned_no_match)
/* Check which byte is a match.  */
	bsf	%eax, %eax

	sub	%rax, %rdx
	jbe	L(return_null)
	add	%rdi, %rax
	add	%rcx, %rax
	ret

	.p2align 4
L(unaligned_no_match):
        /* "rcx" is less than 16.  Calculate "rdx + rcx - 16" by using
	   "rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void
	   possible addition overflow.  */
	neg	%rcx
	add	$16, %rcx
	sub	%rcx, %rdx
	jbe	L(return_null)
	add	$16, %rdi
	sub	$64, %rdx
	jbe	L(exit_loop)

	.p2align 4
L(loop_prolog):
	movdqa	(%rdi), %xmm0
	PCMPEQ	%xmm1, %xmm0
	pmovmskb %xmm0, %eax
	test	%eax, %eax
	jnz	L(matches)

	movdqa	16(%rdi), %xmm2
	PCMPEQ	%xmm1, %xmm2
	pmovmskb %xmm2, %eax
	test	%eax, %eax
	jnz	L(matches16)

	movdqa	32(%rdi), %xmm3
	PCMPEQ	%xmm1, %xmm3
	pmovmskb %xmm3, %eax
	test	%eax, %eax
	jnz	L(matches32)

	movdqa	48(%rdi), %xmm4
	PCMPEQ	%xmm1, %xmm4
	add	$64, %rdi
	pmovmskb %xmm4, %eax
	test	%eax, %eax
	jnz	L(matches0)

	test	$0x3f, %rdi
	jz	L(align64_loop)

	sub	$64, %rdx
	jbe	L(exit_loop)

	movdqa	(%rdi), %xmm0
	PCMPEQ	%xmm1, %xmm0
	pmovmskb %xmm0, %eax
	test	%eax, %eax
	jnz	L(matches)

	movdqa	16(%rdi), %xmm2
	PCMPEQ	%xmm1, %xmm2
	pmovmskb %xmm2, %eax
	test	%eax, %eax
	jnz	L(matches16)

	movdqa	32(%rdi), %xmm3
	PCMPEQ	%xmm1, %xmm3
	pmovmskb %xmm3, %eax
	test	%eax, %eax
	jnz	L(matches32)

	movdqa	48(%rdi), %xmm3
	PCMPEQ	%xmm1, %xmm3
	pmovmskb %xmm3, %eax

	add	$64, %rdi
	test	%eax, %eax
	jnz	L(matches0)

	mov	%rdi, %rcx
	and	$-64, %rdi
	and	$63, %ecx
	add	%rcx, %rdx

	.p2align 4
L(align64_loop):
	sub	$64, %rdx
	jbe	L(exit_loop)
	movdqa	(%rdi), %xmm0
	movdqa	16(%rdi), %xmm2
	movdqa	32(%rdi), %xmm3
	movdqa	48(%rdi), %xmm4

	PCMPEQ	%xmm1, %xmm0
	PCMPEQ	%xmm1, %xmm2
	PCMPEQ	%xmm1, %xmm3
	PCMPEQ	%xmm1, %xmm4

	pmaxub	%xmm0, %xmm3
	pmaxub	%xmm2, %xmm4
	pmaxub	%xmm3, %xmm4
	pmovmskb %xmm4, %eax

	add	$64, %rdi

	test	%eax, %eax
	jz	L(align64_loop)

	sub	$64, %rdi

	pmovmskb %xmm0, %eax
	test	%eax, %eax
	jnz	L(matches)

	pmovmskb %xmm2, %eax
	test	%eax, %eax
	jnz	L(matches16)

	movdqa	32(%rdi), %xmm3
	PCMPEQ	%xmm1, %xmm3

	PCMPEQ	48(%rdi), %xmm1
	pmovmskb %xmm3, %eax
	test	%eax, %eax
	jnz	L(matches32)

	pmovmskb %xmm1, %eax
	bsf	%eax, %eax
	lea	48(%rdi, %rax), %rax
	ret

	.p2align 4
L(exit_loop):
	add	$32, %edx
	jle	L(exit_loop_32)

	movdqa	(%rdi), %xmm0
	PCMPEQ	%xmm1, %xmm0
	pmovmskb %xmm0, %eax
	test	%eax, %eax
	jnz	L(matches)

	movdqa	16(%rdi), %xmm2
	PCMPEQ	%xmm1, %xmm2
	pmovmskb %xmm2, %eax
	test	%eax, %eax
	jnz	L(matches16)

	movdqa	32(%rdi), %xmm3
	PCMPEQ	%xmm1, %xmm3
	pmovmskb %xmm3, %eax
	test	%eax, %eax
	jnz	L(matches32_1)
	sub	$16, %edx
	jle	L(return_null)

	PCMPEQ	48(%rdi), %xmm1
	pmovmskb %xmm1, %eax
	test	%eax, %eax
	jnz	L(matches48_1)
	xor	%eax, %eax
	ret

	.p2align 4
L(exit_loop_32):
	add	$32, %edx
	movdqa	(%rdi), %xmm0
	PCMPEQ	%xmm1, %xmm0
	pmovmskb %xmm0, %eax
	test	%eax, %eax
	jnz	L(matches_1)
	sub	$16, %edx
	jbe	L(return_null)

	PCMPEQ	16(%rdi), %xmm1
	pmovmskb %xmm1, %eax
	test	%eax, %eax
	jnz	L(matches16_1)
	xor	%eax, %eax
	ret

	.p2align 4
L(matches0):
	bsf	%eax, %eax
	lea	-16(%rax, %rdi), %rax
	ret

	.p2align 4
L(matches):
	bsf	%eax, %eax
	add	%rdi, %rax
	ret

	.p2align 4
L(matches16):
	bsf	%eax, %eax
	lea	16(%rax, %rdi), %rax
	ret

	.p2align 4
L(matches32):
	bsf	%eax, %eax
	lea	32(%rax, %rdi), %rax
	ret

	.p2align 4
L(matches_1):
	bsf	%eax, %eax
	sub	%rax, %rdx
	jbe	L(return_null)
	add	%rdi, %rax
	ret

	.p2align 4
L(matches16_1):
	bsf	%eax, %eax
	sub	%rax, %rdx
	jbe	L(return_null)
	lea	16(%rdi, %rax), %rax
	ret

	.p2align 4
L(matches32_1):
	bsf	%eax, %eax
	sub	%rax, %rdx
	jbe	L(return_null)
	lea	32(%rdi, %rax), %rax
	ret

	.p2align 4
L(matches48_1):
	bsf	%eax, %eax
	sub	%rax, %rdx
	jbe	L(return_null)
	lea	48(%rdi, %rax), %rax
	ret

	.p2align 4
L(return_null):
	xor	%eax, %eax
	ret
END(MEMCHR)

#ifndef USE_AS_WMEMCHR
strong_alias (memchr, __memchr)
libc_hidden_builtin_def(memchr)
#endif
Commit	Line	Data
04277e02	1	/* Copyright (C) 2011-2019 Free Software Foundation, Inc.
093ecf92	2	Contributed by Intel Corporation.
322e23db UD	3	This file is part of the GNU C Library.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
322e23db UD	18
	19	#include <sysdep.h>
	20
2f5d20ac L	21	#ifdef USE_AS_WMEMCHR
	22	# define MEMCHR wmemchr
	23	# define PCMPEQ pcmpeqd
	24	#else
	25	# define MEMCHR memchr
	26	# define PCMPEQ pcmpeqb
	27	#endif
	28
093ecf92	29	/* fast SSE2 version with using pmaxub and 64 byte loop */
322e23db UD	30
322e23db UD	31	.text
2f5d20ac	32	ENTRY(MEMCHR)
4f26ef1b L	33	movd %esi, %xmm1
4f26ef1b L	34	mov %edi, %ecx
093ecf92	35
2f5d20ac L	36	#ifdef USE_AS_WMEMCHR
	37	test %rdx, %rdx
	38	jz L(return_null)
	39	shl $2, %rdx
	40	#else
322e23db	41	punpcklbw %xmm1, %xmm1
093ecf92 LD	42	test %rdx, %rdx
093ecf92 LD	43	jz L(return_null)
322e23db	44	punpcklbw %xmm1, %xmm1
2f5d20ac	45	#endif
093ecf92	46
4f26ef1b	47	and $63, %ecx
322e23db	48	pshufd $0, %xmm1, %xmm1
093ecf92	49
4f26ef1b	50	cmp $48, %ecx
093ecf92 LD	51	ja L(crosscache)
	52
	53	movdqu (%rdi), %xmm0
2f5d20ac	54	PCMPEQ %xmm1, %xmm0
093ecf92 LD	55	pmovmskb %xmm0, %eax
	56	test %eax, %eax
	57
	58	jnz L(matches_1)
	59	sub $16, %rdx
	60	jbe L(return_null)
	61	add $16, %rdi
4f26ef1b	62	and $15, %ecx
093ecf92 LD	63	and $-16, %rdi
	64	add %rcx, %rdx
	65	sub $64, %rdx
	66	jbe L(exit_loop)
	67	jmp L(loop_prolog)
	68
	69	.p2align 4
	70	L(crosscache):
4f26ef1b	71	and $15, %ecx
093ecf92 LD	72	and $-16, %rdi
	73	movdqa (%rdi), %xmm0
	74
2f5d20ac	75	PCMPEQ %xmm1, %xmm0
093ecf92 LD	76	/* Check if there is a match. */
	77	pmovmskb %xmm0, %eax
	78	/* Remove the leading bytes. */
	79	sar %cl, %eax
	80	test %eax, %eax
	81	je L(unaligned_no_match)
	82	/* Check which byte is a match. */
	83	bsf %eax, %eax
	84
	85	sub %rax, %rdx
	86	jbe L(return_null)
	87	add %rdi, %rax
	88	add %rcx, %rax
	89	ret
	90
	91	.p2align 4
	92	L(unaligned_no_match):
402bf069 L	93	/* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using
	94	"rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void
	95	possible addition overflow. */
	96	neg %rcx
	97	add $16, %rcx
	98	sub %rcx, %rdx
093ecf92 LD	99	jbe L(return_null)
	100	add $16, %rdi
	101	sub $64, %rdx
	102	jbe L(exit_loop)
	103
	104	.p2align 4
	105	L(loop_prolog):
	106	movdqa (%rdi), %xmm0
2f5d20ac	107	PCMPEQ %xmm1, %xmm0
093ecf92 LD	108	pmovmskb %xmm0, %eax
	109	test %eax, %eax
	110	jnz L(matches)
	111
	112	movdqa 16(%rdi), %xmm2
2f5d20ac	113	PCMPEQ %xmm1, %xmm2
093ecf92 LD	114	pmovmskb %xmm2, %eax
	115	test %eax, %eax
	116	jnz L(matches16)
	117
	118	movdqa 32(%rdi), %xmm3
2f5d20ac	119	PCMPEQ %xmm1, %xmm3
093ecf92 LD	120	pmovmskb %xmm3, %eax
	121	test %eax, %eax
	122	jnz L(matches32)
	123
	124	movdqa 48(%rdi), %xmm4
2f5d20ac	125	PCMPEQ %xmm1, %xmm4
093ecf92 LD	126	add $64, %rdi
	127	pmovmskb %xmm4, %eax
	128	test %eax, %eax
	129	jnz L(matches0)
	130
	131	test $0x3f, %rdi
	132	jz L(align64_loop)
	133
	134	sub $64, %rdx
	135	jbe L(exit_loop)
	136
	137	movdqa (%rdi), %xmm0
2f5d20ac	138	PCMPEQ %xmm1, %xmm0
093ecf92 LD	139	pmovmskb %xmm0, %eax
	140	test %eax, %eax
	141	jnz L(matches)
	142
	143	movdqa 16(%rdi), %xmm2
2f5d20ac	144	PCMPEQ %xmm1, %xmm2
093ecf92 LD	145	pmovmskb %xmm2, %eax
	146	test %eax, %eax
	147	jnz L(matches16)
	148
	149	movdqa 32(%rdi), %xmm3
2f5d20ac	150	PCMPEQ %xmm1, %xmm3
093ecf92 LD	151	pmovmskb %xmm3, %eax
	152	test %eax, %eax
	153	jnz L(matches32)
	154
	155	movdqa 48(%rdi), %xmm3
2f5d20ac	156	PCMPEQ %xmm1, %xmm3
093ecf92 LD	157	pmovmskb %xmm3, %eax
	158
	159	add $64, %rdi
	160	test %eax, %eax
	161	jnz L(matches0)
	162
	163	mov %rdi, %rcx
	164	and $-64, %rdi
4f26ef1b	165	and $63, %ecx
093ecf92 LD	166	add %rcx, %rdx
	167
	168	.p2align 4
	169	L(align64_loop):
	170	sub $64, %rdx
	171	jbe L(exit_loop)
	172	movdqa (%rdi), %xmm0
	173	movdqa 16(%rdi), %xmm2
	174	movdqa 32(%rdi), %xmm3
	175	movdqa 48(%rdi), %xmm4
	176
2f5d20ac L	177	PCMPEQ %xmm1, %xmm0
	178	PCMPEQ %xmm1, %xmm2
	179	PCMPEQ %xmm1, %xmm3
	180	PCMPEQ %xmm1, %xmm4
093ecf92 LD	181
	182	pmaxub %xmm0, %xmm3
	183	pmaxub %xmm2, %xmm4
	184	pmaxub %xmm3, %xmm4
	185	pmovmskb %xmm4, %eax
	186
	187	add $64, %rdi
	188
	189	test %eax, %eax
	190	jz L(align64_loop)
	191
	192	sub $64, %rdi
	193
	194	pmovmskb %xmm0, %eax
	195	test %eax, %eax
	196	jnz L(matches)
	197
	198	pmovmskb %xmm2, %eax
	199	test %eax, %eax
	200	jnz L(matches16)
	201
	202	movdqa 32(%rdi), %xmm3
2f5d20ac	203	PCMPEQ %xmm1, %xmm3
093ecf92	204
2f5d20ac	205	PCMPEQ 48(%rdi), %xmm1
093ecf92 LD	206	pmovmskb %xmm3, %eax
	207	test %eax, %eax
	208	jnz L(matches32)
	209
	210	pmovmskb %xmm1, %eax
	211	bsf %eax, %eax
	212	lea 48(%rdi, %rax), %rax
	213	ret
	214
	215	.p2align 4
	216	L(exit_loop):
4f26ef1b	217	add $32, %edx
093ecf92 LD	218	jle L(exit_loop_32)
	219
	220	movdqa (%rdi), %xmm0
2f5d20ac	221	PCMPEQ %xmm1, %xmm0
093ecf92 LD	222	pmovmskb %xmm0, %eax
	223	test %eax, %eax
	224	jnz L(matches)
	225
	226	movdqa 16(%rdi), %xmm2
2f5d20ac	227	PCMPEQ %xmm1, %xmm2
093ecf92 LD	228	pmovmskb %xmm2, %eax
	229	test %eax, %eax
	230	jnz L(matches16)
	231
	232	movdqa 32(%rdi), %xmm3
2f5d20ac	233	PCMPEQ %xmm1, %xmm3
093ecf92 LD	234	pmovmskb %xmm3, %eax
	235	test %eax, %eax
	236	jnz L(matches32_1)
4f26ef1b	237	sub $16, %edx
093ecf92 LD	238	jle L(return_null)
093ecf92 LD	239
2f5d20ac	240	PCMPEQ 48(%rdi), %xmm1
093ecf92 LD	241	pmovmskb %xmm1, %eax
	242	test %eax, %eax
	243	jnz L(matches48_1)
4f26ef1b	244	xor %eax, %eax
093ecf92 LD	245	ret
	246
	247	.p2align 4
	248	L(exit_loop_32):
4f26ef1b	249	add $32, %edx
093ecf92	250	movdqa (%rdi), %xmm0
2f5d20ac	251	PCMPEQ %xmm1, %xmm0
093ecf92 LD	252	pmovmskb %xmm0, %eax
	253	test %eax, %eax
	254	jnz L(matches_1)
4f26ef1b	255	sub $16, %edx
093ecf92	256	jbe L(return_null)
322e23db	257
2f5d20ac	258	PCMPEQ 16(%rdi), %xmm1
093ecf92 LD	259	pmovmskb %xmm1, %eax
	260	test %eax, %eax
	261	jnz L(matches16_1)
4f26ef1b	262	xor %eax, %eax
322e23db UD	263	ret
322e23db UD	264
093ecf92 LD	265	.p2align 4
	266	L(matches0):
	267	bsf %eax, %eax
	268	lea -16(%rax, %rdi), %rax
322e23db	269	ret
093ecf92 LD	270
	271	.p2align 4
	272	L(matches):
	273	bsf %eax, %eax
	274	add %rdi, %rax
	275	ret
	276
	277	.p2align 4
	278	L(matches16):
	279	bsf %eax, %eax
	280	lea 16(%rax, %rdi), %rax
	281	ret
	282
	283	.p2align 4
	284	L(matches32):
	285	bsf %eax, %eax
	286	lea 32(%rax, %rdi), %rax
	287	ret
	288
	289	.p2align 4
	290	L(matches_1):
	291	bsf %eax, %eax
	292	sub %rax, %rdx
	293	jbe L(return_null)
	294	add %rdi, %rax
	295	ret
	296
	297	.p2align 4
	298	L(matches16_1):
	299	bsf %eax, %eax
	300	sub %rax, %rdx
	301	jbe L(return_null)
	302	lea 16(%rdi, %rax), %rax
	303	ret
	304
	305	.p2align 4
	306	L(matches32_1):
	307	bsf %eax, %eax
	308	sub %rax, %rdx
	309	jbe L(return_null)
	310	lea 32(%rdi, %rax), %rax
	311	ret
	312
	313	.p2align 4
	314	L(matches48_1):
	315	bsf %eax, %eax
	316	sub %rax, %rdx
	317	jbe L(return_null)
	318	lea 48(%rdi, %rax), %rax
	319	ret
	320
	321	.p2align 4
	322	L(return_null):
4f26ef1b	323	xor %eax, %eax
093ecf92	324	ret
2f5d20ac	325	END(MEMCHR)
322e23db	326
2f5d20ac	327	#ifndef USE_AS_WMEMCHR
322e23db	328	strong_alias (memchr, __memchr)
093ecf92	329	libc_hidden_builtin_def(memchr)
2f5d20ac	330	#endif