[thirdparty/glibc.git] / sysdeps / powerpc / powerpc32 / power7 / strchr.S

/* Optimized strchr implementation for PowerPC32/POWER7 using cmpb insn.
   Copyright (C) 2010-2014 Free Software Foundation, Inc.
   Contributed by Luis Machado <luisgpm@br.ibm.com>.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* int [r3] strchr (char *s [r3], int c [r4])  */
	.machine  power7
ENTRY (strchr)
	CALL_MCOUNT
	dcbt	0,r3
	clrrwi	r8,r3,2	      /* Align the address to word boundary.  */
	cmpwi	cr7,r4,0
	lwz	r12,0(r8)     /* Load word from memory.  */
	li	r0,0	      /* Word with null chars to use
				 with cmpb.  */

	rlwinm	r6,r3,3,27,28 /* Calculate padding.  */

	beq	cr7,L(null_match)

	/* Replicate byte to word.  */
	insrdi	r4,r4,8,48
	insrdi	r4,r4,16,32

	/* Now r4 has a word of c bytes and r0 has
	   a word of null bytes.  */

	cmpb	r10,r12,r4     /* Compare each byte against c byte.  */
	cmpb	r11,r12,r0     /* Compare each byte against null byte.  */

	/* Move the words left and right to discard the bits that are
	   not part of the string and to bring them back as zeros.  */
#ifdef __LITTLE_ENDIAN__
	srw	r10,r10,r6
	srw	r11,r11,r6
	slw	r10,r10,r6
	slw	r11,r11,r6
#else
	slw	r10,r10,r6
	slw	r11,r11,r6
	srw	r10,r10,r6
	srw	r11,r11,r6
#endif
	or	r5,r10,r11    /* OR the results to speed things up.  */
	cmpwi	cr7,r5,0      /* If r5 == 0, no c or null bytes
				 have been found.  */
	bne	cr7,L(done)

	mtcrf   0x01,r8

	/* Are we now aligned to a doubleword boundary?  If so, skip to
	   the main loop.  Otherwise, go through the alignment code.  */

	bt	29,L(loop)

	/* Handle WORD2 of pair.  */
	lwzu	r12,4(r8)
	cmpb	r10,r12,r4
	cmpb	r11,r12,r0
	or	r5,r10,r11
	cmpwi	cr7,r5,0
	bne	cr7,L(done)
	b	L(loop)	      /* We branch here (rather than falling through)
				 to skip the nops due to heavy alignment
				 of the loop below.  */

	.p2align  5
L(loop):
	/* Load two words, compare and merge in a
	   single register for speed.  This is an attempt
	   to speed up the null-checking process for bigger strings.  */
	lwz	r12,4(r8)
	lwzu	r9,8(r8)
	cmpb	r10,r12,r4
	cmpb	r11,r12,r0
	cmpb	r6,r9,r4
	cmpb	r7,r9,r0
	or	r12,r10,r11
	or	r9,r6,r7
	or	r5,r12,r9
	cmpwi	cr7,r5,0
	beq	cr7,L(loop)

	/* OK, one (or both) of the words contains a c/null byte.  Check
	   the first word and decrement the address in case the first
	   word really contains a c/null byte.  */

	cmpwi	cr6,r12,0
	addi	r8,r8,-4
	bne	cr6,L(done)

	/* The c/null byte must be in the second word.  Adjust the address
	   again and move the result of cmpb to r10/r11 so we can calculate
	   the pointer.  */

	mr	r10,r6
	mr	r11,r7
	addi	r8,r8,4

	/* r10/r11 have the output of the cmpb instructions, that is,
	   0xff in the same position as the c/null byte in the original
	   word from the string.  Use that to calculate the pointer.  */
L(done):
#ifdef __LITTLE_ENDIAN__
	addi    r3,r10,-1
	andc    r3,r3,r10
	popcntw	r0,r3
	addi    r4,r11,-1
	andc    r4,r4,r11
	cmplw	cr7,r3,r4
	bgt	cr7,L(no_match)
#else
	cntlzw	r0,r10	      /* Count leading zeros before c matches.  */
	cmplw	cr7,r11,r10
	bgt	cr7,L(no_match)
#endif
	srwi	r0,r0,3	      /* Convert leading zeros to bytes.  */
	add	r3,r8,r0      /* Return address of the matching c byte
				 or null in case c was not found.  */
	blr

	.align	4
L(no_match):
	li	r3,0
	blr

/* We are here because strchr was called with a null byte.  */
	.align	4
L(null_match):
	/* r0 has a word of null bytes.  */

	cmpb	r5,r12,r0     /* Compare each byte against null bytes.  */

	/* Move the words left and right to discard the bits that are
	   not part of the string and bring them back as zeros.  */
#ifdef __LITTLE_ENDIAN__
	srw	r5,r5,r6
	slw	r5,r5,r6
#else
	slw	r5,r5,r6
	srw	r5,r5,r6
#endif
	cmpwi	cr7,r5,0      /* If r10 == 0, no c or null bytes
				 have been found.  */
	bne	cr7,L(done_null)

	mtcrf   0x01,r8

	/* Are we now aligned to a doubleword boundary?  If so, skip to
	   the main loop.  Otherwise, go through the alignment code.  */

	bt	29,L(loop_null)

	/* Handle WORD2 of pair.  */
	lwzu	r12,4(r8)
	cmpb    r5,r12,r0
	cmpwi	cr7,r5,0
	bne	cr7,L(done_null)
	b	L(loop_null)  /* We branch here (rather than falling through)
				 to skip the nops due to heavy alignment
				 of the loop below.  */

	/* Main loop to look for the end of the string.  Since it's a
	   small loop (< 8 instructions), align it to 32-bytes.  */
	.p2align  5
L(loop_null):
	/* Load two words, compare and merge in a
	   single register for speed.  This is an attempt
	   to speed up the null-checking process for bigger strings.  */
	lwz	r12,4(r8)
	lwzu    r11,8(r8)
	cmpb	r5,r12,r0
	cmpb	r10,r11,r0
	or	r6,r5,r10
	cmpwi	cr7,r6,0
	beq	cr7,L(loop_null)

	/* OK, one (or both) of the words contains a null byte.  Check
	   the first word and decrement the address in case the first
	   word really contains a null byte.  */

	cmpwi	cr6,r5,0
	addi	r8,r8,-4
	bne	cr6,L(done_null)

	/* The null byte must be in the second word.  Adjust the address
	   again and move the result of cmpb to r10 so we can calculate the
	   pointer.  */

	mr	r5,r10
	addi	r8,r8,4

	/* r5 has the output of the cmpb instruction, that is, it contains
	   0xff in the same position as the null byte in the original
	   word from the string.  Use that to calculate the pointer.  */
L(done_null):
#ifdef __LITTLE_ENDIAN__
	addi    r0,r5,-1
	andc    r0,r0,r5
	popcntw	r0,r0
#else
	cntlzw	r0,r5	      /* Count leading zeros before the match.  */
#endif
	srwi	r0,r0,3	      /* Convert leading zeros to bytes.  */
	add	r3,r8,r0      /* Return address of the matching null byte.  */
	blr
END (strchr)
weak_alias (strchr, index)
libc_hidden_builtin_def (strchr)
Commit	Line	Data
fe2f79db	1	/* Optimized strchr implementation for PowerPC32/POWER7 using cmpb insn.
d4697bc9	2	Copyright (C) 2010-2014 Free Software Foundation, Inc.
fe2f79db LM	3	Contributed by Luis Machado <luisgpm@br.ibm.com>.
	4	This file is part of the GNU C Library.
	5
	6	The GNU C Library is free software; you can redistribute it and/or
	7	modify it under the terms of the GNU Lesser General Public
	8	License as published by the Free Software Foundation; either
	9	version 2.1 of the License, or (at your option) any later version.
	10
	11	The GNU C Library is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	Lesser General Public License for more details.
	15
	16	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	17	License along with the GNU C Library; if not, see
59ba27a6 PE	18	<http://www.gnu.org/licenses/>. */
fe2f79db LM	19
fe2f79db LM	20	#include <sysdep.h>
fe2f79db LM	21
	22	/* int [r3] strchr (char s [r3], int c [r4]) /
	23	.machine power7
b5510883	24	ENTRY (strchr)
fe2f79db LM	25	CALL_MCOUNT
	26	dcbt 0,r3
	27	clrrwi r8,r3,2 /* Align the address to word boundary. */
	28	cmpwi cr7,r4,0
	29	lwz r12,0(r8) /* Load word from memory. */
	30	li r0,0 /* Word with null chars to use
	31	with cmpb. */
	32
	33	rlwinm r6,r3,3,27,28 /* Calculate padding. */
	34
	35	beq cr7,L(null_match)
	36
	37	/* Replicate byte to word. */
664318c3 AM	38	insrdi r4,r4,8,48
664318c3 AM	39	insrdi r4,r4,16,32
fe2f79db LM	40
	41	/* Now r4 has a word of c bytes and r0 has
	42	a word of null bytes. */
	43
	44	cmpb r10,r12,r4 /* Compare each byte against c byte. */
	45	cmpb r11,r12,r0 /* Compare each byte against null byte. */
	46
	47	/* Move the words left and right to discard the bits that are
	48	not part of the string and to bring them back as zeros. */
664318c3 AM	49	#ifdef __LITTLE_ENDIAN__
	50	srw r10,r10,r6
	51	srw r11,r11,r6
	52	slw r10,r10,r6
	53	slw r11,r11,r6
	54	#else
fe2f79db LM	55	slw r10,r10,r6
	56	slw r11,r11,r6
	57	srw r10,r10,r6
	58	srw r11,r11,r6
664318c3	59	#endif
fe2f79db LM	60	or r5,r10,r11 /* OR the results to speed things up. */
	61	cmpwi cr7,r5,0 /* If r5 == 0, no c or null bytes
	62	have been found. */
	63	bne cr7,L(done)
	64
	65	mtcrf 0x01,r8
	66
	67	/* Are we now aligned to a doubleword boundary? If so, skip to
	68	the main loop. Otherwise, go through the alignment code. */
	69
	70	bt 29,L(loop)
	71
	72	/* Handle WORD2 of pair. */
	73	lwzu r12,4(r8)
664318c3	74	cmpb r10,r12,r4
fe2f79db LM	75	cmpb r11,r12,r0
	76	or r5,r10,r11
	77	cmpwi cr7,r5,0
	78	bne cr7,L(done)
	79	b L(loop) /* We branch here (rather than falling through)
	80	to skip the nops due to heavy alignment
	81	of the loop below. */
	82
	83	.p2align 5
	84	L(loop):
	85	/* Load two words, compare and merge in a
	86	single register for speed. This is an attempt
	87	to speed up the null-checking process for bigger strings. */
	88	lwz r12,4(r8)
	89	lwzu r9,8(r8)
	90	cmpb r10,r12,r4
	91	cmpb r11,r12,r0
	92	cmpb r6,r9,r4
	93	cmpb r7,r9,r0
	94	or r12,r10,r11
	95	or r9,r6,r7
	96	or r5,r12,r9
	97	cmpwi cr7,r5,0
	98	beq cr7,L(loop)
	99
	100	/* OK, one (or both) of the words contains a c/null byte. Check
	101	the first word and decrement the address in case the first
	102	word really contains a c/null byte. */
	103
	104	cmpwi cr6,r12,0
	105	addi r8,r8,-4
	106	bne cr6,L(done)
	107
	108	/* The c/null byte must be in the second word. Adjust the address
664318c3 AM	109	again and move the result of cmpb to r10/r11 so we can calculate
664318c3 AM	110	the pointer. */
fe2f79db LM	111
	112	mr r10,r6
	113	mr r11,r7
	114	addi r8,r8,4
	115
664318c3	116	/* r10/r11 have the output of the cmpb instructions, that is,
fe2f79db LM	117	0xff in the same position as the c/null byte in the original
	118	word from the string. Use that to calculate the pointer. */
	119	L(done):
664318c3 AM	120	#ifdef __LITTLE_ENDIAN__
	121	addi r3,r10,-1
	122	andc r3,r3,r10
	123	popcntw r0,r3
	124	addi r4,r11,-1
	125	andc r4,r4,r11
	126	cmplw cr7,r3,r4
	127	bgt cr7,L(no_match)
	128	#else
	129	cntlzw r0,r10 /* Count leading zeros before c matches. */
	130	cmplw cr7,r11,r10
fe2f79db	131	bgt cr7,L(no_match)
664318c3 AM	132	#endif
664318c3 AM	133	srwi r0,r0,3 /* Convert leading zeros to bytes. */
fe2f79db LM	134	add r3,r8,r0 /* Return address of the matching c byte
	135	or null in case c was not found. */
	136	blr
	137
	138	.align 4
	139	L(no_match):
	140	li r3,0
	141	blr
	142
	143	/* We are here because strchr was called with a null byte. */
	144	.align 4
	145	L(null_match):
	146	/* r0 has a word of null bytes. */
	147
	148	cmpb r5,r12,r0 /* Compare each byte against null bytes. */
	149
	150	/* Move the words left and right to discard the bits that are
664318c3 AM	151	not part of the string and bring them back as zeros. */
	152	#ifdef __LITTLE_ENDIAN__
	153	srw r5,r5,r6
	154	slw r5,r5,r6
	155	#else
fe2f79db LM	156	slw r5,r5,r6
fe2f79db LM	157	srw r5,r5,r6
664318c3	158	#endif
fe2f79db LM	159	cmpwi cr7,r5,0 /* If r10 == 0, no c or null bytes
	160	have been found. */
	161	bne cr7,L(done_null)
	162
	163	mtcrf 0x01,r8
	164
	165	/* Are we now aligned to a doubleword boundary? If so, skip to
	166	the main loop. Otherwise, go through the alignment code. */
	167
	168	bt 29,L(loop_null)
	169
	170	/* Handle WORD2 of pair. */
	171	lwzu r12,4(r8)
	172	cmpb r5,r12,r0
	173	cmpwi cr7,r5,0
	174	bne cr7,L(done_null)
	175	b L(loop_null) /* We branch here (rather than falling through)
	176	to skip the nops due to heavy alignment
	177	of the loop below. */
	178
	179	/* Main loop to look for the end of the string. Since it's a
	180	small loop (< 8 instructions), align it to 32-bytes. */
	181	.p2align 5
	182	L(loop_null):
	183	/* Load two words, compare and merge in a
	184	single register for speed. This is an attempt
	185	to speed up the null-checking process for bigger strings. */
	186	lwz r12,4(r8)
	187	lwzu r11,8(r8)
	188	cmpb r5,r12,r0
	189	cmpb r10,r11,r0
	190	or r6,r5,r10
	191	cmpwi cr7,r6,0
	192	beq cr7,L(loop_null)
	193
	194	/* OK, one (or both) of the words contains a null byte. Check
	195	the first word and decrement the address in case the first
	196	word really contains a null byte. */
	197
	198	cmpwi cr6,r5,0
	199	addi r8,r8,-4
	200	bne cr6,L(done_null)
	201
	202	/* The null byte must be in the second word. Adjust the address
	203	again and move the result of cmpb to r10 so we can calculate the
	204	pointer. */
	205
	206	mr r5,r10
	207	addi r8,r8,4
	208
	209	/* r5 has the output of the cmpb instruction, that is, it contains
	210	0xff in the same position as the null byte in the original
	211	word from the string. Use that to calculate the pointer. */
	212	L(done_null):
664318c3 AM	213	#ifdef __LITTLE_ENDIAN__
	214	addi r0,r5,-1
	215	andc r0,r0,r5
	216	popcntw r0,r0
	217	#else
fe2f79db	218	cntlzw r0,r5 /* Count leading zeros before the match. */
664318c3	219	#endif
fe2f79db LM	220	srwi r0,r0,3 /* Convert leading zeros to bytes. */
	221	add r3,r8,r0 /* Return address of the matching null byte. */
	222	blr
b5510883 JM	223	END (strchr)
b5510883 JM	224	weak_alias (strchr, index)
fe2f79db	225	libc_hidden_builtin_def (strchr)