--- /dev/null
+/* Optimized strlen implementation for PowerPC64/POWER9.
+ Copyright (C) 2020 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+#ifndef STRLEN
+# define STRLEN __strlen
+# define DEFINE_STRLEN_HIDDEN_DEF 1
+#endif
+
+/* Implements the function
+
+ int [r3] strlen (const void *s [r3])
+
+ The implementation can load bytes past a matching byte, but only
+ up to the next 64B boundary, so it never crosses a page. */
+
+.machine power9
+ENTRY_TOCLESS (STRLEN, 4)
+ CALL_MCOUNT 2
+
+ vspltisb v18,0
+ vspltisb v19,-1
+
+ neg r5,r3
+ rldicl r9,r5,0,60 /* How many bytes to get source 16B aligned? */
+
+ /* Align data and fill bytes not loaded with non matching char. */
+ lvx v0,0,r3
+ lvsr v1,0,r3
+ vperm v0,v19,v0,v1
+
+ vcmpequb. v6,v0,v18
+ beq cr6,L(aligned)
+
+ vctzlsbb r3,v6
+ blr
+
+ /* Test 64B 16B at a time. The 64B vector loop is optimized for
+ longer strings. Likewise, we check a multiple of 64B to avoid
+ breaking the alignment calculation below. */
+L(aligned):
+ add r4,r3,r9
+ rldicl. r5,r4,60,62 /* Determine the number of 48B loops needed for
+ alignment to 64B. And test for zero. */
+
+ lxv v0+32,0(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail1)
+
+ lxv v0+32,16(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail2)
+
+ lxv v0+32,32(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail3)
+
+ lxv v0+32,48(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail4)
+ addi r4,r4,64
+
+ /* Speculatively generate a fake 16B aligned address to generate the
+ vector byte constant 0,1,..,15 using lvsl during reduction. */
+ li r0,0
+
+ /* Skip the alignment if already 64B aligned. */
+ beq L(loop_64b)
+ mtctr r5
+
+ /* Test 48B per iteration until 64B aligned. */
+ .p2align 5
+L(loop):
+ lxv v0+32,0(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail1)
+
+ lxv v0+32,16(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail2)
+
+ lxv v0+32,32(r4)
+ vcmpequb. v6,v0,v18
+ bne cr6,L(tail3)
+
+ addi r4,r4,48
+ bdnz L(loop)
+
+ .p2align 5
+L(loop_64b):
+ lxv v1+32,0(r4) /* Load 4 quadwords. */
+ lxv v2+32,16(r4)
+ lxv v3+32,32(r4)
+ lxv v4+32,48(r4)
+ vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
+ vminub v6,v3,v4
+ vminub v7,v5,v6
+ vcmpequb. v7,v7,v18 /* Check for NULLs. */
+ addi r4,r4,64 /* Adjust address for the next iteration. */
+ bne cr6,L(vmx_zero)
+
+ lxv v1+32,0(r4) /* Load 4 quadwords. */
+ lxv v2+32,16(r4)
+ lxv v3+32,32(r4)
+ lxv v4+32,48(r4)
+ vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
+ vminub v6,v3,v4
+ vminub v7,v5,v6
+ vcmpequb. v7,v7,v18 /* Check for NULLs. */
+ addi r4,r4,64 /* Adjust address for the next iteration. */
+ bne cr6,L(vmx_zero)
+
+ lxv v1+32,0(r4) /* Load 4 quadwords. */
+ lxv v2+32,16(r4)
+ lxv v3+32,32(r4)
+ lxv v4+32,48(r4)
+ vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
+ vminub v6,v3,v4
+ vminub v7,v5,v6
+ vcmpequb. v7,v7,v18 /* Check for NULLs. */
+ addi r4,r4,64 /* Adjust address for the next iteration. */
+ beq cr6,L(loop_64b)
+
+L(vmx_zero):
+ /* OK, we found a null byte. Let's look for it in the current 64-byte
+ block and mark it in its corresponding VR. */
+ vcmpequb v1,v1,v18
+ vcmpequb v2,v2,v18
+ vcmpequb v3,v3,v18
+ vcmpequb v4,v4,v18
+
+ /* We will now 'compress' the result into a single doubleword, so it
+ can be moved to a GPR for the final calculation. First, we
+ generate an appropriate mask for vbpermq, so we can permute bits into
+ the first halfword. */
+ vspltisb v10,3
+ lvsl v11,0,r0
+ vslb v10,v11,v10
+
+ /* Permute the first bit of each byte into bits 48-63. */
+ vbpermq v1,v1,v10
+ vbpermq v2,v2,v10
+ vbpermq v3,v3,v10
+ vbpermq v4,v4,v10
+
+ /* Shift each component into its correct position for merging. */
+ vsldoi v2,v2,v2,2
+ vsldoi v3,v3,v3,4
+ vsldoi v4,v4,v4,6
+
+ /* Merge the results and move to a GPR. */
+ vor v1,v2,v1
+ vor v2,v3,v4
+ vor v4,v1,v2
+ mfvrd r10,v4
+
+ /* Adjust address to the begninning of the current 64-byte block. */
+ addi r4,r4,-64
+
+ cnttzd r0,r10 /* Count trailing zeros before the match. */
+ subf r5,r3,r4
+ add r3,r5,r0 /* Compute final length. */
+ blr
+
+L(tail1):
+ vctzlsbb r0,v6
+ add r4,r4,r0
+ subf r3,r3,r4
+ blr
+
+L(tail2):
+ vctzlsbb r0,v6
+ add r4,r4,r0
+ addi r4,r4,16
+ subf r3,r3,r4
+ blr
+
+L(tail3):
+ vctzlsbb r0,v6
+ add r4,r4,r0
+ addi r4,r4,32
+ subf r3,r3,r4
+ blr
+
+L(tail4):
+ vctzlsbb r0,v6
+ add r4,r4,r0
+ addi r4,r4,48
+ subf r3,r3,r4
+ blr
+
+END (STRLEN)
+
+#ifdef DEFINE_STRLEN_HIDDEN_DEF
+weak_alias (__strlen, strlen)
+libc_hidden_builtin_def (strlen)
+#endif
projects / thirdparty / glibc.git / commitdiff
