1 /* memchr implemented using NEON.
2 Copyright (C) 2011-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library. If not, see
17 <https://www.gnu.org/licenses/>. */
21 /* For __ARM_NEON__ this file defines memchr. */
23 # define memchr __memchr_neon
24 # undef libc_hidden_builtin_def
25 # define libc_hidden_builtin_def(a)
38 #define result r0 /* Live range does not overlap with srcin */
40 /* Working registers */
41 #define src r1 /* Live range does not overlap with chrin */
43 #define synd r0 /* No overlap with srcin or result */
46 /* Working NEON registers */
49 #define vdata0_0 d2 /* Lower half of vdata0 */
50 #define vdata0_1 d3 /* Upper half of vdata0 */
52 #define vdata1_0 d4 /* Lower half of vhas_chr0 */
53 #define vdata1_1 d5 /* Upper half of vhas_chr0 */
64 * For each 32-byte chunk we calculate a 32-bit syndrome value, with one bit per
65 * byte. Each bit is set if the relevant byte matched the requested character
66 * and cleared otherwise. Since the bits in the syndrome reflect exactly the
67 * order in which things occur in the original string, counting trailing zeros
68 * allows to identify exactly which byte has matched.
75 /* Use a simple loop if there are less than 8 bytes to search. */
78 and chrin, chrin, #0xff
82 blo .Lnotfound /* Return not found if reached end. */
85 bne .Lsmallstr /* Loop again if not found. */
86 /* Otherwise fixup address and return. */
92 vdup.8 vrepchr, chrin /* Duplicate char across all lanes. */
94 * Magic constant 0x8040201008040201 allows us to identify which lane
95 * matches the requested byte.
100 vmov vrepmask0, tmp, soff
101 vmov vrepmask1, tmp, soff
102 /* Work with aligned 32-byte chunks */
104 ands soff, srcin, #31
105 beq .Lloopintro /* Go straight to main loop if it's aligned. */
108 * Input string is not 32-byte aligned. We calculate the syndrome
109 * value for the aligned 32 bytes block containing the first bytes
110 * and mask the irrelevant part.
112 vld1.8 {vdata0, vdata1}, [src:256]!
114 adds cntin, cntin, tmp
115 vceq.i8 vdata0, vdata0, vrepchr
116 vceq.i8 vdata1, vdata1, vrepchr
117 vand vdata0, vdata0, vrepmask
118 vand vdata1, vdata1, vrepmask
119 vpadd.i8 vdata0_0, vdata0_0, vdata0_1
120 vpadd.i8 vdata1_0, vdata1_0, vdata1_1
121 vpadd.i8 vdata0_0, vdata0_0, vdata1_0
122 vpadd.i8 vdata0_0, vdata0_0, vdata0_0
123 vmov synd, vdata0_0[0]
125 /* Clear the soff lower bits */
128 /* The first block can also be the last */
130 /* Have we found something already? */
136 /* 264/265 correspond to d8/d9 for q4 */
137 cfi_adjust_cfa_offset (16)
138 cfi_rel_offset (264, 0)
139 cfi_rel_offset (265, 8)
142 vld1.8 {vdata0, vdata1}, [src:256]!
143 subs cntin, cntin, #32
144 vceq.i8 vdata0, vdata0, vrepchr
145 vceq.i8 vdata1, vdata1, vrepchr
146 /* If we're out of data we finish regardless of the result. */
148 /* Use a fast check for the termination condition. */
149 vorr vend, vdata0, vdata1
150 vorr vend0, vend0, vend1
151 vmov synd, tmp, vend0
153 /* We're not out of data, loop if we haven't found the character. */
158 cfi_adjust_cfa_offset (-16)
162 /* Termination condition found, let's calculate the syndrome value. */
163 vand vdata0, vdata0, vrepmask
164 vand vdata1, vdata1, vrepmask
165 vpadd.i8 vdata0_0, vdata0_0, vdata0_1
166 vpadd.i8 vdata1_0, vdata1_0, vdata1_1
167 vpadd.i8 vdata0_0, vdata0_0, vdata1_0
168 vpadd.i8 vdata0_0, vdata0_0, vdata0_0
169 vmov synd, vdata0_0[0]
171 bhi .Ltail /* Uses the condition code from
172 subs cntin, cntin, #32 above. */
176 /* Clear the (-cntin) upper bits to avoid out-of-bounds matches. */
178 lsl synd, synd, cntin
179 lsrs synd, synd, cntin
181 moveq src, #0 /* If no match, set src to 0 so the retval is 0. */
185 /* Count the trailing zeros using bit reversing */
187 /* Compensate the last post-increment */
189 /* Count the leading zeros */
191 /* Compute the potential result and return */
192 add result, src, synd
197 /* Set result to NULL if not found and return */
202 libc_hidden_builtin_def (memchr)