]>
Commit | Line | Data |
---|---|---|
78df0fcb | 1 | /* Highly optimized version for x86-64. |
04277e02 | 2 | Copyright (C) 1999-2019 Free Software Foundation, Inc. |
78df0fcb AJ |
3 | This file is part of the GNU C Library. |
4 | Based on i686 version contributed by Ulrich Drepper | |
5 | <drepper@cygnus.com>, 1999. | |
7956a3d2 | 6 | Updated with SSE2 support contributed by Intel Corporation. |
78df0fcb AJ |
7 | |
8 | The GNU C Library is free software; you can redistribute it and/or | |
9 | modify it under the terms of the GNU Lesser General Public | |
10 | License as published by the Free Software Foundation; either | |
11 | version 2.1 of the License, or (at your option) any later version. | |
12 | ||
13 | The GNU C Library is distributed in the hope that it will be useful, | |
14 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | Lesser General Public License for more details. | |
17 | ||
18 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 | 19 | License along with the GNU C Library; if not, see |
5a82c748 | 20 | <https://www.gnu.org/licenses/>. */ |
78df0fcb AJ |
21 | |
22 | #include <sysdep.h> | |
23 | #include "asm-syntax.h" | |
78df0fcb | 24 | |
7956a3d2 L |
25 | #undef UPDATE_STRNCMP_COUNTER |
26 | ||
27 | #ifndef LABEL | |
28 | #define LABEL(l) L(l) | |
29 | #endif | |
30 | ||
31 | #ifdef USE_AS_STRNCMP | |
32 | /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz | |
33 | if the new counter > the old one or is 0. */ | |
34 | # define UPDATE_STRNCMP_COUNTER \ | |
35 | /* calculate left number to compare */ \ | |
36 | lea -16(%rcx, %r11), %r9; \ | |
37 | cmp %r9, %r11; \ | |
38 | jb LABEL(strcmp_exitz); \ | |
39 | test %r9, %r9; \ | |
40 | je LABEL(strcmp_exitz); \ | |
41 | mov %r9, %r11 | |
42 | ||
42e08a54 UD |
43 | #elif defined USE_AS_STRCASECMP_L |
44 | # include "locale-defines.h" | |
45 | ||
42e08a54 | 46 | # define UPDATE_STRNCMP_COUNTER |
e9f82e0d UD |
47 | #elif defined USE_AS_STRNCASECMP_L |
48 | # include "locale-defines.h" | |
49 | ||
e9f82e0d UD |
50 | # define UPDATE_STRNCMP_COUNTER \ |
51 | /* calculate left number to compare */ \ | |
52 | lea -16(%rcx, %r11), %r9; \ | |
53 | cmp %r9, %r11; \ | |
54 | jb LABEL(strcmp_exitz); \ | |
55 | test %r9, %r9; \ | |
56 | je LABEL(strcmp_exitz); \ | |
57 | mov %r9, %r11 | |
7956a3d2 L |
58 | #else |
59 | # define UPDATE_STRNCMP_COUNTER | |
60 | # ifndef STRCMP | |
25244f17 | 61 | # define STRCMP strcmp |
7956a3d2 L |
62 | # endif |
63 | #endif | |
64 | ||
0fda545d | 65 | #ifndef USE_SSSE3 |
7956a3d2 | 66 | .text |
0fda545d | 67 | #else |
f69190e7 | 68 | .section .text.ssse3,"ax",@progbits |
0fda545d UD |
69 | #endif |
70 | ||
42e08a54 | 71 | #ifdef USE_AS_STRCASECMP_L |
73507d3a UD |
72 | # ifndef ENTRY2 |
73 | # define ENTRY2(name) ENTRY (name) | |
74 | # define END2(name) END (name) | |
73507d3a UD |
75 | # endif |
76 | ||
e9f82e0d | 77 | ENTRY2 (__strcasecmp) |
42e08a54 | 78 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax |
3cb84561 | 79 | mov %fs:(%rax),%RDX_LP |
42e08a54 | 80 | |
73507d3a | 81 | // XXX 5 byte should be before the function |
42e08a54 UD |
82 | /* 5-byte NOP. */ |
83 | .byte 0x0f,0x1f,0x44,0x00,0x00 | |
73507d3a UD |
84 | END2 (__strcasecmp) |
85 | # ifndef NO_NOLOCALE_ALIAS | |
42e08a54 UD |
86 | weak_alias (__strcasecmp, strcasecmp) |
87 | libc_hidden_def (__strcasecmp) | |
73507d3a | 88 | # endif |
42e08a54 | 89 | /* FALLTHROUGH to strcasecmp_l. */ |
e9f82e0d UD |
90 | #elif defined USE_AS_STRNCASECMP_L |
91 | # ifndef ENTRY2 | |
92 | # define ENTRY2(name) ENTRY (name) | |
93 | # define END2(name) END (name) | |
94 | # endif | |
95 | ||
96 | ENTRY2 (__strncasecmp) | |
97 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax | |
3cb84561 | 98 | mov %fs:(%rax),%RCX_LP |
e9f82e0d UD |
99 | |
100 | // XXX 5 byte should be before the function | |
101 | /* 5-byte NOP. */ | |
102 | .byte 0x0f,0x1f,0x44,0x00,0x00 | |
103 | END2 (__strncasecmp) | |
104 | # ifndef NO_NOLOCALE_ALIAS | |
105 | weak_alias (__strncasecmp, strncasecmp) | |
106 | libc_hidden_def (__strncasecmp) | |
107 | # endif | |
108 | /* FALLTHROUGH to strncasecmp_l. */ | |
42e08a54 UD |
109 | #endif |
110 | ||
29691210 | 111 | ENTRY (STRCMP) |
d8725b1f | 112 | #ifdef USE_AS_STRCASECMP_L |
42e08a54 UD |
113 | /* We have to fall back on the C implementation for locales |
114 | with encodings not matching ASCII for single bytes. */ | |
d8725b1f | 115 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 |
3cb84561 | 116 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rdx), %RAX_LP |
d8725b1f | 117 | # else |
3cb84561 | 118 | mov (%rdx), %RAX_LP |
d8725b1f | 119 | # endif |
34372fc6 | 120 | testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
42e08a54 | 121 | jne __strcasecmp_l_nonascii |
d8725b1f | 122 | #elif defined USE_AS_STRNCASECMP_L |
e9f82e0d UD |
123 | /* We have to fall back on the C implementation for locales |
124 | with encodings not matching ASCII for single bytes. */ | |
d8725b1f | 125 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 |
3cb84561 | 126 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rcx), %RAX_LP |
d8725b1f | 127 | # else |
3cb84561 | 128 | mov (%rcx), %RAX_LP |
d8725b1f | 129 | # endif |
34372fc6 | 130 | testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
e9f82e0d | 131 | jne __strncasecmp_l_nonascii |
d8725b1f | 132 | #endif |
42e08a54 | 133 | |
7956a3d2 L |
134 | /* |
135 | * This implementation uses SSE to compare up to 16 bytes at a time. | |
136 | */ | |
d8725b1f | 137 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
ee915088 | 138 | test %RDX_LP, %RDX_LP |
7956a3d2 | 139 | je LABEL(strcmp_exitz) |
ee915088 | 140 | cmp $1, %RDX_LP |
7956a3d2 | 141 | je LABEL(Byte0) |
ee915088 | 142 | mov %RDX_LP, %R11_LP |
d8725b1f | 143 | #endif |
7956a3d2 L |
144 | mov %esi, %ecx |
145 | mov %edi, %eax | |
146 | /* Use 64bit AND here to avoid long NOP padding. */ | |
147 | and $0x3f, %rcx /* rsi alignment in cache line */ | |
148 | and $0x3f, %rax /* rdi alignment in cache line */ | |
d8725b1f | 149 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
42e08a54 UD |
150 | .section .rodata.cst16,"aM",@progbits,16 |
151 | .align 16 | |
152 | .Lbelowupper: | |
153 | .quad 0x4040404040404040 | |
154 | .quad 0x4040404040404040 | |
155 | .Ltopupper: | |
156 | .quad 0x5b5b5b5b5b5b5b5b | |
157 | .quad 0x5b5b5b5b5b5b5b5b | |
158 | .Ltouppermask: | |
159 | .quad 0x2020202020202020 | |
160 | .quad 0x2020202020202020 | |
161 | .previous | |
162 | movdqa .Lbelowupper(%rip), %xmm5 | |
d8725b1f | 163 | # define UCLOW_reg %xmm5 |
42e08a54 | 164 | movdqa .Ltopupper(%rip), %xmm6 |
d8725b1f | 165 | # define UCHIGH_reg %xmm6 |
42e08a54 | 166 | movdqa .Ltouppermask(%rip), %xmm7 |
d8725b1f L |
167 | # define LCQWORD_reg %xmm7 |
168 | #endif | |
7956a3d2 L |
169 | cmp $0x30, %ecx |
170 | ja LABEL(crosscache) /* rsi: 16-byte load will cross cache line */ | |
171 | cmp $0x30, %eax | |
172 | ja LABEL(crosscache) /* rdi: 16-byte load will cross cache line */ | |
173 | movlpd (%rdi), %xmm1 | |
174 | movlpd (%rsi), %xmm2 | |
175 | movhpd 8(%rdi), %xmm1 | |
176 | movhpd 8(%rsi), %xmm2 | |
d8725b1f L |
177 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
178 | # define TOLOWER(reg1, reg2) \ | |
42e08a54 UD |
179 | movdqa reg1, %xmm8; \ |
180 | movdqa UCHIGH_reg, %xmm9; \ | |
181 | movdqa reg2, %xmm10; \ | |
182 | movdqa UCHIGH_reg, %xmm11; \ | |
183 | pcmpgtb UCLOW_reg, %xmm8; \ | |
184 | pcmpgtb reg1, %xmm9; \ | |
185 | pcmpgtb UCLOW_reg, %xmm10; \ | |
186 | pcmpgtb reg2, %xmm11; \ | |
187 | pand %xmm9, %xmm8; \ | |
188 | pand %xmm11, %xmm10; \ | |
189 | pand LCQWORD_reg, %xmm8; \ | |
190 | pand LCQWORD_reg, %xmm10; \ | |
191 | por %xmm8, reg1; \ | |
192 | por %xmm10, reg2 | |
193 | TOLOWER (%xmm1, %xmm2) | |
d8725b1f L |
194 | #else |
195 | # define TOLOWER(reg1, reg2) | |
196 | #endif | |
7956a3d2 L |
197 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ |
198 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
199 | pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */ | |
200 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ | |
201 | pmovmskb %xmm1, %edx | |
202 | sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */ | |
203 | jnz LABEL(less16bytes) /* If not, find different value or null char */ | |
d8725b1f | 204 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
205 | sub $16, %r11 |
206 | jbe LABEL(strcmp_exitz) /* finish comparision */ | |
d8725b1f | 207 | #endif |
7956a3d2 L |
208 | add $16, %rsi /* prepare to search next 16 bytes */ |
209 | add $16, %rdi /* prepare to search next 16 bytes */ | |
210 | ||
211 | /* | |
212 | * Determine source and destination string offsets from 16-byte alignment. | |
213 | * Use relative offset difference between the two to determine which case | |
214 | * below to use. | |
215 | */ | |
216 | .p2align 4 | |
217 | LABEL(crosscache): | |
218 | and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */ | |
219 | and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */ | |
220 | mov $0xffff, %edx /* for equivalent offset */ | |
221 | xor %r8d, %r8d | |
222 | and $0xf, %ecx /* offset of rsi */ | |
223 | and $0xf, %eax /* offset of rdi */ | |
224 | cmp %eax, %ecx | |
225 | je LABEL(ashr_0) /* rsi and rdi relative offset same */ | |
226 | ja LABEL(bigger) | |
227 | mov %edx, %r8d /* r8d is offset flag for exit tail */ | |
228 | xchg %ecx, %eax | |
229 | xchg %rsi, %rdi | |
230 | LABEL(bigger): | |
231 | lea 15(%rax), %r9 | |
232 | sub %rcx, %r9 | |
233 | lea LABEL(unaligned_table)(%rip), %r10 | |
234 | movslq (%r10, %r9,4), %r9 | |
235 | lea (%r10, %r9), %r10 | |
4ef60d95 | 236 | _CET_NOTRACK jmp *%r10 /* jump to corresponding case */ |
7956a3d2 L |
237 | |
238 | /* | |
239 | * The following cases will be handled by ashr_0 | |
240 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
241 | * n(0~15) n(0~15) 15(15+ n-n) ashr_0 | |
242 | */ | |
243 | .p2align 4 | |
244 | LABEL(ashr_0): | |
245 | ||
246 | movdqa (%rsi), %xmm1 | |
247 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */ | |
248 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
d8725b1f | 249 | #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L |
7956a3d2 | 250 | pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */ |
d8725b1f | 251 | #else |
42e08a54 UD |
252 | movdqa (%rdi), %xmm2 |
253 | TOLOWER (%xmm1, %xmm2) | |
254 | pcmpeqb %xmm2, %xmm1 /* compare 16 bytes for equality */ | |
d8725b1f | 255 | #endif |
7956a3d2 L |
256 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ |
257 | pmovmskb %xmm1, %r9d | |
258 | shr %cl, %edx /* adjust 0xffff for offset */ | |
259 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
260 | sub %r9d, %edx | |
261 | /* | |
262 | * edx must be the same with r9d if in left byte (16-rcx) is equal to | |
263 | * the start from (16-rax) and no null char was seen. | |
264 | */ | |
265 | jne LABEL(less32bytes) /* mismatch or null char */ | |
266 | UPDATE_STRNCMP_COUNTER | |
267 | mov $16, %rcx | |
268 | mov $16, %r9 | |
269 | pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */ | |
270 | ||
271 | /* | |
272 | * Now both strings are aligned at 16-byte boundary. Loop over strings | |
273 | * checking 32-bytes per iteration. | |
274 | */ | |
275 | .p2align 4 | |
276 | LABEL(loop_ashr_0): | |
277 | movdqa (%rsi, %rcx), %xmm1 | |
278 | movdqa (%rdi, %rcx), %xmm2 | |
42e08a54 | 279 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
280 | |
281 | pcmpeqb %xmm1, %xmm0 | |
282 | pcmpeqb %xmm2, %xmm1 | |
283 | psubb %xmm0, %xmm1 | |
284 | pmovmskb %xmm1, %edx | |
285 | sub $0xffff, %edx | |
286 | jnz LABEL(exit) /* mismatch or null char seen */ | |
287 | ||
d8725b1f | 288 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
289 | sub $16, %r11 |
290 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 291 | #endif |
7956a3d2 L |
292 | add $16, %rcx |
293 | movdqa (%rsi, %rcx), %xmm1 | |
294 | movdqa (%rdi, %rcx), %xmm2 | |
42e08a54 | 295 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
296 | |
297 | pcmpeqb %xmm1, %xmm0 | |
298 | pcmpeqb %xmm2, %xmm1 | |
299 | psubb %xmm0, %xmm1 | |
300 | pmovmskb %xmm1, %edx | |
301 | sub $0xffff, %edx | |
302 | jnz LABEL(exit) | |
d8725b1f | 303 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
304 | sub $16, %r11 |
305 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 306 | #endif |
7956a3d2 L |
307 | add $16, %rcx |
308 | jmp LABEL(loop_ashr_0) | |
309 | ||
310 | /* | |
311 | * The following cases will be handled by ashr_1 | |
42e08a54 | 312 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case |
7956a3d2 L |
313 | * n(15) n -15 0(15 +(n-15) - n) ashr_1 |
314 | */ | |
315 | .p2align 4 | |
316 | LABEL(ashr_1): | |
317 | pxor %xmm0, %xmm0 | |
318 | movdqa (%rdi), %xmm2 | |
319 | movdqa (%rsi), %xmm1 | |
320 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
321 | pslldq $15, %xmm2 /* shift first string to align with second */ | |
42e08a54 | 322 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
323 | pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */ |
324 | psubb %xmm0, %xmm2 /* packed sub of comparison results*/ | |
325 | pmovmskb %xmm2, %r9d | |
326 | shr %cl, %edx /* adjust 0xffff for offset */ | |
327 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
328 | sub %r9d, %edx | |
329 | jnz LABEL(less32bytes) /* mismatch or null char seen */ | |
330 | movdqa (%rdi), %xmm3 | |
331 | UPDATE_STRNCMP_COUNTER | |
332 | ||
333 | pxor %xmm0, %xmm0 | |
334 | mov $16, %rcx /* index for loads*/ | |
335 | mov $1, %r9d /* byte position left over from less32bytes case */ | |
336 | /* | |
337 | * Setup %r10 value allows us to detect crossing a page boundary. | |
338 | * When %r10 goes positive we have crossed a page boundary and | |
339 | * need to do a nibble. | |
340 | */ | |
341 | lea 1(%rdi), %r10 | |
342 | and $0xfff, %r10 /* offset into 4K page */ | |
343 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
344 | ||
345 | .p2align 4 | |
346 | LABEL(loop_ashr_1): | |
347 | add $16, %r10 | |
348 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
349 | ||
350 | LABEL(gobble_ashr_1): | |
351 | movdqa (%rsi, %rcx), %xmm1 | |
352 | movdqa (%rdi, %rcx), %xmm2 | |
353 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
354 | ||
d8725b1f | 355 | #ifndef USE_SSSE3 |
7956a3d2 L |
356 | psrldq $1, %xmm3 |
357 | pslldq $15, %xmm2 | |
358 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 359 | #else |
0fda545d | 360 | palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 361 | #endif |
42e08a54 | 362 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
363 | |
364 | pcmpeqb %xmm1, %xmm0 | |
365 | pcmpeqb %xmm2, %xmm1 | |
366 | psubb %xmm0, %xmm1 | |
367 | pmovmskb %xmm1, %edx | |
368 | sub $0xffff, %edx | |
369 | jnz LABEL(exit) | |
370 | ||
d8725b1f | 371 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
372 | sub $16, %r11 |
373 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 374 | #endif |
7956a3d2 L |
375 | add $16, %rcx |
376 | movdqa %xmm4, %xmm3 | |
377 | ||
378 | add $16, %r10 | |
379 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
380 | ||
381 | movdqa (%rsi, %rcx), %xmm1 | |
382 | movdqa (%rdi, %rcx), %xmm2 | |
383 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
384 | ||
d8725b1f | 385 | #ifndef USE_SSSE3 |
7956a3d2 | 386 | psrldq $1, %xmm3 |
0fda545d | 387 | pslldq $15, %xmm2 |
7956a3d2 | 388 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 389 | #else |
0fda545d | 390 | palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 391 | #endif |
42e08a54 | 392 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
393 | |
394 | pcmpeqb %xmm1, %xmm0 | |
395 | pcmpeqb %xmm2, %xmm1 | |
396 | psubb %xmm0, %xmm1 | |
397 | pmovmskb %xmm1, %edx | |
398 | sub $0xffff, %edx | |
399 | jnz LABEL(exit) | |
400 | ||
d8725b1f | 401 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
402 | sub $16, %r11 |
403 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 404 | #endif |
7956a3d2 L |
405 | add $16, %rcx |
406 | movdqa %xmm4, %xmm3 | |
407 | jmp LABEL(loop_ashr_1) | |
408 | ||
409 | /* | |
410 | * Nibble avoids loads across page boundary. This is to avoid a potential | |
411 | * access into unmapped memory. | |
412 | */ | |
413 | .p2align 4 | |
414 | LABEL(nibble_ashr_1): | |
415 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char*/ | |
416 | pmovmskb %xmm0, %edx | |
417 | test $0xfffe, %edx | |
418 | jnz LABEL(ashr_1_exittail) /* find null char*/ | |
419 | ||
d8725b1f | 420 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 421 | cmp $15, %r11 |
7956a3d2 | 422 | jbe LABEL(ashr_1_exittail) |
d8725b1f | 423 | #endif |
7956a3d2 L |
424 | |
425 | pxor %xmm0, %xmm0 | |
426 | sub $0x1000, %r10 /* substract 4K from %r10 */ | |
427 | jmp LABEL(gobble_ashr_1) | |
428 | ||
429 | /* | |
430 | * Once find null char, determine if there is a string mismatch | |
431 | * before the null char. | |
432 | */ | |
433 | .p2align 4 | |
434 | LABEL(ashr_1_exittail): | |
435 | movdqa (%rsi, %rcx), %xmm1 | |
436 | psrldq $1, %xmm0 | |
437 | psrldq $1, %xmm3 | |
438 | jmp LABEL(aftertail) | |
439 | ||
440 | /* | |
441 | * The following cases will be handled by ashr_2 | |
f69190e7 | 442 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case |
7956a3d2 L |
443 | * n(14~15) n -14 1(15 +(n-14) - n) ashr_2 |
444 | */ | |
445 | .p2align 4 | |
446 | LABEL(ashr_2): | |
447 | pxor %xmm0, %xmm0 | |
448 | movdqa (%rdi), %xmm2 | |
449 | movdqa (%rsi), %xmm1 | |
450 | pcmpeqb %xmm1, %xmm0 | |
451 | pslldq $14, %xmm2 | |
42e08a54 | 452 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
453 | pcmpeqb %xmm1, %xmm2 |
454 | psubb %xmm0, %xmm2 | |
455 | pmovmskb %xmm2, %r9d | |
456 | shr %cl, %edx | |
457 | shr %cl, %r9d | |
458 | sub %r9d, %edx | |
459 | jnz LABEL(less32bytes) | |
460 | movdqa (%rdi), %xmm3 | |
461 | UPDATE_STRNCMP_COUNTER | |
462 | ||
463 | pxor %xmm0, %xmm0 | |
464 | mov $16, %rcx /* index for loads */ | |
465 | mov $2, %r9d /* byte position left over from less32bytes case */ | |
466 | /* | |
467 | * Setup %r10 value allows us to detect crossing a page boundary. | |
468 | * When %r10 goes positive we have crossed a page boundary and | |
469 | * need to do a nibble. | |
470 | */ | |
471 | lea 2(%rdi), %r10 | |
472 | and $0xfff, %r10 /* offset into 4K page */ | |
473 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
474 | ||
475 | .p2align 4 | |
476 | LABEL(loop_ashr_2): | |
477 | add $16, %r10 | |
478 | jg LABEL(nibble_ashr_2) | |
479 | ||
480 | LABEL(gobble_ashr_2): | |
481 | movdqa (%rsi, %rcx), %xmm1 | |
482 | movdqa (%rdi, %rcx), %xmm2 | |
483 | movdqa %xmm2, %xmm4 | |
484 | ||
d8725b1f | 485 | #ifndef USE_SSSE3 |
7956a3d2 L |
486 | psrldq $2, %xmm3 |
487 | pslldq $14, %xmm2 | |
0fda545d | 488 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 489 | #else |
0fda545d | 490 | palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 491 | #endif |
42e08a54 | 492 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
493 | |
494 | pcmpeqb %xmm1, %xmm0 | |
495 | pcmpeqb %xmm2, %xmm1 | |
496 | psubb %xmm0, %xmm1 | |
497 | pmovmskb %xmm1, %edx | |
498 | sub $0xffff, %edx | |
499 | jnz LABEL(exit) | |
500 | ||
d8725b1f | 501 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
502 | sub $16, %r11 |
503 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 504 | #endif |
7956a3d2 L |
505 | |
506 | add $16, %rcx | |
507 | movdqa %xmm4, %xmm3 | |
508 | ||
509 | add $16, %r10 | |
510 | jg LABEL(nibble_ashr_2) /* cross page boundary */ | |
511 | ||
512 | movdqa (%rsi, %rcx), %xmm1 | |
513 | movdqa (%rdi, %rcx), %xmm2 | |
514 | movdqa %xmm2, %xmm4 | |
515 | ||
d8725b1f | 516 | #ifndef USE_SSSE3 |
7956a3d2 | 517 | psrldq $2, %xmm3 |
0fda545d UD |
518 | pslldq $14, %xmm2 |
519 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 520 | #else |
0fda545d | 521 | palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 522 | #endif |
42e08a54 | 523 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
524 | |
525 | pcmpeqb %xmm1, %xmm0 | |
526 | pcmpeqb %xmm2, %xmm1 | |
527 | psubb %xmm0, %xmm1 | |
528 | pmovmskb %xmm1, %edx | |
529 | sub $0xffff, %edx | |
530 | jnz LABEL(exit) | |
531 | ||
d8725b1f | 532 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
533 | sub $16, %r11 |
534 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 535 | #endif |
7956a3d2 L |
536 | |
537 | add $16, %rcx | |
538 | movdqa %xmm4, %xmm3 | |
539 | jmp LABEL(loop_ashr_2) | |
540 | ||
541 | .p2align 4 | |
542 | LABEL(nibble_ashr_2): | |
543 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
544 | pmovmskb %xmm0, %edx | |
545 | test $0xfffc, %edx | |
546 | jnz LABEL(ashr_2_exittail) | |
547 | ||
d8725b1f | 548 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 549 | cmp $14, %r11 |
7956a3d2 | 550 | jbe LABEL(ashr_2_exittail) |
d8725b1f | 551 | #endif |
7956a3d2 L |
552 | |
553 | pxor %xmm0, %xmm0 | |
554 | sub $0x1000, %r10 | |
555 | jmp LABEL(gobble_ashr_2) | |
556 | ||
557 | .p2align 4 | |
558 | LABEL(ashr_2_exittail): | |
559 | movdqa (%rsi, %rcx), %xmm1 | |
560 | psrldq $2, %xmm0 | |
561 | psrldq $2, %xmm3 | |
562 | jmp LABEL(aftertail) | |
563 | ||
564 | /* | |
565 | * The following cases will be handled by ashr_3 | |
566 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
567 | * n(13~15) n -13 2(15 +(n-13) - n) ashr_3 | |
568 | */ | |
569 | .p2align 4 | |
570 | LABEL(ashr_3): | |
571 | pxor %xmm0, %xmm0 | |
572 | movdqa (%rdi), %xmm2 | |
573 | movdqa (%rsi), %xmm1 | |
574 | pcmpeqb %xmm1, %xmm0 | |
575 | pslldq $13, %xmm2 | |
42e08a54 | 576 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
577 | pcmpeqb %xmm1, %xmm2 |
578 | psubb %xmm0, %xmm2 | |
579 | pmovmskb %xmm2, %r9d | |
580 | shr %cl, %edx | |
581 | shr %cl, %r9d | |
582 | sub %r9d, %edx | |
583 | jnz LABEL(less32bytes) | |
584 | movdqa (%rdi), %xmm3 | |
585 | ||
586 | UPDATE_STRNCMP_COUNTER | |
587 | ||
588 | pxor %xmm0, %xmm0 | |
589 | mov $16, %rcx /* index for loads */ | |
590 | mov $3, %r9d /* byte position left over from less32bytes case */ | |
591 | /* | |
592 | * Setup %r10 value allows us to detect crossing a page boundary. | |
593 | * When %r10 goes positive we have crossed a page boundary and | |
594 | * need to do a nibble. | |
595 | */ | |
596 | lea 3(%rdi), %r10 | |
597 | and $0xfff, %r10 /* offset into 4K page */ | |
598 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
599 | ||
600 | .p2align 4 | |
601 | LABEL(loop_ashr_3): | |
602 | add $16, %r10 | |
603 | jg LABEL(nibble_ashr_3) | |
604 | ||
605 | LABEL(gobble_ashr_3): | |
606 | movdqa (%rsi, %rcx), %xmm1 | |
607 | movdqa (%rdi, %rcx), %xmm2 | |
608 | movdqa %xmm2, %xmm4 | |
609 | ||
d8725b1f | 610 | #ifndef USE_SSSE3 |
7956a3d2 L |
611 | psrldq $3, %xmm3 |
612 | pslldq $13, %xmm2 | |
0fda545d | 613 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 614 | #else |
0fda545d | 615 | palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 616 | #endif |
42e08a54 | 617 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
618 | |
619 | pcmpeqb %xmm1, %xmm0 | |
620 | pcmpeqb %xmm2, %xmm1 | |
621 | psubb %xmm0, %xmm1 | |
622 | pmovmskb %xmm1, %edx | |
623 | sub $0xffff, %edx | |
624 | jnz LABEL(exit) | |
625 | ||
d8725b1f | 626 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
627 | sub $16, %r11 |
628 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 629 | #endif |
7956a3d2 L |
630 | |
631 | add $16, %rcx | |
632 | movdqa %xmm4, %xmm3 | |
633 | ||
634 | add $16, %r10 | |
635 | jg LABEL(nibble_ashr_3) /* cross page boundary */ | |
636 | ||
637 | movdqa (%rsi, %rcx), %xmm1 | |
638 | movdqa (%rdi, %rcx), %xmm2 | |
639 | movdqa %xmm2, %xmm4 | |
640 | ||
d8725b1f | 641 | #ifndef USE_SSSE3 |
7956a3d2 | 642 | psrldq $3, %xmm3 |
0fda545d UD |
643 | pslldq $13, %xmm2 |
644 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 645 | #else |
0fda545d | 646 | palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 647 | #endif |
42e08a54 | 648 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
649 | |
650 | pcmpeqb %xmm1, %xmm0 | |
651 | pcmpeqb %xmm2, %xmm1 | |
652 | psubb %xmm0, %xmm1 | |
653 | pmovmskb %xmm1, %edx | |
654 | sub $0xffff, %edx | |
655 | jnz LABEL(exit) | |
656 | ||
d8725b1f | 657 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
658 | sub $16, %r11 |
659 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 660 | #endif |
7956a3d2 L |
661 | |
662 | add $16, %rcx | |
663 | movdqa %xmm4, %xmm3 | |
664 | jmp LABEL(loop_ashr_3) | |
665 | ||
666 | .p2align 4 | |
667 | LABEL(nibble_ashr_3): | |
668 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
669 | pmovmskb %xmm0, %edx | |
670 | test $0xfff8, %edx | |
671 | jnz LABEL(ashr_3_exittail) | |
672 | ||
d8725b1f | 673 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 674 | cmp $13, %r11 |
7956a3d2 | 675 | jbe LABEL(ashr_3_exittail) |
d8725b1f | 676 | #endif |
7956a3d2 L |
677 | |
678 | pxor %xmm0, %xmm0 | |
679 | sub $0x1000, %r10 | |
680 | jmp LABEL(gobble_ashr_3) | |
681 | ||
682 | .p2align 4 | |
683 | LABEL(ashr_3_exittail): | |
684 | movdqa (%rsi, %rcx), %xmm1 | |
685 | psrldq $3, %xmm0 | |
686 | psrldq $3, %xmm3 | |
687 | jmp LABEL(aftertail) | |
688 | ||
689 | /* | |
690 | * The following cases will be handled by ashr_4 | |
691 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
692 | * n(12~15) n -12 3(15 +(n-12) - n) ashr_4 | |
693 | */ | |
694 | .p2align 4 | |
695 | LABEL(ashr_4): | |
696 | pxor %xmm0, %xmm0 | |
697 | movdqa (%rdi), %xmm2 | |
698 | movdqa (%rsi), %xmm1 | |
699 | pcmpeqb %xmm1, %xmm0 | |
700 | pslldq $12, %xmm2 | |
42e08a54 | 701 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
702 | pcmpeqb %xmm1, %xmm2 |
703 | psubb %xmm0, %xmm2 | |
704 | pmovmskb %xmm2, %r9d | |
705 | shr %cl, %edx | |
706 | shr %cl, %r9d | |
707 | sub %r9d, %edx | |
708 | jnz LABEL(less32bytes) | |
709 | movdqa (%rdi), %xmm3 | |
710 | ||
711 | UPDATE_STRNCMP_COUNTER | |
712 | ||
713 | pxor %xmm0, %xmm0 | |
714 | mov $16, %rcx /* index for loads */ | |
715 | mov $4, %r9d /* byte position left over from less32bytes case */ | |
716 | /* | |
717 | * Setup %r10 value allows us to detect crossing a page boundary. | |
718 | * When %r10 goes positive we have crossed a page boundary and | |
719 | * need to do a nibble. | |
720 | */ | |
721 | lea 4(%rdi), %r10 | |
722 | and $0xfff, %r10 /* offset into 4K page */ | |
723 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
724 | ||
725 | .p2align 4 | |
726 | LABEL(loop_ashr_4): | |
727 | add $16, %r10 | |
728 | jg LABEL(nibble_ashr_4) | |
729 | ||
730 | LABEL(gobble_ashr_4): | |
731 | movdqa (%rsi, %rcx), %xmm1 | |
732 | movdqa (%rdi, %rcx), %xmm2 | |
733 | movdqa %xmm2, %xmm4 | |
734 | ||
d8725b1f | 735 | #ifndef USE_SSSE3 |
7956a3d2 L |
736 | psrldq $4, %xmm3 |
737 | pslldq $12, %xmm2 | |
0fda545d | 738 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 739 | #else |
0fda545d | 740 | palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 741 | #endif |
42e08a54 | 742 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
743 | |
744 | pcmpeqb %xmm1, %xmm0 | |
745 | pcmpeqb %xmm2, %xmm1 | |
746 | psubb %xmm0, %xmm1 | |
747 | pmovmskb %xmm1, %edx | |
748 | sub $0xffff, %edx | |
749 | jnz LABEL(exit) | |
750 | ||
d8725b1f | 751 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
752 | sub $16, %r11 |
753 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 754 | #endif |
7956a3d2 L |
755 | |
756 | add $16, %rcx | |
757 | movdqa %xmm4, %xmm3 | |
758 | ||
759 | add $16, %r10 | |
760 | jg LABEL(nibble_ashr_4) /* cross page boundary */ | |
761 | ||
762 | movdqa (%rsi, %rcx), %xmm1 | |
763 | movdqa (%rdi, %rcx), %xmm2 | |
764 | movdqa %xmm2, %xmm4 | |
765 | ||
d8725b1f | 766 | #ifndef USE_SSSE3 |
7956a3d2 | 767 | psrldq $4, %xmm3 |
0fda545d UD |
768 | pslldq $12, %xmm2 |
769 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 770 | #else |
0fda545d | 771 | palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 772 | #endif |
42e08a54 | 773 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
774 | |
775 | pcmpeqb %xmm1, %xmm0 | |
776 | pcmpeqb %xmm2, %xmm1 | |
777 | psubb %xmm0, %xmm1 | |
778 | pmovmskb %xmm1, %edx | |
779 | sub $0xffff, %edx | |
780 | jnz LABEL(exit) | |
781 | ||
d8725b1f | 782 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
783 | sub $16, %r11 |
784 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 785 | #endif |
7956a3d2 L |
786 | |
787 | add $16, %rcx | |
788 | movdqa %xmm4, %xmm3 | |
789 | jmp LABEL(loop_ashr_4) | |
790 | ||
791 | .p2align 4 | |
792 | LABEL(nibble_ashr_4): | |
793 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
794 | pmovmskb %xmm0, %edx | |
795 | test $0xfff0, %edx | |
796 | jnz LABEL(ashr_4_exittail) | |
797 | ||
d8725b1f | 798 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 799 | cmp $12, %r11 |
7956a3d2 | 800 | jbe LABEL(ashr_4_exittail) |
d8725b1f | 801 | #endif |
7956a3d2 L |
802 | |
803 | pxor %xmm0, %xmm0 | |
804 | sub $0x1000, %r10 | |
805 | jmp LABEL(gobble_ashr_4) | |
806 | ||
807 | .p2align 4 | |
808 | LABEL(ashr_4_exittail): | |
809 | movdqa (%rsi, %rcx), %xmm1 | |
810 | psrldq $4, %xmm0 | |
811 | psrldq $4, %xmm3 | |
812 | jmp LABEL(aftertail) | |
813 | ||
814 | /* | |
815 | * The following cases will be handled by ashr_5 | |
816 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 817 | * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5 |
7956a3d2 L |
818 | */ |
819 | .p2align 4 | |
820 | LABEL(ashr_5): | |
821 | pxor %xmm0, %xmm0 | |
822 | movdqa (%rdi), %xmm2 | |
823 | movdqa (%rsi), %xmm1 | |
824 | pcmpeqb %xmm1, %xmm0 | |
825 | pslldq $11, %xmm2 | |
42e08a54 | 826 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
827 | pcmpeqb %xmm1, %xmm2 |
828 | psubb %xmm0, %xmm2 | |
829 | pmovmskb %xmm2, %r9d | |
830 | shr %cl, %edx | |
831 | shr %cl, %r9d | |
832 | sub %r9d, %edx | |
833 | jnz LABEL(less32bytes) | |
834 | movdqa (%rdi), %xmm3 | |
835 | ||
836 | UPDATE_STRNCMP_COUNTER | |
837 | ||
838 | pxor %xmm0, %xmm0 | |
839 | mov $16, %rcx /* index for loads */ | |
840 | mov $5, %r9d /* byte position left over from less32bytes case */ | |
841 | /* | |
842 | * Setup %r10 value allows us to detect crossing a page boundary. | |
843 | * When %r10 goes positive we have crossed a page boundary and | |
844 | * need to do a nibble. | |
845 | */ | |
846 | lea 5(%rdi), %r10 | |
847 | and $0xfff, %r10 /* offset into 4K page */ | |
848 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
849 | ||
850 | .p2align 4 | |
851 | LABEL(loop_ashr_5): | |
852 | add $16, %r10 | |
853 | jg LABEL(nibble_ashr_5) | |
854 | ||
855 | LABEL(gobble_ashr_5): | |
856 | movdqa (%rsi, %rcx), %xmm1 | |
857 | movdqa (%rdi, %rcx), %xmm2 | |
858 | movdqa %xmm2, %xmm4 | |
859 | ||
d8725b1f | 860 | #ifndef USE_SSSE3 |
7956a3d2 L |
861 | psrldq $5, %xmm3 |
862 | pslldq $11, %xmm2 | |
0fda545d | 863 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 864 | #else |
0fda545d | 865 | palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 866 | #endif |
42e08a54 | 867 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
868 | |
869 | pcmpeqb %xmm1, %xmm0 | |
870 | pcmpeqb %xmm2, %xmm1 | |
871 | psubb %xmm0, %xmm1 | |
872 | pmovmskb %xmm1, %edx | |
873 | sub $0xffff, %edx | |
874 | jnz LABEL(exit) | |
875 | ||
d8725b1f | 876 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
877 | sub $16, %r11 |
878 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 879 | #endif |
7956a3d2 L |
880 | |
881 | add $16, %rcx | |
882 | movdqa %xmm4, %xmm3 | |
883 | ||
884 | add $16, %r10 | |
885 | jg LABEL(nibble_ashr_5) /* cross page boundary */ | |
886 | ||
887 | movdqa (%rsi, %rcx), %xmm1 | |
888 | movdqa (%rdi, %rcx), %xmm2 | |
889 | movdqa %xmm2, %xmm4 | |
890 | ||
d8725b1f | 891 | #ifndef USE_SSSE3 |
7956a3d2 | 892 | psrldq $5, %xmm3 |
0fda545d UD |
893 | pslldq $11, %xmm2 |
894 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 895 | #else |
0fda545d | 896 | palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 897 | #endif |
42e08a54 | 898 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
899 | |
900 | pcmpeqb %xmm1, %xmm0 | |
901 | pcmpeqb %xmm2, %xmm1 | |
902 | psubb %xmm0, %xmm1 | |
903 | pmovmskb %xmm1, %edx | |
904 | sub $0xffff, %edx | |
905 | jnz LABEL(exit) | |
906 | ||
d8725b1f | 907 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
908 | sub $16, %r11 |
909 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 910 | #endif |
7956a3d2 L |
911 | |
912 | add $16, %rcx | |
913 | movdqa %xmm4, %xmm3 | |
914 | jmp LABEL(loop_ashr_5) | |
915 | ||
916 | .p2align 4 | |
917 | LABEL(nibble_ashr_5): | |
918 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
919 | pmovmskb %xmm0, %edx | |
920 | test $0xffe0, %edx | |
921 | jnz LABEL(ashr_5_exittail) | |
922 | ||
d8725b1f | 923 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 924 | cmp $11, %r11 |
7956a3d2 | 925 | jbe LABEL(ashr_5_exittail) |
d8725b1f | 926 | #endif |
7956a3d2 L |
927 | |
928 | pxor %xmm0, %xmm0 | |
929 | sub $0x1000, %r10 | |
930 | jmp LABEL(gobble_ashr_5) | |
931 | ||
932 | .p2align 4 | |
933 | LABEL(ashr_5_exittail): | |
934 | movdqa (%rsi, %rcx), %xmm1 | |
935 | psrldq $5, %xmm0 | |
936 | psrldq $5, %xmm3 | |
937 | jmp LABEL(aftertail) | |
938 | ||
939 | /* | |
940 | * The following cases will be handled by ashr_6 | |
941 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 942 | * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6 |
7956a3d2 L |
943 | */ |
944 | .p2align 4 | |
945 | LABEL(ashr_6): | |
946 | pxor %xmm0, %xmm0 | |
947 | movdqa (%rdi), %xmm2 | |
948 | movdqa (%rsi), %xmm1 | |
949 | pcmpeqb %xmm1, %xmm0 | |
950 | pslldq $10, %xmm2 | |
42e08a54 | 951 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
952 | pcmpeqb %xmm1, %xmm2 |
953 | psubb %xmm0, %xmm2 | |
954 | pmovmskb %xmm2, %r9d | |
955 | shr %cl, %edx | |
956 | shr %cl, %r9d | |
957 | sub %r9d, %edx | |
958 | jnz LABEL(less32bytes) | |
959 | movdqa (%rdi), %xmm3 | |
960 | ||
961 | UPDATE_STRNCMP_COUNTER | |
962 | ||
963 | pxor %xmm0, %xmm0 | |
964 | mov $16, %rcx /* index for loads */ | |
965 | mov $6, %r9d /* byte position left over from less32bytes case */ | |
966 | /* | |
967 | * Setup %r10 value allows us to detect crossing a page boundary. | |
968 | * When %r10 goes positive we have crossed a page boundary and | |
969 | * need to do a nibble. | |
970 | */ | |
971 | lea 6(%rdi), %r10 | |
972 | and $0xfff, %r10 /* offset into 4K page */ | |
973 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
974 | ||
975 | .p2align 4 | |
976 | LABEL(loop_ashr_6): | |
977 | add $16, %r10 | |
978 | jg LABEL(nibble_ashr_6) | |
979 | ||
980 | LABEL(gobble_ashr_6): | |
981 | movdqa (%rsi, %rcx), %xmm1 | |
982 | movdqa (%rdi, %rcx), %xmm2 | |
983 | movdqa %xmm2, %xmm4 | |
984 | ||
d8725b1f | 985 | #ifndef USE_SSSE3 |
7956a3d2 L |
986 | psrldq $6, %xmm3 |
987 | pslldq $10, %xmm2 | |
0fda545d | 988 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 989 | #else |
0fda545d | 990 | palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 991 | #endif |
42e08a54 | 992 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
993 | |
994 | pcmpeqb %xmm1, %xmm0 | |
995 | pcmpeqb %xmm2, %xmm1 | |
996 | psubb %xmm0, %xmm1 | |
997 | pmovmskb %xmm1, %edx | |
998 | sub $0xffff, %edx | |
999 | jnz LABEL(exit) | |
1000 | ||
d8725b1f | 1001 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1002 | sub $16, %r11 |
1003 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1004 | #endif |
7956a3d2 L |
1005 | |
1006 | add $16, %rcx | |
1007 | movdqa %xmm4, %xmm3 | |
1008 | ||
1009 | add $16, %r10 | |
1010 | jg LABEL(nibble_ashr_6) /* cross page boundary */ | |
1011 | ||
1012 | movdqa (%rsi, %rcx), %xmm1 | |
1013 | movdqa (%rdi, %rcx), %xmm2 | |
1014 | movdqa %xmm2, %xmm4 | |
1015 | ||
d8725b1f | 1016 | #ifndef USE_SSSE3 |
7956a3d2 | 1017 | psrldq $6, %xmm3 |
0fda545d UD |
1018 | pslldq $10, %xmm2 |
1019 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1020 | #else |
0fda545d | 1021 | palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1022 | #endif |
42e08a54 | 1023 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1024 | |
1025 | pcmpeqb %xmm1, %xmm0 | |
1026 | pcmpeqb %xmm2, %xmm1 | |
1027 | psubb %xmm0, %xmm1 | |
1028 | pmovmskb %xmm1, %edx | |
1029 | sub $0xffff, %edx | |
1030 | jnz LABEL(exit) | |
1031 | ||
d8725b1f | 1032 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1033 | sub $16, %r11 |
1034 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1035 | #endif |
7956a3d2 L |
1036 | |
1037 | add $16, %rcx | |
1038 | movdqa %xmm4, %xmm3 | |
1039 | jmp LABEL(loop_ashr_6) | |
1040 | ||
1041 | .p2align 4 | |
1042 | LABEL(nibble_ashr_6): | |
1043 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1044 | pmovmskb %xmm0, %edx | |
1045 | test $0xffc0, %edx | |
1046 | jnz LABEL(ashr_6_exittail) | |
1047 | ||
d8725b1f | 1048 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1049 | cmp $10, %r11 |
7956a3d2 | 1050 | jbe LABEL(ashr_6_exittail) |
d8725b1f | 1051 | #endif |
7956a3d2 L |
1052 | |
1053 | pxor %xmm0, %xmm0 | |
1054 | sub $0x1000, %r10 | |
1055 | jmp LABEL(gobble_ashr_6) | |
1056 | ||
1057 | .p2align 4 | |
1058 | LABEL(ashr_6_exittail): | |
1059 | movdqa (%rsi, %rcx), %xmm1 | |
1060 | psrldq $6, %xmm0 | |
1061 | psrldq $6, %xmm3 | |
1062 | jmp LABEL(aftertail) | |
1063 | ||
1064 | /* | |
1065 | * The following cases will be handled by ashr_7 | |
1066 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1067 | * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7 |
7956a3d2 L |
1068 | */ |
1069 | .p2align 4 | |
1070 | LABEL(ashr_7): | |
1071 | pxor %xmm0, %xmm0 | |
1072 | movdqa (%rdi), %xmm2 | |
1073 | movdqa (%rsi), %xmm1 | |
1074 | pcmpeqb %xmm1, %xmm0 | |
1075 | pslldq $9, %xmm2 | |
42e08a54 | 1076 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1077 | pcmpeqb %xmm1, %xmm2 |
1078 | psubb %xmm0, %xmm2 | |
1079 | pmovmskb %xmm2, %r9d | |
1080 | shr %cl, %edx | |
1081 | shr %cl, %r9d | |
1082 | sub %r9d, %edx | |
1083 | jnz LABEL(less32bytes) | |
1084 | movdqa (%rdi), %xmm3 | |
1085 | ||
1086 | UPDATE_STRNCMP_COUNTER | |
1087 | ||
1088 | pxor %xmm0, %xmm0 | |
1089 | mov $16, %rcx /* index for loads */ | |
1090 | mov $7, %r9d /* byte position left over from less32bytes case */ | |
1091 | /* | |
1092 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1093 | * When %r10 goes positive we have crossed a page boundary and | |
1094 | * need to do a nibble. | |
1095 | */ | |
1096 | lea 7(%rdi), %r10 | |
1097 | and $0xfff, %r10 /* offset into 4K page */ | |
1098 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1099 | ||
1100 | .p2align 4 | |
1101 | LABEL(loop_ashr_7): | |
1102 | add $16, %r10 | |
1103 | jg LABEL(nibble_ashr_7) | |
1104 | ||
1105 | LABEL(gobble_ashr_7): | |
1106 | movdqa (%rsi, %rcx), %xmm1 | |
1107 | movdqa (%rdi, %rcx), %xmm2 | |
1108 | movdqa %xmm2, %xmm4 | |
1109 | ||
d8725b1f | 1110 | #ifndef USE_SSSE3 |
7956a3d2 L |
1111 | psrldq $7, %xmm3 |
1112 | pslldq $9, %xmm2 | |
0fda545d | 1113 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1114 | #else |
0fda545d | 1115 | palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1116 | #endif |
42e08a54 | 1117 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1118 | |
1119 | pcmpeqb %xmm1, %xmm0 | |
1120 | pcmpeqb %xmm2, %xmm1 | |
1121 | psubb %xmm0, %xmm1 | |
1122 | pmovmskb %xmm1, %edx | |
1123 | sub $0xffff, %edx | |
1124 | jnz LABEL(exit) | |
1125 | ||
d8725b1f | 1126 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1127 | sub $16, %r11 |
1128 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1129 | #endif |
7956a3d2 L |
1130 | |
1131 | add $16, %rcx | |
1132 | movdqa %xmm4, %xmm3 | |
1133 | ||
1134 | add $16, %r10 | |
1135 | jg LABEL(nibble_ashr_7) /* cross page boundary */ | |
1136 | ||
1137 | movdqa (%rsi, %rcx), %xmm1 | |
1138 | movdqa (%rdi, %rcx), %xmm2 | |
1139 | movdqa %xmm2, %xmm4 | |
1140 | ||
d8725b1f | 1141 | #ifndef USE_SSSE3 |
7956a3d2 | 1142 | psrldq $7, %xmm3 |
0fda545d UD |
1143 | pslldq $9, %xmm2 |
1144 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1145 | #else |
0fda545d | 1146 | palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1147 | #endif |
42e08a54 | 1148 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1149 | |
1150 | pcmpeqb %xmm1, %xmm0 | |
1151 | pcmpeqb %xmm2, %xmm1 | |
1152 | psubb %xmm0, %xmm1 | |
1153 | pmovmskb %xmm1, %edx | |
1154 | sub $0xffff, %edx | |
1155 | jnz LABEL(exit) | |
1156 | ||
d8725b1f | 1157 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1158 | sub $16, %r11 |
1159 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1160 | #endif |
7956a3d2 L |
1161 | |
1162 | add $16, %rcx | |
1163 | movdqa %xmm4, %xmm3 | |
1164 | jmp LABEL(loop_ashr_7) | |
1165 | ||
1166 | .p2align 4 | |
1167 | LABEL(nibble_ashr_7): | |
1168 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1169 | pmovmskb %xmm0, %edx | |
1170 | test $0xff80, %edx | |
1171 | jnz LABEL(ashr_7_exittail) | |
1172 | ||
d8725b1f | 1173 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1174 | cmp $9, %r11 |
7956a3d2 | 1175 | jbe LABEL(ashr_7_exittail) |
d8725b1f | 1176 | #endif |
7956a3d2 L |
1177 | |
1178 | pxor %xmm0, %xmm0 | |
1179 | sub $0x1000, %r10 | |
1180 | jmp LABEL(gobble_ashr_7) | |
1181 | ||
1182 | .p2align 4 | |
1183 | LABEL(ashr_7_exittail): | |
1184 | movdqa (%rsi, %rcx), %xmm1 | |
1185 | psrldq $7, %xmm0 | |
1186 | psrldq $7, %xmm3 | |
1187 | jmp LABEL(aftertail) | |
1188 | ||
1189 | /* | |
1190 | * The following cases will be handled by ashr_8 | |
1191 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1192 | * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8 |
7956a3d2 L |
1193 | */ |
1194 | .p2align 4 | |
1195 | LABEL(ashr_8): | |
1196 | pxor %xmm0, %xmm0 | |
1197 | movdqa (%rdi), %xmm2 | |
1198 | movdqa (%rsi), %xmm1 | |
1199 | pcmpeqb %xmm1, %xmm0 | |
1200 | pslldq $8, %xmm2 | |
42e08a54 | 1201 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1202 | pcmpeqb %xmm1, %xmm2 |
1203 | psubb %xmm0, %xmm2 | |
1204 | pmovmskb %xmm2, %r9d | |
1205 | shr %cl, %edx | |
1206 | shr %cl, %r9d | |
1207 | sub %r9d, %edx | |
1208 | jnz LABEL(less32bytes) | |
1209 | movdqa (%rdi), %xmm3 | |
1210 | ||
1211 | UPDATE_STRNCMP_COUNTER | |
1212 | ||
1213 | pxor %xmm0, %xmm0 | |
1214 | mov $16, %rcx /* index for loads */ | |
1215 | mov $8, %r9d /* byte position left over from less32bytes case */ | |
1216 | /* | |
1217 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1218 | * When %r10 goes positive we have crossed a page boundary and | |
1219 | * need to do a nibble. | |
1220 | */ | |
1221 | lea 8(%rdi), %r10 | |
1222 | and $0xfff, %r10 /* offset into 4K page */ | |
1223 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1224 | ||
1225 | .p2align 4 | |
1226 | LABEL(loop_ashr_8): | |
1227 | add $16, %r10 | |
1228 | jg LABEL(nibble_ashr_8) | |
1229 | ||
1230 | LABEL(gobble_ashr_8): | |
1231 | movdqa (%rsi, %rcx), %xmm1 | |
1232 | movdqa (%rdi, %rcx), %xmm2 | |
1233 | movdqa %xmm2, %xmm4 | |
1234 | ||
d8725b1f | 1235 | #ifndef USE_SSSE3 |
7956a3d2 L |
1236 | psrldq $8, %xmm3 |
1237 | pslldq $8, %xmm2 | |
0fda545d | 1238 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1239 | #else |
0fda545d | 1240 | palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1241 | #endif |
42e08a54 | 1242 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1243 | |
1244 | pcmpeqb %xmm1, %xmm0 | |
1245 | pcmpeqb %xmm2, %xmm1 | |
1246 | psubb %xmm0, %xmm1 | |
1247 | pmovmskb %xmm1, %edx | |
1248 | sub $0xffff, %edx | |
1249 | jnz LABEL(exit) | |
1250 | ||
d8725b1f | 1251 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1252 | sub $16, %r11 |
1253 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1254 | #endif |
7956a3d2 L |
1255 | |
1256 | add $16, %rcx | |
1257 | movdqa %xmm4, %xmm3 | |
1258 | ||
1259 | add $16, %r10 | |
1260 | jg LABEL(nibble_ashr_8) /* cross page boundary */ | |
1261 | ||
1262 | movdqa (%rsi, %rcx), %xmm1 | |
1263 | movdqa (%rdi, %rcx), %xmm2 | |
1264 | movdqa %xmm2, %xmm4 | |
1265 | ||
d8725b1f | 1266 | #ifndef USE_SSSE3 |
7956a3d2 | 1267 | psrldq $8, %xmm3 |
0fda545d UD |
1268 | pslldq $8, %xmm2 |
1269 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1270 | #else |
0fda545d | 1271 | palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1272 | #endif |
42e08a54 | 1273 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1274 | |
1275 | pcmpeqb %xmm1, %xmm0 | |
1276 | pcmpeqb %xmm2, %xmm1 | |
1277 | psubb %xmm0, %xmm1 | |
1278 | pmovmskb %xmm1, %edx | |
1279 | sub $0xffff, %edx | |
1280 | jnz LABEL(exit) | |
1281 | ||
d8725b1f | 1282 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1283 | sub $16, %r11 |
1284 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1285 | #endif |
7956a3d2 L |
1286 | |
1287 | add $16, %rcx | |
1288 | movdqa %xmm4, %xmm3 | |
1289 | jmp LABEL(loop_ashr_8) | |
1290 | ||
1291 | .p2align 4 | |
1292 | LABEL(nibble_ashr_8): | |
1293 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1294 | pmovmskb %xmm0, %edx | |
1295 | test $0xff00, %edx | |
1296 | jnz LABEL(ashr_8_exittail) | |
1297 | ||
d8725b1f | 1298 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1299 | cmp $8, %r11 |
7956a3d2 | 1300 | jbe LABEL(ashr_8_exittail) |
d8725b1f | 1301 | #endif |
7956a3d2 L |
1302 | |
1303 | pxor %xmm0, %xmm0 | |
1304 | sub $0x1000, %r10 | |
1305 | jmp LABEL(gobble_ashr_8) | |
1306 | ||
1307 | .p2align 4 | |
1308 | LABEL(ashr_8_exittail): | |
1309 | movdqa (%rsi, %rcx), %xmm1 | |
1310 | psrldq $8, %xmm0 | |
1311 | psrldq $8, %xmm3 | |
1312 | jmp LABEL(aftertail) | |
1313 | ||
1314 | /* | |
1315 | * The following cases will be handled by ashr_9 | |
1316 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1317 | * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9 |
7956a3d2 L |
1318 | */ |
1319 | .p2align 4 | |
1320 | LABEL(ashr_9): | |
1321 | pxor %xmm0, %xmm0 | |
1322 | movdqa (%rdi), %xmm2 | |
1323 | movdqa (%rsi), %xmm1 | |
1324 | pcmpeqb %xmm1, %xmm0 | |
1325 | pslldq $7, %xmm2 | |
42e08a54 | 1326 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1327 | pcmpeqb %xmm1, %xmm2 |
1328 | psubb %xmm0, %xmm2 | |
1329 | pmovmskb %xmm2, %r9d | |
1330 | shr %cl, %edx | |
1331 | shr %cl, %r9d | |
1332 | sub %r9d, %edx | |
1333 | jnz LABEL(less32bytes) | |
1334 | movdqa (%rdi), %xmm3 | |
1335 | ||
1336 | UPDATE_STRNCMP_COUNTER | |
1337 | ||
1338 | pxor %xmm0, %xmm0 | |
1339 | mov $16, %rcx /* index for loads */ | |
1340 | mov $9, %r9d /* byte position left over from less32bytes case */ | |
1341 | /* | |
1342 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1343 | * When %r10 goes positive we have crossed a page boundary and | |
1344 | * need to do a nibble. | |
1345 | */ | |
1346 | lea 9(%rdi), %r10 | |
1347 | and $0xfff, %r10 /* offset into 4K page */ | |
1348 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1349 | ||
1350 | .p2align 4 | |
1351 | LABEL(loop_ashr_9): | |
1352 | add $16, %r10 | |
1353 | jg LABEL(nibble_ashr_9) | |
1354 | ||
1355 | LABEL(gobble_ashr_9): | |
1356 | movdqa (%rsi, %rcx), %xmm1 | |
1357 | movdqa (%rdi, %rcx), %xmm2 | |
1358 | movdqa %xmm2, %xmm4 | |
1359 | ||
d8725b1f | 1360 | #ifndef USE_SSSE3 |
7956a3d2 L |
1361 | psrldq $9, %xmm3 |
1362 | pslldq $7, %xmm2 | |
0fda545d | 1363 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1364 | #else |
0fda545d | 1365 | palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1366 | #endif |
42e08a54 | 1367 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1368 | |
1369 | pcmpeqb %xmm1, %xmm0 | |
1370 | pcmpeqb %xmm2, %xmm1 | |
1371 | psubb %xmm0, %xmm1 | |
1372 | pmovmskb %xmm1, %edx | |
1373 | sub $0xffff, %edx | |
1374 | jnz LABEL(exit) | |
1375 | ||
d8725b1f | 1376 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1377 | sub $16, %r11 |
1378 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1379 | #endif |
7956a3d2 L |
1380 | |
1381 | add $16, %rcx | |
1382 | movdqa %xmm4, %xmm3 | |
1383 | ||
1384 | add $16, %r10 | |
1385 | jg LABEL(nibble_ashr_9) /* cross page boundary */ | |
1386 | ||
1387 | movdqa (%rsi, %rcx), %xmm1 | |
1388 | movdqa (%rdi, %rcx), %xmm2 | |
1389 | movdqa %xmm2, %xmm4 | |
1390 | ||
d8725b1f | 1391 | #ifndef USE_SSSE3 |
7956a3d2 | 1392 | psrldq $9, %xmm3 |
0fda545d UD |
1393 | pslldq $7, %xmm2 |
1394 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1395 | #else |
0fda545d | 1396 | palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1397 | #endif |
42e08a54 | 1398 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1399 | |
1400 | pcmpeqb %xmm1, %xmm0 | |
1401 | pcmpeqb %xmm2, %xmm1 | |
1402 | psubb %xmm0, %xmm1 | |
1403 | pmovmskb %xmm1, %edx | |
1404 | sub $0xffff, %edx | |
1405 | jnz LABEL(exit) | |
1406 | ||
d8725b1f | 1407 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1408 | sub $16, %r11 |
1409 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1410 | #endif |
7956a3d2 L |
1411 | |
1412 | add $16, %rcx | |
1413 | movdqa %xmm4, %xmm3 /* store for next cycle */ | |
1414 | jmp LABEL(loop_ashr_9) | |
1415 | ||
1416 | .p2align 4 | |
1417 | LABEL(nibble_ashr_9): | |
1418 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1419 | pmovmskb %xmm0, %edx | |
1420 | test $0xfe00, %edx | |
1421 | jnz LABEL(ashr_9_exittail) | |
1422 | ||
d8725b1f | 1423 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1424 | cmp $7, %r11 |
7956a3d2 | 1425 | jbe LABEL(ashr_9_exittail) |
d8725b1f | 1426 | #endif |
7956a3d2 L |
1427 | |
1428 | pxor %xmm0, %xmm0 | |
1429 | sub $0x1000, %r10 | |
1430 | jmp LABEL(gobble_ashr_9) | |
1431 | ||
1432 | .p2align 4 | |
1433 | LABEL(ashr_9_exittail): | |
1434 | movdqa (%rsi, %rcx), %xmm1 | |
1435 | psrldq $9, %xmm0 | |
1436 | psrldq $9, %xmm3 | |
1437 | jmp LABEL(aftertail) | |
1438 | ||
1439 | /* | |
1440 | * The following cases will be handled by ashr_10 | |
1441 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1442 | * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10 |
7956a3d2 L |
1443 | */ |
1444 | .p2align 4 | |
1445 | LABEL(ashr_10): | |
1446 | pxor %xmm0, %xmm0 | |
1447 | movdqa (%rdi), %xmm2 | |
1448 | movdqa (%rsi), %xmm1 | |
1449 | pcmpeqb %xmm1, %xmm0 | |
1450 | pslldq $6, %xmm2 | |
42e08a54 | 1451 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1452 | pcmpeqb %xmm1, %xmm2 |
1453 | psubb %xmm0, %xmm2 | |
1454 | pmovmskb %xmm2, %r9d | |
1455 | shr %cl, %edx | |
1456 | shr %cl, %r9d | |
1457 | sub %r9d, %edx | |
1458 | jnz LABEL(less32bytes) | |
1459 | movdqa (%rdi), %xmm3 | |
1460 | ||
1461 | UPDATE_STRNCMP_COUNTER | |
1462 | ||
1463 | pxor %xmm0, %xmm0 | |
1464 | mov $16, %rcx /* index for loads */ | |
1465 | mov $10, %r9d /* byte position left over from less32bytes case */ | |
1466 | /* | |
1467 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1468 | * When %r10 goes positive we have crossed a page boundary and | |
1469 | * need to do a nibble. | |
1470 | */ | |
1471 | lea 10(%rdi), %r10 | |
1472 | and $0xfff, %r10 /* offset into 4K page */ | |
1473 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1474 | ||
1475 | .p2align 4 | |
1476 | LABEL(loop_ashr_10): | |
1477 | add $16, %r10 | |
1478 | jg LABEL(nibble_ashr_10) | |
1479 | ||
1480 | LABEL(gobble_ashr_10): | |
1481 | movdqa (%rsi, %rcx), %xmm1 | |
1482 | movdqa (%rdi, %rcx), %xmm2 | |
1483 | movdqa %xmm2, %xmm4 | |
1484 | ||
d8725b1f | 1485 | #ifndef USE_SSSE3 |
7956a3d2 L |
1486 | psrldq $10, %xmm3 |
1487 | pslldq $6, %xmm2 | |
0fda545d | 1488 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1489 | #else |
0fda545d | 1490 | palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1491 | #endif |
42e08a54 | 1492 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1493 | |
1494 | pcmpeqb %xmm1, %xmm0 | |
1495 | pcmpeqb %xmm2, %xmm1 | |
1496 | psubb %xmm0, %xmm1 | |
1497 | pmovmskb %xmm1, %edx | |
1498 | sub $0xffff, %edx | |
1499 | jnz LABEL(exit) | |
1500 | ||
d8725b1f | 1501 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1502 | sub $16, %r11 |
1503 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1504 | #endif |
7956a3d2 L |
1505 | |
1506 | add $16, %rcx | |
1507 | movdqa %xmm4, %xmm3 | |
1508 | ||
1509 | add $16, %r10 | |
1510 | jg LABEL(nibble_ashr_10) /* cross page boundary */ | |
1511 | ||
1512 | movdqa (%rsi, %rcx), %xmm1 | |
1513 | movdqa (%rdi, %rcx), %xmm2 | |
1514 | movdqa %xmm2, %xmm4 | |
1515 | ||
d8725b1f | 1516 | #ifndef USE_SSSE3 |
7956a3d2 | 1517 | psrldq $10, %xmm3 |
0fda545d UD |
1518 | pslldq $6, %xmm2 |
1519 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1520 | #else |
0fda545d | 1521 | palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1522 | #endif |
42e08a54 | 1523 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1524 | |
1525 | pcmpeqb %xmm1, %xmm0 | |
1526 | pcmpeqb %xmm2, %xmm1 | |
1527 | psubb %xmm0, %xmm1 | |
1528 | pmovmskb %xmm1, %edx | |
1529 | sub $0xffff, %edx | |
1530 | jnz LABEL(exit) | |
1531 | ||
d8725b1f | 1532 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1533 | sub $16, %r11 |
1534 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1535 | #endif |
7956a3d2 L |
1536 | |
1537 | add $16, %rcx | |
1538 | movdqa %xmm4, %xmm3 | |
1539 | jmp LABEL(loop_ashr_10) | |
1540 | ||
1541 | .p2align 4 | |
1542 | LABEL(nibble_ashr_10): | |
1543 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1544 | pmovmskb %xmm0, %edx | |
1545 | test $0xfc00, %edx | |
1546 | jnz LABEL(ashr_10_exittail) | |
1547 | ||
d8725b1f | 1548 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1549 | cmp $6, %r11 |
7956a3d2 | 1550 | jbe LABEL(ashr_10_exittail) |
d8725b1f | 1551 | #endif |
7956a3d2 L |
1552 | |
1553 | pxor %xmm0, %xmm0 | |
1554 | sub $0x1000, %r10 | |
1555 | jmp LABEL(gobble_ashr_10) | |
1556 | ||
1557 | .p2align 4 | |
1558 | LABEL(ashr_10_exittail): | |
1559 | movdqa (%rsi, %rcx), %xmm1 | |
1560 | psrldq $10, %xmm0 | |
1561 | psrldq $10, %xmm3 | |
1562 | jmp LABEL(aftertail) | |
1563 | ||
1564 | /* | |
1565 | * The following cases will be handled by ashr_11 | |
1566 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1567 | * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11 |
7956a3d2 L |
1568 | */ |
1569 | .p2align 4 | |
1570 | LABEL(ashr_11): | |
1571 | pxor %xmm0, %xmm0 | |
1572 | movdqa (%rdi), %xmm2 | |
1573 | movdqa (%rsi), %xmm1 | |
1574 | pcmpeqb %xmm1, %xmm0 | |
1575 | pslldq $5, %xmm2 | |
42e08a54 | 1576 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1577 | pcmpeqb %xmm1, %xmm2 |
1578 | psubb %xmm0, %xmm2 | |
1579 | pmovmskb %xmm2, %r9d | |
1580 | shr %cl, %edx | |
1581 | shr %cl, %r9d | |
1582 | sub %r9d, %edx | |
1583 | jnz LABEL(less32bytes) | |
1584 | movdqa (%rdi), %xmm3 | |
1585 | ||
1586 | UPDATE_STRNCMP_COUNTER | |
1587 | ||
1588 | pxor %xmm0, %xmm0 | |
1589 | mov $16, %rcx /* index for loads */ | |
1590 | mov $11, %r9d /* byte position left over from less32bytes case */ | |
1591 | /* | |
1592 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1593 | * When %r10 goes positive we have crossed a page boundary and | |
1594 | * need to do a nibble. | |
1595 | */ | |
1596 | lea 11(%rdi), %r10 | |
1597 | and $0xfff, %r10 /* offset into 4K page */ | |
1598 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1599 | ||
1600 | .p2align 4 | |
1601 | LABEL(loop_ashr_11): | |
1602 | add $16, %r10 | |
1603 | jg LABEL(nibble_ashr_11) | |
1604 | ||
1605 | LABEL(gobble_ashr_11): | |
1606 | movdqa (%rsi, %rcx), %xmm1 | |
1607 | movdqa (%rdi, %rcx), %xmm2 | |
1608 | movdqa %xmm2, %xmm4 | |
1609 | ||
d8725b1f | 1610 | #ifndef USE_SSSE3 |
7956a3d2 L |
1611 | psrldq $11, %xmm3 |
1612 | pslldq $5, %xmm2 | |
0fda545d | 1613 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1614 | #else |
0fda545d | 1615 | palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1616 | #endif |
42e08a54 | 1617 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1618 | |
1619 | pcmpeqb %xmm1, %xmm0 | |
1620 | pcmpeqb %xmm2, %xmm1 | |
1621 | psubb %xmm0, %xmm1 | |
1622 | pmovmskb %xmm1, %edx | |
1623 | sub $0xffff, %edx | |
1624 | jnz LABEL(exit) | |
1625 | ||
d8725b1f | 1626 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1627 | sub $16, %r11 |
1628 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1629 | #endif |
7956a3d2 L |
1630 | |
1631 | add $16, %rcx | |
1632 | movdqa %xmm4, %xmm3 | |
1633 | ||
1634 | add $16, %r10 | |
1635 | jg LABEL(nibble_ashr_11) /* cross page boundary */ | |
1636 | ||
1637 | movdqa (%rsi, %rcx), %xmm1 | |
1638 | movdqa (%rdi, %rcx), %xmm2 | |
1639 | movdqa %xmm2, %xmm4 | |
1640 | ||
d8725b1f | 1641 | #ifndef USE_SSSE3 |
7956a3d2 | 1642 | psrldq $11, %xmm3 |
0fda545d UD |
1643 | pslldq $5, %xmm2 |
1644 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1645 | #else |
0fda545d | 1646 | palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1647 | #endif |
42e08a54 | 1648 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1649 | |
1650 | pcmpeqb %xmm1, %xmm0 | |
1651 | pcmpeqb %xmm2, %xmm1 | |
1652 | psubb %xmm0, %xmm1 | |
1653 | pmovmskb %xmm1, %edx | |
1654 | sub $0xffff, %edx | |
1655 | jnz LABEL(exit) | |
1656 | ||
d8725b1f | 1657 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1658 | sub $16, %r11 |
1659 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1660 | #endif |
7956a3d2 L |
1661 | |
1662 | add $16, %rcx | |
1663 | movdqa %xmm4, %xmm3 | |
1664 | jmp LABEL(loop_ashr_11) | |
1665 | ||
1666 | .p2align 4 | |
1667 | LABEL(nibble_ashr_11): | |
1668 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1669 | pmovmskb %xmm0, %edx | |
1670 | test $0xf800, %edx | |
1671 | jnz LABEL(ashr_11_exittail) | |
1672 | ||
d8725b1f | 1673 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1674 | cmp $5, %r11 |
7956a3d2 | 1675 | jbe LABEL(ashr_11_exittail) |
d8725b1f | 1676 | #endif |
7956a3d2 L |
1677 | |
1678 | pxor %xmm0, %xmm0 | |
1679 | sub $0x1000, %r10 | |
1680 | jmp LABEL(gobble_ashr_11) | |
1681 | ||
1682 | .p2align 4 | |
1683 | LABEL(ashr_11_exittail): | |
1684 | movdqa (%rsi, %rcx), %xmm1 | |
1685 | psrldq $11, %xmm0 | |
1686 | psrldq $11, %xmm3 | |
1687 | jmp LABEL(aftertail) | |
1688 | ||
1689 | /* | |
1690 | * The following cases will be handled by ashr_12 | |
1691 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1692 | * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12 |
7956a3d2 L |
1693 | */ |
1694 | .p2align 4 | |
1695 | LABEL(ashr_12): | |
1696 | pxor %xmm0, %xmm0 | |
1697 | movdqa (%rdi), %xmm2 | |
1698 | movdqa (%rsi), %xmm1 | |
1699 | pcmpeqb %xmm1, %xmm0 | |
1700 | pslldq $4, %xmm2 | |
42e08a54 | 1701 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1702 | pcmpeqb %xmm1, %xmm2 |
1703 | psubb %xmm0, %xmm2 | |
1704 | pmovmskb %xmm2, %r9d | |
1705 | shr %cl, %edx | |
1706 | shr %cl, %r9d | |
1707 | sub %r9d, %edx | |
1708 | jnz LABEL(less32bytes) | |
1709 | movdqa (%rdi), %xmm3 | |
1710 | ||
1711 | UPDATE_STRNCMP_COUNTER | |
1712 | ||
1713 | pxor %xmm0, %xmm0 | |
1714 | mov $16, %rcx /* index for loads */ | |
1715 | mov $12, %r9d /* byte position left over from less32bytes case */ | |
1716 | /* | |
1717 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1718 | * When %r10 goes positive we have crossed a page boundary and | |
1719 | * need to do a nibble. | |
1720 | */ | |
1721 | lea 12(%rdi), %r10 | |
1722 | and $0xfff, %r10 /* offset into 4K page */ | |
1723 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1724 | ||
1725 | .p2align 4 | |
1726 | LABEL(loop_ashr_12): | |
1727 | add $16, %r10 | |
1728 | jg LABEL(nibble_ashr_12) | |
1729 | ||
1730 | LABEL(gobble_ashr_12): | |
1731 | movdqa (%rsi, %rcx), %xmm1 | |
1732 | movdqa (%rdi, %rcx), %xmm2 | |
1733 | movdqa %xmm2, %xmm4 | |
1734 | ||
d8725b1f | 1735 | #ifndef USE_SSSE3 |
7956a3d2 L |
1736 | psrldq $12, %xmm3 |
1737 | pslldq $4, %xmm2 | |
0fda545d | 1738 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1739 | #else |
0fda545d | 1740 | palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1741 | #endif |
42e08a54 | 1742 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1743 | |
1744 | pcmpeqb %xmm1, %xmm0 | |
1745 | pcmpeqb %xmm2, %xmm1 | |
1746 | psubb %xmm0, %xmm1 | |
1747 | pmovmskb %xmm1, %edx | |
1748 | sub $0xffff, %edx | |
1749 | jnz LABEL(exit) | |
1750 | ||
d8725b1f | 1751 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1752 | sub $16, %r11 |
1753 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1754 | #endif |
7956a3d2 L |
1755 | |
1756 | add $16, %rcx | |
1757 | movdqa %xmm4, %xmm3 | |
1758 | ||
1759 | add $16, %r10 | |
1760 | jg LABEL(nibble_ashr_12) /* cross page boundary */ | |
1761 | ||
1762 | movdqa (%rsi, %rcx), %xmm1 | |
1763 | movdqa (%rdi, %rcx), %xmm2 | |
1764 | movdqa %xmm2, %xmm4 | |
1765 | ||
d8725b1f | 1766 | #ifndef USE_SSSE3 |
7956a3d2 | 1767 | psrldq $12, %xmm3 |
0fda545d UD |
1768 | pslldq $4, %xmm2 |
1769 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1770 | #else |
0fda545d | 1771 | palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1772 | #endif |
42e08a54 | 1773 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1774 | |
1775 | pcmpeqb %xmm1, %xmm0 | |
1776 | pcmpeqb %xmm2, %xmm1 | |
1777 | psubb %xmm0, %xmm1 | |
1778 | pmovmskb %xmm1, %edx | |
1779 | sub $0xffff, %edx | |
1780 | jnz LABEL(exit) | |
1781 | ||
d8725b1f | 1782 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1783 | sub $16, %r11 |
1784 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1785 | #endif |
7956a3d2 L |
1786 | |
1787 | add $16, %rcx | |
1788 | movdqa %xmm4, %xmm3 | |
1789 | jmp LABEL(loop_ashr_12) | |
1790 | ||
1791 | .p2align 4 | |
1792 | LABEL(nibble_ashr_12): | |
1793 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1794 | pmovmskb %xmm0, %edx | |
1795 | test $0xf000, %edx | |
1796 | jnz LABEL(ashr_12_exittail) | |
1797 | ||
d8725b1f | 1798 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1799 | cmp $4, %r11 |
7956a3d2 | 1800 | jbe LABEL(ashr_12_exittail) |
d8725b1f | 1801 | #endif |
7956a3d2 L |
1802 | |
1803 | pxor %xmm0, %xmm0 | |
1804 | sub $0x1000, %r10 | |
1805 | jmp LABEL(gobble_ashr_12) | |
1806 | ||
1807 | .p2align 4 | |
1808 | LABEL(ashr_12_exittail): | |
1809 | movdqa (%rsi, %rcx), %xmm1 | |
1810 | psrldq $12, %xmm0 | |
1811 | psrldq $12, %xmm3 | |
1812 | jmp LABEL(aftertail) | |
1813 | ||
1814 | /* | |
1815 | * The following cases will be handled by ashr_13 | |
1816 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1817 | * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13 |
7956a3d2 L |
1818 | */ |
1819 | .p2align 4 | |
1820 | LABEL(ashr_13): | |
1821 | pxor %xmm0, %xmm0 | |
1822 | movdqa (%rdi), %xmm2 | |
1823 | movdqa (%rsi), %xmm1 | |
1824 | pcmpeqb %xmm1, %xmm0 | |
1825 | pslldq $3, %xmm2 | |
42e08a54 | 1826 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1827 | pcmpeqb %xmm1, %xmm2 |
1828 | psubb %xmm0, %xmm2 | |
1829 | pmovmskb %xmm2, %r9d | |
1830 | shr %cl, %edx | |
1831 | shr %cl, %r9d | |
1832 | sub %r9d, %edx | |
1833 | jnz LABEL(less32bytes) | |
1834 | movdqa (%rdi), %xmm3 | |
1835 | ||
1836 | UPDATE_STRNCMP_COUNTER | |
1837 | ||
1838 | pxor %xmm0, %xmm0 | |
1839 | mov $16, %rcx /* index for loads */ | |
1840 | mov $13, %r9d /* byte position left over from less32bytes case */ | |
1841 | /* | |
1842 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1843 | * When %r10 goes positive we have crossed a page boundary and | |
1844 | * need to do a nibble. | |
1845 | */ | |
1846 | lea 13(%rdi), %r10 | |
1847 | and $0xfff, %r10 /* offset into 4K page */ | |
1848 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1849 | ||
1850 | .p2align 4 | |
1851 | LABEL(loop_ashr_13): | |
1852 | add $16, %r10 | |
1853 | jg LABEL(nibble_ashr_13) | |
1854 | ||
1855 | LABEL(gobble_ashr_13): | |
1856 | movdqa (%rsi, %rcx), %xmm1 | |
1857 | movdqa (%rdi, %rcx), %xmm2 | |
1858 | movdqa %xmm2, %xmm4 | |
1859 | ||
d8725b1f | 1860 | #ifndef USE_SSSE3 |
7956a3d2 L |
1861 | psrldq $13, %xmm3 |
1862 | pslldq $3, %xmm2 | |
0fda545d | 1863 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1864 | #else |
0fda545d | 1865 | palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1866 | #endif |
42e08a54 | 1867 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1868 | |
1869 | pcmpeqb %xmm1, %xmm0 | |
1870 | pcmpeqb %xmm2, %xmm1 | |
1871 | psubb %xmm0, %xmm1 | |
1872 | pmovmskb %xmm1, %edx | |
1873 | sub $0xffff, %edx | |
1874 | jnz LABEL(exit) | |
1875 | ||
d8725b1f | 1876 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1877 | sub $16, %r11 |
1878 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1879 | #endif |
7956a3d2 L |
1880 | |
1881 | add $16, %rcx | |
1882 | movdqa %xmm4, %xmm3 | |
1883 | ||
1884 | add $16, %r10 | |
1885 | jg LABEL(nibble_ashr_13) /* cross page boundary */ | |
1886 | ||
1887 | movdqa (%rsi, %rcx), %xmm1 | |
1888 | movdqa (%rdi, %rcx), %xmm2 | |
1889 | movdqa %xmm2, %xmm4 | |
1890 | ||
d8725b1f | 1891 | #ifndef USE_SSSE3 |
7956a3d2 | 1892 | psrldq $13, %xmm3 |
0fda545d UD |
1893 | pslldq $3, %xmm2 |
1894 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1895 | #else |
0fda545d | 1896 | palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1897 | #endif |
42e08a54 | 1898 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1899 | |
1900 | pcmpeqb %xmm1, %xmm0 | |
1901 | pcmpeqb %xmm2, %xmm1 | |
1902 | psubb %xmm0, %xmm1 | |
1903 | pmovmskb %xmm1, %edx | |
1904 | sub $0xffff, %edx | |
1905 | jnz LABEL(exit) | |
1906 | ||
d8725b1f | 1907 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1908 | sub $16, %r11 |
1909 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1910 | #endif |
7956a3d2 L |
1911 | |
1912 | add $16, %rcx | |
1913 | movdqa %xmm4, %xmm3 | |
1914 | jmp LABEL(loop_ashr_13) | |
1915 | ||
1916 | .p2align 4 | |
1917 | LABEL(nibble_ashr_13): | |
1918 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1919 | pmovmskb %xmm0, %edx | |
1920 | test $0xe000, %edx | |
1921 | jnz LABEL(ashr_13_exittail) | |
1922 | ||
d8725b1f | 1923 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1924 | cmp $3, %r11 |
7956a3d2 | 1925 | jbe LABEL(ashr_13_exittail) |
d8725b1f | 1926 | #endif |
7956a3d2 L |
1927 | |
1928 | pxor %xmm0, %xmm0 | |
1929 | sub $0x1000, %r10 | |
1930 | jmp LABEL(gobble_ashr_13) | |
1931 | ||
1932 | .p2align 4 | |
1933 | LABEL(ashr_13_exittail): | |
1934 | movdqa (%rsi, %rcx), %xmm1 | |
1935 | psrldq $13, %xmm0 | |
1936 | psrldq $13, %xmm3 | |
1937 | jmp LABEL(aftertail) | |
1938 | ||
1939 | /* | |
1940 | * The following cases will be handled by ashr_14 | |
1941 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1942 | * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14 |
7956a3d2 L |
1943 | */ |
1944 | .p2align 4 | |
1945 | LABEL(ashr_14): | |
1946 | pxor %xmm0, %xmm0 | |
1947 | movdqa (%rdi), %xmm2 | |
1948 | movdqa (%rsi), %xmm1 | |
1949 | pcmpeqb %xmm1, %xmm0 | |
1950 | pslldq $2, %xmm2 | |
42e08a54 | 1951 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1952 | pcmpeqb %xmm1, %xmm2 |
1953 | psubb %xmm0, %xmm2 | |
1954 | pmovmskb %xmm2, %r9d | |
1955 | shr %cl, %edx | |
1956 | shr %cl, %r9d | |
1957 | sub %r9d, %edx | |
1958 | jnz LABEL(less32bytes) | |
1959 | movdqa (%rdi), %xmm3 | |
1960 | ||
1961 | UPDATE_STRNCMP_COUNTER | |
1962 | ||
1963 | pxor %xmm0, %xmm0 | |
1964 | mov $16, %rcx /* index for loads */ | |
1965 | mov $14, %r9d /* byte position left over from less32bytes case */ | |
1966 | /* | |
1967 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1968 | * When %r10 goes positive we have crossed a page boundary and | |
1969 | * need to do a nibble. | |
1970 | */ | |
1971 | lea 14(%rdi), %r10 | |
1972 | and $0xfff, %r10 /* offset into 4K page */ | |
1973 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1974 | ||
1975 | .p2align 4 | |
1976 | LABEL(loop_ashr_14): | |
1977 | add $16, %r10 | |
1978 | jg LABEL(nibble_ashr_14) | |
1979 | ||
1980 | LABEL(gobble_ashr_14): | |
1981 | movdqa (%rsi, %rcx), %xmm1 | |
1982 | movdqa (%rdi, %rcx), %xmm2 | |
1983 | movdqa %xmm2, %xmm4 | |
1984 | ||
d8725b1f | 1985 | #ifndef USE_SSSE3 |
7956a3d2 L |
1986 | psrldq $14, %xmm3 |
1987 | pslldq $2, %xmm2 | |
0fda545d | 1988 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1989 | #else |
0fda545d | 1990 | palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1991 | #endif |
42e08a54 | 1992 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1993 | |
1994 | pcmpeqb %xmm1, %xmm0 | |
1995 | pcmpeqb %xmm2, %xmm1 | |
1996 | psubb %xmm0, %xmm1 | |
1997 | pmovmskb %xmm1, %edx | |
1998 | sub $0xffff, %edx | |
1999 | jnz LABEL(exit) | |
2000 | ||
d8725b1f | 2001 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2002 | sub $16, %r11 |
2003 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2004 | #endif |
7956a3d2 L |
2005 | |
2006 | add $16, %rcx | |
2007 | movdqa %xmm4, %xmm3 | |
2008 | ||
2009 | add $16, %r10 | |
2010 | jg LABEL(nibble_ashr_14) /* cross page boundary */ | |
2011 | ||
2012 | movdqa (%rsi, %rcx), %xmm1 | |
2013 | movdqa (%rdi, %rcx), %xmm2 | |
2014 | movdqa %xmm2, %xmm4 | |
2015 | ||
d8725b1f | 2016 | #ifndef USE_SSSE3 |
7956a3d2 | 2017 | psrldq $14, %xmm3 |
0fda545d UD |
2018 | pslldq $2, %xmm2 |
2019 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 2020 | #else |
0fda545d | 2021 | palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2022 | #endif |
42e08a54 | 2023 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2024 | |
2025 | pcmpeqb %xmm1, %xmm0 | |
2026 | pcmpeqb %xmm2, %xmm1 | |
2027 | psubb %xmm0, %xmm1 | |
2028 | pmovmskb %xmm1, %edx | |
2029 | sub $0xffff, %edx | |
2030 | jnz LABEL(exit) | |
2031 | ||
d8725b1f | 2032 | #if defined USE_AS_STRNCMP | defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2033 | sub $16, %r11 |
2034 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2035 | #endif |
7956a3d2 L |
2036 | |
2037 | add $16, %rcx | |
2038 | movdqa %xmm4, %xmm3 | |
2039 | jmp LABEL(loop_ashr_14) | |
2040 | ||
2041 | .p2align 4 | |
2042 | LABEL(nibble_ashr_14): | |
2043 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
2044 | pmovmskb %xmm0, %edx | |
2045 | test $0xc000, %edx | |
2046 | jnz LABEL(ashr_14_exittail) | |
2047 | ||
d8725b1f | 2048 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 2049 | cmp $2, %r11 |
7956a3d2 | 2050 | jbe LABEL(ashr_14_exittail) |
d8725b1f | 2051 | #endif |
7956a3d2 L |
2052 | |
2053 | pxor %xmm0, %xmm0 | |
2054 | sub $0x1000, %r10 | |
2055 | jmp LABEL(gobble_ashr_14) | |
2056 | ||
2057 | .p2align 4 | |
2058 | LABEL(ashr_14_exittail): | |
2059 | movdqa (%rsi, %rcx), %xmm1 | |
2060 | psrldq $14, %xmm0 | |
2061 | psrldq $14, %xmm3 | |
2062 | jmp LABEL(aftertail) | |
2063 | ||
2064 | /* | |
2065 | * The following cases will be handled by ashr_15 | |
2066 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 2067 | * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15 |
7956a3d2 L |
2068 | */ |
2069 | .p2align 4 | |
2070 | LABEL(ashr_15): | |
2071 | pxor %xmm0, %xmm0 | |
2072 | movdqa (%rdi), %xmm2 | |
2073 | movdqa (%rsi), %xmm1 | |
2074 | pcmpeqb %xmm1, %xmm0 | |
2075 | pslldq $1, %xmm2 | |
42e08a54 | 2076 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2077 | pcmpeqb %xmm1, %xmm2 |
2078 | psubb %xmm0, %xmm2 | |
2079 | pmovmskb %xmm2, %r9d | |
2080 | shr %cl, %edx | |
2081 | shr %cl, %r9d | |
2082 | sub %r9d, %edx | |
2083 | jnz LABEL(less32bytes) | |
2084 | ||
2085 | movdqa (%rdi), %xmm3 | |
2086 | ||
2087 | UPDATE_STRNCMP_COUNTER | |
2088 | ||
2089 | pxor %xmm0, %xmm0 | |
2090 | mov $16, %rcx /* index for loads */ | |
2091 | mov $15, %r9d /* byte position left over from less32bytes case */ | |
2092 | /* | |
2093 | * Setup %r10 value allows us to detect crossing a page boundary. | |
2094 | * When %r10 goes positive we have crossed a page boundary and | |
2095 | * need to do a nibble. | |
2096 | */ | |
2097 | lea 15(%rdi), %r10 | |
2098 | and $0xfff, %r10 /* offset into 4K page */ | |
2099 | ||
2100 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
2101 | ||
2102 | .p2align 4 | |
2103 | LABEL(loop_ashr_15): | |
2104 | add $16, %r10 | |
2105 | jg LABEL(nibble_ashr_15) | |
2106 | ||
2107 | LABEL(gobble_ashr_15): | |
2108 | movdqa (%rsi, %rcx), %xmm1 | |
2109 | movdqa (%rdi, %rcx), %xmm2 | |
2110 | movdqa %xmm2, %xmm4 | |
2111 | ||
d8725b1f | 2112 | #ifndef USE_SSSE3 |
7956a3d2 L |
2113 | psrldq $15, %xmm3 |
2114 | pslldq $1, %xmm2 | |
0fda545d | 2115 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2116 | #else |
0fda545d | 2117 | palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2118 | #endif |
42e08a54 | 2119 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2120 | |
2121 | pcmpeqb %xmm1, %xmm0 | |
2122 | pcmpeqb %xmm2, %xmm1 | |
2123 | psubb %xmm0, %xmm1 | |
2124 | pmovmskb %xmm1, %edx | |
2125 | sub $0xffff, %edx | |
2126 | jnz LABEL(exit) | |
2127 | ||
d8725b1f | 2128 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2129 | sub $16, %r11 |
2130 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2131 | #endif |
7956a3d2 L |
2132 | |
2133 | add $16, %rcx | |
2134 | movdqa %xmm4, %xmm3 | |
2135 | ||
2136 | add $16, %r10 | |
2137 | jg LABEL(nibble_ashr_15) /* cross page boundary */ | |
2138 | ||
2139 | movdqa (%rsi, %rcx), %xmm1 | |
2140 | movdqa (%rdi, %rcx), %xmm2 | |
2141 | movdqa %xmm2, %xmm4 | |
2142 | ||
d8725b1f | 2143 | #ifndef USE_SSSE3 |
7956a3d2 | 2144 | psrldq $15, %xmm3 |
0fda545d UD |
2145 | pslldq $1, %xmm2 |
2146 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 2147 | #else |
0fda545d | 2148 | palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2149 | #endif |
42e08a54 | 2150 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2151 | |
2152 | pcmpeqb %xmm1, %xmm0 | |
2153 | pcmpeqb %xmm2, %xmm1 | |
2154 | psubb %xmm0, %xmm1 | |
2155 | pmovmskb %xmm1, %edx | |
2156 | sub $0xffff, %edx | |
2157 | jnz LABEL(exit) | |
2158 | ||
d8725b1f | 2159 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2160 | sub $16, %r11 |
2161 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2162 | #endif |
7956a3d2 L |
2163 | |
2164 | add $16, %rcx | |
2165 | movdqa %xmm4, %xmm3 | |
2166 | jmp LABEL(loop_ashr_15) | |
2167 | ||
2168 | .p2align 4 | |
2169 | LABEL(nibble_ashr_15): | |
2170 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
2171 | pmovmskb %xmm0, %edx | |
2172 | test $0x8000, %edx | |
2173 | jnz LABEL(ashr_15_exittail) | |
2174 | ||
d8725b1f | 2175 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 UD |
2176 | cmpq $1, %r11 |
2177 | jbe LABEL(ashr_15_exittail) | |
d8725b1f | 2178 | #endif |
7956a3d2 L |
2179 | |
2180 | pxor %xmm0, %xmm0 | |
2181 | sub $0x1000, %r10 | |
2182 | jmp LABEL(gobble_ashr_15) | |
2183 | ||
2184 | .p2align 4 | |
2185 | LABEL(ashr_15_exittail): | |
2186 | movdqa (%rsi, %rcx), %xmm1 | |
2187 | psrldq $15, %xmm3 | |
2188 | psrldq $15, %xmm0 | |
2189 | ||
2190 | .p2align 4 | |
2191 | LABEL(aftertail): | |
42e08a54 | 2192 | TOLOWER (%xmm1, %xmm3) |
7956a3d2 L |
2193 | pcmpeqb %xmm3, %xmm1 |
2194 | psubb %xmm0, %xmm1 | |
2195 | pmovmskb %xmm1, %edx | |
2196 | not %edx | |
2197 | ||
2198 | .p2align 4 | |
2199 | LABEL(exit): | |
2200 | lea -16(%r9, %rcx), %rax /* locate the exact offset for rdi */ | |
2201 | LABEL(less32bytes): | |
2202 | lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */ | |
2203 | lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */ | |
2204 | test %r8d, %r8d | |
2205 | jz LABEL(ret) | |
2206 | xchg %rsi, %rdi /* recover original order according to flag(%r8d) */ | |
2207 | ||
2208 | .p2align 4 | |
2209 | LABEL(ret): | |
2210 | LABEL(less16bytes): | |
2211 | bsf %rdx, %rdx /* find and store bit index in %rdx */ | |
2212 | ||
d8725b1f | 2213 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2214 | sub %rdx, %r11 |
2215 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2216 | #endif |
7956a3d2 L |
2217 | movzbl (%rsi, %rdx), %ecx |
2218 | movzbl (%rdi, %rdx), %eax | |
2219 | ||
d8725b1f | 2220 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
42e08a54 UD |
2221 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx |
2222 | movl (%rdx,%rcx,4), %ecx | |
2223 | movl (%rdx,%rax,4), %eax | |
d8725b1f | 2224 | #endif |
42e08a54 | 2225 | |
7956a3d2 L |
2226 | sub %ecx, %eax |
2227 | ret | |
2228 | ||
2229 | LABEL(strcmp_exitz): | |
2230 | xor %eax, %eax | |
2231 | ret | |
2232 | ||
2233 | .p2align 4 | |
2234 | LABEL(Byte0): | |
2235 | movzx (%rsi), %ecx | |
2236 | movzx (%rdi), %eax | |
2237 | ||
d8725b1f | 2238 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
e9f82e0d UD |
2239 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx |
2240 | movl (%rdx,%rcx,4), %ecx | |
2241 | movl (%rdx,%rax,4), %eax | |
d8725b1f | 2242 | #endif |
e9f82e0d | 2243 | |
7956a3d2 L |
2244 | sub %ecx, %eax |
2245 | ret | |
29691210 | 2246 | END (STRCMP) |
7956a3d2 L |
2247 | |
2248 | .section .rodata,"a",@progbits | |
2249 | .p2align 3 | |
2250 | LABEL(unaligned_table): | |
2251 | .int LABEL(ashr_1) - LABEL(unaligned_table) | |
2252 | .int LABEL(ashr_2) - LABEL(unaligned_table) | |
2253 | .int LABEL(ashr_3) - LABEL(unaligned_table) | |
2254 | .int LABEL(ashr_4) - LABEL(unaligned_table) | |
2255 | .int LABEL(ashr_5) - LABEL(unaligned_table) | |
2256 | .int LABEL(ashr_6) - LABEL(unaligned_table) | |
2257 | .int LABEL(ashr_7) - LABEL(unaligned_table) | |
2258 | .int LABEL(ashr_8) - LABEL(unaligned_table) | |
2259 | .int LABEL(ashr_9) - LABEL(unaligned_table) | |
2260 | .int LABEL(ashr_10) - LABEL(unaligned_table) | |
2261 | .int LABEL(ashr_11) - LABEL(unaligned_table) | |
2262 | .int LABEL(ashr_12) - LABEL(unaligned_table) | |
2263 | .int LABEL(ashr_13) - LABEL(unaligned_table) | |
2264 | .int LABEL(ashr_14) - LABEL(unaligned_table) | |
2265 | .int LABEL(ashr_15) - LABEL(unaligned_table) | |
2266 | .int LABEL(ashr_0) - LABEL(unaligned_table) | |
7956a3d2 | 2267 | libc_hidden_builtin_def (STRCMP) |