]>
Commit | Line | Data |
---|---|---|
78df0fcb | 1 | /* Highly optimized version for x86-64. |
2b778ceb | 2 | Copyright (C) 1999-2021 Free Software Foundation, Inc. |
78df0fcb | 3 | This file is part of the GNU C Library. |
78df0fcb AJ |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 | 16 | License along with the GNU C Library; if not, see |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
78df0fcb AJ |
18 | |
19 | #include <sysdep.h> | |
20 | #include "asm-syntax.h" | |
78df0fcb | 21 | |
7956a3d2 L |
22 | #undef UPDATE_STRNCMP_COUNTER |
23 | ||
24 | #ifndef LABEL | |
25 | #define LABEL(l) L(l) | |
26 | #endif | |
27 | ||
28 | #ifdef USE_AS_STRNCMP | |
29 | /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz | |
30 | if the new counter > the old one or is 0. */ | |
31 | # define UPDATE_STRNCMP_COUNTER \ | |
32 | /* calculate left number to compare */ \ | |
33 | lea -16(%rcx, %r11), %r9; \ | |
34 | cmp %r9, %r11; \ | |
35 | jb LABEL(strcmp_exitz); \ | |
36 | test %r9, %r9; \ | |
37 | je LABEL(strcmp_exitz); \ | |
38 | mov %r9, %r11 | |
39 | ||
42e08a54 UD |
40 | #elif defined USE_AS_STRCASECMP_L |
41 | # include "locale-defines.h" | |
42 | ||
42e08a54 | 43 | # define UPDATE_STRNCMP_COUNTER |
e9f82e0d UD |
44 | #elif defined USE_AS_STRNCASECMP_L |
45 | # include "locale-defines.h" | |
46 | ||
e9f82e0d UD |
47 | # define UPDATE_STRNCMP_COUNTER \ |
48 | /* calculate left number to compare */ \ | |
49 | lea -16(%rcx, %r11), %r9; \ | |
50 | cmp %r9, %r11; \ | |
51 | jb LABEL(strcmp_exitz); \ | |
52 | test %r9, %r9; \ | |
53 | je LABEL(strcmp_exitz); \ | |
54 | mov %r9, %r11 | |
7956a3d2 L |
55 | #else |
56 | # define UPDATE_STRNCMP_COUNTER | |
57 | # ifndef STRCMP | |
25244f17 | 58 | # define STRCMP strcmp |
7956a3d2 L |
59 | # endif |
60 | #endif | |
61 | ||
0fda545d | 62 | #ifndef USE_SSSE3 |
7956a3d2 | 63 | .text |
0fda545d | 64 | #else |
f69190e7 | 65 | .section .text.ssse3,"ax",@progbits |
0fda545d UD |
66 | #endif |
67 | ||
42e08a54 | 68 | #ifdef USE_AS_STRCASECMP_L |
73507d3a UD |
69 | # ifndef ENTRY2 |
70 | # define ENTRY2(name) ENTRY (name) | |
71 | # define END2(name) END (name) | |
73507d3a UD |
72 | # endif |
73 | ||
e9f82e0d | 74 | ENTRY2 (__strcasecmp) |
42e08a54 | 75 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax |
3cb84561 | 76 | mov %fs:(%rax),%RDX_LP |
42e08a54 | 77 | |
73507d3a | 78 | // XXX 5 byte should be before the function |
42e08a54 UD |
79 | /* 5-byte NOP. */ |
80 | .byte 0x0f,0x1f,0x44,0x00,0x00 | |
73507d3a UD |
81 | END2 (__strcasecmp) |
82 | # ifndef NO_NOLOCALE_ALIAS | |
42e08a54 UD |
83 | weak_alias (__strcasecmp, strcasecmp) |
84 | libc_hidden_def (__strcasecmp) | |
73507d3a | 85 | # endif |
42e08a54 | 86 | /* FALLTHROUGH to strcasecmp_l. */ |
e9f82e0d UD |
87 | #elif defined USE_AS_STRNCASECMP_L |
88 | # ifndef ENTRY2 | |
89 | # define ENTRY2(name) ENTRY (name) | |
90 | # define END2(name) END (name) | |
91 | # endif | |
92 | ||
93 | ENTRY2 (__strncasecmp) | |
94 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax | |
3cb84561 | 95 | mov %fs:(%rax),%RCX_LP |
e9f82e0d UD |
96 | |
97 | // XXX 5 byte should be before the function | |
98 | /* 5-byte NOP. */ | |
99 | .byte 0x0f,0x1f,0x44,0x00,0x00 | |
100 | END2 (__strncasecmp) | |
101 | # ifndef NO_NOLOCALE_ALIAS | |
102 | weak_alias (__strncasecmp, strncasecmp) | |
103 | libc_hidden_def (__strncasecmp) | |
104 | # endif | |
105 | /* FALLTHROUGH to strncasecmp_l. */ | |
42e08a54 UD |
106 | #endif |
107 | ||
29691210 | 108 | ENTRY (STRCMP) |
d8725b1f | 109 | #ifdef USE_AS_STRCASECMP_L |
42e08a54 UD |
110 | /* We have to fall back on the C implementation for locales |
111 | with encodings not matching ASCII for single bytes. */ | |
d8725b1f | 112 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 |
3cb84561 | 113 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rdx), %RAX_LP |
d8725b1f | 114 | # else |
3cb84561 | 115 | mov (%rdx), %RAX_LP |
d8725b1f | 116 | # endif |
34372fc6 | 117 | testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
42e08a54 | 118 | jne __strcasecmp_l_nonascii |
d8725b1f | 119 | #elif defined USE_AS_STRNCASECMP_L |
e9f82e0d UD |
120 | /* We have to fall back on the C implementation for locales |
121 | with encodings not matching ASCII for single bytes. */ | |
d8725b1f | 122 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 |
3cb84561 | 123 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rcx), %RAX_LP |
d8725b1f | 124 | # else |
3cb84561 | 125 | mov (%rcx), %RAX_LP |
d8725b1f | 126 | # endif |
34372fc6 | 127 | testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
e9f82e0d | 128 | jne __strncasecmp_l_nonascii |
d8725b1f | 129 | #endif |
42e08a54 | 130 | |
7956a3d2 L |
131 | /* |
132 | * This implementation uses SSE to compare up to 16 bytes at a time. | |
133 | */ | |
d8725b1f | 134 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
ee915088 | 135 | test %RDX_LP, %RDX_LP |
7956a3d2 | 136 | je LABEL(strcmp_exitz) |
ee915088 | 137 | cmp $1, %RDX_LP |
7956a3d2 | 138 | je LABEL(Byte0) |
ee915088 | 139 | mov %RDX_LP, %R11_LP |
d8725b1f | 140 | #endif |
7956a3d2 L |
141 | mov %esi, %ecx |
142 | mov %edi, %eax | |
143 | /* Use 64bit AND here to avoid long NOP padding. */ | |
144 | and $0x3f, %rcx /* rsi alignment in cache line */ | |
145 | and $0x3f, %rax /* rdi alignment in cache line */ | |
d8725b1f | 146 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
42e08a54 UD |
147 | .section .rodata.cst16,"aM",@progbits,16 |
148 | .align 16 | |
149 | .Lbelowupper: | |
150 | .quad 0x4040404040404040 | |
151 | .quad 0x4040404040404040 | |
152 | .Ltopupper: | |
153 | .quad 0x5b5b5b5b5b5b5b5b | |
154 | .quad 0x5b5b5b5b5b5b5b5b | |
155 | .Ltouppermask: | |
156 | .quad 0x2020202020202020 | |
157 | .quad 0x2020202020202020 | |
158 | .previous | |
159 | movdqa .Lbelowupper(%rip), %xmm5 | |
d8725b1f | 160 | # define UCLOW_reg %xmm5 |
42e08a54 | 161 | movdqa .Ltopupper(%rip), %xmm6 |
d8725b1f | 162 | # define UCHIGH_reg %xmm6 |
42e08a54 | 163 | movdqa .Ltouppermask(%rip), %xmm7 |
d8725b1f L |
164 | # define LCQWORD_reg %xmm7 |
165 | #endif | |
7956a3d2 L |
166 | cmp $0x30, %ecx |
167 | ja LABEL(crosscache) /* rsi: 16-byte load will cross cache line */ | |
168 | cmp $0x30, %eax | |
169 | ja LABEL(crosscache) /* rdi: 16-byte load will cross cache line */ | |
170 | movlpd (%rdi), %xmm1 | |
171 | movlpd (%rsi), %xmm2 | |
172 | movhpd 8(%rdi), %xmm1 | |
173 | movhpd 8(%rsi), %xmm2 | |
d8725b1f L |
174 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
175 | # define TOLOWER(reg1, reg2) \ | |
42e08a54 UD |
176 | movdqa reg1, %xmm8; \ |
177 | movdqa UCHIGH_reg, %xmm9; \ | |
178 | movdqa reg2, %xmm10; \ | |
179 | movdqa UCHIGH_reg, %xmm11; \ | |
180 | pcmpgtb UCLOW_reg, %xmm8; \ | |
181 | pcmpgtb reg1, %xmm9; \ | |
182 | pcmpgtb UCLOW_reg, %xmm10; \ | |
183 | pcmpgtb reg2, %xmm11; \ | |
184 | pand %xmm9, %xmm8; \ | |
185 | pand %xmm11, %xmm10; \ | |
186 | pand LCQWORD_reg, %xmm8; \ | |
187 | pand LCQWORD_reg, %xmm10; \ | |
188 | por %xmm8, reg1; \ | |
189 | por %xmm10, reg2 | |
190 | TOLOWER (%xmm1, %xmm2) | |
d8725b1f L |
191 | #else |
192 | # define TOLOWER(reg1, reg2) | |
193 | #endif | |
7956a3d2 L |
194 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ |
195 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
196 | pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */ | |
197 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ | |
198 | pmovmskb %xmm1, %edx | |
199 | sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */ | |
200 | jnz LABEL(less16bytes) /* If not, find different value or null char */ | |
d8725b1f | 201 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
202 | sub $16, %r11 |
203 | jbe LABEL(strcmp_exitz) /* finish comparision */ | |
d8725b1f | 204 | #endif |
7956a3d2 L |
205 | add $16, %rsi /* prepare to search next 16 bytes */ |
206 | add $16, %rdi /* prepare to search next 16 bytes */ | |
207 | ||
208 | /* | |
209 | * Determine source and destination string offsets from 16-byte alignment. | |
210 | * Use relative offset difference between the two to determine which case | |
211 | * below to use. | |
212 | */ | |
213 | .p2align 4 | |
214 | LABEL(crosscache): | |
215 | and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */ | |
216 | and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */ | |
217 | mov $0xffff, %edx /* for equivalent offset */ | |
218 | xor %r8d, %r8d | |
219 | and $0xf, %ecx /* offset of rsi */ | |
220 | and $0xf, %eax /* offset of rdi */ | |
221 | cmp %eax, %ecx | |
222 | je LABEL(ashr_0) /* rsi and rdi relative offset same */ | |
223 | ja LABEL(bigger) | |
224 | mov %edx, %r8d /* r8d is offset flag for exit tail */ | |
225 | xchg %ecx, %eax | |
226 | xchg %rsi, %rdi | |
227 | LABEL(bigger): | |
228 | lea 15(%rax), %r9 | |
229 | sub %rcx, %r9 | |
230 | lea LABEL(unaligned_table)(%rip), %r10 | |
231 | movslq (%r10, %r9,4), %r9 | |
232 | lea (%r10, %r9), %r10 | |
4ef60d95 | 233 | _CET_NOTRACK jmp *%r10 /* jump to corresponding case */ |
7956a3d2 L |
234 | |
235 | /* | |
236 | * The following cases will be handled by ashr_0 | |
237 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
238 | * n(0~15) n(0~15) 15(15+ n-n) ashr_0 | |
239 | */ | |
240 | .p2align 4 | |
241 | LABEL(ashr_0): | |
242 | ||
243 | movdqa (%rsi), %xmm1 | |
244 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */ | |
245 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
d8725b1f | 246 | #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L |
7956a3d2 | 247 | pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */ |
d8725b1f | 248 | #else |
42e08a54 UD |
249 | movdqa (%rdi), %xmm2 |
250 | TOLOWER (%xmm1, %xmm2) | |
251 | pcmpeqb %xmm2, %xmm1 /* compare 16 bytes for equality */ | |
d8725b1f | 252 | #endif |
7956a3d2 L |
253 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ |
254 | pmovmskb %xmm1, %r9d | |
255 | shr %cl, %edx /* adjust 0xffff for offset */ | |
256 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
257 | sub %r9d, %edx | |
258 | /* | |
259 | * edx must be the same with r9d if in left byte (16-rcx) is equal to | |
260 | * the start from (16-rax) and no null char was seen. | |
261 | */ | |
262 | jne LABEL(less32bytes) /* mismatch or null char */ | |
263 | UPDATE_STRNCMP_COUNTER | |
264 | mov $16, %rcx | |
265 | mov $16, %r9 | |
266 | pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */ | |
267 | ||
268 | /* | |
269 | * Now both strings are aligned at 16-byte boundary. Loop over strings | |
270 | * checking 32-bytes per iteration. | |
271 | */ | |
272 | .p2align 4 | |
273 | LABEL(loop_ashr_0): | |
274 | movdqa (%rsi, %rcx), %xmm1 | |
275 | movdqa (%rdi, %rcx), %xmm2 | |
42e08a54 | 276 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
277 | |
278 | pcmpeqb %xmm1, %xmm0 | |
279 | pcmpeqb %xmm2, %xmm1 | |
280 | psubb %xmm0, %xmm1 | |
281 | pmovmskb %xmm1, %edx | |
282 | sub $0xffff, %edx | |
283 | jnz LABEL(exit) /* mismatch or null char seen */ | |
284 | ||
d8725b1f | 285 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
286 | sub $16, %r11 |
287 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 288 | #endif |
7956a3d2 L |
289 | add $16, %rcx |
290 | movdqa (%rsi, %rcx), %xmm1 | |
291 | movdqa (%rdi, %rcx), %xmm2 | |
42e08a54 | 292 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
293 | |
294 | pcmpeqb %xmm1, %xmm0 | |
295 | pcmpeqb %xmm2, %xmm1 | |
296 | psubb %xmm0, %xmm1 | |
297 | pmovmskb %xmm1, %edx | |
298 | sub $0xffff, %edx | |
299 | jnz LABEL(exit) | |
d8725b1f | 300 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
301 | sub $16, %r11 |
302 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 303 | #endif |
7956a3d2 L |
304 | add $16, %rcx |
305 | jmp LABEL(loop_ashr_0) | |
306 | ||
307 | /* | |
308 | * The following cases will be handled by ashr_1 | |
42e08a54 | 309 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case |
7956a3d2 L |
310 | * n(15) n -15 0(15 +(n-15) - n) ashr_1 |
311 | */ | |
312 | .p2align 4 | |
313 | LABEL(ashr_1): | |
314 | pxor %xmm0, %xmm0 | |
315 | movdqa (%rdi), %xmm2 | |
316 | movdqa (%rsi), %xmm1 | |
317 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
318 | pslldq $15, %xmm2 /* shift first string to align with second */ | |
42e08a54 | 319 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
320 | pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */ |
321 | psubb %xmm0, %xmm2 /* packed sub of comparison results*/ | |
322 | pmovmskb %xmm2, %r9d | |
323 | shr %cl, %edx /* adjust 0xffff for offset */ | |
324 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
325 | sub %r9d, %edx | |
326 | jnz LABEL(less32bytes) /* mismatch or null char seen */ | |
327 | movdqa (%rdi), %xmm3 | |
328 | UPDATE_STRNCMP_COUNTER | |
329 | ||
330 | pxor %xmm0, %xmm0 | |
331 | mov $16, %rcx /* index for loads*/ | |
332 | mov $1, %r9d /* byte position left over from less32bytes case */ | |
333 | /* | |
334 | * Setup %r10 value allows us to detect crossing a page boundary. | |
335 | * When %r10 goes positive we have crossed a page boundary and | |
336 | * need to do a nibble. | |
337 | */ | |
338 | lea 1(%rdi), %r10 | |
339 | and $0xfff, %r10 /* offset into 4K page */ | |
340 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
341 | ||
342 | .p2align 4 | |
343 | LABEL(loop_ashr_1): | |
344 | add $16, %r10 | |
345 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
346 | ||
347 | LABEL(gobble_ashr_1): | |
348 | movdqa (%rsi, %rcx), %xmm1 | |
349 | movdqa (%rdi, %rcx), %xmm2 | |
350 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
351 | ||
d8725b1f | 352 | #ifndef USE_SSSE3 |
7956a3d2 L |
353 | psrldq $1, %xmm3 |
354 | pslldq $15, %xmm2 | |
355 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 356 | #else |
0fda545d | 357 | palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 358 | #endif |
42e08a54 | 359 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
360 | |
361 | pcmpeqb %xmm1, %xmm0 | |
362 | pcmpeqb %xmm2, %xmm1 | |
363 | psubb %xmm0, %xmm1 | |
364 | pmovmskb %xmm1, %edx | |
365 | sub $0xffff, %edx | |
366 | jnz LABEL(exit) | |
367 | ||
d8725b1f | 368 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
369 | sub $16, %r11 |
370 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 371 | #endif |
7956a3d2 L |
372 | add $16, %rcx |
373 | movdqa %xmm4, %xmm3 | |
374 | ||
375 | add $16, %r10 | |
376 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
377 | ||
378 | movdqa (%rsi, %rcx), %xmm1 | |
379 | movdqa (%rdi, %rcx), %xmm2 | |
380 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
381 | ||
d8725b1f | 382 | #ifndef USE_SSSE3 |
7956a3d2 | 383 | psrldq $1, %xmm3 |
0fda545d | 384 | pslldq $15, %xmm2 |
7956a3d2 | 385 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 386 | #else |
0fda545d | 387 | palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 388 | #endif |
42e08a54 | 389 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
390 | |
391 | pcmpeqb %xmm1, %xmm0 | |
392 | pcmpeqb %xmm2, %xmm1 | |
393 | psubb %xmm0, %xmm1 | |
394 | pmovmskb %xmm1, %edx | |
395 | sub $0xffff, %edx | |
396 | jnz LABEL(exit) | |
397 | ||
d8725b1f | 398 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
399 | sub $16, %r11 |
400 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 401 | #endif |
7956a3d2 L |
402 | add $16, %rcx |
403 | movdqa %xmm4, %xmm3 | |
404 | jmp LABEL(loop_ashr_1) | |
405 | ||
406 | /* | |
407 | * Nibble avoids loads across page boundary. This is to avoid a potential | |
408 | * access into unmapped memory. | |
409 | */ | |
410 | .p2align 4 | |
411 | LABEL(nibble_ashr_1): | |
412 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char*/ | |
413 | pmovmskb %xmm0, %edx | |
414 | test $0xfffe, %edx | |
415 | jnz LABEL(ashr_1_exittail) /* find null char*/ | |
416 | ||
d8725b1f | 417 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 418 | cmp $15, %r11 |
7956a3d2 | 419 | jbe LABEL(ashr_1_exittail) |
d8725b1f | 420 | #endif |
7956a3d2 L |
421 | |
422 | pxor %xmm0, %xmm0 | |
423 | sub $0x1000, %r10 /* substract 4K from %r10 */ | |
424 | jmp LABEL(gobble_ashr_1) | |
425 | ||
426 | /* | |
427 | * Once find null char, determine if there is a string mismatch | |
428 | * before the null char. | |
429 | */ | |
430 | .p2align 4 | |
431 | LABEL(ashr_1_exittail): | |
432 | movdqa (%rsi, %rcx), %xmm1 | |
433 | psrldq $1, %xmm0 | |
434 | psrldq $1, %xmm3 | |
435 | jmp LABEL(aftertail) | |
436 | ||
437 | /* | |
438 | * The following cases will be handled by ashr_2 | |
f69190e7 | 439 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case |
7956a3d2 L |
440 | * n(14~15) n -14 1(15 +(n-14) - n) ashr_2 |
441 | */ | |
442 | .p2align 4 | |
443 | LABEL(ashr_2): | |
444 | pxor %xmm0, %xmm0 | |
445 | movdqa (%rdi), %xmm2 | |
446 | movdqa (%rsi), %xmm1 | |
447 | pcmpeqb %xmm1, %xmm0 | |
448 | pslldq $14, %xmm2 | |
42e08a54 | 449 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
450 | pcmpeqb %xmm1, %xmm2 |
451 | psubb %xmm0, %xmm2 | |
452 | pmovmskb %xmm2, %r9d | |
453 | shr %cl, %edx | |
454 | shr %cl, %r9d | |
455 | sub %r9d, %edx | |
456 | jnz LABEL(less32bytes) | |
457 | movdqa (%rdi), %xmm3 | |
458 | UPDATE_STRNCMP_COUNTER | |
459 | ||
460 | pxor %xmm0, %xmm0 | |
461 | mov $16, %rcx /* index for loads */ | |
462 | mov $2, %r9d /* byte position left over from less32bytes case */ | |
463 | /* | |
464 | * Setup %r10 value allows us to detect crossing a page boundary. | |
465 | * When %r10 goes positive we have crossed a page boundary and | |
466 | * need to do a nibble. | |
467 | */ | |
468 | lea 2(%rdi), %r10 | |
469 | and $0xfff, %r10 /* offset into 4K page */ | |
470 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
471 | ||
472 | .p2align 4 | |
473 | LABEL(loop_ashr_2): | |
474 | add $16, %r10 | |
475 | jg LABEL(nibble_ashr_2) | |
476 | ||
477 | LABEL(gobble_ashr_2): | |
478 | movdqa (%rsi, %rcx), %xmm1 | |
479 | movdqa (%rdi, %rcx), %xmm2 | |
480 | movdqa %xmm2, %xmm4 | |
481 | ||
d8725b1f | 482 | #ifndef USE_SSSE3 |
7956a3d2 L |
483 | psrldq $2, %xmm3 |
484 | pslldq $14, %xmm2 | |
0fda545d | 485 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 486 | #else |
0fda545d | 487 | palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 488 | #endif |
42e08a54 | 489 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
490 | |
491 | pcmpeqb %xmm1, %xmm0 | |
492 | pcmpeqb %xmm2, %xmm1 | |
493 | psubb %xmm0, %xmm1 | |
494 | pmovmskb %xmm1, %edx | |
495 | sub $0xffff, %edx | |
496 | jnz LABEL(exit) | |
497 | ||
d8725b1f | 498 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
499 | sub $16, %r11 |
500 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 501 | #endif |
7956a3d2 L |
502 | |
503 | add $16, %rcx | |
504 | movdqa %xmm4, %xmm3 | |
505 | ||
506 | add $16, %r10 | |
507 | jg LABEL(nibble_ashr_2) /* cross page boundary */ | |
508 | ||
509 | movdqa (%rsi, %rcx), %xmm1 | |
510 | movdqa (%rdi, %rcx), %xmm2 | |
511 | movdqa %xmm2, %xmm4 | |
512 | ||
d8725b1f | 513 | #ifndef USE_SSSE3 |
7956a3d2 | 514 | psrldq $2, %xmm3 |
0fda545d UD |
515 | pslldq $14, %xmm2 |
516 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 517 | #else |
0fda545d | 518 | palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 519 | #endif |
42e08a54 | 520 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
521 | |
522 | pcmpeqb %xmm1, %xmm0 | |
523 | pcmpeqb %xmm2, %xmm1 | |
524 | psubb %xmm0, %xmm1 | |
525 | pmovmskb %xmm1, %edx | |
526 | sub $0xffff, %edx | |
527 | jnz LABEL(exit) | |
528 | ||
d8725b1f | 529 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
530 | sub $16, %r11 |
531 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 532 | #endif |
7956a3d2 L |
533 | |
534 | add $16, %rcx | |
535 | movdqa %xmm4, %xmm3 | |
536 | jmp LABEL(loop_ashr_2) | |
537 | ||
538 | .p2align 4 | |
539 | LABEL(nibble_ashr_2): | |
540 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
541 | pmovmskb %xmm0, %edx | |
542 | test $0xfffc, %edx | |
543 | jnz LABEL(ashr_2_exittail) | |
544 | ||
d8725b1f | 545 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 546 | cmp $14, %r11 |
7956a3d2 | 547 | jbe LABEL(ashr_2_exittail) |
d8725b1f | 548 | #endif |
7956a3d2 L |
549 | |
550 | pxor %xmm0, %xmm0 | |
551 | sub $0x1000, %r10 | |
552 | jmp LABEL(gobble_ashr_2) | |
553 | ||
554 | .p2align 4 | |
555 | LABEL(ashr_2_exittail): | |
556 | movdqa (%rsi, %rcx), %xmm1 | |
557 | psrldq $2, %xmm0 | |
558 | psrldq $2, %xmm3 | |
559 | jmp LABEL(aftertail) | |
560 | ||
561 | /* | |
562 | * The following cases will be handled by ashr_3 | |
563 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
564 | * n(13~15) n -13 2(15 +(n-13) - n) ashr_3 | |
565 | */ | |
566 | .p2align 4 | |
567 | LABEL(ashr_3): | |
568 | pxor %xmm0, %xmm0 | |
569 | movdqa (%rdi), %xmm2 | |
570 | movdqa (%rsi), %xmm1 | |
571 | pcmpeqb %xmm1, %xmm0 | |
572 | pslldq $13, %xmm2 | |
42e08a54 | 573 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
574 | pcmpeqb %xmm1, %xmm2 |
575 | psubb %xmm0, %xmm2 | |
576 | pmovmskb %xmm2, %r9d | |
577 | shr %cl, %edx | |
578 | shr %cl, %r9d | |
579 | sub %r9d, %edx | |
580 | jnz LABEL(less32bytes) | |
581 | movdqa (%rdi), %xmm3 | |
582 | ||
583 | UPDATE_STRNCMP_COUNTER | |
584 | ||
585 | pxor %xmm0, %xmm0 | |
586 | mov $16, %rcx /* index for loads */ | |
587 | mov $3, %r9d /* byte position left over from less32bytes case */ | |
588 | /* | |
589 | * Setup %r10 value allows us to detect crossing a page boundary. | |
590 | * When %r10 goes positive we have crossed a page boundary and | |
591 | * need to do a nibble. | |
592 | */ | |
593 | lea 3(%rdi), %r10 | |
594 | and $0xfff, %r10 /* offset into 4K page */ | |
595 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
596 | ||
597 | .p2align 4 | |
598 | LABEL(loop_ashr_3): | |
599 | add $16, %r10 | |
600 | jg LABEL(nibble_ashr_3) | |
601 | ||
602 | LABEL(gobble_ashr_3): | |
603 | movdqa (%rsi, %rcx), %xmm1 | |
604 | movdqa (%rdi, %rcx), %xmm2 | |
605 | movdqa %xmm2, %xmm4 | |
606 | ||
d8725b1f | 607 | #ifndef USE_SSSE3 |
7956a3d2 L |
608 | psrldq $3, %xmm3 |
609 | pslldq $13, %xmm2 | |
0fda545d | 610 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 611 | #else |
0fda545d | 612 | palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 613 | #endif |
42e08a54 | 614 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
615 | |
616 | pcmpeqb %xmm1, %xmm0 | |
617 | pcmpeqb %xmm2, %xmm1 | |
618 | psubb %xmm0, %xmm1 | |
619 | pmovmskb %xmm1, %edx | |
620 | sub $0xffff, %edx | |
621 | jnz LABEL(exit) | |
622 | ||
d8725b1f | 623 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
624 | sub $16, %r11 |
625 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 626 | #endif |
7956a3d2 L |
627 | |
628 | add $16, %rcx | |
629 | movdqa %xmm4, %xmm3 | |
630 | ||
631 | add $16, %r10 | |
632 | jg LABEL(nibble_ashr_3) /* cross page boundary */ | |
633 | ||
634 | movdqa (%rsi, %rcx), %xmm1 | |
635 | movdqa (%rdi, %rcx), %xmm2 | |
636 | movdqa %xmm2, %xmm4 | |
637 | ||
d8725b1f | 638 | #ifndef USE_SSSE3 |
7956a3d2 | 639 | psrldq $3, %xmm3 |
0fda545d UD |
640 | pslldq $13, %xmm2 |
641 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 642 | #else |
0fda545d | 643 | palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 644 | #endif |
42e08a54 | 645 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
646 | |
647 | pcmpeqb %xmm1, %xmm0 | |
648 | pcmpeqb %xmm2, %xmm1 | |
649 | psubb %xmm0, %xmm1 | |
650 | pmovmskb %xmm1, %edx | |
651 | sub $0xffff, %edx | |
652 | jnz LABEL(exit) | |
653 | ||
d8725b1f | 654 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
655 | sub $16, %r11 |
656 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 657 | #endif |
7956a3d2 L |
658 | |
659 | add $16, %rcx | |
660 | movdqa %xmm4, %xmm3 | |
661 | jmp LABEL(loop_ashr_3) | |
662 | ||
663 | .p2align 4 | |
664 | LABEL(nibble_ashr_3): | |
665 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
666 | pmovmskb %xmm0, %edx | |
667 | test $0xfff8, %edx | |
668 | jnz LABEL(ashr_3_exittail) | |
669 | ||
d8725b1f | 670 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 671 | cmp $13, %r11 |
7956a3d2 | 672 | jbe LABEL(ashr_3_exittail) |
d8725b1f | 673 | #endif |
7956a3d2 L |
674 | |
675 | pxor %xmm0, %xmm0 | |
676 | sub $0x1000, %r10 | |
677 | jmp LABEL(gobble_ashr_3) | |
678 | ||
679 | .p2align 4 | |
680 | LABEL(ashr_3_exittail): | |
681 | movdqa (%rsi, %rcx), %xmm1 | |
682 | psrldq $3, %xmm0 | |
683 | psrldq $3, %xmm3 | |
684 | jmp LABEL(aftertail) | |
685 | ||
686 | /* | |
687 | * The following cases will be handled by ashr_4 | |
688 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
689 | * n(12~15) n -12 3(15 +(n-12) - n) ashr_4 | |
690 | */ | |
691 | .p2align 4 | |
692 | LABEL(ashr_4): | |
693 | pxor %xmm0, %xmm0 | |
694 | movdqa (%rdi), %xmm2 | |
695 | movdqa (%rsi), %xmm1 | |
696 | pcmpeqb %xmm1, %xmm0 | |
697 | pslldq $12, %xmm2 | |
42e08a54 | 698 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
699 | pcmpeqb %xmm1, %xmm2 |
700 | psubb %xmm0, %xmm2 | |
701 | pmovmskb %xmm2, %r9d | |
702 | shr %cl, %edx | |
703 | shr %cl, %r9d | |
704 | sub %r9d, %edx | |
705 | jnz LABEL(less32bytes) | |
706 | movdqa (%rdi), %xmm3 | |
707 | ||
708 | UPDATE_STRNCMP_COUNTER | |
709 | ||
710 | pxor %xmm0, %xmm0 | |
711 | mov $16, %rcx /* index for loads */ | |
712 | mov $4, %r9d /* byte position left over from less32bytes case */ | |
713 | /* | |
714 | * Setup %r10 value allows us to detect crossing a page boundary. | |
715 | * When %r10 goes positive we have crossed a page boundary and | |
716 | * need to do a nibble. | |
717 | */ | |
718 | lea 4(%rdi), %r10 | |
719 | and $0xfff, %r10 /* offset into 4K page */ | |
720 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
721 | ||
722 | .p2align 4 | |
723 | LABEL(loop_ashr_4): | |
724 | add $16, %r10 | |
725 | jg LABEL(nibble_ashr_4) | |
726 | ||
727 | LABEL(gobble_ashr_4): | |
728 | movdqa (%rsi, %rcx), %xmm1 | |
729 | movdqa (%rdi, %rcx), %xmm2 | |
730 | movdqa %xmm2, %xmm4 | |
731 | ||
d8725b1f | 732 | #ifndef USE_SSSE3 |
7956a3d2 L |
733 | psrldq $4, %xmm3 |
734 | pslldq $12, %xmm2 | |
0fda545d | 735 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 736 | #else |
0fda545d | 737 | palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 738 | #endif |
42e08a54 | 739 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
740 | |
741 | pcmpeqb %xmm1, %xmm0 | |
742 | pcmpeqb %xmm2, %xmm1 | |
743 | psubb %xmm0, %xmm1 | |
744 | pmovmskb %xmm1, %edx | |
745 | sub $0xffff, %edx | |
746 | jnz LABEL(exit) | |
747 | ||
d8725b1f | 748 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
749 | sub $16, %r11 |
750 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 751 | #endif |
7956a3d2 L |
752 | |
753 | add $16, %rcx | |
754 | movdqa %xmm4, %xmm3 | |
755 | ||
756 | add $16, %r10 | |
757 | jg LABEL(nibble_ashr_4) /* cross page boundary */ | |
758 | ||
759 | movdqa (%rsi, %rcx), %xmm1 | |
760 | movdqa (%rdi, %rcx), %xmm2 | |
761 | movdqa %xmm2, %xmm4 | |
762 | ||
d8725b1f | 763 | #ifndef USE_SSSE3 |
7956a3d2 | 764 | psrldq $4, %xmm3 |
0fda545d UD |
765 | pslldq $12, %xmm2 |
766 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 767 | #else |
0fda545d | 768 | palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 769 | #endif |
42e08a54 | 770 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
771 | |
772 | pcmpeqb %xmm1, %xmm0 | |
773 | pcmpeqb %xmm2, %xmm1 | |
774 | psubb %xmm0, %xmm1 | |
775 | pmovmskb %xmm1, %edx | |
776 | sub $0xffff, %edx | |
777 | jnz LABEL(exit) | |
778 | ||
d8725b1f | 779 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
780 | sub $16, %r11 |
781 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 782 | #endif |
7956a3d2 L |
783 | |
784 | add $16, %rcx | |
785 | movdqa %xmm4, %xmm3 | |
786 | jmp LABEL(loop_ashr_4) | |
787 | ||
788 | .p2align 4 | |
789 | LABEL(nibble_ashr_4): | |
790 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
791 | pmovmskb %xmm0, %edx | |
792 | test $0xfff0, %edx | |
793 | jnz LABEL(ashr_4_exittail) | |
794 | ||
d8725b1f | 795 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 796 | cmp $12, %r11 |
7956a3d2 | 797 | jbe LABEL(ashr_4_exittail) |
d8725b1f | 798 | #endif |
7956a3d2 L |
799 | |
800 | pxor %xmm0, %xmm0 | |
801 | sub $0x1000, %r10 | |
802 | jmp LABEL(gobble_ashr_4) | |
803 | ||
804 | .p2align 4 | |
805 | LABEL(ashr_4_exittail): | |
806 | movdqa (%rsi, %rcx), %xmm1 | |
807 | psrldq $4, %xmm0 | |
808 | psrldq $4, %xmm3 | |
809 | jmp LABEL(aftertail) | |
810 | ||
811 | /* | |
812 | * The following cases will be handled by ashr_5 | |
813 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 814 | * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5 |
7956a3d2 L |
815 | */ |
816 | .p2align 4 | |
817 | LABEL(ashr_5): | |
818 | pxor %xmm0, %xmm0 | |
819 | movdqa (%rdi), %xmm2 | |
820 | movdqa (%rsi), %xmm1 | |
821 | pcmpeqb %xmm1, %xmm0 | |
822 | pslldq $11, %xmm2 | |
42e08a54 | 823 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
824 | pcmpeqb %xmm1, %xmm2 |
825 | psubb %xmm0, %xmm2 | |
826 | pmovmskb %xmm2, %r9d | |
827 | shr %cl, %edx | |
828 | shr %cl, %r9d | |
829 | sub %r9d, %edx | |
830 | jnz LABEL(less32bytes) | |
831 | movdqa (%rdi), %xmm3 | |
832 | ||
833 | UPDATE_STRNCMP_COUNTER | |
834 | ||
835 | pxor %xmm0, %xmm0 | |
836 | mov $16, %rcx /* index for loads */ | |
837 | mov $5, %r9d /* byte position left over from less32bytes case */ | |
838 | /* | |
839 | * Setup %r10 value allows us to detect crossing a page boundary. | |
840 | * When %r10 goes positive we have crossed a page boundary and | |
841 | * need to do a nibble. | |
842 | */ | |
843 | lea 5(%rdi), %r10 | |
844 | and $0xfff, %r10 /* offset into 4K page */ | |
845 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
846 | ||
847 | .p2align 4 | |
848 | LABEL(loop_ashr_5): | |
849 | add $16, %r10 | |
850 | jg LABEL(nibble_ashr_5) | |
851 | ||
852 | LABEL(gobble_ashr_5): | |
853 | movdqa (%rsi, %rcx), %xmm1 | |
854 | movdqa (%rdi, %rcx), %xmm2 | |
855 | movdqa %xmm2, %xmm4 | |
856 | ||
d8725b1f | 857 | #ifndef USE_SSSE3 |
7956a3d2 L |
858 | psrldq $5, %xmm3 |
859 | pslldq $11, %xmm2 | |
0fda545d | 860 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 861 | #else |
0fda545d | 862 | palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 863 | #endif |
42e08a54 | 864 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
865 | |
866 | pcmpeqb %xmm1, %xmm0 | |
867 | pcmpeqb %xmm2, %xmm1 | |
868 | psubb %xmm0, %xmm1 | |
869 | pmovmskb %xmm1, %edx | |
870 | sub $0xffff, %edx | |
871 | jnz LABEL(exit) | |
872 | ||
d8725b1f | 873 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
874 | sub $16, %r11 |
875 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 876 | #endif |
7956a3d2 L |
877 | |
878 | add $16, %rcx | |
879 | movdqa %xmm4, %xmm3 | |
880 | ||
881 | add $16, %r10 | |
882 | jg LABEL(nibble_ashr_5) /* cross page boundary */ | |
883 | ||
884 | movdqa (%rsi, %rcx), %xmm1 | |
885 | movdqa (%rdi, %rcx), %xmm2 | |
886 | movdqa %xmm2, %xmm4 | |
887 | ||
d8725b1f | 888 | #ifndef USE_SSSE3 |
7956a3d2 | 889 | psrldq $5, %xmm3 |
0fda545d UD |
890 | pslldq $11, %xmm2 |
891 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 892 | #else |
0fda545d | 893 | palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 894 | #endif |
42e08a54 | 895 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
896 | |
897 | pcmpeqb %xmm1, %xmm0 | |
898 | pcmpeqb %xmm2, %xmm1 | |
899 | psubb %xmm0, %xmm1 | |
900 | pmovmskb %xmm1, %edx | |
901 | sub $0xffff, %edx | |
902 | jnz LABEL(exit) | |
903 | ||
d8725b1f | 904 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
905 | sub $16, %r11 |
906 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 907 | #endif |
7956a3d2 L |
908 | |
909 | add $16, %rcx | |
910 | movdqa %xmm4, %xmm3 | |
911 | jmp LABEL(loop_ashr_5) | |
912 | ||
913 | .p2align 4 | |
914 | LABEL(nibble_ashr_5): | |
915 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
916 | pmovmskb %xmm0, %edx | |
917 | test $0xffe0, %edx | |
918 | jnz LABEL(ashr_5_exittail) | |
919 | ||
d8725b1f | 920 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 921 | cmp $11, %r11 |
7956a3d2 | 922 | jbe LABEL(ashr_5_exittail) |
d8725b1f | 923 | #endif |
7956a3d2 L |
924 | |
925 | pxor %xmm0, %xmm0 | |
926 | sub $0x1000, %r10 | |
927 | jmp LABEL(gobble_ashr_5) | |
928 | ||
929 | .p2align 4 | |
930 | LABEL(ashr_5_exittail): | |
931 | movdqa (%rsi, %rcx), %xmm1 | |
932 | psrldq $5, %xmm0 | |
933 | psrldq $5, %xmm3 | |
934 | jmp LABEL(aftertail) | |
935 | ||
936 | /* | |
937 | * The following cases will be handled by ashr_6 | |
938 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 939 | * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6 |
7956a3d2 L |
940 | */ |
941 | .p2align 4 | |
942 | LABEL(ashr_6): | |
943 | pxor %xmm0, %xmm0 | |
944 | movdqa (%rdi), %xmm2 | |
945 | movdqa (%rsi), %xmm1 | |
946 | pcmpeqb %xmm1, %xmm0 | |
947 | pslldq $10, %xmm2 | |
42e08a54 | 948 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
949 | pcmpeqb %xmm1, %xmm2 |
950 | psubb %xmm0, %xmm2 | |
951 | pmovmskb %xmm2, %r9d | |
952 | shr %cl, %edx | |
953 | shr %cl, %r9d | |
954 | sub %r9d, %edx | |
955 | jnz LABEL(less32bytes) | |
956 | movdqa (%rdi), %xmm3 | |
957 | ||
958 | UPDATE_STRNCMP_COUNTER | |
959 | ||
960 | pxor %xmm0, %xmm0 | |
961 | mov $16, %rcx /* index for loads */ | |
962 | mov $6, %r9d /* byte position left over from less32bytes case */ | |
963 | /* | |
964 | * Setup %r10 value allows us to detect crossing a page boundary. | |
965 | * When %r10 goes positive we have crossed a page boundary and | |
966 | * need to do a nibble. | |
967 | */ | |
968 | lea 6(%rdi), %r10 | |
969 | and $0xfff, %r10 /* offset into 4K page */ | |
970 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
971 | ||
972 | .p2align 4 | |
973 | LABEL(loop_ashr_6): | |
974 | add $16, %r10 | |
975 | jg LABEL(nibble_ashr_6) | |
976 | ||
977 | LABEL(gobble_ashr_6): | |
978 | movdqa (%rsi, %rcx), %xmm1 | |
979 | movdqa (%rdi, %rcx), %xmm2 | |
980 | movdqa %xmm2, %xmm4 | |
981 | ||
d8725b1f | 982 | #ifndef USE_SSSE3 |
7956a3d2 L |
983 | psrldq $6, %xmm3 |
984 | pslldq $10, %xmm2 | |
0fda545d | 985 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 986 | #else |
0fda545d | 987 | palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 988 | #endif |
42e08a54 | 989 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
990 | |
991 | pcmpeqb %xmm1, %xmm0 | |
992 | pcmpeqb %xmm2, %xmm1 | |
993 | psubb %xmm0, %xmm1 | |
994 | pmovmskb %xmm1, %edx | |
995 | sub $0xffff, %edx | |
996 | jnz LABEL(exit) | |
997 | ||
d8725b1f | 998 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
999 | sub $16, %r11 |
1000 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1001 | #endif |
7956a3d2 L |
1002 | |
1003 | add $16, %rcx | |
1004 | movdqa %xmm4, %xmm3 | |
1005 | ||
1006 | add $16, %r10 | |
1007 | jg LABEL(nibble_ashr_6) /* cross page boundary */ | |
1008 | ||
1009 | movdqa (%rsi, %rcx), %xmm1 | |
1010 | movdqa (%rdi, %rcx), %xmm2 | |
1011 | movdqa %xmm2, %xmm4 | |
1012 | ||
d8725b1f | 1013 | #ifndef USE_SSSE3 |
7956a3d2 | 1014 | psrldq $6, %xmm3 |
0fda545d UD |
1015 | pslldq $10, %xmm2 |
1016 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1017 | #else |
0fda545d | 1018 | palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1019 | #endif |
42e08a54 | 1020 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1021 | |
1022 | pcmpeqb %xmm1, %xmm0 | |
1023 | pcmpeqb %xmm2, %xmm1 | |
1024 | psubb %xmm0, %xmm1 | |
1025 | pmovmskb %xmm1, %edx | |
1026 | sub $0xffff, %edx | |
1027 | jnz LABEL(exit) | |
1028 | ||
d8725b1f | 1029 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1030 | sub $16, %r11 |
1031 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1032 | #endif |
7956a3d2 L |
1033 | |
1034 | add $16, %rcx | |
1035 | movdqa %xmm4, %xmm3 | |
1036 | jmp LABEL(loop_ashr_6) | |
1037 | ||
1038 | .p2align 4 | |
1039 | LABEL(nibble_ashr_6): | |
1040 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1041 | pmovmskb %xmm0, %edx | |
1042 | test $0xffc0, %edx | |
1043 | jnz LABEL(ashr_6_exittail) | |
1044 | ||
d8725b1f | 1045 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1046 | cmp $10, %r11 |
7956a3d2 | 1047 | jbe LABEL(ashr_6_exittail) |
d8725b1f | 1048 | #endif |
7956a3d2 L |
1049 | |
1050 | pxor %xmm0, %xmm0 | |
1051 | sub $0x1000, %r10 | |
1052 | jmp LABEL(gobble_ashr_6) | |
1053 | ||
1054 | .p2align 4 | |
1055 | LABEL(ashr_6_exittail): | |
1056 | movdqa (%rsi, %rcx), %xmm1 | |
1057 | psrldq $6, %xmm0 | |
1058 | psrldq $6, %xmm3 | |
1059 | jmp LABEL(aftertail) | |
1060 | ||
1061 | /* | |
1062 | * The following cases will be handled by ashr_7 | |
1063 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1064 | * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7 |
7956a3d2 L |
1065 | */ |
1066 | .p2align 4 | |
1067 | LABEL(ashr_7): | |
1068 | pxor %xmm0, %xmm0 | |
1069 | movdqa (%rdi), %xmm2 | |
1070 | movdqa (%rsi), %xmm1 | |
1071 | pcmpeqb %xmm1, %xmm0 | |
1072 | pslldq $9, %xmm2 | |
42e08a54 | 1073 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1074 | pcmpeqb %xmm1, %xmm2 |
1075 | psubb %xmm0, %xmm2 | |
1076 | pmovmskb %xmm2, %r9d | |
1077 | shr %cl, %edx | |
1078 | shr %cl, %r9d | |
1079 | sub %r9d, %edx | |
1080 | jnz LABEL(less32bytes) | |
1081 | movdqa (%rdi), %xmm3 | |
1082 | ||
1083 | UPDATE_STRNCMP_COUNTER | |
1084 | ||
1085 | pxor %xmm0, %xmm0 | |
1086 | mov $16, %rcx /* index for loads */ | |
1087 | mov $7, %r9d /* byte position left over from less32bytes case */ | |
1088 | /* | |
1089 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1090 | * When %r10 goes positive we have crossed a page boundary and | |
1091 | * need to do a nibble. | |
1092 | */ | |
1093 | lea 7(%rdi), %r10 | |
1094 | and $0xfff, %r10 /* offset into 4K page */ | |
1095 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1096 | ||
1097 | .p2align 4 | |
1098 | LABEL(loop_ashr_7): | |
1099 | add $16, %r10 | |
1100 | jg LABEL(nibble_ashr_7) | |
1101 | ||
1102 | LABEL(gobble_ashr_7): | |
1103 | movdqa (%rsi, %rcx), %xmm1 | |
1104 | movdqa (%rdi, %rcx), %xmm2 | |
1105 | movdqa %xmm2, %xmm4 | |
1106 | ||
d8725b1f | 1107 | #ifndef USE_SSSE3 |
7956a3d2 L |
1108 | psrldq $7, %xmm3 |
1109 | pslldq $9, %xmm2 | |
0fda545d | 1110 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1111 | #else |
0fda545d | 1112 | palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1113 | #endif |
42e08a54 | 1114 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1115 | |
1116 | pcmpeqb %xmm1, %xmm0 | |
1117 | pcmpeqb %xmm2, %xmm1 | |
1118 | psubb %xmm0, %xmm1 | |
1119 | pmovmskb %xmm1, %edx | |
1120 | sub $0xffff, %edx | |
1121 | jnz LABEL(exit) | |
1122 | ||
d8725b1f | 1123 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1124 | sub $16, %r11 |
1125 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1126 | #endif |
7956a3d2 L |
1127 | |
1128 | add $16, %rcx | |
1129 | movdqa %xmm4, %xmm3 | |
1130 | ||
1131 | add $16, %r10 | |
1132 | jg LABEL(nibble_ashr_7) /* cross page boundary */ | |
1133 | ||
1134 | movdqa (%rsi, %rcx), %xmm1 | |
1135 | movdqa (%rdi, %rcx), %xmm2 | |
1136 | movdqa %xmm2, %xmm4 | |
1137 | ||
d8725b1f | 1138 | #ifndef USE_SSSE3 |
7956a3d2 | 1139 | psrldq $7, %xmm3 |
0fda545d UD |
1140 | pslldq $9, %xmm2 |
1141 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1142 | #else |
0fda545d | 1143 | palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1144 | #endif |
42e08a54 | 1145 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1146 | |
1147 | pcmpeqb %xmm1, %xmm0 | |
1148 | pcmpeqb %xmm2, %xmm1 | |
1149 | psubb %xmm0, %xmm1 | |
1150 | pmovmskb %xmm1, %edx | |
1151 | sub $0xffff, %edx | |
1152 | jnz LABEL(exit) | |
1153 | ||
d8725b1f | 1154 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1155 | sub $16, %r11 |
1156 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1157 | #endif |
7956a3d2 L |
1158 | |
1159 | add $16, %rcx | |
1160 | movdqa %xmm4, %xmm3 | |
1161 | jmp LABEL(loop_ashr_7) | |
1162 | ||
1163 | .p2align 4 | |
1164 | LABEL(nibble_ashr_7): | |
1165 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1166 | pmovmskb %xmm0, %edx | |
1167 | test $0xff80, %edx | |
1168 | jnz LABEL(ashr_7_exittail) | |
1169 | ||
d8725b1f | 1170 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1171 | cmp $9, %r11 |
7956a3d2 | 1172 | jbe LABEL(ashr_7_exittail) |
d8725b1f | 1173 | #endif |
7956a3d2 L |
1174 | |
1175 | pxor %xmm0, %xmm0 | |
1176 | sub $0x1000, %r10 | |
1177 | jmp LABEL(gobble_ashr_7) | |
1178 | ||
1179 | .p2align 4 | |
1180 | LABEL(ashr_7_exittail): | |
1181 | movdqa (%rsi, %rcx), %xmm1 | |
1182 | psrldq $7, %xmm0 | |
1183 | psrldq $7, %xmm3 | |
1184 | jmp LABEL(aftertail) | |
1185 | ||
1186 | /* | |
1187 | * The following cases will be handled by ashr_8 | |
1188 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1189 | * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8 |
7956a3d2 L |
1190 | */ |
1191 | .p2align 4 | |
1192 | LABEL(ashr_8): | |
1193 | pxor %xmm0, %xmm0 | |
1194 | movdqa (%rdi), %xmm2 | |
1195 | movdqa (%rsi), %xmm1 | |
1196 | pcmpeqb %xmm1, %xmm0 | |
1197 | pslldq $8, %xmm2 | |
42e08a54 | 1198 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1199 | pcmpeqb %xmm1, %xmm2 |
1200 | psubb %xmm0, %xmm2 | |
1201 | pmovmskb %xmm2, %r9d | |
1202 | shr %cl, %edx | |
1203 | shr %cl, %r9d | |
1204 | sub %r9d, %edx | |
1205 | jnz LABEL(less32bytes) | |
1206 | movdqa (%rdi), %xmm3 | |
1207 | ||
1208 | UPDATE_STRNCMP_COUNTER | |
1209 | ||
1210 | pxor %xmm0, %xmm0 | |
1211 | mov $16, %rcx /* index for loads */ | |
1212 | mov $8, %r9d /* byte position left over from less32bytes case */ | |
1213 | /* | |
1214 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1215 | * When %r10 goes positive we have crossed a page boundary and | |
1216 | * need to do a nibble. | |
1217 | */ | |
1218 | lea 8(%rdi), %r10 | |
1219 | and $0xfff, %r10 /* offset into 4K page */ | |
1220 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1221 | ||
1222 | .p2align 4 | |
1223 | LABEL(loop_ashr_8): | |
1224 | add $16, %r10 | |
1225 | jg LABEL(nibble_ashr_8) | |
1226 | ||
1227 | LABEL(gobble_ashr_8): | |
1228 | movdqa (%rsi, %rcx), %xmm1 | |
1229 | movdqa (%rdi, %rcx), %xmm2 | |
1230 | movdqa %xmm2, %xmm4 | |
1231 | ||
d8725b1f | 1232 | #ifndef USE_SSSE3 |
7956a3d2 L |
1233 | psrldq $8, %xmm3 |
1234 | pslldq $8, %xmm2 | |
0fda545d | 1235 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1236 | #else |
0fda545d | 1237 | palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1238 | #endif |
42e08a54 | 1239 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1240 | |
1241 | pcmpeqb %xmm1, %xmm0 | |
1242 | pcmpeqb %xmm2, %xmm1 | |
1243 | psubb %xmm0, %xmm1 | |
1244 | pmovmskb %xmm1, %edx | |
1245 | sub $0xffff, %edx | |
1246 | jnz LABEL(exit) | |
1247 | ||
d8725b1f | 1248 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1249 | sub $16, %r11 |
1250 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1251 | #endif |
7956a3d2 L |
1252 | |
1253 | add $16, %rcx | |
1254 | movdqa %xmm4, %xmm3 | |
1255 | ||
1256 | add $16, %r10 | |
1257 | jg LABEL(nibble_ashr_8) /* cross page boundary */ | |
1258 | ||
1259 | movdqa (%rsi, %rcx), %xmm1 | |
1260 | movdqa (%rdi, %rcx), %xmm2 | |
1261 | movdqa %xmm2, %xmm4 | |
1262 | ||
d8725b1f | 1263 | #ifndef USE_SSSE3 |
7956a3d2 | 1264 | psrldq $8, %xmm3 |
0fda545d UD |
1265 | pslldq $8, %xmm2 |
1266 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1267 | #else |
0fda545d | 1268 | palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1269 | #endif |
42e08a54 | 1270 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1271 | |
1272 | pcmpeqb %xmm1, %xmm0 | |
1273 | pcmpeqb %xmm2, %xmm1 | |
1274 | psubb %xmm0, %xmm1 | |
1275 | pmovmskb %xmm1, %edx | |
1276 | sub $0xffff, %edx | |
1277 | jnz LABEL(exit) | |
1278 | ||
d8725b1f | 1279 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1280 | sub $16, %r11 |
1281 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1282 | #endif |
7956a3d2 L |
1283 | |
1284 | add $16, %rcx | |
1285 | movdqa %xmm4, %xmm3 | |
1286 | jmp LABEL(loop_ashr_8) | |
1287 | ||
1288 | .p2align 4 | |
1289 | LABEL(nibble_ashr_8): | |
1290 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1291 | pmovmskb %xmm0, %edx | |
1292 | test $0xff00, %edx | |
1293 | jnz LABEL(ashr_8_exittail) | |
1294 | ||
d8725b1f | 1295 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1296 | cmp $8, %r11 |
7956a3d2 | 1297 | jbe LABEL(ashr_8_exittail) |
d8725b1f | 1298 | #endif |
7956a3d2 L |
1299 | |
1300 | pxor %xmm0, %xmm0 | |
1301 | sub $0x1000, %r10 | |
1302 | jmp LABEL(gobble_ashr_8) | |
1303 | ||
1304 | .p2align 4 | |
1305 | LABEL(ashr_8_exittail): | |
1306 | movdqa (%rsi, %rcx), %xmm1 | |
1307 | psrldq $8, %xmm0 | |
1308 | psrldq $8, %xmm3 | |
1309 | jmp LABEL(aftertail) | |
1310 | ||
1311 | /* | |
1312 | * The following cases will be handled by ashr_9 | |
1313 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1314 | * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9 |
7956a3d2 L |
1315 | */ |
1316 | .p2align 4 | |
1317 | LABEL(ashr_9): | |
1318 | pxor %xmm0, %xmm0 | |
1319 | movdqa (%rdi), %xmm2 | |
1320 | movdqa (%rsi), %xmm1 | |
1321 | pcmpeqb %xmm1, %xmm0 | |
1322 | pslldq $7, %xmm2 | |
42e08a54 | 1323 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1324 | pcmpeqb %xmm1, %xmm2 |
1325 | psubb %xmm0, %xmm2 | |
1326 | pmovmskb %xmm2, %r9d | |
1327 | shr %cl, %edx | |
1328 | shr %cl, %r9d | |
1329 | sub %r9d, %edx | |
1330 | jnz LABEL(less32bytes) | |
1331 | movdqa (%rdi), %xmm3 | |
1332 | ||
1333 | UPDATE_STRNCMP_COUNTER | |
1334 | ||
1335 | pxor %xmm0, %xmm0 | |
1336 | mov $16, %rcx /* index for loads */ | |
1337 | mov $9, %r9d /* byte position left over from less32bytes case */ | |
1338 | /* | |
1339 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1340 | * When %r10 goes positive we have crossed a page boundary and | |
1341 | * need to do a nibble. | |
1342 | */ | |
1343 | lea 9(%rdi), %r10 | |
1344 | and $0xfff, %r10 /* offset into 4K page */ | |
1345 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1346 | ||
1347 | .p2align 4 | |
1348 | LABEL(loop_ashr_9): | |
1349 | add $16, %r10 | |
1350 | jg LABEL(nibble_ashr_9) | |
1351 | ||
1352 | LABEL(gobble_ashr_9): | |
1353 | movdqa (%rsi, %rcx), %xmm1 | |
1354 | movdqa (%rdi, %rcx), %xmm2 | |
1355 | movdqa %xmm2, %xmm4 | |
1356 | ||
d8725b1f | 1357 | #ifndef USE_SSSE3 |
7956a3d2 L |
1358 | psrldq $9, %xmm3 |
1359 | pslldq $7, %xmm2 | |
0fda545d | 1360 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1361 | #else |
0fda545d | 1362 | palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1363 | #endif |
42e08a54 | 1364 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1365 | |
1366 | pcmpeqb %xmm1, %xmm0 | |
1367 | pcmpeqb %xmm2, %xmm1 | |
1368 | psubb %xmm0, %xmm1 | |
1369 | pmovmskb %xmm1, %edx | |
1370 | sub $0xffff, %edx | |
1371 | jnz LABEL(exit) | |
1372 | ||
d8725b1f | 1373 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1374 | sub $16, %r11 |
1375 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1376 | #endif |
7956a3d2 L |
1377 | |
1378 | add $16, %rcx | |
1379 | movdqa %xmm4, %xmm3 | |
1380 | ||
1381 | add $16, %r10 | |
1382 | jg LABEL(nibble_ashr_9) /* cross page boundary */ | |
1383 | ||
1384 | movdqa (%rsi, %rcx), %xmm1 | |
1385 | movdqa (%rdi, %rcx), %xmm2 | |
1386 | movdqa %xmm2, %xmm4 | |
1387 | ||
d8725b1f | 1388 | #ifndef USE_SSSE3 |
7956a3d2 | 1389 | psrldq $9, %xmm3 |
0fda545d UD |
1390 | pslldq $7, %xmm2 |
1391 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1392 | #else |
0fda545d | 1393 | palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1394 | #endif |
42e08a54 | 1395 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1396 | |
1397 | pcmpeqb %xmm1, %xmm0 | |
1398 | pcmpeqb %xmm2, %xmm1 | |
1399 | psubb %xmm0, %xmm1 | |
1400 | pmovmskb %xmm1, %edx | |
1401 | sub $0xffff, %edx | |
1402 | jnz LABEL(exit) | |
1403 | ||
d8725b1f | 1404 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1405 | sub $16, %r11 |
1406 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1407 | #endif |
7956a3d2 L |
1408 | |
1409 | add $16, %rcx | |
1410 | movdqa %xmm4, %xmm3 /* store for next cycle */ | |
1411 | jmp LABEL(loop_ashr_9) | |
1412 | ||
1413 | .p2align 4 | |
1414 | LABEL(nibble_ashr_9): | |
1415 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1416 | pmovmskb %xmm0, %edx | |
1417 | test $0xfe00, %edx | |
1418 | jnz LABEL(ashr_9_exittail) | |
1419 | ||
d8725b1f | 1420 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1421 | cmp $7, %r11 |
7956a3d2 | 1422 | jbe LABEL(ashr_9_exittail) |
d8725b1f | 1423 | #endif |
7956a3d2 L |
1424 | |
1425 | pxor %xmm0, %xmm0 | |
1426 | sub $0x1000, %r10 | |
1427 | jmp LABEL(gobble_ashr_9) | |
1428 | ||
1429 | .p2align 4 | |
1430 | LABEL(ashr_9_exittail): | |
1431 | movdqa (%rsi, %rcx), %xmm1 | |
1432 | psrldq $9, %xmm0 | |
1433 | psrldq $9, %xmm3 | |
1434 | jmp LABEL(aftertail) | |
1435 | ||
1436 | /* | |
1437 | * The following cases will be handled by ashr_10 | |
1438 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1439 | * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10 |
7956a3d2 L |
1440 | */ |
1441 | .p2align 4 | |
1442 | LABEL(ashr_10): | |
1443 | pxor %xmm0, %xmm0 | |
1444 | movdqa (%rdi), %xmm2 | |
1445 | movdqa (%rsi), %xmm1 | |
1446 | pcmpeqb %xmm1, %xmm0 | |
1447 | pslldq $6, %xmm2 | |
42e08a54 | 1448 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1449 | pcmpeqb %xmm1, %xmm2 |
1450 | psubb %xmm0, %xmm2 | |
1451 | pmovmskb %xmm2, %r9d | |
1452 | shr %cl, %edx | |
1453 | shr %cl, %r9d | |
1454 | sub %r9d, %edx | |
1455 | jnz LABEL(less32bytes) | |
1456 | movdqa (%rdi), %xmm3 | |
1457 | ||
1458 | UPDATE_STRNCMP_COUNTER | |
1459 | ||
1460 | pxor %xmm0, %xmm0 | |
1461 | mov $16, %rcx /* index for loads */ | |
1462 | mov $10, %r9d /* byte position left over from less32bytes case */ | |
1463 | /* | |
1464 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1465 | * When %r10 goes positive we have crossed a page boundary and | |
1466 | * need to do a nibble. | |
1467 | */ | |
1468 | lea 10(%rdi), %r10 | |
1469 | and $0xfff, %r10 /* offset into 4K page */ | |
1470 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1471 | ||
1472 | .p2align 4 | |
1473 | LABEL(loop_ashr_10): | |
1474 | add $16, %r10 | |
1475 | jg LABEL(nibble_ashr_10) | |
1476 | ||
1477 | LABEL(gobble_ashr_10): | |
1478 | movdqa (%rsi, %rcx), %xmm1 | |
1479 | movdqa (%rdi, %rcx), %xmm2 | |
1480 | movdqa %xmm2, %xmm4 | |
1481 | ||
d8725b1f | 1482 | #ifndef USE_SSSE3 |
7956a3d2 L |
1483 | psrldq $10, %xmm3 |
1484 | pslldq $6, %xmm2 | |
0fda545d | 1485 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1486 | #else |
0fda545d | 1487 | palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1488 | #endif |
42e08a54 | 1489 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1490 | |
1491 | pcmpeqb %xmm1, %xmm0 | |
1492 | pcmpeqb %xmm2, %xmm1 | |
1493 | psubb %xmm0, %xmm1 | |
1494 | pmovmskb %xmm1, %edx | |
1495 | sub $0xffff, %edx | |
1496 | jnz LABEL(exit) | |
1497 | ||
d8725b1f | 1498 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1499 | sub $16, %r11 |
1500 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1501 | #endif |
7956a3d2 L |
1502 | |
1503 | add $16, %rcx | |
1504 | movdqa %xmm4, %xmm3 | |
1505 | ||
1506 | add $16, %r10 | |
1507 | jg LABEL(nibble_ashr_10) /* cross page boundary */ | |
1508 | ||
1509 | movdqa (%rsi, %rcx), %xmm1 | |
1510 | movdqa (%rdi, %rcx), %xmm2 | |
1511 | movdqa %xmm2, %xmm4 | |
1512 | ||
d8725b1f | 1513 | #ifndef USE_SSSE3 |
7956a3d2 | 1514 | psrldq $10, %xmm3 |
0fda545d UD |
1515 | pslldq $6, %xmm2 |
1516 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1517 | #else |
0fda545d | 1518 | palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1519 | #endif |
42e08a54 | 1520 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1521 | |
1522 | pcmpeqb %xmm1, %xmm0 | |
1523 | pcmpeqb %xmm2, %xmm1 | |
1524 | psubb %xmm0, %xmm1 | |
1525 | pmovmskb %xmm1, %edx | |
1526 | sub $0xffff, %edx | |
1527 | jnz LABEL(exit) | |
1528 | ||
d8725b1f | 1529 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1530 | sub $16, %r11 |
1531 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1532 | #endif |
7956a3d2 L |
1533 | |
1534 | add $16, %rcx | |
1535 | movdqa %xmm4, %xmm3 | |
1536 | jmp LABEL(loop_ashr_10) | |
1537 | ||
1538 | .p2align 4 | |
1539 | LABEL(nibble_ashr_10): | |
1540 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1541 | pmovmskb %xmm0, %edx | |
1542 | test $0xfc00, %edx | |
1543 | jnz LABEL(ashr_10_exittail) | |
1544 | ||
d8725b1f | 1545 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1546 | cmp $6, %r11 |
7956a3d2 | 1547 | jbe LABEL(ashr_10_exittail) |
d8725b1f | 1548 | #endif |
7956a3d2 L |
1549 | |
1550 | pxor %xmm0, %xmm0 | |
1551 | sub $0x1000, %r10 | |
1552 | jmp LABEL(gobble_ashr_10) | |
1553 | ||
1554 | .p2align 4 | |
1555 | LABEL(ashr_10_exittail): | |
1556 | movdqa (%rsi, %rcx), %xmm1 | |
1557 | psrldq $10, %xmm0 | |
1558 | psrldq $10, %xmm3 | |
1559 | jmp LABEL(aftertail) | |
1560 | ||
1561 | /* | |
1562 | * The following cases will be handled by ashr_11 | |
1563 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1564 | * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11 |
7956a3d2 L |
1565 | */ |
1566 | .p2align 4 | |
1567 | LABEL(ashr_11): | |
1568 | pxor %xmm0, %xmm0 | |
1569 | movdqa (%rdi), %xmm2 | |
1570 | movdqa (%rsi), %xmm1 | |
1571 | pcmpeqb %xmm1, %xmm0 | |
1572 | pslldq $5, %xmm2 | |
42e08a54 | 1573 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1574 | pcmpeqb %xmm1, %xmm2 |
1575 | psubb %xmm0, %xmm2 | |
1576 | pmovmskb %xmm2, %r9d | |
1577 | shr %cl, %edx | |
1578 | shr %cl, %r9d | |
1579 | sub %r9d, %edx | |
1580 | jnz LABEL(less32bytes) | |
1581 | movdqa (%rdi), %xmm3 | |
1582 | ||
1583 | UPDATE_STRNCMP_COUNTER | |
1584 | ||
1585 | pxor %xmm0, %xmm0 | |
1586 | mov $16, %rcx /* index for loads */ | |
1587 | mov $11, %r9d /* byte position left over from less32bytes case */ | |
1588 | /* | |
1589 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1590 | * When %r10 goes positive we have crossed a page boundary and | |
1591 | * need to do a nibble. | |
1592 | */ | |
1593 | lea 11(%rdi), %r10 | |
1594 | and $0xfff, %r10 /* offset into 4K page */ | |
1595 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1596 | ||
1597 | .p2align 4 | |
1598 | LABEL(loop_ashr_11): | |
1599 | add $16, %r10 | |
1600 | jg LABEL(nibble_ashr_11) | |
1601 | ||
1602 | LABEL(gobble_ashr_11): | |
1603 | movdqa (%rsi, %rcx), %xmm1 | |
1604 | movdqa (%rdi, %rcx), %xmm2 | |
1605 | movdqa %xmm2, %xmm4 | |
1606 | ||
d8725b1f | 1607 | #ifndef USE_SSSE3 |
7956a3d2 L |
1608 | psrldq $11, %xmm3 |
1609 | pslldq $5, %xmm2 | |
0fda545d | 1610 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1611 | #else |
0fda545d | 1612 | palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1613 | #endif |
42e08a54 | 1614 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1615 | |
1616 | pcmpeqb %xmm1, %xmm0 | |
1617 | pcmpeqb %xmm2, %xmm1 | |
1618 | psubb %xmm0, %xmm1 | |
1619 | pmovmskb %xmm1, %edx | |
1620 | sub $0xffff, %edx | |
1621 | jnz LABEL(exit) | |
1622 | ||
d8725b1f | 1623 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1624 | sub $16, %r11 |
1625 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1626 | #endif |
7956a3d2 L |
1627 | |
1628 | add $16, %rcx | |
1629 | movdqa %xmm4, %xmm3 | |
1630 | ||
1631 | add $16, %r10 | |
1632 | jg LABEL(nibble_ashr_11) /* cross page boundary */ | |
1633 | ||
1634 | movdqa (%rsi, %rcx), %xmm1 | |
1635 | movdqa (%rdi, %rcx), %xmm2 | |
1636 | movdqa %xmm2, %xmm4 | |
1637 | ||
d8725b1f | 1638 | #ifndef USE_SSSE3 |
7956a3d2 | 1639 | psrldq $11, %xmm3 |
0fda545d UD |
1640 | pslldq $5, %xmm2 |
1641 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1642 | #else |
0fda545d | 1643 | palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1644 | #endif |
42e08a54 | 1645 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1646 | |
1647 | pcmpeqb %xmm1, %xmm0 | |
1648 | pcmpeqb %xmm2, %xmm1 | |
1649 | psubb %xmm0, %xmm1 | |
1650 | pmovmskb %xmm1, %edx | |
1651 | sub $0xffff, %edx | |
1652 | jnz LABEL(exit) | |
1653 | ||
d8725b1f | 1654 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1655 | sub $16, %r11 |
1656 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1657 | #endif |
7956a3d2 L |
1658 | |
1659 | add $16, %rcx | |
1660 | movdqa %xmm4, %xmm3 | |
1661 | jmp LABEL(loop_ashr_11) | |
1662 | ||
1663 | .p2align 4 | |
1664 | LABEL(nibble_ashr_11): | |
1665 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1666 | pmovmskb %xmm0, %edx | |
1667 | test $0xf800, %edx | |
1668 | jnz LABEL(ashr_11_exittail) | |
1669 | ||
d8725b1f | 1670 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1671 | cmp $5, %r11 |
7956a3d2 | 1672 | jbe LABEL(ashr_11_exittail) |
d8725b1f | 1673 | #endif |
7956a3d2 L |
1674 | |
1675 | pxor %xmm0, %xmm0 | |
1676 | sub $0x1000, %r10 | |
1677 | jmp LABEL(gobble_ashr_11) | |
1678 | ||
1679 | .p2align 4 | |
1680 | LABEL(ashr_11_exittail): | |
1681 | movdqa (%rsi, %rcx), %xmm1 | |
1682 | psrldq $11, %xmm0 | |
1683 | psrldq $11, %xmm3 | |
1684 | jmp LABEL(aftertail) | |
1685 | ||
1686 | /* | |
1687 | * The following cases will be handled by ashr_12 | |
1688 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1689 | * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12 |
7956a3d2 L |
1690 | */ |
1691 | .p2align 4 | |
1692 | LABEL(ashr_12): | |
1693 | pxor %xmm0, %xmm0 | |
1694 | movdqa (%rdi), %xmm2 | |
1695 | movdqa (%rsi), %xmm1 | |
1696 | pcmpeqb %xmm1, %xmm0 | |
1697 | pslldq $4, %xmm2 | |
42e08a54 | 1698 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1699 | pcmpeqb %xmm1, %xmm2 |
1700 | psubb %xmm0, %xmm2 | |
1701 | pmovmskb %xmm2, %r9d | |
1702 | shr %cl, %edx | |
1703 | shr %cl, %r9d | |
1704 | sub %r9d, %edx | |
1705 | jnz LABEL(less32bytes) | |
1706 | movdqa (%rdi), %xmm3 | |
1707 | ||
1708 | UPDATE_STRNCMP_COUNTER | |
1709 | ||
1710 | pxor %xmm0, %xmm0 | |
1711 | mov $16, %rcx /* index for loads */ | |
1712 | mov $12, %r9d /* byte position left over from less32bytes case */ | |
1713 | /* | |
1714 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1715 | * When %r10 goes positive we have crossed a page boundary and | |
1716 | * need to do a nibble. | |
1717 | */ | |
1718 | lea 12(%rdi), %r10 | |
1719 | and $0xfff, %r10 /* offset into 4K page */ | |
1720 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1721 | ||
1722 | .p2align 4 | |
1723 | LABEL(loop_ashr_12): | |
1724 | add $16, %r10 | |
1725 | jg LABEL(nibble_ashr_12) | |
1726 | ||
1727 | LABEL(gobble_ashr_12): | |
1728 | movdqa (%rsi, %rcx), %xmm1 | |
1729 | movdqa (%rdi, %rcx), %xmm2 | |
1730 | movdqa %xmm2, %xmm4 | |
1731 | ||
d8725b1f | 1732 | #ifndef USE_SSSE3 |
7956a3d2 L |
1733 | psrldq $12, %xmm3 |
1734 | pslldq $4, %xmm2 | |
0fda545d | 1735 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1736 | #else |
0fda545d | 1737 | palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1738 | #endif |
42e08a54 | 1739 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1740 | |
1741 | pcmpeqb %xmm1, %xmm0 | |
1742 | pcmpeqb %xmm2, %xmm1 | |
1743 | psubb %xmm0, %xmm1 | |
1744 | pmovmskb %xmm1, %edx | |
1745 | sub $0xffff, %edx | |
1746 | jnz LABEL(exit) | |
1747 | ||
d8725b1f | 1748 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1749 | sub $16, %r11 |
1750 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1751 | #endif |
7956a3d2 L |
1752 | |
1753 | add $16, %rcx | |
1754 | movdqa %xmm4, %xmm3 | |
1755 | ||
1756 | add $16, %r10 | |
1757 | jg LABEL(nibble_ashr_12) /* cross page boundary */ | |
1758 | ||
1759 | movdqa (%rsi, %rcx), %xmm1 | |
1760 | movdqa (%rdi, %rcx), %xmm2 | |
1761 | movdqa %xmm2, %xmm4 | |
1762 | ||
d8725b1f | 1763 | #ifndef USE_SSSE3 |
7956a3d2 | 1764 | psrldq $12, %xmm3 |
0fda545d UD |
1765 | pslldq $4, %xmm2 |
1766 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1767 | #else |
0fda545d | 1768 | palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1769 | #endif |
42e08a54 | 1770 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1771 | |
1772 | pcmpeqb %xmm1, %xmm0 | |
1773 | pcmpeqb %xmm2, %xmm1 | |
1774 | psubb %xmm0, %xmm1 | |
1775 | pmovmskb %xmm1, %edx | |
1776 | sub $0xffff, %edx | |
1777 | jnz LABEL(exit) | |
1778 | ||
d8725b1f | 1779 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1780 | sub $16, %r11 |
1781 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1782 | #endif |
7956a3d2 L |
1783 | |
1784 | add $16, %rcx | |
1785 | movdqa %xmm4, %xmm3 | |
1786 | jmp LABEL(loop_ashr_12) | |
1787 | ||
1788 | .p2align 4 | |
1789 | LABEL(nibble_ashr_12): | |
1790 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1791 | pmovmskb %xmm0, %edx | |
1792 | test $0xf000, %edx | |
1793 | jnz LABEL(ashr_12_exittail) | |
1794 | ||
d8725b1f | 1795 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1796 | cmp $4, %r11 |
7956a3d2 | 1797 | jbe LABEL(ashr_12_exittail) |
d8725b1f | 1798 | #endif |
7956a3d2 L |
1799 | |
1800 | pxor %xmm0, %xmm0 | |
1801 | sub $0x1000, %r10 | |
1802 | jmp LABEL(gobble_ashr_12) | |
1803 | ||
1804 | .p2align 4 | |
1805 | LABEL(ashr_12_exittail): | |
1806 | movdqa (%rsi, %rcx), %xmm1 | |
1807 | psrldq $12, %xmm0 | |
1808 | psrldq $12, %xmm3 | |
1809 | jmp LABEL(aftertail) | |
1810 | ||
1811 | /* | |
1812 | * The following cases will be handled by ashr_13 | |
1813 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1814 | * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13 |
7956a3d2 L |
1815 | */ |
1816 | .p2align 4 | |
1817 | LABEL(ashr_13): | |
1818 | pxor %xmm0, %xmm0 | |
1819 | movdqa (%rdi), %xmm2 | |
1820 | movdqa (%rsi), %xmm1 | |
1821 | pcmpeqb %xmm1, %xmm0 | |
1822 | pslldq $3, %xmm2 | |
42e08a54 | 1823 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1824 | pcmpeqb %xmm1, %xmm2 |
1825 | psubb %xmm0, %xmm2 | |
1826 | pmovmskb %xmm2, %r9d | |
1827 | shr %cl, %edx | |
1828 | shr %cl, %r9d | |
1829 | sub %r9d, %edx | |
1830 | jnz LABEL(less32bytes) | |
1831 | movdqa (%rdi), %xmm3 | |
1832 | ||
1833 | UPDATE_STRNCMP_COUNTER | |
1834 | ||
1835 | pxor %xmm0, %xmm0 | |
1836 | mov $16, %rcx /* index for loads */ | |
1837 | mov $13, %r9d /* byte position left over from less32bytes case */ | |
1838 | /* | |
1839 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1840 | * When %r10 goes positive we have crossed a page boundary and | |
1841 | * need to do a nibble. | |
1842 | */ | |
1843 | lea 13(%rdi), %r10 | |
1844 | and $0xfff, %r10 /* offset into 4K page */ | |
1845 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1846 | ||
1847 | .p2align 4 | |
1848 | LABEL(loop_ashr_13): | |
1849 | add $16, %r10 | |
1850 | jg LABEL(nibble_ashr_13) | |
1851 | ||
1852 | LABEL(gobble_ashr_13): | |
1853 | movdqa (%rsi, %rcx), %xmm1 | |
1854 | movdqa (%rdi, %rcx), %xmm2 | |
1855 | movdqa %xmm2, %xmm4 | |
1856 | ||
d8725b1f | 1857 | #ifndef USE_SSSE3 |
7956a3d2 L |
1858 | psrldq $13, %xmm3 |
1859 | pslldq $3, %xmm2 | |
0fda545d | 1860 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1861 | #else |
0fda545d | 1862 | palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1863 | #endif |
42e08a54 | 1864 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1865 | |
1866 | pcmpeqb %xmm1, %xmm0 | |
1867 | pcmpeqb %xmm2, %xmm1 | |
1868 | psubb %xmm0, %xmm1 | |
1869 | pmovmskb %xmm1, %edx | |
1870 | sub $0xffff, %edx | |
1871 | jnz LABEL(exit) | |
1872 | ||
d8725b1f | 1873 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1874 | sub $16, %r11 |
1875 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1876 | #endif |
7956a3d2 L |
1877 | |
1878 | add $16, %rcx | |
1879 | movdqa %xmm4, %xmm3 | |
1880 | ||
1881 | add $16, %r10 | |
1882 | jg LABEL(nibble_ashr_13) /* cross page boundary */ | |
1883 | ||
1884 | movdqa (%rsi, %rcx), %xmm1 | |
1885 | movdqa (%rdi, %rcx), %xmm2 | |
1886 | movdqa %xmm2, %xmm4 | |
1887 | ||
d8725b1f | 1888 | #ifndef USE_SSSE3 |
7956a3d2 | 1889 | psrldq $13, %xmm3 |
0fda545d UD |
1890 | pslldq $3, %xmm2 |
1891 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 1892 | #else |
0fda545d | 1893 | palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1894 | #endif |
42e08a54 | 1895 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1896 | |
1897 | pcmpeqb %xmm1, %xmm0 | |
1898 | pcmpeqb %xmm2, %xmm1 | |
1899 | psubb %xmm0, %xmm1 | |
1900 | pmovmskb %xmm1, %edx | |
1901 | sub $0xffff, %edx | |
1902 | jnz LABEL(exit) | |
1903 | ||
d8725b1f | 1904 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1905 | sub $16, %r11 |
1906 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 1907 | #endif |
7956a3d2 L |
1908 | |
1909 | add $16, %rcx | |
1910 | movdqa %xmm4, %xmm3 | |
1911 | jmp LABEL(loop_ashr_13) | |
1912 | ||
1913 | .p2align 4 | |
1914 | LABEL(nibble_ashr_13): | |
1915 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1916 | pmovmskb %xmm0, %edx | |
1917 | test $0xe000, %edx | |
1918 | jnz LABEL(ashr_13_exittail) | |
1919 | ||
d8725b1f | 1920 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 1921 | cmp $3, %r11 |
7956a3d2 | 1922 | jbe LABEL(ashr_13_exittail) |
d8725b1f | 1923 | #endif |
7956a3d2 L |
1924 | |
1925 | pxor %xmm0, %xmm0 | |
1926 | sub $0x1000, %r10 | |
1927 | jmp LABEL(gobble_ashr_13) | |
1928 | ||
1929 | .p2align 4 | |
1930 | LABEL(ashr_13_exittail): | |
1931 | movdqa (%rsi, %rcx), %xmm1 | |
1932 | psrldq $13, %xmm0 | |
1933 | psrldq $13, %xmm3 | |
1934 | jmp LABEL(aftertail) | |
1935 | ||
1936 | /* | |
1937 | * The following cases will be handled by ashr_14 | |
1938 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 1939 | * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14 |
7956a3d2 L |
1940 | */ |
1941 | .p2align 4 | |
1942 | LABEL(ashr_14): | |
1943 | pxor %xmm0, %xmm0 | |
1944 | movdqa (%rdi), %xmm2 | |
1945 | movdqa (%rsi), %xmm1 | |
1946 | pcmpeqb %xmm1, %xmm0 | |
1947 | pslldq $2, %xmm2 | |
42e08a54 | 1948 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1949 | pcmpeqb %xmm1, %xmm2 |
1950 | psubb %xmm0, %xmm2 | |
1951 | pmovmskb %xmm2, %r9d | |
1952 | shr %cl, %edx | |
1953 | shr %cl, %r9d | |
1954 | sub %r9d, %edx | |
1955 | jnz LABEL(less32bytes) | |
1956 | movdqa (%rdi), %xmm3 | |
1957 | ||
1958 | UPDATE_STRNCMP_COUNTER | |
1959 | ||
1960 | pxor %xmm0, %xmm0 | |
1961 | mov $16, %rcx /* index for loads */ | |
1962 | mov $14, %r9d /* byte position left over from less32bytes case */ | |
1963 | /* | |
1964 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1965 | * When %r10 goes positive we have crossed a page boundary and | |
1966 | * need to do a nibble. | |
1967 | */ | |
1968 | lea 14(%rdi), %r10 | |
1969 | and $0xfff, %r10 /* offset into 4K page */ | |
1970 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1971 | ||
1972 | .p2align 4 | |
1973 | LABEL(loop_ashr_14): | |
1974 | add $16, %r10 | |
1975 | jg LABEL(nibble_ashr_14) | |
1976 | ||
1977 | LABEL(gobble_ashr_14): | |
1978 | movdqa (%rsi, %rcx), %xmm1 | |
1979 | movdqa (%rdi, %rcx), %xmm2 | |
1980 | movdqa %xmm2, %xmm4 | |
1981 | ||
d8725b1f | 1982 | #ifndef USE_SSSE3 |
7956a3d2 L |
1983 | psrldq $14, %xmm3 |
1984 | pslldq $2, %xmm2 | |
0fda545d | 1985 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1986 | #else |
0fda545d | 1987 | palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 1988 | #endif |
42e08a54 | 1989 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
1990 | |
1991 | pcmpeqb %xmm1, %xmm0 | |
1992 | pcmpeqb %xmm2, %xmm1 | |
1993 | psubb %xmm0, %xmm1 | |
1994 | pmovmskb %xmm1, %edx | |
1995 | sub $0xffff, %edx | |
1996 | jnz LABEL(exit) | |
1997 | ||
d8725b1f | 1998 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
1999 | sub $16, %r11 |
2000 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2001 | #endif |
7956a3d2 L |
2002 | |
2003 | add $16, %rcx | |
2004 | movdqa %xmm4, %xmm3 | |
2005 | ||
2006 | add $16, %r10 | |
2007 | jg LABEL(nibble_ashr_14) /* cross page boundary */ | |
2008 | ||
2009 | movdqa (%rsi, %rcx), %xmm1 | |
2010 | movdqa (%rdi, %rcx), %xmm2 | |
2011 | movdqa %xmm2, %xmm4 | |
2012 | ||
d8725b1f | 2013 | #ifndef USE_SSSE3 |
7956a3d2 | 2014 | psrldq $14, %xmm3 |
0fda545d UD |
2015 | pslldq $2, %xmm2 |
2016 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 2017 | #else |
0fda545d | 2018 | palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2019 | #endif |
42e08a54 | 2020 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2021 | |
2022 | pcmpeqb %xmm1, %xmm0 | |
2023 | pcmpeqb %xmm2, %xmm1 | |
2024 | psubb %xmm0, %xmm1 | |
2025 | pmovmskb %xmm1, %edx | |
2026 | sub $0xffff, %edx | |
2027 | jnz LABEL(exit) | |
2028 | ||
d8725b1f | 2029 | #if defined USE_AS_STRNCMP | defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2030 | sub $16, %r11 |
2031 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2032 | #endif |
7956a3d2 L |
2033 | |
2034 | add $16, %rcx | |
2035 | movdqa %xmm4, %xmm3 | |
2036 | jmp LABEL(loop_ashr_14) | |
2037 | ||
2038 | .p2align 4 | |
2039 | LABEL(nibble_ashr_14): | |
2040 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
2041 | pmovmskb %xmm0, %edx | |
2042 | test $0xc000, %edx | |
2043 | jnz LABEL(ashr_14_exittail) | |
2044 | ||
d8725b1f | 2045 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 | 2046 | cmp $2, %r11 |
7956a3d2 | 2047 | jbe LABEL(ashr_14_exittail) |
d8725b1f | 2048 | #endif |
7956a3d2 L |
2049 | |
2050 | pxor %xmm0, %xmm0 | |
2051 | sub $0x1000, %r10 | |
2052 | jmp LABEL(gobble_ashr_14) | |
2053 | ||
2054 | .p2align 4 | |
2055 | LABEL(ashr_14_exittail): | |
2056 | movdqa (%rsi, %rcx), %xmm1 | |
2057 | psrldq $14, %xmm0 | |
2058 | psrldq $14, %xmm3 | |
2059 | jmp LABEL(aftertail) | |
2060 | ||
2061 | /* | |
2062 | * The following cases will be handled by ashr_15 | |
2063 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
f69190e7 | 2064 | * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15 |
7956a3d2 L |
2065 | */ |
2066 | .p2align 4 | |
2067 | LABEL(ashr_15): | |
2068 | pxor %xmm0, %xmm0 | |
2069 | movdqa (%rdi), %xmm2 | |
2070 | movdqa (%rsi), %xmm1 | |
2071 | pcmpeqb %xmm1, %xmm0 | |
2072 | pslldq $1, %xmm2 | |
42e08a54 | 2073 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2074 | pcmpeqb %xmm1, %xmm2 |
2075 | psubb %xmm0, %xmm2 | |
2076 | pmovmskb %xmm2, %r9d | |
2077 | shr %cl, %edx | |
2078 | shr %cl, %r9d | |
2079 | sub %r9d, %edx | |
2080 | jnz LABEL(less32bytes) | |
2081 | ||
2082 | movdqa (%rdi), %xmm3 | |
2083 | ||
2084 | UPDATE_STRNCMP_COUNTER | |
2085 | ||
2086 | pxor %xmm0, %xmm0 | |
2087 | mov $16, %rcx /* index for loads */ | |
2088 | mov $15, %r9d /* byte position left over from less32bytes case */ | |
2089 | /* | |
2090 | * Setup %r10 value allows us to detect crossing a page boundary. | |
2091 | * When %r10 goes positive we have crossed a page boundary and | |
2092 | * need to do a nibble. | |
2093 | */ | |
2094 | lea 15(%rdi), %r10 | |
2095 | and $0xfff, %r10 /* offset into 4K page */ | |
2096 | ||
2097 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
2098 | ||
2099 | .p2align 4 | |
2100 | LABEL(loop_ashr_15): | |
2101 | add $16, %r10 | |
2102 | jg LABEL(nibble_ashr_15) | |
2103 | ||
2104 | LABEL(gobble_ashr_15): | |
2105 | movdqa (%rsi, %rcx), %xmm1 | |
2106 | movdqa (%rdi, %rcx), %xmm2 | |
2107 | movdqa %xmm2, %xmm4 | |
2108 | ||
d8725b1f | 2109 | #ifndef USE_SSSE3 |
7956a3d2 L |
2110 | psrldq $15, %xmm3 |
2111 | pslldq $1, %xmm2 | |
0fda545d | 2112 | por %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2113 | #else |
0fda545d | 2114 | palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2115 | #endif |
42e08a54 | 2116 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2117 | |
2118 | pcmpeqb %xmm1, %xmm0 | |
2119 | pcmpeqb %xmm2, %xmm1 | |
2120 | psubb %xmm0, %xmm1 | |
2121 | pmovmskb %xmm1, %edx | |
2122 | sub $0xffff, %edx | |
2123 | jnz LABEL(exit) | |
2124 | ||
d8725b1f | 2125 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2126 | sub $16, %r11 |
2127 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2128 | #endif |
7956a3d2 L |
2129 | |
2130 | add $16, %rcx | |
2131 | movdqa %xmm4, %xmm3 | |
2132 | ||
2133 | add $16, %r10 | |
2134 | jg LABEL(nibble_ashr_15) /* cross page boundary */ | |
2135 | ||
2136 | movdqa (%rsi, %rcx), %xmm1 | |
2137 | movdqa (%rdi, %rcx), %xmm2 | |
2138 | movdqa %xmm2, %xmm4 | |
2139 | ||
d8725b1f | 2140 | #ifndef USE_SSSE3 |
7956a3d2 | 2141 | psrldq $15, %xmm3 |
0fda545d UD |
2142 | pslldq $1, %xmm2 |
2143 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
d8725b1f | 2144 | #else |
0fda545d | 2145 | palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */ |
d8725b1f | 2146 | #endif |
42e08a54 | 2147 | TOLOWER (%xmm1, %xmm2) |
7956a3d2 L |
2148 | |
2149 | pcmpeqb %xmm1, %xmm0 | |
2150 | pcmpeqb %xmm2, %xmm1 | |
2151 | psubb %xmm0, %xmm1 | |
2152 | pmovmskb %xmm1, %edx | |
2153 | sub $0xffff, %edx | |
2154 | jnz LABEL(exit) | |
2155 | ||
d8725b1f | 2156 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2157 | sub $16, %r11 |
2158 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2159 | #endif |
7956a3d2 L |
2160 | |
2161 | add $16, %rcx | |
2162 | movdqa %xmm4, %xmm3 | |
2163 | jmp LABEL(loop_ashr_15) | |
2164 | ||
2165 | .p2align 4 | |
2166 | LABEL(nibble_ashr_15): | |
2167 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
2168 | pmovmskb %xmm0, %edx | |
2169 | test $0x8000, %edx | |
2170 | jnz LABEL(ashr_15_exittail) | |
2171 | ||
d8725b1f | 2172 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
45db99c7 UD |
2173 | cmpq $1, %r11 |
2174 | jbe LABEL(ashr_15_exittail) | |
d8725b1f | 2175 | #endif |
7956a3d2 L |
2176 | |
2177 | pxor %xmm0, %xmm0 | |
2178 | sub $0x1000, %r10 | |
2179 | jmp LABEL(gobble_ashr_15) | |
2180 | ||
2181 | .p2align 4 | |
2182 | LABEL(ashr_15_exittail): | |
2183 | movdqa (%rsi, %rcx), %xmm1 | |
2184 | psrldq $15, %xmm3 | |
2185 | psrldq $15, %xmm0 | |
2186 | ||
2187 | .p2align 4 | |
2188 | LABEL(aftertail): | |
42e08a54 | 2189 | TOLOWER (%xmm1, %xmm3) |
7956a3d2 L |
2190 | pcmpeqb %xmm3, %xmm1 |
2191 | psubb %xmm0, %xmm1 | |
2192 | pmovmskb %xmm1, %edx | |
2193 | not %edx | |
2194 | ||
2195 | .p2align 4 | |
2196 | LABEL(exit): | |
2197 | lea -16(%r9, %rcx), %rax /* locate the exact offset for rdi */ | |
2198 | LABEL(less32bytes): | |
2199 | lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */ | |
2200 | lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */ | |
2201 | test %r8d, %r8d | |
2202 | jz LABEL(ret) | |
2203 | xchg %rsi, %rdi /* recover original order according to flag(%r8d) */ | |
2204 | ||
2205 | .p2align 4 | |
2206 | LABEL(ret): | |
2207 | LABEL(less16bytes): | |
2208 | bsf %rdx, %rdx /* find and store bit index in %rdx */ | |
2209 | ||
d8725b1f | 2210 | #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L |
7956a3d2 L |
2211 | sub %rdx, %r11 |
2212 | jbe LABEL(strcmp_exitz) | |
d8725b1f | 2213 | #endif |
7956a3d2 L |
2214 | movzbl (%rsi, %rdx), %ecx |
2215 | movzbl (%rdi, %rdx), %eax | |
2216 | ||
d8725b1f | 2217 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
42e08a54 UD |
2218 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx |
2219 | movl (%rdx,%rcx,4), %ecx | |
2220 | movl (%rdx,%rax,4), %eax | |
d8725b1f | 2221 | #endif |
42e08a54 | 2222 | |
7956a3d2 L |
2223 | sub %ecx, %eax |
2224 | ret | |
2225 | ||
2226 | LABEL(strcmp_exitz): | |
2227 | xor %eax, %eax | |
2228 | ret | |
2229 | ||
2230 | .p2align 4 | |
2231 | LABEL(Byte0): | |
2232 | movzx (%rsi), %ecx | |
2233 | movzx (%rdi), %eax | |
2234 | ||
d8725b1f | 2235 | #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L |
e9f82e0d UD |
2236 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx |
2237 | movl (%rdx,%rcx,4), %ecx | |
2238 | movl (%rdx,%rax,4), %eax | |
d8725b1f | 2239 | #endif |
e9f82e0d | 2240 | |
7956a3d2 L |
2241 | sub %ecx, %eax |
2242 | ret | |
29691210 | 2243 | END (STRCMP) |
7956a3d2 L |
2244 | |
2245 | .section .rodata,"a",@progbits | |
2246 | .p2align 3 | |
2247 | LABEL(unaligned_table): | |
2248 | .int LABEL(ashr_1) - LABEL(unaligned_table) | |
2249 | .int LABEL(ashr_2) - LABEL(unaligned_table) | |
2250 | .int LABEL(ashr_3) - LABEL(unaligned_table) | |
2251 | .int LABEL(ashr_4) - LABEL(unaligned_table) | |
2252 | .int LABEL(ashr_5) - LABEL(unaligned_table) | |
2253 | .int LABEL(ashr_6) - LABEL(unaligned_table) | |
2254 | .int LABEL(ashr_7) - LABEL(unaligned_table) | |
2255 | .int LABEL(ashr_8) - LABEL(unaligned_table) | |
2256 | .int LABEL(ashr_9) - LABEL(unaligned_table) | |
2257 | .int LABEL(ashr_10) - LABEL(unaligned_table) | |
2258 | .int LABEL(ashr_11) - LABEL(unaligned_table) | |
2259 | .int LABEL(ashr_12) - LABEL(unaligned_table) | |
2260 | .int LABEL(ashr_13) - LABEL(unaligned_table) | |
2261 | .int LABEL(ashr_14) - LABEL(unaligned_table) | |
2262 | .int LABEL(ashr_15) - LABEL(unaligned_table) | |
2263 | .int LABEL(ashr_0) - LABEL(unaligned_table) | |
7956a3d2 | 2264 | libc_hidden_builtin_def (STRCMP) |