]>
Commit | Line | Data |
---|---|---|
11ffcacb | 1 | /* strcmp optimized with SSE2. |
6d7e8eda | 2 | Copyright (C) 2017-2023 Free Software Foundation, Inc. |
11ffcacb L |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
16 | License along with the GNU C Library; if not, see | |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
11ffcacb | 18 | |
ceabdcd1 NG |
19 | #include <isa-level.h> |
20 | ||
21 | /* Continue building at ISA level 2 as the strcmp-sse42 is not always | |
22 | preferable for ISA level == 2 CPUs. */ | |
23 | #if ISA_SHOULD_BUILD (2) | |
d561fbb0 NG |
24 | |
25 | # define STRCMP_ISA _sse2 | |
26 | # include "strcmp-naming.h" | |
27 | ||
11ffcacb L |
28 | # include <sysdep.h> |
29 | ||
d561fbb0 | 30 | # undef UPDATE_STRNCMP_COUNTER |
11ffcacb | 31 | |
d561fbb0 NG |
32 | # ifndef LABEL |
33 | # define LABEL(l) L(l) | |
34 | # endif | |
35 | ||
36 | # ifdef USE_AS_STRNCMP | |
37 | /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz | |
38 | if the new counter > the old one or is 0. */ | |
39 | # define UPDATE_STRNCMP_COUNTER \ | |
40 | /* calculate left number to compare */ \ | |
41 | lea -16(%rcx, %r11), %r9; \ | |
42 | cmp %r9, %r11; \ | |
43 | jb LABEL(strcmp_exitz); \ | |
44 | test %r9, %r9; \ | |
45 | je LABEL(strcmp_exitz); \ | |
46 | mov %r9, %r11 | |
47 | ||
48 | # elif defined USE_AS_STRCASECMP_L | |
49 | # include "locale-defines.h" | |
50 | ||
51 | # define UPDATE_STRNCMP_COUNTER | |
52 | # elif defined USE_AS_STRNCASECMP_L | |
53 | # include "locale-defines.h" | |
54 | ||
55 | # define UPDATE_STRNCMP_COUNTER \ | |
56 | /* calculate left number to compare */ \ | |
57 | lea -16(%rcx, %r11), %r9; \ | |
58 | cmp %r9, %r11; \ | |
59 | jb LABEL(strcmp_exitz); \ | |
60 | test %r9, %r9; \ | |
61 | je LABEL(strcmp_exitz); \ | |
62 | mov %r9, %r11 | |
63 | # else | |
64 | # define UPDATE_STRNCMP_COUNTER | |
65 | # endif | |
66 | ||
67 | .text | |
68 | # ifdef USE_AS_STRCASECMP_L | |
69 | # ifndef ENTRY2 | |
70 | # define ENTRY2(name) ENTRY (name) | |
71 | # define END2(name) END (name) | |
72 | # endif | |
73 | ||
74 | ENTRY2 (STRCASECMP) | |
75 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax | |
76 | mov %fs:(%rax),%RDX_LP | |
77 | ||
78 | /* Either 1 or 5 bytes (dependeing if CET is enabled). */ | |
79 | .p2align 4 | |
80 | END2 (STRCASECMP) | |
81 | /* FALLTHROUGH to strcasecmp_l. */ | |
82 | # elif defined USE_AS_STRNCASECMP_L | |
83 | # ifndef ENTRY2 | |
84 | # define ENTRY2(name) ENTRY (name) | |
85 | # define END2(name) END (name) | |
86 | # endif | |
87 | ||
88 | ENTRY2 (STRCASECMP) | |
89 | movq __libc_tsd_LOCALE@gottpoff(%rip),%rax | |
90 | mov %fs:(%rax),%RCX_LP | |
91 | ||
92 | /* Either 1 or 5 bytes (dependeing if CET is enabled). */ | |
93 | .p2align 4 | |
94 | END2 (STRCASECMP) | |
95 | /* FALLTHROUGH to strncasecmp_l. */ | |
96 | # endif | |
97 | ||
98 | ENTRY (STRCMP) | |
99 | # ifdef USE_AS_STRCASECMP_L | |
100 | /* We have to fall back on the C implementation for locales | |
101 | with encodings not matching ASCII for single bytes. */ | |
102 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 | |
103 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rdx), %RAX_LP | |
104 | # else | |
105 | mov (%rdx), %RAX_LP | |
106 | # endif | |
7775574c | 107 | testb $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
d561fbb0 NG |
108 | jne __strcasecmp_l_nonascii |
109 | # elif defined USE_AS_STRNCASECMP_L | |
110 | /* We have to fall back on the C implementation for locales | |
111 | with encodings not matching ASCII for single bytes. */ | |
112 | # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0 | |
113 | mov LOCALE_T___LOCALES+LC_CTYPE*LP_SIZE(%rcx), %RAX_LP | |
114 | # else | |
115 | mov (%rcx), %RAX_LP | |
116 | # endif | |
7775574c | 117 | testb $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax) |
d561fbb0 NG |
118 | jne __strncasecmp_l_nonascii |
119 | # endif | |
120 | ||
121 | /* | |
122 | * This implementation uses SSE to compare up to 16 bytes at a time. | |
123 | */ | |
124 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
125 | test %RDX_LP, %RDX_LP | |
126 | je LABEL(strcmp_exitz) | |
127 | cmp $1, %RDX_LP | |
128 | je LABEL(Byte0) | |
129 | mov %RDX_LP, %R11_LP | |
130 | # endif | |
131 | mov %esi, %ecx | |
132 | mov %edi, %eax | |
133 | /* Use 64bit AND here to avoid long NOP padding. */ | |
134 | and $0x3f, %rcx /* rsi alignment in cache line */ | |
135 | and $0x3f, %rax /* rdi alignment in cache line */ | |
136 | # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L | |
137 | .section .rodata.cst16,"aM",@progbits,16 | |
138 | .align 16 | |
139 | .Llcase_min: | |
140 | .quad 0x3f3f3f3f3f3f3f3f | |
141 | .quad 0x3f3f3f3f3f3f3f3f | |
142 | .Llcase_max: | |
143 | .quad 0x9999999999999999 | |
144 | .quad 0x9999999999999999 | |
145 | .Lcase_add: | |
146 | .quad 0x2020202020202020 | |
147 | .quad 0x2020202020202020 | |
148 | .previous | |
149 | movdqa .Llcase_min(%rip), %xmm5 | |
150 | # define LCASE_MIN_reg %xmm5 | |
151 | movdqa .Llcase_max(%rip), %xmm6 | |
152 | # define LCASE_MAX_reg %xmm6 | |
153 | movdqa .Lcase_add(%rip), %xmm7 | |
154 | # define CASE_ADD_reg %xmm7 | |
155 | # endif | |
156 | cmp $0x30, %ecx | |
157 | ja LABEL(crosscache) /* rsi: 16-byte load will cross cache line */ | |
158 | cmp $0x30, %eax | |
159 | ja LABEL(crosscache) /* rdi: 16-byte load will cross cache line */ | |
160 | movlpd (%rdi), %xmm1 | |
161 | movlpd (%rsi), %xmm2 | |
162 | movhpd 8(%rdi), %xmm1 | |
163 | movhpd 8(%rsi), %xmm2 | |
164 | # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L | |
165 | # define TOLOWER(reg1, reg2) \ | |
166 | movdqa LCASE_MIN_reg, %xmm8; \ | |
167 | movdqa LCASE_MIN_reg, %xmm9; \ | |
168 | paddb reg1, %xmm8; \ | |
169 | paddb reg2, %xmm9; \ | |
170 | pcmpgtb LCASE_MAX_reg, %xmm8; \ | |
171 | pcmpgtb LCASE_MAX_reg, %xmm9; \ | |
172 | pandn CASE_ADD_reg, %xmm8; \ | |
173 | pandn CASE_ADD_reg, %xmm9; \ | |
174 | paddb %xmm8, reg1; \ | |
175 | paddb %xmm9, reg2 | |
176 | TOLOWER (%xmm1, %xmm2) | |
177 | # else | |
178 | # define TOLOWER(reg1, reg2) | |
179 | # endif | |
180 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ | |
181 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
182 | pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */ | |
183 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ | |
184 | pmovmskb %xmm1, %edx | |
185 | sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */ | |
186 | jnz LABEL(less16bytes) /* If not, find different value or null char */ | |
187 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
188 | sub $16, %r11 | |
189 | jbe LABEL(strcmp_exitz) /* finish comparision */ | |
190 | # endif | |
191 | add $16, %rsi /* prepare to search next 16 bytes */ | |
192 | add $16, %rdi /* prepare to search next 16 bytes */ | |
193 | ||
194 | /* | |
195 | * Determine source and destination string offsets from 16-byte alignment. | |
196 | * Use relative offset difference between the two to determine which case | |
197 | * below to use. | |
198 | */ | |
199 | .p2align 4 | |
200 | LABEL(crosscache): | |
201 | and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */ | |
202 | and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */ | |
203 | mov $0xffff, %edx /* for equivalent offset */ | |
204 | xor %r8d, %r8d | |
205 | and $0xf, %ecx /* offset of rsi */ | |
206 | and $0xf, %eax /* offset of rdi */ | |
207 | cmp %eax, %ecx | |
208 | je LABEL(ashr_0) /* rsi and rdi relative offset same */ | |
209 | ja LABEL(bigger) | |
210 | mov %edx, %r8d /* r8d is offset flag for exit tail */ | |
211 | xchg %ecx, %eax | |
212 | xchg %rsi, %rdi | |
213 | LABEL(bigger): | |
214 | lea 15(%rax), %r9 | |
215 | sub %rcx, %r9 | |
216 | lea LABEL(unaligned_table)(%rip), %r10 | |
217 | movslq (%r10, %r9,4), %r9 | |
218 | lea (%r10, %r9), %r10 | |
219 | _CET_NOTRACK jmp *%r10 /* jump to corresponding case */ | |
220 | ||
221 | /* | |
222 | * The following cases will be handled by ashr_0 | |
223 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
224 | * n(0~15) n(0~15) 15(15+ n-n) ashr_0 | |
225 | */ | |
226 | .p2align 4 | |
227 | LABEL(ashr_0): | |
228 | ||
229 | movdqa (%rsi), %xmm1 | |
230 | pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */ | |
231 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
232 | # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L | |
233 | pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */ | |
234 | # else | |
235 | movdqa (%rdi), %xmm2 | |
236 | TOLOWER (%xmm1, %xmm2) | |
237 | pcmpeqb %xmm2, %xmm1 /* compare 16 bytes for equality */ | |
238 | # endif | |
239 | psubb %xmm0, %xmm1 /* packed sub of comparison results*/ | |
240 | pmovmskb %xmm1, %r9d | |
241 | shr %cl, %edx /* adjust 0xffff for offset */ | |
242 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
243 | sub %r9d, %edx | |
244 | /* | |
245 | * edx must be the same with r9d if in left byte (16-rcx) is equal to | |
246 | * the start from (16-rax) and no null char was seen. | |
247 | */ | |
248 | jne LABEL(less32bytes) /* mismatch or null char */ | |
249 | UPDATE_STRNCMP_COUNTER | |
250 | mov $16, %rcx | |
251 | mov $16, %r9 | |
252 | pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */ | |
253 | ||
254 | /* | |
255 | * Now both strings are aligned at 16-byte boundary. Loop over strings | |
256 | * checking 32-bytes per iteration. | |
257 | */ | |
258 | .p2align 4 | |
259 | LABEL(loop_ashr_0): | |
260 | movdqa (%rsi, %rcx), %xmm1 | |
261 | movdqa (%rdi, %rcx), %xmm2 | |
262 | TOLOWER (%xmm1, %xmm2) | |
263 | ||
264 | pcmpeqb %xmm1, %xmm0 | |
265 | pcmpeqb %xmm2, %xmm1 | |
266 | psubb %xmm0, %xmm1 | |
267 | pmovmskb %xmm1, %edx | |
268 | sub $0xffff, %edx | |
269 | jnz LABEL(exit) /* mismatch or null char seen */ | |
270 | ||
271 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
272 | sub $16, %r11 | |
273 | jbe LABEL(strcmp_exitz) | |
274 | # endif | |
275 | add $16, %rcx | |
276 | movdqa (%rsi, %rcx), %xmm1 | |
277 | movdqa (%rdi, %rcx), %xmm2 | |
278 | TOLOWER (%xmm1, %xmm2) | |
279 | ||
280 | pcmpeqb %xmm1, %xmm0 | |
281 | pcmpeqb %xmm2, %xmm1 | |
282 | psubb %xmm0, %xmm1 | |
283 | pmovmskb %xmm1, %edx | |
284 | sub $0xffff, %edx | |
285 | jnz LABEL(exit) | |
286 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
287 | sub $16, %r11 | |
288 | jbe LABEL(strcmp_exitz) | |
289 | # endif | |
290 | add $16, %rcx | |
291 | jmp LABEL(loop_ashr_0) | |
292 | ||
293 | /* | |
294 | * The following cases will be handled by ashr_1 | |
295 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
296 | * n(15) n -15 0(15 +(n-15) - n) ashr_1 | |
297 | */ | |
298 | .p2align 4 | |
299 | LABEL(ashr_1): | |
300 | pxor %xmm0, %xmm0 | |
301 | movdqa (%rdi), %xmm2 | |
302 | movdqa (%rsi), %xmm1 | |
303 | pcmpeqb %xmm1, %xmm0 /* Any null chars? */ | |
304 | pslldq $15, %xmm2 /* shift first string to align with second */ | |
305 | TOLOWER (%xmm1, %xmm2) | |
306 | pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */ | |
307 | psubb %xmm0, %xmm2 /* packed sub of comparison results*/ | |
308 | pmovmskb %xmm2, %r9d | |
309 | shr %cl, %edx /* adjust 0xffff for offset */ | |
310 | shr %cl, %r9d /* adjust for 16-byte offset */ | |
311 | sub %r9d, %edx | |
312 | jnz LABEL(less32bytes) /* mismatch or null char seen */ | |
313 | movdqa (%rdi), %xmm3 | |
314 | UPDATE_STRNCMP_COUNTER | |
315 | ||
316 | pxor %xmm0, %xmm0 | |
317 | mov $16, %rcx /* index for loads*/ | |
318 | mov $1, %r9d /* byte position left over from less32bytes case */ | |
319 | /* | |
320 | * Setup %r10 value allows us to detect crossing a page boundary. | |
321 | * When %r10 goes positive we have crossed a page boundary and | |
322 | * need to do a nibble. | |
323 | */ | |
324 | lea 1(%rdi), %r10 | |
325 | and $0xfff, %r10 /* offset into 4K page */ | |
326 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
327 | ||
328 | .p2align 4 | |
329 | LABEL(loop_ashr_1): | |
330 | add $16, %r10 | |
331 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
332 | ||
333 | LABEL(gobble_ashr_1): | |
334 | movdqa (%rsi, %rcx), %xmm1 | |
335 | movdqa (%rdi, %rcx), %xmm2 | |
336 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
337 | ||
338 | psrldq $1, %xmm3 | |
339 | pslldq $15, %xmm2 | |
340 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
341 | ||
342 | TOLOWER (%xmm1, %xmm2) | |
343 | ||
344 | pcmpeqb %xmm1, %xmm0 | |
345 | pcmpeqb %xmm2, %xmm1 | |
346 | psubb %xmm0, %xmm1 | |
347 | pmovmskb %xmm1, %edx | |
348 | sub $0xffff, %edx | |
349 | jnz LABEL(exit) | |
350 | ||
351 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
352 | sub $16, %r11 | |
353 | jbe LABEL(strcmp_exitz) | |
354 | # endif | |
355 | add $16, %rcx | |
356 | movdqa %xmm4, %xmm3 | |
357 | ||
358 | add $16, %r10 | |
359 | jg LABEL(nibble_ashr_1) /* cross page boundary */ | |
360 | ||
361 | movdqa (%rsi, %rcx), %xmm1 | |
362 | movdqa (%rdi, %rcx), %xmm2 | |
363 | movdqa %xmm2, %xmm4 /* store for next cycle */ | |
364 | ||
365 | psrldq $1, %xmm3 | |
366 | pslldq $15, %xmm2 | |
367 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
368 | ||
369 | TOLOWER (%xmm1, %xmm2) | |
370 | ||
371 | pcmpeqb %xmm1, %xmm0 | |
372 | pcmpeqb %xmm2, %xmm1 | |
373 | psubb %xmm0, %xmm1 | |
374 | pmovmskb %xmm1, %edx | |
375 | sub $0xffff, %edx | |
376 | jnz LABEL(exit) | |
377 | ||
378 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
379 | sub $16, %r11 | |
380 | jbe LABEL(strcmp_exitz) | |
381 | # endif | |
382 | add $16, %rcx | |
383 | movdqa %xmm4, %xmm3 | |
384 | jmp LABEL(loop_ashr_1) | |
385 | ||
386 | /* | |
387 | * Nibble avoids loads across page boundary. This is to avoid a potential | |
388 | * access into unmapped memory. | |
389 | */ | |
390 | .p2align 4 | |
391 | LABEL(nibble_ashr_1): | |
392 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char*/ | |
393 | pmovmskb %xmm0, %edx | |
394 | test $0xfffe, %edx | |
395 | jnz LABEL(ashr_1_exittail) /* find null char*/ | |
396 | ||
397 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
398 | cmp $15, %r11 | |
399 | jbe LABEL(ashr_1_exittail) | |
400 | # endif | |
401 | ||
402 | pxor %xmm0, %xmm0 | |
403 | sub $0x1000, %r10 /* substract 4K from %r10 */ | |
404 | jmp LABEL(gobble_ashr_1) | |
405 | ||
406 | /* | |
407 | * Once find null char, determine if there is a string mismatch | |
408 | * before the null char. | |
409 | */ | |
410 | .p2align 4 | |
411 | LABEL(ashr_1_exittail): | |
412 | movdqa (%rsi, %rcx), %xmm1 | |
413 | psrldq $1, %xmm0 | |
414 | psrldq $1, %xmm3 | |
415 | jmp LABEL(aftertail) | |
416 | ||
417 | /* | |
418 | * The following cases will be handled by ashr_2 | |
419 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
420 | * n(14~15) n -14 1(15 +(n-14) - n) ashr_2 | |
421 | */ | |
422 | .p2align 4 | |
423 | LABEL(ashr_2): | |
424 | pxor %xmm0, %xmm0 | |
425 | movdqa (%rdi), %xmm2 | |
426 | movdqa (%rsi), %xmm1 | |
427 | pcmpeqb %xmm1, %xmm0 | |
428 | pslldq $14, %xmm2 | |
429 | TOLOWER (%xmm1, %xmm2) | |
430 | pcmpeqb %xmm1, %xmm2 | |
431 | psubb %xmm0, %xmm2 | |
432 | pmovmskb %xmm2, %r9d | |
433 | shr %cl, %edx | |
434 | shr %cl, %r9d | |
435 | sub %r9d, %edx | |
436 | jnz LABEL(less32bytes) | |
437 | movdqa (%rdi), %xmm3 | |
438 | UPDATE_STRNCMP_COUNTER | |
439 | ||
440 | pxor %xmm0, %xmm0 | |
441 | mov $16, %rcx /* index for loads */ | |
442 | mov $2, %r9d /* byte position left over from less32bytes case */ | |
443 | /* | |
444 | * Setup %r10 value allows us to detect crossing a page boundary. | |
445 | * When %r10 goes positive we have crossed a page boundary and | |
446 | * need to do a nibble. | |
447 | */ | |
448 | lea 2(%rdi), %r10 | |
449 | and $0xfff, %r10 /* offset into 4K page */ | |
450 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
451 | ||
452 | .p2align 4 | |
453 | LABEL(loop_ashr_2): | |
454 | add $16, %r10 | |
455 | jg LABEL(nibble_ashr_2) | |
456 | ||
457 | LABEL(gobble_ashr_2): | |
458 | movdqa (%rsi, %rcx), %xmm1 | |
459 | movdqa (%rdi, %rcx), %xmm2 | |
460 | movdqa %xmm2, %xmm4 | |
461 | ||
462 | psrldq $2, %xmm3 | |
463 | pslldq $14, %xmm2 | |
464 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
465 | ||
466 | TOLOWER (%xmm1, %xmm2) | |
467 | ||
468 | pcmpeqb %xmm1, %xmm0 | |
469 | pcmpeqb %xmm2, %xmm1 | |
470 | psubb %xmm0, %xmm1 | |
471 | pmovmskb %xmm1, %edx | |
472 | sub $0xffff, %edx | |
473 | jnz LABEL(exit) | |
474 | ||
475 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
476 | sub $16, %r11 | |
477 | jbe LABEL(strcmp_exitz) | |
478 | # endif | |
479 | ||
480 | add $16, %rcx | |
481 | movdqa %xmm4, %xmm3 | |
482 | ||
483 | add $16, %r10 | |
484 | jg LABEL(nibble_ashr_2) /* cross page boundary */ | |
485 | ||
486 | movdqa (%rsi, %rcx), %xmm1 | |
487 | movdqa (%rdi, %rcx), %xmm2 | |
488 | movdqa %xmm2, %xmm4 | |
489 | ||
490 | psrldq $2, %xmm3 | |
491 | pslldq $14, %xmm2 | |
492 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
493 | ||
494 | TOLOWER (%xmm1, %xmm2) | |
495 | ||
496 | pcmpeqb %xmm1, %xmm0 | |
497 | pcmpeqb %xmm2, %xmm1 | |
498 | psubb %xmm0, %xmm1 | |
499 | pmovmskb %xmm1, %edx | |
500 | sub $0xffff, %edx | |
501 | jnz LABEL(exit) | |
502 | ||
503 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
504 | sub $16, %r11 | |
505 | jbe LABEL(strcmp_exitz) | |
506 | # endif | |
507 | ||
508 | add $16, %rcx | |
509 | movdqa %xmm4, %xmm3 | |
510 | jmp LABEL(loop_ashr_2) | |
511 | ||
512 | .p2align 4 | |
513 | LABEL(nibble_ashr_2): | |
514 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
515 | pmovmskb %xmm0, %edx | |
516 | test $0xfffc, %edx | |
517 | jnz LABEL(ashr_2_exittail) | |
518 | ||
519 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
520 | cmp $14, %r11 | |
521 | jbe LABEL(ashr_2_exittail) | |
522 | # endif | |
523 | ||
524 | pxor %xmm0, %xmm0 | |
525 | sub $0x1000, %r10 | |
526 | jmp LABEL(gobble_ashr_2) | |
527 | ||
528 | .p2align 4 | |
529 | LABEL(ashr_2_exittail): | |
530 | movdqa (%rsi, %rcx), %xmm1 | |
531 | psrldq $2, %xmm0 | |
532 | psrldq $2, %xmm3 | |
533 | jmp LABEL(aftertail) | |
534 | ||
535 | /* | |
536 | * The following cases will be handled by ashr_3 | |
537 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
538 | * n(13~15) n -13 2(15 +(n-13) - n) ashr_3 | |
539 | */ | |
540 | .p2align 4 | |
541 | LABEL(ashr_3): | |
542 | pxor %xmm0, %xmm0 | |
543 | movdqa (%rdi), %xmm2 | |
544 | movdqa (%rsi), %xmm1 | |
545 | pcmpeqb %xmm1, %xmm0 | |
546 | pslldq $13, %xmm2 | |
547 | TOLOWER (%xmm1, %xmm2) | |
548 | pcmpeqb %xmm1, %xmm2 | |
549 | psubb %xmm0, %xmm2 | |
550 | pmovmskb %xmm2, %r9d | |
551 | shr %cl, %edx | |
552 | shr %cl, %r9d | |
553 | sub %r9d, %edx | |
554 | jnz LABEL(less32bytes) | |
555 | movdqa (%rdi), %xmm3 | |
556 | ||
557 | UPDATE_STRNCMP_COUNTER | |
558 | ||
559 | pxor %xmm0, %xmm0 | |
560 | mov $16, %rcx /* index for loads */ | |
561 | mov $3, %r9d /* byte position left over from less32bytes case */ | |
562 | /* | |
563 | * Setup %r10 value allows us to detect crossing a page boundary. | |
564 | * When %r10 goes positive we have crossed a page boundary and | |
565 | * need to do a nibble. | |
566 | */ | |
567 | lea 3(%rdi), %r10 | |
568 | and $0xfff, %r10 /* offset into 4K page */ | |
569 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
570 | ||
571 | .p2align 4 | |
572 | LABEL(loop_ashr_3): | |
573 | add $16, %r10 | |
574 | jg LABEL(nibble_ashr_3) | |
575 | ||
576 | LABEL(gobble_ashr_3): | |
577 | movdqa (%rsi, %rcx), %xmm1 | |
578 | movdqa (%rdi, %rcx), %xmm2 | |
579 | movdqa %xmm2, %xmm4 | |
580 | ||
581 | psrldq $3, %xmm3 | |
582 | pslldq $13, %xmm2 | |
583 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
584 | ||
585 | TOLOWER (%xmm1, %xmm2) | |
586 | ||
587 | pcmpeqb %xmm1, %xmm0 | |
588 | pcmpeqb %xmm2, %xmm1 | |
589 | psubb %xmm0, %xmm1 | |
590 | pmovmskb %xmm1, %edx | |
591 | sub $0xffff, %edx | |
592 | jnz LABEL(exit) | |
593 | ||
594 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
595 | sub $16, %r11 | |
596 | jbe LABEL(strcmp_exitz) | |
597 | # endif | |
598 | ||
599 | add $16, %rcx | |
600 | movdqa %xmm4, %xmm3 | |
601 | ||
602 | add $16, %r10 | |
603 | jg LABEL(nibble_ashr_3) /* cross page boundary */ | |
604 | ||
605 | movdqa (%rsi, %rcx), %xmm1 | |
606 | movdqa (%rdi, %rcx), %xmm2 | |
607 | movdqa %xmm2, %xmm4 | |
608 | ||
609 | psrldq $3, %xmm3 | |
610 | pslldq $13, %xmm2 | |
611 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
612 | ||
613 | TOLOWER (%xmm1, %xmm2) | |
614 | ||
615 | pcmpeqb %xmm1, %xmm0 | |
616 | pcmpeqb %xmm2, %xmm1 | |
617 | psubb %xmm0, %xmm1 | |
618 | pmovmskb %xmm1, %edx | |
619 | sub $0xffff, %edx | |
620 | jnz LABEL(exit) | |
621 | ||
622 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
623 | sub $16, %r11 | |
624 | jbe LABEL(strcmp_exitz) | |
625 | # endif | |
626 | ||
627 | add $16, %rcx | |
628 | movdqa %xmm4, %xmm3 | |
629 | jmp LABEL(loop_ashr_3) | |
630 | ||
631 | .p2align 4 | |
632 | LABEL(nibble_ashr_3): | |
633 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
634 | pmovmskb %xmm0, %edx | |
635 | test $0xfff8, %edx | |
636 | jnz LABEL(ashr_3_exittail) | |
637 | ||
638 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
639 | cmp $13, %r11 | |
640 | jbe LABEL(ashr_3_exittail) | |
641 | # endif | |
642 | ||
643 | pxor %xmm0, %xmm0 | |
644 | sub $0x1000, %r10 | |
645 | jmp LABEL(gobble_ashr_3) | |
646 | ||
647 | .p2align 4 | |
648 | LABEL(ashr_3_exittail): | |
649 | movdqa (%rsi, %rcx), %xmm1 | |
650 | psrldq $3, %xmm0 | |
651 | psrldq $3, %xmm3 | |
652 | jmp LABEL(aftertail) | |
653 | ||
654 | /* | |
655 | * The following cases will be handled by ashr_4 | |
656 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
657 | * n(12~15) n -12 3(15 +(n-12) - n) ashr_4 | |
658 | */ | |
659 | .p2align 4 | |
660 | LABEL(ashr_4): | |
661 | pxor %xmm0, %xmm0 | |
662 | movdqa (%rdi), %xmm2 | |
663 | movdqa (%rsi), %xmm1 | |
664 | pcmpeqb %xmm1, %xmm0 | |
665 | pslldq $12, %xmm2 | |
666 | TOLOWER (%xmm1, %xmm2) | |
667 | pcmpeqb %xmm1, %xmm2 | |
668 | psubb %xmm0, %xmm2 | |
669 | pmovmskb %xmm2, %r9d | |
670 | shr %cl, %edx | |
671 | shr %cl, %r9d | |
672 | sub %r9d, %edx | |
673 | jnz LABEL(less32bytes) | |
674 | movdqa (%rdi), %xmm3 | |
675 | ||
676 | UPDATE_STRNCMP_COUNTER | |
677 | ||
678 | pxor %xmm0, %xmm0 | |
679 | mov $16, %rcx /* index for loads */ | |
680 | mov $4, %r9d /* byte position left over from less32bytes case */ | |
681 | /* | |
682 | * Setup %r10 value allows us to detect crossing a page boundary. | |
683 | * When %r10 goes positive we have crossed a page boundary and | |
684 | * need to do a nibble. | |
685 | */ | |
686 | lea 4(%rdi), %r10 | |
687 | and $0xfff, %r10 /* offset into 4K page */ | |
688 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
689 | ||
690 | .p2align 4 | |
691 | LABEL(loop_ashr_4): | |
692 | add $16, %r10 | |
693 | jg LABEL(nibble_ashr_4) | |
694 | ||
695 | LABEL(gobble_ashr_4): | |
696 | movdqa (%rsi, %rcx), %xmm1 | |
697 | movdqa (%rdi, %rcx), %xmm2 | |
698 | movdqa %xmm2, %xmm4 | |
699 | ||
700 | psrldq $4, %xmm3 | |
701 | pslldq $12, %xmm2 | |
702 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
703 | ||
704 | TOLOWER (%xmm1, %xmm2) | |
705 | ||
706 | pcmpeqb %xmm1, %xmm0 | |
707 | pcmpeqb %xmm2, %xmm1 | |
708 | psubb %xmm0, %xmm1 | |
709 | pmovmskb %xmm1, %edx | |
710 | sub $0xffff, %edx | |
711 | jnz LABEL(exit) | |
712 | ||
713 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
714 | sub $16, %r11 | |
715 | jbe LABEL(strcmp_exitz) | |
716 | # endif | |
717 | ||
718 | add $16, %rcx | |
719 | movdqa %xmm4, %xmm3 | |
720 | ||
721 | add $16, %r10 | |
722 | jg LABEL(nibble_ashr_4) /* cross page boundary */ | |
723 | ||
724 | movdqa (%rsi, %rcx), %xmm1 | |
725 | movdqa (%rdi, %rcx), %xmm2 | |
726 | movdqa %xmm2, %xmm4 | |
727 | ||
728 | psrldq $4, %xmm3 | |
729 | pslldq $12, %xmm2 | |
730 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
731 | ||
732 | TOLOWER (%xmm1, %xmm2) | |
733 | ||
734 | pcmpeqb %xmm1, %xmm0 | |
735 | pcmpeqb %xmm2, %xmm1 | |
736 | psubb %xmm0, %xmm1 | |
737 | pmovmskb %xmm1, %edx | |
738 | sub $0xffff, %edx | |
739 | jnz LABEL(exit) | |
740 | ||
741 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
742 | sub $16, %r11 | |
743 | jbe LABEL(strcmp_exitz) | |
744 | # endif | |
745 | ||
746 | add $16, %rcx | |
747 | movdqa %xmm4, %xmm3 | |
748 | jmp LABEL(loop_ashr_4) | |
749 | ||
750 | .p2align 4 | |
751 | LABEL(nibble_ashr_4): | |
752 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
753 | pmovmskb %xmm0, %edx | |
754 | test $0xfff0, %edx | |
755 | jnz LABEL(ashr_4_exittail) | |
756 | ||
757 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
758 | cmp $12, %r11 | |
759 | jbe LABEL(ashr_4_exittail) | |
760 | # endif | |
761 | ||
762 | pxor %xmm0, %xmm0 | |
763 | sub $0x1000, %r10 | |
764 | jmp LABEL(gobble_ashr_4) | |
765 | ||
766 | .p2align 4 | |
767 | LABEL(ashr_4_exittail): | |
768 | movdqa (%rsi, %rcx), %xmm1 | |
769 | psrldq $4, %xmm0 | |
770 | psrldq $4, %xmm3 | |
771 | jmp LABEL(aftertail) | |
772 | ||
773 | /* | |
774 | * The following cases will be handled by ashr_5 | |
775 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
776 | * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5 | |
777 | */ | |
778 | .p2align 4 | |
779 | LABEL(ashr_5): | |
780 | pxor %xmm0, %xmm0 | |
781 | movdqa (%rdi), %xmm2 | |
782 | movdqa (%rsi), %xmm1 | |
783 | pcmpeqb %xmm1, %xmm0 | |
784 | pslldq $11, %xmm2 | |
785 | TOLOWER (%xmm1, %xmm2) | |
786 | pcmpeqb %xmm1, %xmm2 | |
787 | psubb %xmm0, %xmm2 | |
788 | pmovmskb %xmm2, %r9d | |
789 | shr %cl, %edx | |
790 | shr %cl, %r9d | |
791 | sub %r9d, %edx | |
792 | jnz LABEL(less32bytes) | |
793 | movdqa (%rdi), %xmm3 | |
794 | ||
795 | UPDATE_STRNCMP_COUNTER | |
796 | ||
797 | pxor %xmm0, %xmm0 | |
798 | mov $16, %rcx /* index for loads */ | |
799 | mov $5, %r9d /* byte position left over from less32bytes case */ | |
800 | /* | |
801 | * Setup %r10 value allows us to detect crossing a page boundary. | |
802 | * When %r10 goes positive we have crossed a page boundary and | |
803 | * need to do a nibble. | |
804 | */ | |
805 | lea 5(%rdi), %r10 | |
806 | and $0xfff, %r10 /* offset into 4K page */ | |
807 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
808 | ||
809 | .p2align 4 | |
810 | LABEL(loop_ashr_5): | |
811 | add $16, %r10 | |
812 | jg LABEL(nibble_ashr_5) | |
813 | ||
814 | LABEL(gobble_ashr_5): | |
815 | movdqa (%rsi, %rcx), %xmm1 | |
816 | movdqa (%rdi, %rcx), %xmm2 | |
817 | movdqa %xmm2, %xmm4 | |
818 | ||
819 | psrldq $5, %xmm3 | |
820 | pslldq $11, %xmm2 | |
821 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
822 | ||
823 | TOLOWER (%xmm1, %xmm2) | |
824 | ||
825 | pcmpeqb %xmm1, %xmm0 | |
826 | pcmpeqb %xmm2, %xmm1 | |
827 | psubb %xmm0, %xmm1 | |
828 | pmovmskb %xmm1, %edx | |
829 | sub $0xffff, %edx | |
830 | jnz LABEL(exit) | |
831 | ||
832 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
833 | sub $16, %r11 | |
834 | jbe LABEL(strcmp_exitz) | |
835 | # endif | |
836 | ||
837 | add $16, %rcx | |
838 | movdqa %xmm4, %xmm3 | |
839 | ||
840 | add $16, %r10 | |
841 | jg LABEL(nibble_ashr_5) /* cross page boundary */ | |
842 | ||
843 | movdqa (%rsi, %rcx), %xmm1 | |
844 | movdqa (%rdi, %rcx), %xmm2 | |
845 | movdqa %xmm2, %xmm4 | |
846 | ||
847 | psrldq $5, %xmm3 | |
848 | pslldq $11, %xmm2 | |
849 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
850 | ||
851 | TOLOWER (%xmm1, %xmm2) | |
852 | ||
853 | pcmpeqb %xmm1, %xmm0 | |
854 | pcmpeqb %xmm2, %xmm1 | |
855 | psubb %xmm0, %xmm1 | |
856 | pmovmskb %xmm1, %edx | |
857 | sub $0xffff, %edx | |
858 | jnz LABEL(exit) | |
859 | ||
860 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
861 | sub $16, %r11 | |
862 | jbe LABEL(strcmp_exitz) | |
863 | # endif | |
864 | ||
865 | add $16, %rcx | |
866 | movdqa %xmm4, %xmm3 | |
867 | jmp LABEL(loop_ashr_5) | |
868 | ||
869 | .p2align 4 | |
870 | LABEL(nibble_ashr_5): | |
871 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
872 | pmovmskb %xmm0, %edx | |
873 | test $0xffe0, %edx | |
874 | jnz LABEL(ashr_5_exittail) | |
875 | ||
876 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
877 | cmp $11, %r11 | |
878 | jbe LABEL(ashr_5_exittail) | |
879 | # endif | |
880 | ||
881 | pxor %xmm0, %xmm0 | |
882 | sub $0x1000, %r10 | |
883 | jmp LABEL(gobble_ashr_5) | |
884 | ||
885 | .p2align 4 | |
886 | LABEL(ashr_5_exittail): | |
887 | movdqa (%rsi, %rcx), %xmm1 | |
888 | psrldq $5, %xmm0 | |
889 | psrldq $5, %xmm3 | |
890 | jmp LABEL(aftertail) | |
891 | ||
892 | /* | |
893 | * The following cases will be handled by ashr_6 | |
894 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
895 | * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6 | |
896 | */ | |
897 | .p2align 4 | |
898 | LABEL(ashr_6): | |
899 | pxor %xmm0, %xmm0 | |
900 | movdqa (%rdi), %xmm2 | |
901 | movdqa (%rsi), %xmm1 | |
902 | pcmpeqb %xmm1, %xmm0 | |
903 | pslldq $10, %xmm2 | |
904 | TOLOWER (%xmm1, %xmm2) | |
905 | pcmpeqb %xmm1, %xmm2 | |
906 | psubb %xmm0, %xmm2 | |
907 | pmovmskb %xmm2, %r9d | |
908 | shr %cl, %edx | |
909 | shr %cl, %r9d | |
910 | sub %r9d, %edx | |
911 | jnz LABEL(less32bytes) | |
912 | movdqa (%rdi), %xmm3 | |
913 | ||
914 | UPDATE_STRNCMP_COUNTER | |
915 | ||
916 | pxor %xmm0, %xmm0 | |
917 | mov $16, %rcx /* index for loads */ | |
918 | mov $6, %r9d /* byte position left over from less32bytes case */ | |
919 | /* | |
920 | * Setup %r10 value allows us to detect crossing a page boundary. | |
921 | * When %r10 goes positive we have crossed a page boundary and | |
922 | * need to do a nibble. | |
923 | */ | |
924 | lea 6(%rdi), %r10 | |
925 | and $0xfff, %r10 /* offset into 4K page */ | |
926 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
927 | ||
928 | .p2align 4 | |
929 | LABEL(loop_ashr_6): | |
930 | add $16, %r10 | |
931 | jg LABEL(nibble_ashr_6) | |
932 | ||
933 | LABEL(gobble_ashr_6): | |
934 | movdqa (%rsi, %rcx), %xmm1 | |
935 | movdqa (%rdi, %rcx), %xmm2 | |
936 | movdqa %xmm2, %xmm4 | |
937 | ||
938 | psrldq $6, %xmm3 | |
939 | pslldq $10, %xmm2 | |
940 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
941 | ||
942 | TOLOWER (%xmm1, %xmm2) | |
943 | ||
944 | pcmpeqb %xmm1, %xmm0 | |
945 | pcmpeqb %xmm2, %xmm1 | |
946 | psubb %xmm0, %xmm1 | |
947 | pmovmskb %xmm1, %edx | |
948 | sub $0xffff, %edx | |
949 | jnz LABEL(exit) | |
950 | ||
951 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
952 | sub $16, %r11 | |
953 | jbe LABEL(strcmp_exitz) | |
954 | # endif | |
955 | ||
956 | add $16, %rcx | |
957 | movdqa %xmm4, %xmm3 | |
958 | ||
959 | add $16, %r10 | |
960 | jg LABEL(nibble_ashr_6) /* cross page boundary */ | |
961 | ||
962 | movdqa (%rsi, %rcx), %xmm1 | |
963 | movdqa (%rdi, %rcx), %xmm2 | |
964 | movdqa %xmm2, %xmm4 | |
965 | ||
966 | psrldq $6, %xmm3 | |
967 | pslldq $10, %xmm2 | |
968 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
969 | ||
970 | TOLOWER (%xmm1, %xmm2) | |
971 | ||
972 | pcmpeqb %xmm1, %xmm0 | |
973 | pcmpeqb %xmm2, %xmm1 | |
974 | psubb %xmm0, %xmm1 | |
975 | pmovmskb %xmm1, %edx | |
976 | sub $0xffff, %edx | |
977 | jnz LABEL(exit) | |
978 | ||
979 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
980 | sub $16, %r11 | |
981 | jbe LABEL(strcmp_exitz) | |
982 | # endif | |
983 | ||
984 | add $16, %rcx | |
985 | movdqa %xmm4, %xmm3 | |
986 | jmp LABEL(loop_ashr_6) | |
987 | ||
988 | .p2align 4 | |
989 | LABEL(nibble_ashr_6): | |
990 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
991 | pmovmskb %xmm0, %edx | |
992 | test $0xffc0, %edx | |
993 | jnz LABEL(ashr_6_exittail) | |
994 | ||
995 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
996 | cmp $10, %r11 | |
997 | jbe LABEL(ashr_6_exittail) | |
998 | # endif | |
999 | ||
1000 | pxor %xmm0, %xmm0 | |
1001 | sub $0x1000, %r10 | |
1002 | jmp LABEL(gobble_ashr_6) | |
1003 | ||
1004 | .p2align 4 | |
1005 | LABEL(ashr_6_exittail): | |
1006 | movdqa (%rsi, %rcx), %xmm1 | |
1007 | psrldq $6, %xmm0 | |
1008 | psrldq $6, %xmm3 | |
1009 | jmp LABEL(aftertail) | |
1010 | ||
1011 | /* | |
1012 | * The following cases will be handled by ashr_7 | |
1013 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1014 | * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7 | |
1015 | */ | |
1016 | .p2align 4 | |
1017 | LABEL(ashr_7): | |
1018 | pxor %xmm0, %xmm0 | |
1019 | movdqa (%rdi), %xmm2 | |
1020 | movdqa (%rsi), %xmm1 | |
1021 | pcmpeqb %xmm1, %xmm0 | |
1022 | pslldq $9, %xmm2 | |
1023 | TOLOWER (%xmm1, %xmm2) | |
1024 | pcmpeqb %xmm1, %xmm2 | |
1025 | psubb %xmm0, %xmm2 | |
1026 | pmovmskb %xmm2, %r9d | |
1027 | shr %cl, %edx | |
1028 | shr %cl, %r9d | |
1029 | sub %r9d, %edx | |
1030 | jnz LABEL(less32bytes) | |
1031 | movdqa (%rdi), %xmm3 | |
1032 | ||
1033 | UPDATE_STRNCMP_COUNTER | |
1034 | ||
1035 | pxor %xmm0, %xmm0 | |
1036 | mov $16, %rcx /* index for loads */ | |
1037 | mov $7, %r9d /* byte position left over from less32bytes case */ | |
1038 | /* | |
1039 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1040 | * When %r10 goes positive we have crossed a page boundary and | |
1041 | * need to do a nibble. | |
1042 | */ | |
1043 | lea 7(%rdi), %r10 | |
1044 | and $0xfff, %r10 /* offset into 4K page */ | |
1045 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1046 | ||
1047 | .p2align 4 | |
1048 | LABEL(loop_ashr_7): | |
1049 | add $16, %r10 | |
1050 | jg LABEL(nibble_ashr_7) | |
1051 | ||
1052 | LABEL(gobble_ashr_7): | |
1053 | movdqa (%rsi, %rcx), %xmm1 | |
1054 | movdqa (%rdi, %rcx), %xmm2 | |
1055 | movdqa %xmm2, %xmm4 | |
1056 | ||
1057 | psrldq $7, %xmm3 | |
1058 | pslldq $9, %xmm2 | |
1059 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1060 | ||
1061 | TOLOWER (%xmm1, %xmm2) | |
1062 | ||
1063 | pcmpeqb %xmm1, %xmm0 | |
1064 | pcmpeqb %xmm2, %xmm1 | |
1065 | psubb %xmm0, %xmm1 | |
1066 | pmovmskb %xmm1, %edx | |
1067 | sub $0xffff, %edx | |
1068 | jnz LABEL(exit) | |
1069 | ||
1070 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1071 | sub $16, %r11 | |
1072 | jbe LABEL(strcmp_exitz) | |
1073 | # endif | |
1074 | ||
1075 | add $16, %rcx | |
1076 | movdqa %xmm4, %xmm3 | |
1077 | ||
1078 | add $16, %r10 | |
1079 | jg LABEL(nibble_ashr_7) /* cross page boundary */ | |
1080 | ||
1081 | movdqa (%rsi, %rcx), %xmm1 | |
1082 | movdqa (%rdi, %rcx), %xmm2 | |
1083 | movdqa %xmm2, %xmm4 | |
1084 | ||
1085 | psrldq $7, %xmm3 | |
1086 | pslldq $9, %xmm2 | |
1087 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1088 | ||
1089 | TOLOWER (%xmm1, %xmm2) | |
1090 | ||
1091 | pcmpeqb %xmm1, %xmm0 | |
1092 | pcmpeqb %xmm2, %xmm1 | |
1093 | psubb %xmm0, %xmm1 | |
1094 | pmovmskb %xmm1, %edx | |
1095 | sub $0xffff, %edx | |
1096 | jnz LABEL(exit) | |
1097 | ||
1098 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1099 | sub $16, %r11 | |
1100 | jbe LABEL(strcmp_exitz) | |
1101 | # endif | |
1102 | ||
1103 | add $16, %rcx | |
1104 | movdqa %xmm4, %xmm3 | |
1105 | jmp LABEL(loop_ashr_7) | |
1106 | ||
1107 | .p2align 4 | |
1108 | LABEL(nibble_ashr_7): | |
1109 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1110 | pmovmskb %xmm0, %edx | |
1111 | test $0xff80, %edx | |
1112 | jnz LABEL(ashr_7_exittail) | |
1113 | ||
1114 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1115 | cmp $9, %r11 | |
1116 | jbe LABEL(ashr_7_exittail) | |
1117 | # endif | |
1118 | ||
1119 | pxor %xmm0, %xmm0 | |
1120 | sub $0x1000, %r10 | |
1121 | jmp LABEL(gobble_ashr_7) | |
1122 | ||
1123 | .p2align 4 | |
1124 | LABEL(ashr_7_exittail): | |
1125 | movdqa (%rsi, %rcx), %xmm1 | |
1126 | psrldq $7, %xmm0 | |
1127 | psrldq $7, %xmm3 | |
1128 | jmp LABEL(aftertail) | |
1129 | ||
1130 | /* | |
1131 | * The following cases will be handled by ashr_8 | |
1132 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1133 | * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8 | |
1134 | */ | |
1135 | .p2align 4 | |
1136 | LABEL(ashr_8): | |
1137 | pxor %xmm0, %xmm0 | |
1138 | movdqa (%rdi), %xmm2 | |
1139 | movdqa (%rsi), %xmm1 | |
1140 | pcmpeqb %xmm1, %xmm0 | |
1141 | pslldq $8, %xmm2 | |
1142 | TOLOWER (%xmm1, %xmm2) | |
1143 | pcmpeqb %xmm1, %xmm2 | |
1144 | psubb %xmm0, %xmm2 | |
1145 | pmovmskb %xmm2, %r9d | |
1146 | shr %cl, %edx | |
1147 | shr %cl, %r9d | |
1148 | sub %r9d, %edx | |
1149 | jnz LABEL(less32bytes) | |
1150 | movdqa (%rdi), %xmm3 | |
1151 | ||
1152 | UPDATE_STRNCMP_COUNTER | |
1153 | ||
1154 | pxor %xmm0, %xmm0 | |
1155 | mov $16, %rcx /* index for loads */ | |
1156 | mov $8, %r9d /* byte position left over from less32bytes case */ | |
1157 | /* | |
1158 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1159 | * When %r10 goes positive we have crossed a page boundary and | |
1160 | * need to do a nibble. | |
1161 | */ | |
1162 | lea 8(%rdi), %r10 | |
1163 | and $0xfff, %r10 /* offset into 4K page */ | |
1164 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1165 | ||
1166 | .p2align 4 | |
1167 | LABEL(loop_ashr_8): | |
1168 | add $16, %r10 | |
1169 | jg LABEL(nibble_ashr_8) | |
1170 | ||
1171 | LABEL(gobble_ashr_8): | |
1172 | movdqa (%rsi, %rcx), %xmm1 | |
1173 | movdqa (%rdi, %rcx), %xmm2 | |
1174 | movdqa %xmm2, %xmm4 | |
1175 | ||
1176 | psrldq $8, %xmm3 | |
1177 | pslldq $8, %xmm2 | |
1178 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1179 | ||
1180 | TOLOWER (%xmm1, %xmm2) | |
1181 | ||
1182 | pcmpeqb %xmm1, %xmm0 | |
1183 | pcmpeqb %xmm2, %xmm1 | |
1184 | psubb %xmm0, %xmm1 | |
1185 | pmovmskb %xmm1, %edx | |
1186 | sub $0xffff, %edx | |
1187 | jnz LABEL(exit) | |
1188 | ||
1189 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1190 | sub $16, %r11 | |
1191 | jbe LABEL(strcmp_exitz) | |
1192 | # endif | |
11ffcacb | 1193 | |
d561fbb0 NG |
1194 | add $16, %rcx |
1195 | movdqa %xmm4, %xmm3 | |
1196 | ||
1197 | add $16, %r10 | |
1198 | jg LABEL(nibble_ashr_8) /* cross page boundary */ | |
1199 | ||
1200 | movdqa (%rsi, %rcx), %xmm1 | |
1201 | movdqa (%rdi, %rcx), %xmm2 | |
1202 | movdqa %xmm2, %xmm4 | |
1203 | ||
1204 | psrldq $8, %xmm3 | |
1205 | pslldq $8, %xmm2 | |
1206 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1207 | ||
1208 | TOLOWER (%xmm1, %xmm2) | |
1209 | ||
1210 | pcmpeqb %xmm1, %xmm0 | |
1211 | pcmpeqb %xmm2, %xmm1 | |
1212 | psubb %xmm0, %xmm1 | |
1213 | pmovmskb %xmm1, %edx | |
1214 | sub $0xffff, %edx | |
1215 | jnz LABEL(exit) | |
1216 | ||
1217 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1218 | sub $16, %r11 | |
1219 | jbe LABEL(strcmp_exitz) | |
1220 | # endif | |
1221 | ||
1222 | add $16, %rcx | |
1223 | movdqa %xmm4, %xmm3 | |
1224 | jmp LABEL(loop_ashr_8) | |
1225 | ||
1226 | .p2align 4 | |
1227 | LABEL(nibble_ashr_8): | |
1228 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1229 | pmovmskb %xmm0, %edx | |
1230 | test $0xff00, %edx | |
1231 | jnz LABEL(ashr_8_exittail) | |
1232 | ||
1233 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1234 | cmp $8, %r11 | |
1235 | jbe LABEL(ashr_8_exittail) | |
1236 | # endif | |
1237 | ||
1238 | pxor %xmm0, %xmm0 | |
1239 | sub $0x1000, %r10 | |
1240 | jmp LABEL(gobble_ashr_8) | |
1241 | ||
1242 | .p2align 4 | |
1243 | LABEL(ashr_8_exittail): | |
1244 | movdqa (%rsi, %rcx), %xmm1 | |
1245 | psrldq $8, %xmm0 | |
1246 | psrldq $8, %xmm3 | |
1247 | jmp LABEL(aftertail) | |
1248 | ||
1249 | /* | |
1250 | * The following cases will be handled by ashr_9 | |
1251 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1252 | * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9 | |
1253 | */ | |
1254 | .p2align 4 | |
1255 | LABEL(ashr_9): | |
1256 | pxor %xmm0, %xmm0 | |
1257 | movdqa (%rdi), %xmm2 | |
1258 | movdqa (%rsi), %xmm1 | |
1259 | pcmpeqb %xmm1, %xmm0 | |
1260 | pslldq $7, %xmm2 | |
1261 | TOLOWER (%xmm1, %xmm2) | |
1262 | pcmpeqb %xmm1, %xmm2 | |
1263 | psubb %xmm0, %xmm2 | |
1264 | pmovmskb %xmm2, %r9d | |
1265 | shr %cl, %edx | |
1266 | shr %cl, %r9d | |
1267 | sub %r9d, %edx | |
1268 | jnz LABEL(less32bytes) | |
1269 | movdqa (%rdi), %xmm3 | |
1270 | ||
1271 | UPDATE_STRNCMP_COUNTER | |
1272 | ||
1273 | pxor %xmm0, %xmm0 | |
1274 | mov $16, %rcx /* index for loads */ | |
1275 | mov $9, %r9d /* byte position left over from less32bytes case */ | |
1276 | /* | |
1277 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1278 | * When %r10 goes positive we have crossed a page boundary and | |
1279 | * need to do a nibble. | |
1280 | */ | |
1281 | lea 9(%rdi), %r10 | |
1282 | and $0xfff, %r10 /* offset into 4K page */ | |
1283 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1284 | ||
1285 | .p2align 4 | |
1286 | LABEL(loop_ashr_9): | |
1287 | add $16, %r10 | |
1288 | jg LABEL(nibble_ashr_9) | |
1289 | ||
1290 | LABEL(gobble_ashr_9): | |
1291 | movdqa (%rsi, %rcx), %xmm1 | |
1292 | movdqa (%rdi, %rcx), %xmm2 | |
1293 | movdqa %xmm2, %xmm4 | |
1294 | ||
1295 | psrldq $9, %xmm3 | |
1296 | pslldq $7, %xmm2 | |
1297 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1298 | ||
1299 | TOLOWER (%xmm1, %xmm2) | |
1300 | ||
1301 | pcmpeqb %xmm1, %xmm0 | |
1302 | pcmpeqb %xmm2, %xmm1 | |
1303 | psubb %xmm0, %xmm1 | |
1304 | pmovmskb %xmm1, %edx | |
1305 | sub $0xffff, %edx | |
1306 | jnz LABEL(exit) | |
1307 | ||
1308 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1309 | sub $16, %r11 | |
1310 | jbe LABEL(strcmp_exitz) | |
1311 | # endif | |
1312 | ||
1313 | add $16, %rcx | |
1314 | movdqa %xmm4, %xmm3 | |
1315 | ||
1316 | add $16, %r10 | |
1317 | jg LABEL(nibble_ashr_9) /* cross page boundary */ | |
1318 | ||
1319 | movdqa (%rsi, %rcx), %xmm1 | |
1320 | movdqa (%rdi, %rcx), %xmm2 | |
1321 | movdqa %xmm2, %xmm4 | |
1322 | ||
1323 | psrldq $9, %xmm3 | |
1324 | pslldq $7, %xmm2 | |
1325 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1326 | ||
1327 | TOLOWER (%xmm1, %xmm2) | |
1328 | ||
1329 | pcmpeqb %xmm1, %xmm0 | |
1330 | pcmpeqb %xmm2, %xmm1 | |
1331 | psubb %xmm0, %xmm1 | |
1332 | pmovmskb %xmm1, %edx | |
1333 | sub $0xffff, %edx | |
1334 | jnz LABEL(exit) | |
1335 | ||
1336 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1337 | sub $16, %r11 | |
1338 | jbe LABEL(strcmp_exitz) | |
1339 | # endif | |
1340 | ||
1341 | add $16, %rcx | |
1342 | movdqa %xmm4, %xmm3 /* store for next cycle */ | |
1343 | jmp LABEL(loop_ashr_9) | |
1344 | ||
1345 | .p2align 4 | |
1346 | LABEL(nibble_ashr_9): | |
1347 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1348 | pmovmskb %xmm0, %edx | |
1349 | test $0xfe00, %edx | |
1350 | jnz LABEL(ashr_9_exittail) | |
1351 | ||
1352 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1353 | cmp $7, %r11 | |
1354 | jbe LABEL(ashr_9_exittail) | |
1355 | # endif | |
1356 | ||
1357 | pxor %xmm0, %xmm0 | |
1358 | sub $0x1000, %r10 | |
1359 | jmp LABEL(gobble_ashr_9) | |
1360 | ||
1361 | .p2align 4 | |
1362 | LABEL(ashr_9_exittail): | |
1363 | movdqa (%rsi, %rcx), %xmm1 | |
1364 | psrldq $9, %xmm0 | |
1365 | psrldq $9, %xmm3 | |
1366 | jmp LABEL(aftertail) | |
1367 | ||
1368 | /* | |
1369 | * The following cases will be handled by ashr_10 | |
1370 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1371 | * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10 | |
1372 | */ | |
1373 | .p2align 4 | |
1374 | LABEL(ashr_10): | |
1375 | pxor %xmm0, %xmm0 | |
1376 | movdqa (%rdi), %xmm2 | |
1377 | movdqa (%rsi), %xmm1 | |
1378 | pcmpeqb %xmm1, %xmm0 | |
1379 | pslldq $6, %xmm2 | |
1380 | TOLOWER (%xmm1, %xmm2) | |
1381 | pcmpeqb %xmm1, %xmm2 | |
1382 | psubb %xmm0, %xmm2 | |
1383 | pmovmskb %xmm2, %r9d | |
1384 | shr %cl, %edx | |
1385 | shr %cl, %r9d | |
1386 | sub %r9d, %edx | |
1387 | jnz LABEL(less32bytes) | |
1388 | movdqa (%rdi), %xmm3 | |
1389 | ||
1390 | UPDATE_STRNCMP_COUNTER | |
1391 | ||
1392 | pxor %xmm0, %xmm0 | |
1393 | mov $16, %rcx /* index for loads */ | |
1394 | mov $10, %r9d /* byte position left over from less32bytes case */ | |
1395 | /* | |
1396 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1397 | * When %r10 goes positive we have crossed a page boundary and | |
1398 | * need to do a nibble. | |
1399 | */ | |
1400 | lea 10(%rdi), %r10 | |
1401 | and $0xfff, %r10 /* offset into 4K page */ | |
1402 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1403 | ||
1404 | .p2align 4 | |
1405 | LABEL(loop_ashr_10): | |
1406 | add $16, %r10 | |
1407 | jg LABEL(nibble_ashr_10) | |
1408 | ||
1409 | LABEL(gobble_ashr_10): | |
1410 | movdqa (%rsi, %rcx), %xmm1 | |
1411 | movdqa (%rdi, %rcx), %xmm2 | |
1412 | movdqa %xmm2, %xmm4 | |
1413 | ||
1414 | psrldq $10, %xmm3 | |
1415 | pslldq $6, %xmm2 | |
1416 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1417 | ||
1418 | TOLOWER (%xmm1, %xmm2) | |
1419 | ||
1420 | pcmpeqb %xmm1, %xmm0 | |
1421 | pcmpeqb %xmm2, %xmm1 | |
1422 | psubb %xmm0, %xmm1 | |
1423 | pmovmskb %xmm1, %edx | |
1424 | sub $0xffff, %edx | |
1425 | jnz LABEL(exit) | |
1426 | ||
1427 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1428 | sub $16, %r11 | |
1429 | jbe LABEL(strcmp_exitz) | |
1430 | # endif | |
1431 | ||
1432 | add $16, %rcx | |
1433 | movdqa %xmm4, %xmm3 | |
1434 | ||
1435 | add $16, %r10 | |
1436 | jg LABEL(nibble_ashr_10) /* cross page boundary */ | |
1437 | ||
1438 | movdqa (%rsi, %rcx), %xmm1 | |
1439 | movdqa (%rdi, %rcx), %xmm2 | |
1440 | movdqa %xmm2, %xmm4 | |
1441 | ||
1442 | psrldq $10, %xmm3 | |
1443 | pslldq $6, %xmm2 | |
1444 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1445 | ||
1446 | TOLOWER (%xmm1, %xmm2) | |
1447 | ||
1448 | pcmpeqb %xmm1, %xmm0 | |
1449 | pcmpeqb %xmm2, %xmm1 | |
1450 | psubb %xmm0, %xmm1 | |
1451 | pmovmskb %xmm1, %edx | |
1452 | sub $0xffff, %edx | |
1453 | jnz LABEL(exit) | |
1454 | ||
1455 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1456 | sub $16, %r11 | |
1457 | jbe LABEL(strcmp_exitz) | |
1458 | # endif | |
1459 | ||
1460 | add $16, %rcx | |
1461 | movdqa %xmm4, %xmm3 | |
1462 | jmp LABEL(loop_ashr_10) | |
1463 | ||
1464 | .p2align 4 | |
1465 | LABEL(nibble_ashr_10): | |
1466 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1467 | pmovmskb %xmm0, %edx | |
1468 | test $0xfc00, %edx | |
1469 | jnz LABEL(ashr_10_exittail) | |
1470 | ||
1471 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1472 | cmp $6, %r11 | |
1473 | jbe LABEL(ashr_10_exittail) | |
1474 | # endif | |
1475 | ||
1476 | pxor %xmm0, %xmm0 | |
1477 | sub $0x1000, %r10 | |
1478 | jmp LABEL(gobble_ashr_10) | |
1479 | ||
1480 | .p2align 4 | |
1481 | LABEL(ashr_10_exittail): | |
1482 | movdqa (%rsi, %rcx), %xmm1 | |
1483 | psrldq $10, %xmm0 | |
1484 | psrldq $10, %xmm3 | |
1485 | jmp LABEL(aftertail) | |
1486 | ||
1487 | /* | |
1488 | * The following cases will be handled by ashr_11 | |
1489 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1490 | * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11 | |
1491 | */ | |
1492 | .p2align 4 | |
1493 | LABEL(ashr_11): | |
1494 | pxor %xmm0, %xmm0 | |
1495 | movdqa (%rdi), %xmm2 | |
1496 | movdqa (%rsi), %xmm1 | |
1497 | pcmpeqb %xmm1, %xmm0 | |
1498 | pslldq $5, %xmm2 | |
1499 | TOLOWER (%xmm1, %xmm2) | |
1500 | pcmpeqb %xmm1, %xmm2 | |
1501 | psubb %xmm0, %xmm2 | |
1502 | pmovmskb %xmm2, %r9d | |
1503 | shr %cl, %edx | |
1504 | shr %cl, %r9d | |
1505 | sub %r9d, %edx | |
1506 | jnz LABEL(less32bytes) | |
1507 | movdqa (%rdi), %xmm3 | |
1508 | ||
1509 | UPDATE_STRNCMP_COUNTER | |
1510 | ||
1511 | pxor %xmm0, %xmm0 | |
1512 | mov $16, %rcx /* index for loads */ | |
1513 | mov $11, %r9d /* byte position left over from less32bytes case */ | |
1514 | /* | |
1515 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1516 | * When %r10 goes positive we have crossed a page boundary and | |
1517 | * need to do a nibble. | |
1518 | */ | |
1519 | lea 11(%rdi), %r10 | |
1520 | and $0xfff, %r10 /* offset into 4K page */ | |
1521 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1522 | ||
1523 | .p2align 4 | |
1524 | LABEL(loop_ashr_11): | |
1525 | add $16, %r10 | |
1526 | jg LABEL(nibble_ashr_11) | |
1527 | ||
1528 | LABEL(gobble_ashr_11): | |
1529 | movdqa (%rsi, %rcx), %xmm1 | |
1530 | movdqa (%rdi, %rcx), %xmm2 | |
1531 | movdqa %xmm2, %xmm4 | |
1532 | ||
1533 | psrldq $11, %xmm3 | |
1534 | pslldq $5, %xmm2 | |
1535 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1536 | ||
1537 | TOLOWER (%xmm1, %xmm2) | |
1538 | ||
1539 | pcmpeqb %xmm1, %xmm0 | |
1540 | pcmpeqb %xmm2, %xmm1 | |
1541 | psubb %xmm0, %xmm1 | |
1542 | pmovmskb %xmm1, %edx | |
1543 | sub $0xffff, %edx | |
1544 | jnz LABEL(exit) | |
1545 | ||
1546 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1547 | sub $16, %r11 | |
1548 | jbe LABEL(strcmp_exitz) | |
1549 | # endif | |
1550 | ||
1551 | add $16, %rcx | |
1552 | movdqa %xmm4, %xmm3 | |
1553 | ||
1554 | add $16, %r10 | |
1555 | jg LABEL(nibble_ashr_11) /* cross page boundary */ | |
1556 | ||
1557 | movdqa (%rsi, %rcx), %xmm1 | |
1558 | movdqa (%rdi, %rcx), %xmm2 | |
1559 | movdqa %xmm2, %xmm4 | |
1560 | ||
1561 | psrldq $11, %xmm3 | |
1562 | pslldq $5, %xmm2 | |
1563 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1564 | ||
1565 | TOLOWER (%xmm1, %xmm2) | |
1566 | ||
1567 | pcmpeqb %xmm1, %xmm0 | |
1568 | pcmpeqb %xmm2, %xmm1 | |
1569 | psubb %xmm0, %xmm1 | |
1570 | pmovmskb %xmm1, %edx | |
1571 | sub $0xffff, %edx | |
1572 | jnz LABEL(exit) | |
1573 | ||
1574 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1575 | sub $16, %r11 | |
1576 | jbe LABEL(strcmp_exitz) | |
1577 | # endif | |
1578 | ||
1579 | add $16, %rcx | |
1580 | movdqa %xmm4, %xmm3 | |
1581 | jmp LABEL(loop_ashr_11) | |
1582 | ||
1583 | .p2align 4 | |
1584 | LABEL(nibble_ashr_11): | |
1585 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1586 | pmovmskb %xmm0, %edx | |
1587 | test $0xf800, %edx | |
1588 | jnz LABEL(ashr_11_exittail) | |
1589 | ||
1590 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1591 | cmp $5, %r11 | |
1592 | jbe LABEL(ashr_11_exittail) | |
1593 | # endif | |
1594 | ||
1595 | pxor %xmm0, %xmm0 | |
1596 | sub $0x1000, %r10 | |
1597 | jmp LABEL(gobble_ashr_11) | |
1598 | ||
1599 | .p2align 4 | |
1600 | LABEL(ashr_11_exittail): | |
1601 | movdqa (%rsi, %rcx), %xmm1 | |
1602 | psrldq $11, %xmm0 | |
1603 | psrldq $11, %xmm3 | |
1604 | jmp LABEL(aftertail) | |
1605 | ||
1606 | /* | |
1607 | * The following cases will be handled by ashr_12 | |
1608 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1609 | * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12 | |
1610 | */ | |
1611 | .p2align 4 | |
1612 | LABEL(ashr_12): | |
1613 | pxor %xmm0, %xmm0 | |
1614 | movdqa (%rdi), %xmm2 | |
1615 | movdqa (%rsi), %xmm1 | |
1616 | pcmpeqb %xmm1, %xmm0 | |
1617 | pslldq $4, %xmm2 | |
1618 | TOLOWER (%xmm1, %xmm2) | |
1619 | pcmpeqb %xmm1, %xmm2 | |
1620 | psubb %xmm0, %xmm2 | |
1621 | pmovmskb %xmm2, %r9d | |
1622 | shr %cl, %edx | |
1623 | shr %cl, %r9d | |
1624 | sub %r9d, %edx | |
1625 | jnz LABEL(less32bytes) | |
1626 | movdqa (%rdi), %xmm3 | |
1627 | ||
1628 | UPDATE_STRNCMP_COUNTER | |
1629 | ||
1630 | pxor %xmm0, %xmm0 | |
1631 | mov $16, %rcx /* index for loads */ | |
1632 | mov $12, %r9d /* byte position left over from less32bytes case */ | |
1633 | /* | |
1634 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1635 | * When %r10 goes positive we have crossed a page boundary and | |
1636 | * need to do a nibble. | |
1637 | */ | |
1638 | lea 12(%rdi), %r10 | |
1639 | and $0xfff, %r10 /* offset into 4K page */ | |
1640 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1641 | ||
1642 | .p2align 4 | |
1643 | LABEL(loop_ashr_12): | |
1644 | add $16, %r10 | |
1645 | jg LABEL(nibble_ashr_12) | |
1646 | ||
1647 | LABEL(gobble_ashr_12): | |
1648 | movdqa (%rsi, %rcx), %xmm1 | |
1649 | movdqa (%rdi, %rcx), %xmm2 | |
1650 | movdqa %xmm2, %xmm4 | |
1651 | ||
1652 | psrldq $12, %xmm3 | |
1653 | pslldq $4, %xmm2 | |
1654 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1655 | ||
1656 | TOLOWER (%xmm1, %xmm2) | |
1657 | ||
1658 | pcmpeqb %xmm1, %xmm0 | |
1659 | pcmpeqb %xmm2, %xmm1 | |
1660 | psubb %xmm0, %xmm1 | |
1661 | pmovmskb %xmm1, %edx | |
1662 | sub $0xffff, %edx | |
1663 | jnz LABEL(exit) | |
1664 | ||
1665 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1666 | sub $16, %r11 | |
1667 | jbe LABEL(strcmp_exitz) | |
1668 | # endif | |
1669 | ||
1670 | add $16, %rcx | |
1671 | movdqa %xmm4, %xmm3 | |
1672 | ||
1673 | add $16, %r10 | |
1674 | jg LABEL(nibble_ashr_12) /* cross page boundary */ | |
1675 | ||
1676 | movdqa (%rsi, %rcx), %xmm1 | |
1677 | movdqa (%rdi, %rcx), %xmm2 | |
1678 | movdqa %xmm2, %xmm4 | |
1679 | ||
1680 | psrldq $12, %xmm3 | |
1681 | pslldq $4, %xmm2 | |
1682 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1683 | ||
1684 | TOLOWER (%xmm1, %xmm2) | |
1685 | ||
1686 | pcmpeqb %xmm1, %xmm0 | |
1687 | pcmpeqb %xmm2, %xmm1 | |
1688 | psubb %xmm0, %xmm1 | |
1689 | pmovmskb %xmm1, %edx | |
1690 | sub $0xffff, %edx | |
1691 | jnz LABEL(exit) | |
1692 | ||
1693 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1694 | sub $16, %r11 | |
1695 | jbe LABEL(strcmp_exitz) | |
1696 | # endif | |
1697 | ||
1698 | add $16, %rcx | |
1699 | movdqa %xmm4, %xmm3 | |
1700 | jmp LABEL(loop_ashr_12) | |
1701 | ||
1702 | .p2align 4 | |
1703 | LABEL(nibble_ashr_12): | |
1704 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1705 | pmovmskb %xmm0, %edx | |
1706 | test $0xf000, %edx | |
1707 | jnz LABEL(ashr_12_exittail) | |
1708 | ||
1709 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1710 | cmp $4, %r11 | |
1711 | jbe LABEL(ashr_12_exittail) | |
1712 | # endif | |
1713 | ||
1714 | pxor %xmm0, %xmm0 | |
1715 | sub $0x1000, %r10 | |
1716 | jmp LABEL(gobble_ashr_12) | |
1717 | ||
1718 | .p2align 4 | |
1719 | LABEL(ashr_12_exittail): | |
1720 | movdqa (%rsi, %rcx), %xmm1 | |
1721 | psrldq $12, %xmm0 | |
1722 | psrldq $12, %xmm3 | |
1723 | jmp LABEL(aftertail) | |
1724 | ||
1725 | /* | |
1726 | * The following cases will be handled by ashr_13 | |
1727 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1728 | * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13 | |
1729 | */ | |
1730 | .p2align 4 | |
1731 | LABEL(ashr_13): | |
1732 | pxor %xmm0, %xmm0 | |
1733 | movdqa (%rdi), %xmm2 | |
1734 | movdqa (%rsi), %xmm1 | |
1735 | pcmpeqb %xmm1, %xmm0 | |
1736 | pslldq $3, %xmm2 | |
1737 | TOLOWER (%xmm1, %xmm2) | |
1738 | pcmpeqb %xmm1, %xmm2 | |
1739 | psubb %xmm0, %xmm2 | |
1740 | pmovmskb %xmm2, %r9d | |
1741 | shr %cl, %edx | |
1742 | shr %cl, %r9d | |
1743 | sub %r9d, %edx | |
1744 | jnz LABEL(less32bytes) | |
1745 | movdqa (%rdi), %xmm3 | |
1746 | ||
1747 | UPDATE_STRNCMP_COUNTER | |
1748 | ||
1749 | pxor %xmm0, %xmm0 | |
1750 | mov $16, %rcx /* index for loads */ | |
1751 | mov $13, %r9d /* byte position left over from less32bytes case */ | |
1752 | /* | |
1753 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1754 | * When %r10 goes positive we have crossed a page boundary and | |
1755 | * need to do a nibble. | |
1756 | */ | |
1757 | lea 13(%rdi), %r10 | |
1758 | and $0xfff, %r10 /* offset into 4K page */ | |
1759 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1760 | ||
1761 | .p2align 4 | |
1762 | LABEL(loop_ashr_13): | |
1763 | add $16, %r10 | |
1764 | jg LABEL(nibble_ashr_13) | |
1765 | ||
1766 | LABEL(gobble_ashr_13): | |
1767 | movdqa (%rsi, %rcx), %xmm1 | |
1768 | movdqa (%rdi, %rcx), %xmm2 | |
1769 | movdqa %xmm2, %xmm4 | |
1770 | ||
1771 | psrldq $13, %xmm3 | |
1772 | pslldq $3, %xmm2 | |
1773 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1774 | ||
1775 | TOLOWER (%xmm1, %xmm2) | |
1776 | ||
1777 | pcmpeqb %xmm1, %xmm0 | |
1778 | pcmpeqb %xmm2, %xmm1 | |
1779 | psubb %xmm0, %xmm1 | |
1780 | pmovmskb %xmm1, %edx | |
1781 | sub $0xffff, %edx | |
1782 | jnz LABEL(exit) | |
1783 | ||
1784 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1785 | sub $16, %r11 | |
1786 | jbe LABEL(strcmp_exitz) | |
1787 | # endif | |
1788 | ||
1789 | add $16, %rcx | |
1790 | movdqa %xmm4, %xmm3 | |
1791 | ||
1792 | add $16, %r10 | |
1793 | jg LABEL(nibble_ashr_13) /* cross page boundary */ | |
1794 | ||
1795 | movdqa (%rsi, %rcx), %xmm1 | |
1796 | movdqa (%rdi, %rcx), %xmm2 | |
1797 | movdqa %xmm2, %xmm4 | |
1798 | ||
1799 | psrldq $13, %xmm3 | |
1800 | pslldq $3, %xmm2 | |
1801 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1802 | ||
1803 | TOLOWER (%xmm1, %xmm2) | |
1804 | ||
1805 | pcmpeqb %xmm1, %xmm0 | |
1806 | pcmpeqb %xmm2, %xmm1 | |
1807 | psubb %xmm0, %xmm1 | |
1808 | pmovmskb %xmm1, %edx | |
1809 | sub $0xffff, %edx | |
1810 | jnz LABEL(exit) | |
1811 | ||
1812 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1813 | sub $16, %r11 | |
1814 | jbe LABEL(strcmp_exitz) | |
1815 | # endif | |
1816 | ||
1817 | add $16, %rcx | |
1818 | movdqa %xmm4, %xmm3 | |
1819 | jmp LABEL(loop_ashr_13) | |
1820 | ||
1821 | .p2align 4 | |
1822 | LABEL(nibble_ashr_13): | |
1823 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1824 | pmovmskb %xmm0, %edx | |
1825 | test $0xe000, %edx | |
1826 | jnz LABEL(ashr_13_exittail) | |
1827 | ||
1828 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1829 | cmp $3, %r11 | |
1830 | jbe LABEL(ashr_13_exittail) | |
1831 | # endif | |
1832 | ||
1833 | pxor %xmm0, %xmm0 | |
1834 | sub $0x1000, %r10 | |
1835 | jmp LABEL(gobble_ashr_13) | |
1836 | ||
1837 | .p2align 4 | |
1838 | LABEL(ashr_13_exittail): | |
1839 | movdqa (%rsi, %rcx), %xmm1 | |
1840 | psrldq $13, %xmm0 | |
1841 | psrldq $13, %xmm3 | |
1842 | jmp LABEL(aftertail) | |
1843 | ||
1844 | /* | |
1845 | * The following cases will be handled by ashr_14 | |
1846 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1847 | * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14 | |
1848 | */ | |
1849 | .p2align 4 | |
1850 | LABEL(ashr_14): | |
1851 | pxor %xmm0, %xmm0 | |
1852 | movdqa (%rdi), %xmm2 | |
1853 | movdqa (%rsi), %xmm1 | |
1854 | pcmpeqb %xmm1, %xmm0 | |
1855 | pslldq $2, %xmm2 | |
1856 | TOLOWER (%xmm1, %xmm2) | |
1857 | pcmpeqb %xmm1, %xmm2 | |
1858 | psubb %xmm0, %xmm2 | |
1859 | pmovmskb %xmm2, %r9d | |
1860 | shr %cl, %edx | |
1861 | shr %cl, %r9d | |
1862 | sub %r9d, %edx | |
1863 | jnz LABEL(less32bytes) | |
1864 | movdqa (%rdi), %xmm3 | |
1865 | ||
1866 | UPDATE_STRNCMP_COUNTER | |
1867 | ||
1868 | pxor %xmm0, %xmm0 | |
1869 | mov $16, %rcx /* index for loads */ | |
1870 | mov $14, %r9d /* byte position left over from less32bytes case */ | |
1871 | /* | |
1872 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1873 | * When %r10 goes positive we have crossed a page boundary and | |
1874 | * need to do a nibble. | |
1875 | */ | |
1876 | lea 14(%rdi), %r10 | |
1877 | and $0xfff, %r10 /* offset into 4K page */ | |
1878 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
1879 | ||
1880 | .p2align 4 | |
1881 | LABEL(loop_ashr_14): | |
1882 | add $16, %r10 | |
1883 | jg LABEL(nibble_ashr_14) | |
1884 | ||
1885 | LABEL(gobble_ashr_14): | |
1886 | movdqa (%rsi, %rcx), %xmm1 | |
1887 | movdqa (%rdi, %rcx), %xmm2 | |
1888 | movdqa %xmm2, %xmm4 | |
1889 | ||
1890 | psrldq $14, %xmm3 | |
1891 | pslldq $2, %xmm2 | |
1892 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1893 | ||
1894 | TOLOWER (%xmm1, %xmm2) | |
1895 | ||
1896 | pcmpeqb %xmm1, %xmm0 | |
1897 | pcmpeqb %xmm2, %xmm1 | |
1898 | psubb %xmm0, %xmm1 | |
1899 | pmovmskb %xmm1, %edx | |
1900 | sub $0xffff, %edx | |
1901 | jnz LABEL(exit) | |
1902 | ||
1903 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1904 | sub $16, %r11 | |
1905 | jbe LABEL(strcmp_exitz) | |
1906 | # endif | |
1907 | ||
1908 | add $16, %rcx | |
1909 | movdqa %xmm4, %xmm3 | |
1910 | ||
1911 | add $16, %r10 | |
1912 | jg LABEL(nibble_ashr_14) /* cross page boundary */ | |
1913 | ||
1914 | movdqa (%rsi, %rcx), %xmm1 | |
1915 | movdqa (%rdi, %rcx), %xmm2 | |
1916 | movdqa %xmm2, %xmm4 | |
1917 | ||
1918 | psrldq $14, %xmm3 | |
1919 | pslldq $2, %xmm2 | |
1920 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
1921 | ||
1922 | TOLOWER (%xmm1, %xmm2) | |
1923 | ||
1924 | pcmpeqb %xmm1, %xmm0 | |
1925 | pcmpeqb %xmm2, %xmm1 | |
1926 | psubb %xmm0, %xmm1 | |
1927 | pmovmskb %xmm1, %edx | |
1928 | sub $0xffff, %edx | |
1929 | jnz LABEL(exit) | |
1930 | ||
1931 | # if defined USE_AS_STRNCMP | defined USE_AS_STRNCASECMP_L | |
1932 | sub $16, %r11 | |
1933 | jbe LABEL(strcmp_exitz) | |
1934 | # endif | |
1935 | ||
1936 | add $16, %rcx | |
1937 | movdqa %xmm4, %xmm3 | |
1938 | jmp LABEL(loop_ashr_14) | |
1939 | ||
1940 | .p2align 4 | |
1941 | LABEL(nibble_ashr_14): | |
1942 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
1943 | pmovmskb %xmm0, %edx | |
1944 | test $0xc000, %edx | |
1945 | jnz LABEL(ashr_14_exittail) | |
1946 | ||
1947 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
1948 | cmp $2, %r11 | |
1949 | jbe LABEL(ashr_14_exittail) | |
1950 | # endif | |
1951 | ||
1952 | pxor %xmm0, %xmm0 | |
1953 | sub $0x1000, %r10 | |
1954 | jmp LABEL(gobble_ashr_14) | |
1955 | ||
1956 | .p2align 4 | |
1957 | LABEL(ashr_14_exittail): | |
1958 | movdqa (%rsi, %rcx), %xmm1 | |
1959 | psrldq $14, %xmm0 | |
1960 | psrldq $14, %xmm3 | |
1961 | jmp LABEL(aftertail) | |
1962 | ||
1963 | /* | |
1964 | * The following cases will be handled by ashr_15 | |
1965 | * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case | |
1966 | * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15 | |
1967 | */ | |
1968 | .p2align 4 | |
1969 | LABEL(ashr_15): | |
1970 | pxor %xmm0, %xmm0 | |
1971 | movdqa (%rdi), %xmm2 | |
1972 | movdqa (%rsi), %xmm1 | |
1973 | pcmpeqb %xmm1, %xmm0 | |
1974 | pslldq $1, %xmm2 | |
1975 | TOLOWER (%xmm1, %xmm2) | |
1976 | pcmpeqb %xmm1, %xmm2 | |
1977 | psubb %xmm0, %xmm2 | |
1978 | pmovmskb %xmm2, %r9d | |
1979 | shr %cl, %edx | |
1980 | shr %cl, %r9d | |
1981 | sub %r9d, %edx | |
1982 | jnz LABEL(less32bytes) | |
1983 | ||
1984 | movdqa (%rdi), %xmm3 | |
1985 | ||
1986 | UPDATE_STRNCMP_COUNTER | |
1987 | ||
1988 | pxor %xmm0, %xmm0 | |
1989 | mov $16, %rcx /* index for loads */ | |
1990 | mov $15, %r9d /* byte position left over from less32bytes case */ | |
1991 | /* | |
1992 | * Setup %r10 value allows us to detect crossing a page boundary. | |
1993 | * When %r10 goes positive we have crossed a page boundary and | |
1994 | * need to do a nibble. | |
1995 | */ | |
1996 | lea 15(%rdi), %r10 | |
1997 | and $0xfff, %r10 /* offset into 4K page */ | |
1998 | ||
1999 | sub $0x1000, %r10 /* subtract 4K pagesize */ | |
2000 | ||
2001 | .p2align 4 | |
2002 | LABEL(loop_ashr_15): | |
2003 | add $16, %r10 | |
2004 | jg LABEL(nibble_ashr_15) | |
2005 | ||
2006 | LABEL(gobble_ashr_15): | |
2007 | movdqa (%rsi, %rcx), %xmm1 | |
2008 | movdqa (%rdi, %rcx), %xmm2 | |
2009 | movdqa %xmm2, %xmm4 | |
2010 | ||
2011 | psrldq $15, %xmm3 | |
2012 | pslldq $1, %xmm2 | |
2013 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
2014 | ||
2015 | TOLOWER (%xmm1, %xmm2) | |
2016 | ||
2017 | pcmpeqb %xmm1, %xmm0 | |
2018 | pcmpeqb %xmm2, %xmm1 | |
2019 | psubb %xmm0, %xmm1 | |
2020 | pmovmskb %xmm1, %edx | |
2021 | sub $0xffff, %edx | |
2022 | jnz LABEL(exit) | |
2023 | ||
2024 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
2025 | sub $16, %r11 | |
2026 | jbe LABEL(strcmp_exitz) | |
2027 | # endif | |
2028 | ||
2029 | add $16, %rcx | |
2030 | movdqa %xmm4, %xmm3 | |
2031 | ||
2032 | add $16, %r10 | |
2033 | jg LABEL(nibble_ashr_15) /* cross page boundary */ | |
2034 | ||
2035 | movdqa (%rsi, %rcx), %xmm1 | |
2036 | movdqa (%rdi, %rcx), %xmm2 | |
2037 | movdqa %xmm2, %xmm4 | |
2038 | ||
2039 | psrldq $15, %xmm3 | |
2040 | pslldq $1, %xmm2 | |
2041 | por %xmm3, %xmm2 /* merge into one 16byte value */ | |
2042 | ||
2043 | TOLOWER (%xmm1, %xmm2) | |
2044 | ||
2045 | pcmpeqb %xmm1, %xmm0 | |
2046 | pcmpeqb %xmm2, %xmm1 | |
2047 | psubb %xmm0, %xmm1 | |
2048 | pmovmskb %xmm1, %edx | |
2049 | sub $0xffff, %edx | |
2050 | jnz LABEL(exit) | |
2051 | ||
2052 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
2053 | sub $16, %r11 | |
2054 | jbe LABEL(strcmp_exitz) | |
2055 | # endif | |
2056 | ||
2057 | add $16, %rcx | |
2058 | movdqa %xmm4, %xmm3 | |
2059 | jmp LABEL(loop_ashr_15) | |
2060 | ||
2061 | .p2align 4 | |
2062 | LABEL(nibble_ashr_15): | |
2063 | pcmpeqb %xmm3, %xmm0 /* check nibble for null char */ | |
2064 | pmovmskb %xmm0, %edx | |
2065 | test $0x8000, %edx | |
2066 | jnz LABEL(ashr_15_exittail) | |
2067 | ||
2068 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
2069 | cmpq $1, %r11 | |
2070 | jbe LABEL(ashr_15_exittail) | |
2071 | # endif | |
2072 | ||
2073 | pxor %xmm0, %xmm0 | |
2074 | sub $0x1000, %r10 | |
2075 | jmp LABEL(gobble_ashr_15) | |
2076 | ||
2077 | .p2align 4 | |
2078 | LABEL(ashr_15_exittail): | |
2079 | movdqa (%rsi, %rcx), %xmm1 | |
2080 | psrldq $15, %xmm3 | |
2081 | psrldq $15, %xmm0 | |
2082 | ||
2083 | .p2align 4 | |
2084 | LABEL(aftertail): | |
2085 | TOLOWER (%xmm1, %xmm3) | |
2086 | pcmpeqb %xmm3, %xmm1 | |
2087 | psubb %xmm0, %xmm1 | |
2088 | pmovmskb %xmm1, %edx | |
2089 | not %edx | |
2090 | ||
2091 | .p2align 4 | |
2092 | LABEL(exit): | |
2093 | lea -16(%r9, %rcx), %rax /* locate the exact offset for rdi */ | |
2094 | LABEL(less32bytes): | |
2095 | lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */ | |
2096 | lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */ | |
2097 | test %r8d, %r8d | |
2098 | jz LABEL(ret) | |
2099 | xchg %rsi, %rdi /* recover original order according to flag(%r8d) */ | |
2100 | ||
2101 | .p2align 4 | |
2102 | LABEL(ret): | |
2103 | LABEL(less16bytes): | |
2104 | bsf %rdx, %rdx /* find and store bit index in %rdx */ | |
2105 | ||
2106 | # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L | |
2107 | sub %rdx, %r11 | |
2108 | jbe LABEL(strcmp_exitz) | |
2109 | # endif | |
2110 | movzbl (%rsi, %rdx), %ecx | |
2111 | movzbl (%rdi, %rdx), %eax | |
2112 | ||
2113 | # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L | |
2114 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx | |
2115 | movl (%rdx,%rcx,4), %ecx | |
2116 | movl (%rdx,%rax,4), %eax | |
2117 | # endif | |
2118 | ||
2119 | sub %ecx, %eax | |
2120 | ret | |
2121 | ||
2122 | LABEL(strcmp_exitz): | |
2123 | xor %eax, %eax | |
2124 | ret | |
2125 | ||
2126 | .p2align 4 | |
2127 | LABEL(Byte0): | |
2128 | movzbl (%rsi), %ecx | |
2129 | movzbl (%rdi), %eax | |
2130 | ||
2131 | # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L | |
2132 | leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx | |
2133 | movl (%rdx,%rcx,4), %ecx | |
2134 | movl (%rdx,%rax,4), %eax | |
2135 | # endif | |
2136 | ||
2137 | sub %ecx, %eax | |
2138 | ret | |
2139 | END (STRCMP) | |
2140 | ||
2141 | .section .rodata,"a",@progbits | |
2142 | .p2align 3 | |
2143 | LABEL(unaligned_table): | |
2144 | .int LABEL(ashr_1) - LABEL(unaligned_table) | |
2145 | .int LABEL(ashr_2) - LABEL(unaligned_table) | |
2146 | .int LABEL(ashr_3) - LABEL(unaligned_table) | |
2147 | .int LABEL(ashr_4) - LABEL(unaligned_table) | |
2148 | .int LABEL(ashr_5) - LABEL(unaligned_table) | |
2149 | .int LABEL(ashr_6) - LABEL(unaligned_table) | |
2150 | .int LABEL(ashr_7) - LABEL(unaligned_table) | |
2151 | .int LABEL(ashr_8) - LABEL(unaligned_table) | |
2152 | .int LABEL(ashr_9) - LABEL(unaligned_table) | |
2153 | .int LABEL(ashr_10) - LABEL(unaligned_table) | |
2154 | .int LABEL(ashr_11) - LABEL(unaligned_table) | |
2155 | .int LABEL(ashr_12) - LABEL(unaligned_table) | |
2156 | .int LABEL(ashr_13) - LABEL(unaligned_table) | |
2157 | .int LABEL(ashr_14) - LABEL(unaligned_table) | |
2158 | .int LABEL(ashr_15) - LABEL(unaligned_table) | |
2159 | .int LABEL(ashr_0) - LABEL(unaligned_table) | |
2160 | #endif |