]>
Commit | Line | Data |
---|---|---|
04277e02 | 1 | /* Copyright (C) 2012-2019 Free Software Foundation, Inc. |
38fecb39 MS |
2 | |
3 | This file is part of the GNU C Library. | |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
16 | License along with the GNU C Library. If not, see | |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
38fecb39 MS |
18 | |
19 | /* Assumptions: | |
20 | * | |
21 | * ARMv8-a, AArch64 | |
22 | */ | |
23 | ||
24 | #include <sysdep.h> | |
25 | ||
26 | #define REP8_01 0x0101010101010101 | |
27 | #define REP8_7f 0x7f7f7f7f7f7f7f7f | |
28 | #define REP8_80 0x8080808080808080 | |
29 | ||
30 | /* Parameters and result. */ | |
31 | #define src1 x0 | |
32 | #define src2 x1 | |
33 | #define result x0 | |
34 | ||
35 | /* Internal variables. */ | |
36 | #define data1 x2 | |
37 | #define data1w w2 | |
38 | #define data2 x3 | |
39 | #define data2w w3 | |
40 | #define has_nul x4 | |
41 | #define diff x5 | |
42 | #define syndrome x6 | |
43 | #define tmp1 x7 | |
44 | #define tmp2 x8 | |
45 | #define tmp3 x9 | |
46 | #define zeroones x10 | |
47 | #define pos x11 | |
48 | ||
49 | /* Start of performance-critical section -- one 64B cache line. */ | |
50 | ENTRY_ALIGN(strcmp, 6) | |
51 | ||
389d1f1b SE |
52 | DELOUSE (0) |
53 | DELOUSE (1) | |
38fecb39 MS |
54 | eor tmp1, src1, src2 |
55 | mov zeroones, #REP8_01 | |
56 | tst tmp1, #7 | |
57 | b.ne L(misaligned8) | |
58 | ands tmp1, src1, #7 | |
59 | b.ne L(mutual_align) | |
60 | /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
61 | (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
62 | can be done in parallel across the entire word. */ | |
63 | L(loop_aligned): | |
64 | ldr data1, [src1], #8 | |
65 | ldr data2, [src2], #8 | |
66 | L(start_realigned): | |
67 | sub tmp1, data1, zeroones | |
68 | orr tmp2, data1, #REP8_7f | |
69 | eor diff, data1, data2 /* Non-zero if differences found. */ | |
70 | bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
71 | orr syndrome, diff, has_nul | |
72 | cbz syndrome, L(loop_aligned) | |
73 | /* End of performance-critical section -- one 64B cache line. */ | |
74 | ||
2bce01eb | 75 | L(end): |
38fecb39 MS |
76 | #ifndef __AARCH64EB__ |
77 | rev syndrome, syndrome | |
78 | rev data1, data1 | |
79 | /* The MS-non-zero bit of the syndrome marks either the first bit | |
80 | that is different, or the top bit of the first zero byte. | |
81 | Shifting left now will bring the critical information into the | |
82 | top bits. */ | |
83 | clz pos, syndrome | |
84 | rev data2, data2 | |
85 | lsl data1, data1, pos | |
86 | lsl data2, data2, pos | |
87 | /* But we need to zero-extend (char is unsigned) the value and then | |
88 | perform a signed 32-bit subtraction. */ | |
89 | lsr data1, data1, #56 | |
90 | sub result, data1, data2, lsr #56 | |
91 | RET | |
92 | #else | |
93 | /* For big-endian we cannot use the trick with the syndrome value | |
94 | as carry-propagation can corrupt the upper bits if the trailing | |
95 | bytes in the string contain 0x01. */ | |
96 | /* However, if there is no NUL byte in the dword, we can generate | |
97 | the result directly. We can't just subtract the bytes as the | |
98 | MSB might be significant. */ | |
99 | cbnz has_nul, 1f | |
100 | cmp data1, data2 | |
101 | cset result, ne | |
102 | cneg result, result, lo | |
103 | RET | |
104 | 1: | |
105 | /* Re-compute the NUL-byte detection, using a byte-reversed value. */ | |
106 | rev tmp3, data1 | |
107 | sub tmp1, tmp3, zeroones | |
108 | orr tmp2, tmp3, #REP8_7f | |
109 | bic has_nul, tmp1, tmp2 | |
110 | rev has_nul, has_nul | |
111 | orr syndrome, diff, has_nul | |
112 | clz pos, syndrome | |
113 | /* The MS-non-zero bit of the syndrome marks either the first bit | |
114 | that is different, or the top bit of the first zero byte. | |
115 | Shifting left now will bring the critical information into the | |
116 | top bits. */ | |
117 | lsl data1, data1, pos | |
118 | lsl data2, data2, pos | |
119 | /* But we need to zero-extend (char is unsigned) the value and then | |
120 | perform a signed 32-bit subtraction. */ | |
121 | lsr data1, data1, #56 | |
122 | sub result, data1, data2, lsr #56 | |
123 | RET | |
124 | #endif | |
125 | ||
126 | L(mutual_align): | |
127 | /* Sources are mutually aligned, but are not currently at an | |
128 | alignment boundary. Round down the addresses and then mask off | |
129 | the bytes that preceed the start point. */ | |
130 | bic src1, src1, #7 | |
131 | bic src2, src2, #7 | |
132 | lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ | |
133 | ldr data1, [src1], #8 | |
134 | neg tmp1, tmp1 /* Bits to alignment -64. */ | |
135 | ldr data2, [src2], #8 | |
136 | mov tmp2, #~0 | |
137 | #ifdef __AARCH64EB__ | |
138 | /* Big-endian. Early bytes are at MSB. */ | |
139 | lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ | |
140 | #else | |
141 | /* Little-endian. Early bytes are at LSB. */ | |
142 | lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ | |
143 | #endif | |
144 | orr data1, data1, tmp2 | |
145 | orr data2, data2, tmp2 | |
146 | b L(start_realigned) | |
147 | ||
148 | L(misaligned8): | |
2bce01eb SP |
149 | /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always |
150 | checking to make sure that we don't access beyond page boundary in | |
151 | SRC2. */ | |
152 | tst src1, #7 | |
153 | b.eq L(loop_misaligned) | |
154 | L(do_misaligned): | |
38fecb39 MS |
155 | ldrb data1w, [src1], #1 |
156 | ldrb data2w, [src2], #1 | |
157 | cmp data1w, #1 | |
158 | ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
2bce01eb SP |
159 | b.ne L(done) |
160 | tst src1, #7 | |
6ca24c43 | 161 | b.ne L(do_misaligned) |
2bce01eb SP |
162 | |
163 | L(loop_misaligned): | |
164 | /* Test if we are within the last dword of the end of a 4K page. If | |
165 | yes then jump back to the misaligned loop to copy a byte at a time. */ | |
166 | and tmp1, src2, #0xff8 | |
167 | eor tmp1, tmp1, #0xff8 | |
168 | cbz tmp1, L(do_misaligned) | |
169 | ldr data1, [src1], #8 | |
170 | ldr data2, [src2], #8 | |
171 | ||
172 | sub tmp1, data1, zeroones | |
173 | orr tmp2, data1, #REP8_7f | |
174 | eor diff, data1, data2 /* Non-zero if differences found. */ | |
175 | bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
176 | orr syndrome, diff, has_nul | |
177 | cbz syndrome, L(loop_misaligned) | |
178 | b L(end) | |
179 | ||
180 | L(done): | |
38fecb39 MS |
181 | sub result, data1, data2 |
182 | RET | |
183 | END(strcmp) | |
184 | libc_hidden_builtin_def (strcmp) |