]>
Commit | Line | Data |
---|---|---|
83056630 | 1 | /* memset/bzero with unaligned store and rep stosb |
04277e02 | 2 | Copyright (C) 2016-2019 Free Software Foundation, Inc. |
83056630 L |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
16 | License along with the GNU C Library; if not, see | |
17 | <http://www.gnu.org/licenses/>. */ | |
18 | ||
19 | /* memset is implemented as: | |
20 | 1. Use overlapping store to avoid branch. | |
eb2c88c7 L |
21 | 2. If size is less than VEC, use integer register stores. |
22 | 3. If size is from VEC_SIZE to 2 * VEC_SIZE, use 2 VEC stores. | |
23 | 4. If size is from 2 * VEC_SIZE to 4 * VEC_SIZE, use 4 VEC stores. | |
24 | 5. If size is more to 4 * VEC_SIZE, align to 4 * VEC_SIZE with | |
83056630 L |
25 | 4 VEC stores and store 4 * VEC at a time until done. */ |
26 | ||
27 | #include <sysdep.h> | |
28 | ||
4af1bb06 L |
29 | #ifndef MEMSET_CHK_SYMBOL |
30 | # define MEMSET_CHK_SYMBOL(p,s) MEMSET_SYMBOL(p, s) | |
31 | #endif | |
32 | ||
ef9c4cb6 L |
33 | #ifndef WMEMSET_CHK_SYMBOL |
34 | # define WMEMSET_CHK_SYMBOL(p,s) WMEMSET_SYMBOL(p, s) | |
35 | #endif | |
36 | ||
83056630 L |
37 | #ifndef VZEROUPPER |
38 | # if VEC_SIZE > 16 | |
39 | # define VZEROUPPER vzeroupper | |
40 | # else | |
41 | # define VZEROUPPER | |
42 | # endif | |
43 | #endif | |
44 | ||
45 | #ifndef VZEROUPPER_SHORT_RETURN | |
46 | # if VEC_SIZE > 16 | |
47 | # define VZEROUPPER_SHORT_RETURN vzeroupper | |
48 | # else | |
49 | # define VZEROUPPER_SHORT_RETURN rep | |
50 | # endif | |
51 | #endif | |
52 | ||
53 | #ifndef MOVQ | |
54 | # if VEC_SIZE > 16 | |
55 | # define MOVQ vmovq | |
56 | # else | |
57 | # define MOVQ movq | |
58 | # endif | |
59 | #endif | |
60 | ||
61 | /* Threshold to use Enhanced REP STOSB. Since there is overhead to set | |
62 | up REP STOSB operation, REP STOSB isn't faster on short data. The | |
63 | memset micro benchmark in glibc shows that 2KB is the approximate | |
64 | value above which REP STOSB becomes faster on processors with | |
65 | Enhanced REP STOSB. Since the stored value is fixed, larger register | |
66 | size has minimal impact on threshold. */ | |
67 | #ifndef REP_STOSB_THRESHOLD | |
68 | # define REP_STOSB_THRESHOLD 2048 | |
69 | #endif | |
70 | ||
71 | #ifndef SECTION | |
72 | # error SECTION is not defined! | |
73 | #endif | |
74 | ||
83056630 | 75 | .section SECTION(.text),"ax",@progbits |
5e8c5bb1 | 76 | #if VEC_SIZE == 16 && IS_IN (libc) |
83056630 L |
77 | ENTRY (__bzero) |
78 | movq %rdi, %rax /* Set return value. */ | |
79 | movq %rsi, %rdx /* Set n. */ | |
80 | pxor %xmm0, %xmm0 | |
81 | jmp L(entry_from_bzero) | |
82 | END (__bzero) | |
83 | weak_alias (__bzero, bzero) | |
84 | #endif | |
85 | ||
ef9c4cb6 L |
86 | #if IS_IN (libc) |
87 | # if defined SHARED | |
88 | ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) | |
89 | cmpq %rdx, %rcx | |
90 | jb HIDDEN_JUMPTARGET (__chk_fail) | |
91 | END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned)) | |
92 | # endif | |
93 | ||
94 | ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned)) | |
95 | shlq $2, %rdx | |
96 | WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) | |
97 | jmp L(entry_from_bzero) | |
98 | END (WMEMSET_SYMBOL (__wmemset, unaligned)) | |
99 | #endif | |
100 | ||
83056630 | 101 | #if defined SHARED && IS_IN (libc) |
4af1bb06 | 102 | ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) |
83056630 L |
103 | cmpq %rdx, %rcx |
104 | jb HIDDEN_JUMPTARGET (__chk_fail) | |
4af1bb06 | 105 | END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned)) |
83056630 L |
106 | #endif |
107 | ||
108 | ENTRY (MEMSET_SYMBOL (__memset, unaligned)) | |
ef9c4cb6 | 109 | MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) |
83056630 L |
110 | L(entry_from_bzero): |
111 | cmpq $VEC_SIZE, %rdx | |
112 | jb L(less_vec) | |
113 | cmpq $(VEC_SIZE * 2), %rdx | |
114 | ja L(more_2x_vec) | |
115 | /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ | |
116 | VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) | |
117 | VMOVU %VEC(0), (%rdi) | |
118 | VZEROUPPER | |
119 | ret | |
4af1bb06 | 120 | #if defined USE_MULTIARCH && IS_IN (libc) |
83056630 L |
121 | END (MEMSET_SYMBOL (__memset, unaligned)) |
122 | ||
4af1bb06 | 123 | # if VEC_SIZE == 16 |
93e46f87 L |
124 | ENTRY (__memset_chk_erms) |
125 | cmpq %rdx, %rcx | |
126 | jb HIDDEN_JUMPTARGET (__chk_fail) | |
127 | END (__memset_chk_erms) | |
128 | ||
83056630 L |
129 | /* Only used to measure performance of REP STOSB. */ |
130 | ENTRY (__memset_erms) | |
727b38df L |
131 | /* Skip zero length. */ |
132 | testq %rdx, %rdx | |
133 | jnz L(stosb) | |
134 | movq %rdi, %rax | |
135 | ret | |
4af1bb06 | 136 | # else |
50d7d351 L |
137 | /* Provide a hidden symbol to debugger. */ |
138 | .hidden MEMSET_SYMBOL (__memset, erms) | |
83056630 | 139 | ENTRY (MEMSET_SYMBOL (__memset, erms)) |
4af1bb06 | 140 | # endif |
83056630 | 141 | L(stosb): |
02b78ff7 L |
142 | /* Issue vzeroupper before rep stosb. */ |
143 | VZEROUPPER | |
83056630 L |
144 | movq %rdx, %rcx |
145 | movzbl %sil, %eax | |
146 | movq %rdi, %rdx | |
147 | rep stosb | |
148 | movq %rdx, %rax | |
149 | ret | |
4af1bb06 | 150 | # if VEC_SIZE == 16 |
83056630 | 151 | END (__memset_erms) |
4af1bb06 | 152 | # else |
83056630 | 153 | END (MEMSET_SYMBOL (__memset, erms)) |
4af1bb06 | 154 | # endif |
83056630 | 155 | |
4af1bb06 L |
156 | # if defined SHARED && IS_IN (libc) |
157 | ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) | |
83056630 L |
158 | cmpq %rdx, %rcx |
159 | jb HIDDEN_JUMPTARGET (__chk_fail) | |
4af1bb06 L |
160 | END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms)) |
161 | # endif | |
83056630 L |
162 | |
163 | ENTRY (MEMSET_SYMBOL (__memset, unaligned_erms)) | |
ef9c4cb6 | 164 | MEMSET_VDUP_TO_VEC0_AND_SET_RETURN (%esi, %rdi) |
83056630 L |
165 | cmpq $VEC_SIZE, %rdx |
166 | jb L(less_vec) | |
167 | cmpq $(VEC_SIZE * 2), %rdx | |
168 | ja L(stosb_more_2x_vec) | |
169 | /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ | |
170 | VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) | |
171 | VMOVU %VEC(0), (%rdi) | |
172 | VZEROUPPER | |
173 | ret | |
174 | ||
83056630 L |
175 | L(stosb_more_2x_vec): |
176 | cmpq $REP_STOSB_THRESHOLD, %rdx | |
eb2c88c7 | 177 | ja L(stosb) |
4af1bb06 | 178 | #endif |
83056630 L |
179 | L(more_2x_vec): |
180 | cmpq $(VEC_SIZE * 4), %rdx | |
181 | ja L(loop_start) | |
182 | VMOVU %VEC(0), (%rdi) | |
183 | VMOVU %VEC(0), VEC_SIZE(%rdi) | |
184 | VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) | |
185 | VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) | |
186 | L(return): | |
187 | VZEROUPPER | |
188 | ret | |
189 | ||
83056630 L |
190 | L(loop_start): |
191 | leaq (VEC_SIZE * 4)(%rdi), %rcx | |
192 | VMOVU %VEC(0), (%rdi) | |
193 | andq $-(VEC_SIZE * 4), %rcx | |
194 | VMOVU %VEC(0), -VEC_SIZE(%rdi,%rdx) | |
195 | VMOVU %VEC(0), VEC_SIZE(%rdi) | |
196 | VMOVU %VEC(0), -(VEC_SIZE * 2)(%rdi,%rdx) | |
197 | VMOVU %VEC(0), (VEC_SIZE * 2)(%rdi) | |
198 | VMOVU %VEC(0), -(VEC_SIZE * 3)(%rdi,%rdx) | |
199 | VMOVU %VEC(0), (VEC_SIZE * 3)(%rdi) | |
200 | VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi,%rdx) | |
201 | addq %rdi, %rdx | |
202 | andq $-(VEC_SIZE * 4), %rdx | |
203 | cmpq %rdx, %rcx | |
83056630 | 204 | je L(return) |
83056630 L |
205 | L(loop): |
206 | VMOVA %VEC(0), (%rcx) | |
207 | VMOVA %VEC(0), VEC_SIZE(%rcx) | |
208 | VMOVA %VEC(0), (VEC_SIZE * 2)(%rcx) | |
209 | VMOVA %VEC(0), (VEC_SIZE * 3)(%rcx) | |
210 | addq $(VEC_SIZE * 4), %rcx | |
211 | cmpq %rcx, %rdx | |
212 | jne L(loop) | |
213 | VZEROUPPER_SHORT_RETURN | |
214 | ret | |
215 | L(less_vec): | |
216 | /* Less than 1 VEC. */ | |
217 | # if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 | |
218 | # error Unsupported VEC_SIZE! | |
219 | # endif | |
220 | # if VEC_SIZE > 32 | |
221 | cmpb $32, %dl | |
222 | jae L(between_32_63) | |
223 | # endif | |
224 | # if VEC_SIZE > 16 | |
225 | cmpb $16, %dl | |
226 | jae L(between_16_31) | |
227 | # endif | |
228 | MOVQ %xmm0, %rcx | |
229 | cmpb $8, %dl | |
230 | jae L(between_8_15) | |
231 | cmpb $4, %dl | |
232 | jae L(between_4_7) | |
233 | cmpb $1, %dl | |
234 | ja L(between_2_3) | |
235 | jb 1f | |
236 | movb %cl, (%rdi) | |
237 | 1: | |
238 | VZEROUPPER | |
239 | ret | |
240 | # if VEC_SIZE > 32 | |
241 | /* From 32 to 63. No branch when size == 32. */ | |
242 | L(between_32_63): | |
243 | vmovdqu %ymm0, -32(%rdi,%rdx) | |
244 | vmovdqu %ymm0, (%rdi) | |
245 | VZEROUPPER | |
246 | ret | |
247 | # endif | |
248 | # if VEC_SIZE > 16 | |
249 | /* From 16 to 31. No branch when size == 16. */ | |
250 | L(between_16_31): | |
251 | vmovdqu %xmm0, -16(%rdi,%rdx) | |
252 | vmovdqu %xmm0, (%rdi) | |
253 | VZEROUPPER | |
254 | ret | |
255 | # endif | |
256 | /* From 8 to 15. No branch when size == 8. */ | |
257 | L(between_8_15): | |
258 | movq %rcx, -8(%rdi,%rdx) | |
259 | movq %rcx, (%rdi) | |
260 | VZEROUPPER | |
261 | ret | |
262 | L(between_4_7): | |
263 | /* From 4 to 7. No branch when size == 4. */ | |
264 | movl %ecx, -4(%rdi,%rdx) | |
265 | movl %ecx, (%rdi) | |
266 | VZEROUPPER | |
267 | ret | |
268 | L(between_2_3): | |
269 | /* From 2 to 3. No branch when size == 2. */ | |
270 | movw %cx, -2(%rdi,%rdx) | |
271 | movw %cx, (%rdi) | |
272 | VZEROUPPER | |
273 | ret | |
274 | END (MEMSET_SYMBOL (__memset, unaligned_erms)) |