1 /* Optimized memset implementation for PowerPC.
2 Copyright (C) 1997-2020 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
21 /* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));
24 The memset is done in four sizes: byte (8 bits), word (32 bits),
25 32-byte blocks (256 bits) and __cache_line_size (128, 256, 1024 bits).
26 There is a special case for setting whole cache lines to 0, which
27 takes advantage of the dcbz instruction. */
33 #define rRTN r3 /* initial value of 1st argument */
34 #define rMEMP0 r3 /* original value of 1st arg */
35 #define rCHR r4 /* char to set in each byte */
36 #define rLEN r5 /* length of region to set */
37 #define rMEMP r6 /* address at which we are storing */
38 #define rALIGN r7 /* number of bytes we are setting now (when aligning) */
41 #define rPOS32 r7 /* constant +32 for clearing with dcbz */
42 #define rNEG64 r8 /* constant -64 for clearing with dcbz */
43 #define rNEG32 r9 /* constant -32 for clearing with dcbz */
45 #define rGOT r9 /* Address of the Global Offset Table. */
46 #define rCLS r8 /* Cache line size obtained from static. */
47 #define rCLM r9 /* Cache line size mask to check for cache alignment. */
49 /* take care of case for size <= 4 */
51 andi. rALIGN, rMEMP0, 3
54 /* align to word boundary */
56 rlwimi rCHR, rCHR, 8, 16, 23
57 beq+ L(aligned) /* 8th instruction from .align */
59 subfic rALIGN, rALIGN, 4
60 add rMEMP, rMEMP, rALIGN
61 sub rLEN, rLEN, rALIGN
65 L(g0): sth rCHR, -2(rMEMP) /* 16th instruction from .align */
66 /* take care of case for size < 31 */
69 rlwimi rCHR, rCHR, 16, 0, 15
71 /* align to cache line boundary... */
72 andi. rALIGN, rMEMP, 0x1C
73 subfic rALIGN, rALIGN, 0x20
76 add rMEMP, rMEMP, rALIGN
77 sub rLEN, rLEN, rALIGN
78 cmplwi cr1, rALIGN, 0x10
84 stw rCHR, -4(rMEMP2) /* 32nd instruction from .align */
87 stwu rCHR, -16(rMEMP2)
88 L(a2): bf 29, L(caligned)
90 /* now aligned to a cache line. */
93 clrrwi. rALIGN, rLEN, 5
94 mtcrf 0x01, rLEN /* 40th instruction from .align */
96 /* Check if we can use the special case for clearing memory using dcbz.
97 This requires that we know the correct cache line size for this
98 processor. Getting the __cache_line_size may require establishing GOT
99 addressability, so branch out of line to set this up. */
100 beq cr1, L(checklinesize)
102 /* Store blocks of 32-bytes (256-bits) starting on a 32-byte boundary.
103 Can't assume that rCHR is zero or that the cache line size is either
104 32-bytes or even known. */
108 beq L(medium) /* we may not actually get to do a full line */
109 clrlwi. rLEN, rLEN, 27
110 add rMEMP, rMEMP, rALIGN
112 bdz L(cloopdone) /* 48th instruction from .align */
114 /* We can't use dcbz here as we don't know the cache line size. We can
115 use "data cache block touch for store", which is safe. */
116 L(c3): dcbtst rNEG64, rMEMP
121 nop /* let 601 fetch last 4 instructions of loop */
123 stw rCHR, -24(rMEMP) /* 56th instruction from .align */
124 nop /* let 601 fetch first 8 instructions of loop */
126 stwu rCHR, -32(rMEMP)
132 stw rCHR, -16(rMEMP) /* 64th instruction from .align */
137 stwu rCHR, -32(rMEMP)
139 add rMEMP, rMEMP, rALIGN
140 b L(medium_tail2) /* 72nd instruction from .align */
144 /* Clear cache lines of memory in 128-byte chunks.
145 This code is optimized for processors with 32-byte cache lines.
146 It is further optimized for the 601 processor, which requires
147 some care in how the code is aligned in the i-cache. */
149 clrlwi rLEN, rLEN, 27
151 srwi. rTMP, rALIGN, 7
155 cmplwi cr1, rLEN, 16 /* 8 */
158 addi rMEMP, rMEMP, 0x20
159 L(z0): li rNEG32, -0x20
163 addi rMEMP, rMEMP, 0x40 /* 16 */
164 L(z1): cmplwi cr5, rLEN, 0
169 addi rMEMP, rMEMP, 0x80
178 /* Memset of 4 bytes or less. */
193 /* Memset of 0-31 bytes. */
198 add rMEMP, rMEMP, rLEN
200 bt- 31, L(medium_31t)
201 bt- 30, L(medium_30t)
203 bt- 29, L(medium_29t)
205 bge- cr1, L(medium_27t)
207 stw rCHR, -4(rMEMP) /* 8th instruction from .align */
213 bf- 30, L(medium_30f)
216 bf- 29, L(medium_29f)
219 blt- cr1, L(medium_27f) /* 16th instruction from .align */
224 stwu rCHR, -16(rMEMP)
235 /* If the remaining length is less the 32 bytes then don't bother getting
236 the cache line size. */
238 /* Establishes GOT addressability so we can load __cache_line_size
239 from static. This value was set from the aux vector during startup. */
240 SETUP_GOT_ACCESS(rGOT,got_label)
241 addis rGOT,rGOT,__cache_line_size-got_label@ha
242 lwz rCLS,__cache_line_size-got_label@l(rGOT)
245 /* Load __cache_line_size from static. This value was set from the
246 aux vector during startup. */
247 lis rCLS,__cache_line_size@ha
248 /* If the remaining length is less the 32 bytes then don't bother getting
249 the cache line size. */
251 lwz rCLS,__cache_line_size@l(rCLS)
254 /* If the cache line size was not set then goto to L(nondcbz), which is
255 safe for any cache line size. */
259 /* If the cache line size is 32 bytes then goto to L(zloopstart),
260 which is coded specifically for 32-byte lines (and 601). */
262 beq cr1,L(zloopstart)
264 /* Now we know the cache line size and it is not 32-bytes. However
265 we may not yet be aligned to the cache line and may have a partial
266 line to fill. Touch it 1st to fetch the cache line. */
273 blt cr1,L(handletail32)
275 /* We are not aligned to start of a cache line yet. Store 32-byte
276 of data and test again. */
289 /* Now we are aligned to the cache line and can use dcbz. */
292 blt cr1,L(handletail32)
298 /* We are here because; the cache line size was set, it was not
299 32-bytes, and the remainder (rLEN) is now less than the actual cache
300 line size. Set up the preconditions for L(nondcbz) and go there to
301 store the remaining bytes. */
303 clrrwi. rALIGN, rLEN, 5
307 libc_hidden_builtin_def (memset)