]>
Commit | Line | Data |
---|---|---|
49889fb2 NG |
1 | /* strcat(dest, src) -- Append SRC on the end of DEST. |
2 | Optimized for x86-64. | |
6d7e8eda | 3 | Copyright (C) 2002-2023 Free Software Foundation, Inc. |
9ed0aa15 L |
4 | This file is part of the GNU C Library. |
5 | ||
6 | The GNU C Library is free software; you can redistribute it and/or | |
7 | modify it under the terms of the GNU Lesser General Public | |
8 | License as published by the Free Software Foundation; either | |
9 | version 2.1 of the License, or (at your option) any later version. | |
10 | ||
11 | The GNU C Library is distributed in the hope that it will be useful, | |
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | Lesser General Public License for more details. | |
15 | ||
16 | You should have received a copy of the GNU Lesser General Public | |
17 | License along with the GNU C Library; if not, see | |
5a82c748 | 18 | <https://www.gnu.org/licenses/>. */ |
9ed0aa15 | 19 | |
49889fb2 NG |
20 | #include <isa-level.h> |
21 | ||
22 | #if ISA_SHOULD_BUILD (1) | |
23 | ||
24 | # include <sysdep.h> | |
25 | ||
72a48ec0 | 26 | # ifndef STRCAT |
49889fb2 | 27 | # define STRCAT __strcat_sse2 |
72a48ec0 | 28 | # endif |
9ed0aa15 | 29 | |
49889fb2 | 30 | /* Will be removed when new strcpy implementation gets merged. */ |
72a48ec0 NG |
31 | |
32 | .text | |
33 | ENTRY (STRCAT) | |
34 | movq %rdi, %rcx /* Dest. register. */ | |
35 | andl $7, %ecx /* mask alignment bits */ | |
36 | movq %rdi, %rax /* Duplicate destination pointer. */ | |
37 | movq $0xfefefefefefefeff,%r8 | |
38 | ||
39 | /* First step: Find end of destination. */ | |
40 | jz 4f /* aligned => start loop */ | |
41 | ||
42 | neg %ecx /* We need to align to 8 bytes. */ | |
43 | addl $8,%ecx | |
44 | /* Search the first bytes directly. */ | |
45 | 0: cmpb $0x0,(%rax) /* is byte NUL? */ | |
46 | je 2f /* yes => start copy */ | |
47 | incq %rax /* increment pointer */ | |
48 | decl %ecx | |
49 | jnz 0b | |
50 | ||
51 | ||
52 | ||
53 | /* Now the source is aligned. Scan for NUL byte. */ | |
54 | .p2align 4 | |
55 | 4: | |
56 | /* First unroll. */ | |
57 | movq (%rax), %rcx /* get double word (= 8 bytes) in question */ | |
58 | addq $8,%rax /* adjust pointer for next word */ | |
59 | movq %r8, %rdx /* magic value */ | |
60 | addq %rcx, %rdx /* add the magic value to the word. We get | |
61 | carry bits reported for each byte which | |
62 | is *not* 0 */ | |
63 | jnc 3f /* highest byte is NUL => return pointer */ | |
64 | xorq %rcx, %rdx /* (word+magic)^word */ | |
65 | orq %r8, %rdx /* set all non-carry bits */ | |
66 | incq %rdx /* add 1: if one carry bit was *not* set | |
67 | the addition will not result in 0. */ | |
68 | jnz 3f /* found NUL => return pointer */ | |
69 | ||
70 | /* Second unroll. */ | |
71 | movq (%rax), %rcx /* get double word (= 8 bytes) in question */ | |
72 | addq $8,%rax /* adjust pointer for next word */ | |
73 | movq %r8, %rdx /* magic value */ | |
74 | addq %rcx, %rdx /* add the magic value to the word. We get | |
75 | carry bits reported for each byte which | |
76 | is *not* 0 */ | |
77 | jnc 3f /* highest byte is NUL => return pointer */ | |
78 | xorq %rcx, %rdx /* (word+magic)^word */ | |
79 | orq %r8, %rdx /* set all non-carry bits */ | |
80 | incq %rdx /* add 1: if one carry bit was *not* set | |
81 | the addition will not result in 0. */ | |
82 | jnz 3f /* found NUL => return pointer */ | |
83 | ||
84 | /* Third unroll. */ | |
85 | movq (%rax), %rcx /* get double word (= 8 bytes) in question */ | |
86 | addq $8,%rax /* adjust pointer for next word */ | |
87 | movq %r8, %rdx /* magic value */ | |
88 | addq %rcx, %rdx /* add the magic value to the word. We get | |
89 | carry bits reported for each byte which | |
90 | is *not* 0 */ | |
91 | jnc 3f /* highest byte is NUL => return pointer */ | |
92 | xorq %rcx, %rdx /* (word+magic)^word */ | |
93 | orq %r8, %rdx /* set all non-carry bits */ | |
94 | incq %rdx /* add 1: if one carry bit was *not* set | |
95 | the addition will not result in 0. */ | |
96 | jnz 3f /* found NUL => return pointer */ | |
97 | ||
98 | /* Fourth unroll. */ | |
99 | movq (%rax), %rcx /* get double word (= 8 bytes) in question */ | |
100 | addq $8,%rax /* adjust pointer for next word */ | |
101 | movq %r8, %rdx /* magic value */ | |
102 | addq %rcx, %rdx /* add the magic value to the word. We get | |
103 | carry bits reported for each byte which | |
104 | is *not* 0 */ | |
105 | jnc 3f /* highest byte is NUL => return pointer */ | |
106 | xorq %rcx, %rdx /* (word+magic)^word */ | |
107 | orq %r8, %rdx /* set all non-carry bits */ | |
108 | incq %rdx /* add 1: if one carry bit was *not* set | |
109 | the addition will not result in 0. */ | |
110 | jz 4b /* no NUL found => continue loop */ | |
111 | ||
112 | .p2align 4 /* Align, it's a jump target. */ | |
113 | 3: subq $8,%rax /* correct pointer increment. */ | |
114 | ||
115 | testb %cl, %cl /* is first byte NUL? */ | |
116 | jz 2f /* yes => return */ | |
117 | incq %rax /* increment pointer */ | |
118 | ||
119 | testb %ch, %ch /* is second byte NUL? */ | |
120 | jz 2f /* yes => return */ | |
121 | incq %rax /* increment pointer */ | |
122 | ||
123 | testl $0x00ff0000, %ecx /* is third byte NUL? */ | |
124 | jz 2f /* yes => return pointer */ | |
125 | incq %rax /* increment pointer */ | |
126 | ||
127 | testl $0xff000000, %ecx /* is fourth byte NUL? */ | |
128 | jz 2f /* yes => return pointer */ | |
129 | incq %rax /* increment pointer */ | |
130 | ||
131 | shrq $32, %rcx /* look at other half. */ | |
132 | ||
133 | testb %cl, %cl /* is first byte NUL? */ | |
134 | jz 2f /* yes => return */ | |
135 | incq %rax /* increment pointer */ | |
136 | ||
137 | testb %ch, %ch /* is second byte NUL? */ | |
138 | jz 2f /* yes => return */ | |
139 | incq %rax /* increment pointer */ | |
140 | ||
141 | testl $0xff0000, %ecx /* is third byte NUL? */ | |
142 | jz 2f /* yes => return pointer */ | |
143 | incq %rax /* increment pointer */ | |
144 | ||
145 | 2: | |
146 | /* Second step: Copy source to destination. */ | |
147 | ||
148 | movq %rsi, %rcx /* duplicate */ | |
149 | andl $7,%ecx /* mask alignment bits */ | |
150 | movq %rax, %rdx /* move around */ | |
151 | jz 22f /* aligned => start loop */ | |
152 | ||
153 | neg %ecx /* align to 8 bytes. */ | |
154 | addl $8, %ecx | |
155 | /* Align the source pointer. */ | |
156 | 21: | |
157 | movb (%rsi), %al /* Fetch a byte */ | |
158 | testb %al, %al /* Is it NUL? */ | |
159 | movb %al, (%rdx) /* Store it */ | |
160 | jz 24f /* If it was NUL, done! */ | |
161 | incq %rsi | |
162 | incq %rdx | |
163 | decl %ecx | |
164 | jnz 21b | |
165 | ||
166 | /* Now the sources is aligned. Unfortunatly we cannot force | |
167 | to have both source and destination aligned, so ignore the | |
168 | alignment of the destination. */ | |
169 | .p2align 4 | |
170 | 22: | |
171 | /* 1st unroll. */ | |
172 | movq (%rsi), %rax /* Read double word (8 bytes). */ | |
173 | addq $8, %rsi /* Adjust pointer for next word. */ | |
174 | movq %rax, %r9 /* Save a copy for NUL finding. */ | |
175 | addq %r8, %r9 /* add the magic value to the word. We get | |
176 | carry bits reported for each byte which | |
177 | is *not* 0 */ | |
178 | jnc 23f /* highest byte is NUL => return pointer */ | |
179 | xorq %rax, %r9 /* (word+magic)^word */ | |
180 | orq %r8, %r9 /* set all non-carry bits */ | |
181 | incq %r9 /* add 1: if one carry bit was *not* set | |
182 | the addition will not result in 0. */ | |
183 | ||
184 | jnz 23f /* found NUL => return pointer */ | |
185 | ||
186 | movq %rax, (%rdx) /* Write value to destination. */ | |
187 | addq $8, %rdx /* Adjust pointer. */ | |
188 | ||
189 | /* 2nd unroll. */ | |
190 | movq (%rsi), %rax /* Read double word (8 bytes). */ | |
191 | addq $8, %rsi /* Adjust pointer for next word. */ | |
192 | movq %rax, %r9 /* Save a copy for NUL finding. */ | |
193 | addq %r8, %r9 /* add the magic value to the word. We get | |
194 | carry bits reported for each byte which | |
195 | is *not* 0 */ | |
196 | jnc 23f /* highest byte is NUL => return pointer */ | |
197 | xorq %rax, %r9 /* (word+magic)^word */ | |
198 | orq %r8, %r9 /* set all non-carry bits */ | |
199 | incq %r9 /* add 1: if one carry bit was *not* set | |
200 | the addition will not result in 0. */ | |
201 | ||
202 | jnz 23f /* found NUL => return pointer */ | |
203 | ||
204 | movq %rax, (%rdx) /* Write value to destination. */ | |
205 | addq $8, %rdx /* Adjust pointer. */ | |
206 | ||
207 | /* 3rd unroll. */ | |
208 | movq (%rsi), %rax /* Read double word (8 bytes). */ | |
209 | addq $8, %rsi /* Adjust pointer for next word. */ | |
210 | movq %rax, %r9 /* Save a copy for NUL finding. */ | |
211 | addq %r8, %r9 /* add the magic value to the word. We get | |
212 | carry bits reported for each byte which | |
213 | is *not* 0 */ | |
214 | jnc 23f /* highest byte is NUL => return pointer */ | |
215 | xorq %rax, %r9 /* (word+magic)^word */ | |
216 | orq %r8, %r9 /* set all non-carry bits */ | |
217 | incq %r9 /* add 1: if one carry bit was *not* set | |
218 | the addition will not result in 0. */ | |
219 | ||
220 | jnz 23f /* found NUL => return pointer */ | |
221 | ||
222 | movq %rax, (%rdx) /* Write value to destination. */ | |
223 | addq $8, %rdx /* Adjust pointer. */ | |
224 | ||
225 | /* 4th unroll. */ | |
226 | movq (%rsi), %rax /* Read double word (8 bytes). */ | |
227 | addq $8, %rsi /* Adjust pointer for next word. */ | |
228 | movq %rax, %r9 /* Save a copy for NUL finding. */ | |
229 | addq %r8, %r9 /* add the magic value to the word. We get | |
230 | carry bits reported for each byte which | |
231 | is *not* 0 */ | |
232 | jnc 23f /* highest byte is NUL => return pointer */ | |
233 | xorq %rax, %r9 /* (word+magic)^word */ | |
234 | orq %r8, %r9 /* set all non-carry bits */ | |
235 | incq %r9 /* add 1: if one carry bit was *not* set | |
236 | the addition will not result in 0. */ | |
237 | ||
238 | jnz 23f /* found NUL => return pointer */ | |
239 | ||
240 | movq %rax, (%rdx) /* Write value to destination. */ | |
241 | addq $8, %rdx /* Adjust pointer. */ | |
242 | jmp 22b /* Next iteration. */ | |
243 | ||
244 | /* Do the last few bytes. %rax contains the value to write. | |
245 | The loop is unrolled twice. */ | |
246 | .p2align 4 | |
247 | 23: | |
248 | movb %al, (%rdx) /* 1st byte. */ | |
249 | testb %al, %al /* Is it NUL. */ | |
250 | jz 24f /* yes, finish. */ | |
251 | incq %rdx /* Increment destination. */ | |
252 | movb %ah, (%rdx) /* 2nd byte. */ | |
253 | testb %ah, %ah /* Is it NUL?. */ | |
254 | jz 24f /* yes, finish. */ | |
255 | incq %rdx /* Increment destination. */ | |
256 | shrq $16, %rax /* Shift... */ | |
257 | jmp 23b /* and look at next two bytes in %rax. */ | |
9ed0aa15 | 258 | |
9ed0aa15 | 259 | |
72a48ec0 NG |
260 | 24: |
261 | movq %rdi, %rax /* Source is return value. */ | |
262 | retq | |
263 | END (STRCAT) | |
49889fb2 | 264 | #endif |