]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 | 2 | #include <linux/string.h> |
e683014c | 3 | #include <linux/export.h> |
1da177e4 LT |
4 | |
5 | #undef memcpy | |
6 | #undef memset | |
7 | ||
a9143296 | 8 | __visible void *memcpy(void *to, const void *from, size_t n) |
1da177e4 | 9 | { |
6974f0c4 | 10 | #if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE) |
1da177e4 LT |
11 | return __memcpy3d(to, from, n); |
12 | #else | |
13 | return __memcpy(to, from, n); | |
14 | #endif | |
15 | } | |
16 | EXPORT_SYMBOL(memcpy); | |
17 | ||
a9143296 | 18 | __visible void *memset(void *s, int c, size_t count) |
1da177e4 LT |
19 | { |
20 | return __memset(s, c, count); | |
21 | } | |
22 | EXPORT_SYMBOL(memset); | |
23 | ||
a9143296 | 24 | __visible void *memmove(void *dest, const void *src, size_t n) |
1da177e4 | 25 | { |
3b4b682b ML |
26 | int d0,d1,d2,d3,d4,d5; |
27 | char *ret = dest; | |
28 | ||
29 | __asm__ __volatile__( | |
d50ba368 | 30 | /* Handle more 16 bytes in loop */ |
3b4b682b ML |
31 | "cmp $0x10, %0\n\t" |
32 | "jb 1f\n\t" | |
33 | ||
34 | /* Decide forward/backward copy mode */ | |
35 | "cmp %2, %1\n\t" | |
36 | "jb 2f\n\t" | |
37 | ||
38 | /* | |
39 | * movs instruction have many startup latency | |
40 | * so we handle small size by general register. | |
41 | */ | |
42 | "cmp $680, %0\n\t" | |
43 | "jb 3f\n\t" | |
44 | /* | |
45 | * movs instruction is only good for aligned case. | |
46 | */ | |
47 | "mov %1, %3\n\t" | |
48 | "xor %2, %3\n\t" | |
49 | "and $0xff, %3\n\t" | |
50 | "jz 4f\n\t" | |
51 | "3:\n\t" | |
52 | "sub $0x10, %0\n\t" | |
53 | ||
54 | /* | |
bb916ff7 | 55 | * We gobble 16 bytes forward in each loop. |
3b4b682b ML |
56 | */ |
57 | "3:\n\t" | |
58 | "sub $0x10, %0\n\t" | |
59 | "mov 0*4(%1), %3\n\t" | |
60 | "mov 1*4(%1), %4\n\t" | |
61 | "mov %3, 0*4(%2)\n\t" | |
62 | "mov %4, 1*4(%2)\n\t" | |
63 | "mov 2*4(%1), %3\n\t" | |
64 | "mov 3*4(%1), %4\n\t" | |
65 | "mov %3, 2*4(%2)\n\t" | |
66 | "mov %4, 3*4(%2)\n\t" | |
67 | "lea 0x10(%1), %1\n\t" | |
68 | "lea 0x10(%2), %2\n\t" | |
69 | "jae 3b\n\t" | |
70 | "add $0x10, %0\n\t" | |
71 | "jmp 1f\n\t" | |
72 | ||
73 | /* | |
74 | * Handle data forward by movs. | |
75 | */ | |
76 | ".p2align 4\n\t" | |
77 | "4:\n\t" | |
78 | "mov -4(%1, %0), %3\n\t" | |
79 | "lea -4(%2, %0), %4\n\t" | |
80 | "shr $2, %0\n\t" | |
81 | "rep movsl\n\t" | |
82 | "mov %3, (%4)\n\t" | |
83 | "jmp 11f\n\t" | |
84 | /* | |
85 | * Handle data backward by movs. | |
86 | */ | |
87 | ".p2align 4\n\t" | |
88 | "6:\n\t" | |
89 | "mov (%1), %3\n\t" | |
90 | "mov %2, %4\n\t" | |
91 | "lea -4(%1, %0), %1\n\t" | |
92 | "lea -4(%2, %0), %2\n\t" | |
93 | "shr $2, %0\n\t" | |
94 | "std\n\t" | |
95 | "rep movsl\n\t" | |
96 | "mov %3,(%4)\n\t" | |
97 | "cld\n\t" | |
98 | "jmp 11f\n\t" | |
99 | ||
100 | /* | |
101 | * Start to prepare for backward copy. | |
102 | */ | |
103 | ".p2align 4\n\t" | |
104 | "2:\n\t" | |
105 | "cmp $680, %0\n\t" | |
106 | "jb 5f\n\t" | |
107 | "mov %1, %3\n\t" | |
108 | "xor %2, %3\n\t" | |
109 | "and $0xff, %3\n\t" | |
110 | "jz 6b\n\t" | |
111 | ||
112 | /* | |
113 | * Calculate copy position to tail. | |
114 | */ | |
115 | "5:\n\t" | |
116 | "add %0, %1\n\t" | |
117 | "add %0, %2\n\t" | |
118 | "sub $0x10, %0\n\t" | |
119 | ||
120 | /* | |
bb916ff7 | 121 | * We gobble 16 bytes backward in each loop. |
3b4b682b ML |
122 | */ |
123 | "7:\n\t" | |
124 | "sub $0x10, %0\n\t" | |
125 | ||
126 | "mov -1*4(%1), %3\n\t" | |
127 | "mov -2*4(%1), %4\n\t" | |
128 | "mov %3, -1*4(%2)\n\t" | |
129 | "mov %4, -2*4(%2)\n\t" | |
130 | "mov -3*4(%1), %3\n\t" | |
131 | "mov -4*4(%1), %4\n\t" | |
132 | "mov %3, -3*4(%2)\n\t" | |
133 | "mov %4, -4*4(%2)\n\t" | |
134 | "lea -0x10(%1), %1\n\t" | |
135 | "lea -0x10(%2), %2\n\t" | |
136 | "jae 7b\n\t" | |
137 | /* | |
138 | * Calculate copy position to head. | |
139 | */ | |
140 | "add $0x10, %0\n\t" | |
141 | "sub %0, %1\n\t" | |
142 | "sub %0, %2\n\t" | |
143 | ||
144 | /* | |
145 | * Move data from 8 bytes to 15 bytes. | |
146 | */ | |
147 | ".p2align 4\n\t" | |
148 | "1:\n\t" | |
149 | "cmp $8, %0\n\t" | |
150 | "jb 8f\n\t" | |
151 | "mov 0*4(%1), %3\n\t" | |
152 | "mov 1*4(%1), %4\n\t" | |
153 | "mov -2*4(%1, %0), %5\n\t" | |
154 | "mov -1*4(%1, %0), %1\n\t" | |
155 | ||
156 | "mov %3, 0*4(%2)\n\t" | |
157 | "mov %4, 1*4(%2)\n\t" | |
158 | "mov %5, -2*4(%2, %0)\n\t" | |
159 | "mov %1, -1*4(%2, %0)\n\t" | |
160 | "jmp 11f\n\t" | |
161 | ||
162 | /* | |
163 | * Move data from 4 bytes to 7 bytes. | |
164 | */ | |
165 | ".p2align 4\n\t" | |
166 | "8:\n\t" | |
167 | "cmp $4, %0\n\t" | |
168 | "jb 9f\n\t" | |
169 | "mov 0*4(%1), %3\n\t" | |
170 | "mov -1*4(%1, %0), %4\n\t" | |
171 | "mov %3, 0*4(%2)\n\t" | |
172 | "mov %4, -1*4(%2, %0)\n\t" | |
173 | "jmp 11f\n\t" | |
174 | ||
175 | /* | |
176 | * Move data from 2 bytes to 3 bytes. | |
177 | */ | |
178 | ".p2align 4\n\t" | |
179 | "9:\n\t" | |
180 | "cmp $2, %0\n\t" | |
181 | "jb 10f\n\t" | |
182 | "movw 0*2(%1), %%dx\n\t" | |
183 | "movw -1*2(%1, %0), %%bx\n\t" | |
184 | "movw %%dx, 0*2(%2)\n\t" | |
185 | "movw %%bx, -1*2(%2, %0)\n\t" | |
186 | "jmp 11f\n\t" | |
187 | ||
188 | /* | |
189 | * Move data for 1 byte. | |
190 | */ | |
191 | ".p2align 4\n\t" | |
192 | "10:\n\t" | |
193 | "cmp $1, %0\n\t" | |
194 | "jb 11f\n\t" | |
195 | "movb (%1), %%cl\n\t" | |
196 | "movb %%cl, (%2)\n\t" | |
197 | ".p2align 4\n\t" | |
198 | "11:" | |
199 | : "=&c" (d0), "=&S" (d1), "=&D" (d2), | |
200 | "=r" (d3),"=r" (d4), "=r"(d5) | |
201 | :"0" (n), | |
202 | "1" (src), | |
203 | "2" (dest) | |
204 | :"memory"); | |
205 | ||
206 | return ret; | |
207 | ||
1da177e4 LT |
208 | } |
209 | EXPORT_SYMBOL(memmove); |