]>
Commit | Line | Data |
---|---|---|
28f540f4 | 1 | /* memcopy.h -- definitions for memory copy functions. Generic C version. |
04277e02 | 2 | Copyright (C) 1991-2019 Free Software Foundation, Inc. |
41bdb6e2 | 3 | This file is part of the GNU C Library. |
28f540f4 RM |
4 | Contributed by Torbjorn Granlund (tege@sics.se). |
5 | ||
c84142e8 | 6 | The GNU C Library is free software; you can redistribute it and/or |
41bdb6e2 AJ |
7 | modify it under the terms of the GNU Lesser General Public |
8 | License as published by the Free Software Foundation; either | |
9 | version 2.1 of the License, or (at your option) any later version. | |
28f540f4 | 10 | |
c84142e8 UD |
11 | The GNU C Library is distributed in the hope that it will be useful, |
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
41bdb6e2 | 14 | Lesser General Public License for more details. |
28f540f4 | 15 | |
41bdb6e2 | 16 | You should have received a copy of the GNU Lesser General Public |
59ba27a6 PE |
17 | License along with the GNU C Library; if not, see |
18 | <http://www.gnu.org/licenses/>. */ | |
28f540f4 | 19 | |
c8df52ec DM |
20 | #ifndef _MEMCOPY_H |
21 | #define _MEMCOPY_H 1 | |
22 | ||
28f540f4 RM |
23 | /* The strategy of the memory functions is: |
24 | ||
25 | 1. Copy bytes until the destination pointer is aligned. | |
26 | ||
27 | 2. Copy words in unrolled loops. If the source and destination | |
28 | are not aligned in the same way, use word memory operations, | |
29 | but shift and merge two read words before writing. | |
30 | ||
31 | 3. Copy the few remaining bytes. | |
32 | ||
33 | This is fast on processors that have at least 10 registers for | |
34 | allocation by GCC, and that can access memory at reg+const in one | |
35 | instruction. | |
36 | ||
37 | I made an "exhaustive" test of this memmove when I wrote it, | |
38 | exhaustive in the sense that I tried all alignment and length | |
39 | combinations, with and without overlap. */ | |
40 | ||
41 | #include <sys/cdefs.h> | |
42 | #include <endian.h> | |
99f8dc92 | 43 | #include <pagecopy.h> |
28f540f4 RM |
44 | |
45 | /* The macros defined in this file are: | |
46 | ||
47 | BYTE_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_to_copy) | |
48 | ||
49 | BYTE_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_to_copy) | |
50 | ||
51 | WORD_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_remaining, nbytes_to_copy) | |
52 | ||
53 | WORD_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_remaining, nbytes_to_copy) | |
54 | ||
55 | MERGE(old_word, sh_1, new_word, sh_2) | |
56 | [I fail to understand. I feel stupid. --roland] | |
57 | */ | |
58 | ||
59 | /* Type to use for aligned memory operations. | |
60 | This should normally be the biggest type supported by a single load | |
61 | and store. */ | |
62 | #define op_t unsigned long int | |
c4f50205 | 63 | #define OPSIZ (sizeof (op_t)) |
28f540f4 RM |
64 | |
65 | /* Type to use for unaligned operations. */ | |
66 | typedef unsigned char byte; | |
67 | ||
28f540f4 RM |
68 | #if __BYTE_ORDER == __LITTLE_ENDIAN |
69 | #define MERGE(w0, sh_1, w1, sh_2) (((w0) >> (sh_1)) | ((w1) << (sh_2))) | |
70 | #endif | |
71 | #if __BYTE_ORDER == __BIG_ENDIAN | |
72 | #define MERGE(w0, sh_1, w1, sh_2) (((w0) << (sh_1)) | ((w1) >> (sh_2))) | |
73 | #endif | |
74 | ||
75 | /* Copy exactly NBYTES bytes from SRC_BP to DST_BP, | |
76 | without any assumptions about alignment of the pointers. */ | |
77 | #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes) \ | |
78 | do \ | |
79 | { \ | |
80 | size_t __nbytes = (nbytes); \ | |
81 | while (__nbytes > 0) \ | |
82 | { \ | |
83 | byte __x = ((byte *) src_bp)[0]; \ | |
84 | src_bp += 1; \ | |
85 | __nbytes -= 1; \ | |
86 | ((byte *) dst_bp)[0] = __x; \ | |
87 | dst_bp += 1; \ | |
88 | } \ | |
89 | } while (0) | |
90 | ||
91 | /* Copy exactly NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR, | |
92 | beginning at the bytes right before the pointers and continuing towards | |
93 | smaller addresses. Don't assume anything about alignment of the | |
94 | pointers. */ | |
95 | #define BYTE_COPY_BWD(dst_ep, src_ep, nbytes) \ | |
96 | do \ | |
97 | { \ | |
98 | size_t __nbytes = (nbytes); \ | |
99 | while (__nbytes > 0) \ | |
100 | { \ | |
101 | byte __x; \ | |
102 | src_ep -= 1; \ | |
103 | __x = ((byte *) src_ep)[0]; \ | |
104 | dst_ep -= 1; \ | |
105 | __nbytes -= 1; \ | |
106 | ((byte *) dst_ep)[0] = __x; \ | |
107 | } \ | |
108 | } while (0) | |
109 | ||
110 | /* Copy *up to* NBYTES bytes from SRC_BP to DST_BP, with | |
111 | the assumption that DST_BP is aligned on an OPSIZ multiple. If | |
112 | not all bytes could be easily copied, store remaining number of bytes | |
113 | in NBYTES_LEFT, otherwise store 0. */ | |
b8b21bde L |
114 | extern void _wordcopy_fwd_aligned (long int, long int, size_t) |
115 | attribute_hidden __THROW; | |
116 | extern void _wordcopy_fwd_dest_aligned (long int, long int, size_t) | |
117 | attribute_hidden __THROW; | |
28f540f4 RM |
118 | #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes) \ |
119 | do \ | |
120 | { \ | |
121 | if (src_bp % OPSIZ == 0) \ | |
122 | _wordcopy_fwd_aligned (dst_bp, src_bp, (nbytes) / OPSIZ); \ | |
123 | else \ | |
124 | _wordcopy_fwd_dest_aligned (dst_bp, src_bp, (nbytes) / OPSIZ); \ | |
125 | src_bp += (nbytes) & -OPSIZ; \ | |
126 | dst_bp += (nbytes) & -OPSIZ; \ | |
127 | (nbytes_left) = (nbytes) % OPSIZ; \ | |
128 | } while (0) | |
129 | ||
130 | /* Copy *up to* NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR, | |
131 | beginning at the words (of type op_t) right before the pointers and | |
132 | continuing towards smaller addresses. May take advantage of that | |
133 | DST_END_PTR is aligned on an OPSIZ multiple. If not all bytes could be | |
134 | easily copied, store remaining number of bytes in NBYTES_REMAINING, | |
135 | otherwise store 0. */ | |
b8b21bde L |
136 | extern void _wordcopy_bwd_aligned (long int, long int, size_t) |
137 | attribute_hidden __THROW; | |
138 | extern void _wordcopy_bwd_dest_aligned (long int, long int, size_t) | |
139 | attribute_hidden __THROW; | |
28f540f4 RM |
140 | #define WORD_COPY_BWD(dst_ep, src_ep, nbytes_left, nbytes) \ |
141 | do \ | |
142 | { \ | |
143 | if (src_ep % OPSIZ == 0) \ | |
144 | _wordcopy_bwd_aligned (dst_ep, src_ep, (nbytes) / OPSIZ); \ | |
145 | else \ | |
146 | _wordcopy_bwd_dest_aligned (dst_ep, src_ep, (nbytes) / OPSIZ); \ | |
147 | src_ep -= (nbytes) & -OPSIZ; \ | |
148 | dst_ep -= (nbytes) & -OPSIZ; \ | |
149 | (nbytes_left) = (nbytes) % OPSIZ; \ | |
150 | } while (0) | |
151 | ||
99f8dc92 SP |
152 | /* The macro PAGE_COPY_FWD_MAYBE (dstp, srcp, nbytes_left, nbytes) is invoked |
153 | like WORD_COPY_FWD et al. The pointers should be at least word aligned. | |
154 | This will check if virtual copying by pages can and should be done and do it | |
155 | if so. The pointers will be aligned to PAGE_SIZE bytes. The macro requires | |
156 | that pagecopy.h defines at least PAGE_COPY_THRESHOLD to 0. If | |
157 | PAGE_COPY_THRESHOLD is non-zero, the header must also define PAGE_COPY_FWD | |
158 | and PAGE_SIZE. | |
159 | */ | |
160 | #if PAGE_COPY_THRESHOLD | |
161 | ||
162 | # include <assert.h> | |
163 | ||
164 | # define PAGE_COPY_FWD_MAYBE(dstp, srcp, nbytes_left, nbytes) \ | |
165 | do \ | |
166 | { \ | |
a04549c1 JM |
167 | if ((nbytes) >= PAGE_COPY_THRESHOLD \ |
168 | && PAGE_OFFSET ((dstp) - (srcp)) == 0) \ | |
99f8dc92 SP |
169 | { \ |
170 | /* The amount to copy is past the threshold for copying \ | |
171 | pages virtually with kernel VM operations, and the \ | |
172 | source and destination addresses have the same alignment. */ \ | |
173 | size_t nbytes_before = PAGE_OFFSET (-(dstp)); \ | |
174 | if (nbytes_before != 0) \ | |
175 | { \ | |
176 | /* First copy the words before the first page boundary. */ \ | |
177 | WORD_COPY_FWD (dstp, srcp, nbytes_left, nbytes_before); \ | |
178 | assert (nbytes_left == 0); \ | |
179 | nbytes -= nbytes_before; \ | |
180 | } \ | |
181 | PAGE_COPY_FWD (dstp, srcp, nbytes_left, nbytes); \ | |
182 | } \ | |
183 | } while (0) | |
184 | ||
185 | /* The page size is always a power of two, so we can avoid modulo division. */ | |
186 | # define PAGE_OFFSET(n) ((n) & (PAGE_SIZE - 1)) | |
187 | ||
188 | #else | |
189 | ||
190 | # define PAGE_COPY_FWD_MAYBE(dstp, srcp, nbytes_left, nbytes) /* nada */ | |
191 | ||
192 | #endif | |
28f540f4 RM |
193 | |
194 | /* Threshold value for when to enter the unrolled loops. */ | |
195 | #define OP_T_THRES 16 | |
c8df52ec | 196 | |
96baf6ff SP |
197 | /* Set to 1 if memcpy is safe to use for forward-copying memmove with |
198 | overlapping addresses. This is 0 by default because memcpy implementations | |
199 | are generally not safe for overlapping addresses. */ | |
64df73c2 SP |
200 | #define MEMCPY_OK_FOR_FWD_MEMMOVE 0 |
201 | ||
c8df52ec | 202 | #endif /* memcopy.h */ |