]>
Commit | Line | Data |
---|---|---|
99dee823 | 1 | /* Copyright (C) 2012-2021 Free Software Foundation, Inc. |
48310492 RH |
2 | Contributed by Richard Henderson <rth@redhat.com>. |
3 | ||
4 | This file is part of the GNU Atomic Library (libatomic). | |
5 | ||
6 | Libatomic is free software; you can redistribute it and/or modify it | |
7 | under the terms of the GNU General Public License as published by | |
8 | the Free Software Foundation; either version 3, or (at your option) | |
9 | any later version. | |
10 | ||
11 | Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
13 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | more details. | |
15 | ||
16 | Under Section 7 of GPL version 3, you are granted additional | |
17 | permissions described in the GCC Runtime Library Exception, version | |
18 | 3.1, as published by the Free Software Foundation. | |
19 | ||
20 | You should have received a copy of the GNU General Public License and | |
21 | a copy of the GCC Runtime Library Exception along with this program; | |
22 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
23 | <http://www.gnu.org/licenses/>. */ | |
24 | ||
25 | /* This file contains data types and function declarations that are | |
26 | private to the implementation of libatomic. */ | |
27 | ||
28 | #ifndef LIBATOMIC_H | |
29 | #define LIBATOMIC_H 1 | |
30 | ||
31 | #include "auto-config.h" | |
32 | #include <stdbool.h> | |
33 | #include <stdint.h> | |
34 | #include <stddef.h> | |
35 | #include <limits.h> | |
36 | #include <string.h> | |
37 | ||
38 | ||
39 | /* Symbol concatenation macros. */ | |
40 | #define C2_(X,Y) X ## Y | |
41 | #define C2(X,Y) C2_(X,Y) | |
42 | #define C3_(X,Y,Z) X ## Y ## Z | |
43 | #define C3(X,Y,Z) C3_(X,Y,Z) | |
44 | #define C4_(W,X,Y,Z) W ## X ## Y ## Z | |
45 | #define C4(W,X,Y,Z) C4_(W,X,Y,Z) | |
46 | ||
47 | /* Stringification macros. */ | |
48 | #define S2(X) #X | |
49 | #define S(X) S2(X) | |
50 | ||
51 | /* All of the primitive types on which we operate. */ | |
52 | typedef unsigned U_1 __attribute__((mode(QI))); | |
53 | #if HAVE_INT2 | |
54 | typedef unsigned U_2 __attribute__((mode(HI))); | |
55 | #endif | |
56 | #if HAVE_INT4 | |
57 | typedef unsigned U_4 __attribute__((mode(SI))); | |
58 | #endif | |
59 | #if HAVE_INT8 | |
60 | typedef unsigned U_8 __attribute__((mode(DI))); | |
61 | #endif | |
62 | #if HAVE_INT16 | |
63 | typedef unsigned U_16 __attribute__((mode(TI))); | |
64 | #endif | |
65 | ||
66 | /* The widest type that we support. */ | |
67 | #if HAVE_INT16 | |
68 | # define MAX_SIZE 16 | |
69 | #elif HAVE_INT8 | |
70 | # define MAX_SIZE 8 | |
71 | #elif HAVE_INT4 | |
72 | # define MAX_SIZE 4 | |
73 | #elif HAVE_INT2 | |
74 | # define MAX_SIZE 2 | |
75 | #else | |
76 | # define MAX_SIZE 1 | |
77 | #endif | |
78 | typedef C2(U_,MAX_SIZE) U_MAX; | |
79 | ||
80 | /* Provide dummy fallback types so that stuff is syntactically correct | |
81 | without having to overdo the ifdefs. The code using these should | |
82 | always be protected with the HAVE_INT{n} macros. */ | |
83 | #if !HAVE_INT2 | |
84 | typedef U_MAX U_2; | |
85 | #endif | |
86 | #if !HAVE_INT4 | |
87 | typedef U_MAX U_4; | |
88 | #endif | |
89 | #if !HAVE_INT8 | |
90 | typedef U_MAX U_8; | |
91 | #endif | |
92 | #if !HAVE_INT16 | |
93 | typedef U_MAX U_16; | |
94 | #endif | |
95 | ||
96 | union max_size_u | |
97 | { | |
98 | U_1 b[MAX_SIZE]; | |
99 | U_2 i2; | |
100 | U_4 i4; | |
101 | U_8 i8; | |
102 | U_16 i16; | |
103 | }; | |
104 | ||
105 | /* The "word" size of the machine. */ | |
106 | typedef unsigned UWORD __attribute__((mode(word))); | |
107 | ||
108 | /* Macros for handing sub-word sized quantities. */ | |
109 | #define MASK_1 ((UWORD)0xff) | |
110 | #define MASK_2 ((UWORD)0xffff) | |
111 | #define MASK_4 ((UWORD)0xffffffff) | |
15545563 | 112 | #define MASK_8 ((UWORD)0xffffffffffffffff) |
48310492 RH |
113 | #define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT)) |
114 | #define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT)) | |
115 | #define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT)) | |
15545563 | 116 | #define INVERT_MASK_8 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 8) * CHAR_BIT)) |
48310492 RH |
117 | |
118 | /* Most of the files in this library are compiled multiple times with | |
119 | N defined to be a power of 2 between 1 and 16. The SIZE macro is | |
120 | then used to append _N to the symbol being manipulated. */ | |
121 | #define SIZE(X) C3(X,_,N) | |
122 | #define WSIZE(X) C3(X,_,WORDSIZE) | |
123 | #define PTR(N,X) ((C2(U_,N) *)X) | |
124 | ||
125 | /* And thus, the type on which this compilation will be operating. */ | |
126 | #define ITYPE SIZE(I) | |
127 | #define UTYPE SIZE(U) | |
128 | ||
129 | /* Utility macros for GCC attributes. */ | |
130 | #define UNUSED __attribute__((unused)) | |
131 | #ifdef HAVE_ATTRIBUTE_VISIBILITY | |
132 | # define HIDDEN __attribute__((visibility("hidden"))) | |
133 | #else | |
134 | # define HIDDEN | |
135 | #endif | |
136 | ||
137 | /* Occasionally we have to play games with internal and external symbol | |
138 | names, in order to work around builtin functions of the same name. | |
139 | This macro sets the external name of the function appropriately. */ | |
140 | #define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X))) | |
141 | ||
142 | /* Locking for a "small" operation. In the bare-metal single processor | |
143 | cases this could be implemented by disabling interrupts. Thus the extra | |
144 | word passed between the two functions, saving the interrupt level. | |
145 | It is assumed that the object being locked does not cross the locking | |
146 | granularity. | |
147 | ||
148 | Not actually declared here so that they can be defined static inline | |
149 | in a target-specfic <host-config.h>. | |
150 | ||
151 | UWORD protect_start (void *ptr); | |
152 | void protect_end (void *ptr, UWORD); | |
153 | */ | |
154 | ||
155 | /* Locking for a "large' operation. This should always be some sort of | |
156 | test-and-set operation, as we assume that the interrupt latency would | |
157 | be unreasonably large. */ | |
158 | void libat_lock_n (void *ptr, size_t n); | |
159 | void libat_unlock_n (void *ptr, size_t n); | |
160 | ||
161 | /* We'll need to declare all of the sized functions a few times... */ | |
162 | #define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N)) | |
163 | #define DECLARE_ALL_SIZED_(N,T) \ | |
164 | DECLARE_1(T, C2(load_,N), (T *mptr, int)); \ | |
165 | DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \ | |
166 | DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \ | |
167 | DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \ | |
168 | DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \ | |
169 | DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \ | |
170 | DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \ | |
171 | DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \ | |
172 | DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \ | |
173 | DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \ | |
174 | DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \ | |
175 | DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \ | |
176 | DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \ | |
177 | DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \ | |
178 | DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \ | |
179 | DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \ | |
180 | DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int)) | |
181 | ||
182 | /* All sized operations are implemented in hidden functions prefixed with | |
183 | "libat_". These are either renamed or aliased to the expected prefix | |
184 | of "__atomic". Some amount of renaming is required to avoid hiding or | |
185 | conflicting with the builtins of the same name, but this additional | |
186 | use of hidden symbols (where appropriate) avoids unnecessary PLT entries | |
187 | on relevant targets. */ | |
188 | ||
189 | #if IFUNC_ALT | |
190 | # define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN | |
191 | #elif defined(HAVE_ATTRIBUTE_ALIAS) | |
192 | # define MAN(X) HIDDEN | |
193 | #else | |
194 | # define MAN(X) ASMNAME(C2(__atomic_,X)) | |
195 | #endif | |
196 | ||
197 | #if !defined(N) && HAVE_IFUNC | |
198 | # define DECLARE_1(RET,NAME,ARGS) \ | |
199 | RET C2(libat_,NAME) ARGS MAN(NAME); \ | |
200 | RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME)) | |
201 | #else | |
202 | # define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME) | |
203 | #endif | |
204 | ||
205 | /* Prefix to use when calling internal, possibly ifunc'ed functions. */ | |
206 | #if HAVE_IFUNC | |
207 | # define local_ ifunc_ | |
208 | #else | |
209 | # define local_ libat_ | |
210 | #endif | |
211 | ||
212 | DECLARE_ALL_SIZED(1); | |
213 | DECLARE_ALL_SIZED(2); | |
214 | DECLARE_ALL_SIZED(4); | |
215 | DECLARE_ALL_SIZED(8); | |
216 | DECLARE_ALL_SIZED(16); | |
217 | ||
218 | #undef DECLARE_1 | |
219 | #undef DECLARE_ALL_SIZED | |
220 | #undef DECLARE_ALL_SIZED_ | |
221 | ||
222 | /* And the generic sized versions. */ | |
223 | void libat_load (size_t, void *, void *, int) MAN(load); | |
224 | void libat_store (size_t, void *, void *, int) MAN(store); | |
225 | void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange); | |
226 | bool libat_compare_exchange (size_t, void *, void *, void *, int, int) | |
227 | MAN(compare_exchange); | |
228 | bool libat_is_lock_free (size_t, void *) MAN(is_lock_free); | |
229 | ||
230 | #undef MAN | |
231 | ||
232 | #include <host-config.h> | |
233 | ||
234 | /* We don't have IFUNC_NCOND until after host-config.h. */ | |
235 | #if !HAVE_IFUNC | |
236 | # define IFUNC_NCOND(N) 0 | |
237 | #endif | |
238 | ||
239 | #if IFUNC_ALT | |
240 | # define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */ | |
241 | #elif defined(N) && IFUNC_NCOND(N) | |
242 | # if IFUNC_NCOND(N) == 1 | |
243 | # define GEN_SELECTOR(X) \ | |
244 | extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ | |
141abc6f | 245 | static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ |
48310492 RH |
246 | { \ |
247 | if (IFUNC_COND_1) \ | |
248 | return C3(libat_,X,_i1); \ | |
249 | return C2(libat_,X); \ | |
250 | } | |
251 | # elif IFUNC_NCOND(N) == 2 | |
252 | # define GEN_SELECTOR(X) \ | |
253 | extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ | |
254 | extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \ | |
141abc6f | 255 | static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ |
48310492 RH |
256 | { \ |
257 | if (IFUNC_COND_1) \ | |
258 | return C3(libat_,X,_i1); \ | |
259 | if (IFUNC_COND_2) \ | |
260 | return C3(libat_,X,_i2); \ | |
261 | return C2(libat_,X); \ | |
262 | } | |
263 | # elif IFUNC_NCOND(N) == 3 | |
264 | # define GEN_SELECTOR(X) \ | |
265 | extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ | |
266 | extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \ | |
267 | extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \ | |
141abc6f | 268 | static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ |
48310492 RH |
269 | { \ |
270 | if (IFUNC_COND_1) \ | |
271 | return C3(libat_,X,_i1); \ | |
272 | if (IFUNC_COND_2) \ | |
273 | return C3(libat_,X,_i2); \ | |
274 | if (IFUNC_COND_3) \ | |
275 | return C3(libat_,X,_i3); \ | |
276 | return C2(libat_,X); \ | |
277 | } | |
278 | # else | |
279 | # error "Unsupported number of ifunc alternatives." | |
280 | # endif | |
281 | # define EXPORT_ALIAS(X) \ | |
282 | GEN_SELECTOR(X) \ | |
283 | typeof(C2(libat_,X)) C2(ifunc_,X) \ | |
284 | ASMNAME(C2(__atomic_,X)) \ | |
285 | __attribute__((ifunc(S(C2(select_,X))))) | |
286 | #elif defined(HAVE_ATTRIBUTE_ALIAS) | |
287 | # define EXPORT_ALIAS(X) \ | |
288 | extern typeof(C2(libat_,X)) C2(export_,X) \ | |
289 | ASMNAME(C2(__atomic_,X)) \ | |
290 | __attribute__((alias(S(C2(libat_,X))))) | |
291 | #else | |
292 | # define EXPORT_ALIAS(X) /* original symbol is exported */ | |
293 | #endif | |
294 | ||
295 | #endif /* LIBATOMIC_H */ |