]>
Commit | Line | Data |
---|---|---|
7adcbafe | 1 | /* Copyright (C) 2008-2022 Free Software Foundation, Inc. |
0a35513e AH |
2 | Contributed by Richard Henderson <rth@redhat.com>. |
3 | ||
4 | This file is part of the GNU Transactional Memory Library (libitm). | |
5 | ||
6 | Libitm is free software; you can redistribute it and/or modify it | |
7 | under the terms of the GNU General Public License as published by | |
8 | the Free Software Foundation; either version 3 of the License, or | |
9 | (at your option) any later version. | |
10 | ||
11 | Libitm is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
13 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | more details. | |
15 | ||
16 | Under Section 7 of GPL version 3, you are granted additional | |
17 | permissions described in the GCC Runtime Library Exception, version | |
18 | 3.1, as published by the Free Software Foundation. | |
19 | ||
20 | You should have received a copy of the GNU General Public License and | |
21 | a copy of the GCC Runtime Library Exception along with this program; | |
22 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
23 | <http://www.gnu.org/licenses/>. */ | |
24 | ||
25 | #include "libitm_i.h" | |
26 | ||
27 | // Avoid a dependency on libstdc++ for the pure virtuals in abi_dispatch. | |
28 | extern "C" void HIDDEN | |
29 | __cxa_pure_virtual () | |
30 | { | |
31 | abort (); | |
32 | } | |
33 | ||
34 | using namespace GTM; | |
35 | ||
36 | namespace { | |
37 | ||
38 | // This group consists of the serial, serialirr, and serialirr_onwrite | |
39 | // methods, which all need no global state (except what is already provided | |
40 | // by the serial mode implementation). | |
41 | struct serial_mg : public method_group | |
42 | { | |
43 | virtual void init() { } | |
44 | virtual void fini() { } | |
45 | }; | |
46 | ||
47 | static serial_mg o_serial_mg; | |
48 | ||
49 | ||
50 | class serialirr_dispatch : public abi_dispatch | |
51 | { | |
52 | public: | |
b679c813 TR |
53 | serialirr_dispatch() : abi_dispatch(false, true, true, false, |
54 | gtm_thread::STATE_SERIAL | gtm_thread::STATE_IRREVOCABLE, &o_serial_mg) | |
0a35513e AH |
55 | { } |
56 | ||
57 | protected: | |
58 | serialirr_dispatch(bool ro, bool wt, bool uninstrumented, | |
b679c813 TR |
59 | bool closed_nesting, uint32_t requires_serial, method_group* mg) : |
60 | abi_dispatch(ro, wt, uninstrumented, closed_nesting, requires_serial, mg) | |
61 | { } | |
0a35513e AH |
62 | |
63 | // Transactional loads and stores simply access memory directly. | |
64 | // These methods are static to avoid indirect calls, and will be used by the | |
65 | // virtual ABI dispatch methods or by static direct-access methods created | |
66 | // below. | |
67 | template <typename V> static V load(const V* addr, ls_modifier mod) | |
68 | { | |
69 | return *addr; | |
70 | } | |
71 | template <typename V> static void store(V* addr, const V value, | |
72 | ls_modifier mod) | |
73 | { | |
74 | *addr = value; | |
75 | } | |
76 | ||
77 | public: | |
78 | static void memtransfer_static(void *dst, const void* src, size_t size, | |
79 | bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod) | |
80 | { | |
81 | if (!may_overlap) | |
82 | ::memcpy(dst, src, size); | |
83 | else | |
84 | ::memmove(dst, src, size); | |
85 | } | |
86 | ||
87 | static void memset_static(void *dst, int c, size_t size, ls_modifier mod) | |
88 | { | |
89 | ::memset(dst, c, size); | |
90 | } | |
91 | ||
92 | CREATE_DISPATCH_METHODS(virtual, ) | |
93 | CREATE_DISPATCH_METHODS_MEM() | |
94 | ||
95 | virtual gtm_restart_reason begin_or_restart() { return NO_RESTART; } | |
96 | virtual bool trycommit(gtm_word& priv_time) { return true; } | |
97 | virtual void rollback(gtm_transaction_cp *cp) { abort(); } | |
629e4729 | 98 | virtual bool snapshot_most_recent() { return true; } |
0a35513e AH |
99 | |
100 | virtual abi_dispatch* closed_nesting_alternative() | |
101 | { | |
102 | // For nested transactions with an instrumented code path, we can do | |
103 | // undo logging. | |
104 | return GTM::dispatch_serial(); | |
105 | } | |
106 | }; | |
107 | ||
108 | class serial_dispatch : public abi_dispatch | |
109 | { | |
110 | protected: | |
111 | static void log(const void *addr, size_t len) | |
112 | { | |
11f30bb0 TR |
113 | gtm_thread *tx = gtm_thr(); |
114 | tx->undolog.log(addr, len); | |
0a35513e AH |
115 | } |
116 | ||
117 | template <typename V> static V load(const V* addr, ls_modifier mod) | |
118 | { | |
119 | return *addr; | |
120 | } | |
121 | template <typename V> static void store(V* addr, const V value, | |
122 | ls_modifier mod) | |
123 | { | |
124 | if (mod != WaW) | |
125 | log(addr, sizeof(V)); | |
126 | *addr = value; | |
127 | } | |
128 | ||
129 | public: | |
130 | static void memtransfer_static(void *dst, const void* src, size_t size, | |
131 | bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod) | |
132 | { | |
133 | if (dst_mod != WaW && dst_mod != NONTXNAL) | |
134 | log(dst, size); | |
135 | if (!may_overlap) | |
136 | ::memcpy(dst, src, size); | |
137 | else | |
138 | ::memmove(dst, src, size); | |
139 | } | |
140 | ||
141 | static void memset_static(void *dst, int c, size_t size, ls_modifier mod) | |
142 | { | |
143 | if (mod != WaW) | |
144 | log(dst, size); | |
145 | ::memset(dst, c, size); | |
146 | } | |
147 | ||
148 | virtual gtm_restart_reason begin_or_restart() { return NO_RESTART; } | |
149 | virtual bool trycommit(gtm_word& priv_time) { return true; } | |
150 | // Local undo will handle this. | |
151 | // trydropreference() need not be changed either. | |
152 | virtual void rollback(gtm_transaction_cp *cp) { } | |
629e4729 | 153 | virtual bool snapshot_most_recent() { return true; } |
0a35513e AH |
154 | |
155 | CREATE_DISPATCH_METHODS(virtual, ) | |
156 | CREATE_DISPATCH_METHODS_MEM() | |
157 | ||
b679c813 TR |
158 | serial_dispatch() : abi_dispatch(false, true, false, true, |
159 | gtm_thread::STATE_SERIAL, &o_serial_mg) | |
160 | { } | |
0a35513e AH |
161 | }; |
162 | ||
163 | ||
164 | // Like serialirr_dispatch but does not requests serial-irrevocable mode until | |
165 | // the first write in the transaction. Can be useful for read-mostly workloads | |
166 | // and testing, but is likely too simple to be of general purpose. | |
167 | class serialirr_onwrite_dispatch : public serialirr_dispatch | |
168 | { | |
169 | public: | |
170 | serialirr_onwrite_dispatch() : | |
b679c813 | 171 | serialirr_dispatch(false, true, false, false, 0, &o_serial_mg) { } |
0a35513e AH |
172 | |
173 | protected: | |
174 | static void pre_write() | |
175 | { | |
176 | gtm_thread *tx = gtm_thr(); | |
177 | if (!(tx->state & (gtm_thread::STATE_SERIAL | |
178 | | gtm_thread::STATE_IRREVOCABLE))) | |
179 | tx->serialirr_mode(); | |
180 | } | |
181 | ||
182 | // Transactional loads access memory directly. | |
183 | // Transactional stores switch to serial mode first. | |
184 | template <typename V> static void store(V* addr, const V value, | |
185 | ls_modifier mod) | |
186 | { | |
187 | pre_write(); | |
188 | serialirr_dispatch::store(addr, value, mod); | |
189 | } | |
190 | ||
191 | public: | |
192 | static void memtransfer_static(void *dst, const void* src, size_t size, | |
193 | bool may_overlap, ls_modifier dst_mod, ls_modifier src_mod) | |
194 | { | |
195 | pre_write(); | |
196 | serialirr_dispatch::memtransfer_static(dst, src, size, may_overlap, | |
197 | dst_mod, src_mod); | |
198 | } | |
199 | ||
200 | static void memset_static(void *dst, int c, size_t size, ls_modifier mod) | |
201 | { | |
202 | pre_write(); | |
203 | serialirr_dispatch::memset_static(dst, c, size, mod); | |
204 | } | |
205 | ||
206 | CREATE_DISPATCH_METHODS(virtual, ) | |
207 | CREATE_DISPATCH_METHODS_MEM() | |
208 | ||
209 | virtual void rollback(gtm_transaction_cp *cp) | |
210 | { | |
211 | gtm_thread *tx = gtm_thr(); | |
212 | if (tx->state & gtm_thread::STATE_IRREVOCABLE) | |
213 | abort(); | |
214 | } | |
629e4729 TR |
215 | |
216 | virtual bool snapshot_most_recent() { return true; } | |
0a35513e AH |
217 | }; |
218 | ||
64fbcc74 TR |
219 | // This group is pure HTM with serial mode as a fallback. There is no |
220 | // difference to serial_mg except that we need to enable or disable the HTM | |
221 | // fastpath. See gtm_thread::begin_transaction. | |
222 | struct htm_mg : public method_group | |
223 | { | |
224 | virtual void init() | |
225 | { | |
226 | // Enable the HTM fastpath if the HW is available. The fastpath is | |
227 | // initially disabled. | |
228 | #ifdef USE_HTM_FASTPATH | |
6041f70a | 229 | gtm_thread::serial_lock.set_htm_fastpath(htm_init()); |
64fbcc74 TR |
230 | #endif |
231 | } | |
232 | virtual void fini() | |
233 | { | |
234 | // Disable the HTM fastpath. | |
6041f70a | 235 | gtm_thread::serial_lock.set_htm_fastpath(0); |
64fbcc74 TR |
236 | } |
237 | }; | |
238 | ||
239 | static htm_mg o_htm_mg; | |
240 | ||
241 | // We just need the subclass to associate it with the HTM method group that | |
242 | // sets up the HTM fast path. This will use serial_dispatch as fallback for | |
243 | // transactions that might get canceled; it has a different method group, but | |
244 | // this is harmless for serial dispatchs because they never abort. | |
245 | class htm_dispatch : public serialirr_dispatch | |
246 | { | |
247 | public: | |
248 | htm_dispatch() : serialirr_dispatch(false, true, false, false, | |
249 | gtm_thread::STATE_SERIAL | gtm_thread::STATE_IRREVOCABLE, &o_htm_mg) | |
250 | { } | |
251 | }; | |
252 | ||
0a35513e AH |
253 | } // anon namespace |
254 | ||
255 | static const serialirr_dispatch o_serialirr_dispatch; | |
256 | static const serial_dispatch o_serial_dispatch; | |
257 | static const serialirr_onwrite_dispatch o_serialirr_onwrite_dispatch; | |
64fbcc74 | 258 | static const htm_dispatch o_htm_dispatch; |
0a35513e AH |
259 | |
260 | abi_dispatch * | |
261 | GTM::dispatch_serialirr () | |
262 | { | |
263 | return const_cast<serialirr_dispatch *>(&o_serialirr_dispatch); | |
264 | } | |
265 | ||
266 | abi_dispatch * | |
267 | GTM::dispatch_serial () | |
268 | { | |
269 | return const_cast<serial_dispatch *>(&o_serial_dispatch); | |
270 | } | |
271 | ||
272 | abi_dispatch * | |
273 | GTM::dispatch_serialirr_onwrite () | |
274 | { | |
275 | return | |
276 | const_cast<serialirr_onwrite_dispatch *>(&o_serialirr_onwrite_dispatch); | |
277 | } | |
278 | ||
64fbcc74 TR |
279 | abi_dispatch * |
280 | GTM::dispatch_htm () | |
281 | { | |
282 | return const_cast<htm_dispatch *>(&o_htm_dispatch); | |
283 | } | |
284 | ||
0a35513e AH |
285 | // Put the transaction into serial-irrevocable mode. |
286 | ||
287 | void | |
288 | GTM::gtm_thread::serialirr_mode () | |
289 | { | |
290 | struct abi_dispatch *disp = abi_disp (); | |
0a35513e | 291 | |
64fbcc74 TR |
292 | #if defined(USE_HTM_FASTPATH) |
293 | // HTM fastpath. If we are executing a HW transaction, don't go serial but | |
294 | // continue. See gtm_thread::begin_transaction. | |
6041f70a | 295 | if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled())) |
64fbcc74 TR |
296 | return; |
297 | #endif | |
298 | ||
0a35513e AH |
299 | if (this->state & STATE_SERIAL) |
300 | { | |
301 | if (this->state & STATE_IRREVOCABLE) | |
302 | return; | |
303 | ||
304 | // Try to commit the dispatch-specific part of the transaction, as we | |
305 | // would do for an outermost commit. | |
306 | // We're already serial, so we don't need to ensure privatization safety | |
307 | // for other transactions here. | |
308 | gtm_word priv_time = 0; | |
1f82422a | 309 | bool ok __attribute__((unused)) = disp->trycommit (priv_time); |
0a35513e AH |
310 | // Given that we're already serial, the trycommit better work. |
311 | assert (ok); | |
0a35513e AH |
312 | } |
313 | else if (serial_lock.write_upgrade (this)) | |
314 | { | |
315 | this->state |= STATE_SERIAL; | |
316 | // Try to commit the dispatch-specific part of the transaction, as we | |
317 | // would do for an outermost commit. | |
318 | // We have successfully upgraded to serial mode, so we don't need to | |
319 | // ensure privatization safety for other transactions here. | |
610e3901 TR |
320 | // However, we are still a reader (wrt. privatization safety) until we |
321 | // have either committed or restarted, so finish the upgrade after that. | |
0a35513e | 322 | gtm_word priv_time = 0; |
610e3901 TR |
323 | if (!disp->trycommit (priv_time)) |
324 | restart (RESTART_SERIAL_IRR, true); | |
325 | gtm_thread::serial_lock.write_upgrade_finish(this); | |
0a35513e | 326 | } |
0a35513e | 327 | else |
610e3901 TR |
328 | restart (RESTART_SERIAL_IRR, false); |
329 | ||
330 | this->state |= (STATE_SERIAL | STATE_IRREVOCABLE); | |
331 | set_abi_disp (dispatch_serialirr ()); | |
0a35513e AH |
332 | } |
333 | ||
334 | void ITM_REGPARM | |
335 | _ITM_changeTransactionMode (_ITM_transactionState state) | |
336 | { | |
337 | assert (state == modeSerialIrrevocable); | |
338 | gtm_thr()->serialirr_mode (); | |
339 | } |