]>
Commit | Line | Data |
---|---|---|
a945c346 | 1 | /* Copyright (C) 2008-2024 Free Software Foundation, Inc. |
0a35513e AH |
2 | Contributed by Richard Henderson <rth@redhat.com>. |
3 | ||
4 | This file is part of the GNU Transactional Memory Library (libitm). | |
5 | ||
6 | Libitm is free software; you can redistribute it and/or modify it | |
7 | under the terms of the GNU General Public License as published by | |
8 | the Free Software Foundation; either version 3 of the License, or | |
9 | (at your option) any later version. | |
10 | ||
11 | Libitm is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
13 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | more details. | |
15 | ||
16 | Under Section 7 of GPL version 3, you are granted additional | |
17 | permissions described in the GCC Runtime Library Exception, version | |
18 | 3.1, as published by the Free Software Foundation. | |
19 | ||
20 | You should have received a copy of the GNU General Public License and | |
21 | a copy of the GCC Runtime Library Exception along with this program; | |
22 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
23 | <http://www.gnu.org/licenses/>. */ | |
24 | ||
4bdd090f RH |
25 | |
26 | #include "asmcfi.h" | |
bec9ec3f | 27 | #include "config.h" |
c51081fc | 28 | #include "cet.h" |
4bdd090f | 29 | |
0b41ebef IS |
30 | #define CONCAT1(a, b) CONCAT2(a, b) |
31 | #define CONCAT2(a, b) a ## b | |
32 | ||
33 | #ifdef __USER_LABEL_PREFIX__ | |
34 | # define SYM(x) CONCAT1 (__USER_LABEL_PREFIX__, x) | |
35 | #else | |
36 | # define SYM(x) x | |
37 | #endif | |
38 | ||
39 | #ifdef __ELF__ | |
40 | # define TYPE(x) .type SYM(x), @function | |
41 | # define SIZE(x) .size SYM(x), . - SYM(x) | |
42 | # ifdef HAVE_ATTRIBUTE_VISIBILITY | |
43 | # define HIDDEN(x) .hidden SYM(x) | |
44 | # else | |
45 | # define HIDDEN(x) | |
46 | # endif | |
47 | #else | |
48 | # define TYPE(x) | |
49 | # define SIZE(x) | |
50 | # ifdef __MACH__ | |
51 | # define HIDDEN(x) .private_extern SYM(x) | |
52 | # else | |
53 | # define HIDDEN(x) | |
54 | # endif | |
55 | #endif | |
56 | ||
bec9ec3f TR |
57 | /* These are duplicates of the canonical definitions in libitm.h. Note that |
58 | the code relies on pr_uninstrumentedCode == a_runUninstrumentedCode. */ | |
59 | #define pr_uninstrumentedCode 0x02 | |
60 | #define pr_hasNoAbort 0x08 | |
61 | #define pr_HTMRetryableAbort 0x800000 | |
62 | #define pr_HTMRetriedAfterAbort 0x1000000 | |
63 | #define a_runInstrumentedCode 0x01 | |
64 | #define a_runUninstrumentedCode 0x02 | |
65 | #define a_tryHTMFastPath 0x20 | |
66 | ||
67 | #define _XABORT_EXPLICIT (1 << 0) | |
68 | #define _XABORT_RETRY (1 << 1) | |
69 | ||
0a35513e | 70 | .text |
9bdc6343 RH |
71 | |
72 | .align 4 | |
0b41ebef | 73 | .globl SYM(_ITM_beginTransaction) |
0a35513e | 74 | |
0b41ebef | 75 | SYM(_ITM_beginTransaction): |
4bdd090f | 76 | cfi_startproc |
c51081fc | 77 | _CET_ENDBR |
0a35513e | 78 | #ifdef __x86_64__ |
bec9ec3f TR |
79 | #ifdef HAVE_AS_RTM |
80 | /* Custom HTM fast path. We start the HW transaction here and let | |
81 | gtm_thread::begin_transaction (aka GTM_begin_transaction) decide | |
82 | how to proceed on aborts: We either retry the fast path, or fall | |
83 | back to another execution method. RTM restores all registers after | |
84 | a HW transaction abort, so we can do the SW setjmp after aborts, | |
85 | and we have to because we might choose a SW fall back. However, | |
6041f70a TR |
86 | we have to explicitly save/restore the first argument (edi). |
87 | The htm_fastpath field is the second int in gtm_rwlock. */ | |
88 | cmpl $0, (SYM(gtm_serial_lock)+4)(%rip) | |
bec9ec3f TR |
89 | jz .Lno_htm |
90 | testl $pr_hasNoAbort, %edi | |
91 | jz .Lno_htm | |
92 | .Lhtm_fastpath: | |
93 | xbegin .Ltxn_abort | |
94 | /* Monitor the serial lock (specifically, the 32b writer/summary field | |
95 | at its start), and only continue if there is no serial-mode | |
96 | transaction. Note that we might be just a nested transaction and | |
97 | our outermost transaction might be in serial mode; we check for | |
98 | this case in the retry policy implementation. */ | |
99 | cmpl $0, SYM(gtm_serial_lock)(%rip) | |
100 | jnz 1f | |
6041f70a TR |
101 | /* Now also check that HW transactions are still allowed to run (see |
102 | gtm_thread::begin_transaction for why this is necessary). */ | |
103 | cmpl $0, (SYM(gtm_serial_lock)+4)(%rip) | |
104 | jz 1f | |
bec9ec3f TR |
105 | /* Everything is good. Run the transaction, preferably using the |
106 | uninstrumented code path. Note that the following works because | |
107 | pr_uninstrumentedCode == a_runUninstrumentedCode. */ | |
108 | andl $pr_uninstrumentedCode, %edi | |
109 | mov $a_runInstrumentedCode, %eax | |
110 | cmovnz %edi, %eax | |
111 | ret | |
6041f70a TR |
112 | /* There is a serial-mode transaction or HW transactions are not |
113 | allowed anymore, so abort (see htm_abort() regarding the abort | |
114 | code). */ | |
bec9ec3f TR |
115 | 1: xabort $0xff |
116 | .Ltxn_abort: | |
117 | /* If it might make sense to retry the HTM fast path, let the C++ | |
118 | code decide. */ | |
119 | testl $(_XABORT_RETRY|_XABORT_EXPLICIT), %eax | |
120 | jz .Lno_htm | |
121 | orl $pr_HTMRetryableAbort, %edi | |
122 | /* Let the C++ code handle the retry policy. */ | |
123 | .Lno_htm: | |
124 | #endif | |
0a35513e | 125 | leaq 8(%rsp), %rax |
bec9ec3f TR |
126 | subq $72, %rsp |
127 | cfi_adjust_cfa_offset(72) | |
128 | /* Store edi for future HTM fast path retries. We use a stack slot | |
129 | lower than the jmpbuf so that the jmpbuf's rip field will overlap | |
130 | with the proper return address on the stack. */ | |
cc03e55b | 131 | movl %edi, (%rsp) |
bec9ec3f TR |
132 | /* Save the jmpbuf for any non-HTM-fastpath execution method. |
133 | Because rsp-based addressing is 1 byte larger and we've got rax | |
134 | handy, use it. */ | |
cc03e55b IT |
135 | movq %rax, -72(%rax) |
136 | movq %rbx, -64(%rax) | |
137 | movq %rbp, -56(%rax) | |
138 | movq %r12, -48(%rax) | |
139 | movq %r13, -40(%rax) | |
140 | movq %r14, -32(%rax) | |
141 | movq %r15, -24(%rax) | |
142 | xorq %rdx, %rdx | |
c51081fc | 143 | /* Save zero or shadow stack pointer in the new field. */ |
ffc2fc06 | 144 | #if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0 |
c51081fc IT |
145 | rdsspq %rdx |
146 | #endif | |
cc03e55b IT |
147 | movq %rdx, -16(%rax) |
148 | leaq -72(%rax), %rsi | |
0b41ebef | 149 | call SYM(GTM_begin_transaction) |
cc03e55b | 150 | movl (%rsp), %edi |
bec9ec3f TR |
151 | addq $72, %rsp |
152 | cfi_adjust_cfa_offset(-72) | |
153 | #ifdef HAVE_AS_RTM | |
154 | /* If a_tryHTMFastPath was returned, then we need to retry the | |
155 | fast path. We also restore edi and set pr_HTMRetriedAfterAbort | |
156 | to state that we have retried the fast path already (it's harmless | |
157 | if this bit is set even if we don't retry the fast path because it | |
158 | is checked iff pr_HTMRetryableAbort is set). We clear | |
159 | pr_HTMRetryableAbort because it applies to a previous HW | |
160 | transaction attempt. */ | |
161 | cmpl $a_tryHTMFastPath, %eax | |
162 | jnz 2f | |
163 | andl $(0xffffffff-pr_HTMRetryableAbort), %edi | |
164 | orl $pr_HTMRetriedAfterAbort, %edi | |
165 | jmp .Lhtm_fastpath | |
166 | 2: | |
167 | #endif | |
0a35513e AH |
168 | #else |
169 | leal 4(%esp), %ecx | |
5752c591 | 170 | movl 4(%esp), %eax |
0a35513e | 171 | subl $28, %esp |
4bdd090f | 172 | cfi_def_cfa_offset(32) |
cc03e55b IT |
173 | movl %ecx, 4(%esp) |
174 | movl %ebx, 8(%esp) | |
175 | movl %esi, 12(%esp) | |
176 | movl %edi, 16(%esp) | |
177 | movl %ebp, 20(%esp) | |
178 | xorl %edx, %edx | |
c51081fc | 179 | /* Save zero or shadow stack pointer in the new field. */ |
ffc2fc06 | 180 | #if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0 |
c51081fc IT |
181 | rdsspd %edx |
182 | #endif | |
cc03e55b IT |
183 | movl %edx, 24(%esp) |
184 | leal 4(%esp), %edx | |
59659b59 | 185 | #if defined HAVE_ATTRIBUTE_VISIBILITY || !defined __PIC__ |
0b41ebef | 186 | call SYM(GTM_begin_transaction) |
59659b59 RO |
187 | #elif defined __ELF__ |
188 | call 1f | |
189 | 1: popl %ebx | |
190 | addl $_GLOBAL_OFFSET_TABLE_+[.-1b], %ebx | |
0b41ebef | 191 | call SYM(GTM_begin_transaction)@PLT |
cc03e55b | 192 | movl 8(%esp), %ebx |
59659b59 RO |
193 | #else |
194 | # error "Unsupported PIC sequence" | |
195 | #endif | |
0a35513e | 196 | addl $28, %esp |
4bdd090f | 197 | cfi_def_cfa_offset(4) |
0a35513e | 198 | #endif |
0c609a21 | 199 | ret |
4bdd090f | 200 | cfi_endproc |
9bdc6343 | 201 | |
0b41ebef IS |
202 | TYPE(_ITM_beginTransaction) |
203 | SIZE(_ITM_beginTransaction) | |
0a35513e | 204 | |
9bdc6343 | 205 | .align 4 |
0b41ebef | 206 | .globl SYM(GTM_longjmp) |
0a35513e | 207 | |
0b41ebef | 208 | SYM(GTM_longjmp): |
4bdd090f | 209 | cfi_startproc |
c51081fc | 210 | _CET_ENDBR |
0a35513e | 211 | #ifdef __x86_64__ |
062f93f2 | 212 | movq (%rsi), %rcx |
9848887a UB |
213 | movq 8(%rsi), %rbx |
214 | movq 16(%rsi), %rbp | |
215 | movq 24(%rsi), %r12 | |
216 | movq 32(%rsi), %r13 | |
217 | movq 40(%rsi), %r14 | |
218 | movq 48(%rsi), %r15 | |
1f319dba | 219 | movl %edi, %eax |
72824d5e | 220 | cfi_def_cfa(%rsi, 0) |
cc03e55b | 221 | cfi_offset(%rip, 64) |
1f319dba | 222 | cfi_register(%rsp, %rcx) |
72824d5e | 223 | movq %rcx, %rsp |
ffc2fc06 | 224 | #if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0 |
c51081fc IT |
225 | /* Check if Shadow Stack is enabled. */ |
226 | xorq %rcx, %rcx | |
227 | rdsspq %rcx | |
228 | testq %rcx, %rcx | |
229 | je .L1 | |
230 | /* Calculate number of frames to skip. */ | |
231 | subq 56(%rsi), %rcx | |
232 | negq %rcx | |
233 | shrq $3, %rcx | |
234 | incq %rcx | |
235 | /* If # of frames is greater 255 then loop | |
236 | and adjust. */ | |
237 | cmpq $255, %rcx | |
0ef9f21d | 238 | jbe .L3 |
c51081fc IT |
239 | movl $255, %edi |
240 | .p2align 4,,10 | |
241 | .p2align 3 | |
242 | .L4: | |
243 | incsspq %rdi | |
244 | subq $255, %rcx | |
245 | cmpq $255, %rcx | |
0ef9f21d | 246 | ja .L4 |
c51081fc IT |
247 | .L3: |
248 | incsspq %rcx | |
249 | .L1: | |
250 | #endif | |
cc03e55b | 251 | jmp *64(%rsi) |
0a35513e | 252 | #else |
0a35513e AH |
253 | movl (%edx), %ecx |
254 | movl 4(%edx), %ebx | |
255 | movl 8(%edx), %esi | |
256 | movl 12(%edx), %edi | |
257 | movl 16(%edx), %ebp | |
72824d5e | 258 | cfi_def_cfa(%edx, 0) |
cc03e55b | 259 | cfi_offset(%eip, 24) |
1f319dba | 260 | cfi_register(%esp, %ecx) |
72824d5e | 261 | movl %ecx, %esp |
ffc2fc06 | 262 | #if defined __SHSTK__ && defined __CET__ && (__CET__ & 2) != 0 |
c51081fc IT |
263 | /* Check if Shadow Stack is enabled. */ |
264 | xorl %ecx, %ecx | |
265 | rdsspd %ecx | |
266 | testl %ecx, %ecx | |
267 | je .L1 | |
268 | /* Calculate # of frames to skip. */ | |
269 | subl 20(%edx), %ecx | |
270 | negl %ecx | |
271 | shrl $2, %ecx | |
272 | incl %ecx | |
273 | /* If # of frames is greater 255 then loop | |
274 | and adjust. */ | |
275 | cmpl $255, %ecx | |
0ef9f21d | 276 | jbe .L3 |
c51081fc IT |
277 | pushl %eax |
278 | movl $255, %eax | |
279 | .p2align 4,,10 | |
280 | .p2align 3 | |
281 | .L4: | |
282 | incsspd %eax | |
283 | subl $255, %ecx | |
284 | cmpl $255, %ecx | |
0ef9f21d | 285 | ja .L4 |
c51081fc IT |
286 | popl %eax |
287 | .L3: | |
288 | incsspd %ecx | |
289 | .L1: | |
290 | #endif | |
cc03e55b | 291 | jmp *24(%edx) |
0a35513e | 292 | #endif |
4bdd090f | 293 | cfi_endproc |
9bdc6343 | 294 | |
0b41ebef IS |
295 | TYPE(GTM_longjmp) |
296 | HIDDEN(GTM_longjmp) | |
297 | SIZE(GTM_longjmp) | |
0a35513e | 298 | |
9bdc6343 | 299 | #ifdef __linux__ |
0a35513e | 300 | .section .note.GNU-stack, "", @progbits |
9bdc6343 | 301 | #endif |