]>
Commit | Line | Data |
---|---|---|
688903eb | 1 | /* Copyright (C) 2002-2018 Free Software Foundation, Inc. |
76a50749 UD |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
76a50749 UD |
18 | |
19 | #include <sysdep.h> | |
326132db | 20 | #include <pthread-errnos.h> |
e51deae7 UD |
21 | #include <kernel-features.h> |
22 | #include <lowlevellock.h> | |
76a50749 | 23 | |
17557282 RH |
24 | #include <stap-probe.h> |
25 | ||
76a50749 UD |
26 | .text |
27 | ||
e51deae7 UD |
28 | #ifdef __ASSUME_PRIVATE_FUTEX |
29 | # define LOAD_PRIVATE_FUTEX_WAIT(reg) \ | |
30 | movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg | |
31 | # define LOAD_PRIVATE_FUTEX_WAKE(reg) \ | |
32 | movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg | |
33 | # define LOAD_FUTEX_WAIT(reg) \ | |
34 | xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg | |
45f77606 UD |
35 | # define LOAD_FUTEX_WAIT_ABS(reg) \ |
36 | xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg | |
e51deae7 UD |
37 | # define LOAD_FUTEX_WAKE(reg) \ |
38 | xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg | |
39 | #else | |
40 | # if FUTEX_WAIT == 0 | |
41 | # define LOAD_PRIVATE_FUTEX_WAIT(reg) \ | |
42 | movl %gs:PRIVATE_FUTEX, reg | |
9356d063 | 43 | # else |
e51deae7 UD |
44 | # define LOAD_PRIVATE_FUTEX_WAIT(reg) \ |
45 | movl %gs:PRIVATE_FUTEX, reg ; \ | |
46 | orl $FUTEX_WAIT, reg | |
9356d063 | 47 | # endif |
e51deae7 UD |
48 | # define LOAD_PRIVATE_FUTEX_WAKE(reg) \ |
49 | movl %gs:PRIVATE_FUTEX, reg ; \ | |
50 | orl $FUTEX_WAKE, reg | |
5a8075b1 UD |
51 | # if FUTEX_WAIT == 0 |
52 | # define LOAD_FUTEX_WAIT(reg) \ | |
e51deae7 UD |
53 | xorl $FUTEX_PRIVATE_FLAG, reg ; \ |
54 | andl %gs:PRIVATE_FUTEX, reg | |
5a8075b1 UD |
55 | # else |
56 | # define LOAD_FUTEX_WAIT(reg) \ | |
e51deae7 UD |
57 | xorl $FUTEX_PRIVATE_FLAG, reg ; \ |
58 | andl %gs:PRIVATE_FUTEX, reg ; \ | |
59 | orl $FUTEX_WAIT, reg | |
5a8075b1 | 60 | # endif |
45f77606 UD |
61 | # define LOAD_FUTEX_WAIT_ABS(reg) \ |
62 | xorl $FUTEX_PRIVATE_FLAG, reg ; \ | |
63 | andl %gs:PRIVATE_FUTEX, reg ; \ | |
1828530f | 64 | orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg |
5a8075b1 | 65 | # define LOAD_FUTEX_WAKE(reg) \ |
e51deae7 UD |
66 | xorl $FUTEX_PRIVATE_FLAG, reg ; \ |
67 | andl %gs:PRIVATE_FUTEX, reg ; \ | |
68 | orl $FUTEX_WAKE, reg | |
5a8075b1 | 69 | #endif |
76a50749 | 70 | |
e51deae7 UD |
71 | .globl __lll_lock_wait_private |
72 | .type __lll_lock_wait_private,@function | |
73 | .hidden __lll_lock_wait_private | |
76a50749 | 74 | .align 16 |
e51deae7 | 75 | __lll_lock_wait_private: |
cd248c3f | 76 | cfi_startproc |
76a50749 | 77 | pushl %edx |
cd248c3f | 78 | cfi_adjust_cfa_offset(4) |
c0df57e1 | 79 | pushl %ebx |
cd248c3f | 80 | cfi_adjust_cfa_offset(4) |
c0df57e1 | 81 | pushl %esi |
cd248c3f UD |
82 | cfi_adjust_cfa_offset(4) |
83 | cfi_offset(%edx, -8) | |
84 | cfi_offset(%ebx, -12) | |
85 | cfi_offset(%esi, -16) | |
76a50749 | 86 | |
c0df57e1 | 87 | movl $2, %edx |
76a50749 UD |
88 | movl %ecx, %ebx |
89 | xorl %esi, %esi /* No timeout. */ | |
e51deae7 | 90 | LOAD_PRIVATE_FUTEX_WAIT (%ecx) |
71451de2 | 91 | |
a8fd5a02 | 92 | cmpl %edx, %eax /* NB: %edx == 2 */ |
ebddb424 | 93 | jne 2f |
71451de2 | 94 | |
17557282 RH |
95 | 1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx) |
96 | movl $SYS_futex, %eax | |
097eca29 | 97 | ENTER_KERNEL |
76a50749 | 98 | |
ebddb424 | 99 | 2: movl %edx, %eax |
a8fd5a02 | 100 | xchgl %eax, (%ebx) /* NB: lock is implied */ |
76a50749 | 101 | |
a8fd5a02 | 102 | testl %eax, %eax |
ab9a9ff8 | 103 | jnz 1b |
76a50749 | 104 | |
ebddb424 | 105 | popl %esi |
cd248c3f UD |
106 | cfi_adjust_cfa_offset(-4) |
107 | cfi_restore(%esi) | |
c0df57e1 | 108 | popl %ebx |
cd248c3f UD |
109 | cfi_adjust_cfa_offset(-4) |
110 | cfi_restore(%ebx) | |
c0df57e1 | 111 | popl %edx |
cd248c3f UD |
112 | cfi_adjust_cfa_offset(-4) |
113 | cfi_restore(%edx) | |
76a50749 | 114 | ret |
cd248c3f | 115 | cfi_endproc |
e51deae7 | 116 | .size __lll_lock_wait_private,.-__lll_lock_wait_private |
71451de2 | 117 | |
4f41c682 | 118 | #if !IS_IN (libc) |
e51deae7 UD |
119 | .globl __lll_lock_wait |
120 | .type __lll_lock_wait,@function | |
121 | .hidden __lll_lock_wait | |
71451de2 | 122 | .align 16 |
e51deae7 UD |
123 | __lll_lock_wait: |
124 | cfi_startproc | |
125 | pushl %edx | |
126 | cfi_adjust_cfa_offset(4) | |
127 | pushl %ebx | |
128 | cfi_adjust_cfa_offset(4) | |
129 | pushl %esi | |
130 | cfi_adjust_cfa_offset(4) | |
131 | cfi_offset(%edx, -8) | |
132 | cfi_offset(%ebx, -12) | |
133 | cfi_offset(%esi, -16) | |
134 | ||
135 | movl %edx, %ebx | |
136 | movl $2, %edx | |
137 | xorl %esi, %esi /* No timeout. */ | |
138 | LOAD_FUTEX_WAIT (%ecx) | |
139 | ||
140 | cmpl %edx, %eax /* NB: %edx == 2 */ | |
141 | jne 2f | |
142 | ||
143 | 1: movl $SYS_futex, %eax | |
144 | ENTER_KERNEL | |
145 | ||
146 | 2: movl %edx, %eax | |
147 | xchgl %eax, (%ebx) /* NB: lock is implied */ | |
148 | ||
149 | testl %eax, %eax | |
150 | jnz 1b | |
151 | ||
152 | popl %esi | |
153 | cfi_adjust_cfa_offset(-4) | |
154 | cfi_restore(%esi) | |
155 | popl %ebx | |
156 | cfi_adjust_cfa_offset(-4) | |
157 | cfi_restore(%ebx) | |
158 | popl %edx | |
159 | cfi_adjust_cfa_offset(-4) | |
160 | cfi_restore(%edx) | |
161 | ret | |
162 | cfi_endproc | |
163 | .size __lll_lock_wait,.-__lll_lock_wait | |
164 | ||
45f77606 UD |
165 | /* %ecx: futex |
166 | %esi: flags | |
167 | %edx: timeout | |
168 | %eax: futex value | |
169 | */ | |
e51deae7 UD |
170 | .globl __lll_timedlock_wait |
171 | .type __lll_timedlock_wait,@function | |
172 | .hidden __lll_timedlock_wait | |
173 | .align 16 | |
174 | __lll_timedlock_wait: | |
cd248c3f | 175 | cfi_startproc |
45f77606 UD |
176 | pushl %ebp |
177 | cfi_adjust_cfa_offset(4) | |
178 | cfi_rel_offset(%ebp, 0) | |
179 | pushl %ebx | |
180 | cfi_adjust_cfa_offset(4) | |
181 | cfi_rel_offset(%ebx, 0) | |
182 | ||
183 | # ifndef __ASSUME_FUTEX_CLOCK_REALTIME | |
184 | # ifdef PIC | |
185 | LOAD_PIC_REG (bx) | |
186 | cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx) | |
187 | # else | |
188 | cmpl $0, __have_futex_clock_realtime | |
189 | # endif | |
190 | je .Lreltmo | |
191 | # endif | |
192 | ||
2983d85e AS |
193 | cmpl $0, (%edx) |
194 | js 8f | |
195 | ||
45f77606 UD |
196 | movl %ecx, %ebx |
197 | movl %esi, %ecx | |
198 | movl %edx, %esi | |
199 | movl $0xffffffff, %ebp | |
200 | LOAD_FUTEX_WAIT_ABS (%ecx) | |
201 | ||
202 | movl $2, %edx | |
203 | cmpl %edx, %eax | |
204 | jne 2f | |
205 | ||
206 | 1: movl $SYS_futex, %eax | |
207 | movl $2, %edx | |
208 | ENTER_KERNEL | |
209 | ||
210 | 2: xchgl %edx, (%ebx) /* NB: lock is implied */ | |
211 | ||
212 | testl %edx, %edx | |
213 | jz 3f | |
214 | ||
215 | cmpl $-ETIMEDOUT, %eax | |
216 | je 4f | |
217 | cmpl $-EINVAL, %eax | |
218 | jne 1b | |
219 | 4: movl %eax, %edx | |
220 | negl %edx | |
221 | ||
222 | 3: movl %edx, %eax | |
223 | 7: popl %ebx | |
224 | cfi_adjust_cfa_offset(-4) | |
225 | cfi_restore(%ebx) | |
226 | popl %ebp | |
227 | cfi_adjust_cfa_offset(-4) | |
228 | cfi_restore(%ebp) | |
229 | ret | |
230 | ||
2983d85e AS |
231 | 8: movl $ETIMEDOUT, %eax |
232 | jmp 7b | |
233 | ||
45f77606 UD |
234 | # ifndef __ASSUME_FUTEX_CLOCK_REALTIME |
235 | .Lreltmo: | |
71451de2 UD |
236 | /* Check for a valid timeout value. */ |
237 | cmpl $1000000000, 4(%edx) | |
238 | jae 3f | |
239 | ||
71451de2 | 240 | pushl %esi |
cd248c3f | 241 | cfi_adjust_cfa_offset(4) |
45f77606 UD |
242 | cfi_rel_offset(%esi, 0) |
243 | pushl %edi | |
cd248c3f | 244 | cfi_adjust_cfa_offset(4) |
45f77606 | 245 | cfi_rel_offset(%edi, 0) |
71451de2 UD |
246 | |
247 | /* Stack frame for the timespec and timeval structs. */ | |
248 | subl $8, %esp | |
cd248c3f | 249 | cfi_adjust_cfa_offset(8) |
71451de2 UD |
250 | |
251 | movl %ecx, %ebp | |
252 | movl %edx, %edi | |
253 | ||
c012be6f UD |
254 | movl $2, %edx |
255 | xchgl %edx, (%ebp) | |
256 | ||
257 | test %edx, %edx | |
258 | je 6f | |
259 | ||
71451de2 UD |
260 | 1: |
261 | /* Get current time. */ | |
262 | movl %esp, %ebx | |
263 | xorl %ecx, %ecx | |
e51deae7 | 264 | movl $__NR_gettimeofday, %eax |
71451de2 UD |
265 | ENTER_KERNEL |
266 | ||
267 | /* Compute relative timeout. */ | |
268 | movl 4(%esp), %eax | |
269 | movl $1000, %edx | |
270 | mul %edx /* Milli seconds to nano seconds. */ | |
271 | movl (%edi), %ecx | |
272 | movl 4(%edi), %edx | |
273 | subl (%esp), %ecx | |
274 | subl %eax, %edx | |
275 | jns 4f | |
276 | addl $1000000000, %edx | |
277 | subl $1, %ecx | |
278 | 4: testl %ecx, %ecx | |
c012be6f | 279 | js 2f /* Time is already up. */ |
71451de2 UD |
280 | |
281 | /* Store relative timeout. */ | |
282 | movl %ecx, (%esp) | |
283 | movl %edx, 4(%esp) | |
284 | ||
c012be6f | 285 | /* Futex call. */ |
71451de2 | 286 | movl %ebp, %ebx |
71451de2 | 287 | movl $2, %edx |
71451de2 | 288 | movl %esp, %esi |
e51deae7 | 289 | movl 16(%esp), %ecx |
5a8075b1 | 290 | LOAD_FUTEX_WAIT (%ecx) |
71451de2 UD |
291 | movl $SYS_futex, %eax |
292 | ENTER_KERNEL | |
71451de2 | 293 | |
c012be6f UD |
294 | /* NB: %edx == 2 */ |
295 | xchgl %edx, (%ebp) | |
71451de2 | 296 | |
c012be6f UD |
297 | testl %edx, %edx |
298 | je 6f | |
299 | ||
300 | cmpl $-ETIMEDOUT, %eax | |
301 | jne 1b | |
302 | 2: movl $ETIMEDOUT, %edx | |
71451de2 UD |
303 | |
304 | 6: addl $8, %esp | |
cd248c3f | 305 | cfi_adjust_cfa_offset(-8) |
45f77606 | 306 | popl %edi |
cd248c3f | 307 | cfi_adjust_cfa_offset(-4) |
45f77606 | 308 | cfi_restore(%edi) |
71451de2 | 309 | popl %esi |
cd248c3f UD |
310 | cfi_adjust_cfa_offset(-4) |
311 | cfi_restore(%esi) | |
45f77606 | 312 | 7: popl %ebx |
cd248c3f | 313 | cfi_adjust_cfa_offset(-4) |
45f77606 UD |
314 | cfi_restore(%ebx) |
315 | popl %ebp | |
316 | cfi_adjust_cfa_offset(-4) | |
317 | cfi_restore(%ebp) | |
c012be6f | 318 | movl %edx, %eax |
71451de2 UD |
319 | ret |
320 | ||
45f77606 UD |
321 | 3: movl $EINVAL, %edx |
322 | jmp 7b | |
323 | # endif | |
cd248c3f | 324 | cfi_endproc |
e51deae7 | 325 | .size __lll_timedlock_wait,.-__lll_timedlock_wait |
71451de2 | 326 | #endif |
76a50749 | 327 | |
e51deae7 UD |
328 | .globl __lll_unlock_wake_private |
329 | .type __lll_unlock_wake_private,@function | |
330 | .hidden __lll_unlock_wake_private | |
331 | .align 16 | |
332 | __lll_unlock_wake_private: | |
333 | cfi_startproc | |
334 | pushl %ebx | |
335 | cfi_adjust_cfa_offset(4) | |
336 | pushl %ecx | |
337 | cfi_adjust_cfa_offset(4) | |
338 | pushl %edx | |
339 | cfi_adjust_cfa_offset(4) | |
340 | cfi_offset(%ebx, -8) | |
341 | cfi_offset(%ecx, -12) | |
342 | cfi_offset(%edx, -16) | |
343 | ||
344 | movl %eax, %ebx | |
345 | movl $0, (%eax) | |
346 | LOAD_PRIVATE_FUTEX_WAKE (%ecx) | |
347 | movl $1, %edx /* Wake one thread. */ | |
348 | movl $SYS_futex, %eax | |
349 | ENTER_KERNEL | |
350 | ||
351 | popl %edx | |
352 | cfi_adjust_cfa_offset(-4) | |
353 | cfi_restore(%edx) | |
354 | popl %ecx | |
355 | cfi_adjust_cfa_offset(-4) | |
356 | cfi_restore(%ecx) | |
357 | popl %ebx | |
358 | cfi_adjust_cfa_offset(-4) | |
359 | cfi_restore(%ebx) | |
360 | ret | |
361 | cfi_endproc | |
362 | .size __lll_unlock_wake_private,.-__lll_unlock_wake_private | |
76a50749 | 363 | |
4f41c682 | 364 | #if !IS_IN (libc) |
e51deae7 UD |
365 | .globl __lll_unlock_wake |
366 | .type __lll_unlock_wake,@function | |
367 | .hidden __lll_unlock_wake | |
bd8bb78b | 368 | .align 16 |
e51deae7 | 369 | __lll_unlock_wake: |
cd248c3f | 370 | cfi_startproc |
76a50749 | 371 | pushl %ebx |
cd248c3f | 372 | cfi_adjust_cfa_offset(4) |
76a50749 | 373 | pushl %ecx |
cd248c3f | 374 | cfi_adjust_cfa_offset(4) |
76a50749 | 375 | pushl %edx |
cd248c3f UD |
376 | cfi_adjust_cfa_offset(4) |
377 | cfi_offset(%ebx, -8) | |
378 | cfi_offset(%ecx, -12) | |
379 | cfi_offset(%edx, -16) | |
76a50749 UD |
380 | |
381 | movl %eax, %ebx | |
71451de2 | 382 | movl $0, (%eax) |
5a8075b1 | 383 | LOAD_FUTEX_WAKE (%ecx) |
76a50749 | 384 | movl $1, %edx /* Wake one thread. */ |
76a50749 | 385 | movl $SYS_futex, %eax |
097eca29 | 386 | ENTER_KERNEL |
76a50749 UD |
387 | |
388 | popl %edx | |
cd248c3f UD |
389 | cfi_adjust_cfa_offset(-4) |
390 | cfi_restore(%edx) | |
76a50749 | 391 | popl %ecx |
cd248c3f UD |
392 | cfi_adjust_cfa_offset(-4) |
393 | cfi_restore(%ecx) | |
76a50749 | 394 | popl %ebx |
cd248c3f UD |
395 | cfi_adjust_cfa_offset(-4) |
396 | cfi_restore(%ebx) | |
76a50749 | 397 | ret |
cd248c3f | 398 | cfi_endproc |
e51deae7 | 399 | .size __lll_unlock_wake,.-__lll_unlock_wake |
76a50749 | 400 | |
76a50749 UD |
401 | .globl __lll_timedwait_tid |
402 | .type __lll_timedwait_tid,@function | |
403 | .hidden __lll_timedwait_tid | |
bd8bb78b | 404 | .align 16 |
76a50749 UD |
405 | __lll_timedwait_tid: |
406 | pushl %edi | |
407 | pushl %esi | |
408 | pushl %ebx | |
409 | pushl %ebp | |
410 | ||
411 | movl %eax, %ebp | |
412 | movl %edx, %edi | |
413 | subl $8, %esp | |
414 | ||
415 | /* Get current time. */ | |
416 | 2: movl %esp, %ebx | |
417 | xorl %ecx, %ecx | |
e51deae7 | 418 | movl $__NR_gettimeofday, %eax |
097eca29 | 419 | ENTER_KERNEL |
76a50749 UD |
420 | |
421 | /* Compute relative timeout. */ | |
422 | movl 4(%esp), %eax | |
423 | movl $1000, %edx | |
424 | mul %edx /* Milli seconds to nano seconds. */ | |
425 | movl (%edi), %ecx | |
426 | movl 4(%edi), %edx | |
427 | subl (%esp), %ecx | |
428 | subl %eax, %edx | |
429 | jns 5f | |
430 | addl $1000000000, %edx | |
ccf1d573 | 431 | subl $1, %ecx |
76a50749 UD |
432 | 5: testl %ecx, %ecx |
433 | js 6f /* Time is already up. */ | |
434 | ||
435 | movl %ecx, (%esp) /* Store relative timeout. */ | |
436 | movl %edx, 4(%esp) | |
437 | ||
438 | movl (%ebp), %edx | |
439 | testl %edx, %edx | |
440 | jz 4f | |
441 | ||
442 | movl %esp, %esi | |
5a8075b1 UD |
443 | /* XXX The kernel so far uses global futex for the wakeup at |
444 | all times. */ | |
76a50749 UD |
445 | xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ |
446 | movl %ebp, %ebx | |
447 | movl $SYS_futex, %eax | |
097eca29 | 448 | ENTER_KERNEL |
76a50749 | 449 | |
76a50749 UD |
450 | cmpl $0, (%ebx) |
451 | jne 1f | |
452 | 4: xorl %eax, %eax | |
453 | ||
454 | 3: addl $8, %esp | |
455 | popl %ebp | |
456 | popl %ebx | |
457 | popl %esi | |
458 | popl %edi | |
459 | ret | |
460 | ||
3273832c | 461 | 1: cmpl $-ETIMEDOUT, %eax |
76a50749 UD |
462 | jne 2b |
463 | 6: movl $ETIMEDOUT, %eax | |
464 | jmp 3b | |
465 | .size __lll_timedwait_tid,.-__lll_timedwait_tid | |
9356d063 | 466 | #endif |