]>
Commit | Line | Data |
---|---|---|
04277e02 | 1 | /* Copyright (C) 2002-2019 Free Software Foundation, Inc. |
76a50749 UD |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
76a50749 | 18 | |
3892d906 | 19 | #include <assert.h> |
76a50749 | 20 | #include <errno.h> |
1bcfb5a5 | 21 | #include <stdlib.h> |
df47504c | 22 | #include <unistd.h> |
ca06321d | 23 | #include <sys/param.h> |
b894c2ea | 24 | #include <not-cancel.h> |
76a50749 | 25 | #include "pthreadP.h" |
4eb984d3 | 26 | #include <atomic.h> |
76a50749 | 27 | #include <lowlevellock.h> |
5acf7263 | 28 | #include <stap-probe.h> |
76a50749 | 29 | |
e8c659d7 AK |
30 | #ifndef lll_lock_elision |
31 | #define lll_lock_elision(lock, try_lock, private) ({ \ | |
32 | lll_lock (lock, private); 0; }) | |
33 | #endif | |
34 | ||
35 | #ifndef lll_trylock_elision | |
36 | #define lll_trylock_elision(a,t) lll_trylock(a) | |
37 | #endif | |
76a50749 | 38 | |
65810f0e TR |
39 | /* Some of the following definitions differ when pthread_mutex_cond_lock.c |
40 | includes this file. */ | |
69431c9a | 41 | #ifndef LLL_MUTEX_LOCK |
5bd8a249 UD |
42 | # define LLL_MUTEX_LOCK(mutex) \ |
43 | lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) | |
44 | # define LLL_MUTEX_TRYLOCK(mutex) \ | |
45 | lll_trylock ((mutex)->__data.__lock) | |
65810f0e | 46 | # define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0 |
e8c659d7 AK |
47 | # define LLL_MUTEX_LOCK_ELISION(mutex) \ |
48 | lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \ | |
49 | PTHREAD_MUTEX_PSHARED (mutex)) | |
50 | # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \ | |
51 | lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \ | |
52 | PTHREAD_MUTEX_PSHARED (mutex)) | |
69431c9a UD |
53 | #endif |
54 | ||
e8c659d7 AK |
55 | #ifndef FORCE_ELISION |
56 | #define FORCE_ELISION(m, s) | |
57 | #endif | |
69431c9a | 58 | |
6de79a49 UD |
59 | static int __pthread_mutex_lock_full (pthread_mutex_t *mutex) |
60 | __attribute_noinline__; | |
61 | ||
76a50749 | 62 | int |
9dd346ff | 63 | __pthread_mutex_lock (pthread_mutex_t *mutex) |
76a50749 | 64 | { |
403b4feb SL |
65 | /* See concurrency notes regarding mutex type which is loaded from __kind |
66 | in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */ | |
e8c659d7 | 67 | unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); |
5acf7263 RM |
68 | |
69 | LIBC_PROBE (mutex_entry, 1, mutex); | |
70 | ||
e8c659d7 AK |
71 | if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP |
72 | | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) | |
6de79a49 UD |
73 | return __pthread_mutex_lock_full (mutex); |
74 | ||
a1ffb40e | 75 | if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP)) |
6de79a49 | 76 | { |
e8c659d7 | 77 | FORCE_ELISION (mutex, goto elision); |
6de79a49 UD |
78 | simple: |
79 | /* Normal mutex. */ | |
80 | LLL_MUTEX_LOCK (mutex); | |
81 | assert (mutex->__data.__owner == 0); | |
82 | } | |
e8c659d7 | 83 | #ifdef HAVE_ELISION |
a1ffb40e | 84 | else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP)) |
e8c659d7 AK |
85 | { |
86 | elision: __attribute__((unused)) | |
87 | /* This case can never happen on a system without elision, | |
88 | as the mutex type initialization functions will not | |
89 | allow to set the elision flags. */ | |
075b9322 | 90 | /* Don't record owner or users for elision case. This is a |
e8c659d7 AK |
91 | tail call. */ |
92 | return LLL_MUTEX_LOCK_ELISION (mutex); | |
93 | } | |
94 | #endif | |
95 | else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) | |
96 | == PTHREAD_MUTEX_RECURSIVE_NP, 1)) | |
76a50749 UD |
97 | { |
98 | /* Recursive mutex. */ | |
e8c659d7 | 99 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); |
6de79a49 | 100 | |
76a50749 | 101 | /* Check whether we already hold the mutex. */ |
9a7178d6 | 102 | if (mutex->__data.__owner == id) |
76a50749 UD |
103 | { |
104 | /* Just bump the counter. */ | |
a1ffb40e | 105 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
76a50749 UD |
106 | /* Overflow of the counter. */ |
107 | return EAGAIN; | |
108 | ||
109 | ++mutex->__data.__count; | |
76a50749 | 110 | |
3892d906 | 111 | return 0; |
76a50749 | 112 | } |
3892d906 UD |
113 | |
114 | /* We have to get the mutex. */ | |
5bd8a249 | 115 | LLL_MUTEX_LOCK (mutex); |
3892d906 | 116 | |
1bcfb5a5 | 117 | assert (mutex->__data.__owner == 0); |
3892d906 | 118 | mutex->__data.__count = 1; |
6de79a49 | 119 | } |
e8c659d7 AK |
120 | else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) |
121 | == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) | |
6de79a49 | 122 | { |
2c0b891a UD |
123 | if (! __is_smp) |
124 | goto simple; | |
125 | ||
5bd8a249 | 126 | if (LLL_MUTEX_TRYLOCK (mutex) != 0) |
2c0b891a UD |
127 | { |
128 | int cnt = 0; | |
6310e6be | 129 | int max_cnt = MIN (max_adaptive_count (), |
2c0b891a UD |
130 | mutex->__data.__spins * 2 + 10); |
131 | do | |
132 | { | |
133 | if (cnt++ >= max_cnt) | |
134 | { | |
5bd8a249 | 135 | LLL_MUTEX_LOCK (mutex); |
2c0b891a UD |
136 | break; |
137 | } | |
4eb984d3 | 138 | atomic_spin_nop (); |
2c0b891a | 139 | } |
5bd8a249 | 140 | while (LLL_MUTEX_TRYLOCK (mutex) != 0); |
2c0b891a UD |
141 | |
142 | mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; | |
143 | } | |
1bcfb5a5 | 144 | assert (mutex->__data.__owner == 0); |
6de79a49 UD |
145 | } |
146 | else | |
147 | { | |
e8c659d7 AK |
148 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); |
149 | assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP); | |
6de79a49 | 150 | /* Check whether we already hold the mutex. */ |
a1ffb40e | 151 | if (__glibc_unlikely (mutex->__data.__owner == id)) |
6de79a49 UD |
152 | return EDEADLK; |
153 | goto simple; | |
154 | } | |
1bcfb5a5 | 155 | |
e8c659d7 AK |
156 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); |
157 | ||
6de79a49 UD |
158 | /* Record the ownership. */ |
159 | mutex->__data.__owner = id; | |
160 | #ifndef NO_INCR | |
161 | ++mutex->__data.__nusers; | |
162 | #endif | |
163 | ||
5acf7263 RM |
164 | LIBC_PROBE (mutex_acquired, 1, mutex); |
165 | ||
6de79a49 UD |
166 | return 0; |
167 | } | |
168 | ||
169 | static int | |
170 | __pthread_mutex_lock_full (pthread_mutex_t *mutex) | |
171 | { | |
172 | int oldval; | |
173 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); | |
174 | ||
175 | switch (PTHREAD_MUTEX_TYPE (mutex)) | |
176 | { | |
0f6699ea UD |
177 | case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP: |
178 | case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP: | |
179 | case PTHREAD_MUTEX_ROBUST_NORMAL_NP: | |
180 | case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP: | |
181 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, | |
182 | &mutex->__data.__list.__next); | |
8f9450a0 TR |
183 | /* We need to set op_pending before starting the operation. Also |
184 | see comments at ENQUEUE_MUTEX. */ | |
185 | __asm ("" ::: "memory"); | |
0f6699ea | 186 | |
683040c3 | 187 | oldval = mutex->__data.__lock; |
353683a2 TR |
188 | /* This is set to FUTEX_WAITERS iff we might have shared the |
189 | FUTEX_WAITERS flag with other threads, and therefore need to keep it | |
190 | set to avoid lost wake-ups. We have the same requirement in the | |
65810f0e TR |
191 | simple mutex algorithm. |
192 | We start with value zero for a normal mutex, and FUTEX_WAITERS if we | |
193 | are building the special case mutexes for use from within condition | |
194 | variables. */ | |
195 | unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER; | |
196 | while (1) | |
1bcfb5a5 | 197 | { |
65810f0e TR |
198 | /* Try to acquire the lock through a CAS from 0 (not acquired) to |
199 | our TID | assume_other_futex_waiters. */ | |
5920a4a6 CD |
200 | if (__glibc_likely (oldval == 0)) |
201 | { | |
202 | oldval | |
203 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, | |
204 | id | assume_other_futex_waiters, 0); | |
205 | if (__glibc_likely (oldval == 0)) | |
206 | break; | |
207 | } | |
65810f0e | 208 | |
683040c3 UD |
209 | if ((oldval & FUTEX_OWNER_DIED) != 0) |
210 | { | |
211 | /* The previous owner died. Try locking the mutex. */ | |
0f6699ea UD |
212 | int newval = id; |
213 | #ifdef NO_INCR | |
353683a2 TR |
214 | /* We are not taking assume_other_futex_waiters into accoount |
215 | here simply because we'll set FUTEX_WAITERS anyway. */ | |
0f6699ea | 216 | newval |= FUTEX_WAITERS; |
113ad5fc | 217 | #else |
353683a2 | 218 | newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters; |
0f6699ea UD |
219 | #endif |
220 | ||
221 | newval | |
052757bf | 222 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
0f6699ea UD |
223 | newval, oldval); |
224 | ||
225 | if (newval != oldval) | |
683040c3 | 226 | { |
683040c3 | 227 | oldval = newval; |
65810f0e | 228 | continue; |
683040c3 | 229 | } |
1bcfb5a5 | 230 | |
683040c3 UD |
231 | /* We got the mutex. */ |
232 | mutex->__data.__count = 1; | |
233 | /* But it is inconsistent unless marked otherwise. */ | |
234 | mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; | |
235 | ||
8f9450a0 TR |
236 | /* We must not enqueue the mutex before we have acquired it. |
237 | Also see comments at ENQUEUE_MUTEX. */ | |
238 | __asm ("" ::: "memory"); | |
683040c3 | 239 | ENQUEUE_MUTEX (mutex); |
8f9450a0 TR |
240 | /* We need to clear op_pending after we enqueue the mutex. */ |
241 | __asm ("" ::: "memory"); | |
0f6699ea | 242 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
683040c3 UD |
243 | |
244 | /* Note that we deliberately exit here. If we fall | |
245 | through to the end of the function __nusers would be | |
246 | incremented which is not correct because the old | |
247 | owner has to be discounted. If we are not supposed | |
248 | to increment __nusers we actually have to decrement | |
249 | it here. */ | |
250 | #ifdef NO_INCR | |
251 | --mutex->__data.__nusers; | |
252 | #endif | |
1bcfb5a5 | 253 | |
683040c3 UD |
254 | return EOWNERDEAD; |
255 | } | |
1bcfb5a5 | 256 | |
683040c3 | 257 | /* Check whether we already hold the mutex. */ |
a1ffb40e | 258 | if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id)) |
683040c3 | 259 | { |
5bd8a249 UD |
260 | int kind = PTHREAD_MUTEX_TYPE (mutex); |
261 | if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) | |
0f6699ea | 262 | { |
8f9450a0 TR |
263 | /* We do not need to ensure ordering wrt another memory |
264 | access. Also see comments at ENQUEUE_MUTEX. */ | |
0f6699ea UD |
265 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
266 | NULL); | |
267 | return EDEADLK; | |
268 | } | |
1bcfb5a5 | 269 | |
5bd8a249 | 270 | if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) |
683040c3 | 271 | { |
8f9450a0 TR |
272 | /* We do not need to ensure ordering wrt another memory |
273 | access. */ | |
0f6699ea UD |
274 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
275 | NULL); | |
276 | ||
683040c3 | 277 | /* Just bump the counter. */ |
a1ffb40e | 278 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
683040c3 UD |
279 | /* Overflow of the counter. */ |
280 | return EAGAIN; | |
1bcfb5a5 | 281 | |
683040c3 | 282 | ++mutex->__data.__count; |
1bcfb5a5 | 283 | |
683040c3 UD |
284 | return 0; |
285 | } | |
286 | } | |
1bcfb5a5 | 287 | |
65810f0e TR |
288 | /* We cannot acquire the mutex nor has its owner died. Thus, try |
289 | to block using futexes. Set FUTEX_WAITERS if necessary so that | |
290 | other threads are aware that there are potentially threads | |
291 | blocked on the futex. Restart if oldval changed in the | |
292 | meantime. */ | |
293 | if ((oldval & FUTEX_WAITERS) == 0) | |
683040c3 | 294 | { |
65810f0e TR |
295 | if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock, |
296 | oldval | FUTEX_WAITERS, | |
297 | oldval) | |
298 | != 0) | |
299 | { | |
300 | oldval = mutex->__data.__lock; | |
301 | continue; | |
302 | } | |
303 | oldval |= FUTEX_WAITERS; | |
683040c3 | 304 | } |
65810f0e TR |
305 | |
306 | /* It is now possible that we share the FUTEX_WAITERS flag with | |
307 | another thread; therefore, update assume_other_futex_waiters so | |
308 | that we do not forget about this when handling other cases | |
309 | above and thus do not cause lost wake-ups. */ | |
310 | assume_other_futex_waiters |= FUTEX_WAITERS; | |
311 | ||
312 | /* Block using the futex and reload current lock value. */ | |
313 | lll_futex_wait (&mutex->__data.__lock, oldval, | |
314 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); | |
315 | oldval = mutex->__data.__lock; | |
316 | } | |
317 | ||
318 | /* We have acquired the mutex; check if it is still consistent. */ | |
319 | if (__builtin_expect (mutex->__data.__owner | |
320 | == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) | |
321 | { | |
322 | /* This mutex is now not recoverable. */ | |
323 | mutex->__data.__count = 0; | |
324 | int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex); | |
325 | lll_unlock (mutex->__data.__lock, private); | |
8f9450a0 TR |
326 | /* FIXME This violates the mutex destruction requirements. See |
327 | __pthread_mutex_unlock_full. */ | |
65810f0e TR |
328 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
329 | return ENOTRECOVERABLE; | |
1bcfb5a5 UD |
330 | } |
331 | ||
683040c3 | 332 | mutex->__data.__count = 1; |
8f9450a0 TR |
333 | /* We must not enqueue the mutex before we have acquired it. |
334 | Also see comments at ENQUEUE_MUTEX. */ | |
335 | __asm ("" ::: "memory"); | |
1bcfb5a5 | 336 | ENQUEUE_MUTEX (mutex); |
8f9450a0 TR |
337 | /* We need to clear op_pending after we enqueue the mutex. */ |
338 | __asm ("" ::: "memory"); | |
0f6699ea | 339 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
2c0b891a | 340 | break; |
dcc73a8d | 341 | |
184ee940 RM |
342 | /* The PI support requires the Linux futex system call. If that's not |
343 | available, pthread_mutex_init should never have allowed the type to | |
344 | be set. So it will get the default case for an invalid type. */ | |
345 | #ifdef __NR_futex | |
df47504c UD |
346 | case PTHREAD_MUTEX_PI_RECURSIVE_NP: |
347 | case PTHREAD_MUTEX_PI_ERRORCHECK_NP: | |
348 | case PTHREAD_MUTEX_PI_NORMAL_NP: | |
349 | case PTHREAD_MUTEX_PI_ADAPTIVE_NP: | |
350 | case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP: | |
351 | case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP: | |
352 | case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP: | |
353 | case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP: | |
354 | { | |
403b4feb SL |
355 | int kind, robust; |
356 | { | |
357 | /* See concurrency notes regarding __kind in struct __pthread_mutex_s | |
358 | in sysdeps/nptl/bits/thread-shared-types.h. */ | |
359 | int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); | |
360 | kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP; | |
361 | robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP; | |
362 | } | |
df47504c UD |
363 | |
364 | if (robust) | |
8f9450a0 TR |
365 | { |
366 | /* Note: robust PI futexes are signaled by setting bit 0. */ | |
367 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, | |
368 | (void *) (((uintptr_t) &mutex->__data.__list.__next) | |
369 | | 1)); | |
370 | /* We need to set op_pending before starting the operation. Also | |
371 | see comments at ENQUEUE_MUTEX. */ | |
372 | __asm ("" ::: "memory"); | |
373 | } | |
df47504c UD |
374 | |
375 | oldval = mutex->__data.__lock; | |
376 | ||
377 | /* Check whether we already hold the mutex. */ | |
a1ffb40e | 378 | if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id)) |
df47504c UD |
379 | { |
380 | if (kind == PTHREAD_MUTEX_ERRORCHECK_NP) | |
381 | { | |
8f9450a0 TR |
382 | /* We do not need to ensure ordering wrt another memory |
383 | access. */ | |
df47504c UD |
384 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
385 | return EDEADLK; | |
386 | } | |
387 | ||
388 | if (kind == PTHREAD_MUTEX_RECURSIVE_NP) | |
389 | { | |
8f9450a0 TR |
390 | /* We do not need to ensure ordering wrt another memory |
391 | access. */ | |
df47504c UD |
392 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
393 | ||
394 | /* Just bump the counter. */ | |
a1ffb40e | 395 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
df47504c UD |
396 | /* Overflow of the counter. */ |
397 | return EAGAIN; | |
398 | ||
399 | ++mutex->__data.__count; | |
400 | ||
401 | return 0; | |
402 | } | |
403 | } | |
404 | ||
405 | int newval = id; | |
184ee940 | 406 | # ifdef NO_INCR |
df47504c | 407 | newval |= FUTEX_WAITERS; |
184ee940 | 408 | # endif |
052757bf | 409 | oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
df47504c UD |
410 | newval, 0); |
411 | ||
412 | if (oldval != 0) | |
413 | { | |
414 | /* The mutex is locked. The kernel will now take care of | |
415 | everything. */ | |
efac1fce UD |
416 | int private = (robust |
417 | ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) | |
418 | : PTHREAD_MUTEX_PSHARED (mutex)); | |
df47504c UD |
419 | INTERNAL_SYSCALL_DECL (__err); |
420 | int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, | |
efac1fce UD |
421 | __lll_private_flag (FUTEX_LOCK_PI, |
422 | private), 1, 0); | |
df47504c UD |
423 | |
424 | if (INTERNAL_SYSCALL_ERROR_P (e, __err) | |
425 | && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH | |
426 | || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)) | |
427 | { | |
428 | assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK | |
429 | || (kind != PTHREAD_MUTEX_ERRORCHECK_NP | |
430 | && kind != PTHREAD_MUTEX_RECURSIVE_NP)); | |
431 | /* ESRCH can happen only for non-robust PI mutexes where | |
432 | the owner of the lock died. */ | |
433 | assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust); | |
434 | ||
435 | /* Delay the thread indefinitely. */ | |
436 | while (1) | |
08d6eb46 | 437 | __pause_nocancel (); |
df47504c UD |
438 | } |
439 | ||
440 | oldval = mutex->__data.__lock; | |
441 | ||
442 | assert (robust || (oldval & FUTEX_OWNER_DIED) == 0); | |
443 | } | |
444 | ||
a1ffb40e | 445 | if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED)) |
df47504c UD |
446 | { |
447 | atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED); | |
448 | ||
449 | /* We got the mutex. */ | |
450 | mutex->__data.__count = 1; | |
451 | /* But it is inconsistent unless marked otherwise. */ | |
452 | mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; | |
453 | ||
8f9450a0 TR |
454 | /* We must not enqueue the mutex before we have acquired it. |
455 | Also see comments at ENQUEUE_MUTEX. */ | |
456 | __asm ("" ::: "memory"); | |
df47504c | 457 | ENQUEUE_MUTEX_PI (mutex); |
8f9450a0 TR |
458 | /* We need to clear op_pending after we enqueue the mutex. */ |
459 | __asm ("" ::: "memory"); | |
df47504c UD |
460 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
461 | ||
462 | /* Note that we deliberately exit here. If we fall | |
463 | through to the end of the function __nusers would be | |
464 | incremented which is not correct because the old owner | |
465 | has to be discounted. If we are not supposed to | |
466 | increment __nusers we actually have to decrement it here. */ | |
184ee940 | 467 | # ifdef NO_INCR |
df47504c | 468 | --mutex->__data.__nusers; |
184ee940 | 469 | # endif |
df47504c UD |
470 | |
471 | return EOWNERDEAD; | |
472 | } | |
473 | ||
474 | if (robust | |
475 | && __builtin_expect (mutex->__data.__owner | |
476 | == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) | |
477 | { | |
478 | /* This mutex is now not recoverable. */ | |
479 | mutex->__data.__count = 0; | |
480 | ||
481 | INTERNAL_SYSCALL_DECL (__err); | |
482 | INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, | |
efac1fce | 483 | __lll_private_flag (FUTEX_UNLOCK_PI, |
6de79a49 | 484 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)), |
efac1fce | 485 | 0, 0); |
df47504c | 486 | |
8f9450a0 TR |
487 | /* To the kernel, this will be visible after the kernel has |
488 | acquired the mutex in the syscall. */ | |
df47504c UD |
489 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
490 | return ENOTRECOVERABLE; | |
491 | } | |
492 | ||
493 | mutex->__data.__count = 1; | |
494 | if (robust) | |
495 | { | |
8f9450a0 TR |
496 | /* We must not enqueue the mutex before we have acquired it. |
497 | Also see comments at ENQUEUE_MUTEX. */ | |
498 | __asm ("" ::: "memory"); | |
df47504c | 499 | ENQUEUE_MUTEX_PI (mutex); |
8f9450a0 TR |
500 | /* We need to clear op_pending after we enqueue the mutex. */ |
501 | __asm ("" ::: "memory"); | |
df47504c UD |
502 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
503 | } | |
504 | } | |
505 | break; | |
184ee940 | 506 | #endif /* __NR_futex. */ |
df47504c | 507 | |
f17efcb4 UD |
508 | case PTHREAD_MUTEX_PP_RECURSIVE_NP: |
509 | case PTHREAD_MUTEX_PP_ERRORCHECK_NP: | |
510 | case PTHREAD_MUTEX_PP_NORMAL_NP: | |
511 | case PTHREAD_MUTEX_PP_ADAPTIVE_NP: | |
512 | { | |
403b4feb SL |
513 | /* See concurrency notes regarding __kind in struct __pthread_mutex_s |
514 | in sysdeps/nptl/bits/thread-shared-types.h. */ | |
515 | int kind = atomic_load_relaxed (&(mutex->__data.__kind)) | |
516 | & PTHREAD_MUTEX_KIND_MASK_NP; | |
f17efcb4 UD |
517 | |
518 | oldval = mutex->__data.__lock; | |
519 | ||
520 | /* Check whether we already hold the mutex. */ | |
521 | if (mutex->__data.__owner == id) | |
522 | { | |
523 | if (kind == PTHREAD_MUTEX_ERRORCHECK_NP) | |
524 | return EDEADLK; | |
525 | ||
526 | if (kind == PTHREAD_MUTEX_RECURSIVE_NP) | |
527 | { | |
528 | /* Just bump the counter. */ | |
a1ffb40e | 529 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
f17efcb4 UD |
530 | /* Overflow of the counter. */ |
531 | return EAGAIN; | |
532 | ||
533 | ++mutex->__data.__count; | |
534 | ||
535 | return 0; | |
536 | } | |
537 | } | |
538 | ||
539 | int oldprio = -1, ceilval; | |
540 | do | |
541 | { | |
542 | int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) | |
543 | >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT; | |
544 | ||
545 | if (__pthread_current_priority () > ceiling) | |
546 | { | |
547 | if (oldprio != -1) | |
548 | __pthread_tpp_change_priority (oldprio, -1); | |
549 | return EINVAL; | |
550 | } | |
551 | ||
6de79a49 | 552 | int retval = __pthread_tpp_change_priority (oldprio, ceiling); |
f17efcb4 UD |
553 | if (retval) |
554 | return retval; | |
555 | ||
556 | ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT; | |
557 | oldprio = ceiling; | |
558 | ||
559 | oldval | |
052757bf | 560 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
f17efcb4 UD |
561 | #ifdef NO_INCR |
562 | ceilval | 2, | |
563 | #else | |
564 | ceilval | 1, | |
565 | #endif | |
566 | ceilval); | |
567 | ||
568 | if (oldval == ceilval) | |
569 | break; | |
570 | ||
571 | do | |
572 | { | |
573 | oldval | |
052757bf | 574 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
f17efcb4 UD |
575 | ceilval | 2, |
576 | ceilval | 1); | |
577 | ||
578 | if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval) | |
579 | break; | |
580 | ||
581 | if (oldval != ceilval) | |
835abc5c | 582 | lll_futex_wait (&mutex->__data.__lock, ceilval | 2, |
5bd8a249 | 583 | PTHREAD_MUTEX_PSHARED (mutex)); |
f17efcb4 | 584 | } |
052757bf | 585 | while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
f17efcb4 UD |
586 | ceilval | 2, ceilval) |
587 | != ceilval); | |
588 | } | |
589 | while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval); | |
590 | ||
591 | assert (mutex->__data.__owner == 0); | |
592 | mutex->__data.__count = 1; | |
593 | } | |
594 | break; | |
595 | ||
dcc73a8d UD |
596 | default: |
597 | /* Correct code cannot set any other type. */ | |
598 | return EINVAL; | |
76a50749 UD |
599 | } |
600 | ||
3892d906 | 601 | /* Record the ownership. */ |
3892d906 UD |
602 | mutex->__data.__owner = id; |
603 | #ifndef NO_INCR | |
604 | ++mutex->__data.__nusers; | |
605 | #endif | |
606 | ||
5acf7263 RM |
607 | LIBC_PROBE (mutex_acquired, 1, mutex); |
608 | ||
6de79a49 | 609 | return 0; |
76a50749 | 610 | } |
69431c9a | 611 | #ifndef __pthread_mutex_lock |
fa872e1b | 612 | weak_alias (__pthread_mutex_lock, pthread_mutex_lock) |
4d17e683 | 613 | hidden_def (__pthread_mutex_lock) |
69431c9a | 614 | #endif |
b0948ffd UD |
615 | |
616 | ||
617 | #ifdef NO_INCR | |
618 | void | |
9dd346ff | 619 | __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex) |
b0948ffd | 620 | { |
403b4feb SL |
621 | /* See concurrency notes regarding __kind in struct __pthread_mutex_s |
622 | in sysdeps/nptl/bits/thread-shared-types.h. */ | |
623 | int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); | |
624 | assert ((mutex_kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0); | |
625 | assert ((mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0); | |
626 | assert ((mutex_kind & PTHREAD_MUTEX_PSHARED_BIT) == 0); | |
b0948ffd UD |
627 | |
628 | /* Record the ownership. */ | |
629 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); | |
630 | mutex->__data.__owner = id; | |
631 | ||
403b4feb | 632 | if (mutex_kind == PTHREAD_MUTEX_PI_RECURSIVE_NP) |
b0948ffd UD |
633 | ++mutex->__data.__count; |
634 | } | |
635 | #endif |