]>
Commit | Line | Data |
---|---|---|
b168057a | 1 | /* Copyright (C) 2002-2015 Free Software Foundation, Inc. |
76a50749 UD |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
59ba27a6 PE |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ | |
76a50749 | 18 | |
6de79a49 | 19 | #include <assert.h> |
76a50749 | 20 | #include <errno.h> |
1bcfb5a5 | 21 | #include <stdlib.h> |
76a50749 UD |
22 | #include "pthreadP.h" |
23 | #include <lowlevellock.h> | |
5acf7263 | 24 | #include <stap-probe.h> |
76a50749 | 25 | |
e8c659d7 | 26 | #ifndef lll_unlock_elision |
e5e6bea2 | 27 | #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; }) |
e8c659d7 AK |
28 | #endif |
29 | ||
6de79a49 UD |
30 | static int |
31 | internal_function | |
32 | __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) | |
33 | __attribute_noinline__; | |
76a50749 UD |
34 | |
35 | int | |
3a4f2043 | 36 | internal_function attribute_hidden |
9dd346ff | 37 | __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr) |
76a50749 | 38 | { |
e8c659d7 AK |
39 | int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); |
40 | if (__builtin_expect (type & | |
41 | ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) | |
6de79a49 | 42 | return __pthread_mutex_unlock_full (mutex, decr); |
1bcfb5a5 | 43 | |
6de79a49 UD |
44 | if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP) |
45 | == PTHREAD_MUTEX_TIMED_NP) | |
46 | { | |
47 | /* Always reset the owner field. */ | |
48 | normal: | |
49 | mutex->__data.__owner = 0; | |
50 | if (decr) | |
51 | /* One less user. */ | |
52 | --mutex->__data.__nusers; | |
53 | ||
54 | /* Unlock. */ | |
55 | lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); | |
5acf7263 RM |
56 | |
57 | LIBC_PROBE (mutex_release, 1, mutex); | |
58 | ||
6de79a49 UD |
59 | return 0; |
60 | } | |
a1ffb40e | 61 | else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP)) |
e8c659d7 AK |
62 | { |
63 | /* Don't reset the owner/users fields for elision. */ | |
e5e6bea2 | 64 | return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision, |
e8c659d7 AK |
65 | PTHREAD_MUTEX_PSHARED (mutex)); |
66 | } | |
67 | else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) | |
68 | == PTHREAD_MUTEX_RECURSIVE_NP, 1)) | |
76a50749 | 69 | { |
76a50749 | 70 | /* Recursive mutex. */ |
61623643 | 71 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) |
76a50749 UD |
72 | return EPERM; |
73 | ||
74 | if (--mutex->__data.__count != 0) | |
75 | /* We still hold the mutex. */ | |
76 | return 0; | |
683040c3 | 77 | goto normal; |
6de79a49 | 78 | } |
e8c659d7 AK |
79 | else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) |
80 | == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) | |
6de79a49 UD |
81 | goto normal; |
82 | else | |
83 | { | |
76a50749 | 84 | /* Error checking mutex. */ |
6de79a49 | 85 | assert (type == PTHREAD_MUTEX_ERRORCHECK_NP); |
61623643 | 86 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid) |
e51deae7 | 87 | || ! lll_islocked (mutex->__data.__lock)) |
76a50749 | 88 | return EPERM; |
6de79a49 UD |
89 | goto normal; |
90 | } | |
91 | } | |
76a50749 | 92 | |
683040c3 | 93 | |
6de79a49 UD |
94 | static int |
95 | internal_function | |
96 | __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) | |
97 | { | |
98 | int newowner = 0; | |
dcc73a8d | 99 | |
6de79a49 UD |
100 | switch (PTHREAD_MUTEX_TYPE (mutex)) |
101 | { | |
0f6699ea | 102 | case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP: |
1bcfb5a5 | 103 | /* Recursive mutex. */ |
683040c3 | 104 | if ((mutex->__data.__lock & FUTEX_TID_MASK) |
0f6699ea UD |
105 | == THREAD_GETMEM (THREAD_SELF, tid) |
106 | && __builtin_expect (mutex->__data.__owner | |
107 | == PTHREAD_MUTEX_INCONSISTENT, 0)) | |
1bcfb5a5 UD |
108 | { |
109 | if (--mutex->__data.__count != 0) | |
110 | /* We still hold the mutex. */ | |
111 | return ENOTRECOVERABLE; | |
112 | ||
113 | goto notrecoverable; | |
114 | } | |
115 | ||
116 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) | |
117 | return EPERM; | |
118 | ||
119 | if (--mutex->__data.__count != 0) | |
120 | /* We still hold the mutex. */ | |
121 | return 0; | |
122 | ||
123 | goto robust; | |
124 | ||
0f6699ea UD |
125 | case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP: |
126 | case PTHREAD_MUTEX_ROBUST_NORMAL_NP: | |
127 | case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP: | |
683040c3 UD |
128 | if ((mutex->__data.__lock & FUTEX_TID_MASK) |
129 | != THREAD_GETMEM (THREAD_SELF, tid) | |
e51deae7 | 130 | || ! lll_islocked (mutex->__data.__lock)) |
1bcfb5a5 UD |
131 | return EPERM; |
132 | ||
1bcfb5a5 UD |
133 | /* If the previous owner died and the caller did not succeed in |
134 | making the state consistent, mark the mutex as unrecoverable | |
135 | and make all waiters. */ | |
136 | if (__builtin_expect (mutex->__data.__owner | |
683040c3 | 137 | == PTHREAD_MUTEX_INCONSISTENT, 0)) |
1bcfb5a5 UD |
138 | notrecoverable: |
139 | newowner = PTHREAD_MUTEX_NOTRECOVERABLE; | |
140 | ||
141 | robust: | |
142 | /* Remove mutex from the list. */ | |
0f6699ea UD |
143 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
144 | &mutex->__data.__list.__next); | |
1bcfb5a5 | 145 | DEQUEUE_MUTEX (mutex); |
683040c3 UD |
146 | |
147 | mutex->__data.__owner = newowner; | |
148 | if (decr) | |
149 | /* One less user. */ | |
150 | --mutex->__data.__nusers; | |
151 | ||
152 | /* Unlock. */ | |
5bd8a249 UD |
153 | lll_robust_unlock (mutex->__data.__lock, |
154 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); | |
0f6699ea UD |
155 | |
156 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); | |
1bcfb5a5 UD |
157 | break; |
158 | ||
184ee940 RM |
159 | /* The PI support requires the Linux futex system call. If that's not |
160 | available, pthread_mutex_init should never have allowed the type to | |
161 | be set. So it will get the default case for an invalid type. */ | |
162 | #ifdef __NR_futex | |
df47504c UD |
163 | case PTHREAD_MUTEX_PI_RECURSIVE_NP: |
164 | /* Recursive mutex. */ | |
165 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) | |
166 | return EPERM; | |
167 | ||
168 | if (--mutex->__data.__count != 0) | |
169 | /* We still hold the mutex. */ | |
170 | return 0; | |
b0948ffd | 171 | goto continue_pi_non_robust; |
df47504c UD |
172 | |
173 | case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP: | |
174 | /* Recursive mutex. */ | |
175 | if ((mutex->__data.__lock & FUTEX_TID_MASK) | |
176 | == THREAD_GETMEM (THREAD_SELF, tid) | |
177 | && __builtin_expect (mutex->__data.__owner | |
178 | == PTHREAD_MUTEX_INCONSISTENT, 0)) | |
179 | { | |
180 | if (--mutex->__data.__count != 0) | |
181 | /* We still hold the mutex. */ | |
182 | return ENOTRECOVERABLE; | |
183 | ||
184 | goto pi_notrecoverable; | |
185 | } | |
186 | ||
187 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) | |
188 | return EPERM; | |
189 | ||
190 | if (--mutex->__data.__count != 0) | |
191 | /* We still hold the mutex. */ | |
192 | return 0; | |
193 | ||
b0948ffd | 194 | goto continue_pi_robust; |
df47504c UD |
195 | |
196 | case PTHREAD_MUTEX_PI_ERRORCHECK_NP: | |
197 | case PTHREAD_MUTEX_PI_NORMAL_NP: | |
198 | case PTHREAD_MUTEX_PI_ADAPTIVE_NP: | |
199 | case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP: | |
200 | case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP: | |
201 | case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP: | |
202 | if ((mutex->__data.__lock & FUTEX_TID_MASK) | |
203 | != THREAD_GETMEM (THREAD_SELF, tid) | |
e51deae7 | 204 | || ! lll_islocked (mutex->__data.__lock)) |
df47504c UD |
205 | return EPERM; |
206 | ||
207 | /* If the previous owner died and the caller did not succeed in | |
208 | making the state consistent, mark the mutex as unrecoverable | |
209 | and make all waiters. */ | |
210 | if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0 | |
211 | && __builtin_expect (mutex->__data.__owner | |
212 | == PTHREAD_MUTEX_INCONSISTENT, 0)) | |
213 | pi_notrecoverable: | |
214 | newowner = PTHREAD_MUTEX_NOTRECOVERABLE; | |
215 | ||
df47504c UD |
216 | if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0) |
217 | { | |
b0948ffd | 218 | continue_pi_robust: |
df47504c UD |
219 | /* Remove mutex from the list. |
220 | Note: robust PI futexes are signaled by setting bit 0. */ | |
221 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, | |
222 | (void *) (((uintptr_t) &mutex->__data.__list.__next) | |
223 | | 1)); | |
224 | DEQUEUE_MUTEX (mutex); | |
225 | } | |
226 | ||
b0948ffd | 227 | continue_pi_non_robust: |
df47504c UD |
228 | mutex->__data.__owner = newowner; |
229 | if (decr) | |
230 | /* One less user. */ | |
231 | --mutex->__data.__nusers; | |
232 | ||
233 | /* Unlock. */ | |
234 | if ((mutex->__data.__lock & FUTEX_WAITERS) != 0 | |
e73e694e | 235 | || atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock, 0, |
df47504c UD |
236 | THREAD_GETMEM (THREAD_SELF, |
237 | tid))) | |
238 | { | |
efac1fce UD |
239 | int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP; |
240 | int private = (robust | |
241 | ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) | |
242 | : PTHREAD_MUTEX_PSHARED (mutex)); | |
df47504c UD |
243 | INTERNAL_SYSCALL_DECL (__err); |
244 | INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock, | |
efac1fce | 245 | __lll_private_flag (FUTEX_UNLOCK_PI, private)); |
df47504c UD |
246 | } |
247 | ||
248 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); | |
249 | break; | |
184ee940 | 250 | #endif /* __NR_futex. */ |
df47504c | 251 | |
f17efcb4 UD |
252 | case PTHREAD_MUTEX_PP_RECURSIVE_NP: |
253 | /* Recursive mutex. */ | |
254 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) | |
255 | return EPERM; | |
256 | ||
257 | if (--mutex->__data.__count != 0) | |
258 | /* We still hold the mutex. */ | |
259 | return 0; | |
260 | goto pp; | |
261 | ||
262 | case PTHREAD_MUTEX_PP_ERRORCHECK_NP: | |
263 | /* Error checking mutex. */ | |
264 | if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid) | |
265 | || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0) | |
266 | return EPERM; | |
267 | /* FALLTHROUGH */ | |
268 | ||
269 | case PTHREAD_MUTEX_PP_NORMAL_NP: | |
270 | case PTHREAD_MUTEX_PP_ADAPTIVE_NP: | |
271 | /* Always reset the owner field. */ | |
272 | pp: | |
273 | mutex->__data.__owner = 0; | |
274 | ||
275 | if (decr) | |
276 | /* One less user. */ | |
277 | --mutex->__data.__nusers; | |
278 | ||
279 | /* Unlock. */ | |
280 | int newval, oldval; | |
281 | do | |
282 | { | |
283 | oldval = mutex->__data.__lock; | |
284 | newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK; | |
285 | } | |
e73e694e | 286 | while (atomic_compare_and_exchange_bool_rel (&mutex->__data.__lock, |
f17efcb4 UD |
287 | newval, oldval)); |
288 | ||
289 | if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1) | |
835abc5c | 290 | lll_futex_wake (&mutex->__data.__lock, 1, |
5bd8a249 | 291 | PTHREAD_MUTEX_PSHARED (mutex)); |
f17efcb4 UD |
292 | |
293 | int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT; | |
5acf7263 RM |
294 | |
295 | LIBC_PROBE (mutex_release, 1, mutex); | |
296 | ||
f17efcb4 UD |
297 | return __pthread_tpp_change_priority (oldprio, -1); |
298 | ||
dcc73a8d UD |
299 | default: |
300 | /* Correct code cannot set any other type. */ | |
301 | return EINVAL; | |
a334319f | 302 | } |
0ecb606c | 303 | |
5acf7263 | 304 | LIBC_PROBE (mutex_release, 1, mutex); |
76a50749 UD |
305 | return 0; |
306 | } | |
61623643 UD |
307 | |
308 | ||
309 | int | |
9dd346ff | 310 | __pthread_mutex_unlock (pthread_mutex_t *mutex) |
61623643 | 311 | { |
8f31c0ef | 312 | return __pthread_mutex_unlock_usercnt (mutex, 1); |
61623643 | 313 | } |
76a50749 | 314 | strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock) |
4d17e683 | 315 | hidden_def (__pthread_mutex_unlock) |