<https://www.gnu.org/licenses/>. */
#include <ctype.h>
+#include <elision-conf.h>
#include <libc-early-init.h>
#include <libc-internal.h>
+#include <lowlevellock.h>
#include <sys/single_threaded.h>
#ifdef SHARED
#ifdef SHARED
__libc_initial = initial;
#endif
+
+#if ENABLE_ELISION_SUPPORT
+ __lll_elision_init ();
+#endif
}
routines = \
alloca_cutoff \
+ elision-conf \
+ elision-lock \
+ elision-timed \
+ elision-trylock \
+ elision-unlock \
forward \
futex-internal \
libc-cancellation \
__libc_current_sigrtmin_private;
__libc_dl_error_tsd;
__libc_pthread_init;
+ __lll_clocklock_elision;
+ __lll_lock_elision;
__lll_lock_wait;
__lll_lock_wait_private;
+ __lll_trylock_elision;
+ __lll_unlock_elision;
__pthread_attr_copy;
__pthread_attr_destroy;
__pthread_attr_init;
__pthread_attr_setsigmask_internal;
__pthread_cond_destroy; # Used by the C11 threads.
__pthread_cond_init; # Used by the C11 threads.
+ __pthread_force_elision;
__pthread_getattr_default_np;
}
}
-/* Elided version of pthread_mutex_lock.
- Copyright (C) 2014-2021 Free Software Foundation, Inc.
+/* elision-conf.c: Lock elision tunable parameters. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include <force-elision.h>
-
-#include <nptl/pthread_mutex_lock.c>
-/* Elided version of pthread_mutex_lock.
- Copyright (C) 2015-2021 Free Software Foundation, Inc.
+/* elision-lock.c: Lock elision locking. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include <force-elision.h>
-
-#include <nptl/pthread_mutex_lock.c>
-/* Elided version of pthread_mutex_trylock.
- Copyright (C) 2015-2021 Free Software Foundation, Inc.
+/* elision-lock.c: Lock elision timed locking. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include <force-elision.h>
-
-#include <nptl/pthread_mutex_trylock.c>
-/* Elided version of pthread_mutex_lock.
- Copyright (C) 2011-2021 Free Software Foundation, Inc.
+/* elision-lock.c: Lock elision locking attempts. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include "force-elision.h"
-
-#include "nptl/pthread_mutex_lock.c"
--- /dev/null
+/* elision-lock.c: Lock elision unlocking support. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
}
}
libc_hidden_def (__lll_lock_wait)
+
+#if ENABLE_ELISION_SUPPORT
+int __pthread_force_elision __attribute__ ((nocommon));
+libc_hidden_data_def (__pthread_force_elision)
+#endif
#include <futex-internal.h>
#include <stap-probe.h>
-#ifndef lll_lock_elision
-#define lll_lock_elision(lock, try_lock, private) ({ \
- lll_lock (lock, private); 0; })
-#endif
-
-#ifndef lll_trylock_elision
-#define lll_trylock_elision(a,t) lll_trylock(a)
-#endif
-
/* Some of the following definitions differ when pthread_mutex_cond_lock.c
includes this file. */
#ifndef LLL_MUTEX_LOCK
PTHREAD_MUTEX_PSHARED (mutex))
#endif
-#ifndef FORCE_ELISION
-#define FORCE_ELISION(m, s)
-#endif
-
static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
__attribute_noinline__;
LLL_MUTEX_LOCK (mutex);
assert (mutex->__data.__owner == 0);
}
-#ifdef HAVE_ELISION
+#if ENABLE_ELISION_SUPPORT
else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
{
elision: __attribute__((unused))
#include <stap-probe.h>
-#ifndef lll_clocklock_elision
-#define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \
- __futex_clocklock64 (&(futex), clockid, abstime, private)
-#endif
-
-#ifndef lll_trylock_elision
-#define lll_trylock_elision(a,t) lll_trylock(a)
-#endif
-
-#ifndef FORCE_ELISION
-#define FORCE_ELISION(m, s)
-#endif
-
int
__pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
clockid_t clockid,
#include <lowlevellock.h>
#include <futex-internal.h>
-#ifndef lll_trylock_elision
-#define lll_trylock_elision(a,t) lll_trylock(a)
-#endif
-
-#ifndef FORCE_ELISION
-#define FORCE_ELISION(m, s)
-#endif
-
int
__pthread_mutex_trylock (pthread_mutex_t *mutex)
{
#include <stap-probe.h>
#include <futex-internal.h>
-#ifndef lll_unlock_elision
-#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
-#endif
-
static int
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
__attribute_noinline__;
#define _LOWLEVELLOCK_H 1
#include <atomic.h>
+#include <elision-conf.h>
#include <lowlevellock-futex.h>
#include <time.h>
#define LLL_LOCK_INITIALIZER (0)
#define LLL_LOCK_INITIALIZER_LOCKED (1)
+/* Elision support. */
+
+#if ENABLE_ELISION_SUPPORT
+/* Force elision for all new locks. This is used to decide whether
+ existing DEFAULT locks should be automatically upgraded to elision
+ in pthread_mutex_lock. Disabled for suid programs. Only used when
+ elision is available. */
+extern int __pthread_force_elision;
+libc_hidden_proto (__pthread_force_elision)
+
+extern void __lll_elision_init (void) attribute_hidden;
+extern int __lll_clocklock_elision (int *futex, short *adapt_count,
+ clockid_t clockid,
+ const struct __timespec64 *timeout,
+ int private);
+libc_hidden_proto (__lll_clocklock_elision)
+
+extern int __lll_lock_elision (int *futex, short *adapt_count, int private);
+libc_hidden_proto (__lll_lock_elision)
+
+# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT
+extern int __lll_unlock_elision (int *lock, short *adapt_count, int private);
+# else
+extern int __lll_unlock_elision (int *lock, int private);
+# endif
+libc_hidden_proto (__lll_unlock_elision)
+
+extern int __lll_trylock_elision (int *lock, short *adapt_count);
+libc_hidden_proto (__lll_trylock_elision)
+
+# define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
+ __lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
+# define lll_lock_elision(futex, adapt_count, private) \
+ __lll_lock_elision (&(futex), &(adapt_count), private)
+# define lll_trylock_elision(futex, adapt_count) \
+ __lll_trylock_elision (&(futex), &(adapt_count))
+# if ELISION_UNLOCK_NEEDS_ADAPT_COUNT
+# define lll_unlock_elision(futex, adapt_count, private) \
+ __lll_unlock_elision (&(futex), &(adapt_count), private)
+# else
+# define lll_unlock_elision(futex, adapt_count, private) \
+ __lll_unlock_elision (&(futex), private)
+# endif
+
+/* Automatically enable elision for existing user lock kinds. */
+# define FORCE_ELISION(m, s) \
+ if (__pthread_force_elision) \
+ { \
+ /* See concurrency notes regarding __kind in \
+ struct __pthread_mutex_s in \
+ sysdeps/nptl/bits/thread-shared-types.h. \
+ \
+ There are the following cases for the kind of a mutex \
+ (The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
+ PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
+ only one of both flags can be set): \
+ - both flags are not set: \
+ This is the first lock operation for this mutex. Enable \
+ elision as it is not enabled so far. \
+ Note: It can happen that multiple threads are calling e.g. \
+ pthread_mutex_lock at the same time as the first lock \
+ operation for this mutex. Then elision is enabled for this \
+ mutex by multiple threads. Storing with relaxed MO is enough \
+ as all threads will store the same new value for the kind of \
+ the mutex. But we have to ensure that we always use the \
+ elision path regardless if this thread has enabled elision or \
+ another one. \
+ \
+ - PTHREAD_MUTEX_ELISION_NP flag is set: \
+ Elision was already enabled for this mutex by a previous lock \
+ operation. See case above. Just use the elision path. \
+ \
+ - PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
+ Elision was explicitly disabled by pthread_mutexattr_settype. \
+ Do not use the elision path. \
+ Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
+ changed after mutex initialization. */ \
+ int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
+ if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
+ { \
+ mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
+ atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
+ } \
+ if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
+ { \
+ s; \
+ } \
+ }
+
+#else /* !ENABLE_ELISION_SUPPORT */
+
+# define lll_clocklock_elision(futex, adapt_count, clockid, abstime, private) \
+ __futex_clocklock64 (&(futex), clockid, abstime, private)
+# define lll_lock_elision(lock, try_lock, private) \
+ ({ lll_lock (lock, private); 0; })
+# define lll_trylock_elision(a,t) lll_trylock(a)
+# define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
+# define FORCE_ELISION(m, s)
+
+#endif /* !ENABLE_ELISION_SUPPORT */
+
#endif /* lowlevellock.h */
-/* Elided version of pthread_mutex_timedlock.
- Copyright (C) 2015-2021 Free Software Foundation, Inc.
+/* elision-conf.h: Lock elision configuration. Stub version.
+ Copyright (C) 2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-#include <elision-conf.h>
-#include <force-elision.h>
+#ifndef _ELISION_CONF_H
+#define _ELISION_CONF_H 1
-#include <nptl/pthread_mutex_timedlock.c>
+/* No elision support by default. */
+#define ENABLE_ELISION_SUPPORT 0
+
+/* Whether __lll_unlock_elision expects a pointer argument to the
+ adaptive counter. Here, an unused arbitrary value. */
+#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0
+
+#endif
ifeq ($(subdir),nptl)
libpthread-routines += sysdep
-libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
- elision-trylock
libpthread-shared-only-routines += sysdep
endif
.skip_trylock_internal_abort = 3,
};
-/* Force elision for all new locks. This is used to decide whether existing
- DEFAULT locks should be automatically use elision in pthread_mutex_lock().
- Disabled for suid programs. Only used when elision is available. */
-
-int __pthread_force_elision attribute_hidden = 0;
-
#if HAVE_TUNABLES
static inline void
__always_inline
/* Initialize elision. */
-static void
-elision_init (int argc __attribute__ ((unused)),
- char **argv __attribute__ ((unused)),
- char **environ)
+void
+__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
if (!__pthread_force_elision)
__elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */
}
-
-#ifdef SHARED
-# define INIT_SECTION ".init_array"
-# define MAYBE_CONST
-#else
-# define INIT_SECTION ".preinit_array"
-# define MAYBE_CONST const
-#endif
-
-void (*MAYBE_CONST __pthread_init_array []) (int, char **, char **)
- __attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
-{
- &elision_init
-};
#include <pthread.h>
#include <time.h>
+#define ENABLE_ELISION_SUPPORT 1
+#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1
+
/* Should make sure there is no false sharing on this. */
struct elision_config
{
extern struct elision_config __elision_aconf attribute_hidden;
-extern int __pthread_force_elision attribute_hidden;
-
-/* Tell the test suite to test elision for this architecture. */
-#define HAVE_ELISION 1
-
#endif
#include <elision-conf.h>
#include "htm.h"
-#if !defined(LLL_LOCK) && !defined(EXTRAARG)
-/* Make sure the configuration code is always linked in for static
- libraries. */
-#include "elision-conf.c"
-#endif
-
#ifndef EXTRAARG
# define EXTRAARG
#endif
use_lock:
return LLL_LOCK ((*lock), pshared);
}
+libc_hidden_def (__lll_lock_elision)
use_lock:
return lll_trylock (*futex);
}
+libc_hidden_def (__lll_trylock_elision)
}
return 0;
}
+libc_hidden_def (__lll_unlock_elision)
+++ /dev/null
-/* force-elision.h: Automatic enabling of elision for mutexes
- Copyright (C) 2015-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* Automatically enable elision for existing user lock kinds. */
-#define FORCE_ELISION(m, s) \
- if (__pthread_force_elision) \
- { \
- /* See concurrency notes regarding __kind in \
- struct __pthread_mutex_s in \
- sysdeps/nptl/bits/thread-shared-types.h. \
- \
- There are the following cases for the kind of a mutex \
- (The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
- PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
- only one of both flags can be set): \
- - both flags are not set: \
- This is the first lock operation for this mutex. Enable \
- elision as it is not enabled so far. \
- Note: It can happen that multiple threads are calling e.g. \
- pthread_mutex_lock at the same time as the first lock \
- operation for this mutex. Then elision is enabled for this \
- mutex by multiple threads. Storing with relaxed MO is enough \
- as all threads will store the same new value for the kind of \
- the mutex. But we have to ensure that we always use the \
- elision path regardless if this thread has enabled elision or \
- another one. \
- \
- - PTHREAD_MUTEX_ELISION_NP flag is set: \
- Elision was already enabled for this mutex by a previous lock \
- operation. See case above. Just use the elision path. \
- \
- - PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
- Elision was explicitly disabled by pthread_mutexattr_settype. \
- Do not use the elision path. \
- Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
- changed after mutex initialization. */ \
- int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
- { \
- mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
- atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
- } \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
- { \
- s; \
- } \
- }
+++ /dev/null
-/* PowerPC specific lock definitions.
- Copyright (C) 2015-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#ifndef _POWERPC_LOWLEVELLOCK_H
-#define _POWERPC_LOWLEVELLOCK_H 1
-
-#include <sysdeps/nptl/lowlevellock.h>
-
-/* Transactional lock elision definitions. */
-extern int __lll_clocklock_elision
- (int *futex, short *adapt_count,
- clockid_t clockid, const struct __timespec64 *timeout, int private)
- attribute_hidden;
-
-#define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
- __lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
-
-extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_unlock_elision (int *lock, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_trylock_elision(int *lock, short *adapt_count)
- attribute_hidden;
-
-#define lll_lock_elision(futex, adapt_count, private) \
- __lll_lock_elision (&(futex), &(adapt_count), private)
-#define lll_unlock_elision(futex, adapt_count, private) \
- __lll_unlock_elision (&(futex), &(adapt_count), private)
-#define lll_trylock_elision(futex, adapt_count) \
- __lll_trylock_elision (&(futex), &(adapt_count))
-
-#endif
+++ /dev/null
-/* Copyright (C) 2015-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* The cond lock is not actually elided yet, but we still need to handle
- already elided locks. */
-#include <elision-conf.h>
-
-#include <nptl/pthread_mutex_cond_lock.c>
endif
ifeq ($(subdir),nptl)
-libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
- elision-trylock
-
elision-CFLAGS = -mhtm -msoft-float
CFLAGS-elision-lock.c = $(elision-CFLAGS)
CFLAGS-elision-timed.c = $(elision-CFLAGS)
.skip_trylock_internal_abort = 3,
};
-/* Force elision for all new locks. This is used to decide whether existing
- DEFAULT locks should be automatically upgraded to elision in
- pthread_mutex_lock(). Disabled for suid programs. Only used when elision
- is available. */
-
-int __pthread_force_elision attribute_hidden = 0;
-
#if HAVE_TUNABLES
static inline void
__always_inline
/* Initialize elison. */
-static void
-elision_init (int argc __attribute__ ((unused)),
- char **argv __attribute__ ((unused)),
- char **environ)
+void
+__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
if (!__pthread_force_elision)
__elision_aconf.try_tbegin = 0; /* Disable elision on rwlocks. */
}
-
-#ifdef SHARED
-# define INIT_SECTION ".init_array"
-# define MAYBE_CONST
-#else
-# define INIT_SECTION ".preinit_array"
-# define MAYBE_CONST const
-#endif
-
-void (*MAYBE_CONST __pthread_init_array []) (int, char **, char **)
- __attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
-{
- &elision_init
-};
#include <pthread.h>
#include <time.h>
+#define ENABLE_ELISION_SUPPORT 1
+#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 1
+
/* Should make sure there is no false sharing on this. */
struct elision_config
extern struct elision_config __elision_aconf attribute_hidden;
-extern int __pthread_force_elision attribute_hidden;
-
-/* Tell the test suite to test elision for this architecture. */
-#define HAVE_ELISION 1
-
#endif
#include <elision-conf.h>
#include <stdint.h>
-#if !defined(LLL_LOCK) && !defined(EXTRAARG)
-/* Make sure the configuration code is always linked in for static
- libraries. */
-#include "elision-conf.c"
-#endif
-
#ifndef EXTRAARG
#define EXTRAARG
#endif
succeed. */
return LLL_LOCK ((*futex), private);
}
+libc_hidden_def (__lll_lock_elision)
succeed. */
return lll_trylock (*futex);
}
+libc_hidden_def (__lll_trylock_elision)
}
return 0;
}
+libc_hidden_def (__lll_unlock_elision)
+++ /dev/null
-/* Automatic enabling of elision for mutexes
- Copyright (C) 2014-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* Automatically enable elision for existing user lock kinds. */
-#define FORCE_ELISION(m, s) \
- if (__pthread_force_elision) \
- { \
- /* See concurrency notes regarding __kind in \
- struct __pthread_mutex_s in \
- sysdeps/nptl/bits/thread-shared-types.h. \
- \
- There are the following cases for the kind of a mutex \
- (The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
- PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
- only one of both flags can be set): \
- - both flags are not set: \
- This is the first lock operation for this mutex. Enable \
- elision as it is not enabled so far. \
- Note: It can happen that multiple threads are calling e.g. \
- pthread_mutex_lock at the same time as the first lock \
- operation for this mutex. Then elision is enabled for this \
- mutex by multiple threads. Storing with relaxed MO is enough \
- as all threads will store the same new value for the kind of \
- the mutex. But we have to ensure that we always use the \
- elision path regardless if this thread has enabled elision or \
- another one. \
- \
- - PTHREAD_MUTEX_ELISION_NP flag is set: \
- Elision was already enabled for this mutex by a previous lock \
- operation. See case above. Just use the elision path. \
- \
- - PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
- Elision was explicitly disabled by pthread_mutexattr_settype. \
- Do not use the elision path. \
- Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
- changed after mutex initialization. */ \
- int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
- { \
- mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
- atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
- } \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
- { \
- s; \
- } \
- }
+++ /dev/null
-/* Copyright (C) 2003-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#ifndef _S390_LOWLEVELLOCK_H
-#define _S390_LOWLEVELLOCK_H 1
-
-#include <sysdeps/nptl/lowlevellock.h>
-
-/* Transactional lock elision definitions. */
-extern int __lll_clocklock_elision
- (int *futex, short *adapt_count,
- clockid_t clockid, const struct __timespec64 *timeout, int private)
- attribute_hidden;
-
-# define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
- __lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
-
-extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_unlock_elision(int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_trylock_elision(int *futex, short *adapt_count)
- attribute_hidden;
-
-# define lll_lock_elision(futex, adapt_count, private) \
- __lll_lock_elision (&(futex), &(adapt_count), private)
-# define lll_unlock_elision(futex, adapt_count, private) \
- __lll_unlock_elision (&(futex), &(adapt_count), private)
-# define lll_trylock_elision(futex, adapt_count) \
- __lll_trylock_elision(&(futex), &(adapt_count))
-
-#endif /* lowlevellock.h */
+++ /dev/null
-/* Copyright (C) 2014-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* The cond lock is not actually elided yet, but we still need to handle
- already elided locks. */
-#include <elision-conf.h>
-
-#include <nptl/pthread_mutex_cond_lock.c>
+++ /dev/null
-/* Elided version of pthread_mutex_timedlock.
- Copyright (C) 2014-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include <force-elision.h>
-
-#include <nptl/pthread_mutex_timedlock.c>
+++ /dev/null
-/* Elided version of pthread_mutex_trylock.
- Copyright (C) 2014-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include <force-elision.h>
-
-#include <nptl/pthread_mutex_trylock.c>
endif
ifeq ($(subdir),nptl)
-libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
- elision-trylock
CFLAGS-elision-lock.c += -mrtm
CFLAGS-elision-unlock.c += -mrtm
CFLAGS-elision-timed.c += -mrtm
.skip_trylock_internal_abort = 3,
};
-/* Force elision for all new locks. This is used to decide whether existing
- DEFAULT locks should be automatically upgraded to elision in
- pthread_mutex_lock(). Disabled for suid programs. Only used when elision
- is available. */
-
-int __pthread_force_elision attribute_hidden = 0;
-
#if HAVE_TUNABLES
static __always_inline void
do_set_elision_enable (int32_t elision_enable)
/* Initialize elision. */
-static void
-elision_init (int argc __attribute__ ((unused)),
- char **argv __attribute__ ((unused)),
- char **environ)
+void
+__lll_elision_init (void)
{
#if HAVE_TUNABLES
/* Elision depends on tunables and must be explicitly turned on by setting
if (!__pthread_force_elision)
__elision_aconf.retry_try_xbegin = 0; /* Disable elision on rwlocks. */
}
-
-#ifdef SHARED
-# define INIT_SECTION ".init_array"
-#else
-# define INIT_SECTION ".preinit_array"
-#endif
-
-void (*const __pthread_init_array []) (int, char **, char **)
- __attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) =
-{
- &elision_init
-};
#include <pthread.h>
#include <time.h>
+#define ENABLE_ELISION_SUPPORT 1
+#define ELISION_UNLOCK_NEEDS_ADAPT_COUNT 0
+
/* Should make sure there is no false sharing on this. */
struct elision_config
extern struct elision_config __elision_aconf attribute_hidden;
-extern int __pthread_force_elision attribute_hidden;
-
-/* Tell the test suite to test elision for this architecture. */
-#define HAVE_ELISION 1
-
#endif
#include "hle.h"
#include <elision-conf.h>
-#if !defined(LLL_LOCK) && !defined(EXTRAARG)
-/* Make sure the configuration code is always linked in for static
- libraries. */
-#include "elision-conf.c"
-#endif
-
#ifndef EXTRAARG
#define EXTRAARG
#endif
/* Use a normal lock here. */
return LLL_LOCK ((*futex), private);
}
+libc_hidden_def (__lll_lock_elision)
return lll_trylock (*futex);
}
+libc_hidden_def (__lll_trylock_elision)
lll_unlock ((*lock), private);
return 0;
}
+libc_hidden_def (__lll_unlock_elision)
+++ /dev/null
-/* force-elision.h: Automatic enabling of elision for mutexes
- Copyright (C) 2013-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* Automatically enable elision for existing user lock kinds. */
-#define FORCE_ELISION(m, s) \
- if (__pthread_force_elision) \
- { \
- /* See concurrency notes regarding __kind in \
- struct __pthread_mutex_s in \
- sysdeps/nptl/bits/thread-shared-types.h. \
- \
- There are the following cases for the kind of a mutex \
- (The mask PTHREAD_MUTEX_ELISION_FLAGS_NP covers the flags \
- PTHREAD_MUTEX_ELISION_NP and PTHREAD_MUTEX_NO_ELISION_NP where \
- only one of both flags can be set): \
- - both flags are not set: \
- This is the first lock operation for this mutex. Enable \
- elision as it is not enabled so far. \
- Note: It can happen that multiple threads are calling e.g. \
- pthread_mutex_lock at the same time as the first lock \
- operation for this mutex. Then elision is enabled for this \
- mutex by multiple threads. Storing with relaxed MO is enough \
- as all threads will store the same new value for the kind of \
- the mutex. But we have to ensure that we always use the \
- elision path regardless if this thread has enabled elision or \
- another one. \
- \
- - PTHREAD_MUTEX_ELISION_NP flag is set: \
- Elision was already enabled for this mutex by a previous lock \
- operation. See case above. Just use the elision path. \
- \
- - PTHREAD_MUTEX_NO_ELISION_NP flag is set: \
- Elision was explicitly disabled by pthread_mutexattr_settype. \
- Do not use the elision path. \
- Note: The flag PTHREAD_MUTEX_NO_ELISION_NP will never be \
- changed after mutex initialization. */ \
- int mutex_kind = atomic_load_relaxed (&((m)->__data.__kind)); \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \
- { \
- mutex_kind |= PTHREAD_MUTEX_ELISION_NP; \
- atomic_store_relaxed (&((m)->__data.__kind), mutex_kind); \
- } \
- if ((mutex_kind & PTHREAD_MUTEX_ELISION_NP) != 0) \
- { \
- s; \
- } \
- }
__lll_unlock (&(lock), private); \
}))
-extern int __lll_clocklock_elision (int *futex, short *adapt_count,
- clockid_t clockid,
- const struct __timespec64 *timeout,
- int private) attribute_hidden;
-
-#define lll_clocklock_elision(futex, adapt_count, clockid, timeout, private) \
- __lll_clocklock_elision (&(futex), &(adapt_count), clockid, timeout, private)
-
-extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_unlock_elision (int *lock, int private)
- attribute_hidden;
-
-extern int __lll_trylock_elision (int *lock, short *adapt_count)
- attribute_hidden;
-
-#define lll_lock_elision(futex, adapt_count, private) \
- __lll_lock_elision (&(futex), &(adapt_count), private)
-#define lll_unlock_elision(futex, adapt_count, private) \
- __lll_unlock_elision (&(futex), private)
-#define lll_trylock_elision(futex, adapt_count) \
- __lll_trylock_elision (&(futex), &(adapt_count))
-
#endif /* !__ASSEMBLER__ */
#endif /* lowlevellock.h */
+++ /dev/null
-/* Copyright (C) 2013-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-/* The cond lock is not actually elided yet, but we still need to handle
- already elided locks. */
-#include <elision-conf.h>
-
-#include <nptl/pthread_mutex_cond_lock.c>
+++ /dev/null
-/* Elided version of pthread_mutex_timedlock.
- Copyright (C) 2011-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include "force-elision.h"
-
-#include "nptl/pthread_mutex_timedlock.c"
+++ /dev/null
-/* Elided version of pthread_mutex_trylock.
- Copyright (C) 2011-2021 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <https://www.gnu.org/licenses/>. */
-
-#include <elision-conf.h>
-#include "force-elision.h"
-
-#include "nptl/pthread_mutex_trylock.c"
projects / thirdparty / glibc.git / commitdiff
