misc: Optimize internal usage of __libc_single_threaded

By adding an internal alias to avoid the GOT indirection.
On some architecture, __libc_single_thread may be accessed through
copy relocations and thus it requires to update also the copies
default copy.

This is done by adding a new internal macro,
libc_hidden_data_{proto,def}, which has an addition argument that
specifies the alias name (instead of default __GI_ one).

Checked on x86_64-linux-gnu and i686-linux-gnu.

Reviewed-by: Fangrui Song <maskray@google.com>

diff --git a/elf/libc_early_init.c b/elf/libc_early_init.c
index 3c4a19cf6bccc3a3617b50e53186fd0f726515bb..affc46fefc816d5fbb04d030e88295c3d944300b 100644 (file)
--- a/elf/libc_early_init.c
+++ b/elf/libc_early_init.c
@@ -38,7 +38,7 @@ __libc_early_init (_Bool initial)
   __libc_single_threaded = initial;
 
 #ifdef SHARED
-  __libc_initial = initial;
+  __libc_single_threaded_internal = __libc_initial = initial;
 #endif
 
   __pthread_early_init ();

diff --git a/include/libc-symbols.h b/include/libc-symbols.h
index 4bb3d8c7ba55d9c07a6193db537b57c44a97996d..4fde8394718454d9e11a6485155fa4e7c83922fb 100644 (file)
--- a/include/libc-symbols.h
+++ b/include/libc-symbols.h
@@ -534,11 +534,15 @@ for linking")
   __attribute__ ((visibility ("hidden"), ##attrs))
 #  define hidden_proto(name, attrs...) \
   __hidden_proto (name, , __GI_##name, ##attrs)
+#  define hidden_proto_alias(name, alias, attrs...) \
+  __hidden_proto_alias (name, , alias, ##attrs)
 #  define hidden_tls_proto(name, attrs...) \
   __hidden_proto (name, __thread, __GI_##name, ##attrs)
 #  define __hidden_proto(name, thread, internal, attrs...)          \
   extern thread __typeof (name) name __asm__ (__hidden_asmname (#internal)) \
   __hidden_proto_hiddenattr (attrs);
+#  define __hidden_proto_alias(name, thread, internal, attrs...)            \
+  extern thread __typeof (name) internal __hidden_proto_hiddenattr (attrs);
 #  define __hidden_asmname(name) \
   __hidden_asmname1 (__USER_LABEL_PREFIX__, name)
 #  define __hidden_asmname1(prefix, name) __hidden_asmname2(prefix, name)
@@ -554,7 +558,10 @@ for linking")
 #  define hidden_ver(local, name)      __hidden_ver1(local, __GI_##name, name);
 #  define hidden_data_ver(local, name) hidden_ver(local, name)
 #  define hidden_def(name)             __hidden_ver1(__GI_##name, name, name);
+#  define hidden_def_alias(name, internal) \
+  strong_alias (name, internal)
 #  define hidden_data_def(name)                hidden_def(name)
+#  define hidden_data_def_alias(name, alias) hidden_def_alias(name, alias)
 #  define hidden_tls_def(name)                         \
   __hidden_ver2 (__thread, __GI_##name, name, name);
 #  define hidden_weak(name) \
@@ -581,9 +588,11 @@ for linking")
    hidden_proto doesn't make sense for assembly but the equivalent
    is to call via the HIDDEN_JUMPTARGET macro instead of JUMPTARGET.  */
 #  define hidden_def(name)     strong_alias (name, __GI_##name)
+#  define hidden_def_alias(name, alias) strong_alias (name, alias)
 #  define hidden_weak(name)    hidden_def (name)
 #  define hidden_ver(local, name) strong_alias (local, __GI_##name)
 #  define hidden_data_def(name)        strong_data_alias (name, __GI_##name)
+#  define hidden_data_def_alias(name, alias) strong_data_alias (name, alias)
 #  define hidden_tls_def(name) hidden_data_def (name)
 #  define hidden_data_weak(name)       hidden_data_def (name)
 #  define hidden_data_ver(local, name) strong_data_alias (local, __GI_##name)
@@ -598,12 +607,17 @@ for linking")
   __attribute__ ((visibility ("hidden"), ##attrs))
 #   define hidden_proto(name, attrs...) \
   __hidden_proto (name, , name, ##attrs)
+#  define hidden_proto_alias(name, alias, attrs...) \
+  __hidden_proto_alias (name, , alias, ##attrs)
 #   define hidden_tls_proto(name, attrs...) \
   __hidden_proto (name, __thread, name, ##attrs)
 #  define __hidden_proto(name, thread, internal, attrs...)          \
   extern thread __typeof (name) name __hidden_proto_hiddenattr (attrs);
+#  define __hidden_proto_alias(name, thread, internal, attrs...)     \
+  extern thread __typeof (name) internal __hidden_proto_hiddenattr (attrs);
 # else
 #   define hidden_proto(name, attrs...)
+#   define hidden_proto_alias(name, alias, attrs...)
 #   define hidden_tls_proto(name, attrs...)
 # endif
 # else
@@ -611,9 +625,11 @@ for linking")
 # endif /* Not  __ASSEMBLER__ */
 # define hidden_weak(name)
 # define hidden_def(name)
+# define hidden_def_alias(name, alias)
 # define hidden_ver(local, name)
 # define hidden_data_weak(name)
 # define hidden_data_def(name)
+# define hidden_data_def_alias(name, alias)
 # define hidden_tls_def(name)
 # define hidden_data_ver(local, name)
 # define hidden_nolink(name, lib, version)
@@ -621,22 +637,28 @@ for linking")
 
 #if IS_IN (libc)
 # define libc_hidden_proto(name, attrs...) hidden_proto (name, ##attrs)
+# define libc_hidden_proto_alias(name, alias, attrs...) \
+   hidden_proto_alias (name, alias, ##attrs)
 # define libc_hidden_tls_proto(name, attrs...) hidden_tls_proto (name, ##attrs)
 # define libc_hidden_def(name) hidden_def (name)
+# define libc_hidden_def_alias(name, alias) hidden_def_alias (name, alias)
 # define libc_hidden_weak(name) hidden_weak (name)
 # define libc_hidden_nolink_sunrpc(name, version) hidden_nolink (name, libc, version)
 # define libc_hidden_ver(local, name) hidden_ver (local, name)
 # define libc_hidden_data_def(name) hidden_data_def (name)
+# define libc_hidden_data_def_alias(name, alias) hidden_data_def_alias (name, alias)
 # define libc_hidden_tls_def(name) hidden_tls_def (name)
 # define libc_hidden_data_weak(name) hidden_data_weak (name)
 # define libc_hidden_data_ver(local, name) hidden_data_ver (local, name)
 #else
 # define libc_hidden_proto(name, attrs...)
+# define libc_hidden_proto_alias(name, alias, attrs...)
 # define libc_hidden_tls_proto(name, attrs...)
 # define libc_hidden_def(name)
 # define libc_hidden_weak(name)
 # define libc_hidden_ver(local, name)
 # define libc_hidden_data_def(name)
+# define libc_hidden_data_def_alias(name, alias)
 # define libc_hidden_tls_def(name)
 # define libc_hidden_data_weak(name)
 # define libc_hidden_data_ver(local, name)

diff --git a/include/sys/single_threaded.h b/include/sys/single_threaded.h
index 18f69724828b1b888d9f80d9b1cce7c374b87fd9..2015742be07fc5e7363159b5d2f7e1a881d4cabb 100644 (file)
--- a/include/sys/single_threaded.h
+++ b/include/sys/single_threaded.h
@@ -1 +1,12 @@
 #include <misc/sys/single_threaded.h>
+
+#ifndef _ISOMAC
+
+libc_hidden_proto_alias (__libc_single_threaded,
+                        __libc_single_threaded_internal);
+
+#if !defined SHARED || !IS_IN(libc)
+# define __libc_single_threaded_internal __libc_single_threaded
+#endif
+
+#endif

diff --git a/misc/single_threaded.c b/misc/single_threaded.c
index 96ada9137b453ca1091f37f381dfbac71943ecbf..9b0746a69cba9c421ebef2a0b9f3cc22245a1247 100644 (file)
--- a/misc/single_threaded.c
+++ b/misc/single_threaded.c
@@ -25,3 +25,5 @@ char __libc_single_threaded;
 #else
 char __libc_single_threaded = 1;
 #endif
+libc_hidden_data_def_alias (__libc_single_threaded,
+                           __libc_single_threaded_internal)

diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index e7a099acb7ba903a8a414211c711cafa96f94d9b..a06d611e32c64f363afb95d45335b5e2d5b537bc 100644 (file)
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -624,9 +624,12 @@ __pthread_create_2_1 (pthread_t *newthread, const pthread_attr_t *attr,
 
   /* Avoid a data race in the multi-threaded case, and call the
      deferred initialization only once.  */
-  if (__libc_single_threaded)
+  if (__libc_single_threaded_internal)
     {
       late_init ();
+      __libc_single_threaded_internal = 0;
+      /* __libc_single_threaded can be accessed through copy relocations, so
+        it requires to update the external copy.  */
       __libc_single_threaded = 0;
     }
 

diff --git a/posix/fork.c b/posix/fork.c
index e1be3422ea458aa8f0af4d11fc5751147d55a1c1..987916a175accdbb487416de041699a096bf17c5 100644 (file)
--- a/posix/fork.c
+++ b/posix/fork.c
@@ -45,7 +45,7 @@ __libc_fork (void)
      requirement for fork (Austin Group tracker issue #62) this is
      best effort to make is async-signal-safe at least for single-thread
      case.  */
-  bool multiple_threads = __libc_single_threaded == 0;
+  bool multiple_threads = __libc_single_threaded_internal == 0;
   uint64_t lastrun;
 
   lastrun = __run_prefork_handlers (multiple_threads);

diff --git a/sysdeps/nptl/setxid.h b/sysdeps/nptl/setxid.h
index b821d6bc074876dcfbc3ba44ac03d94268e5399f..b87cad7b18529857618015793492a8c8fc4d9c0f 100644 (file)
--- a/sysdeps/nptl/setxid.h
+++ b/sysdeps/nptl/setxid.h
@@ -29,7 +29,7 @@
 #define INLINE_SETXID_SYSCALL(name, nr, args...) \
   ({                                                                   \
     int __result;                                                      \
-    if (!__libc_single_threaded)                                       \
+    if (!__libc_single_threaded_internal)                              \
       {                                                                        \
        struct xid_command __cmd;                                       \
        __cmd.syscall_no = __NR_##name;                                 \