linux: Inline __syscall_internal_cancel and __syscall_cancel

It improves some interception tools such as valgrind, however on
multithread the __syscall_cancel_arch is called.

The result libc.so has a slight larger code size:

ABI             master        patched         diff        increase
aarch64        1658673        1669121        10448           0.63%
x86_64         1976656        1985744         9088           0.46%
i686           2233622        2251130        17508           0.78%
powerpc64le    2382448        2396768        14320           0.60%

It mimics internally how cancellable entrypoints were implemented
before 89b53077d2a58f00e7debdfe58afabe953dac60d, where cancellation
handlign were done inline in the syscall wraper.

diff --git a/elf/Makefile b/elf/Makefile
index ed1b0223da67fcbdecebb6018420bf7bf1428edd..de7020b3bbf6aa8bbc645d31a06ffe37b121f2f2 100644 (file)
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -1467,7 +1467,9 @@ $(objpfx)dl-allobjs.os: $(all-rtld-routines:%=$(objpfx)%.os)
 # when compiled for libc.
 rtld-stubbed-symbols = \
   __libc_assert_fail \
-  __syscall_cancel \
+  __libc_single_threaded_internal \
+  __syscall_cancel_arch \
+  __syscall_do_cancel \
   calloc \
   free \
   malloc \

diff --git a/nptl/cancellation.c b/nptl/cancellation.c
index 156e63dcf0dc9d9c858deb15c5aca4f00c7c190a..7aa6ea77544890d80df3805e88f331f82a792f93 100644 (file)
--- a/nptl/cancellation.c
+++ b/nptl/cancellation.c
@@ -19,66 +19,6 @@
 #include <stdlib.h>
 #include "pthreadP.h"
 
-/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
-   returns the syscall value or its negative error code.  */
-long int
-__internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-                          __syscall_arg_t a3, __syscall_arg_t a4,
-                          __syscall_arg_t a5, __syscall_arg_t a6,
-                          __SYSCALL_CANCEL7_ARG_DEF
-                          __syscall_arg_t nr)
-{
-  long int result;
-  struct pthread *pd = THREAD_SELF;
-
-  /* If cancellation is not enabled, call the syscall directly and also
-     for thread terminatation to avoid call __syscall_do_cancel while
-     executing cleanup handlers.  */
-  int ch = atomic_load_relaxed (&pd->cancelhandling);
-  if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
-    {
-      result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
-                                         __SYSCALL_CANCEL7_ARCH_ARG7);
-      if (INTERNAL_SYSCALL_ERROR_P (result))
-       return -INTERNAL_SYSCALL_ERRNO (result);
-      return result;
-    }
-
-  /* Call the arch-specific entry points that contains the globals markers
-     to be checked by SIGCANCEL handler.  */
-  result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
-                                 a6 __SYSCALL_CANCEL7_ARCH_ARG7);
-
-  /* If the cancellable syscall was interrupted by SIGCANCEL and it has no
-     side-effect, cancel the thread if cancellation is enabled.  */
-  ch = atomic_load_relaxed (&pd->cancelhandling);
-  /* The behaviour here assumes that EINTR is returned only if there are no
-     visible side effects.  POSIX Issue 7 has not yet provided any stronger
-     language for close, and in theory the close syscall could return EINTR
-     and leave the file descriptor open (conforming and leaks).  It expects
-     that no such kernel is used with glibc.  */
-  if (result == -EINTR && cancel_enabled_and_canceled (ch))
-    __syscall_do_cancel ();
-
-  return result;
-}
-
-/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
-   syscall expected success value (usually 0) or, in case of failure, -1 and
-   sets errno to syscall return value.  */
-long int
-__syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-                 __syscall_arg_t a3, __syscall_arg_t a4,
-                 __syscall_arg_t a5, __syscall_arg_t a6,
-                 __SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
-{
-  int r = __internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
-                                    __SYSCALL_CANCEL7_ARG nr);
-  return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
-        ? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
-        : r;
-}
-
 /* Called by __syscall_cancel_arch or function above start the thread
    cancellation.  */
 _Noreturn void

diff --git a/nptl/futex-internal.c b/nptl/futex-internal.c
index b81ca0700addf759fa9297d7ebeb6e858c2dfe81..6d8ef7335bcd9a9390f503f40b35899541a55f2f 100644 (file)
--- a/nptl/futex-internal.c
+++ b/nptl/futex-internal.c
@@ -17,7 +17,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <errno.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <time.h>
 #include <futex-internal.h>
 #include <kernel-features.h>

diff --git a/nptl/sem_waitcommon.c b/nptl/sem_waitcommon.c
index 95450ee35d0fa13545626702941eeffd36633663..0d59e81da1d142028b752c6b0b19076b7b611047 100644 (file)
--- a/nptl/sem_waitcommon.c
+++ b/nptl/sem_waitcommon.c
@@ -18,7 +18,7 @@
 
 #include <kernel-features.h>
 #include <errno.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <futex-internal.h>
 #include <internaltypes.h>
 #include <semaphore.h>

diff --git a/sysdeps/unix/sysdep.h b/sysdeps/unix/sysdep.h
index 2cc98725c386f2f00e46cdca66a80d4a4beebb57..8c1642f7a4757c0f6a9103686aca065435d2064d 100644 (file)
--- a/sysdeps/unix/sysdep.h
+++ b/sysdeps/unix/sysdep.h
@@ -154,42 +154,31 @@
 # define __SYSCALL_CANCEL7_ARG7
 # define __SYSCALL_CANCEL7_ARCH_ARG7
 #endif
-long int __internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
-                                   __syscall_arg_t a3, __syscall_arg_t a4,
-                                   __syscall_arg_t a5, __syscall_arg_t a6,
-                                   __SYSCALL_CANCEL7_ARG_DEF
-                                   __syscall_arg_t nr) attribute_hidden;
-
-long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
-                          __syscall_arg_t arg3, __syscall_arg_t arg4,
-                          __syscall_arg_t arg5, __syscall_arg_t arg6,
-                          __SYSCALL_CANCEL7_ARG_DEF
-                          __syscall_arg_t nr) attribute_hidden;
 
 #define __SYSCALL_CANCEL0(name)                                                \
-  __syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL1(name, a1)                                    \
-  __syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,                         \
-                   __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,                           \
+                 __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL2(name, a1, a2) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,                        \
-                   __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,                  \
+                 __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL3(name, a1, a2, a3) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0,       \
-                   __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0, \
+                 __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL4(name, a1, a2, a3, a4) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),                        \
-                   __SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),                  \
+                 __SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4),     \
-                   __SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4),       \
+                 __SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),    \
-                   __SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG       \
-                   __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),      \
+                 __SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG \
+                 __NR_##name)
 #define __SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7)            \
-  __syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),    \
-                   __SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
+  syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4),      \
+                 __SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
 
 #define __SYSCALL_CANCEL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n
 #define __SYSCALL_CANCEL_NARGS(...) \
@@ -206,33 +195,33 @@ long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
   __SYSCALL_CANCEL_DISP (__SYSCALL_CANCEL, __VA_ARGS__)
 
 #define __INTERNAL_SYSCALL_CANCEL0(name)                               \
-  __internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG   \
+  internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG     \
                             __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL1(name, a1)                           \
-  __internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,                        \
-                            __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0,                  \
+                          __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL2(name, a1, a2)                       \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0,       \
-                            __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \
+                          __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL3(name, a1, a2, a3)                   \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0,    \
-                            0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0,      \
+                          0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL4(name, a1, a2, a3, a4)               \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),       \
-                            __SSC(a4), 0, 0,                           \
-                            __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
+                          __SSC(a4), 0, 0,                             \
+                          __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5)           \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),       \
-                            __SSC(a4), __SSC (a5), 0,                  \
-                            __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
+                          __SSC(a4), __SSC (a5), 0,                    \
+                          __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6)       \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),       \
-                            __SSC (a4), __SSC (a5), __SSC (a6),        \
-                            __SYSCALL_CANCEL7_ARG __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
+                          __SSC (a4), __SSC (a5), __SSC (a6),  \
+                          __SYSCALL_CANCEL7_ARG __NR_##name)
 #define __INTERNAL_SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \
-  __internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),     \
-                            __SSC (a4), __SSC (a5), __SSC (a6),     \
-                            __SSC (a7), __NR_##name)
+  internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3),     \
+                          __SSC (a4), __SSC (a5), __SSC (a6),     \
+                          __SSC (a7), __NR_##name)
 
 /* Issue a cancellable syscall defined by syscall number NAME plus any other
    argument required.  If an error occurs its value is returned as an negative

diff --git a/sysdeps/unix/sysv/linux/epoll_pwait2.c b/sysdeps/unix/sysv/linux/epoll_pwait2.c
index bbb5f044a6be5429fb7660817fcba083a3ba6cbe..a58a3597d9ff94b8e181d34d9f016dd9160d7a4c 100644 (file)
--- a/sysdeps/unix/sysv/linux/epoll_pwait2.c
+++ b/sysdeps/unix/sysv/linux/epoll_pwait2.c
@@ -17,7 +17,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <sys/epoll.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 
 int
 __epoll_pwait2_time64 (int fd, struct epoll_event *ev, int maxev,

diff --git a/sysdeps/unix/sysv/linux/recvmmsg.c b/sysdeps/unix/sysv/linux/recvmmsg.c
index 3b00bb2fe09cfd537721e0cfeb47b29322825699..7ce2de65a29f631acffd8c8d6d5447ab92d3ad40 100644 (file)
--- a/sysdeps/unix/sysv/linux/recvmmsg.c
+++ b/sysdeps/unix/sysv/linux/recvmmsg.c
@@ -16,7 +16,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <sys/socket.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 #include <socketcall.h>
 
 static int

diff --git a/sysdeps/unix/sysv/linux/sigtimedwait.c b/sysdeps/unix/sysv/linux/sigtimedwait.c
index ad76628890cbbd1a0976d452dde5f5a7bebabfa1..5781c63aa6d91bbdca1360a5840321e86c91ede3 100644 (file)
--- a/sysdeps/unix/sysv/linux/sigtimedwait.c
+++ b/sysdeps/unix/sysv/linux/sigtimedwait.c
@@ -16,7 +16,7 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include <signal.h>
-#include <sysdep.h>
+#include <sysdep-cancel.h>
 
 int
 __sigtimedwait64 (const sigset_t *set, siginfo_t *info,

diff --git a/sysdeps/unix/sysv/linux/sysdep-cancel.h b/sysdeps/unix/sysv/linux/sysdep-cancel.h
index 96c9a904b8311ce2113eb8d60ed57eeaeea45a3a..6e3f0455c275fe46fb6e02dc5f7fdf4f784492ee 100644 (file)
--- a/sysdeps/unix/sysv/linux/sysdep-cancel.h
+++ b/sysdeps/unix/sysv/linux/sysdep-cancel.h
@@ -1,4 +1,4 @@
-/* Single-thread optimization definitions.  Linux version.
+/* Cancellable syscall definitions  Linux version.
    Copyright (C) 2017-2025 Free Software Foundation, Inc.
 
    This file is part of the GNU C Library.
@@ -21,5 +21,66 @@
 #define _SYSDEP_CANCEL_H
 
 #include <sysdep.h>
+#include "pthreadP.h"
+
+/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
+   returns the syscall value or its negative error code.  */
+static __always_inline long int
+internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
+                        __syscall_arg_t a3, __syscall_arg_t a4,
+                        __syscall_arg_t a5, __syscall_arg_t a6,
+                        __SYSCALL_CANCEL7_ARG_DEF
+                        __syscall_arg_t nr)
+{
+  long int result;
+  struct pthread *pd = THREAD_SELF;
+
+  /* If cancellation is not enabled, call the syscall directly and also
+     for thread terminatation to avoid call __syscall_do_cancel while
+     executing cleanup handlers.  */
+  int ch = atomic_load_relaxed (&pd->cancelhandling);
+  if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
+    {
+      result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
+                                         __SYSCALL_CANCEL7_ARCH_ARG7);
+      if (INTERNAL_SYSCALL_ERROR_P (result))
+       return -INTERNAL_SYSCALL_ERRNO (result);
+      return result;
+    }
+
+  /* Call the arch-specific entry points that contains the globals markers
+     to be checked by SIGCANCEL handler.  */
+  result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
+                                 a6 __SYSCALL_CANCEL7_ARCH_ARG7);
+
+  /* If the cancellable syscall was interrupted by SIGCANCEL and it has no
+     side-effect, cancel the thread if cancellation is enabled.  */
+  ch = atomic_load_relaxed (&pd->cancelhandling);
+  /* The behaviour here assumes that EINTR is returned only if there are no
+     visible side effects.  POSIX Issue 7 has not yet provided any stronger
+     language for close, and in theory the close syscall could return EINTR
+     and leave the file descriptor open (conforming and leaks).  It expects
+     that no such kernel is used with glibc.  */
+  if (result == -EINTR && cancel_enabled_and_canceled (ch))
+    __syscall_do_cancel ();
+
+  return result;
+}
+
+/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
+   syscall expected success value (usually 0) or, in case of failure, -1 and
+   sets errno to syscall return value.  */
+static __always_inline long int
+syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
+               __syscall_arg_t a3, __syscall_arg_t a4,
+               __syscall_arg_t a5, __syscall_arg_t a6,
+               __SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
+{
+  int r = internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
+                                  __SYSCALL_CANCEL7_ARG nr);
+  return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
+        ? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
+        : r;
+}
 
 #endif