nptl: Introduce <rseq-access.h> for RSEQ_* accessors

In preparation to move the rseq area to the 'extra TLS' block, we need
accessors based on the thread pointer and the rseq offset. The ONCE
variant of the accessors ensures single-copy atomicity for loads and
stores which is required for all fields once the registration is active.

A separate header is required to allow including <atomic.h> which
results in an include loop when added to <tcb-access.h>.

Signed-off-by: Michael Jeanson <mjeanson@efficios.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Florian Weimer <fweimer@redhat.com>

diff --git a/sysdeps/i386/nptl/rseq-access.h b/sysdeps/i386/nptl/rseq-access.h
new file mode 100644 (file)
index 0000000..5e7e09d
--- /dev/null
+++ b/sysdeps/i386/nptl/rseq-access.h
@@ -0,0 +1,98 @@
+/* RSEQ_* accessors.  i386 version.
+   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define __RSEQ_GETMEM(member) \
+  ({ __typeof (RSEQ_SELF()->member) __value;                                 \
+     if (sizeof (__value) == 1)                                                      \
+       asm volatile ("movb %%gs:%P2(%3),%b0"                                 \
+                    : "=q" (__value)                                         \
+                    : "0" (0), "i" (offsetof (struct rseq_area, member)),   \
+                    "r" (__rseq_offset));                                    \
+     else if (sizeof (__value) == 4)                                         \
+       asm volatile ("movl %%gs:%P1(%2),%0"                                  \
+                    : "=r" (__value)                                         \
+                    : "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else /* 8 */                                                            \
+       {                                                                     \
+        asm volatile  ("movl %%gs:%P1(%2),%%eax\n\t"                         \
+                       "movl %%gs:4+%P1(%2),%%edx"                           \
+                       : "=&A" (__value)                                     \
+                       : "i" (offsetof (struct rseq_area, member)),          \
+                         "r" (__rseq_offset));                               \
+       }                                                                     \
+     __value; })
+
+/* Read member of the RSEQ area directly.  */
+#define RSEQ_GETMEM(member) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     __RSEQ_GETMEM(member); })
+
+/* Read member of the RSEQ area directly, with single-copy atomicity semantics.
+   Static assert for types >= 64 bits since they can't be loaded atomically on
+   x86-32.  */
+#define RSEQ_GETMEM_ONCE(member) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4,                    \
+                    "size of rseq data");                                    \
+     __RSEQ_GETMEM(member); })
+
+#define __RSEQ_SETMEM(member, value) \
+  ({                                                                         \
+     if (sizeof (RSEQ_SELF()->member) == 1)                                  \
+       asm volatile ("movb %b0,%%gs:%P1(%2)" :                               \
+                    : "iq" (value),                                          \
+                      "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else if (sizeof (RSEQ_SELF()->member) == 4)                             \
+       asm volatile ("movl %0,%%gs:%P1(%2)" :                                \
+                    : "ir" (value),                                          \
+                      "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else /* 8 */                                                            \
+       {                                                                     \
+        asm volatile ("movl %%eax,%%gs:%P1(%2)\n\t"                          \
+                      "movl %%edx,%%gs:4+%P1(%2)" :                          \
+                      : "A" ((uint64_t) cast_to_integer (value)),            \
+                        "i" (offsetof (struct rseq_area, member)),           \
+                        "r" (__rseq_offset));                                \
+       }})
+
+/* Set member of the RSEQ area directly.  */
+#define RSEQ_SETMEM(member, value) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     __RSEQ_SETMEM(member, value); })
+
+/* Set member of the RSEQ area directly, with single-copy atomicity semantics.
+   Static assert for types >= 64 bits since they can't be stored atomically on
+   x86-32.  */
+#define RSEQ_SETMEM_ONCE(member, value) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4,                    \
+                    "size of rseq data");                                    \
+     __RSEQ_SETMEM(member, value); })

diff --git a/sysdeps/nptl/rseq-access.h b/sysdeps/nptl/rseq-access.h
new file mode 100644 (file)
index 0000000..450f2dc
--- /dev/null
+++ b/sysdeps/nptl/rseq-access.h
@@ -0,0 +1,56 @@
+/* RSEQ_* accessors.  Generic version.
+   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <atomic.h>
+
+/* Read member of the RSEQ area directly.  */
+#define RSEQ_GETMEM(member) \
+  RSEQ_SELF()->member
+
+/* Set member of the RSEQ area directly.  */
+#define RSEQ_SETMEM(member, value) \
+  RSEQ_SELF()->member = (value)
+
+/* Static assert for types that can't be loaded/stored atomically on the
+   current architecture.  */
+#if __HAVE_64B_ATOMICS
+#define __RSEQ_ASSERT_ATOMIC(member) \
+   _Static_assert (sizeof (RSEQ_SELF()->member) == 1                         \
+                  || sizeof (RSEQ_SELF()->member) == 4                       \
+                  || sizeof (RSEQ_SELF()->member) == 8,                      \
+                  "size of rseq data")
+#else
+#define __RSEQ_ASSERT_ATOMIC(member) \
+   _Static_assert (sizeof (RSEQ_SELF()->member) == 1                         \
+                  || sizeof (RSEQ_SELF()->member) == 4,                      \
+                  "size of rseq data")
+#endif
+
+/* Read member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_GETMEM_ONCE(member) \
+  ({                                                                         \
+     __RSEQ_ASSERT_ATOMIC(member);                                           \
+     (*(volatile __typeof (RSEQ_SELF()->member) *)&RSEQ_SELF()->member);      \
+  })
+
+/* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_SETMEM_ONCE(member, value) \
+  ({                                                                         \
+     __RSEQ_ASSERT_ATOMIC(member);                                           \
+     (*(volatile __typeof (RSEQ_SELF()->member) *)&RSEQ_SELF()->member = (value)); \
+  })

diff --git a/sysdeps/unix/sysv/linux/rseq-internal.h b/sysdeps/unix/sysv/linux/rseq-internal.h
index 3993431707563199161fbc8836c3fb9e16e5e90f..00be15cfc8cefcd9a7cda60efeb29fb9f51ce7be 100644 (file)
--- a/sysdeps/unix/sysv/linux/rseq-internal.h
+++ b/sysdeps/unix/sysv/linux/rseq-internal.h
@@ -25,6 +25,7 @@
 #include <stdio.h>
 #include <sys/rseq.h>
 #include <ldsodefs.h>
+#include <thread_pointer.h>
 
 /* Minimum size of the rseq area allocation required by the syscall.  The
    actually used rseq feature size may be less (20 bytes initially).  */
@@ -59,6 +60,13 @@ extern ptrdiff_t _rseq_offset attribute_hidden;
 rtld_hidden_proto (__rseq_size)
 rtld_hidden_proto (__rseq_offset)
 
+/* Returns a pointer to the current thread rseq area.  */
+static inline struct rseq_area *
+RSEQ_SELF (void)
+{
+  return (struct rseq_area *) ((char *) __thread_pointer () + __rseq_offset);
+}
+
 #ifdef RSEQ_SIG
 static inline bool
 rseq_register_current_thread (struct pthread *self, bool do_rseq)

diff --git a/sysdeps/x86_64/nptl/rseq-access.h b/sysdeps/x86_64/nptl/rseq-access.h
new file mode 100644 (file)
index 0000000..535e362
--- /dev/null
+++ b/sysdeps/x86_64/nptl/rseq-access.h
@@ -0,0 +1,77 @@
+/* RSEQ_* accessors.  x86_64 version.
+   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Read member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_GETMEM_ONCE(member) \
+  ({ __typeof (RSEQ_SELF()->member) __value;                                 \
+     _Static_assert (sizeof (__value) == 1                                   \
+                    || sizeof (__value) == 4                                 \
+                    || sizeof (__value) == 8,                                \
+                    "size of rseq data");                            \
+     if (sizeof (__value) == 1)                                                      \
+       asm volatile ("movb %%fs:%P2(%q3),%b0"                                \
+                    : "=q" (__value)                                         \
+                    : "0" (0), "i" (offsetof (struct rseq_area, member)),    \
+                      "r" (__rseq_offset));                                  \
+     else if (sizeof (__value) == 4)                                         \
+       asm volatile ("movl %%fs:%P1(%q2),%0"                                 \
+                    : "=r" (__value)                                         \
+                    : "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else /* 8 */                                                            \
+       {                                                                     \
+        asm volatile ("movq %%fs:%P1(%q2),%q0"                               \
+                      : "=r" (__value)                                       \
+                      : "i" (offsetof (struct rseq_area, member)),           \
+                        "r" (__rseq_offset));                                \
+       }                                                                     \
+     __value; })
+
+/* Read member of the RSEQ area directly.  */
+#define RSEQ_GETMEM(member) RSEQ_GETMEM_ONCE(member)
+
+/* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_SETMEM_ONCE(member, value) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                            \
+     if (sizeof (RSEQ_SELF()->member) == 1)                                  \
+       asm volatile ("movb %b0,%%fs:%P1(%q2)" :                                      \
+                    : "iq" (value),                                          \
+                      "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else if (sizeof (RSEQ_SELF()->member) == 4)                             \
+       asm volatile ("movl %0,%%fs:%P1(%q2)" :                               \
+                    : IMM_MODE (value),                                      \
+                      "i" (offsetof (struct rseq_area, member)),             \
+                      "r" (__rseq_offset));                                  \
+     else /* 8 */                                                            \
+       {                                                                     \
+        /* Since movq takes a signed 32-bit immediate or a register source   \
+           operand, use "er" constraint for 32-bit signed integer constant   \
+           or register.  */                                                  \
+        asm volatile ("movq %q0,%%fs:%P1(%q2)" :                             \
+                      : "er" ((uint64_t) cast_to_integer (value)),           \
+                        "i" (offsetof (struct rseq_area, member)),           \
+                        "r" (__rseq_offset));                                \
+       }})
+
+/* Set member of the RSEQ area directly.  */
+#define RSEQ_SETMEM(member, value) RSEQ_SETMEM_ONCE(member, value)