x86_64: Use __seg_fs qualifiers in NPTL accessors

Use __seg_fs named address space qualifiers to cast NPTL accessors
to %fs: prefixed addresses.  Use volatile access only where
strictly necessary.

Use existing assembly RSEQ_* accessors for x32 to
work around the GCC bug:

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121613

because negative value in __rseq_offset is used
as an offset from %fs.

Co-Authored-By: H.J. Lu <hjl.tools@gmail.com>
Signed-off-by: H.J. Lu <hjl.tools@gmail.com>
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Cc: Florian Weimer <fweimer@redhat.com>
Cc: Carlos O'Donell <carlos@redhat.com>

diff --git a/sysdeps/unix/sysv/linux/x86_64/64/Implies b/sysdeps/unix/sysv/linux/x86_64/64/Implies
new file mode 100644 (file)
index 0000000..f379d95
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/64/Implies
@@ -0,0 +1 @@
+x86_64/64/nptl

diff --git a/sysdeps/x86_64/64/nptl/rseq-access.h b/sysdeps/x86_64/64/nptl/rseq-access.h
new file mode 100644 (file)
index 0000000..214cb96
--- /dev/null
+++ b/sysdeps/x86_64/64/nptl/rseq-access.h
@@ -0,0 +1,61 @@
+/* RSEQ_* accessors.  x86_64 version.
+   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Read member of the RSEQ area directly.  */
+#define RSEQ_GETMEM(member) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     (*(__typeof (RSEQ_SELF()->member) __seg_fs *)                           \
+      (__rseq_offset + offsetof (struct rseq_area, member)));                \
+  })
+
+/* Read member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_GETMEM_ONCE(member) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *)                  \
+      (__rseq_offset + offsetof (struct rseq_area, member)));                \
+  })
+
+/* Set member of the RSEQ area directly.  */
+#define RSEQ_SETMEM(member, value) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     (*(__typeof (RSEQ_SELF()->member) __seg_fs *)                           \
+      (__rseq_offset + offsetof (struct rseq_area, member)) = (value));              \
+  })
+
+/* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_SETMEM_ONCE(member, value) \
+  ({                                                                         \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1                       \
+                    || sizeof (RSEQ_SELF()->member) == 4                     \
+                    || sizeof (RSEQ_SELF()->member) == 8,                    \
+                    "size of rseq data");                                    \
+     (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *)                  \
+      (__rseq_offset + offsetof (struct rseq_area, member)) = (value));              \
+  })

diff --git a/sysdeps/x86_64/nptl/tcb-access.h b/sysdeps/x86_64/nptl/tcb-access.h
index defd76f54fb5c1d91cd8034078bcda420a984698..445f328ac209f1c9dea0877e908dc8f2ee909fc7 100644 (file)
--- a/sysdeps/x86_64/nptl/tcb-access.h
+++ b/sysdeps/x86_64/nptl/tcb-access.h
@@ -17,116 +17,55 @@
    <https://www.gnu.org/licenses/>.  */
 
 /* Read member of the thread descriptor directly.  */
-# define THREAD_GETMEM(descr, member) \
-  ({ __typeof (descr->member) __value;                                       \
-     _Static_assert (sizeof (__value) == 1                                   \
-                    || sizeof (__value) == 4                                 \
-                    || sizeof (__value) == 8,                                \
+#define THREAD_GETMEM(descr, member) \
+  ({                                                                         \
+     _Static_assert (sizeof (descr->member) == 1                                     \
+                    || sizeof (descr->member) == 4                           \
+                    || sizeof (descr->member) == 8,                          \
                     "size of per-thread data");                              \
-     if (sizeof (__value) == 1)                                                      \
-       asm volatile ("movb %%fs:%P2,%b0"                                     \
-                    : "=q" (__value)                                         \
-                    : "0" (0), "i" (offsetof (struct pthread, member)));     \
-     else if (sizeof (__value) == 4)                                         \
-       asm volatile ("movl %%fs:%P1,%0"                                              \
-                    : "=r" (__value)                                         \
-                    : "i" (offsetof (struct pthread, member)));              \
-     else /* 8 */                                                                    \
-       {                                                                     \
-        asm volatile ("movq %%fs:%P1,%q0"                                    \
-                      : "=r" (__value)                                       \
-                      : "i" (offsetof (struct pthread, member)));            \
-       }                                                                     \
-     __value; })
+     (*(__typeof (descr->member) __seg_fs *)                                 \
+      offsetof (struct pthread, member));                                    \
+  })
 
-/* THREAD_GETMEM already forces a read.  */
-#define THREAD_GETMEM_VOLATILE(descr, member) THREAD_GETMEM (descr, member)
+#define THREAD_GETMEM_VOLATILE(descr, member) \
+  ({                                                                         \
+     _Static_assert (sizeof (descr->member) == 1                             \
+                    || sizeof (descr->member) == 4                           \
+                    || sizeof (descr->member) == 8,                          \
+                    "size of per-thread data");                              \
+     (*(volatile __typeof (descr->member) __seg_fs *)                        \
+      offsetof (struct pthread, member));                                    \
+  })
 
 /* Same as THREAD_GETMEM, but the member offset can be non-constant.  */
-# define THREAD_GETMEM_NC(descr, member, idx) \
-  ({ __typeof (descr->member[0]) __value;                                    \
-     _Static_assert (sizeof (__value) == 1                                   \
-                    || sizeof (__value) == 4                                 \
-                    || sizeof (__value) == 8,                                \
+#define THREAD_GETMEM_NC(descr, member, idx) \
+  ({                                                                         \
+     _Static_assert (sizeof (descr->member[0]) == 1                          \
+                    || sizeof (descr->member[0]) == 4                        \
+                    || sizeof (descr->member[0]) == 8,                       \
                     "size of per-thread data");                              \
-     if (sizeof (__value) == 1)                                                      \
-       asm volatile ("movb %%fs:%P2(%q3),%b0"                                \
-                    : "=q" (__value)                                         \
-                    : "0" (0), "i" (offsetof (struct pthread, member[0])),   \
-                      "r" (idx));                                            \
-     else if (sizeof (__value) == 4)                                         \
-       asm volatile ("movl %%fs:%P1(,%q2,4),%0"                                      \
-                    : "=r" (__value)                                         \
-                    : "i" (offsetof (struct pthread, member[0])), "r" (idx));\
-     else /* 8 */                                                            \
-       {                                                                     \
-        asm volatile ("movq %%fs:%P1(,%q2,8),%q0"                            \
-                      : "=r" (__value)                                       \
-                      : "i" (offsetof (struct pthread, member[0])),          \
-                        "r" (idx));                                          \
-       }                                                                     \
-     __value; })
-
-
-/* Loading addresses of objects on x86-64 needs to be treated special
-   when generating PIC code.  */
-#ifdef __pic__
-# define IMM_MODE "nr"
-#else
-# define IMM_MODE "ir"
-#endif
-
+     (*(__typeof (descr->member[0]) __seg_fs *)                                      \
+      offsetof (struct pthread, member[idx]));                               \
+  })
 
 /* Set member of the thread descriptor directly.  */
-# define THREAD_SETMEM(descr, member, value) \
+#define THREAD_SETMEM(descr, member, value) \
   ({                                                                         \
      _Static_assert (sizeof (descr->member) == 1                             \
                     || sizeof (descr->member) == 4                           \
                     || sizeof (descr->member) == 8,                          \
                     "size of per-thread data");                              \
-     if (sizeof (descr->member) == 1)                                        \
-       asm volatile ("movb %b0,%%fs:%P1" :                                   \
-                    : "iq" (value),                                          \
-                      "i" (offsetof (struct pthread, member)));              \
-     else if (sizeof (descr->member) == 4)                                   \
-       asm volatile ("movl %0,%%fs:%P1" :                                    \
-                    : IMM_MODE (value),                                      \
-                      "i" (offsetof (struct pthread, member)));              \
-     else /* 8 */                                                            \
-       {                                                                     \
-        /* Since movq takes a signed 32-bit immediate or a register source   \
-           operand, use "er" constraint for 32-bit signed integer constant   \
-           or register.  */                                                  \
-        asm volatile ("movq %q0,%%fs:%P1" :                                  \
-                      : "er" ((uint64_t) cast_to_integer (value)),           \
-                        "i" (offsetof (struct pthread, member)));            \
-       }})
-
+     (*(__typeof (descr->member) __seg_fs *)                                 \
+      offsetof (struct pthread, member) = (value));                          \
+  })
 
 /* Same as THREAD_SETMEM, but the member offset can be non-constant.  */
-# define THREAD_SETMEM_NC(descr, member, idx, value) \
+#define THREAD_SETMEM_NC(descr, member, idx, value) \
   ({                                                                         \
      _Static_assert (sizeof (descr->member[0]) == 1                          \
                     || sizeof (descr->member[0]) == 4                        \
                     || sizeof (descr->member[0]) == 8,                       \
                     "size of per-thread data");                              \
-     if (sizeof (descr->member[0]) == 1)                                     \
-       asm volatile ("movb %b0,%%fs:%P1(%q2)" :                                      \
-                    : "iq" (value),                                          \
-                      "i" (offsetof (struct pthread, member[0])),            \
-                      "r" (idx));                                            \
-     else if (sizeof (descr->member[0]) == 4)                                \
-       asm volatile ("movl %0,%%fs:%P1(,%q2,4)" :                            \
-                    : IMM_MODE (value),                                      \
-                      "i" (offsetof (struct pthread, member[0])),            \
-                      "r" (idx));                                            \
-     else /* 8 */                                                            \
-       {                                                                     \
-        /* Since movq takes a signed 32-bit immediate or a register source   \
-           operand, use "er" constraint for 32-bit signed integer constant   \
-           or register.  */                                                  \
-        asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" :                          \
-                      : "er" ((uint64_t) cast_to_integer (value)),           \
-                        "i" (offsetof (struct pthread, member[0])),          \
-                        "r" (idx));                                          \
-       }})
+     (*(__typeof (descr->member[0]) __seg_fs *)                                      \
+      offsetof (struct pthread, member[idx]) = (value));                     \
+  })

diff --git a/sysdeps/x86_64/nptl/rseq-access.h b/sysdeps/x86_64/x32/nptl/rseq-access.h
similarity index 92%
rename from sysdeps/x86_64/nptl/rseq-access.h
rename to sysdeps/x86_64/x32/nptl/rseq-access.h
index bc966b2972d18ee524626d85bd17096beb2c5e98..8386ebd4bfdcbdec4d6a0cea73971e9deaea18ac 100644 (file)
--- a/sysdeps/x86_64/nptl/rseq-access.h
+++ b/sysdeps/x86_64/x32/nptl/rseq-access.h
@@ -1,5 +1,5 @@
-/* RSEQ_* accessors.  x86_64 version.
-   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+/* RSEQ_* accessors.  x32 version.
+   Copyright (C) 2025 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -45,6 +45,14 @@
 /* Read member of the RSEQ area directly.  */
 #define RSEQ_GETMEM(member) RSEQ_GETMEM_ONCE(member)
 
+/* Loading addresses of objects on x86-64 needs to be treated special
+   when generating PIC code.  */
+#ifdef __pic__
+# define IMM_MODE "nr"
+#else
+# define IMM_MODE "ir"
+#endif
+
 /* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
 #define RSEQ_SETMEM_ONCE(member, value) \
   ({                                                                         \