From 3997c50f0bb2a254fe35f26506495204f87e432e Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 22 Aug 2025 07:33:13 +0200 Subject: [PATCH] x86_64: Use __seg_fs qualifiers in NPTL accessors Use __seg_fs named address space qualifiers to cast NPTL accessors to %fs: prefixed addresses. Use volatile access only where strictly necessary. Use existing assembly RSEQ_* accessors for x32 to work around the GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121613 because negative value in __rseq_offset is used as an offset from %fs. Co-Authored-By: H.J. Lu Signed-off-by: H.J. Lu Signed-off-by: Uros Bizjak Reviewed-by: H.J. Lu Cc: Florian Weimer Cc: Carlos O'Donell --- sysdeps/unix/sysv/linux/x86_64/64/Implies | 1 + sysdeps/x86_64/64/nptl/rseq-access.h | 61 ++++++++++ sysdeps/x86_64/nptl/tcb-access.h | 127 +++++--------------- sysdeps/x86_64/{ => x32}/nptl/rseq-access.h | 12 +- 4 files changed, 105 insertions(+), 96 deletions(-) create mode 100644 sysdeps/unix/sysv/linux/x86_64/64/Implies create mode 100644 sysdeps/x86_64/64/nptl/rseq-access.h rename sysdeps/x86_64/{ => x32}/nptl/rseq-access.h (92%) diff --git a/sysdeps/unix/sysv/linux/x86_64/64/Implies b/sysdeps/unix/sysv/linux/x86_64/64/Implies new file mode 100644 index 0000000000..f379d95141 --- /dev/null +++ b/sysdeps/unix/sysv/linux/x86_64/64/Implies @@ -0,0 +1 @@ +x86_64/64/nptl diff --git a/sysdeps/x86_64/64/nptl/rseq-access.h b/sysdeps/x86_64/64/nptl/rseq-access.h new file mode 100644 index 0000000000..214cb96797 --- /dev/null +++ b/sysdeps/x86_64/64/nptl/rseq-access.h @@ -0,0 +1,61 @@ +/* RSEQ_* accessors. x86_64 version. + Copyright (C) 2002-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* Read member of the RSEQ area directly. */ +#define RSEQ_GETMEM(member) \ + ({ \ + _Static_assert (sizeof (RSEQ_SELF()->member) == 1 \ + || sizeof (RSEQ_SELF()->member) == 4 \ + || sizeof (RSEQ_SELF()->member) == 8, \ + "size of rseq data"); \ + (*(__typeof (RSEQ_SELF()->member) __seg_fs *) \ + (__rseq_offset + offsetof (struct rseq_area, member))); \ + }) + +/* Read member of the RSEQ area directly, with single-copy atomicity semantics. */ +#define RSEQ_GETMEM_ONCE(member) \ + ({ \ + _Static_assert (sizeof (RSEQ_SELF()->member) == 1 \ + || sizeof (RSEQ_SELF()->member) == 4 \ + || sizeof (RSEQ_SELF()->member) == 8, \ + "size of rseq data"); \ + (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *) \ + (__rseq_offset + offsetof (struct rseq_area, member))); \ + }) + +/* Set member of the RSEQ area directly. */ +#define RSEQ_SETMEM(member, value) \ + ({ \ + _Static_assert (sizeof (RSEQ_SELF()->member) == 1 \ + || sizeof (RSEQ_SELF()->member) == 4 \ + || sizeof (RSEQ_SELF()->member) == 8, \ + "size of rseq data"); \ + (*(__typeof (RSEQ_SELF()->member) __seg_fs *) \ + (__rseq_offset + offsetof (struct rseq_area, member)) = (value)); \ + }) + +/* Set member of the RSEQ area directly, with single-copy atomicity semantics. */ +#define RSEQ_SETMEM_ONCE(member, value) \ + ({ \ + _Static_assert (sizeof (RSEQ_SELF()->member) == 1 \ + || sizeof (RSEQ_SELF()->member) == 4 \ + || sizeof (RSEQ_SELF()->member) == 8, \ + "size of rseq data"); \ + (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *) \ + (__rseq_offset + offsetof (struct rseq_area, member)) = (value)); \ + }) diff --git a/sysdeps/x86_64/nptl/tcb-access.h b/sysdeps/x86_64/nptl/tcb-access.h index defd76f54f..445f328ac2 100644 --- a/sysdeps/x86_64/nptl/tcb-access.h +++ b/sysdeps/x86_64/nptl/tcb-access.h @@ -17,116 +17,55 @@ . */ /* Read member of the thread descriptor directly. */ -# define THREAD_GETMEM(descr, member) \ - ({ __typeof (descr->member) __value; \ - _Static_assert (sizeof (__value) == 1 \ - || sizeof (__value) == 4 \ - || sizeof (__value) == 8, \ +#define THREAD_GETMEM(descr, member) \ + ({ \ + _Static_assert (sizeof (descr->member) == 1 \ + || sizeof (descr->member) == 4 \ + || sizeof (descr->member) == 8, \ "size of per-thread data"); \ - if (sizeof (__value) == 1) \ - asm volatile ("movb %%fs:%P2,%b0" \ - : "=q" (__value) \ - : "0" (0), "i" (offsetof (struct pthread, member))); \ - else if (sizeof (__value) == 4) \ - asm volatile ("movl %%fs:%P1,%0" \ - : "=r" (__value) \ - : "i" (offsetof (struct pthread, member))); \ - else /* 8 */ \ - { \ - asm volatile ("movq %%fs:%P1,%q0" \ - : "=r" (__value) \ - : "i" (offsetof (struct pthread, member))); \ - } \ - __value; }) + (*(__typeof (descr->member) __seg_fs *) \ + offsetof (struct pthread, member)); \ + }) -/* THREAD_GETMEM already forces a read. */ -#define THREAD_GETMEM_VOLATILE(descr, member) THREAD_GETMEM (descr, member) +#define THREAD_GETMEM_VOLATILE(descr, member) \ + ({ \ + _Static_assert (sizeof (descr->member) == 1 \ + || sizeof (descr->member) == 4 \ + || sizeof (descr->member) == 8, \ + "size of per-thread data"); \ + (*(volatile __typeof (descr->member) __seg_fs *) \ + offsetof (struct pthread, member)); \ + }) /* Same as THREAD_GETMEM, but the member offset can be non-constant. */ -# define THREAD_GETMEM_NC(descr, member, idx) \ - ({ __typeof (descr->member[0]) __value; \ - _Static_assert (sizeof (__value) == 1 \ - || sizeof (__value) == 4 \ - || sizeof (__value) == 8, \ +#define THREAD_GETMEM_NC(descr, member, idx) \ + ({ \ + _Static_assert (sizeof (descr->member[0]) == 1 \ + || sizeof (descr->member[0]) == 4 \ + || sizeof (descr->member[0]) == 8, \ "size of per-thread data"); \ - if (sizeof (__value) == 1) \ - asm volatile ("movb %%fs:%P2(%q3),%b0" \ - : "=q" (__value) \ - : "0" (0), "i" (offsetof (struct pthread, member[0])), \ - "r" (idx)); \ - else if (sizeof (__value) == 4) \ - asm volatile ("movl %%fs:%P1(,%q2,4),%0" \ - : "=r" (__value) \ - : "i" (offsetof (struct pthread, member[0])), "r" (idx));\ - else /* 8 */ \ - { \ - asm volatile ("movq %%fs:%P1(,%q2,8),%q0" \ - : "=r" (__value) \ - : "i" (offsetof (struct pthread, member[0])), \ - "r" (idx)); \ - } \ - __value; }) - - -/* Loading addresses of objects on x86-64 needs to be treated special - when generating PIC code. */ -#ifdef __pic__ -# define IMM_MODE "nr" -#else -# define IMM_MODE "ir" -#endif - + (*(__typeof (descr->member[0]) __seg_fs *) \ + offsetof (struct pthread, member[idx])); \ + }) /* Set member of the thread descriptor directly. */ -# define THREAD_SETMEM(descr, member, value) \ +#define THREAD_SETMEM(descr, member, value) \ ({ \ _Static_assert (sizeof (descr->member) == 1 \ || sizeof (descr->member) == 4 \ || sizeof (descr->member) == 8, \ "size of per-thread data"); \ - if (sizeof (descr->member) == 1) \ - asm volatile ("movb %b0,%%fs:%P1" : \ - : "iq" (value), \ - "i" (offsetof (struct pthread, member))); \ - else if (sizeof (descr->member) == 4) \ - asm volatile ("movl %0,%%fs:%P1" : \ - : IMM_MODE (value), \ - "i" (offsetof (struct pthread, member))); \ - else /* 8 */ \ - { \ - /* Since movq takes a signed 32-bit immediate or a register source \ - operand, use "er" constraint for 32-bit signed integer constant \ - or register. */ \ - asm volatile ("movq %q0,%%fs:%P1" : \ - : "er" ((uint64_t) cast_to_integer (value)), \ - "i" (offsetof (struct pthread, member))); \ - }}) - + (*(__typeof (descr->member) __seg_fs *) \ + offsetof (struct pthread, member) = (value)); \ + }) /* Same as THREAD_SETMEM, but the member offset can be non-constant. */ -# define THREAD_SETMEM_NC(descr, member, idx, value) \ +#define THREAD_SETMEM_NC(descr, member, idx, value) \ ({ \ _Static_assert (sizeof (descr->member[0]) == 1 \ || sizeof (descr->member[0]) == 4 \ || sizeof (descr->member[0]) == 8, \ "size of per-thread data"); \ - if (sizeof (descr->member[0]) == 1) \ - asm volatile ("movb %b0,%%fs:%P1(%q2)" : \ - : "iq" (value), \ - "i" (offsetof (struct pthread, member[0])), \ - "r" (idx)); \ - else if (sizeof (descr->member[0]) == 4) \ - asm volatile ("movl %0,%%fs:%P1(,%q2,4)" : \ - : IMM_MODE (value), \ - "i" (offsetof (struct pthread, member[0])), \ - "r" (idx)); \ - else /* 8 */ \ - { \ - /* Since movq takes a signed 32-bit immediate or a register source \ - operand, use "er" constraint for 32-bit signed integer constant \ - or register. */ \ - asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" : \ - : "er" ((uint64_t) cast_to_integer (value)), \ - "i" (offsetof (struct pthread, member[0])), \ - "r" (idx)); \ - }}) + (*(__typeof (descr->member[0]) __seg_fs *) \ + offsetof (struct pthread, member[idx]) = (value)); \ + }) diff --git a/sysdeps/x86_64/nptl/rseq-access.h b/sysdeps/x86_64/x32/nptl/rseq-access.h similarity index 92% rename from sysdeps/x86_64/nptl/rseq-access.h rename to sysdeps/x86_64/x32/nptl/rseq-access.h index bc966b2972..8386ebd4bf 100644 --- a/sysdeps/x86_64/nptl/rseq-access.h +++ b/sysdeps/x86_64/x32/nptl/rseq-access.h @@ -1,5 +1,5 @@ -/* RSEQ_* accessors. x86_64 version. - Copyright (C) 2002-2025 Free Software Foundation, Inc. +/* RSEQ_* accessors. x32 version. + Copyright (C) 2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -45,6 +45,14 @@ /* Read member of the RSEQ area directly. */ #define RSEQ_GETMEM(member) RSEQ_GETMEM_ONCE(member) +/* Loading addresses of objects on x86-64 needs to be treated special + when generating PIC code. */ +#ifdef __pic__ +# define IMM_MODE "nr" +#else +# define IMM_MODE "ir" +#endif + /* Set member of the RSEQ area directly, with single-copy atomicity semantics. */ #define RSEQ_SETMEM_ONCE(member, value) \ ({ \ -- 2.47.2