elf: Support THP segment load with madvise enabled THP

The current THP segment load approach works only when THP is enabled
with always in the kernel.  If THP is enabled with madvise in the
kernel, to enable THP segment load in an application, madvise should
be called with MADV_HUGEPAGE on all THP eligible PT_LOAD segments:

1. Define DL_MAP_DEFAULT_THP_PAGESIZE in hugepages.h and default it to 0.
If DL_MAP_DEFAULT_THP_PAGESIZE is defined, assume kernel THP madvise mode.
If kernel THP mode is always or never, there is an extra madvise call
which has no impact.  DL_MAP_DEFAULT_THP_PAGESIZE is defined for x86-64
and 64-bit loongarch.
2. Update _dl_map_segment_align to support madvise THP mode.  This fixes
BZ #34079.
3. Call _dl_executable_postprocess in rtld_setup_main_map for dynamic
executables and in LIBC_START_MAIN for static executables, which calls
madvise with MADV_HUGEPAGE on all THP eligible PT_LOAD segments in
executable.  This fixes BZ #34080 for both dynamic and static executables.
4. Call _dl_postprocess_loadcmd_extra in _dl_postprocess_loadcmd, which
calls madvise with MADV_HUGEPAGE on all THP eligible PT_LOAD segments
when loading an object after they have been mapped in.  This fixes
BZ #34080 for shared objects.
5. Set the maximum page alignment on THP tests to THP page size as the
default maximum page alignment may be smaller than THP page size.
6. Add tests to verify that large executable PT_LOAD segments in
executables are mapped at addresses aligned to THP page size when the
kernel is configured to use THP in "always" mode or "madvise" mode by
inspecting /proc/self/maps to check that the mapping address is aligned
to THP page size reported by the kernel.  Also verify that madvise is
called with MADV_HUGEPAGE when the glibc tunable glibc.elf.thp=1 is used
and madvise isn't called with MADV_HUGEPAGE when the glibc tunable
glibc.elf.thp=0 is used.

Skip these tests if THP page size cannot be determined or if THP is not
enabled in "always" mode nor "madvise" mode.

Quote WANG Rui <wangrui@loongson.cn>:

From benchmarking a clang build of the Linux kernel on x86_64 with
your patch in THP madvise mode, I observed that iTLB misses were
reduced, similar to what we see in THP always mode.

NB: Some THP tests fail on arm due to limitations of arm32 kABI:

https://sourceware.org/bugzilla/show_bug.cgi?id=34096

Signed-off-by: H.J. Lu <hjl.tools@gmail.com>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

diff --git a/csu/libc-start.c b/csu/libc-start.c
index 1bd2184e460afdcf78ba8d5343af3b1a78239ddc..67aa9812468f7677fb35f3cab2474a0c0ff34174 100644 (file)
--- a/csu/libc-start.c
+++ b/csu/libc-start.c
@@ -204,6 +204,7 @@ call_fini (void *unused)
 #endif /* !SHARED */
 
 #include <libc-start.h>
 #endif /* !SHARED */
 
 #include <libc-start.h>

+#include <dl-exec-post.h>

 
 STATIC int LIBC_START_MAIN (int (*main) (int, char **, char **
                                         MAIN_AUXVEC_DECL),
 
 STATIC int LIBC_START_MAIN (int (*main) (int, char **, char **
                                         MAIN_AUXVEC_DECL),


@@ -307,6 +308,11 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
   _dl_relocate_static_pie_ifunc ();
   ARCH_SETUP_IREL ();
 
   _dl_relocate_static_pie_ifunc ();
   ARCH_SETUP_IREL ();
 

+  /* This must run after the IFUNC relocations: _dl_executable_postprocess
+     may call IFUNC-resolved routines.  */
+  struct link_map *main_map = _dl_get_dl_main_map ();
+  _dl_executable_postprocess (main_map, GL(dl_phdr), GL(dl_phnum));
+

   /* Initialize libpthread if linked in.  */
   if (__pthread_initialize_minimal != NULL)
     __pthread_initialize_minimal ();
   /* Initialize libpthread if linked in.  */
   if (__pthread_initialize_minimal != NULL)
     __pthread_initialize_minimal ();

diff --git a/elf/dl-load.h b/elf/dl-load.h
index 80ae5db4b3d2ff0c705ae246b16349f1fd1b1618..84b82e183df645f94fcd480a91f8d33dd0b19443 100644 (file)
--- a/elf/dl-load.h
+++ b/elf/dl-load.h
@@ -112,6 +112,7 @@ struct loadcmd
   int prot;                             /* PROT_* bits.  */
 };
 
   int prot;                             /* PROT_* bits.  */
 };
 

+#include <dl-load-post.h>

 
 /* Iterator for program header segments.  Initialize with
    _dl_pt_load_iterator_init, then either walk PT_LOAD segments via
 
 /* Iterator for program header segments.  Initialize with
    _dl_pt_load_iterator_init, then either walk PT_LOAD segments via


@@ -215,6 +216,8 @@ _dl_postprocess_loadcmd (struct link_map *l, const ElfW(Ehdr) *header,
     /* Found the program header in this segment.  */
     l->l_phdr = (void *) (uintptr_t) (c->mapstart + header->e_phoff
                                       - c->mapoff);
     /* Found the program header in this segment.  */
     l->l_phdr = (void *) (uintptr_t) (c->mapstart + header->e_phoff
                                       - c->mapoff);

+
+  _dl_postprocess_loadcmd_extra (l, c);

 }
 
 
 }
 
 

diff --git a/elf/dl-support.c b/elf/dl-support.c
index 0508d6113b538df3dd7e683b714e5c7a7b0e04a0..b57fd74670bbf8f853777161f449926acae4d39f 100644 (file)
--- a/elf/dl-support.c
+++ b/elf/dl-support.c
@@ -179,6 +179,12 @@ int _dl_stack_cache_lock;
 #endif
 struct dl_scope_free_list *_dl_scope_free_list;
 
 #endif
 struct dl_scope_free_list *_dl_scope_free_list;
 

+#ifdef HAVE_THP
+enum dl_elf_thp_control_t _dl_elf_thp_control;
+enum thp_mode_t _dl_thp_mode;
+size_t _dl_elf_thp_pagesize;
+#endif
+

 #ifdef NEED_DL_SYSINFO
 /* Needed for improved syscall handling on at least x86/Linux.  NB: Don't
    initialize it here to avoid RELATIVE relocation in static PIE.  */
 #ifdef NEED_DL_SYSINFO
 /* Needed for improved syscall handling on at least x86/Linux.  NB: Don't
    initialize it here to avoid RELATIVE relocation in static PIE.  */

diff --git a/elf/rtld.c b/elf/rtld.c
index f2be6239b3181efd9561dfbeab5635ef80a7de84..e5ba71fef188b744dd821e88e80b2b83570bef00 100644 (file)
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -53,6 +53,7 @@
 #include <dl-find_object.h>
 #include <dl-audit-check.h>
 #include <dl-call_tls_init_tp.h>
 #include <dl-find_object.h>
 #include <dl-audit-check.h>
 #include <dl-call_tls_init_tp.h>

+#include <dl-exec-post.h>

 
 #include <assert.h>
 
 
 #include <assert.h>
 


@@ -1203,14 +1204,8 @@ rtld_setup_main_map (struct link_map *main_map)
        main_map->l_relro_size = ph->p_memsz;
        break;
       }
        main_map->l_relro_size = ph->p_memsz;
        break;
       }

-  /* Process program headers again, but scan them backwards since
-     PT_GNU_PROPERTY is close to the end of program headers.   */
-  for (const ElfW(Phdr) *ph = &phdr[phnum]; ph != phdr; --ph)
-    if (ph[-1].p_type == PT_GNU_PROPERTY)
-      {
-       _dl_process_pt_gnu_property (main_map, -1, &ph[-1]);
-       break;
-      }
+
+  _dl_executable_postprocess (main_map, phdr, phnum);

 
   /* Adjust the address of the TLS initialization image in case
      the executable is actually an ET_DYN object.  */
 
   /* Adjust the address of the TLS initialization image in case
      the executable is actually an ET_DYN object.  */


@@ -1583,6 +1578,9 @@ dl_main (const ElfW(Phdr) *phdr,
        {
          RTLD_TIMING_VAR (start);
          rtld_timer_start (&start);
        {
          RTLD_TIMING_VAR (start);
          rtld_timer_start (&start);

+#ifdef HAVE_THP
+         _dl_get_thp_config ();
+#endif

          _dl_map_object (NULL, rtld_progname, lt_executable, 0,
                          __RTLD_OPENEXEC, LM_ID_BASE);
          rtld_timer_stop (&load_time, start);
          _dl_map_object (NULL, rtld_progname, lt_executable, 0,
                          __RTLD_OPENEXEC, LM_ID_BASE);
          rtld_timer_stop (&load_time, start);

diff --git a/support/support_check_hugetlb.c b/support/support_check_hugetlb.c
index c73946c894cd79f1f58b3eda1edfd1333cabf876..b1c2a8e7cd7daafe73151c7877939e1a9fcd1017 100644 (file)
--- a/support/support_check_hugetlb.c
+++ b/support/support_check_hugetlb.c
@@ -27,23 +27,67 @@
 #include <support/support_check_hugetlb.h>
 #include <support/check.h>
 
 #include <support/support_check_hugetlb.h>
 #include <support/check.h>
 

-bool
-support_thp_work_madvise (void)
+enum thp_mode_t
+support_get_thp_mode (void)

 {
 {

-  int fd = open ("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+  int fd = open ("/sys/kernel/mm/transparent_hugepage/enabled",
+                O_RDONLY, 0);

   if (fd == -1)
   if (fd == -1)

-    return false;
+    return thp_mode_not_supported;

 
 

-#define MODE_MADVISE "always [madvise] never\n"
-#define MODE_ALWAYS  "[always] madvise never\n"
+  static const char mode_always[]  = "[always] madvise never\n";
+  static const char mode_madvise[] = "always [madvise] never\n";
+  static const char mode_never[]   = "always madvise [never]\n";

 
 

-  char str[sizeof(MODE_MADVISE)];
+  char str[sizeof(mode_always)];

   ssize_t s = read (fd, str, sizeof (str));
   close (fd);
   ssize_t s = read (fd, str, sizeof (str));
   close (fd);

-  if (s != sizeof (str) - 1)
-    return false;
+  if (s < 0 || s >= sizeof str)
+    return thp_mode_not_supported;

   str[s] = '\0';
   str[s] = '\0';

-  return strcmp (str, MODE_MADVISE) == 0 || strcmp (str, MODE_ALWAYS) == 0;
+
+  if (s == sizeof (mode_always) - 1)
+    {
+      if (strcmp (str, mode_always) == 0)
+       return thp_mode_always;
+      else if (strcmp (str, mode_madvise) == 0)
+       return thp_mode_madvise;
+      else if (strcmp (str, mode_never) == 0)
+       return thp_mode_never;
+    }
+  return thp_mode_not_supported;
+}
+
+unsigned long int
+support_get_thp_size (void)
+{
+  int fd = open ("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
+                 O_RDONLY, 0);
+  if (fd == -1)
+    return 0;
+
+  char str[INT_BUFSIZE_BOUND (unsigned long int)];
+  ssize_t s = read (fd, str, sizeof (str));
+  close (fd);
+  if (s < 0)
+    return 0;
+
+  unsigned long int r = 0;
+  for (ssize_t i = 0; i < s; i++)
+    {
+      if (str[i] == '\n')
+    break;
+      r *= 10;
+      r += str[i] - '0';
+    }
+  return r;
+}
+
+bool
+support_thp_work_madvise (void)
+{
+  enum thp_mode_t mode = support_get_thp_mode ();
+  return mode == thp_mode_always || mode == thp_mode_madvise;

 }
 
 bool
 }
 
 bool


@@ -56,7 +100,7 @@ support_hugepages_reserved (void)
   char str[INT_BUFSIZE_BOUND(unsigned long int)];
   ssize_t s = read (fd, str, sizeof (str));
   close (fd);
   char str[INT_BUFSIZE_BOUND(unsigned long int)];
   ssize_t s = read (fd, str, sizeof (str));
   close (fd);

-  if (s >= sizeof str || s < 0)
+  if (s < 0 || s >= sizeof str)

     return false;
   str[s] = '\0';
   unsigned long int n = 0;
     return false;
   str[s] = '\0';
   unsigned long int n = 0;

diff --git a/support/support_check_hugetlb.h b/support/support_check_hugetlb.h
index 2c5f5cb80cc7fcfd4971a302e2a37dd83b9d1829..6f55405c27ba614b92e3f1a15a1b413f6f49c461 100644 (file)
--- a/support/support_check_hugetlb.h
+++ b/support/support_check_hugetlb.h
@@ -21,6 +21,10 @@
 
 #include <stdbool.h>
 #include <sys/cdefs.h>
 
 #include <stdbool.h>
 #include <sys/cdefs.h>

+#undef attribute_hidden
+#define attribute_hidden
+#include <hugepages.h> /* For enum thp_mode_t.  */
+#undef attribute_hidden

 
 __BEGIN_DECLS
 
 
 __BEGIN_DECLS
 


@@ -43,6 +47,14 @@ bool support_hugepages_reserved (void);
    so it is safe to call unconditionally.  */
 void support_check_malloc_hugetlb (void);
 
    so it is safe to call unconditionally.  */
 void support_check_malloc_hugetlb (void);
 

+/* Returns Transparent Huge Pages (THP) mode from
+   /sys/kernel/mm/transparent_hugepage/enabled.  */
+extern enum thp_mode_t support_get_thp_mode (void);
+
+/* Returns Transparent Huge Pages (THP) page size from
+   /sys/kernel/mm/transparent_hugepage/hpage_pmd_size.  */
+extern unsigned long int support_get_thp_size (void);
+

 __END_DECLS
 
 #endif /* SUPPORT_SUPPORT_CHECK_HUGETLB_H */
 __END_DECLS
 
 #endif /* SUPPORT_SUPPORT_CHECK_HUGETLB_H */

diff --git a/sysdeps/generic/dl-exec-post.h b/sysdeps/generic/dl-exec-post.h
new file mode 100644 (file)
index 0000000..f5dcdc0
--- /dev/null
+++ b/sysdeps/generic/dl-exec-post.h
@@ -0,0 +1,34 @@
+/* _dl_executable_postprocess.  Generic version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+static inline void
+_dl_executable_postprocess (struct link_map *main_map,
+                           const ElfW(Phdr) *phdr, ElfW(Word) phnum)
+{
+#ifdef SHARED
+  /* Process program headers again, but scan them backwards since
+     PT_GNU_PROPERTY is close to the end of program headers.   */
+  for (const ElfW(Phdr) *ph = &phdr[phnum]; ph != phdr; --ph)
+    if (ph[-1].p_type == PT_GNU_PROPERTY)
+      {
+       _dl_process_pt_gnu_property (main_map, -1, &ph[-1]);
+       break;
+      }
+#endif
+}

diff --git a/sysdeps/generic/dl-load-post.h b/sysdeps/generic/dl-load-post.h
new file mode 100644 (file)
index 0000000..1a6d205
--- /dev/null
+++ b/sysdeps/generic/dl-load-post.h
@@ -0,0 +1,23 @@
+/* _dl_postprocess_loadcmd_extra.  Generic version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+static inline void
+_dl_postprocess_loadcmd_extra (struct link_map *l, const struct loadcmd *c)
+{
+}

diff --git a/sysdeps/generic/hugepages.h b/sysdeps/generic/hugepages.h
index 5fc9b5c8deab331487caa2f19636634c66c38e6e..8e54661527bc80606908a3d68e9c8e512502582d 100644 (file)
--- a/sysdeps/generic/hugepages.h
+++ b/sysdeps/generic/hugepages.h
@@ -21,15 +21,26 @@
 
 #include <stddef.h>
 
 
 #include <stddef.h>
 

+/* The THP segment load control mode.  */
+enum dl_elf_thp_control_t
+{
+  /* To be enabled or disabled by GLIBC_TUNABLES.  */
+  dl_elf_thp_control_default = 0,
+  /* Enabled by GLIBC_TUNABLES=glibc.elf.thp=1.  */
+  dl_elf_thp_control_enabled,
+    /* Disabled by GLIBC_TUNABLES=glibc.elf.thp=0.  */
+  dl_elf_thp_control_disabled
+};
+

 /* Return the default transparent huge page size.  */
 unsigned long int __get_thp_size (void) attribute_hidden;
 
 enum thp_mode_t
 {
 /* Return the default transparent huge page size.  */
 unsigned long int __get_thp_size (void) attribute_hidden;
 
 enum thp_mode_t
 {

+  thp_mode_not_supported = 0,

   thp_mode_always,
   thp_mode_madvise,
   thp_mode_always,
   thp_mode_madvise,

-  thp_mode_never,
-  thp_mode_not_supported
+  thp_mode_never

 };
 
 enum thp_mode_t __get_thp_mode (void) attribute_hidden;
 };
 
 enum thp_mode_t __get_thp_mode (void) attribute_hidden;


@@ -45,6 +56,10 @@ void __get_hugepage_config (size_t requested, size_t *pagesize, int *flags)
 # define MALLOC_DEFAULT_THP_PAGESIZE   0
 #endif
 
 # define MALLOC_DEFAULT_THP_PAGESIZE   0
 #endif
 

+#ifndef DL_MAP_DEFAULT_THP_PAGESIZE
+# define DL_MAP_DEFAULT_THP_PAGESIZE   0
+#endif
+

 #ifndef MAX_THP_PAGESIZE
 # define MAX_THP_PAGESIZE      (32 * 1024 * 1024)
 #endif
 #ifndef MAX_THP_PAGESIZE
 # define MAX_THP_PAGESIZE      (32 * 1024 * 1024)
 #endif

diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
index 24529db8a1ed45cb76bb7f754de69f6150008a1a..f94247ad9fb7f8908091583efb02f04bbd36f0de 100644 (file)
--- a/sysdeps/generic/ldsodefs.h
+++ b/sysdeps/generic/ldsodefs.h
@@ -39,6 +39,7 @@
 #include <libc-lock.h>
 #include <hp-timing.h>
 #include <list_t.h>
 #include <libc-lock.h>
 #include <hp-timing.h>
 #include <list_t.h>

+#include <hugepages.h>

 
 __BEGIN_DECLS
 
 
 __BEGIN_DECLS
 


@@ -477,6 +478,14 @@ struct rtld_global
   EXTERN struct __pthread **_dl_pthread_threads;
   __mach_rwlock_define (EXTERN, _dl_pthread_threads_lock)
 #endif
   EXTERN struct __pthread **_dl_pthread_threads;
   __mach_rwlock_define (EXTERN, _dl_pthread_threads_lock)
 #endif

+#ifdef HAVE_THP
+  /* The THP segment load control.  */
+  EXTERN enum dl_elf_thp_control_t _dl_elf_thp_control;
+  /* The kernel THP mode.  */
+  EXTERN enum thp_mode_t _dl_thp_mode;
+  /* Page size used for THP segment load.  */
+  EXTERN size_t _dl_elf_thp_pagesize;
+#endif

 #ifdef SHARED
 };
 # define __rtld_global_attribute__
 #ifdef SHARED
 };
 # define __rtld_global_attribute__

diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile
index 7d87888dab57fb930fba4b1b10ac7491dd4cdb32..a048dbc7323e2b2747616e1b7528334f2a999e35 100644 (file)
--- a/sysdeps/unix/sysv/linux/Makefile
+++ b/sysdeps/unix/sysv/linux/Makefile
@@ -697,11 +697,15 @@ $(objpfx)pldd: $(objpfx)xmalloc.o
 tests += \
   tst-rseq-tls-range \
   tst-rseq-tls-range-4096 \
 tests += \
   tst-rseq-tls-range \
   tst-rseq-tls-range-4096 \

+  tst-thp-1 \
+  tst-thp-1-pde \
+  tst-thp-1-static \

   tst-thp-align \
 # tests
 tests-static += \
   tst-rseq-tls-range-4096-static \
   tst-rseq-tls-range-static \
   tst-thp-align \
 # tests
 tests-static += \
   tst-rseq-tls-range-4096-static \
   tst-rseq-tls-range-static \

+  tst-thp-1-static \

 # tests-static
 modules-names += \
   tst-rseq-tls-range-mod \
 # tests-static
 modules-names += \
   tst-rseq-tls-range-mod \


@@ -711,15 +715,12 @@ CFLAGS-tst-rseq-tls-range.c += -DMAIN_TLS_ALIGN=4
 CFLAGS-tst-rseq-tls-range-4096.c += -DMAIN_TLS_ALIGN=4096
 CFLAGS-tst-rseq-tls-range-static.c += -DMAIN_TLS_ALIGN=4
 CFLAGS-tst-rseq-tls-range-4096-static.c += -DMAIN_TLS_ALIGN=4096
 CFLAGS-tst-rseq-tls-range-4096.c += -DMAIN_TLS_ALIGN=4096
 CFLAGS-tst-rseq-tls-range-static.c += -DMAIN_TLS_ALIGN=4
 CFLAGS-tst-rseq-tls-range-4096-static.c += -DMAIN_TLS_ALIGN=4096

-LDFLAGS-tst-thp-size-mod.so += -Wl,-z,noseparate-code

 $(objpfx)tst-rseq-tls-range.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-4096.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-static.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-4096-static.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-4096.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-static.out: $(objpfx)tst-rseq-tls-range-mod.so
 $(objpfx)tst-rseq-tls-range-4096-static.out: $(objpfx)tst-rseq-tls-range-mod.so

-$(objpfx)tst-thp-align.out: $(objpfx)tst-thp-size-mod.so

 tst-rseq-tls-range-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)
 tst-rseq-tls-range-4096-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)
 tst-rseq-tls-range-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)
 tst-rseq-tls-range-4096-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)

-tst-thp-align-ENV = GLIBC_TUNABLES=glibc.elf.thp=1

 
 test-internal-extras += tst-nolink-libc
 ifeq ($(run-built-tests),yes)
 
 test-internal-extras += tst-nolink-libc
 ifeq ($(run-built-tests),yes)


@@ -728,6 +729,186 @@ tests-special += \
   $(objpfx)tst-nolink-libc-2.out \
   # tests-special
 endif
   $(objpfx)tst-nolink-libc-2.out \
   # tests-special
 endif

+
+ifndef THP-PAGE-SIZE
+# Align PT_LOAD segments in THP tests to THP page size so that kernel will
+# map PIE to the address aligned to THP page size.  Default THP page size
+# to 2MB which can be overridden in Makefile in subdirectories.
+THP-PAGE-SIZE = 0x200000
+endif
+
+THP-PAGE-SIZE-LDFLAGS = -Wl,-z,max-page-size=$(THP-PAGE-SIZE)
+
+# -Wl,-z,max-page-size=$(THP-PAGE-SIZE) alone doesn't work for PDE when
+# text-segment address is lower than the maximum page size:
+# https://sourceware.org/bugzilla/show_bug.cgi?id=34184
+ifneq (,$(load-address-ldflag))
+LOAD-THP-ADDRESS-LDFLAGS = $(load-address-ldflag)=$(THP-PAGE-SIZE)
+endif
+
+LDFLAGS-tst-thp-size-mod.so = -Wl,-z,noseparate-code \
+                             $(THP-PAGE-SIZE-LDFLAGS)
+tst-thp-align-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+$(objpfx)tst-thp-align.out: $(objpfx)tst-thp-size-mod.so
+
+tests += \
+  tst-thp-1-no-s-code \
+  tst-thp-1-no-s-code-pde \
+  tst-thp-1-no-s-code-static \
+# tests
+tests-static += \
+  tst-thp-1-no-s-code-static \
+# tests-static
+
+ifeq (yes,$(with-lld))
+# -z noseparate-code is a nop for lld.  It doesn't create an executable
+# segment placed at 0 file offset.
+test-xfail-tst-thp-1-no-s-code = yes
+test-xfail-tst-thp-1-no-s-code-pde = yes
+test-xfail-tst-thp-1-no-s-code-static = yes
+endif
+
+LDFLAGS-tst-thp-1 = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS)
+LDFLAGS-tst-thp-1-pde = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS) \
+                       $(LOAD-THP-ADDRESS-LDFLAGS)
+LDFLAGS-tst-thp-1-static = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS)
+ifneq (yes,$(enable-static-pie))
+LDFLAGS-tst-thp-1-static += $(LOAD-THP-ADDRESS-LDFLAGS)
+endif
+LDFLAGS-tst-thp-1-no-s-code = -Wl,-z,noseparate-code \
+                             $(THP-PAGE-SIZE-LDFLAGS)
+LDFLAGS-tst-thp-1-no-s-code-pde = -Wl,-z,noseparate-code \
+                                 $(THP-PAGE-SIZE-LDFLAGS) \
+                                 $(LOAD-THP-ADDRESS-LDFLAGS)
+LDFLAGS-tst-thp-1-no-s-code-static = -Wl,-z,noseparate-code \
+                                    $(THP-PAGE-SIZE-LDFLAGS)
+ifneq (yes,$(enable-static-pie))
+LDFLAGS-tst-thp-1-no-s-code-static += $(LOAD-THP-ADDRESS-LDFLAGS)
+endif
+
+$(objpfx)tst-thp-1-no-s-code: $(objpfx)tst-thp-size-mod.o
+$(objpfx)tst-thp-1-no-s-code-pde: $(objpfx)tst-thp-size-mod.o
+$(objpfx)tst-thp-1-no-s-code-static: $(objpfx)tst-thp-size-mod.o
+
+tst-thp-1-no-s-code-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+tst-thp-1-no-s-code-pde-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+tst-thp-1-no-s-code-static-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+
+tst-thp-1-no-s-code-pde-no-pie = yes
+
+tst-thp-1-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+tst-thp-1-pde-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+tst-thp-1-static-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
+
+$(objpfx)tst-thp-1: $(objpfx)tst-thp-size-mod.o
+$(objpfx)tst-thp-1-pde: $(objpfx)tst-thp-size-mod.o
+$(objpfx)tst-thp-1-static: $(objpfx)tst-thp-size-mod.o
+
+tst-thp-1-pde-no-pie = yes
+
+# Don't run strace tests for cross-compiling.
+ifeq (no,$(cross-compiling))
+thp-kernel-status = $(shell grep madvise /sys/kernel/mm/transparent_hugepage/enabled)
+# Verify that madvise is called with MADV_HUGEPAGE when THP is enabled
+# under madvise THP kernel.
+ifneq ($(findstring [madvise],$(thp-kernel-status)),)
+tests-special += \
+  $(objpfx)strace-tst-thp-1-disabled.out \
+  $(objpfx)strace-tst-thp-1-enabled.out \
+  $(objpfx)strace-tst-thp-1-pde-disabled.out \
+  $(objpfx)strace-tst-thp-1-pde-enabled.out \
+  $(objpfx)strace-tst-thp-1-static-disabled.out \
+  $(objpfx)strace-tst-thp-1-static-enabled.out \
+  $(objpfx)strace-tst-thp-align-default.out \
+  $(objpfx)strace-tst-thp-align-disabled.out \
+  $(objpfx)strace-tst-thp-align-enabled.out \
+# tests-special
+
+ifeq (yes,$(build-hardcoded-path-in-tests))
+strace-test-via-rtld-prefix =
+else
+strace-test-via-rtld-prefix = $(rtld-prefix)
+endif
+
+$(objpfx)strace-tst-thp-1-enabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-1
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
+               $(objpfx)tst-thp-1 > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-1-disabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-1
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
+               $(objpfx)tst-thp-1 > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-1-pde-enabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-1-pde
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
+               $(objpfx)tst-thp-1-pde > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-1-pde-disabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-1-pde
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
+               $(objpfx)tst-thp-1-pde > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-1-static-enabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh \
+  $(objpfx)tst-thp-1-static
+       $(SHELL) $< '' '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
+               $(objpfx)tst-thp-1-static > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-1-static-disabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh \
+  $(objpfx)tst-thp-1-static
+       $(SHELL) $< '' '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
+               $(objpfx)tst-thp-1-static > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-align-default.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-align
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' '$(run-program-env)' \
+               $(objpfx)tst-thp-align > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-align-enabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-align
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
+               $(objpfx)tst-thp-align > $@; \
+         $(evaluate-test)
+
+$(objpfx)strace-tst-thp-align-disabled.out: \
+  $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
+  $(objpfx)tst-thp-align
+       $(SHELL) $< '$(strace-test-via-rtld-prefix)' \
+               '$(test-wrapper-env)' \
+               '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
+               $(objpfx)tst-thp-align > $@; \
+         $(evaluate-test)
+endif # [madvise]
+endif # $(cross-compiling)

 endif # $(subdir) == elf
 
 ifeq ($(subdir),rt)
 endif # $(subdir) == elf
 
 ifeq ($(subdir),rt)

diff --git a/sysdeps/unix/sysv/linux/arm/Makefile b/sysdeps/unix/sysv/linux/arm/Makefile
index e73ce4f81114e7894e16119e4fa4d5e61572830d..428949fefd3336b1103df8655ed97cdb9b8a4180 100644 (file)
--- a/sysdeps/unix/sysv/linux/arm/Makefile
+++ b/sysdeps/unix/sysv/linux/arm/Makefile
@@ -3,6 +3,13 @@ sysdep-rtld-routines += aeabi_read_tp libc-do-syscall
 # The test uses INTERNAL_SYSCALL_CALL.  In thumb mode, this uses
 # an undefined reference to __libc_do_syscall.
 CFLAGS-tst-nolink-libc.c += -marm
 # The test uses INTERNAL_SYSCALL_CALL.  In thumb mode, this uses
 # an undefined reference to __libc_do_syscall.
 CFLAGS-tst-nolink-libc.c += -marm

+
+ifeq (yes,$(enable-static-pie))
+# These static PIE tests fail on arm due to limitations of arm32 kABI:
+# https://sourceware.org/bugzilla/show_bug.cgi?id=34096
+test-xfail-tst-thp-1-no-s-code-static = yes
+test-xfail-tst-thp-1-static = yes
+endif

 endif
 
 ifeq ($(subdir),misc)
 endif
 
 ifeq ($(subdir),misc)

diff --git a/sysdeps/unix/sysv/linux/dl-exec-post.h b/sysdeps/unix/sysv/linux/dl-exec-post.h
new file mode 100644 (file)
index 0000000..dd8a379
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/dl-exec-post.h
@@ -0,0 +1,137 @@
+/* _dl_executable_postprocess.  Linux version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+static inline void
+_dl_get_thp_config (void)
+{
+  /* Check if there is GLIBC_TUNABLES=glibc.elf.thp=[0|1].  */
+  if (TUNABLE_GET_FULL (glibc, elf, thp, int32_t, NULL) == 0)
+    GL(dl_elf_thp_control) = dl_elf_thp_control_disabled;
+  else
+    GL(dl_elf_thp_control) = dl_elf_thp_control_enabled;
+
+  /* Return if the tunable is not set or THP is disabled by the
+     tunable.  */
+  if (GL(dl_elf_thp_control) == dl_elf_thp_control_disabled)
+    return;
+
+  verify (DL_MAP_DEFAULT_THP_PAGESIZE <= MAX_THP_PAGESIZE);
+
+  /* NB: Accessing /sys/kernel/mm files is quite expensive and the file
+     may not be accessible in containers.  If DL_MAP_DEFAULT_THP_PAGESIZE
+     is non-zero, assume THP mode is madvise and always call madvise.
+     Since madvise is a fast system call, it adds only a small overhead
+     compared to the cost of accessing /sys/kernel/mm files.  */
+  if (DL_MAP_DEFAULT_THP_PAGESIZE != 0)
+    {
+      GL(dl_elf_thp_pagesize) = DL_MAP_DEFAULT_THP_PAGESIZE;
+      GL(dl_thp_mode) = thp_mode_madvise;
+    }
+  else
+    {
+      GL(dl_thp_mode) = __get_thp_mode ();
+      if (GL(dl_thp_mode) == thp_mode_always
+         || GL(dl_thp_mode) == thp_mode_madvise)
+       {
+         GL(dl_elf_thp_pagesize) = __get_thp_size ();
+         /* We cap the huge page size at MAX_THP_PAGESIZE to avoid
+            over-aligning on systems with very large normal pages
+            (like 64K pages with 512M huge pages).  */
+         if (GL(dl_elf_thp_pagesize) > MAX_THP_PAGESIZE)
+           GL(dl_elf_thp_pagesize) = 0;
+       }
+      else
+       GL(dl_elf_thp_pagesize) = 0;
+
+      if (GL(dl_elf_thp_pagesize) == 0)
+       {
+         GL(dl_elf_thp_control) = dl_elf_thp_control_disabled;
+         GL(dl_thp_mode) = thp_mode_not_supported;
+       }
+    }
+}
+
+static inline void
+_dl_executable_postprocess (struct link_map *main_map,
+                           const ElfW(Phdr) *phdr, ElfW(Word) phnum)
+{
+  /* NB: In static executable, PT_GNU_PROPERTY is processed in target
+     libc-start.h if it is needed by target.  When ld.so is used, if
+     a target doesn't need PT_GNU_PROPERTY, _dl_process_pt_gnu_property
+     is an empty function.  */
+#ifdef SHARED
+  /* Process program headers again, but scan them backwards since
+     PT_GNU_PROPERTY is close to the end of program headers.   */
+  for (const ElfW(Phdr) *ph = &phdr[phnum]; ph != phdr; --ph)
+    if (ph[-1].p_type == PT_GNU_PROPERTY)
+      {
+       _dl_process_pt_gnu_property (main_map, -1, &ph[-1]);
+       break;
+      }
+#endif
+
+  /* If THP state was not yet initialized, the main executable was mapped
+     by the kernel; in that case this function is the only place that can
+     apply MADV_HUGEPAGE to the main executable's segments.  Otherwise,
+     _dl_get_thp_config has already run earlier in dl_main and
+     _dl_map_segments has just mapped the main executable, so
+     _dl_postprocess_loadcmd_extra has already done the madvise pass; do
+     not repeat it here.  */
+  if (GL(dl_elf_thp_control) != dl_elf_thp_control_default)
+    return;
+
+   _dl_get_thp_config ();
+
+  /* Return if THP segment load isn't enabled.  */
+  if (GL(dl_elf_thp_control) != dl_elf_thp_control_enabled)
+    return;
+
+  /* NB: If DL_MAP_DEFAULT_THP_PAGESIZE is non-zero, dl_thp_mode is set
+     to thp_mode_madvise.  */
+  if (DL_MAP_DEFAULT_THP_PAGESIZE == 0
+      && GL(dl_thp_mode) != thp_mode_madvise)
+    return;
+
+  /* When we get here, the main executable have been mapped in.  Call
+     madvise with MADV_HUGEPAGE for all THP eligible PT_LOAD segments.  */
+
+  const ElfW(Phdr) *ph;
+
+  size_t thp_pagesize = GL(dl_elf_thp_pagesize);
+
+  /* Call __madvise if offset and address of the PT_LOAD segment are
+     aligned to THP page size and it is read-only.  */
+  for (ph = phdr; ph < &phdr[phnum]; ++ph)
+    if (ph->p_type == PT_LOAD
+       && ph->p_memsz >= thp_pagesize
+       && ((ph->p_vaddr | ph->p_offset) & (thp_pagesize - 1)) == 0
+       && (ph->p_flags & (PF_W | PF_R)) == PF_R)
+      {
+       int ret = __madvise ((void *) (main_map->l_addr + ph->p_vaddr),
+                            ph->p_memsz, MADV_HUGEPAGE);
+       if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
+         _dl_debug_printf ("\
+madvise (0x%0*lx, 0x%0*lx, MADV_HUGEPAGE) returns %d\n",
+(int) sizeof (void *) * 2,
+(unsigned long int) (main_map->l_addr + ph->p_vaddr),
+(int) sizeof (void *) * 2,
+(unsigned long int) ph->p_memsz,
+ret);
+      }
+}

diff --git a/sysdeps/unix/sysv/linux/dl-load-post.h b/sysdeps/unix/sysv/linux/dl-load-post.h
new file mode 100644 (file)
index 0000000..741c4dd
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/dl-load-post.h
@@ -0,0 +1,42 @@
+/* _dl_postprocess_loadcmd_extra.  Linux version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+static bool _dl_segment_thp_eligible (const struct loadcmd *, size_t);
+
+/* After L has been mapped in, call madvise with MADV_HUGEPAGE for THP
+   madvise mode if L is THP eligible.  */
+
+static inline void
+_dl_postprocess_loadcmd_extra (struct link_map *l, const struct loadcmd *c)
+{
+  if (GL(dl_thp_mode) == thp_mode_madvise
+      && _dl_segment_thp_eligible (c, GL(dl_elf_thp_pagesize)))
+    {
+      int ret = __madvise ((void *) (l->l_addr + c->mapstart),
+                          c->mapend - c->mapstart, MADV_HUGEPAGE);
+      if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_FILES))
+       _dl_debug_printf ("\
+  madvise (0x%0*lx, 0x%0*lx, MADV_HUGEPAGE) returns %d\n",
+  (int) sizeof (void *) * 2,
+  (unsigned long int) (l->l_addr + c->mapstart),
+  (int) sizeof (void *) * 2,
+  (unsigned long int) (c->mapend - c->mapstart),
+  ret);
+    }
+}

diff --git a/sysdeps/unix/sysv/linux/dl-map-segment-align.c b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
index a39e74d91b37486f3d2227a5c4619cc354f4f1bd..c062ba0c58e04aadf7746b15da3ead9259283525 100644 (file)
--- a/sysdeps/unix/sysv/linux/dl-map-segment-align.c
+++ b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
@@ -17,38 +17,24 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 

+#include <ldsodefs.h>

 #include <dl-map-segment-align.h>
 #include <dl-map-segment-align.h>

-#include <dl-tunables.h>
-#include <hugepages.h>
+
+/* Return the alignment of the PT_LOAD segment for THP.  P_ALIGN_MAX is
+   the maximum p_align value in the PT_LOAD segment.  */

 
 ElfW (Addr)
 _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
 {
 
 ElfW (Addr)
 _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
 {

-  static enum thp_mode_t thp_mode = thp_mode_not_supported;
-  static unsigned long int thp_pagesize;
+  size_t thp_pagesize = GL(dl_elf_thp_pagesize);

 
 

-  if (TUNABLE_GET (glibc, elf, thp, int32_t, NULL) == 0)
+  if (GL(dl_elf_thp_control) != dl_elf_thp_control_enabled
+      || p_align_max >= thp_pagesize)

     return p_align_max;
 
     return p_align_max;
 

-  if (__glibc_unlikely (thp_mode == thp_mode_not_supported
-                        || thp_pagesize == 0))
-    {
-      unsigned long int default_thp_pagesize = DL_MAP_DEFAULT_THP_PAGESIZE;
-      thp_mode = default_thp_pagesize ? thp_mode_always : __get_thp_mode ();
-      thp_pagesize = default_thp_pagesize ? : __get_thp_size ();
-    }
-
-  /* Aligning load segments that are large enough to the PMD size helps
-     improve THP eligibility and reduces TLB pressure.
-     We cap the huge page size at MAX_THP_PAGESIZE to avoid over-aligning
-     on systems with very large normal pages (like 64K pages with 512M
-     huge pages). */
-  if (thp_mode == thp_mode_always
-      && thp_pagesize <= MAX_THP_PAGESIZE
-      && ((c->mapstart | c->mapoff) & (thp_pagesize - 1)) == 0
-      && (c->mapend - c->mapstart) >= thp_pagesize
-      && p_align_max < thp_pagesize
-      && (c->prot & PROT_WRITE) == 0)
+  /* Return true if the segment is THP eligible.  It helps improve THP
+     eligibility and reduces TLB pressure.  */
+  if (_dl_segment_thp_eligible (c, thp_pagesize))

     return thp_pagesize;
 
   return p_align_max;
     return thp_pagesize;
 
   return p_align_max;

diff --git a/sysdeps/unix/sysv/linux/dl-map-segment-align.h b/sysdeps/unix/sysv/linux/dl-map-segment-align.h
index d9b05181b7842660a031913a61f4f0f5172a7fd1..9a7fc3a10ef6b0747abd1e3c9714eb9e6204a363 100644 (file)
--- a/sysdeps/unix/sysv/linux/dl-map-segment-align.h
+++ b/sysdeps/unix/sysv/linux/dl-map-segment-align.h
@@ -19,9 +19,18 @@
 
 #include <dl-load.h>
 
 
 #include <dl-load.h>
 

-#ifndef DL_MAP_DEFAULT_THP_PAGESIZE
-# define DL_MAP_DEFAULT_THP_PAGESIZE   0
-#endif
-

 extern ElfW (Addr) _dl_map_segment_align
   (const struct loadcmd *, ElfW (Addr)) attribute_hidden;
 extern ElfW (Addr) _dl_map_segment_align
   (const struct loadcmd *, ElfW (Addr)) attribute_hidden;

+
+/* Return true only if the loadcmd C is THP eligible with THP page size
+   THP_PAGESIZE, which means that it is read-only, its size >= THP page
+   size, its offset and address of the loadcmd C are aligned to THP page
+   size.  */
+
+static inline bool
+_dl_segment_thp_eligible (const struct loadcmd *c, size_t thp_pagesize)
+{
+  return ((c->prot & (PROT_WRITE | PROT_READ)) == PROT_READ
+         && (c->mapend - c->mapstart) >= thp_pagesize
+         && ((c->mapstart | c->mapoff) & (thp_pagesize - 1)) == 0);
+}

diff --git a/sysdeps/unix/sysv/linux/ldsodefs.h b/sysdeps/unix/sysv/linux/ldsodefs.h
index c63b64943270280c21d1a1833ec6a291a18e29df..e39d9afe34d64fe5c20702fa9faacb0f959dd3e5 100644 (file)
--- a/sysdeps/unix/sysv/linux/ldsodefs.h
+++ b/sysdeps/unix/sysv/linux/ldsodefs.h
@@ -21,6 +21,9 @@
 /* We have the auxiliary vector.  */
 #define HAVE_AUX_VECTOR
 
 /* We have the auxiliary vector.  */
 #define HAVE_AUX_VECTOR
 

+/* We have transparent huge page.  */
+#define HAVE_THP
+

 /* Get the real definitions.  */
 #include_next <ldsodefs.h>
 
 /* Get the real definitions.  */
 #include_next <ldsodefs.h>
 

diff --git a/sysdeps/unix/sysv/linux/loongarch/Makefile b/sysdeps/unix/sysv/linux/loongarch/Makefile
index 0d5f087862db8569339d19c2846cb1017d4cbf36..d5beb624401f6e63ad23fa1c7307f5b660da6479 100644 (file)
--- a/sysdeps/unix/sysv/linux/loongarch/Makefile
+++ b/sysdeps/unix/sysv/linux/loongarch/Makefile
@@ -12,3 +12,6 @@ abi-ilp32s-condition  := __WORDSIZE == 32 && defined __loongarch_soft_float
 abi-ilp32d-condition   := __WORDSIZE == 32 && defined __loongarch_double_float
 abi-lp64s-condition    := __WORDSIZE == 64 && defined __loongarch_soft_float
 abi-lp64d-condition    := __WORDSIZE == 64 && defined __loongarch_double_float
 abi-ilp32d-condition   := __WORDSIZE == 32 && defined __loongarch_double_float
 abi-lp64s-condition    := __WORDSIZE == 64 && defined __loongarch_soft_float
 abi-lp64d-condition    := __WORDSIZE == 64 && defined __loongarch_double_float

+
+# Align THP tests to 32MB.
+THP-PAGE-SIZE = 0x2000000

diff --git a/sysdeps/unix/sysv/linux/loongarch/lp64/dl-map-segment-align.h b/sysdeps/unix/sysv/linux/loongarch/lp64/hugepages.h
similarity index 90%
rename from sysdeps/unix/sysv/linux/loongarch/lp64/dl-map-segment-align.h
rename to sysdeps/unix/sysv/linux/loongarch/lp64/hugepages.h
index c51ee4ac47eb11708ab85a8d7cd2f48d37e75750..30252a9b86d7c2442ed25b4a16636fe8b823f84b 100644 (file)
--- a/sysdeps/unix/sysv/linux/loongarch/lp64/dl-map-segment-align.h
+++ b/sysdeps/unix/sysv/linux/loongarch/lp64/hugepages.h
@@ -1,4 +1,4 @@
-/* _dl_map_segment_align.  LoongArch64 Linux version.
+/* Huge Page support.  LoongArch64 Linux version.

    Copyright (C) 2026 Free Software Foundation, Inc.
    Copyright The GNU Toolchain Authors.
    This file is part of the GNU C Library.
    Copyright (C) 2026 Free Software Foundation, Inc.
    Copyright The GNU Toolchain Authors.
    This file is part of the GNU C Library.


@@ -19,4 +19,4 @@
 
 #define DL_MAP_DEFAULT_THP_PAGESIZE (32 * 1024 * 1024)
 
 
 #define DL_MAP_DEFAULT_THP_PAGESIZE (32 * 1024 * 1024)
 

-#include_next <dl-map-segment-align.h>
+#include_next <hugepages.h>

diff --git a/sysdeps/unix/sysv/linux/strace-tst-thp.sh b/sysdeps/unix/sysv/linux/strace-tst-thp.sh
new file mode 100644 (file)
index 0000000..bff7d3e
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/strace-tst-thp.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+# Run THP test under strace to verify control of the THP segment load.
+# Copyright (C) 2026 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <https://www.gnu.org/licenses/>.
+
+set -e
+
+rtld="$1"
+test_wrapper_env="$2"
+run_program_env="$3"
+test_prog="$4"
+
+cmd="${test_wrapper_env} ${run_program_env} strace -X raw ${rtld} ${test_prog}"
+
+TIMEOUTFACTOR=${TIMEOUTFACTOR:-1}
+
+case x"${run_program_env}" in
+*glibc.elf.thp=1*)
+  strace_expected=yes
+  ;;
+*)
+  strace_expected=no
+  ;;
+esac
+
+# Verify strace is not just present, but works in this environment.  If
+# not, skip the test.
+/bin/sh -c \
+ "${test_wrapper_env} ${run_program_env} \
+  strace -X raw -e trace=none -- /bin/true" > /dev/null 2>&1 || exit 77
+
+# Finally the actual test inside the test environment, using the just
+# build ld.so and new libraries to run the THP test under strace.
+if /bin/sh -c \
+  "timeout -k 4 $((3*$TIMEOUTFACTOR)) ${cmd} --direct 2>&1 \
+   | grep -E \"madvise\(0x[0-9a-f]+, [0-9]+, 0xe)\""; then
+  if test ${strace_expected} = yes; then
+    exit 0
+  else
+    exit 1
+  fi
+else
+  if test ${strace_expected} = no; then
+    exit 0
+  else
+    exit 1
+  fi
+fi

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-pde.c b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-pde.c
new file mode 100644 (file)
index 0000000..3fd01e9
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-pde.c
@@ -0,0 +1,19 @@
+/* Test PDE with THP segment load linked with -Wl,-z,noseparate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-1.c"

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-static.c b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-static.c
new file mode 100644 (file)
index 0000000..d0ae0f1
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code-static.c
@@ -0,0 +1,19 @@
+/* Test static with THP segment load linked with -Wl,-z,noseparate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-1.c"

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code.c b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code.c
new file mode 100644 (file)
index 0000000..5eb1e00
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1-no-s-code.c
@@ -0,0 +1,19 @@
+/* Test THP segment load linked with -Wl,-z,noseparate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-1.c"

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1-pde.c b/sysdeps/unix/sysv/linux/tst-thp-1-pde.c
new file mode 100644 (file)
index 0000000..d854dd4
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1-pde.c
@@ -0,0 +1,19 @@
+/* Test PDE with THP segment load linked with -Wl,-z,separate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-1.c"

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1-static.c b/sysdeps/unix/sysv/linux/tst-thp-1-static.c
new file mode 100644 (file)
index 0000000..66d7e12
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1-static.c
@@ -0,0 +1,19 @@
+/* Test static with THP segment load linked with -Wl,-z,separate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-1.c"

diff --git a/sysdeps/unix/sysv/linux/tst-thp-1.c b/sysdeps/unix/sysv/linux/tst-thp-1.c
new file mode 100644 (file)
index 0000000..49eea70
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-1.c
@@ -0,0 +1,28 @@
+/* Test THP segment load linked with -Wl,-z,separate-code.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "tst-thp-align-check.h"
+
+static int
+do_test (void)
+{
+  check_align ("tst-thp-1");
+  return 0;
+}
+
+#include <support/test-driver.c>

diff --git a/sysdeps/unix/sysv/linux/tst-thp-align-check.h b/sysdeps/unix/sysv/linux/tst-thp-align-check.h
new file mode 100644 (file)
index 0000000..c71ac88
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/tst-thp-align-check.h
@@ -0,0 +1,67 @@
+/* Test the THP compatible alignment of PT_LOAD segments.
+
+   Copyright (C) 2026 Free Software Foundation, Inc.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <intprops.h>
+#include <inttypes.h>
+#include <support/support_check_hugetlb.h>
+#include <support/check.h>
+#include <support/xstdio.h>
+#include <support/xunistd.h>
+
+static void
+check_align (const char *name)
+{
+  unsigned long int thp_size = support_get_thp_size ();
+  enum thp_mode_t thp_mode = support_get_thp_mode ();
+
+  if (thp_size == 0)
+    FAIL_UNSUPPORTED ("unable to get THP size.\n");
+
+  if (thp_size > MAX_THP_PAGESIZE)
+    FAIL_UNSUPPORTED ("THP size exceeds MAX_THP_PAGESIZE.\n");
+
+  if (thp_mode != thp_mode_always && thp_mode != thp_mode_madvise)
+    FAIL_UNSUPPORTED ("THP mode is not always nor madvise.\n");
+
+  FILE *f = xfopen ("/proc/self/maps", "r");
+  char *line = NULL;
+  size_t len;
+
+  while (xgetline (&line, &len, f))
+    {
+      uintptr_t from, to;
+      char *prot = NULL, *path = NULL;
+      int r = sscanf (line, "%" SCNxPTR "-%" SCNxPTR "%ms%*s%*s%*s%ms",
+                      &from, &to, &prot, &path);
+
+      TEST_VERIFY (r == 3 || r == 4);
+
+      if (path != NULL && strstr (prot, "x") && strstr (path, name))
+        TEST_COMPARE (from % thp_size, 0);
+
+      free (prot);
+      free (path);
+    }
+
+  free (line);
+  xfclose (f);
+}

diff --git a/sysdeps/unix/sysv/linux/tst-thp-align.c b/sysdeps/unix/sysv/linux/tst-thp-align.c
index 0b3f18e000fb586cc37dca4a76a093496c1b13d8..2e44109ba660bd68071aa8d6c5648399aac4215f 100644 (file)
--- a/sysdeps/unix/sysv/linux/tst-thp-align.c
+++ b/sysdeps/unix/sysv/linux/tst-thp-align.c
@@ -16,129 +16,10 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 

-#include <fcntl.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <intprops.h>
-#include <inttypes.h>
-#include <support/check.h>

 #include <support/xdlfcn.h>
 #include <support/xdlfcn.h>

-#include <support/xstdio.h>
-#include <support/xunistd.h>
+#include "tst-thp-align-check.h"

 
 #define THP_SIZE_MOD_NAME "tst-thp-size-mod.so"
 
 #define THP_SIZE_MOD_NAME "tst-thp-size-mod.so"

-#define MAX_THP_PAGESIZE (32 * 1024 * 1024)
-
-enum thp_mode_t
-{
-  thp_mode_always,
-  thp_mode_madvise,
-  thp_mode_never,
-  thp_mode_not_supported
-};
-
-static unsigned long int
-get_thp_size (void)
-{
-  int fd = open ("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
-                 O_RDONLY, 0);
-  if (fd == -1)
-    return 0;
-
-  char str[INT_BUFSIZE_BOUND (unsigned long int)];
-  ssize_t s = read (fd, str, sizeof (str));
-  close (fd);
-  if (s < 0)
-    return 0;
-
-  unsigned long int r = 0;
-  for (ssize_t i = 0; i < s; i++)
-    {
-      if (str[i] == '\n')
-    break;
-      r *= 10;
-      r += str[i] - '0';
-    }
-  return r;
-}
-
-static enum thp_mode_t
-get_thp_mode (void)
-{
-  int fd = open ("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY, 0);
-  if (fd == -1)
-    return thp_mode_not_supported;
-
-  static const char mode_always[]  = "[always] madvise never\n";
-  static const char mode_madvise[] = "always [madvise] never\n";
-  static const char mode_never[]   = "always madvise [never]\n";
-
-  char str[sizeof(mode_always)];
-  ssize_t s = read (fd, str, sizeof (str));
-  if (s >= sizeof str || s < 0)
-    return thp_mode_not_supported;
-  str[s] = '\0';
-  close (fd);
-
-  if (s == sizeof (mode_always) - 1)
-    {
-      if (strcmp (str, mode_always) == 0)
-    return thp_mode_always;
-      else if (strcmp (str, mode_madvise) == 0)
-    return thp_mode_madvise;
-      else if (strcmp (str, mode_never) == 0)
-    return thp_mode_never;
-    }
-  return thp_mode_not_supported;
-}
-
-static void
-check_align (void)
-{
-  unsigned long int thp_size = get_thp_size ();
-  enum thp_mode_t thp_mode = get_thp_mode ();
-
-  if (thp_size == 0)
-    {
-      FAIL_UNSUPPORTED ("unable to get THP size.\n");
-      return;
-    }
-
-  if (thp_size > MAX_THP_PAGESIZE)
-    {
-      FAIL_UNSUPPORTED ("THP size exceeds MAX_THP_PAGESIZE.\n");
-      return;
-    }
-
-  if (thp_mode != thp_mode_always)
-    {
-      FAIL_UNSUPPORTED ("THP mode is not always.\n");
-      return;
-    }
-
-  FILE *f = xfopen ("/proc/self/maps", "r");
-  char *line = NULL;
-  size_t len;
-
-  while (xgetline (&line, &len, f))
-    {
-      uintptr_t from, to;
-      char *prot = NULL, *path = NULL;
-      int r = sscanf (line, "%" SCNxPTR "-%" SCNxPTR "%ms%*s%*s%*s%ms",
-                      &from, &to, &prot, &path);
-
-      TEST_VERIFY (r == 3 || r == 4);
-
-      if (strstr (prot, "x") && strstr (path, THP_SIZE_MOD_NAME))
-        TEST_COMPARE (from % thp_size, 0);
-
-      free (path);
-    }
-
-  free (line);
-  xfclose (f);
-}

 
 static int
 do_test (void)
 
 static int
 do_test (void)


@@ -146,7 +27,7 @@ do_test (void)
   void *dl;
 
   dl = xdlopen (THP_SIZE_MOD_NAME, RTLD_NOW);
   void *dl;
 
   dl = xdlopen (THP_SIZE_MOD_NAME, RTLD_NOW);

-  check_align ();
+  check_align (THP_SIZE_MOD_NAME);

   xdlclose (dl);
 
   return 0;
   xdlclose (dl);
 
   return 0;

diff --git a/sysdeps/unix/sysv/linux/x86_64/hugepages.h b/sysdeps/unix/sysv/linux/x86_64/hugepages.h
new file mode 100644 (file)
index 0000000..74a3833
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/hugepages.h
@@ -0,0 +1,22 @@
+/* Huge Page support.  Linux/x86-64 version.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   Copyright The GNU Toolchain Authors.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define DL_MAP_DEFAULT_THP_PAGESIZE (2 * 1024 * 1024)
+
+#include_next <hugepages.h>