1 /* Manage function descriptors. Generic version.
2 Copyright (C) 1999-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
23 #include <sys/param.h>
27 #include <elf/dynamic-link.h>
29 #include <dl-runtime.h>
30 #include <dl-unmap-segments.h>
32 #include <libc-pointer-arith.h>
34 #ifndef ELF_MACHINE_BOOT_FPTR_TABLE_LEN
35 /* ELF_MACHINE_BOOT_FPTR_TABLE_LEN should be greater than the number of
36 dynamic symbols in ld.so. */
37 # define ELF_MACHINE_BOOT_FPTR_TABLE_LEN 256
40 #ifndef ELF_MACHINE_LOAD_ADDRESS
41 # error "ELF_MACHINE_LOAD_ADDRESS is not defined."
44 #ifndef COMPARE_AND_SWAP
45 # define COMPARE_AND_SWAP(ptr, old, new) \
46 (catomic_compare_and_exchange_bool_acq (ptr, new, old) == 0)
49 ElfW(Addr
) _dl_boot_fptr_table
[ELF_MACHINE_BOOT_FPTR_TABLE_LEN
];
53 struct fdesc_table
*root
;
54 struct fdesc
*free_list
;
55 unsigned int npages
; /* # of pages to allocate */
56 /* the next to members MUST be consecutive! */
57 struct fdesc_table boot_table
;
58 struct fdesc boot_fdescs
[1024];
63 /* Address of .boot_table is not known until runtime. */
66 .root
= &local
.boot_table
,
71 .len
= sizeof (local
.boot_fdescs
) / sizeof (local
.boot_fdescs
[0]),
76 /* Create a new fdesc table and return a pointer to the first fdesc
77 entry. The fdesc lock must have been acquired already. */
79 static struct fdesc_table
*
80 new_fdesc_table (struct local
*l
, size_t *size
)
82 size_t old_npages
= l
->npages
;
83 size_t new_npages
= old_npages
+ old_npages
;
84 struct fdesc_table
*new_table
;
86 /* If someone has just created a new table, we return NULL to tell
87 the caller to use the new table. */
88 if (! COMPARE_AND_SWAP (&l
->npages
, old_npages
, new_npages
))
89 return (struct fdesc_table
*) NULL
;
91 *size
= old_npages
* GLRO(dl_pagesize
);
92 new_table
= __mmap (NULL
, *size
,
93 PROT_READ
| PROT_WRITE
, MAP_ANON
| MAP_PRIVATE
, -1, 0);
94 if (new_table
== MAP_FAILED
)
95 _dl_signal_error (errno
, NULL
, NULL
,
96 N_("cannot map pages for fdesc table"));
99 = (*size
- sizeof (*new_table
)) / sizeof (struct fdesc
);
100 new_table
->first_unused
= 1;
104 /* Must call _dl_fptr_init before using any other function. */
110 ELF_MACHINE_LOAD_ADDRESS (l
, local
);
111 l
->root
= &l
->boot_table
;
115 make_fdesc (ElfW(Addr
) ip
, ElfW(Addr
) gp
)
117 struct fdesc
*fdesc
= NULL
;
118 struct fdesc_table
*root
;
122 ELF_MACHINE_LOAD_ADDRESS (l
, local
);
128 old
= root
->first_unused
;
129 if (old
>= root
->len
)
131 else if (COMPARE_AND_SWAP (&root
->first_unused
, old
, old
+ 1))
133 fdesc
= &root
->fdesc
[old
];
140 /* Get it from free-list. */
143 fdesc
= l
->free_list
;
147 while (! COMPARE_AND_SWAP ((ElfW(Addr
) *) &l
->free_list
,
148 (ElfW(Addr
)) fdesc
, fdesc
->ip
));
152 /* Create a new fdesc table. */
154 struct fdesc_table
*new_table
= new_fdesc_table (l
, &size
);
156 if (new_table
== NULL
)
159 new_table
->next
= root
;
160 if (! COMPARE_AND_SWAP ((ElfW(Addr
) *) &l
->root
,
162 (ElfW(Addr
)) new_table
))
164 /* Someone has just installed a new table. Return NULL to
165 tell the caller to use the new table. */
166 __munmap (new_table
, size
);
170 /* Note that the first entry was reserved while allocating the
171 memory for the new page. */
172 fdesc
= &new_table
->fdesc
[0];
179 return (ElfW(Addr
)) fdesc
;
183 static inline ElfW(Addr
) * __attribute__ ((always_inline
))
184 make_fptr_table (struct link_map
*map
)
186 const ElfW(Sym
) *symtab
= (const void *) D_PTR (map
, l_info
[DT_SYMTAB
]);
187 const char *strtab
= (const void *) D_PTR (map
, l_info
[DT_STRTAB
]);
188 ElfW(Addr
) *fptr_table
;
191 const ElfW(Sym
) *symtabend
;
193 /* Determine the end of the dynamic symbol table using the hash. */
194 if (map
->l_info
[DT_HASH
] != NULL
)
195 symtabend
= (symtab
+ ((Elf_Symndx
*) D_PTR (map
, l_info
[DT_HASH
]))[1]);
197 /* There is no direct way to determine the number of symbols in the
198 dynamic symbol table and no hash table is present. The ELF
199 binary is ill-formed but what shall we do? Use the beginning of
200 the string table which generally follows the symbol table. */
201 symtabend
= (const ElfW(Sym
) *) strtab
;
203 len
= (((char *) symtabend
- (char *) symtab
)
204 / map
->l_info
[DT_SYMENT
]->d_un
.d_val
);
205 size
= ALIGN_UP (len
* sizeof (fptr_table
[0]), GLRO(dl_pagesize
));
207 /* We don't support systems without MAP_ANON. We avoid using malloc
208 because this might get called before malloc is setup. */
209 fptr_table
= __mmap (NULL
, size
,
210 PROT_READ
| PROT_WRITE
, MAP_ANON
| MAP_PRIVATE
,
212 if (fptr_table
== MAP_FAILED
)
213 _dl_signal_error (errno
, NULL
, NULL
,
214 N_("cannot map pages for fptr table"));
216 if (COMPARE_AND_SWAP ((ElfW(Addr
) *) &map
->l_mach
.fptr_table
,
217 (ElfW(Addr
)) NULL
, (ElfW(Addr
)) fptr_table
))
218 map
->l_mach
.fptr_table_len
= len
;
220 __munmap (fptr_table
, len
* sizeof (fptr_table
[0]));
222 return map
->l_mach
.fptr_table
;
227 _dl_make_fptr (struct link_map
*map
, const ElfW(Sym
) *sym
,
230 ElfW(Addr
) *ftab
= map
->l_mach
.fptr_table
;
231 const ElfW(Sym
) *symtab
;
235 if (__builtin_expect (ftab
== NULL
, 0))
236 ftab
= make_fptr_table (map
);
238 symtab
= (const void *) D_PTR (map
, l_info
[DT_SYMTAB
]);
239 symidx
= sym
- symtab
;
241 if (symidx
>= map
->l_mach
.fptr_table_len
)
242 _dl_signal_error (0, NULL
, NULL
,
243 N_("internal error: symidx out of range of fptr table"));
245 while (ftab
[symidx
] == 0)
247 /* GOT has already been relocated in elf_get_dynamic_info -
248 don't try to relocate it again. */
250 = make_fdesc (ip
, map
->l_info
[DT_PLTGOT
]->d_un
.d_ptr
);
252 if (__builtin_expect (COMPARE_AND_SWAP (&ftab
[symidx
], (ElfW(Addr
)) NULL
,
255 /* No one has updated the entry and the new function
256 descriptor has been installed. */
259 = (const void *) D_PTR (map
, l_info
[DT_STRTAB
]);
261 ELF_MACHINE_LOAD_ADDRESS (l
, local
);
262 if (l
->root
!= &l
->boot_table
263 || l
->boot_table
.first_unused
> 20)
264 _dl_debug_printf ("created fdesc symbol `%s' at %lx\n",
265 strtab
+ sym
->st_name
, ftab
[symidx
]);
271 /* We created a duplicated function descriptor. We put it on
273 struct fdesc
*f
= (struct fdesc
*) fdesc
;
275 ELF_MACHINE_LOAD_ADDRESS (l
, local
);
278 f
->ip
= (ElfW(Addr
)) l
->free_list
;
279 while (! COMPARE_AND_SWAP ((ElfW(Addr
) *) &l
->free_list
,
289 _dl_unmap (struct link_map
*map
)
291 ElfW(Addr
) *ftab
= map
->l_mach
.fptr_table
;
292 struct fdesc
*head
= NULL
, *tail
= NULL
;
295 _dl_unmap_segments (map
);
300 /* String together the fdesc structures that are being freed. */
301 for (i
= 0; i
< map
->l_mach
.fptr_table_len
; ++i
)
305 *(struct fdesc
**) ftab
[i
] = head
;
306 head
= (struct fdesc
*) ftab
[i
];
312 /* Prepend the new list to the free_list: */
315 tail
->ip
= (ElfW(Addr
)) local
.free_list
;
316 while (! COMPARE_AND_SWAP ((ElfW(Addr
) *) &local
.free_list
,
317 tail
->ip
, (ElfW(Addr
)) head
));
319 __munmap (ftab
, (map
->l_mach
.fptr_table_len
320 * sizeof (map
->l_mach
.fptr_table
[0])));
322 map
->l_mach
.fptr_table
= NULL
;
325 extern ElfW(Addr
) _dl_fixup (struct link_map
*, ElfW(Word
)) attribute_hidden
;
327 static inline Elf32_Addr
328 elf_machine_resolve (void)
333 " addil L'_dl_runtime_resolve - ($PIC_pcrel$0 - 1),%0\n"
334 "1: ldo R'_dl_runtime_resolve - ($PIC_pcrel$0 - 5)(%%r1),%0\n"
335 : "=r" (addr
) : : "r1");
341 _dl_read_access_allowed (unsigned int *addr
)
345 asm ("proberi (%1),3,%0" : "=r" (result
) : "r" (addr
) : );
351 _dl_lookup_address (const void *address
)
353 ElfW(Addr
) addr
= (ElfW(Addr
)) address
;
354 ElfW(Word
) reloc_arg
;
355 unsigned int *desc
, *gptr
;
357 /* Return ADDR if the least-significant two bits of ADDR are not consistent
358 with ADDR being a linker defined function pointer. The normal value for
359 a code address in a backtrace is 3. */
360 if (((uintptr_t) addr
& 3) != 2)
363 /* Handle special case where ADDR points to page 0. */
364 if ((uintptr_t) addr
< 4096)
367 /* Clear least-significant two bits from descriptor address. */
368 desc
= (unsigned int *) ((uintptr_t) addr
& ~3);
369 if (!_dl_read_access_allowed (desc
))
372 /* First load the relocation offset. */
373 reloc_arg
= (ElfW(Word
)) desc
[1];
374 atomic_full_barrier();
376 /* Then load first word of candidate descriptor. It should be a pointer
377 with word alignment and point to memory that can be read. */
378 gptr
= (unsigned int *) desc
[0];
379 if (((uintptr_t) gptr
& 3) != 0
380 || !_dl_read_access_allowed (gptr
))
383 /* See if descriptor requires resolution. The following trampoline is
384 used in each global offset table for function resolution:
391 .word _dl_runtime_resolve
392 .word "_dl_runtime_resolve ltp"
394 .word "struct link map address" */
395 if (gptr
[0] == 0xea9f1fdd /* b,l .-12,r20 */
396 && gptr
[1] == 0xd6801c1e /* depwi 0,31,2,r20 */
397 && (ElfW(Addr
)) gptr
[2] == elf_machine_resolve ())
399 struct link_map
*l
= (struct link_map
*) gptr
[5];
401 /* If gp has been resolved, we need to hunt for relocation offset. */
402 if (!(reloc_arg
& PA_GP_RELOC
))
403 reloc_arg
= _dl_fix_reloc_arg ((struct fdesc
*) addr
, l
);
405 _dl_fixup (l
, reloc_arg
);
408 return (ElfW(Addr
)) desc
[0];
410 rtld_hidden_def (_dl_lookup_address
)