]>
Commit | Line | Data |
---|---|---|
7adcbafe | 1 | /* Copyright (C) 2012-2022 Free Software Foundation, Inc. |
2077db1b CT |
2 | |
3 | This file is part of GCC. | |
4 | ||
5 | GCC is free software; you can redistribute it and/or modify | |
6 | it under the terms of the GNU General Public License as published by | |
7 | the Free Software Foundation; either version 3, or (at your option) | |
8 | any later version. | |
9 | ||
10 | GCC is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | GNU General Public License for more details. | |
14 | ||
15 | Under Section 7 of GPL version 3, you are granted additional | |
16 | permissions described in the GCC Runtime Library Exception, version | |
17 | 3.1, as published by the Free Software Foundation. | |
18 | ||
19 | You should have received a copy of the GNU General Public License and | |
20 | a copy of the GCC Runtime Library Exception along with this program; | |
21 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
22 | <http://www.gnu.org/licenses/>. */ | |
23 | ||
24 | /* This file is part of the vtable security feature implementation. | |
25 | The vtable security feature is designed to detect when a virtual | |
26 | call is about to be made through an invalid vtable pointer | |
27 | (possibly due to data corruption or malicious attacks). The | |
28 | compiler finds every virtual call, and inserts a verification call | |
29 | before the virtual call. The verification call takes the actual | |
30 | vtable pointer value in the object through which the virtual call | |
31 | is being made, and compares the vtable pointer against a set of all | |
32 | valid vtable pointers that the object could contain (this set is | |
33 | based on the declared type of the object). If the pointer is in | |
34 | the valid set, execution is allowed to continue; otherwise the | |
35 | program is halted. | |
36 | ||
37 | There are several pieces needed in order to make this work: 1. For | |
38 | every virtual class in the program (i.e. a class that contains | |
39 | virtual methods), we need to build the set of all possible valid | |
40 | vtables that an object of that class could point to. This includes | |
41 | vtables for any class(es) that inherit from the class under | |
42 | consideration. 2. For every such data set we build up, we need a | |
43 | way to find and reference the data set. This is complicated by the | |
44 | fact that the real vtable addresses are not known until runtime, | |
45 | when the program is loaded into memory, but we need to reference the | |
46 | sets at compile time when we are inserting verification calls into | |
47 | the program. 3. We need to find every virtual call in the program, | |
48 | and insert the verification call (with the appropriate arguments) | |
49 | before the virtual call. 4. We need some runtime library pieces: | |
50 | the code to build up the data sets at runtime; the code to actually | |
51 | perform the verification using the data sets; and some code to set | |
52 | protections on the data sets, so they themselves do not become | |
53 | hacker targets. | |
54 | ||
55 | To find and reference the set of valid vtable pointers for any given | |
56 | virtual class, we create a special global varible for each virtual | |
57 | class. We refer to this as the "vtable map variable" for that | |
58 | class. The vtable map variable has the type "void *", and is | |
59 | initialized by the compiler to NULL. At runtime when the set of | |
60 | valid vtable pointers for a virtual class, e.g. class Foo, is built, | |
61 | the vtable map variable for class Foo is made to point to the set. | |
62 | During compile time, when the compiler is inserting verification | |
63 | calls into the program, it passes the vtable map variable for the | |
64 | appropriate class to the verification call, so that at runtime the | |
65 | verification call can find the appropriate data set. | |
66 | ||
67 | The actual set of valid vtable pointers for a polymorphic class, | |
68 | e.g. class Foo, cannot be built until runtime, when the vtables get | |
69 | loaded into memory and their addresses are known. But the knowledge | |
70 | about which vtables belong in which class' hierarchy is only known | |
71 | at compile time. Therefore at compile time we collect class | |
72 | hierarchy and vtable information about every virtual class, and we | |
73 | generate calls to build up the data sets at runtime. To build the | |
74 | data sets, we call one of the functions we add to the runtime | |
75 | library, __VLTRegisterPair. __VLTRegisterPair takes two arguments, | |
76 | a vtable map variable and the address of a vtable. If the vtable | |
77 | map variable is currently NULL, it creates a new data set (hash | |
78 | table), makes the vtable map variable point to the new data set, and | |
79 | inserts the vtable address into the data set. If the vtable map | |
80 | variable is not NULL, it just inserts the vtable address into the | |
81 | data set. In order to make sure that our data sets are built before | |
82 | any verification calls happen, we create a special constructor | |
83 | initialization function for each compilation unit, give it a very | |
84 | high initialization priority, and insert all of our calls to | |
85 | __VLTRegisterPair into our special constructor initialization | |
86 | function. */ | |
87 | ||
88 | /* This file contains the main externally visible runtime library | |
89 | functions for vtable verification: __VLTChangePermission, | |
90 | __VLTRegisterPair, and __VLTVerifyVtablePointer. It also contains | |
91 | debug versions __VLTRegisterPairDebug and | |
92 | __VLTVerifyVtablePointerDebug, which have extra parameters in order | |
93 | to make it easier to debug verification failures. | |
94 | ||
95 | The final piece of functionality implemented in this file is symbol | |
96 | resolution for multiple instances of the same vtable map variable. | |
97 | If the same virtual class is used in two different compilation | |
98 | units, then each compilation unit will create a vtable map variable | |
99 | for the class. We need all instances of the same vtable map | |
100 | variable to point to the same (single) set of valid vtable | |
101 | pointers for the class, so we wrote our own hashtable-based symbol | |
102 | resolution for vtable map variables (with a tiny optimization in | |
103 | the case where there is only one instance of the variable). | |
104 | ||
105 | There are two other important pieces to the runtime for vtable | |
106 | verification besides the main pieces that go into libstdc++.so: two | |
107 | special tiny shared libraries, libvtv_init.so and libvtv_stubs.so. | |
108 | libvtv_init.so is built from vtv_init.cc. It is designed to help | |
109 | minimize the calls made to mprotect (see the comments in | |
110 | vtv_init.cc for more details). Anything compiled with | |
111 | "-fvtable-verify=std" must be linked with libvtv_init.so (the gcc | |
112 | driver has been modified to do this). vtv_stubs.so is built from | |
113 | vtv_stubs.cc. It replaces the main runtime functions | |
114 | (__VLTChangePermissino, __VLTRegisterPair and | |
115 | __VLTVerifyVtablePointer) with stub functions that do nothing. If | |
116 | a programmer has a library that was built with verification, but | |
117 | wishes to not have verification turned on, the programmer can link | |
118 | in the vtv_stubs.so library. */ | |
119 | ||
120 | #include <stdlib.h> | |
121 | #include <stdio.h> | |
122 | #include <string.h> | |
f7f049fa | 123 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
902c7559 | 124 | #define WIN32_LEAN_AND_MEAN |
f7f049fa CT |
125 | #include <windows.h> |
126 | #include <winternl.h> | |
127 | #include <psapi.h> | |
128 | #else | |
2077db1b | 129 | #include <execinfo.h> |
f7f049fa | 130 | #endif |
2077db1b CT |
131 | |
132 | #include <unistd.h> | |
f7f049fa | 133 | #if !defined (__CYGWIN__) && !defined (__MINGW32__) |
2077db1b | 134 | #include <sys/mman.h> |
2077db1b | 135 | #include <link.h> |
f7f049fa CT |
136 | #endif |
137 | #include <errno.h> | |
2077db1b CT |
138 | #include <fcntl.h> |
139 | #include <limits.h> | |
140 | ||
141 | /* For gthreads suppport */ | |
142 | #include <bits/c++config.h> | |
143 | #include <ext/concurrence.h> | |
144 | ||
145 | #include "vtv_utils.h" | |
146 | #include "vtv_malloc.h" | |
147 | #include "vtv_set.h" | |
148 | #include "vtv_map.h" | |
149 | #include "vtv_rts.h" | |
150 | #include "vtv_fail.h" | |
151 | ||
152 | #include "vtv-change-permission.h" | |
153 | ||
e5ef217c RO |
154 | #ifdef HAVE_GETEXECNAME |
155 | const char *program_invocation_name; | |
156 | #endif | |
157 | ||
158 | #ifdef HAVE___FORTIFY_FAIL | |
2077db1b CT |
159 | extern "C" { |
160 | ||
161 | /* __fortify_fail is a function in glibc that calls __libc_message, | |
162 | causing it to print out a program termination error message | |
163 | (including the name of the binary being terminated), a stack | |
164 | trace where the error occurred, and a memory map dump. Ideally | |
165 | we would have called __libc_message directly, but that function | |
166 | does not appear to be accessible to functions outside glibc, | |
167 | whereas __fortify_fail is. We call __fortify_fail from | |
168 | __vtv_really_fail. We looked at calling __libc_fatal, which is | |
169 | externally accessible, but it does not do the back trace and | |
170 | memory dump. */ | |
171 | ||
172 | extern void __fortify_fail (const char *) __attribute__((noreturn)); | |
173 | ||
174 | } /* extern "C" */ | |
e5ef217c RO |
175 | #else |
176 | #if defined (__CYGWIN__) || defined (__MINGW32__) | |
177 | // porting: fix link error to libc | |
178 | void __fortify_fail (const char * msg){ | |
179 | OutputDebugString(msg); | |
180 | abort(); | |
181 | } | |
182 | #else | |
183 | // FIXME: Provide backtrace via libbacktrace? | |
184 | void __fortify_fail (const char *msg) { | |
185 | write (2, msg, strlen (msg)); | |
186 | abort (); | |
187 | } | |
188 | #endif | |
f7f049fa | 189 | #endif |
2077db1b CT |
190 | |
191 | /* The following variables are used only for debugging and performance | |
192 | tuning purposes. Therefore they do not need to be "protected". | |
193 | They cannot be used to attack the vtable verification system and if | |
194 | they become corrupted it will not affect the correctness or | |
195 | security of any of the rest of the vtable verification feature. */ | |
196 | ||
197 | unsigned int num_calls_to_regset = 0; | |
198 | unsigned int num_calls_to_regpair = 0; | |
199 | unsigned int num_calls_to_verify_vtable = 0; | |
200 | unsigned long long regset_cycles = 0; | |
201 | unsigned long long regpair_cycles = 0; | |
202 | unsigned long long verify_vtable_cycles = 0; | |
203 | ||
204 | /* Be careful about initialization of statics in this file. Some of | |
205 | the routines below are called before any runtime initialization for | |
206 | statics in this file will be done. For example, dont try to | |
207 | initialize any of these statics with a runtime call (for ex: | |
208 | sysconf). The initialization will happen after calls to the routines | |
209 | to protect/unprotec the vtabla_map variables */ | |
210 | ||
211 | /* No need to mark the following variables with VTV_PROTECTED_VAR. | |
212 | These are either const or are only used for debugging/tracing. | |
213 | debugging/tracing will not be ON on production environments */ | |
214 | ||
215 | static const bool debug_hash = HASHTABLE_STATS; | |
2077db1b CT |
216 | |
217 | #ifdef VTV_DEBUG | |
218 | static const int debug_functions = 1; | |
219 | static const int debug_init = 1; | |
220 | static const int debug_verify_vtable = 1; | |
b0cca5ec CT |
221 | #else |
222 | static const int debug_functions = 0; | |
223 | static const int debug_init = 0; | |
224 | static const int debug_verify_vtable = 0; | |
2077db1b CT |
225 | #endif |
226 | ||
227 | /* Global file descriptor variables for logging, tracing and debugging. */ | |
228 | ||
229 | static int init_log_fd = -1; | |
230 | static int verify_vtable_log_fd = -1; | |
231 | ||
232 | /* This holds a formatted error logging message, to be written to the | |
233 | vtable verify failures log. */ | |
234 | static char debug_log_message[1024]; | |
235 | ||
236 | ||
237 | #ifdef __GTHREAD_MUTEX_INIT | |
238 | static __gthread_mutex_t change_permissions_lock = __GTHREAD_MUTEX_INIT; | |
239 | #else | |
240 | static __gthread_mutex_t change_permissions_lock; | |
241 | #endif | |
242 | ||
243 | ||
244 | #ifndef VTV_STATS | |
245 | #define VTV_STATS 0 | |
246 | #endif | |
247 | ||
248 | #if VTV_STATS | |
249 | ||
250 | static inline unsigned long long | |
251 | get_cycle_count (void) | |
252 | { | |
253 | return rdtsc(); | |
254 | } | |
255 | ||
256 | static inline void | |
257 | accumulate_cycle_count (unsigned long long *sum, unsigned long long start) | |
258 | { | |
259 | unsigned long long end = rdtsc(); | |
260 | *sum = *sum + (end - start); | |
261 | } | |
262 | ||
263 | static inline void | |
264 | increment_num_calls (unsigned int *num_calls) | |
265 | { | |
266 | *num_calls = *num_calls + 1; | |
267 | } | |
268 | ||
269 | #else | |
270 | ||
271 | static inline unsigned long long | |
272 | get_cycle_count (void) | |
273 | { | |
274 | return (unsigned long long) 0; | |
275 | } | |
276 | ||
277 | static inline void | |
278 | accumulate_cycle_count (unsigned long long *sum __attribute__((__unused__)), | |
279 | unsigned long long start __attribute__((__unused__))) | |
280 | { | |
281 | /* Do nothing. */ | |
282 | } | |
283 | ||
284 | static inline void | |
285 | increment_num_calls (unsigned int *num_calls __attribute__((__unused__))) | |
286 | { | |
287 | /* Do nothing. */ | |
288 | } | |
289 | ||
290 | #endif | |
291 | ||
292 | /* Types needed by insert_only_hash_sets. */ | |
293 | typedef uintptr_t int_vptr; | |
294 | ||
295 | /* The set of valid vtable pointers for each virtual class is stored | |
296 | in a hash table. This is the hashing function used for the hash | |
297 | table. For more information on the implementation of the hash | |
298 | table, see the class insert_only_hash_sets in vtv_set.h. */ | |
299 | ||
300 | struct vptr_hash | |
301 | { | |
302 | /* Hash function, used to convert vtable pointer, V, (a memory | |
303 | address) into an index into the hash table. */ | |
304 | size_t | |
305 | operator() (int_vptr v) const | |
306 | { | |
307 | const uint32_t x = 0x7a35e4d9; | |
308 | const int shift = (sizeof (v) == 8) ? 23 : 21; | |
309 | v = x * v; | |
310 | return v ^ (v >> shift); | |
311 | } | |
312 | }; | |
313 | ||
314 | /* This is the memory allocator used to create the hash table data | |
315 | sets of valid vtable pointers. We use VTV_malloc in order to keep | |
316 | track of which pages have been allocated, so we can update the | |
317 | protections on those pages appropriately. See the class | |
318 | insert_only_hash_sets in vtv_set.h for more information. */ | |
319 | ||
320 | struct vptr_set_alloc | |
321 | { | |
322 | /* Memory allocator operator. N is the number of bytes to be | |
323 | allocated. */ | |
324 | void * | |
325 | operator() (size_t n) const | |
326 | { | |
327 | return __vtv_malloc (n); | |
328 | } | |
329 | }; | |
330 | ||
331 | /* Instantiate the template classes (in vtv_set.h) for our particular | |
332 | hash table needs. */ | |
333 | typedef insert_only_hash_sets<int_vptr, vptr_hash, vptr_set_alloc> vtv_sets; | |
334 | typedef vtv_sets::insert_only_hash_set vtv_set; | |
335 | typedef vtv_set * vtv_set_handle; | |
336 | typedef vtv_set_handle * vtv_set_handle_handle; | |
337 | ||
338 | /* Records for caching the section header information that we have | |
339 | read out of the file(s) on disk (in dl_iterate_phdr_callback), to | |
340 | avoid having to re-open and re-read the same file multiple | |
341 | times. */ | |
342 | ||
343 | struct sect_hdr_data | |
344 | { | |
f7f049fa CT |
345 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
346 | uintptr_t dlpi_addr; /* The header address in the INFO record, | |
347 | passed in from dl_iterate_phdr. */ | |
348 | uintptr_t mp_low; /* Start address of the .vtable_map_vars | |
349 | section in memory. */ | |
350 | #else | |
2077db1b CT |
351 | ElfW (Addr) dlpi_addr; /* The header address in the INFO record, |
352 | passed in from dl_iterate_phdr. */ | |
353 | ElfW (Addr) mp_low; /* Start address of the .vtable_map_vars | |
354 | section in memory. */ | |
f7f049fa | 355 | #endif |
2077db1b CT |
356 | size_t mp_size; /* Size of the .vtable_map_vars section in |
357 | memory. */ | |
358 | }; | |
359 | ||
360 | /* Array for caching the section header information, read from file, | |
361 | to avoid re-opening and re-reading the same file over-and-over | |
362 | again. */ | |
363 | ||
364 | #define MAX_ENTRIES 250 | |
365 | static struct sect_hdr_data vtv_sect_info_cache[MAX_ENTRIES] VTV_PROTECTED_VAR; | |
366 | ||
367 | unsigned int num_cache_entries VTV_PROTECTED_VAR = 0; | |
368 | ||
369 | /* This function takes the LOAD_ADDR for an object opened by the | |
370 | dynamic loader, and checks the array of cached file data to see if | |
371 | there is an entry with the same addres. If it finds such an entry, | |
372 | it returns the record for that entry; otherwise it returns | |
373 | NULL. */ | |
374 | ||
f7f049fa CT |
375 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
376 | struct sect_hdr_data * | |
377 | search_cached_file_data (uintptr_t load_addr) | |
378 | #else | |
2077db1b CT |
379 | struct sect_hdr_data * |
380 | search_cached_file_data (ElfW (Addr) load_addr) | |
f7f049fa | 381 | #endif |
2077db1b CT |
382 | { |
383 | unsigned int i; | |
384 | for (i = 0; i < num_cache_entries; ++i) | |
385 | { | |
386 | if (vtv_sect_info_cache[i].dlpi_addr == load_addr) | |
387 | return &(vtv_sect_info_cache[i]); | |
388 | } | |
389 | ||
390 | return NULL; | |
391 | } | |
392 | ||
393 | /* This function tries to read COUNT bytes out of the file referred to | |
394 | by FD into the buffer BUF. It returns the actual number of bytes | |
395 | it succeeded in reading. */ | |
396 | ||
397 | static size_t | |
398 | ReadPersistent (int fd, void *buf, size_t count) | |
399 | { | |
400 | char *buf0 = (char *) buf; | |
401 | size_t num_bytes = 0; | |
402 | while (num_bytes < count) | |
403 | { | |
404 | int len; | |
405 | len = read (fd, buf0 + num_bytes, count - num_bytes); | |
406 | if (len < 0) | |
407 | return -1; | |
408 | if (len == 0) | |
409 | break; | |
410 | num_bytes += len; | |
411 | } | |
412 | ||
413 | return num_bytes; | |
414 | } | |
415 | ||
416 | /* This function tries to read COUNT bytes, starting at OFFSET from | |
417 | the file referred to by FD, and put them into BUF. It calls | |
418 | ReadPersistent to help it do so. It returns the actual number of | |
419 | bytes read, or -1 if it fails altogether. */ | |
420 | ||
421 | static size_t | |
422 | ReadFromOffset (int fd, void *buf, const size_t count, const off_t offset) | |
423 | { | |
424 | off_t off = lseek (fd, offset, SEEK_SET); | |
425 | if (off != (off_t) -1) | |
426 | return ReadPersistent (fd, buf, count); | |
427 | return -1; | |
428 | } | |
429 | ||
430 | /* The function takes a MESSAGE and attempts to write it to the vtable | |
431 | memory protection log (for debugging purposes). If the file is not | |
432 | open, it attempts to open the file first. */ | |
433 | ||
434 | static void | |
435 | log_memory_protection_data (char *message) | |
436 | { | |
437 | static int log_fd = -1; | |
438 | ||
439 | if (log_fd == -1) | |
8bc16536 | 440 | log_fd = __vtv_open_log ("vtv_memory_protection_data.log"); |
2077db1b CT |
441 | |
442 | __vtv_add_to_log (log_fd, "%s", message); | |
443 | } | |
444 | ||
f7f049fa CT |
445 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
446 | static void | |
447 | read_section_offset_and_length (char *name, | |
448 | uintptr_t addr, | |
449 | const char *sect_name, | |
450 | int mprotect_flags, | |
451 | off_t *sect_offset, | |
452 | WORD *sect_len) | |
453 | { | |
454 | bool found = false; | |
455 | struct sect_hdr_data *cached_data = NULL; | |
456 | ||
457 | /* Check to see if we already have the data for this file. */ | |
458 | cached_data = search_cached_file_data (addr); | |
459 | ||
460 | if (cached_data) | |
461 | { | |
462 | *sect_offset = cached_data->mp_low; | |
463 | *sect_len = cached_data->mp_size; | |
464 | return; | |
465 | } | |
466 | ||
467 | // check for DOS Header magic bytes | |
468 | if (*(WORD *)addr == 0x5A4D) | |
469 | { | |
470 | int name_len = strlen (sect_name); | |
471 | int fd = -1; | |
472 | ||
473 | /* Attempt to open the binary file on disk. */ | |
474 | if (strlen (name) == 0) | |
475 | { | |
476 | return; | |
477 | } | |
478 | else | |
479 | fd = open (name, O_RDONLY | O_BINARY); | |
480 | ||
481 | if (fd != -1) | |
482 | { | |
483 | /* Find the section header information in memory. */ | |
484 | PIMAGE_DOS_HEADER pDosHeader = (PIMAGE_DOS_HEADER)addr; | |
485 | PIMAGE_NT_HEADERS pNtHeaders = (PIMAGE_NT_HEADERS)((char *)addr | |
486 | + pDosHeader->e_lfanew); | |
487 | PIMAGE_FILE_HEADER pFileHeader = &pNtHeaders->FileHeader; | |
488 | ||
489 | DWORD PointerToStringTable = pFileHeader->PointerToSymbolTable | |
490 | + (pFileHeader->NumberOfSymbols*0x12); | |
491 | ||
492 | PIMAGE_SECTION_HEADER sect_hdr = | |
493 | (PIMAGE_SECTION_HEADER)((char *)&pNtHeaders->OptionalHeader | |
494 | + pFileHeader->SizeOfOptionalHeader); | |
495 | ||
496 | /* Loop through all the section headers, looking for one whose | |
497 | name is ".vtable_map_vars". */ | |
498 | ||
499 | for (int i = 0; i < pFileHeader->NumberOfSections && !found; ++i) | |
500 | { | |
501 | char header_name[64]; | |
502 | ||
503 | /* Check if we have to get the section name from the COFF string | |
504 | table. */ | |
505 | if (sect_hdr[i].Name[0] == '/') | |
506 | { | |
507 | if (atoi((const char*)sect_hdr[i].Name+1) == 0) | |
508 | { | |
509 | continue; | |
510 | } | |
511 | ||
512 | off_t name_offset = PointerToStringTable | |
513 | + atoi((const char*)sect_hdr[i].Name+1); | |
514 | ||
515 | size_t bytes_read = ReadFromOffset (fd, &header_name, 64, | |
516 | name_offset); | |
517 | ||
518 | VTV_ASSERT (bytes_read > 0); | |
519 | } | |
520 | else | |
521 | { | |
522 | memcpy (&header_name, sect_hdr[i].Name, | |
523 | sizeof (sect_hdr[i].Name)); | |
524 | } | |
525 | ||
526 | if (memcmp (header_name, sect_name, name_len) == 0) | |
527 | { | |
528 | /* We found the section; get its load offset and | |
529 | size. */ | |
530 | *sect_offset = sect_hdr[i].VirtualAddress; | |
531 | if (sect_hdr[i].Misc.VirtualSize % VTV_PAGE_SIZE != 0) | |
532 | *sect_len = sect_hdr[i].Misc.VirtualSize + VTV_PAGE_SIZE | |
533 | - (sect_hdr[i].Misc.VirtualSize % VTV_PAGE_SIZE); | |
534 | else | |
535 | *sect_len = sect_hdr[i].Misc.VirtualSize; | |
536 | found = true; | |
537 | } | |
538 | } | |
539 | close (fd); | |
540 | } | |
541 | } | |
542 | ||
543 | if (*sect_offset != 0 && *sect_len != 0) | |
544 | { | |
545 | /* Calculate the page location in memory, making sure the | |
546 | address is page-aligned. */ | |
547 | uintptr_t start_addr = addr + *sect_offset; | |
548 | *sect_offset = start_addr & ~(VTV_PAGE_SIZE - 1); | |
549 | *sect_len = *sect_len - 1; | |
550 | ||
551 | /* Since we got this far, we must not have found these pages in | |
552 | the cache, so add them to it. NOTE: We could get here either | |
553 | while making everything read-only or while making everything | |
554 | read-write. We will only update the cache if we get here on | |
555 | a read-write (to make absolutely sure the cache is writable | |
556 | -- also the read-write pass should come before the read-only | |
557 | pass). */ | |
558 | if ((mprotect_flags & PROT_WRITE) | |
559 | && num_cache_entries < MAX_ENTRIES) | |
560 | { | |
561 | vtv_sect_info_cache[num_cache_entries].dlpi_addr = addr; | |
562 | vtv_sect_info_cache[num_cache_entries].mp_low = *sect_offset; | |
563 | vtv_sect_info_cache[num_cache_entries].mp_size = *sect_len; | |
564 | num_cache_entries++; | |
565 | } | |
566 | } | |
567 | } | |
568 | #else | |
2077db1b CT |
569 | static void |
570 | read_section_offset_and_length (struct dl_phdr_info *info, | |
571 | const char *sect_name, | |
572 | int mprotect_flags, | |
573 | off_t *sect_offset, | |
574 | ElfW (Word) *sect_len) | |
575 | { | |
576 | char program_name[PATH_MAX]; | |
577 | char *cptr; | |
578 | bool found = false; | |
579 | struct sect_hdr_data *cached_data = NULL; | |
580 | const ElfW (Phdr) *phdr_info = info->dlpi_phdr; | |
581 | const ElfW (Ehdr) *ehdr_info = | |
582 | (const ElfW (Ehdr) *) (info->dlpi_addr + info->dlpi_phdr[0].p_vaddr | |
583 | - info->dlpi_phdr[0].p_offset); | |
584 | ||
585 | ||
586 | /* Get the name of the main executable. This may or may not include | |
587 | arguments passed to the program. Find the first space, assume it | |
588 | is the start of the argument list, and change it to a '\0'. */ | |
e5ef217c RO |
589 | #ifdef HAVE_GETEXECNAME |
590 | program_invocation_name = getexecname (); | |
591 | #endif | |
2077db1b CT |
592 | snprintf (program_name, sizeof (program_name), program_invocation_name); |
593 | ||
594 | /* Check to see if we already have the data for this file. */ | |
595 | cached_data = search_cached_file_data (info->dlpi_addr); | |
596 | ||
597 | if (cached_data) | |
598 | { | |
599 | *sect_offset = cached_data->mp_low; | |
600 | *sect_len = cached_data->mp_size; | |
601 | return; | |
602 | } | |
603 | ||
604 | /* Find the first non-escaped space in the program name and make it | |
605 | the end of the string. */ | |
606 | cptr = strchr (program_name, ' '); | |
607 | if (cptr != NULL && cptr[-1] != '\\') | |
608 | cptr[0] = '\0'; | |
609 | ||
610 | if ((phdr_info->p_type == PT_PHDR || phdr_info->p_type == PT_LOAD) | |
611 | && (ehdr_info->e_shoff && ehdr_info->e_shnum)) | |
612 | { | |
613 | int name_len = strlen (sect_name); | |
614 | int fd = -1; | |
615 | ||
616 | /* Attempt to open the binary file on disk. */ | |
617 | if (strlen (info->dlpi_name) == 0) | |
618 | { | |
619 | /* If the constructor initialization function was put into | |
620 | the preinit array, then this function will get called | |
621 | while handling preinit array stuff, in which case | |
622 | program_invocation_name has not been initialized. In | |
623 | that case we can get the filename of the executable from | |
624 | "/proc/self/exe". */ | |
625 | if (strlen (program_name) > 0) | |
626 | { | |
627 | if (phdr_info->p_type == PT_PHDR) | |
628 | fd = open (program_name, O_RDONLY); | |
629 | } | |
630 | else | |
631 | fd = open ("/proc/self/exe", O_RDONLY); | |
632 | } | |
633 | else | |
634 | fd = open (info->dlpi_name, O_RDONLY); | |
635 | ||
636 | if (fd != -1) | |
637 | { | |
638 | /* Find the section header information in the file. */ | |
639 | ElfW (Half) strtab_idx = ehdr_info->e_shstrndx; | |
640 | ElfW (Shdr) shstrtab; | |
641 | off_t shstrtab_offset = ehdr_info->e_shoff + | |
642 | (ehdr_info->e_shentsize * strtab_idx); | |
643 | size_t bytes_read = ReadFromOffset (fd, &shstrtab, sizeof (shstrtab), | |
644 | shstrtab_offset); | |
645 | VTV_ASSERT (bytes_read == sizeof (shstrtab)); | |
646 | ||
647 | ElfW (Shdr) sect_hdr; | |
648 | ||
649 | /* This code will be needed once we have crated libvtv.so. */ | |
650 | bool is_libvtv = false; | |
651 | ||
652 | /* | |
653 | if (strstr (info->dlpi_name, "libvtv.so")) | |
654 | is_libvtv = true; | |
655 | */ | |
656 | ||
657 | /* Loop through all the section headers, looking for one whose | |
658 | name is ".vtable_map_vars". */ | |
659 | ||
660 | for (int i = 0; i < ehdr_info->e_shnum && !found; ++i) | |
661 | { | |
662 | off_t offset = ehdr_info->e_shoff + (ehdr_info->e_shentsize * i); | |
663 | ||
664 | bytes_read = ReadFromOffset (fd, §_hdr, sizeof (sect_hdr), | |
665 | offset); | |
666 | ||
667 | VTV_ASSERT (bytes_read == sizeof (sect_hdr)); | |
668 | ||
669 | char header_name[64]; | |
670 | off_t name_offset = shstrtab.sh_offset + sect_hdr.sh_name; | |
671 | ||
672 | bytes_read = ReadFromOffset (fd, &header_name, 64, name_offset); | |
673 | ||
674 | VTV_ASSERT (bytes_read > 0); | |
675 | ||
676 | if (memcmp (header_name, sect_name, name_len) == 0) | |
677 | { | |
678 | /* We found the section; get its load offset and | |
679 | size. */ | |
680 | *sect_offset = sect_hdr.sh_addr; | |
681 | if (!is_libvtv) | |
e5ef217c RO |
682 | { |
683 | VTV_ASSERT (sect_hdr.sh_size - VTV_PAGE_SIZE >= 0); | |
684 | *sect_len = sect_hdr.sh_size - VTV_PAGE_SIZE; | |
685 | } | |
2077db1b CT |
686 | else |
687 | *sect_len = sect_hdr.sh_size; | |
688 | found = true; | |
689 | } | |
690 | } | |
691 | close (fd); | |
692 | } | |
693 | } | |
694 | ||
695 | if (*sect_offset != 0 && *sect_len != 0) | |
696 | { | |
697 | /* Calculate the page location in memory, making sure the | |
698 | address is page-aligned. */ | |
699 | ElfW (Addr) start_addr = (const ElfW (Addr)) info->dlpi_addr | |
700 | + *sect_offset; | |
701 | *sect_offset = start_addr & ~(VTV_PAGE_SIZE - 1); | |
702 | *sect_len = *sect_len - 1; | |
703 | ||
704 | /* Since we got this far, we must not have found these pages in | |
705 | the cache, so add them to it. NOTE: We could get here either | |
706 | while making everything read-only or while making everything | |
707 | read-write. We will only update the cache if we get here on | |
708 | a read-write (to make absolutely sure the cache is writable | |
709 | -- also the read-write pass should come before the read-only | |
710 | pass). */ | |
711 | if ((mprotect_flags & PROT_WRITE) | |
712 | && num_cache_entries < MAX_ENTRIES) | |
713 | { | |
714 | vtv_sect_info_cache[num_cache_entries].dlpi_addr = info->dlpi_addr; | |
715 | vtv_sect_info_cache[num_cache_entries].mp_low = *sect_offset; | |
716 | vtv_sect_info_cache[num_cache_entries].mp_size = *sect_len; | |
717 | num_cache_entries++; | |
718 | } | |
719 | } | |
720 | } | |
f7f049fa CT |
721 | #endif |
722 | ||
723 | #if defined (__CYGWIN__) || defined (__MINGW32__) | |
724 | /* This function is used to iterate over all loaded modules and searches | |
725 | for a section called ".vtable_map_vars". The only interaction with | |
726 | the binary file on disk of the module is to read section names in the | |
727 | COFF string table. If the module contains a ".vtable_map_vars" section, | |
728 | read section offset and size from the section header of the loaded module. | |
729 | Call 'mprotect' on those pages, setting the protection either to | |
730 | read-only or read-write, depending on what's in data. | |
731 | The calls to change the protection occur in vtv_unprotect_vtable_vars | |
732 | and vtv_protect_vtable_vars. */ | |
733 | ||
734 | static int | |
735 | iterate_modules (void *data) | |
736 | { | |
737 | int * mprotect_flags = (int *) data; | |
738 | off_t map_sect_offset = 0; | |
739 | WORD map_sect_len = 0; | |
740 | char buffer[1024]; | |
741 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
742 | HMODULE hMods[1024]; | |
743 | HANDLE hProcess; | |
744 | DWORD cbNeeded; | |
2077db1b | 745 | |
f7f049fa CT |
746 | hProcess = GetCurrentProcess (); |
747 | ||
748 | if (NULL == hProcess) | |
749 | return 0; | |
750 | ||
751 | if (EnumProcessModules (hProcess, hMods, sizeof (hMods), &cbNeeded)) | |
752 | { | |
753 | /* Iterate over all loaded modules. */ | |
754 | for (unsigned int i = 0; i < (cbNeeded / sizeof (HMODULE)); i++) | |
755 | { | |
756 | char szModName[MAX_PATH]; | |
757 | ||
758 | if (GetModuleFileNameExA (hProcess, hMods[i], szModName, | |
759 | sizeof (szModName))) | |
760 | { | |
761 | map_sect_offset = 0; | |
762 | map_sect_len = 0; | |
763 | read_section_offset_and_length (szModName, | |
764 | (uintptr_t) hMods[i], | |
765 | map_sect_name, | |
766 | *mprotect_flags, | |
767 | &map_sect_offset, | |
768 | &map_sect_len); | |
769 | ||
770 | if (debug_functions) | |
771 | { | |
772 | snprintf (buffer, sizeof(buffer), | |
773 | " Looking at load module %s to change permissions to %s\n", | |
774 | szModName, | |
775 | (*mprotect_flags & PROT_WRITE) ? "READ/WRITE" : "READ-ONLY"); | |
776 | log_memory_protection_data (buffer); | |
777 | } | |
778 | ||
779 | /* See if we actually found the section. */ | |
780 | if (map_sect_offset && map_sect_len) | |
781 | { | |
782 | unsigned long long start; | |
783 | int result; | |
784 | ||
785 | if (debug_functions) | |
786 | { | |
787 | snprintf (buffer, sizeof (buffer), | |
788 | " (%s): Protecting %p to %p\n", | |
789 | szModName, | |
790 | (void *) map_sect_offset, | |
791 | (void *) (map_sect_offset + map_sect_len)); | |
792 | log_memory_protection_data (buffer); | |
793 | } | |
794 | ||
795 | /* Change the protections on the pages for the section. */ | |
796 | ||
797 | start = get_cycle_count (); | |
798 | result = mprotect ((void *) map_sect_offset, map_sect_len, | |
799 | *mprotect_flags); | |
800 | accumulate_cycle_count (&mprotect_cycles, start); | |
801 | if (result == -1) | |
802 | { | |
803 | if (debug_functions) | |
804 | { | |
805 | snprintf (buffer, sizeof (buffer), | |
e5ef217c | 806 | "Failed call to mprotect for %s error: ", |
f7f049fa CT |
807 | (*mprotect_flags & PROT_WRITE) ? |
808 | "READ/WRITE" : "READ-ONLY"); | |
809 | log_memory_protection_data (buffer); | |
810 | perror(NULL); | |
811 | } | |
812 | VTV_error(); | |
813 | } | |
814 | else | |
815 | { | |
816 | if (debug_functions) | |
817 | { | |
818 | snprintf (buffer, sizeof (buffer), | |
819 | "mprotect'ed range [%p, %p]\n", | |
820 | (void *) map_sect_offset, | |
821 | (char *) map_sect_offset + map_sect_len); | |
822 | log_memory_protection_data (buffer); | |
823 | } | |
824 | } | |
825 | increment_num_calls (&num_calls_to_mprotect); | |
e5ef217c RO |
826 | num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1) |
827 | / VTV_PAGE_SIZE; | |
f7f049fa CT |
828 | continue; |
829 | } | |
830 | } | |
831 | } | |
832 | } | |
833 | ||
834 | CloseHandle(hProcess); | |
835 | ||
836 | return 0; | |
837 | } | |
838 | #else | |
2077db1b CT |
839 | /* This is the callback function used by dl_iterate_phdr (which is |
840 | called from vtv_unprotect_vtable_vars and vtv_protect_vtable_vars). | |
841 | It attempts to find the binary file on disk for the INFO record | |
842 | that dl_iterate_phdr passes in; open the binary file, and read its | |
843 | section header information. If the file contains a | |
844 | ".vtable_map_vars" section, read the section offset and size. Use | |
845 | the section offset and size, in conjunction with the data in INFO | |
846 | to locate the pages in memory where the section is. Call | |
847 | 'mprotect' on those pages, setting the protection either to | |
848 | read-only or read-write, depending on what's in DATA. */ | |
849 | ||
850 | static int | |
851 | dl_iterate_phdr_callback (struct dl_phdr_info *info, size_t, void *data) | |
852 | { | |
853 | int * mprotect_flags = (int *) data; | |
854 | off_t map_sect_offset = 0; | |
855 | ElfW (Word) map_sect_len = 0; | |
856 | char buffer[1024]; | |
857 | char program_name[1024]; | |
858 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
859 | ||
860 | /* Check to see if this is the record for the Linux Virtual Dynamic | |
861 | Shared Object (linux-vdso.so.1), which exists only in memory (and | |
862 | therefore cannot be read from disk). */ | |
863 | ||
864 | if (strcmp (info->dlpi_name, "linux-vdso.so.1") == 0) | |
865 | return 0; | |
866 | ||
867 | if (strlen (info->dlpi_name) == 0 | |
868 | && info->dlpi_addr != 0) | |
869 | return 0; | |
870 | ||
871 | /* Get the name of the main executable. This may or may not include | |
872 | arguments passed to the program. Find the first space, assume it | |
873 | is the start of the argument list, and change it to a '\0'. */ | |
e5ef217c RO |
874 | #ifdef HAVE_GETEXECNAME |
875 | program_invocation_name = getexecname (); | |
876 | #endif | |
2077db1b CT |
877 | snprintf (program_name, sizeof (program_name), program_invocation_name); |
878 | ||
879 | read_section_offset_and_length (info, map_sect_name, *mprotect_flags, | |
880 | &map_sect_offset, &map_sect_len); | |
881 | ||
882 | if (debug_functions) | |
883 | { | |
884 | snprintf (buffer, sizeof(buffer), | |
885 | " Looking at load module %s to change permissions to %s\n", | |
886 | ((strlen (info->dlpi_name) == 0) ? program_name | |
887 | : info->dlpi_name), | |
888 | (*mprotect_flags & PROT_WRITE) ? "READ/WRITE" : "READ-ONLY"); | |
889 | log_memory_protection_data (buffer); | |
890 | } | |
891 | ||
892 | /* See if we actually found the section. */ | |
893 | if (map_sect_offset && map_sect_len) | |
894 | { | |
895 | unsigned long long start; | |
896 | int result; | |
897 | ||
898 | if (debug_functions) | |
899 | { | |
900 | snprintf (buffer, sizeof (buffer), | |
901 | " (%s): Protecting %p to %p\n", | |
902 | ((strlen (info->dlpi_name) == 0) ? program_name | |
903 | : info->dlpi_name), | |
904 | (void *) map_sect_offset, | |
905 | (void *) (map_sect_offset + map_sect_len)); | |
906 | log_memory_protection_data (buffer); | |
907 | } | |
908 | ||
909 | /* Change the protections on the pages for the section. */ | |
910 | ||
911 | start = get_cycle_count (); | |
912 | result = mprotect ((void *) map_sect_offset, map_sect_len, | |
913 | *mprotect_flags); | |
914 | accumulate_cycle_count (&mprotect_cycles, start); | |
915 | if (result == -1) | |
916 | { | |
917 | if (debug_functions) | |
918 | { | |
919 | snprintf (buffer, sizeof (buffer), | |
e5ef217c | 920 | "Failed call to mprotect for %s error: ", |
2077db1b CT |
921 | (*mprotect_flags & PROT_WRITE) ? |
922 | "READ/WRITE" : "READ-ONLY"); | |
923 | log_memory_protection_data (buffer); | |
924 | perror(NULL); | |
925 | } | |
926 | VTV_error(); | |
927 | } | |
928 | else | |
929 | { | |
930 | if (debug_functions) | |
931 | { | |
932 | snprintf (buffer, sizeof (buffer), | |
933 | "mprotect'ed range [%p, %p]\n", | |
934 | (void *) map_sect_offset, | |
935 | (char *) map_sect_offset + map_sect_len); | |
936 | log_memory_protection_data (buffer); | |
937 | } | |
938 | } | |
939 | increment_num_calls (&num_calls_to_mprotect); | |
e5ef217c | 940 | num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1) / VTV_PAGE_SIZE; |
2077db1b CT |
941 | } |
942 | ||
943 | return 0; | |
944 | } | |
f7f049fa | 945 | #endif |
2077db1b CT |
946 | |
947 | /* This function explicitly changes the protection (read-only or read-write) | |
948 | on the vtv_sect_info_cache, which is used for speeding up look ups in the | |
949 | function dl_iterate_phdr_callback. This data structure needs to be | |
950 | explicitly made read-write before any calls to dl_iterate_phdr_callback, | |
951 | because otherwise it may still be read-only when dl_iterate_phdr_callback | |
952 | attempts to write to it. | |
953 | ||
954 | More detailed explanation: dl_iterate_phdr_callback finds all the | |
955 | .vtable_map_vars sections in all loaded objects (including the main program) | |
956 | and (depending on where it was called from) either makes all the pages in the | |
957 | sections read-write or read-only. The vtv_sect_info_cache should be in the | |
958 | .vtable_map_vars section for libstdc++.so, which means that normally it would | |
959 | be read-only until libstdc++.so is processed by dl_iterate_phdr_callback | |
960 | (on the read-write pass), after which it will be writable. But if any loaded | |
961 | object gets processed before libstdc++.so, it will attempt to update the | |
962 | data cache, which will still be read-only, and cause a seg fault. Hence | |
963 | we need a special function, called before dl_iterate_phdr_callback, that | |
964 | will make the data cache writable. */ | |
965 | ||
966 | static void | |
967 | change_protections_on_phdr_cache (int protection_flag) | |
968 | { | |
969 | char * low_address = (char *) &(vtv_sect_info_cache); | |
970 | size_t cache_size = MAX_ENTRIES * sizeof (struct sect_hdr_data); | |
971 | ||
f7f049fa | 972 | low_address = (char *) ((uintptr_t) low_address & ~(VTV_PAGE_SIZE - 1)); |
2077db1b CT |
973 | |
974 | if (mprotect ((void *) low_address, cache_size, protection_flag) == -1) | |
975 | VTV_error (); | |
976 | } | |
977 | ||
978 | /* Unprotect all the vtable map vars and other side data that is used | |
979 | to keep the core hash_map data. All of these data have been put | |
980 | into relro sections */ | |
981 | ||
982 | static void | |
983 | vtv_unprotect_vtable_vars (void) | |
984 | { | |
985 | int mprotect_flags; | |
986 | ||
987 | mprotect_flags = PROT_READ | PROT_WRITE; | |
988 | change_protections_on_phdr_cache (mprotect_flags); | |
f7f049fa CT |
989 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
990 | iterate_modules ((void *) &mprotect_flags); | |
991 | #else | |
2077db1b | 992 | dl_iterate_phdr (dl_iterate_phdr_callback, (void *) &mprotect_flags); |
f7f049fa | 993 | #endif |
2077db1b CT |
994 | } |
995 | ||
996 | /* Protect all the vtable map vars and other side data that is used | |
997 | to keep the core hash_map data. All of these data have been put | |
998 | into relro sections */ | |
999 | ||
1000 | static void | |
1001 | vtv_protect_vtable_vars (void) | |
1002 | { | |
1003 | int mprotect_flags; | |
1004 | ||
1005 | mprotect_flags = PROT_READ; | |
f7f049fa CT |
1006 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
1007 | iterate_modules ((void *) &mprotect_flags); | |
1008 | #else | |
2077db1b | 1009 | dl_iterate_phdr (dl_iterate_phdr_callback, (void *) &mprotect_flags); |
f7f049fa | 1010 | #endif |
2077db1b CT |
1011 | change_protections_on_phdr_cache (mprotect_flags); |
1012 | } | |
1013 | ||
1014 | #ifndef __GTHREAD_MUTEX_INIT | |
1015 | static void | |
1016 | initialize_change_permissions_mutexes () | |
1017 | { | |
1018 | __GTHREAD_MUTEX_INIT_FUNCTION (&change_permissions_lock); | |
1019 | } | |
1020 | #endif | |
1021 | ||
1022 | /* Variables needed for getting the statistics about the hashtable set. */ | |
1023 | #if HASHTABLE_STATS | |
1024 | _AtomicStatCounter stat_contains = 0; | |
1025 | _AtomicStatCounter stat_insert = 0; | |
1026 | _AtomicStatCounter stat_resize = 0; | |
1027 | _AtomicStatCounter stat_create = 0; | |
1028 | _AtomicStatCounter stat_probes_in_non_trivial_set = 0; | |
1029 | _AtomicStatCounter stat_contains_size0 = 0; | |
1030 | _AtomicStatCounter stat_contains_size1 = 0; | |
1031 | _AtomicStatCounter stat_contains_size2 = 0; | |
1032 | _AtomicStatCounter stat_contains_size3 = 0; | |
1033 | _AtomicStatCounter stat_contains_size4 = 0; | |
1034 | _AtomicStatCounter stat_contains_size5 = 0; | |
1035 | _AtomicStatCounter stat_contains_size6 = 0; | |
1036 | _AtomicStatCounter stat_contains_size7 = 0; | |
1037 | _AtomicStatCounter stat_contains_size8 = 0; | |
1038 | _AtomicStatCounter stat_contains_size9 = 0; | |
1039 | _AtomicStatCounter stat_contains_size10 = 0; | |
1040 | _AtomicStatCounter stat_contains_size11 = 0; | |
1041 | _AtomicStatCounter stat_contains_size12 = 0; | |
1042 | _AtomicStatCounter stat_contains_size13_or_more = 0; | |
1043 | _AtomicStatCounter stat_contains_sizes = 0; | |
1044 | _AtomicStatCounter stat_grow_from_size0_to_1 = 0; | |
1045 | _AtomicStatCounter stat_grow_from_size1_to_2 = 0; | |
1046 | _AtomicStatCounter stat_double_the_number_of_buckets = 0; | |
1047 | _AtomicStatCounter stat_insert_found_hash_collision = 0; | |
1048 | _AtomicStatCounter stat_contains_in_non_trivial_set = 0; | |
1049 | _AtomicStatCounter stat_insert_key_that_was_already_present = 0; | |
1050 | #endif | |
1051 | /* Record statistics about the hash table sets, for debugging. */ | |
1052 | ||
1053 | static void | |
1054 | log_set_stats (void) | |
1055 | { | |
1056 | #if HASHTABLE_STATS | |
1057 | if (set_log_fd == -1) | |
1058 | set_log_fd = __vtv_open_log ("vtv_set_stats.log"); | |
1059 | ||
1060 | __vtv_add_to_log (set_log_fd, "---\n%s\n", | |
1061 | insert_only_hash_tables_stats().c_str()); | |
1062 | #endif | |
1063 | } | |
1064 | ||
1065 | /* Change the permissions on all the pages we have allocated for the | |
1066 | data sets and all the ".vtable_map_var" sections in memory (which | |
1067 | contain our vtable map variables). PERM indicates whether to make | |
1068 | the permissions read-only or read-write. */ | |
1069 | ||
1070 | extern "C" /* This is only being applied to __VLTChangePermission*/ | |
1071 | void | |
1072 | __VLTChangePermission (int perm) | |
1073 | { | |
1074 | if (debug_functions) | |
1075 | { | |
1076 | if (perm == __VLTP_READ_WRITE) | |
e5ef217c | 1077 | fprintf (stdout, "Changing VLT permissions to Read-Write.\n"); |
2077db1b | 1078 | else if (perm == __VLTP_READ_ONLY) |
e5ef217c | 1079 | fprintf (stdout, "Changing VLT permissions to Read-Only.\n"); |
2077db1b CT |
1080 | |
1081 | else | |
1082 | fprintf (stdout, "Unrecognized permissions value: %d\n", perm); | |
1083 | } | |
1084 | ||
1085 | #ifndef __GTHREAD_MUTEX_INIT | |
1086 | static __gthread_once_t mutex_once VTV_PROTECTED_VAR = __GTHREAD_ONCE_INIT; | |
1087 | ||
1088 | __gthread_once (&mutex_once, initialize_change_permissions_mutexes); | |
1089 | #endif | |
1090 | ||
1091 | /* Ordering of these unprotect/protect calls is very important. | |
1092 | You first need to unprotect all the map vars and side | |
1093 | structures before you do anything with the core data | |
1094 | structures (hash_maps) */ | |
1095 | ||
1096 | if (perm == __VLTP_READ_WRITE) | |
1097 | { | |
1098 | /* TODO: Need to revisit this code for dlopen. It most probably | |
1099 | is not unlocking the protected vtable vars after for load | |
1100 | module that is not the first load module. */ | |
1101 | __gthread_mutex_lock (&change_permissions_lock); | |
1102 | ||
1103 | vtv_unprotect_vtable_vars (); | |
1104 | __vtv_malloc_init (); | |
1105 | __vtv_malloc_unprotect (); | |
1106 | ||
1107 | } | |
1108 | else if (perm == __VLTP_READ_ONLY) | |
1109 | { | |
1110 | if (debug_hash) | |
1111 | log_set_stats(); | |
1112 | ||
1113 | __vtv_malloc_protect (); | |
1114 | vtv_protect_vtable_vars (); | |
1115 | ||
1116 | __gthread_mutex_unlock (&change_permissions_lock); | |
1117 | } | |
1118 | } | |
1119 | ||
1120 | /* This is the memory allocator used to create the hash table that | |
1121 | maps from vtable map variable name to the data set that vtable map | |
1122 | variable should point to. This is part of our vtable map variable | |
1123 | symbol resolution, which is necessary because the same vtable map | |
1124 | variable may be created by multiple compilation units and we need a | |
1125 | method to make sure that all vtable map variables for a particular | |
1126 | class point to the same data set at runtime. */ | |
1127 | ||
1128 | struct insert_only_hash_map_allocator | |
1129 | { | |
1130 | /* N is the number of bytes to allocate. */ | |
1131 | void * | |
1132 | alloc (size_t n) const | |
1133 | { | |
1134 | return __vtv_malloc (n); | |
1135 | } | |
1136 | ||
1137 | /* P points to the memory to be deallocated; N is the number of | |
1138 | bytes to deallocate. */ | |
1139 | void | |
1140 | dealloc (void *p, size_t) const | |
1141 | { | |
1142 | __vtv_free (p); | |
1143 | } | |
1144 | }; | |
1145 | ||
1146 | /* Explicitly instantiate this class since this file is compiled with | |
1147 | -fno-implicit-templates. These are for the hash table that is used | |
1148 | to do vtable map variable symbol resolution. */ | |
1149 | template class insert_only_hash_map <vtv_set_handle *, | |
1150 | insert_only_hash_map_allocator >; | |
1151 | typedef insert_only_hash_map <vtv_set_handle *, | |
1152 | insert_only_hash_map_allocator > s2s; | |
1153 | typedef const s2s::key_type vtv_symbol_key; | |
1154 | ||
1155 | static s2s * vtv_symbol_unification_map VTV_PROTECTED_VAR = NULL; | |
1156 | ||
1157 | const unsigned long SET_HANDLE_HANDLE_BIT = 0x2; | |
1158 | ||
1159 | /* In the case where a vtable map variable is the only instance of the | |
1160 | variable we have seen, it points directly to the set of valid | |
1161 | vtable pointers. All subsequent instances of the 'same' vtable map | |
1162 | variable point to the first vtable map variable. This function, | |
1163 | given a vtable map variable PTR, checks a bit to see whether it's | |
1164 | pointing directly to the data set or to the first vtable map | |
1165 | variable. */ | |
1166 | ||
1167 | static inline bool | |
1168 | is_set_handle_handle (void * ptr) | |
1169 | { | |
f7f049fa | 1170 | return ((uintptr_t) ptr & SET_HANDLE_HANDLE_BIT) |
2077db1b CT |
1171 | == SET_HANDLE_HANDLE_BIT; |
1172 | } | |
1173 | ||
1174 | /* Returns the actual pointer value of a vtable map variable, PTR (see | |
1175 | comments for is_set_handle_handle for more details). */ | |
1176 | ||
1177 | static inline vtv_set_handle * | |
1178 | ptr_from_set_handle_handle (void * ptr) | |
1179 | { | |
f7f049fa | 1180 | return (vtv_set_handle *) ((uintptr_t) ptr & ~SET_HANDLE_HANDLE_BIT); |
2077db1b CT |
1181 | } |
1182 | ||
1183 | /* Given a vtable map variable, PTR, this function sets the bit that | |
1184 | says this is the second (or later) instance of a vtable map | |
1185 | variable. */ | |
1186 | ||
1187 | static inline vtv_set_handle_handle | |
1188 | set_handle_handle (vtv_set_handle * ptr) | |
1189 | { | |
f7f049fa | 1190 | return (vtv_set_handle_handle) ((uintptr_t) ptr | SET_HANDLE_HANDLE_BIT); |
2077db1b CT |
1191 | } |
1192 | ||
1193 | static inline void | |
1194 | register_set_common (void **set_handle_ptr, size_t num_args, | |
1195 | void **vtable_ptr_array, bool debug) | |
1196 | { | |
1197 | /* Now figure out what pointer to use for the set pointer, for the | |
1198 | inserts. */ | |
1199 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1200 | ||
1201 | if (debug) | |
1202 | VTV_DEBUG_ASSERT (vtv_symbol_unification_map != NULL); | |
1203 | ||
1204 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1205 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1206 | else | |
1207 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1208 | ||
1209 | /* Now we've got the set and it's initialized, add the vtable | |
1210 | pointers. */ | |
1211 | for (size_t index = 0; index < num_args; ++index) | |
1212 | { | |
1213 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr_array[index]; | |
1214 | vtv_sets::insert (vtbl_ptr, handle_ptr); | |
1215 | } | |
1216 | } | |
1217 | ||
1218 | static inline void | |
1219 | register_pair_common (void **set_handle_ptr, const void *vtable_ptr, | |
1220 | const char *set_symbol_name, const char *vtable_name, | |
1221 | bool debug) | |
1222 | { | |
1223 | /* Now we've got the set and it's initialized, add the vtable | |
1224 | pointer (assuming that it's not NULL...It may be NULL, as we may | |
1225 | have called this function merely to initialize the set | |
1226 | pointer). */ | |
1227 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1228 | if (vtbl_ptr) | |
1229 | { | |
1230 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1231 | if (debug) | |
1232 | VTV_DEBUG_ASSERT (vtv_symbol_unification_map != NULL); | |
1233 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1234 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1235 | else | |
1236 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1237 | ||
1238 | vtv_sets::insert (vtbl_ptr, handle_ptr); | |
1239 | } | |
1240 | ||
1241 | if (debug && debug_init) | |
1242 | { | |
1243 | if (init_log_fd == -1) | |
1244 | init_log_fd = __vtv_open_log("vtv_init.log"); | |
1245 | ||
1246 | __vtv_add_to_log(init_log_fd, | |
1247 | "Registered %s : %s (%p) 2 level deref = %s\n", | |
1248 | set_symbol_name, vtable_name, vtbl_ptr, | |
1249 | is_set_handle_handle(*set_handle_ptr) ? "yes" : "no" ); | |
1250 | } | |
1251 | } | |
1252 | ||
1253 | /* This routine initializes a set handle to a vtable set. It makes | |
1254 | sure that there is only one set handle for a particular set by | |
1255 | using a map from set name to pointer to set handle. Since there | |
1256 | will be multiple copies of the pointer to the set handle (one per | |
1257 | compilation unit that uses it), it makes sure to initialize all the | |
1258 | pointers to the set handle so that the set handle is unique. To | |
1259 | make this a little more efficient and avoid a level of indirection | |
1260 | in some cases, the first pointer to handle for a particular handle | |
1261 | becomes the handle itself and the other pointers will point to the | |
1262 | set handle. This is the debug version of this function, so it | |
1263 | outputs extra debugging messages and logging. SET_HANDLE_PTR is | |
1264 | the address of the vtable map variable, SET_SYMBOL_KEY is the hash | |
1265 | table key (containing the name of the map variable and the hash | |
1266 | value) and SIZE_HINT is a guess for the best initial size for the | |
1267 | set of vtable pointers that SET_HANDLE_POINTER will point to. */ | |
1268 | ||
1269 | static inline void | |
1270 | init_set_symbol_debug (void **set_handle_ptr, const void *set_symbol_key, | |
1271 | size_t size_hint) | |
1272 | { | |
1273 | VTV_DEBUG_ASSERT (set_handle_ptr); | |
1274 | ||
1275 | if (vtv_symbol_unification_map == NULL) | |
1276 | { | |
1277 | /* TODO: For now we have chosen 1024, but we need to come up with a | |
1278 | better initial size for this. */ | |
1279 | vtv_symbol_unification_map = s2s::create (1024); | |
1280 | VTV_DEBUG_ASSERT(vtv_symbol_unification_map); | |
1281 | } | |
1282 | ||
1283 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1284 | vtv_symbol_key *symbol_key_ptr = (vtv_symbol_key *) set_symbol_key; | |
1285 | ||
1286 | const s2s::value_type * map_value_ptr = | |
1287 | vtv_symbol_unification_map->get (symbol_key_ptr); | |
1288 | char buffer[200]; | |
1289 | if (map_value_ptr == NULL) | |
1290 | { | |
1291 | if (*handle_ptr != NULL) | |
1292 | { | |
1293 | snprintf (buffer, sizeof (buffer), | |
1294 | "*** Found non-NULL local set ptr %p missing for symbol" | |
1295 | " %.*s", | |
1296 | *handle_ptr, symbol_key_ptr->n, symbol_key_ptr->bytes); | |
1297 | __vtv_log_verification_failure (buffer, true); | |
1298 | VTV_DEBUG_ASSERT (0); | |
1299 | } | |
1300 | } | |
1301 | else if (*handle_ptr != NULL && | |
1302 | (handle_ptr != *map_value_ptr && | |
1303 | ptr_from_set_handle_handle (*handle_ptr) != *map_value_ptr)) | |
1304 | { | |
1305 | VTV_DEBUG_ASSERT (*map_value_ptr != NULL); | |
1306 | snprintf (buffer, sizeof(buffer), | |
1307 | "*** Found diffence between local set ptr %p and set ptr %p" | |
1308 | "for symbol %.*s", | |
1309 | *handle_ptr, *map_value_ptr, | |
1310 | symbol_key_ptr->n, symbol_key_ptr->bytes); | |
1311 | __vtv_log_verification_failure (buffer, true); | |
1312 | VTV_DEBUG_ASSERT (0); | |
1313 | } | |
1314 | else if (*handle_ptr == NULL) | |
1315 | { | |
1316 | /* Execution should not reach this point. */ | |
1317 | } | |
1318 | ||
1319 | if (*handle_ptr != NULL) | |
1320 | { | |
1321 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1322 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1323 | else | |
1324 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1325 | vtv_sets::resize (size_hint, handle_ptr); | |
1326 | return; | |
1327 | } | |
1328 | ||
1329 | VTV_DEBUG_ASSERT (*handle_ptr == NULL); | |
1330 | if (map_value_ptr != NULL) | |
1331 | { | |
1332 | if (*map_value_ptr == handle_ptr) | |
1333 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1334 | else | |
1335 | { | |
1336 | /* The one level handle to the set already exists. So, we | |
1337 | are adding one level of indirection here and we will | |
1338 | store a pointer to the one level handle here. */ | |
1339 | ||
1340 | vtv_set_handle_handle * handle_handle_ptr = | |
1341 | (vtv_set_handle_handle *)handle_ptr; | |
1342 | *handle_handle_ptr = set_handle_handle(*map_value_ptr); | |
1343 | VTV_DEBUG_ASSERT(*handle_handle_ptr != NULL); | |
1344 | ||
1345 | /* The handle can itself be NULL if the set has only | |
1346 | been initiazlied with size hint == 1. */ | |
1347 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1348 | } | |
1349 | } | |
1350 | else | |
1351 | { | |
1352 | /* We will create a new set. So, in this case handle_ptr is the | |
1353 | one level pointer to the set handle. Create copy of map name | |
1354 | in case the memory where this comes from gets unmapped by | |
1355 | dlclose. */ | |
1356 | size_t map_key_len = symbol_key_ptr->n + sizeof (vtv_symbol_key); | |
1357 | void *map_key = __vtv_malloc (map_key_len); | |
1358 | ||
1359 | memcpy (map_key, symbol_key_ptr, map_key_len); | |
1360 | ||
1361 | s2s::value_type *value_ptr; | |
1362 | vtv_symbol_unification_map = | |
1363 | vtv_symbol_unification_map->find_or_add_key ((vtv_symbol_key *)map_key, | |
1364 | &value_ptr); | |
1365 | *value_ptr = handle_ptr; | |
1366 | ||
1367 | /* TODO: We should verify the return value. */ | |
1368 | vtv_sets::create (size_hint, handle_ptr); | |
1369 | VTV_DEBUG_ASSERT (size_hint <= 1 || *handle_ptr != NULL); | |
1370 | } | |
1371 | ||
1372 | if (debug_init) | |
1373 | { | |
1374 | if (init_log_fd == -1) | |
1375 | init_log_fd = __vtv_open_log ("vtv_init.log"); | |
1376 | ||
1377 | __vtv_add_to_log (init_log_fd, | |
1378 | "Init handle:%p for symbol:%.*s hash:%u size_hint:%lu" | |
1379 | "number of symbols:%lu \n", | |
1380 | set_handle_ptr, symbol_key_ptr->n, | |
1381 | symbol_key_ptr->bytes, symbol_key_ptr->hash, size_hint, | |
1382 | vtv_symbol_unification_map->size ()); | |
1383 | } | |
1384 | } | |
1385 | ||
1386 | ||
1387 | /* This routine initializes a set handle to a vtable set. It makes | |
1388 | sure that there is only one set handle for a particular set by | |
1389 | using a map from set name to pointer to set handle. Since there | |
1390 | will be multiple copies of the pointer to the set handle (one per | |
1391 | compilation unit that uses it), it makes sure to initialize all the | |
1392 | pointers to the set handle so that the set handle is unique. To | |
1393 | make this a little more efficient and avoid a level of indirection | |
1394 | in some cases, the first pointer to handle for a particular handle | |
1395 | becomes the handle itself and the other pointers will point to the | |
1396 | set handle. This is the debug version of this function, so it | |
1397 | outputs extra debugging messages and logging. SET_HANDLE_PTR is | |
1398 | the address of the vtable map variable, SET_SYMBOL_KEY is the hash | |
1399 | table key (containing the name of the map variable and the hash | |
1400 | value) and SIZE_HINT is a guess for the best initial size for the | |
1401 | set of vtable pointers that SET_HANDLE_POINTER will point to. */ | |
1402 | ||
1403 | void | |
1404 | __VLTRegisterSetDebug (void **set_handle_ptr, const void *set_symbol_key, | |
1405 | size_t size_hint, size_t num_args, | |
1406 | void **vtable_ptr_array) | |
1407 | { | |
1408 | unsigned long long start = get_cycle_count (); | |
1409 | increment_num_calls (&num_calls_to_regset); | |
1410 | ||
1411 | VTV_DEBUG_ASSERT(set_handle_ptr != NULL); | |
1412 | init_set_symbol_debug (set_handle_ptr, set_symbol_key, size_hint); | |
1413 | ||
1414 | register_set_common (set_handle_ptr, num_args, vtable_ptr_array, true); | |
1415 | ||
1416 | accumulate_cycle_count (®set_cycles, start); | |
1417 | } | |
1418 | ||
1419 | /* This function takes a the address of a vtable map variable | |
1420 | (SET_HANDLE_PTR), a VTABLE_PTR to add to the data set, the name of | |
1421 | the vtable map variable (SET_SYMBOL_NAME) and the name of the | |
1422 | vtable (VTABLE_NAME) being pointed to. If the vtable map variable | |
1423 | is NULL it creates a new data set and initializes the variable, | |
1424 | otherwise it uses our symbol unification to find the right data | |
1425 | set; in either case it then adds the vtable pointer to the set. | |
1426 | The other two parameters are used for debugging information. */ | |
1427 | ||
1428 | void | |
1429 | __VLTRegisterPairDebug (void **set_handle_ptr, const void *set_symbol_key, | |
1430 | size_t size_hint, const void *vtable_ptr, | |
1431 | const char *set_symbol_name, const char *vtable_name) | |
1432 | { | |
1433 | unsigned long long start = get_cycle_count (); | |
1434 | increment_num_calls (&num_calls_to_regpair); | |
1435 | ||
1436 | VTV_DEBUG_ASSERT(set_handle_ptr != NULL); | |
1437 | init_set_symbol_debug (set_handle_ptr, set_symbol_key, size_hint); | |
1438 | ||
1439 | register_pair_common (set_handle_ptr, vtable_ptr, set_symbol_name, vtable_name, | |
1440 | true); | |
1441 | ||
1442 | accumulate_cycle_count (®pair_cycles, start); | |
1443 | } | |
1444 | ||
1445 | ||
1446 | /* This is the debug version of the verification function. It takes | |
1447 | the address of a vtable map variable (SET_HANDLE_PTR) and a | |
1448 | VTABLE_PTR to validate, as well as the name of the vtable map | |
1449 | variable (SET_SYMBOL_NAME) and VTABLE_NAME, which are used for | |
1450 | debugging messages. It checks to see if VTABLE_PTR is in the set | |
1451 | pointed to by SET_HANDLE_PTR. If so, it returns VTABLE_PTR, | |
1452 | otherwise it calls __vtv_verify_fail, which usually logs error | |
1453 | messages and calls abort. */ | |
1454 | ||
1455 | const void * | |
1456 | __VLTVerifyVtablePointerDebug (void **set_handle_ptr, const void *vtable_ptr, | |
1457 | const char *set_symbol_name, | |
1458 | const char *vtable_name) | |
1459 | { | |
1460 | unsigned long long start = get_cycle_count (); | |
1461 | VTV_DEBUG_ASSERT (set_handle_ptr != NULL && *set_handle_ptr != NULL); | |
1462 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1463 | ||
1464 | increment_num_calls (&num_calls_to_verify_vtable); | |
1465 | vtv_set_handle *handle_ptr; | |
1466 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1467 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1468 | else | |
1469 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1470 | ||
1471 | if (vtv_sets::contains (vtbl_ptr, handle_ptr)) | |
1472 | { | |
1473 | if (debug_verify_vtable) | |
1474 | { | |
1475 | if (verify_vtable_log_fd == -1) | |
1476 | __vtv_open_log ("vtv_verify_vtable.log"); | |
1477 | __vtv_add_to_log (verify_vtable_log_fd, | |
1478 | "Verified %s %s value = %p\n", | |
1479 | set_symbol_name, vtable_name, vtable_ptr); | |
1480 | } | |
1481 | } | |
1482 | else | |
1483 | { | |
1484 | /* We failed to find the vtable pointer in the set of valid | |
1485 | pointers. Log the error data and call the failure | |
1486 | function. */ | |
1487 | snprintf (debug_log_message, sizeof (debug_log_message), | |
1488 | "Looking for %s in %s\n", vtable_name, set_symbol_name); | |
1489 | __vtv_verify_fail_debug (set_handle_ptr, vtable_ptr, debug_log_message); | |
1490 | ||
1491 | /* Normally __vtv_verify_fail_debug will call abort, so we won't | |
1492 | execute the return below. If we get this far, the assumption | |
1493 | is that the programmer has replaced __vtv_verify_fail_debug | |
1494 | with some kind of secondary verification AND this secondary | |
1495 | verification succeeded, so the vtable pointer is valid. */ | |
1496 | } | |
1497 | accumulate_cycle_count (&verify_vtable_cycles, start); | |
1498 | ||
1499 | return vtable_ptr; | |
1500 | } | |
1501 | ||
1502 | /* This routine initializes a set handle to a vtable set. It makes | |
1503 | sure that there is only one set handle for a particular set by | |
1504 | using a map from set name to pointer to set handle. Since there | |
1505 | will be multiple copies of the pointer to the set handle (one per | |
1506 | compilation unit that uses it), it makes sure to initialize all the | |
1507 | pointers to the set handle so that the set handle is unique. To | |
1508 | make this a little more efficient and avoid a level of indirection | |
1509 | in some cases, the first pointer to handle for a particular handle | |
1510 | becomes the handle itself and the other pointers will point to the | |
1511 | set handle. SET_HANDLE_PTR is the address of the vtable map | |
1512 | variable, SET_SYMBOL_KEY is the hash table key (containing the name | |
1513 | of the map variable and the hash value) and SIZE_HINT is a guess | |
1514 | for the best initial size for the set of vtable pointers that | |
1515 | SET_HANDLE_POINTER will point to.*/ | |
1516 | ||
1517 | static inline void | |
1518 | init_set_symbol (void **set_handle_ptr, const void *set_symbol_key, | |
1519 | size_t size_hint) | |
1520 | { | |
1521 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1522 | ||
1523 | if (*handle_ptr != NULL) | |
1524 | { | |
1525 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1526 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1527 | else | |
1528 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1529 | vtv_sets::resize (size_hint, handle_ptr); | |
1530 | return; | |
1531 | } | |
1532 | ||
1533 | if (vtv_symbol_unification_map == NULL) | |
1534 | vtv_symbol_unification_map = s2s::create (1024); | |
1535 | ||
1536 | vtv_symbol_key *symbol_key_ptr = (vtv_symbol_key *) set_symbol_key; | |
1537 | const s2s::value_type *map_value_ptr = | |
1538 | vtv_symbol_unification_map->get (symbol_key_ptr); | |
1539 | ||
1540 | if (map_value_ptr != NULL) | |
1541 | { | |
1542 | if (*map_value_ptr == handle_ptr) | |
1543 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1544 | else | |
1545 | { | |
1546 | /* The one level handle to the set already exists. So, we | |
1547 | are adding one level of indirection here and we will | |
1548 | store a pointer to the one level pointer here. */ | |
1549 | vtv_set_handle_handle *handle_handle_ptr = | |
1550 | (vtv_set_handle_handle *) handle_ptr; | |
1551 | *handle_handle_ptr = set_handle_handle (*map_value_ptr); | |
1552 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1553 | } | |
1554 | } | |
1555 | else | |
1556 | { | |
1557 | /* We will create a new set. So, in this case handle_ptr is the | |
1558 | one level pointer to the set handle. Create copy of map name | |
1559 | in case the memory where this comes from gets unmapped by | |
1560 | dlclose. */ | |
1561 | size_t map_key_len = symbol_key_ptr->n + sizeof (vtv_symbol_key); | |
1562 | void * map_key = __vtv_malloc (map_key_len); | |
1563 | memcpy (map_key, symbol_key_ptr, map_key_len); | |
1564 | ||
1565 | s2s::value_type * value_ptr; | |
1566 | vtv_symbol_unification_map = | |
1567 | vtv_symbol_unification_map->find_or_add_key ((vtv_symbol_key *)map_key, | |
1568 | &value_ptr); | |
1569 | ||
1570 | *value_ptr = handle_ptr; | |
1571 | ||
1572 | /* TODO: We should verify the return value. */ | |
1573 | vtv_sets::create (size_hint, handle_ptr); | |
1574 | } | |
1575 | } | |
1576 | ||
1577 | /* This routine initializes a set handle to a vtable set. It makes | |
1578 | sure that there is only one set handle for a particular set by | |
1579 | using a map from set name to pointer to set handle. Since there | |
1580 | will be multiple copies of the pointer to the set handle (one per | |
1581 | compilation unit that uses it), it makes sure to initialize all the | |
1582 | pointers to the set handle so that the set handle is unique. To | |
1583 | make this a little more efficient and avoid a level of indirection | |
1584 | in some cases, the first pointer to handle for a particular handle | |
1585 | becomes the handle itself and the other pointers will point to the | |
1586 | set handle. SET_HANDLE_PTR is the address of the vtable map | |
1587 | variable, SET_SYMBOL_KEY is the hash table key (containing the name | |
1588 | of the map variable and the hash value) and SIZE_HINT is a guess | |
1589 | for the best initial size for the set of vtable pointers that | |
1590 | SET_HANDLE_POINTER will point to.*/ | |
1591 | ||
1592 | ||
1593 | void | |
1594 | __VLTRegisterSet (void **set_handle_ptr, const void *set_symbol_key, | |
1595 | size_t size_hint, size_t num_args, void **vtable_ptr_array) | |
1596 | { | |
1597 | unsigned long long start = get_cycle_count (); | |
1598 | increment_num_calls (&num_calls_to_regset); | |
1599 | ||
1600 | init_set_symbol (set_handle_ptr, set_symbol_key, size_hint); | |
1601 | register_set_common (set_handle_ptr, num_args, vtable_ptr_array, false); | |
1602 | ||
1603 | accumulate_cycle_count (®set_cycles, start); | |
1604 | } | |
1605 | ||
1606 | ||
1607 | ||
1608 | /* This function takes a the address of a vtable map variable | |
1609 | (SET_HANDLE_PTR) and a VTABLE_PTR. If the vtable map variable is | |
1610 | NULL it creates a new data set and initializes the variable, | |
1611 | otherwise it uses our symbol unification to find the right data | |
1612 | set; in either case it then adds the vtable pointer to the set. */ | |
1613 | ||
1614 | void | |
1615 | __VLTRegisterPair (void **set_handle_ptr, const void *set_symbol_key, | |
1616 | size_t size_hint, const void *vtable_ptr) | |
1617 | { | |
1618 | unsigned long long start = get_cycle_count (); | |
1619 | increment_num_calls (&num_calls_to_regpair); | |
1620 | ||
1621 | init_set_symbol (set_handle_ptr, set_symbol_key, size_hint); | |
1622 | register_pair_common (set_handle_ptr, vtable_ptr, NULL, NULL, false); | |
1623 | ||
1624 | accumulate_cycle_count (®pair_cycles, start); | |
1625 | } | |
1626 | ||
1627 | /* This is the main verification function. It takes the address of a | |
1628 | vtable map variable (SET_HANDLE_PTR) and a VTABLE_PTR to validate. | |
1629 | It checks to see if VTABLE_PTR is in the set pointed to by | |
1630 | SET_HANDLE_PTR. If so, it returns VTABLE_PTR, otherwise it calls | |
1631 | __vtv_verify_fail, which usually logs error messages and calls | |
1632 | abort. Since this function gets called VERY frequently, it is | |
1633 | important for it to be as efficient as possible. */ | |
1634 | ||
1635 | const void * | |
1636 | __VLTVerifyVtablePointer (void ** set_handle_ptr, const void * vtable_ptr) | |
1637 | { | |
1638 | unsigned long long start = get_cycle_count (); | |
1639 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1640 | ||
1641 | vtv_set_handle *handle_ptr; | |
1642 | increment_num_calls (&num_calls_to_verify_vtable); | |
1643 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1644 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1645 | else | |
1646 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1647 | ||
1648 | if (!vtv_sets::contains (vtbl_ptr, handle_ptr)) | |
1649 | { | |
1650 | __vtv_verify_fail ((void **) handle_ptr, vtable_ptr); | |
1651 | /* Normally __vtv_verify_fail will call abort, so we won't | |
1652 | execute the return below. If we get this far, the assumption | |
1653 | is that the programmer has replaced __vtv_verify_fail with | |
1654 | some kind of secondary verification AND this secondary | |
1655 | verification succeeded, so the vtable pointer is valid. */ | |
1656 | } | |
1657 | accumulate_cycle_count (&verify_vtable_cycles, start); | |
1658 | ||
1659 | return vtable_ptr; | |
1660 | } | |
1661 | ||
1662 | static int page_count_2 = 0; | |
1663 | ||
f7f049fa | 1664 | #if !defined (__CYGWIN__) && !defined (__MINGW32__) |
2077db1b CT |
1665 | static int |
1666 | dl_iterate_phdr_count_pages (struct dl_phdr_info *info, | |
1667 | size_t unused __attribute__ ((__unused__)), | |
1668 | void *data) | |
1669 | { | |
1670 | int *mprotect_flags = (int *) data; | |
1671 | off_t map_sect_offset = 0; | |
1672 | ElfW (Word) map_sect_len = 0; | |
1673 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
1674 | ||
1675 | /* Check to see if this is the record for the Linux Virtual Dynamic | |
1676 | Shared Object (linux-vdso.so.1), which exists only in memory (and | |
1677 | therefore cannot be read from disk). */ | |
1678 | ||
1679 | if (strcmp (info->dlpi_name, "linux-vdso.so.1") == 0) | |
1680 | return 0; | |
1681 | ||
1682 | if (strlen (info->dlpi_name) == 0 | |
1683 | && info->dlpi_addr != 0) | |
1684 | return 0; | |
1685 | ||
1686 | read_section_offset_and_length (info, map_sect_name, *mprotect_flags, | |
1687 | &map_sect_offset, &map_sect_len); | |
1688 | ||
1689 | /* See if we actually found the section. */ | |
1690 | if (map_sect_len) | |
1691 | page_count_2 += (map_sect_len + VTV_PAGE_SIZE - 1) / VTV_PAGE_SIZE; | |
1692 | ||
1693 | return 0; | |
1694 | } | |
f7f049fa | 1695 | #endif |
2077db1b CT |
1696 | |
1697 | static void | |
1698 | count_all_pages (void) | |
1699 | { | |
1700 | int mprotect_flags; | |
1701 | ||
1702 | mprotect_flags = PROT_READ; | |
1703 | page_count_2 = 0; | |
1704 | ||
f7f049fa CT |
1705 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
1706 | iterate_modules ((void *) &mprotect_flags); | |
1707 | #else | |
2077db1b | 1708 | dl_iterate_phdr (dl_iterate_phdr_count_pages, (void *) &mprotect_flags); |
f7f049fa | 1709 | #endif |
2077db1b CT |
1710 | page_count_2 += __vtv_count_mmapped_pages (); |
1711 | } | |
1712 | ||
1713 | void | |
1714 | __VLTDumpStats (void) | |
1715 | { | |
1716 | int log_fd = __vtv_open_log ("vtv-runtime-stats.log"); | |
1717 | ||
1718 | if (log_fd != -1) | |
1719 | { | |
1720 | count_all_pages (); | |
1721 | __vtv_add_to_log (log_fd, | |
1722 | "Calls: mprotect (%d) regset (%d) regpair (%d)" | |
1723 | " verify_vtable (%d)\n", | |
1724 | num_calls_to_mprotect, num_calls_to_regset, | |
1725 | num_calls_to_regpair, num_calls_to_verify_vtable); | |
1726 | __vtv_add_to_log (log_fd, | |
1727 | "Cycles: mprotect (%lld) regset (%lld) " | |
1728 | "regpair (%lld) verify_vtable (%lld)\n", | |
1729 | mprotect_cycles, regset_cycles, regpair_cycles, | |
1730 | verify_vtable_cycles); | |
1731 | __vtv_add_to_log (log_fd, | |
1732 | "Pages protected (1): %d\n", num_pages_protected); | |
1733 | __vtv_add_to_log (log_fd, "Pages protected (2): %d\n", page_count_2); | |
1734 | ||
1735 | close (log_fd); | |
1736 | } | |
1737 | } | |
1738 | ||
1739 | /* This function is called from __VLTVerifyVtablePointerDebug; it | |
1740 | sends as much debugging information as it can to the error log | |
1741 | file, then calls __vtv_verify_fail. SET_HANDLE_PTR is the pointer | |
1742 | to the set of valid vtable pointers, VTBL_PTR is the pointer that | |
1743 | was not found in the set, and DEBUG_MSG is the message to be | |
1744 | written to the log file before failing. n */ | |
1745 | ||
1746 | void | |
1747 | __vtv_verify_fail_debug (void **set_handle_ptr, const void *vtbl_ptr, | |
1748 | const char *debug_msg) | |
1749 | { | |
1750 | __vtv_log_verification_failure (debug_msg, false); | |
1751 | ||
1752 | /* Call the public interface in case it has been overwritten by | |
1753 | user. */ | |
1754 | __vtv_verify_fail (set_handle_ptr, vtbl_ptr); | |
1755 | ||
1756 | __vtv_log_verification_failure ("Returned from __vtv_verify_fail." | |
1757 | " Secondary verification succeeded.\n", false); | |
1758 | } | |
1759 | ||
1760 | /* This function calls __fortify_fail with a FAILURE_MSG and then | |
1761 | calls abort. */ | |
1762 | ||
1763 | void | |
1764 | __vtv_really_fail (const char *failure_msg) | |
1765 | { | |
1766 | __fortify_fail (failure_msg); | |
1767 | ||
1768 | /* We should never get this far; __fortify_fail calls __libc_message | |
1769 | which prints out a back trace and a memory dump and then is | |
1770 | supposed to call abort, but let's play it safe anyway and call abort | |
1771 | ourselves. */ | |
1772 | abort (); | |
1773 | } | |
1774 | ||
1775 | /* This function takes an error MSG, a vtable map variable | |
1776 | (DATA_SET_PTR) and a vtable pointer (VTBL_PTR). It is called when | |
1777 | an attempt to verify VTBL_PTR with the set pointed to by | |
1778 | DATA_SET_PTR failed. It outputs a failure message with the | |
1779 | addresses involved, and calls __vtv_really_fail. */ | |
1780 | ||
1781 | static void | |
1782 | vtv_fail (const char *msg, void **data_set_ptr, const void *vtbl_ptr) | |
1783 | { | |
1784 | char buffer[128]; | |
1785 | int buf_len; | |
1786 | const char *format_str = | |
1787 | "*** Unable to verify vtable pointer (%p) in set (%p) *** \n"; | |
1788 | ||
1789 | snprintf (buffer, sizeof (buffer), format_str, vtbl_ptr, | |
1790 | is_set_handle_handle(*data_set_ptr) ? | |
1791 | ptr_from_set_handle_handle (*data_set_ptr) : | |
1792 | *data_set_ptr); | |
1793 | buf_len = strlen (buffer); | |
1794 | /* Send this to to stderr. */ | |
1795 | write (2, buffer, buf_len); | |
1796 | ||
1797 | #ifndef VTV_NO_ABORT | |
1798 | __vtv_really_fail (msg); | |
1799 | #endif | |
1800 | } | |
1801 | ||
1802 | /* Send information about what we were trying to do when verification | |
1803 | failed to the error log, then call vtv_fail. This function can be | |
1804 | overwritten/replaced by the user, to implement a secondary | |
1805 | verification function instead. DATA_SET_PTR is the vtable map | |
1806 | variable used for the failed verification, and VTBL_PTR is the | |
1807 | vtable pointer that was not found in the set. */ | |
1808 | ||
1809 | void | |
1810 | __vtv_verify_fail (void **data_set_ptr, const void *vtbl_ptr) | |
1811 | { | |
1812 | char log_msg[256]; | |
1813 | snprintf (log_msg, sizeof (log_msg), "Looking for vtable %p in set %p.\n", | |
1814 | vtbl_ptr, | |
1815 | is_set_handle_handle (*data_set_ptr) ? | |
1816 | ptr_from_set_handle_handle (*data_set_ptr) : | |
1817 | *data_set_ptr); | |
1818 | __vtv_log_verification_failure (log_msg, false); | |
1819 | ||
1820 | const char *format_str = | |
1821 | "*** Unable to verify vtable pointer (%p) in set (%p) *** \n"; | |
1822 | snprintf (log_msg, sizeof (log_msg), format_str, vtbl_ptr, *data_set_ptr); | |
1823 | __vtv_log_verification_failure (log_msg, false); | |
1824 | __vtv_log_verification_failure (" Backtrace: \n", true); | |
1825 | ||
1826 | const char *fail_msg = "Potential vtable pointer corruption detected!!\n"; | |
1827 | vtv_fail (fail_msg, data_set_ptr, vtbl_ptr); | |
1828 | } |