]>
Commit | Line | Data |
---|---|---|
fbd26352 | 1 | /* Copyright (C) 2012-2019 Free Software Foundation, Inc. |
b710ec85 | 2 | |
3 | This file is part of GCC. | |
4 | ||
5 | GCC is free software; you can redistribute it and/or modify | |
6 | it under the terms of the GNU General Public License as published by | |
7 | the Free Software Foundation; either version 3, or (at your option) | |
8 | any later version. | |
9 | ||
10 | GCC is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | GNU General Public License for more details. | |
14 | ||
15 | Under Section 7 of GPL version 3, you are granted additional | |
16 | permissions described in the GCC Runtime Library Exception, version | |
17 | 3.1, as published by the Free Software Foundation. | |
18 | ||
19 | You should have received a copy of the GNU General Public License and | |
20 | a copy of the GCC Runtime Library Exception along with this program; | |
21 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
22 | <http://www.gnu.org/licenses/>. */ | |
23 | ||
24 | /* This file is part of the vtable security feature implementation. | |
25 | The vtable security feature is designed to detect when a virtual | |
26 | call is about to be made through an invalid vtable pointer | |
27 | (possibly due to data corruption or malicious attacks). The | |
28 | compiler finds every virtual call, and inserts a verification call | |
29 | before the virtual call. The verification call takes the actual | |
30 | vtable pointer value in the object through which the virtual call | |
31 | is being made, and compares the vtable pointer against a set of all | |
32 | valid vtable pointers that the object could contain (this set is | |
33 | based on the declared type of the object). If the pointer is in | |
34 | the valid set, execution is allowed to continue; otherwise the | |
35 | program is halted. | |
36 | ||
37 | There are several pieces needed in order to make this work: 1. For | |
38 | every virtual class in the program (i.e. a class that contains | |
39 | virtual methods), we need to build the set of all possible valid | |
40 | vtables that an object of that class could point to. This includes | |
41 | vtables for any class(es) that inherit from the class under | |
42 | consideration. 2. For every such data set we build up, we need a | |
43 | way to find and reference the data set. This is complicated by the | |
44 | fact that the real vtable addresses are not known until runtime, | |
45 | when the program is loaded into memory, but we need to reference the | |
46 | sets at compile time when we are inserting verification calls into | |
47 | the program. 3. We need to find every virtual call in the program, | |
48 | and insert the verification call (with the appropriate arguments) | |
49 | before the virtual call. 4. We need some runtime library pieces: | |
50 | the code to build up the data sets at runtime; the code to actually | |
51 | perform the verification using the data sets; and some code to set | |
52 | protections on the data sets, so they themselves do not become | |
53 | hacker targets. | |
54 | ||
55 | To find and reference the set of valid vtable pointers for any given | |
56 | virtual class, we create a special global varible for each virtual | |
57 | class. We refer to this as the "vtable map variable" for that | |
58 | class. The vtable map variable has the type "void *", and is | |
59 | initialized by the compiler to NULL. At runtime when the set of | |
60 | valid vtable pointers for a virtual class, e.g. class Foo, is built, | |
61 | the vtable map variable for class Foo is made to point to the set. | |
62 | During compile time, when the compiler is inserting verification | |
63 | calls into the program, it passes the vtable map variable for the | |
64 | appropriate class to the verification call, so that at runtime the | |
65 | verification call can find the appropriate data set. | |
66 | ||
67 | The actual set of valid vtable pointers for a polymorphic class, | |
68 | e.g. class Foo, cannot be built until runtime, when the vtables get | |
69 | loaded into memory and their addresses are known. But the knowledge | |
70 | about which vtables belong in which class' hierarchy is only known | |
71 | at compile time. Therefore at compile time we collect class | |
72 | hierarchy and vtable information about every virtual class, and we | |
73 | generate calls to build up the data sets at runtime. To build the | |
74 | data sets, we call one of the functions we add to the runtime | |
75 | library, __VLTRegisterPair. __VLTRegisterPair takes two arguments, | |
76 | a vtable map variable and the address of a vtable. If the vtable | |
77 | map variable is currently NULL, it creates a new data set (hash | |
78 | table), makes the vtable map variable point to the new data set, and | |
79 | inserts the vtable address into the data set. If the vtable map | |
80 | variable is not NULL, it just inserts the vtable address into the | |
81 | data set. In order to make sure that our data sets are built before | |
82 | any verification calls happen, we create a special constructor | |
83 | initialization function for each compilation unit, give it a very | |
84 | high initialization priority, and insert all of our calls to | |
85 | __VLTRegisterPair into our special constructor initialization | |
86 | function. */ | |
87 | ||
88 | /* This file contains the main externally visible runtime library | |
89 | functions for vtable verification: __VLTChangePermission, | |
90 | __VLTRegisterPair, and __VLTVerifyVtablePointer. It also contains | |
91 | debug versions __VLTRegisterPairDebug and | |
92 | __VLTVerifyVtablePointerDebug, which have extra parameters in order | |
93 | to make it easier to debug verification failures. | |
94 | ||
95 | The final piece of functionality implemented in this file is symbol | |
96 | resolution for multiple instances of the same vtable map variable. | |
97 | If the same virtual class is used in two different compilation | |
98 | units, then each compilation unit will create a vtable map variable | |
99 | for the class. We need all instances of the same vtable map | |
100 | variable to point to the same (single) set of valid vtable | |
101 | pointers for the class, so we wrote our own hashtable-based symbol | |
102 | resolution for vtable map variables (with a tiny optimization in | |
103 | the case where there is only one instance of the variable). | |
104 | ||
105 | There are two other important pieces to the runtime for vtable | |
106 | verification besides the main pieces that go into libstdc++.so: two | |
107 | special tiny shared libraries, libvtv_init.so and libvtv_stubs.so. | |
108 | libvtv_init.so is built from vtv_init.cc. It is designed to help | |
109 | minimize the calls made to mprotect (see the comments in | |
110 | vtv_init.cc for more details). Anything compiled with | |
111 | "-fvtable-verify=std" must be linked with libvtv_init.so (the gcc | |
112 | driver has been modified to do this). vtv_stubs.so is built from | |
113 | vtv_stubs.cc. It replaces the main runtime functions | |
114 | (__VLTChangePermissino, __VLTRegisterPair and | |
115 | __VLTVerifyVtablePointer) with stub functions that do nothing. If | |
116 | a programmer has a library that was built with verification, but | |
117 | wishes to not have verification turned on, the programmer can link | |
118 | in the vtv_stubs.so library. */ | |
119 | ||
120 | #include <stdlib.h> | |
121 | #include <stdio.h> | |
122 | #include <string.h> | |
5be42fa9 | 123 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
124 | #include <windows.h> | |
125 | #include <winternl.h> | |
126 | #include <psapi.h> | |
127 | #else | |
b710ec85 | 128 | #include <execinfo.h> |
5be42fa9 | 129 | #endif |
b710ec85 | 130 | |
131 | #include <unistd.h> | |
5be42fa9 | 132 | #if !defined (__CYGWIN__) && !defined (__MINGW32__) |
b710ec85 | 133 | #include <sys/mman.h> |
b710ec85 | 134 | #include <link.h> |
5be42fa9 | 135 | #endif |
136 | #include <errno.h> | |
b710ec85 | 137 | #include <fcntl.h> |
138 | #include <limits.h> | |
139 | ||
140 | /* For gthreads suppport */ | |
141 | #include <bits/c++config.h> | |
142 | #include <ext/concurrence.h> | |
143 | ||
144 | #include "vtv_utils.h" | |
145 | #include "vtv_malloc.h" | |
146 | #include "vtv_set.h" | |
147 | #include "vtv_map.h" | |
148 | #include "vtv_rts.h" | |
149 | #include "vtv_fail.h" | |
150 | ||
151 | #include "vtv-change-permission.h" | |
152 | ||
08de21a4 | 153 | #ifdef HAVE_GETEXECNAME |
154 | const char *program_invocation_name; | |
155 | #endif | |
156 | ||
157 | #ifdef HAVE___FORTIFY_FAIL | |
b710ec85 | 158 | extern "C" { |
159 | ||
160 | /* __fortify_fail is a function in glibc that calls __libc_message, | |
161 | causing it to print out a program termination error message | |
162 | (including the name of the binary being terminated), a stack | |
163 | trace where the error occurred, and a memory map dump. Ideally | |
164 | we would have called __libc_message directly, but that function | |
165 | does not appear to be accessible to functions outside glibc, | |
166 | whereas __fortify_fail is. We call __fortify_fail from | |
167 | __vtv_really_fail. We looked at calling __libc_fatal, which is | |
168 | externally accessible, but it does not do the back trace and | |
169 | memory dump. */ | |
170 | ||
171 | extern void __fortify_fail (const char *) __attribute__((noreturn)); | |
172 | ||
173 | } /* extern "C" */ | |
08de21a4 | 174 | #else |
175 | #if defined (__CYGWIN__) || defined (__MINGW32__) | |
176 | // porting: fix link error to libc | |
177 | void __fortify_fail (const char * msg){ | |
178 | OutputDebugString(msg); | |
179 | abort(); | |
180 | } | |
181 | #else | |
182 | // FIXME: Provide backtrace via libbacktrace? | |
183 | void __fortify_fail (const char *msg) { | |
184 | write (2, msg, strlen (msg)); | |
185 | abort (); | |
186 | } | |
187 | #endif | |
5be42fa9 | 188 | #endif |
b710ec85 | 189 | |
190 | /* The following variables are used only for debugging and performance | |
191 | tuning purposes. Therefore they do not need to be "protected". | |
192 | They cannot be used to attack the vtable verification system and if | |
193 | they become corrupted it will not affect the correctness or | |
194 | security of any of the rest of the vtable verification feature. */ | |
195 | ||
196 | unsigned int num_calls_to_regset = 0; | |
197 | unsigned int num_calls_to_regpair = 0; | |
198 | unsigned int num_calls_to_verify_vtable = 0; | |
199 | unsigned long long regset_cycles = 0; | |
200 | unsigned long long regpair_cycles = 0; | |
201 | unsigned long long verify_vtable_cycles = 0; | |
202 | ||
203 | /* Be careful about initialization of statics in this file. Some of | |
204 | the routines below are called before any runtime initialization for | |
205 | statics in this file will be done. For example, dont try to | |
206 | initialize any of these statics with a runtime call (for ex: | |
207 | sysconf). The initialization will happen after calls to the routines | |
208 | to protect/unprotec the vtabla_map variables */ | |
209 | ||
210 | /* No need to mark the following variables with VTV_PROTECTED_VAR. | |
211 | These are either const or are only used for debugging/tracing. | |
212 | debugging/tracing will not be ON on production environments */ | |
213 | ||
214 | static const bool debug_hash = HASHTABLE_STATS; | |
b710ec85 | 215 | |
216 | #ifdef VTV_DEBUG | |
217 | static const int debug_functions = 1; | |
218 | static const int debug_init = 1; | |
219 | static const int debug_verify_vtable = 1; | |
68a98a70 | 220 | #else |
221 | static const int debug_functions = 0; | |
222 | static const int debug_init = 0; | |
223 | static const int debug_verify_vtable = 0; | |
b710ec85 | 224 | #endif |
225 | ||
226 | /* Global file descriptor variables for logging, tracing and debugging. */ | |
227 | ||
228 | static int init_log_fd = -1; | |
229 | static int verify_vtable_log_fd = -1; | |
230 | ||
231 | /* This holds a formatted error logging message, to be written to the | |
232 | vtable verify failures log. */ | |
233 | static char debug_log_message[1024]; | |
234 | ||
235 | ||
236 | #ifdef __GTHREAD_MUTEX_INIT | |
237 | static __gthread_mutex_t change_permissions_lock = __GTHREAD_MUTEX_INIT; | |
238 | #else | |
239 | static __gthread_mutex_t change_permissions_lock; | |
240 | #endif | |
241 | ||
242 | ||
243 | #ifndef VTV_STATS | |
244 | #define VTV_STATS 0 | |
245 | #endif | |
246 | ||
247 | #if VTV_STATS | |
248 | ||
249 | static inline unsigned long long | |
250 | get_cycle_count (void) | |
251 | { | |
252 | return rdtsc(); | |
253 | } | |
254 | ||
255 | static inline void | |
256 | accumulate_cycle_count (unsigned long long *sum, unsigned long long start) | |
257 | { | |
258 | unsigned long long end = rdtsc(); | |
259 | *sum = *sum + (end - start); | |
260 | } | |
261 | ||
262 | static inline void | |
263 | increment_num_calls (unsigned int *num_calls) | |
264 | { | |
265 | *num_calls = *num_calls + 1; | |
266 | } | |
267 | ||
268 | #else | |
269 | ||
270 | static inline unsigned long long | |
271 | get_cycle_count (void) | |
272 | { | |
273 | return (unsigned long long) 0; | |
274 | } | |
275 | ||
276 | static inline void | |
277 | accumulate_cycle_count (unsigned long long *sum __attribute__((__unused__)), | |
278 | unsigned long long start __attribute__((__unused__))) | |
279 | { | |
280 | /* Do nothing. */ | |
281 | } | |
282 | ||
283 | static inline void | |
284 | increment_num_calls (unsigned int *num_calls __attribute__((__unused__))) | |
285 | { | |
286 | /* Do nothing. */ | |
287 | } | |
288 | ||
289 | #endif | |
290 | ||
291 | /* Types needed by insert_only_hash_sets. */ | |
292 | typedef uintptr_t int_vptr; | |
293 | ||
294 | /* The set of valid vtable pointers for each virtual class is stored | |
295 | in a hash table. This is the hashing function used for the hash | |
296 | table. For more information on the implementation of the hash | |
297 | table, see the class insert_only_hash_sets in vtv_set.h. */ | |
298 | ||
299 | struct vptr_hash | |
300 | { | |
301 | /* Hash function, used to convert vtable pointer, V, (a memory | |
302 | address) into an index into the hash table. */ | |
303 | size_t | |
304 | operator() (int_vptr v) const | |
305 | { | |
306 | const uint32_t x = 0x7a35e4d9; | |
307 | const int shift = (sizeof (v) == 8) ? 23 : 21; | |
308 | v = x * v; | |
309 | return v ^ (v >> shift); | |
310 | } | |
311 | }; | |
312 | ||
313 | /* This is the memory allocator used to create the hash table data | |
314 | sets of valid vtable pointers. We use VTV_malloc in order to keep | |
315 | track of which pages have been allocated, so we can update the | |
316 | protections on those pages appropriately. See the class | |
317 | insert_only_hash_sets in vtv_set.h for more information. */ | |
318 | ||
319 | struct vptr_set_alloc | |
320 | { | |
321 | /* Memory allocator operator. N is the number of bytes to be | |
322 | allocated. */ | |
323 | void * | |
324 | operator() (size_t n) const | |
325 | { | |
326 | return __vtv_malloc (n); | |
327 | } | |
328 | }; | |
329 | ||
330 | /* Instantiate the template classes (in vtv_set.h) for our particular | |
331 | hash table needs. */ | |
332 | typedef insert_only_hash_sets<int_vptr, vptr_hash, vptr_set_alloc> vtv_sets; | |
333 | typedef vtv_sets::insert_only_hash_set vtv_set; | |
334 | typedef vtv_set * vtv_set_handle; | |
335 | typedef vtv_set_handle * vtv_set_handle_handle; | |
336 | ||
337 | /* Records for caching the section header information that we have | |
338 | read out of the file(s) on disk (in dl_iterate_phdr_callback), to | |
339 | avoid having to re-open and re-read the same file multiple | |
340 | times. */ | |
341 | ||
342 | struct sect_hdr_data | |
343 | { | |
5be42fa9 | 344 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
345 | uintptr_t dlpi_addr; /* The header address in the INFO record, | |
346 | passed in from dl_iterate_phdr. */ | |
347 | uintptr_t mp_low; /* Start address of the .vtable_map_vars | |
348 | section in memory. */ | |
349 | #else | |
b710ec85 | 350 | ElfW (Addr) dlpi_addr; /* The header address in the INFO record, |
351 | passed in from dl_iterate_phdr. */ | |
352 | ElfW (Addr) mp_low; /* Start address of the .vtable_map_vars | |
353 | section in memory. */ | |
5be42fa9 | 354 | #endif |
b710ec85 | 355 | size_t mp_size; /* Size of the .vtable_map_vars section in |
356 | memory. */ | |
357 | }; | |
358 | ||
359 | /* Array for caching the section header information, read from file, | |
360 | to avoid re-opening and re-reading the same file over-and-over | |
361 | again. */ | |
362 | ||
363 | #define MAX_ENTRIES 250 | |
364 | static struct sect_hdr_data vtv_sect_info_cache[MAX_ENTRIES] VTV_PROTECTED_VAR; | |
365 | ||
366 | unsigned int num_cache_entries VTV_PROTECTED_VAR = 0; | |
367 | ||
368 | /* This function takes the LOAD_ADDR for an object opened by the | |
369 | dynamic loader, and checks the array of cached file data to see if | |
370 | there is an entry with the same addres. If it finds such an entry, | |
371 | it returns the record for that entry; otherwise it returns | |
372 | NULL. */ | |
373 | ||
5be42fa9 | 374 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
375 | struct sect_hdr_data * | |
376 | search_cached_file_data (uintptr_t load_addr) | |
377 | #else | |
b710ec85 | 378 | struct sect_hdr_data * |
379 | search_cached_file_data (ElfW (Addr) load_addr) | |
5be42fa9 | 380 | #endif |
b710ec85 | 381 | { |
382 | unsigned int i; | |
383 | for (i = 0; i < num_cache_entries; ++i) | |
384 | { | |
385 | if (vtv_sect_info_cache[i].dlpi_addr == load_addr) | |
386 | return &(vtv_sect_info_cache[i]); | |
387 | } | |
388 | ||
389 | return NULL; | |
390 | } | |
391 | ||
392 | /* This function tries to read COUNT bytes out of the file referred to | |
393 | by FD into the buffer BUF. It returns the actual number of bytes | |
394 | it succeeded in reading. */ | |
395 | ||
396 | static size_t | |
397 | ReadPersistent (int fd, void *buf, size_t count) | |
398 | { | |
399 | char *buf0 = (char *) buf; | |
400 | size_t num_bytes = 0; | |
401 | while (num_bytes < count) | |
402 | { | |
403 | int len; | |
404 | len = read (fd, buf0 + num_bytes, count - num_bytes); | |
405 | if (len < 0) | |
406 | return -1; | |
407 | if (len == 0) | |
408 | break; | |
409 | num_bytes += len; | |
410 | } | |
411 | ||
412 | return num_bytes; | |
413 | } | |
414 | ||
415 | /* This function tries to read COUNT bytes, starting at OFFSET from | |
416 | the file referred to by FD, and put them into BUF. It calls | |
417 | ReadPersistent to help it do so. It returns the actual number of | |
418 | bytes read, or -1 if it fails altogether. */ | |
419 | ||
420 | static size_t | |
421 | ReadFromOffset (int fd, void *buf, const size_t count, const off_t offset) | |
422 | { | |
423 | off_t off = lseek (fd, offset, SEEK_SET); | |
424 | if (off != (off_t) -1) | |
425 | return ReadPersistent (fd, buf, count); | |
426 | return -1; | |
427 | } | |
428 | ||
429 | /* The function takes a MESSAGE and attempts to write it to the vtable | |
430 | memory protection log (for debugging purposes). If the file is not | |
431 | open, it attempts to open the file first. */ | |
432 | ||
433 | static void | |
434 | log_memory_protection_data (char *message) | |
435 | { | |
436 | static int log_fd = -1; | |
437 | ||
438 | if (log_fd == -1) | |
83628c60 | 439 | log_fd = __vtv_open_log ("vtv_memory_protection_data.log"); |
b710ec85 | 440 | |
441 | __vtv_add_to_log (log_fd, "%s", message); | |
442 | } | |
443 | ||
5be42fa9 | 444 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
445 | static void | |
446 | read_section_offset_and_length (char *name, | |
447 | uintptr_t addr, | |
448 | const char *sect_name, | |
449 | int mprotect_flags, | |
450 | off_t *sect_offset, | |
451 | WORD *sect_len) | |
452 | { | |
453 | bool found = false; | |
454 | struct sect_hdr_data *cached_data = NULL; | |
455 | ||
456 | /* Check to see if we already have the data for this file. */ | |
457 | cached_data = search_cached_file_data (addr); | |
458 | ||
459 | if (cached_data) | |
460 | { | |
461 | *sect_offset = cached_data->mp_low; | |
462 | *sect_len = cached_data->mp_size; | |
463 | return; | |
464 | } | |
465 | ||
466 | // check for DOS Header magic bytes | |
467 | if (*(WORD *)addr == 0x5A4D) | |
468 | { | |
469 | int name_len = strlen (sect_name); | |
470 | int fd = -1; | |
471 | ||
472 | /* Attempt to open the binary file on disk. */ | |
473 | if (strlen (name) == 0) | |
474 | { | |
475 | return; | |
476 | } | |
477 | else | |
478 | fd = open (name, O_RDONLY | O_BINARY); | |
479 | ||
480 | if (fd != -1) | |
481 | { | |
482 | /* Find the section header information in memory. */ | |
483 | PIMAGE_DOS_HEADER pDosHeader = (PIMAGE_DOS_HEADER)addr; | |
484 | PIMAGE_NT_HEADERS pNtHeaders = (PIMAGE_NT_HEADERS)((char *)addr | |
485 | + pDosHeader->e_lfanew); | |
486 | PIMAGE_FILE_HEADER pFileHeader = &pNtHeaders->FileHeader; | |
487 | ||
488 | DWORD PointerToStringTable = pFileHeader->PointerToSymbolTable | |
489 | + (pFileHeader->NumberOfSymbols*0x12); | |
490 | ||
491 | PIMAGE_SECTION_HEADER sect_hdr = | |
492 | (PIMAGE_SECTION_HEADER)((char *)&pNtHeaders->OptionalHeader | |
493 | + pFileHeader->SizeOfOptionalHeader); | |
494 | ||
495 | /* Loop through all the section headers, looking for one whose | |
496 | name is ".vtable_map_vars". */ | |
497 | ||
498 | for (int i = 0; i < pFileHeader->NumberOfSections && !found; ++i) | |
499 | { | |
500 | char header_name[64]; | |
501 | ||
502 | /* Check if we have to get the section name from the COFF string | |
503 | table. */ | |
504 | if (sect_hdr[i].Name[0] == '/') | |
505 | { | |
506 | if (atoi((const char*)sect_hdr[i].Name+1) == 0) | |
507 | { | |
508 | continue; | |
509 | } | |
510 | ||
511 | off_t name_offset = PointerToStringTable | |
512 | + atoi((const char*)sect_hdr[i].Name+1); | |
513 | ||
514 | size_t bytes_read = ReadFromOffset (fd, &header_name, 64, | |
515 | name_offset); | |
516 | ||
517 | VTV_ASSERT (bytes_read > 0); | |
518 | } | |
519 | else | |
520 | { | |
521 | memcpy (&header_name, sect_hdr[i].Name, | |
522 | sizeof (sect_hdr[i].Name)); | |
523 | } | |
524 | ||
525 | if (memcmp (header_name, sect_name, name_len) == 0) | |
526 | { | |
527 | /* We found the section; get its load offset and | |
528 | size. */ | |
529 | *sect_offset = sect_hdr[i].VirtualAddress; | |
530 | if (sect_hdr[i].Misc.VirtualSize % VTV_PAGE_SIZE != 0) | |
531 | *sect_len = sect_hdr[i].Misc.VirtualSize + VTV_PAGE_SIZE | |
532 | - (sect_hdr[i].Misc.VirtualSize % VTV_PAGE_SIZE); | |
533 | else | |
534 | *sect_len = sect_hdr[i].Misc.VirtualSize; | |
535 | found = true; | |
536 | } | |
537 | } | |
538 | close (fd); | |
539 | } | |
540 | } | |
541 | ||
542 | if (*sect_offset != 0 && *sect_len != 0) | |
543 | { | |
544 | /* Calculate the page location in memory, making sure the | |
545 | address is page-aligned. */ | |
546 | uintptr_t start_addr = addr + *sect_offset; | |
547 | *sect_offset = start_addr & ~(VTV_PAGE_SIZE - 1); | |
548 | *sect_len = *sect_len - 1; | |
549 | ||
550 | /* Since we got this far, we must not have found these pages in | |
551 | the cache, so add them to it. NOTE: We could get here either | |
552 | while making everything read-only or while making everything | |
553 | read-write. We will only update the cache if we get here on | |
554 | a read-write (to make absolutely sure the cache is writable | |
555 | -- also the read-write pass should come before the read-only | |
556 | pass). */ | |
557 | if ((mprotect_flags & PROT_WRITE) | |
558 | && num_cache_entries < MAX_ENTRIES) | |
559 | { | |
560 | vtv_sect_info_cache[num_cache_entries].dlpi_addr = addr; | |
561 | vtv_sect_info_cache[num_cache_entries].mp_low = *sect_offset; | |
562 | vtv_sect_info_cache[num_cache_entries].mp_size = *sect_len; | |
563 | num_cache_entries++; | |
564 | } | |
565 | } | |
566 | } | |
567 | #else | |
b710ec85 | 568 | static void |
569 | read_section_offset_and_length (struct dl_phdr_info *info, | |
570 | const char *sect_name, | |
571 | int mprotect_flags, | |
572 | off_t *sect_offset, | |
573 | ElfW (Word) *sect_len) | |
574 | { | |
575 | char program_name[PATH_MAX]; | |
576 | char *cptr; | |
577 | bool found = false; | |
578 | struct sect_hdr_data *cached_data = NULL; | |
579 | const ElfW (Phdr) *phdr_info = info->dlpi_phdr; | |
580 | const ElfW (Ehdr) *ehdr_info = | |
581 | (const ElfW (Ehdr) *) (info->dlpi_addr + info->dlpi_phdr[0].p_vaddr | |
582 | - info->dlpi_phdr[0].p_offset); | |
583 | ||
584 | ||
585 | /* Get the name of the main executable. This may or may not include | |
586 | arguments passed to the program. Find the first space, assume it | |
587 | is the start of the argument list, and change it to a '\0'. */ | |
08de21a4 | 588 | #ifdef HAVE_GETEXECNAME |
589 | program_invocation_name = getexecname (); | |
590 | #endif | |
b710ec85 | 591 | snprintf (program_name, sizeof (program_name), program_invocation_name); |
592 | ||
593 | /* Check to see if we already have the data for this file. */ | |
594 | cached_data = search_cached_file_data (info->dlpi_addr); | |
595 | ||
596 | if (cached_data) | |
597 | { | |
598 | *sect_offset = cached_data->mp_low; | |
599 | *sect_len = cached_data->mp_size; | |
600 | return; | |
601 | } | |
602 | ||
603 | /* Find the first non-escaped space in the program name and make it | |
604 | the end of the string. */ | |
605 | cptr = strchr (program_name, ' '); | |
606 | if (cptr != NULL && cptr[-1] != '\\') | |
607 | cptr[0] = '\0'; | |
608 | ||
609 | if ((phdr_info->p_type == PT_PHDR || phdr_info->p_type == PT_LOAD) | |
610 | && (ehdr_info->e_shoff && ehdr_info->e_shnum)) | |
611 | { | |
612 | int name_len = strlen (sect_name); | |
613 | int fd = -1; | |
614 | ||
615 | /* Attempt to open the binary file on disk. */ | |
616 | if (strlen (info->dlpi_name) == 0) | |
617 | { | |
618 | /* If the constructor initialization function was put into | |
619 | the preinit array, then this function will get called | |
620 | while handling preinit array stuff, in which case | |
621 | program_invocation_name has not been initialized. In | |
622 | that case we can get the filename of the executable from | |
623 | "/proc/self/exe". */ | |
624 | if (strlen (program_name) > 0) | |
625 | { | |
626 | if (phdr_info->p_type == PT_PHDR) | |
627 | fd = open (program_name, O_RDONLY); | |
628 | } | |
629 | else | |
630 | fd = open ("/proc/self/exe", O_RDONLY); | |
631 | } | |
632 | else | |
633 | fd = open (info->dlpi_name, O_RDONLY); | |
634 | ||
635 | if (fd != -1) | |
636 | { | |
637 | /* Find the section header information in the file. */ | |
638 | ElfW (Half) strtab_idx = ehdr_info->e_shstrndx; | |
639 | ElfW (Shdr) shstrtab; | |
640 | off_t shstrtab_offset = ehdr_info->e_shoff + | |
641 | (ehdr_info->e_shentsize * strtab_idx); | |
642 | size_t bytes_read = ReadFromOffset (fd, &shstrtab, sizeof (shstrtab), | |
643 | shstrtab_offset); | |
644 | VTV_ASSERT (bytes_read == sizeof (shstrtab)); | |
645 | ||
646 | ElfW (Shdr) sect_hdr; | |
647 | ||
648 | /* This code will be needed once we have crated libvtv.so. */ | |
649 | bool is_libvtv = false; | |
650 | ||
651 | /* | |
652 | if (strstr (info->dlpi_name, "libvtv.so")) | |
653 | is_libvtv = true; | |
654 | */ | |
655 | ||
656 | /* Loop through all the section headers, looking for one whose | |
657 | name is ".vtable_map_vars". */ | |
658 | ||
659 | for (int i = 0; i < ehdr_info->e_shnum && !found; ++i) | |
660 | { | |
661 | off_t offset = ehdr_info->e_shoff + (ehdr_info->e_shentsize * i); | |
662 | ||
663 | bytes_read = ReadFromOffset (fd, §_hdr, sizeof (sect_hdr), | |
664 | offset); | |
665 | ||
666 | VTV_ASSERT (bytes_read == sizeof (sect_hdr)); | |
667 | ||
668 | char header_name[64]; | |
669 | off_t name_offset = shstrtab.sh_offset + sect_hdr.sh_name; | |
670 | ||
671 | bytes_read = ReadFromOffset (fd, &header_name, 64, name_offset); | |
672 | ||
673 | VTV_ASSERT (bytes_read > 0); | |
674 | ||
675 | if (memcmp (header_name, sect_name, name_len) == 0) | |
676 | { | |
677 | /* We found the section; get its load offset and | |
678 | size. */ | |
679 | *sect_offset = sect_hdr.sh_addr; | |
680 | if (!is_libvtv) | |
08de21a4 | 681 | { |
682 | VTV_ASSERT (sect_hdr.sh_size - VTV_PAGE_SIZE >= 0); | |
683 | *sect_len = sect_hdr.sh_size - VTV_PAGE_SIZE; | |
684 | } | |
b710ec85 | 685 | else |
686 | *sect_len = sect_hdr.sh_size; | |
687 | found = true; | |
688 | } | |
689 | } | |
690 | close (fd); | |
691 | } | |
692 | } | |
693 | ||
694 | if (*sect_offset != 0 && *sect_len != 0) | |
695 | { | |
696 | /* Calculate the page location in memory, making sure the | |
697 | address is page-aligned. */ | |
698 | ElfW (Addr) start_addr = (const ElfW (Addr)) info->dlpi_addr | |
699 | + *sect_offset; | |
700 | *sect_offset = start_addr & ~(VTV_PAGE_SIZE - 1); | |
701 | *sect_len = *sect_len - 1; | |
702 | ||
703 | /* Since we got this far, we must not have found these pages in | |
704 | the cache, so add them to it. NOTE: We could get here either | |
705 | while making everything read-only or while making everything | |
706 | read-write. We will only update the cache if we get here on | |
707 | a read-write (to make absolutely sure the cache is writable | |
708 | -- also the read-write pass should come before the read-only | |
709 | pass). */ | |
710 | if ((mprotect_flags & PROT_WRITE) | |
711 | && num_cache_entries < MAX_ENTRIES) | |
712 | { | |
713 | vtv_sect_info_cache[num_cache_entries].dlpi_addr = info->dlpi_addr; | |
714 | vtv_sect_info_cache[num_cache_entries].mp_low = *sect_offset; | |
715 | vtv_sect_info_cache[num_cache_entries].mp_size = *sect_len; | |
716 | num_cache_entries++; | |
717 | } | |
718 | } | |
719 | } | |
5be42fa9 | 720 | #endif |
721 | ||
722 | #if defined (__CYGWIN__) || defined (__MINGW32__) | |
723 | /* This function is used to iterate over all loaded modules and searches | |
724 | for a section called ".vtable_map_vars". The only interaction with | |
725 | the binary file on disk of the module is to read section names in the | |
726 | COFF string table. If the module contains a ".vtable_map_vars" section, | |
727 | read section offset and size from the section header of the loaded module. | |
728 | Call 'mprotect' on those pages, setting the protection either to | |
729 | read-only or read-write, depending on what's in data. | |
730 | The calls to change the protection occur in vtv_unprotect_vtable_vars | |
731 | and vtv_protect_vtable_vars. */ | |
732 | ||
733 | static int | |
734 | iterate_modules (void *data) | |
735 | { | |
736 | int * mprotect_flags = (int *) data; | |
737 | off_t map_sect_offset = 0; | |
738 | WORD map_sect_len = 0; | |
739 | char buffer[1024]; | |
740 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
741 | HMODULE hMods[1024]; | |
742 | HANDLE hProcess; | |
743 | DWORD cbNeeded; | |
b710ec85 | 744 | |
5be42fa9 | 745 | hProcess = GetCurrentProcess (); |
746 | ||
747 | if (NULL == hProcess) | |
748 | return 0; | |
749 | ||
750 | if (EnumProcessModules (hProcess, hMods, sizeof (hMods), &cbNeeded)) | |
751 | { | |
752 | /* Iterate over all loaded modules. */ | |
753 | for (unsigned int i = 0; i < (cbNeeded / sizeof (HMODULE)); i++) | |
754 | { | |
755 | char szModName[MAX_PATH]; | |
756 | ||
757 | if (GetModuleFileNameExA (hProcess, hMods[i], szModName, | |
758 | sizeof (szModName))) | |
759 | { | |
760 | map_sect_offset = 0; | |
761 | map_sect_len = 0; | |
762 | read_section_offset_and_length (szModName, | |
763 | (uintptr_t) hMods[i], | |
764 | map_sect_name, | |
765 | *mprotect_flags, | |
766 | &map_sect_offset, | |
767 | &map_sect_len); | |
768 | ||
769 | if (debug_functions) | |
770 | { | |
771 | snprintf (buffer, sizeof(buffer), | |
772 | " Looking at load module %s to change permissions to %s\n", | |
773 | szModName, | |
774 | (*mprotect_flags & PROT_WRITE) ? "READ/WRITE" : "READ-ONLY"); | |
775 | log_memory_protection_data (buffer); | |
776 | } | |
777 | ||
778 | /* See if we actually found the section. */ | |
779 | if (map_sect_offset && map_sect_len) | |
780 | { | |
781 | unsigned long long start; | |
782 | int result; | |
783 | ||
784 | if (debug_functions) | |
785 | { | |
786 | snprintf (buffer, sizeof (buffer), | |
787 | " (%s): Protecting %p to %p\n", | |
788 | szModName, | |
789 | (void *) map_sect_offset, | |
790 | (void *) (map_sect_offset + map_sect_len)); | |
791 | log_memory_protection_data (buffer); | |
792 | } | |
793 | ||
794 | /* Change the protections on the pages for the section. */ | |
795 | ||
796 | start = get_cycle_count (); | |
797 | result = mprotect ((void *) map_sect_offset, map_sect_len, | |
798 | *mprotect_flags); | |
799 | accumulate_cycle_count (&mprotect_cycles, start); | |
800 | if (result == -1) | |
801 | { | |
802 | if (debug_functions) | |
803 | { | |
804 | snprintf (buffer, sizeof (buffer), | |
08de21a4 | 805 | "Failed call to mprotect for %s error: ", |
5be42fa9 | 806 | (*mprotect_flags & PROT_WRITE) ? |
807 | "READ/WRITE" : "READ-ONLY"); | |
808 | log_memory_protection_data (buffer); | |
809 | perror(NULL); | |
810 | } | |
811 | VTV_error(); | |
812 | } | |
813 | else | |
814 | { | |
815 | if (debug_functions) | |
816 | { | |
817 | snprintf (buffer, sizeof (buffer), | |
818 | "mprotect'ed range [%p, %p]\n", | |
819 | (void *) map_sect_offset, | |
820 | (char *) map_sect_offset + map_sect_len); | |
821 | log_memory_protection_data (buffer); | |
822 | } | |
823 | } | |
824 | increment_num_calls (&num_calls_to_mprotect); | |
08de21a4 | 825 | num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1) |
826 | / VTV_PAGE_SIZE; | |
5be42fa9 | 827 | continue; |
828 | } | |
829 | } | |
830 | } | |
831 | } | |
832 | ||
833 | CloseHandle(hProcess); | |
834 | ||
835 | return 0; | |
836 | } | |
837 | #else | |
b710ec85 | 838 | /* This is the callback function used by dl_iterate_phdr (which is |
839 | called from vtv_unprotect_vtable_vars and vtv_protect_vtable_vars). | |
840 | It attempts to find the binary file on disk for the INFO record | |
841 | that dl_iterate_phdr passes in; open the binary file, and read its | |
842 | section header information. If the file contains a | |
843 | ".vtable_map_vars" section, read the section offset and size. Use | |
844 | the section offset and size, in conjunction with the data in INFO | |
845 | to locate the pages in memory where the section is. Call | |
846 | 'mprotect' on those pages, setting the protection either to | |
847 | read-only or read-write, depending on what's in DATA. */ | |
848 | ||
849 | static int | |
850 | dl_iterate_phdr_callback (struct dl_phdr_info *info, size_t, void *data) | |
851 | { | |
852 | int * mprotect_flags = (int *) data; | |
853 | off_t map_sect_offset = 0; | |
854 | ElfW (Word) map_sect_len = 0; | |
855 | char buffer[1024]; | |
856 | char program_name[1024]; | |
857 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
858 | ||
859 | /* Check to see if this is the record for the Linux Virtual Dynamic | |
860 | Shared Object (linux-vdso.so.1), which exists only in memory (and | |
861 | therefore cannot be read from disk). */ | |
862 | ||
863 | if (strcmp (info->dlpi_name, "linux-vdso.so.1") == 0) | |
864 | return 0; | |
865 | ||
866 | if (strlen (info->dlpi_name) == 0 | |
867 | && info->dlpi_addr != 0) | |
868 | return 0; | |
869 | ||
870 | /* Get the name of the main executable. This may or may not include | |
871 | arguments passed to the program. Find the first space, assume it | |
872 | is the start of the argument list, and change it to a '\0'. */ | |
08de21a4 | 873 | #ifdef HAVE_GETEXECNAME |
874 | program_invocation_name = getexecname (); | |
875 | #endif | |
b710ec85 | 876 | snprintf (program_name, sizeof (program_name), program_invocation_name); |
877 | ||
878 | read_section_offset_and_length (info, map_sect_name, *mprotect_flags, | |
879 | &map_sect_offset, &map_sect_len); | |
880 | ||
881 | if (debug_functions) | |
882 | { | |
883 | snprintf (buffer, sizeof(buffer), | |
884 | " Looking at load module %s to change permissions to %s\n", | |
885 | ((strlen (info->dlpi_name) == 0) ? program_name | |
886 | : info->dlpi_name), | |
887 | (*mprotect_flags & PROT_WRITE) ? "READ/WRITE" : "READ-ONLY"); | |
888 | log_memory_protection_data (buffer); | |
889 | } | |
890 | ||
891 | /* See if we actually found the section. */ | |
892 | if (map_sect_offset && map_sect_len) | |
893 | { | |
894 | unsigned long long start; | |
895 | int result; | |
896 | ||
897 | if (debug_functions) | |
898 | { | |
899 | snprintf (buffer, sizeof (buffer), | |
900 | " (%s): Protecting %p to %p\n", | |
901 | ((strlen (info->dlpi_name) == 0) ? program_name | |
902 | : info->dlpi_name), | |
903 | (void *) map_sect_offset, | |
904 | (void *) (map_sect_offset + map_sect_len)); | |
905 | log_memory_protection_data (buffer); | |
906 | } | |
907 | ||
908 | /* Change the protections on the pages for the section. */ | |
909 | ||
910 | start = get_cycle_count (); | |
911 | result = mprotect ((void *) map_sect_offset, map_sect_len, | |
912 | *mprotect_flags); | |
913 | accumulate_cycle_count (&mprotect_cycles, start); | |
914 | if (result == -1) | |
915 | { | |
916 | if (debug_functions) | |
917 | { | |
918 | snprintf (buffer, sizeof (buffer), | |
08de21a4 | 919 | "Failed call to mprotect for %s error: ", |
b710ec85 | 920 | (*mprotect_flags & PROT_WRITE) ? |
921 | "READ/WRITE" : "READ-ONLY"); | |
922 | log_memory_protection_data (buffer); | |
923 | perror(NULL); | |
924 | } | |
925 | VTV_error(); | |
926 | } | |
927 | else | |
928 | { | |
929 | if (debug_functions) | |
930 | { | |
931 | snprintf (buffer, sizeof (buffer), | |
932 | "mprotect'ed range [%p, %p]\n", | |
933 | (void *) map_sect_offset, | |
934 | (char *) map_sect_offset + map_sect_len); | |
935 | log_memory_protection_data (buffer); | |
936 | } | |
937 | } | |
938 | increment_num_calls (&num_calls_to_mprotect); | |
08de21a4 | 939 | num_pages_protected += (map_sect_len + VTV_PAGE_SIZE - 1) / VTV_PAGE_SIZE; |
b710ec85 | 940 | } |
941 | ||
942 | return 0; | |
943 | } | |
5be42fa9 | 944 | #endif |
b710ec85 | 945 | |
946 | /* This function explicitly changes the protection (read-only or read-write) | |
947 | on the vtv_sect_info_cache, which is used for speeding up look ups in the | |
948 | function dl_iterate_phdr_callback. This data structure needs to be | |
949 | explicitly made read-write before any calls to dl_iterate_phdr_callback, | |
950 | because otherwise it may still be read-only when dl_iterate_phdr_callback | |
951 | attempts to write to it. | |
952 | ||
953 | More detailed explanation: dl_iterate_phdr_callback finds all the | |
954 | .vtable_map_vars sections in all loaded objects (including the main program) | |
955 | and (depending on where it was called from) either makes all the pages in the | |
956 | sections read-write or read-only. The vtv_sect_info_cache should be in the | |
957 | .vtable_map_vars section for libstdc++.so, which means that normally it would | |
958 | be read-only until libstdc++.so is processed by dl_iterate_phdr_callback | |
959 | (on the read-write pass), after which it will be writable. But if any loaded | |
960 | object gets processed before libstdc++.so, it will attempt to update the | |
961 | data cache, which will still be read-only, and cause a seg fault. Hence | |
962 | we need a special function, called before dl_iterate_phdr_callback, that | |
963 | will make the data cache writable. */ | |
964 | ||
965 | static void | |
966 | change_protections_on_phdr_cache (int protection_flag) | |
967 | { | |
968 | char * low_address = (char *) &(vtv_sect_info_cache); | |
969 | size_t cache_size = MAX_ENTRIES * sizeof (struct sect_hdr_data); | |
970 | ||
5be42fa9 | 971 | low_address = (char *) ((uintptr_t) low_address & ~(VTV_PAGE_SIZE - 1)); |
b710ec85 | 972 | |
973 | if (mprotect ((void *) low_address, cache_size, protection_flag) == -1) | |
974 | VTV_error (); | |
975 | } | |
976 | ||
977 | /* Unprotect all the vtable map vars and other side data that is used | |
978 | to keep the core hash_map data. All of these data have been put | |
979 | into relro sections */ | |
980 | ||
981 | static void | |
982 | vtv_unprotect_vtable_vars (void) | |
983 | { | |
984 | int mprotect_flags; | |
985 | ||
986 | mprotect_flags = PROT_READ | PROT_WRITE; | |
987 | change_protections_on_phdr_cache (mprotect_flags); | |
5be42fa9 | 988 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
989 | iterate_modules ((void *) &mprotect_flags); | |
990 | #else | |
b710ec85 | 991 | dl_iterate_phdr (dl_iterate_phdr_callback, (void *) &mprotect_flags); |
5be42fa9 | 992 | #endif |
b710ec85 | 993 | } |
994 | ||
995 | /* Protect all the vtable map vars and other side data that is used | |
996 | to keep the core hash_map data. All of these data have been put | |
997 | into relro sections */ | |
998 | ||
999 | static void | |
1000 | vtv_protect_vtable_vars (void) | |
1001 | { | |
1002 | int mprotect_flags; | |
1003 | ||
1004 | mprotect_flags = PROT_READ; | |
5be42fa9 | 1005 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
1006 | iterate_modules ((void *) &mprotect_flags); | |
1007 | #else | |
b710ec85 | 1008 | dl_iterate_phdr (dl_iterate_phdr_callback, (void *) &mprotect_flags); |
5be42fa9 | 1009 | #endif |
b710ec85 | 1010 | change_protections_on_phdr_cache (mprotect_flags); |
1011 | } | |
1012 | ||
1013 | #ifndef __GTHREAD_MUTEX_INIT | |
1014 | static void | |
1015 | initialize_change_permissions_mutexes () | |
1016 | { | |
1017 | __GTHREAD_MUTEX_INIT_FUNCTION (&change_permissions_lock); | |
1018 | } | |
1019 | #endif | |
1020 | ||
1021 | /* Variables needed for getting the statistics about the hashtable set. */ | |
1022 | #if HASHTABLE_STATS | |
1023 | _AtomicStatCounter stat_contains = 0; | |
1024 | _AtomicStatCounter stat_insert = 0; | |
1025 | _AtomicStatCounter stat_resize = 0; | |
1026 | _AtomicStatCounter stat_create = 0; | |
1027 | _AtomicStatCounter stat_probes_in_non_trivial_set = 0; | |
1028 | _AtomicStatCounter stat_contains_size0 = 0; | |
1029 | _AtomicStatCounter stat_contains_size1 = 0; | |
1030 | _AtomicStatCounter stat_contains_size2 = 0; | |
1031 | _AtomicStatCounter stat_contains_size3 = 0; | |
1032 | _AtomicStatCounter stat_contains_size4 = 0; | |
1033 | _AtomicStatCounter stat_contains_size5 = 0; | |
1034 | _AtomicStatCounter stat_contains_size6 = 0; | |
1035 | _AtomicStatCounter stat_contains_size7 = 0; | |
1036 | _AtomicStatCounter stat_contains_size8 = 0; | |
1037 | _AtomicStatCounter stat_contains_size9 = 0; | |
1038 | _AtomicStatCounter stat_contains_size10 = 0; | |
1039 | _AtomicStatCounter stat_contains_size11 = 0; | |
1040 | _AtomicStatCounter stat_contains_size12 = 0; | |
1041 | _AtomicStatCounter stat_contains_size13_or_more = 0; | |
1042 | _AtomicStatCounter stat_contains_sizes = 0; | |
1043 | _AtomicStatCounter stat_grow_from_size0_to_1 = 0; | |
1044 | _AtomicStatCounter stat_grow_from_size1_to_2 = 0; | |
1045 | _AtomicStatCounter stat_double_the_number_of_buckets = 0; | |
1046 | _AtomicStatCounter stat_insert_found_hash_collision = 0; | |
1047 | _AtomicStatCounter stat_contains_in_non_trivial_set = 0; | |
1048 | _AtomicStatCounter stat_insert_key_that_was_already_present = 0; | |
1049 | #endif | |
1050 | /* Record statistics about the hash table sets, for debugging. */ | |
1051 | ||
1052 | static void | |
1053 | log_set_stats (void) | |
1054 | { | |
1055 | #if HASHTABLE_STATS | |
1056 | if (set_log_fd == -1) | |
1057 | set_log_fd = __vtv_open_log ("vtv_set_stats.log"); | |
1058 | ||
1059 | __vtv_add_to_log (set_log_fd, "---\n%s\n", | |
1060 | insert_only_hash_tables_stats().c_str()); | |
1061 | #endif | |
1062 | } | |
1063 | ||
1064 | /* Change the permissions on all the pages we have allocated for the | |
1065 | data sets and all the ".vtable_map_var" sections in memory (which | |
1066 | contain our vtable map variables). PERM indicates whether to make | |
1067 | the permissions read-only or read-write. */ | |
1068 | ||
1069 | extern "C" /* This is only being applied to __VLTChangePermission*/ | |
1070 | void | |
1071 | __VLTChangePermission (int perm) | |
1072 | { | |
1073 | if (debug_functions) | |
1074 | { | |
1075 | if (perm == __VLTP_READ_WRITE) | |
08de21a4 | 1076 | fprintf (stdout, "Changing VLT permissions to Read-Write.\n"); |
b710ec85 | 1077 | else if (perm == __VLTP_READ_ONLY) |
08de21a4 | 1078 | fprintf (stdout, "Changing VLT permissions to Read-Only.\n"); |
b710ec85 | 1079 | |
1080 | else | |
1081 | fprintf (stdout, "Unrecognized permissions value: %d\n", perm); | |
1082 | } | |
1083 | ||
1084 | #ifndef __GTHREAD_MUTEX_INIT | |
1085 | static __gthread_once_t mutex_once VTV_PROTECTED_VAR = __GTHREAD_ONCE_INIT; | |
1086 | ||
1087 | __gthread_once (&mutex_once, initialize_change_permissions_mutexes); | |
1088 | #endif | |
1089 | ||
1090 | /* Ordering of these unprotect/protect calls is very important. | |
1091 | You first need to unprotect all the map vars and side | |
1092 | structures before you do anything with the core data | |
1093 | structures (hash_maps) */ | |
1094 | ||
1095 | if (perm == __VLTP_READ_WRITE) | |
1096 | { | |
1097 | /* TODO: Need to revisit this code for dlopen. It most probably | |
1098 | is not unlocking the protected vtable vars after for load | |
1099 | module that is not the first load module. */ | |
1100 | __gthread_mutex_lock (&change_permissions_lock); | |
1101 | ||
1102 | vtv_unprotect_vtable_vars (); | |
1103 | __vtv_malloc_init (); | |
1104 | __vtv_malloc_unprotect (); | |
1105 | ||
1106 | } | |
1107 | else if (perm == __VLTP_READ_ONLY) | |
1108 | { | |
1109 | if (debug_hash) | |
1110 | log_set_stats(); | |
1111 | ||
1112 | __vtv_malloc_protect (); | |
1113 | vtv_protect_vtable_vars (); | |
1114 | ||
1115 | __gthread_mutex_unlock (&change_permissions_lock); | |
1116 | } | |
1117 | } | |
1118 | ||
1119 | /* This is the memory allocator used to create the hash table that | |
1120 | maps from vtable map variable name to the data set that vtable map | |
1121 | variable should point to. This is part of our vtable map variable | |
1122 | symbol resolution, which is necessary because the same vtable map | |
1123 | variable may be created by multiple compilation units and we need a | |
1124 | method to make sure that all vtable map variables for a particular | |
1125 | class point to the same data set at runtime. */ | |
1126 | ||
1127 | struct insert_only_hash_map_allocator | |
1128 | { | |
1129 | /* N is the number of bytes to allocate. */ | |
1130 | void * | |
1131 | alloc (size_t n) const | |
1132 | { | |
1133 | return __vtv_malloc (n); | |
1134 | } | |
1135 | ||
1136 | /* P points to the memory to be deallocated; N is the number of | |
1137 | bytes to deallocate. */ | |
1138 | void | |
1139 | dealloc (void *p, size_t) const | |
1140 | { | |
1141 | __vtv_free (p); | |
1142 | } | |
1143 | }; | |
1144 | ||
1145 | /* Explicitly instantiate this class since this file is compiled with | |
1146 | -fno-implicit-templates. These are for the hash table that is used | |
1147 | to do vtable map variable symbol resolution. */ | |
1148 | template class insert_only_hash_map <vtv_set_handle *, | |
1149 | insert_only_hash_map_allocator >; | |
1150 | typedef insert_only_hash_map <vtv_set_handle *, | |
1151 | insert_only_hash_map_allocator > s2s; | |
1152 | typedef const s2s::key_type vtv_symbol_key; | |
1153 | ||
1154 | static s2s * vtv_symbol_unification_map VTV_PROTECTED_VAR = NULL; | |
1155 | ||
1156 | const unsigned long SET_HANDLE_HANDLE_BIT = 0x2; | |
1157 | ||
1158 | /* In the case where a vtable map variable is the only instance of the | |
1159 | variable we have seen, it points directly to the set of valid | |
1160 | vtable pointers. All subsequent instances of the 'same' vtable map | |
1161 | variable point to the first vtable map variable. This function, | |
1162 | given a vtable map variable PTR, checks a bit to see whether it's | |
1163 | pointing directly to the data set or to the first vtable map | |
1164 | variable. */ | |
1165 | ||
1166 | static inline bool | |
1167 | is_set_handle_handle (void * ptr) | |
1168 | { | |
5be42fa9 | 1169 | return ((uintptr_t) ptr & SET_HANDLE_HANDLE_BIT) |
b710ec85 | 1170 | == SET_HANDLE_HANDLE_BIT; |
1171 | } | |
1172 | ||
1173 | /* Returns the actual pointer value of a vtable map variable, PTR (see | |
1174 | comments for is_set_handle_handle for more details). */ | |
1175 | ||
1176 | static inline vtv_set_handle * | |
1177 | ptr_from_set_handle_handle (void * ptr) | |
1178 | { | |
5be42fa9 | 1179 | return (vtv_set_handle *) ((uintptr_t) ptr & ~SET_HANDLE_HANDLE_BIT); |
b710ec85 | 1180 | } |
1181 | ||
1182 | /* Given a vtable map variable, PTR, this function sets the bit that | |
1183 | says this is the second (or later) instance of a vtable map | |
1184 | variable. */ | |
1185 | ||
1186 | static inline vtv_set_handle_handle | |
1187 | set_handle_handle (vtv_set_handle * ptr) | |
1188 | { | |
5be42fa9 | 1189 | return (vtv_set_handle_handle) ((uintptr_t) ptr | SET_HANDLE_HANDLE_BIT); |
b710ec85 | 1190 | } |
1191 | ||
1192 | static inline void | |
1193 | register_set_common (void **set_handle_ptr, size_t num_args, | |
1194 | void **vtable_ptr_array, bool debug) | |
1195 | { | |
1196 | /* Now figure out what pointer to use for the set pointer, for the | |
1197 | inserts. */ | |
1198 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1199 | ||
1200 | if (debug) | |
1201 | VTV_DEBUG_ASSERT (vtv_symbol_unification_map != NULL); | |
1202 | ||
1203 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1204 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1205 | else | |
1206 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1207 | ||
1208 | /* Now we've got the set and it's initialized, add the vtable | |
1209 | pointers. */ | |
1210 | for (size_t index = 0; index < num_args; ++index) | |
1211 | { | |
1212 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr_array[index]; | |
1213 | vtv_sets::insert (vtbl_ptr, handle_ptr); | |
1214 | } | |
1215 | } | |
1216 | ||
1217 | static inline void | |
1218 | register_pair_common (void **set_handle_ptr, const void *vtable_ptr, | |
1219 | const char *set_symbol_name, const char *vtable_name, | |
1220 | bool debug) | |
1221 | { | |
1222 | /* Now we've got the set and it's initialized, add the vtable | |
1223 | pointer (assuming that it's not NULL...It may be NULL, as we may | |
1224 | have called this function merely to initialize the set | |
1225 | pointer). */ | |
1226 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1227 | if (vtbl_ptr) | |
1228 | { | |
1229 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1230 | if (debug) | |
1231 | VTV_DEBUG_ASSERT (vtv_symbol_unification_map != NULL); | |
1232 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1233 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1234 | else | |
1235 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1236 | ||
1237 | vtv_sets::insert (vtbl_ptr, handle_ptr); | |
1238 | } | |
1239 | ||
1240 | if (debug && debug_init) | |
1241 | { | |
1242 | if (init_log_fd == -1) | |
1243 | init_log_fd = __vtv_open_log("vtv_init.log"); | |
1244 | ||
1245 | __vtv_add_to_log(init_log_fd, | |
1246 | "Registered %s : %s (%p) 2 level deref = %s\n", | |
1247 | set_symbol_name, vtable_name, vtbl_ptr, | |
1248 | is_set_handle_handle(*set_handle_ptr) ? "yes" : "no" ); | |
1249 | } | |
1250 | } | |
1251 | ||
1252 | /* This routine initializes a set handle to a vtable set. It makes | |
1253 | sure that there is only one set handle for a particular set by | |
1254 | using a map from set name to pointer to set handle. Since there | |
1255 | will be multiple copies of the pointer to the set handle (one per | |
1256 | compilation unit that uses it), it makes sure to initialize all the | |
1257 | pointers to the set handle so that the set handle is unique. To | |
1258 | make this a little more efficient and avoid a level of indirection | |
1259 | in some cases, the first pointer to handle for a particular handle | |
1260 | becomes the handle itself and the other pointers will point to the | |
1261 | set handle. This is the debug version of this function, so it | |
1262 | outputs extra debugging messages and logging. SET_HANDLE_PTR is | |
1263 | the address of the vtable map variable, SET_SYMBOL_KEY is the hash | |
1264 | table key (containing the name of the map variable and the hash | |
1265 | value) and SIZE_HINT is a guess for the best initial size for the | |
1266 | set of vtable pointers that SET_HANDLE_POINTER will point to. */ | |
1267 | ||
1268 | static inline void | |
1269 | init_set_symbol_debug (void **set_handle_ptr, const void *set_symbol_key, | |
1270 | size_t size_hint) | |
1271 | { | |
1272 | VTV_DEBUG_ASSERT (set_handle_ptr); | |
1273 | ||
1274 | if (vtv_symbol_unification_map == NULL) | |
1275 | { | |
1276 | /* TODO: For now we have chosen 1024, but we need to come up with a | |
1277 | better initial size for this. */ | |
1278 | vtv_symbol_unification_map = s2s::create (1024); | |
1279 | VTV_DEBUG_ASSERT(vtv_symbol_unification_map); | |
1280 | } | |
1281 | ||
1282 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1283 | vtv_symbol_key *symbol_key_ptr = (vtv_symbol_key *) set_symbol_key; | |
1284 | ||
1285 | const s2s::value_type * map_value_ptr = | |
1286 | vtv_symbol_unification_map->get (symbol_key_ptr); | |
1287 | char buffer[200]; | |
1288 | if (map_value_ptr == NULL) | |
1289 | { | |
1290 | if (*handle_ptr != NULL) | |
1291 | { | |
1292 | snprintf (buffer, sizeof (buffer), | |
1293 | "*** Found non-NULL local set ptr %p missing for symbol" | |
1294 | " %.*s", | |
1295 | *handle_ptr, symbol_key_ptr->n, symbol_key_ptr->bytes); | |
1296 | __vtv_log_verification_failure (buffer, true); | |
1297 | VTV_DEBUG_ASSERT (0); | |
1298 | } | |
1299 | } | |
1300 | else if (*handle_ptr != NULL && | |
1301 | (handle_ptr != *map_value_ptr && | |
1302 | ptr_from_set_handle_handle (*handle_ptr) != *map_value_ptr)) | |
1303 | { | |
1304 | VTV_DEBUG_ASSERT (*map_value_ptr != NULL); | |
1305 | snprintf (buffer, sizeof(buffer), | |
1306 | "*** Found diffence between local set ptr %p and set ptr %p" | |
1307 | "for symbol %.*s", | |
1308 | *handle_ptr, *map_value_ptr, | |
1309 | symbol_key_ptr->n, symbol_key_ptr->bytes); | |
1310 | __vtv_log_verification_failure (buffer, true); | |
1311 | VTV_DEBUG_ASSERT (0); | |
1312 | } | |
1313 | else if (*handle_ptr == NULL) | |
1314 | { | |
1315 | /* Execution should not reach this point. */ | |
1316 | } | |
1317 | ||
1318 | if (*handle_ptr != NULL) | |
1319 | { | |
1320 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1321 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1322 | else | |
1323 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1324 | vtv_sets::resize (size_hint, handle_ptr); | |
1325 | return; | |
1326 | } | |
1327 | ||
1328 | VTV_DEBUG_ASSERT (*handle_ptr == NULL); | |
1329 | if (map_value_ptr != NULL) | |
1330 | { | |
1331 | if (*map_value_ptr == handle_ptr) | |
1332 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1333 | else | |
1334 | { | |
1335 | /* The one level handle to the set already exists. So, we | |
1336 | are adding one level of indirection here and we will | |
1337 | store a pointer to the one level handle here. */ | |
1338 | ||
1339 | vtv_set_handle_handle * handle_handle_ptr = | |
1340 | (vtv_set_handle_handle *)handle_ptr; | |
1341 | *handle_handle_ptr = set_handle_handle(*map_value_ptr); | |
1342 | VTV_DEBUG_ASSERT(*handle_handle_ptr != NULL); | |
1343 | ||
1344 | /* The handle can itself be NULL if the set has only | |
1345 | been initiazlied with size hint == 1. */ | |
1346 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1347 | } | |
1348 | } | |
1349 | else | |
1350 | { | |
1351 | /* We will create a new set. So, in this case handle_ptr is the | |
1352 | one level pointer to the set handle. Create copy of map name | |
1353 | in case the memory where this comes from gets unmapped by | |
1354 | dlclose. */ | |
1355 | size_t map_key_len = symbol_key_ptr->n + sizeof (vtv_symbol_key); | |
1356 | void *map_key = __vtv_malloc (map_key_len); | |
1357 | ||
1358 | memcpy (map_key, symbol_key_ptr, map_key_len); | |
1359 | ||
1360 | s2s::value_type *value_ptr; | |
1361 | vtv_symbol_unification_map = | |
1362 | vtv_symbol_unification_map->find_or_add_key ((vtv_symbol_key *)map_key, | |
1363 | &value_ptr); | |
1364 | *value_ptr = handle_ptr; | |
1365 | ||
1366 | /* TODO: We should verify the return value. */ | |
1367 | vtv_sets::create (size_hint, handle_ptr); | |
1368 | VTV_DEBUG_ASSERT (size_hint <= 1 || *handle_ptr != NULL); | |
1369 | } | |
1370 | ||
1371 | if (debug_init) | |
1372 | { | |
1373 | if (init_log_fd == -1) | |
1374 | init_log_fd = __vtv_open_log ("vtv_init.log"); | |
1375 | ||
1376 | __vtv_add_to_log (init_log_fd, | |
1377 | "Init handle:%p for symbol:%.*s hash:%u size_hint:%lu" | |
1378 | "number of symbols:%lu \n", | |
1379 | set_handle_ptr, symbol_key_ptr->n, | |
1380 | symbol_key_ptr->bytes, symbol_key_ptr->hash, size_hint, | |
1381 | vtv_symbol_unification_map->size ()); | |
1382 | } | |
1383 | } | |
1384 | ||
1385 | ||
1386 | /* This routine initializes a set handle to a vtable set. It makes | |
1387 | sure that there is only one set handle for a particular set by | |
1388 | using a map from set name to pointer to set handle. Since there | |
1389 | will be multiple copies of the pointer to the set handle (one per | |
1390 | compilation unit that uses it), it makes sure to initialize all the | |
1391 | pointers to the set handle so that the set handle is unique. To | |
1392 | make this a little more efficient and avoid a level of indirection | |
1393 | in some cases, the first pointer to handle for a particular handle | |
1394 | becomes the handle itself and the other pointers will point to the | |
1395 | set handle. This is the debug version of this function, so it | |
1396 | outputs extra debugging messages and logging. SET_HANDLE_PTR is | |
1397 | the address of the vtable map variable, SET_SYMBOL_KEY is the hash | |
1398 | table key (containing the name of the map variable and the hash | |
1399 | value) and SIZE_HINT is a guess for the best initial size for the | |
1400 | set of vtable pointers that SET_HANDLE_POINTER will point to. */ | |
1401 | ||
1402 | void | |
1403 | __VLTRegisterSetDebug (void **set_handle_ptr, const void *set_symbol_key, | |
1404 | size_t size_hint, size_t num_args, | |
1405 | void **vtable_ptr_array) | |
1406 | { | |
1407 | unsigned long long start = get_cycle_count (); | |
1408 | increment_num_calls (&num_calls_to_regset); | |
1409 | ||
1410 | VTV_DEBUG_ASSERT(set_handle_ptr != NULL); | |
1411 | init_set_symbol_debug (set_handle_ptr, set_symbol_key, size_hint); | |
1412 | ||
1413 | register_set_common (set_handle_ptr, num_args, vtable_ptr_array, true); | |
1414 | ||
1415 | accumulate_cycle_count (®set_cycles, start); | |
1416 | } | |
1417 | ||
1418 | /* This function takes a the address of a vtable map variable | |
1419 | (SET_HANDLE_PTR), a VTABLE_PTR to add to the data set, the name of | |
1420 | the vtable map variable (SET_SYMBOL_NAME) and the name of the | |
1421 | vtable (VTABLE_NAME) being pointed to. If the vtable map variable | |
1422 | is NULL it creates a new data set and initializes the variable, | |
1423 | otherwise it uses our symbol unification to find the right data | |
1424 | set; in either case it then adds the vtable pointer to the set. | |
1425 | The other two parameters are used for debugging information. */ | |
1426 | ||
1427 | void | |
1428 | __VLTRegisterPairDebug (void **set_handle_ptr, const void *set_symbol_key, | |
1429 | size_t size_hint, const void *vtable_ptr, | |
1430 | const char *set_symbol_name, const char *vtable_name) | |
1431 | { | |
1432 | unsigned long long start = get_cycle_count (); | |
1433 | increment_num_calls (&num_calls_to_regpair); | |
1434 | ||
1435 | VTV_DEBUG_ASSERT(set_handle_ptr != NULL); | |
1436 | init_set_symbol_debug (set_handle_ptr, set_symbol_key, size_hint); | |
1437 | ||
1438 | register_pair_common (set_handle_ptr, vtable_ptr, set_symbol_name, vtable_name, | |
1439 | true); | |
1440 | ||
1441 | accumulate_cycle_count (®pair_cycles, start); | |
1442 | } | |
1443 | ||
1444 | ||
1445 | /* This is the debug version of the verification function. It takes | |
1446 | the address of a vtable map variable (SET_HANDLE_PTR) and a | |
1447 | VTABLE_PTR to validate, as well as the name of the vtable map | |
1448 | variable (SET_SYMBOL_NAME) and VTABLE_NAME, which are used for | |
1449 | debugging messages. It checks to see if VTABLE_PTR is in the set | |
1450 | pointed to by SET_HANDLE_PTR. If so, it returns VTABLE_PTR, | |
1451 | otherwise it calls __vtv_verify_fail, which usually logs error | |
1452 | messages and calls abort. */ | |
1453 | ||
1454 | const void * | |
1455 | __VLTVerifyVtablePointerDebug (void **set_handle_ptr, const void *vtable_ptr, | |
1456 | const char *set_symbol_name, | |
1457 | const char *vtable_name) | |
1458 | { | |
1459 | unsigned long long start = get_cycle_count (); | |
1460 | VTV_DEBUG_ASSERT (set_handle_ptr != NULL && *set_handle_ptr != NULL); | |
1461 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1462 | ||
1463 | increment_num_calls (&num_calls_to_verify_vtable); | |
1464 | vtv_set_handle *handle_ptr; | |
1465 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1466 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1467 | else | |
1468 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1469 | ||
1470 | if (vtv_sets::contains (vtbl_ptr, handle_ptr)) | |
1471 | { | |
1472 | if (debug_verify_vtable) | |
1473 | { | |
1474 | if (verify_vtable_log_fd == -1) | |
1475 | __vtv_open_log ("vtv_verify_vtable.log"); | |
1476 | __vtv_add_to_log (verify_vtable_log_fd, | |
1477 | "Verified %s %s value = %p\n", | |
1478 | set_symbol_name, vtable_name, vtable_ptr); | |
1479 | } | |
1480 | } | |
1481 | else | |
1482 | { | |
1483 | /* We failed to find the vtable pointer in the set of valid | |
1484 | pointers. Log the error data and call the failure | |
1485 | function. */ | |
1486 | snprintf (debug_log_message, sizeof (debug_log_message), | |
1487 | "Looking for %s in %s\n", vtable_name, set_symbol_name); | |
1488 | __vtv_verify_fail_debug (set_handle_ptr, vtable_ptr, debug_log_message); | |
1489 | ||
1490 | /* Normally __vtv_verify_fail_debug will call abort, so we won't | |
1491 | execute the return below. If we get this far, the assumption | |
1492 | is that the programmer has replaced __vtv_verify_fail_debug | |
1493 | with some kind of secondary verification AND this secondary | |
1494 | verification succeeded, so the vtable pointer is valid. */ | |
1495 | } | |
1496 | accumulate_cycle_count (&verify_vtable_cycles, start); | |
1497 | ||
1498 | return vtable_ptr; | |
1499 | } | |
1500 | ||
1501 | /* This routine initializes a set handle to a vtable set. It makes | |
1502 | sure that there is only one set handle for a particular set by | |
1503 | using a map from set name to pointer to set handle. Since there | |
1504 | will be multiple copies of the pointer to the set handle (one per | |
1505 | compilation unit that uses it), it makes sure to initialize all the | |
1506 | pointers to the set handle so that the set handle is unique. To | |
1507 | make this a little more efficient and avoid a level of indirection | |
1508 | in some cases, the first pointer to handle for a particular handle | |
1509 | becomes the handle itself and the other pointers will point to the | |
1510 | set handle. SET_HANDLE_PTR is the address of the vtable map | |
1511 | variable, SET_SYMBOL_KEY is the hash table key (containing the name | |
1512 | of the map variable and the hash value) and SIZE_HINT is a guess | |
1513 | for the best initial size for the set of vtable pointers that | |
1514 | SET_HANDLE_POINTER will point to.*/ | |
1515 | ||
1516 | static inline void | |
1517 | init_set_symbol (void **set_handle_ptr, const void *set_symbol_key, | |
1518 | size_t size_hint) | |
1519 | { | |
1520 | vtv_set_handle *handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1521 | ||
1522 | if (*handle_ptr != NULL) | |
1523 | { | |
1524 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1525 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1526 | else | |
1527 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1528 | vtv_sets::resize (size_hint, handle_ptr); | |
1529 | return; | |
1530 | } | |
1531 | ||
1532 | if (vtv_symbol_unification_map == NULL) | |
1533 | vtv_symbol_unification_map = s2s::create (1024); | |
1534 | ||
1535 | vtv_symbol_key *symbol_key_ptr = (vtv_symbol_key *) set_symbol_key; | |
1536 | const s2s::value_type *map_value_ptr = | |
1537 | vtv_symbol_unification_map->get (symbol_key_ptr); | |
1538 | ||
1539 | if (map_value_ptr != NULL) | |
1540 | { | |
1541 | if (*map_value_ptr == handle_ptr) | |
1542 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1543 | else | |
1544 | { | |
1545 | /* The one level handle to the set already exists. So, we | |
1546 | are adding one level of indirection here and we will | |
1547 | store a pointer to the one level pointer here. */ | |
1548 | vtv_set_handle_handle *handle_handle_ptr = | |
1549 | (vtv_set_handle_handle *) handle_ptr; | |
1550 | *handle_handle_ptr = set_handle_handle (*map_value_ptr); | |
1551 | vtv_sets::resize (size_hint, *map_value_ptr); | |
1552 | } | |
1553 | } | |
1554 | else | |
1555 | { | |
1556 | /* We will create a new set. So, in this case handle_ptr is the | |
1557 | one level pointer to the set handle. Create copy of map name | |
1558 | in case the memory where this comes from gets unmapped by | |
1559 | dlclose. */ | |
1560 | size_t map_key_len = symbol_key_ptr->n + sizeof (vtv_symbol_key); | |
1561 | void * map_key = __vtv_malloc (map_key_len); | |
1562 | memcpy (map_key, symbol_key_ptr, map_key_len); | |
1563 | ||
1564 | s2s::value_type * value_ptr; | |
1565 | vtv_symbol_unification_map = | |
1566 | vtv_symbol_unification_map->find_or_add_key ((vtv_symbol_key *)map_key, | |
1567 | &value_ptr); | |
1568 | ||
1569 | *value_ptr = handle_ptr; | |
1570 | ||
1571 | /* TODO: We should verify the return value. */ | |
1572 | vtv_sets::create (size_hint, handle_ptr); | |
1573 | } | |
1574 | } | |
1575 | ||
1576 | /* This routine initializes a set handle to a vtable set. It makes | |
1577 | sure that there is only one set handle for a particular set by | |
1578 | using a map from set name to pointer to set handle. Since there | |
1579 | will be multiple copies of the pointer to the set handle (one per | |
1580 | compilation unit that uses it), it makes sure to initialize all the | |
1581 | pointers to the set handle so that the set handle is unique. To | |
1582 | make this a little more efficient and avoid a level of indirection | |
1583 | in some cases, the first pointer to handle for a particular handle | |
1584 | becomes the handle itself and the other pointers will point to the | |
1585 | set handle. SET_HANDLE_PTR is the address of the vtable map | |
1586 | variable, SET_SYMBOL_KEY is the hash table key (containing the name | |
1587 | of the map variable and the hash value) and SIZE_HINT is a guess | |
1588 | for the best initial size for the set of vtable pointers that | |
1589 | SET_HANDLE_POINTER will point to.*/ | |
1590 | ||
1591 | ||
1592 | void | |
1593 | __VLTRegisterSet (void **set_handle_ptr, const void *set_symbol_key, | |
1594 | size_t size_hint, size_t num_args, void **vtable_ptr_array) | |
1595 | { | |
1596 | unsigned long long start = get_cycle_count (); | |
1597 | increment_num_calls (&num_calls_to_regset); | |
1598 | ||
1599 | init_set_symbol (set_handle_ptr, set_symbol_key, size_hint); | |
1600 | register_set_common (set_handle_ptr, num_args, vtable_ptr_array, false); | |
1601 | ||
1602 | accumulate_cycle_count (®set_cycles, start); | |
1603 | } | |
1604 | ||
1605 | ||
1606 | ||
1607 | /* This function takes a the address of a vtable map variable | |
1608 | (SET_HANDLE_PTR) and a VTABLE_PTR. If the vtable map variable is | |
1609 | NULL it creates a new data set and initializes the variable, | |
1610 | otherwise it uses our symbol unification to find the right data | |
1611 | set; in either case it then adds the vtable pointer to the set. */ | |
1612 | ||
1613 | void | |
1614 | __VLTRegisterPair (void **set_handle_ptr, const void *set_symbol_key, | |
1615 | size_t size_hint, const void *vtable_ptr) | |
1616 | { | |
1617 | unsigned long long start = get_cycle_count (); | |
1618 | increment_num_calls (&num_calls_to_regpair); | |
1619 | ||
1620 | init_set_symbol (set_handle_ptr, set_symbol_key, size_hint); | |
1621 | register_pair_common (set_handle_ptr, vtable_ptr, NULL, NULL, false); | |
1622 | ||
1623 | accumulate_cycle_count (®pair_cycles, start); | |
1624 | } | |
1625 | ||
1626 | /* This is the main verification function. It takes the address of a | |
1627 | vtable map variable (SET_HANDLE_PTR) and a VTABLE_PTR to validate. | |
1628 | It checks to see if VTABLE_PTR is in the set pointed to by | |
1629 | SET_HANDLE_PTR. If so, it returns VTABLE_PTR, otherwise it calls | |
1630 | __vtv_verify_fail, which usually logs error messages and calls | |
1631 | abort. Since this function gets called VERY frequently, it is | |
1632 | important for it to be as efficient as possible. */ | |
1633 | ||
1634 | const void * | |
1635 | __VLTVerifyVtablePointer (void ** set_handle_ptr, const void * vtable_ptr) | |
1636 | { | |
1637 | unsigned long long start = get_cycle_count (); | |
1638 | int_vptr vtbl_ptr = (int_vptr) vtable_ptr; | |
1639 | ||
1640 | vtv_set_handle *handle_ptr; | |
1641 | increment_num_calls (&num_calls_to_verify_vtable); | |
1642 | if (!is_set_handle_handle (*set_handle_ptr)) | |
1643 | handle_ptr = (vtv_set_handle *) set_handle_ptr; | |
1644 | else | |
1645 | handle_ptr = ptr_from_set_handle_handle (*set_handle_ptr); | |
1646 | ||
1647 | if (!vtv_sets::contains (vtbl_ptr, handle_ptr)) | |
1648 | { | |
1649 | __vtv_verify_fail ((void **) handle_ptr, vtable_ptr); | |
1650 | /* Normally __vtv_verify_fail will call abort, so we won't | |
1651 | execute the return below. If we get this far, the assumption | |
1652 | is that the programmer has replaced __vtv_verify_fail with | |
1653 | some kind of secondary verification AND this secondary | |
1654 | verification succeeded, so the vtable pointer is valid. */ | |
1655 | } | |
1656 | accumulate_cycle_count (&verify_vtable_cycles, start); | |
1657 | ||
1658 | return vtable_ptr; | |
1659 | } | |
1660 | ||
1661 | static int page_count_2 = 0; | |
1662 | ||
5be42fa9 | 1663 | #if !defined (__CYGWIN__) && !defined (__MINGW32__) |
b710ec85 | 1664 | static int |
1665 | dl_iterate_phdr_count_pages (struct dl_phdr_info *info, | |
1666 | size_t unused __attribute__ ((__unused__)), | |
1667 | void *data) | |
1668 | { | |
1669 | int *mprotect_flags = (int *) data; | |
1670 | off_t map_sect_offset = 0; | |
1671 | ElfW (Word) map_sect_len = 0; | |
1672 | const char *map_sect_name = VTV_PROTECTED_VARS_SECTION; | |
1673 | ||
1674 | /* Check to see if this is the record for the Linux Virtual Dynamic | |
1675 | Shared Object (linux-vdso.so.1), which exists only in memory (and | |
1676 | therefore cannot be read from disk). */ | |
1677 | ||
1678 | if (strcmp (info->dlpi_name, "linux-vdso.so.1") == 0) | |
1679 | return 0; | |
1680 | ||
1681 | if (strlen (info->dlpi_name) == 0 | |
1682 | && info->dlpi_addr != 0) | |
1683 | return 0; | |
1684 | ||
1685 | read_section_offset_and_length (info, map_sect_name, *mprotect_flags, | |
1686 | &map_sect_offset, &map_sect_len); | |
1687 | ||
1688 | /* See if we actually found the section. */ | |
1689 | if (map_sect_len) | |
1690 | page_count_2 += (map_sect_len + VTV_PAGE_SIZE - 1) / VTV_PAGE_SIZE; | |
1691 | ||
1692 | return 0; | |
1693 | } | |
5be42fa9 | 1694 | #endif |
b710ec85 | 1695 | |
1696 | static void | |
1697 | count_all_pages (void) | |
1698 | { | |
1699 | int mprotect_flags; | |
1700 | ||
1701 | mprotect_flags = PROT_READ; | |
1702 | page_count_2 = 0; | |
1703 | ||
5be42fa9 | 1704 | #if defined (__CYGWIN__) || defined (__MINGW32__) |
1705 | iterate_modules ((void *) &mprotect_flags); | |
1706 | #else | |
b710ec85 | 1707 | dl_iterate_phdr (dl_iterate_phdr_count_pages, (void *) &mprotect_flags); |
5be42fa9 | 1708 | #endif |
b710ec85 | 1709 | page_count_2 += __vtv_count_mmapped_pages (); |
1710 | } | |
1711 | ||
1712 | void | |
1713 | __VLTDumpStats (void) | |
1714 | { | |
1715 | int log_fd = __vtv_open_log ("vtv-runtime-stats.log"); | |
1716 | ||
1717 | if (log_fd != -1) | |
1718 | { | |
1719 | count_all_pages (); | |
1720 | __vtv_add_to_log (log_fd, | |
1721 | "Calls: mprotect (%d) regset (%d) regpair (%d)" | |
1722 | " verify_vtable (%d)\n", | |
1723 | num_calls_to_mprotect, num_calls_to_regset, | |
1724 | num_calls_to_regpair, num_calls_to_verify_vtable); | |
1725 | __vtv_add_to_log (log_fd, | |
1726 | "Cycles: mprotect (%lld) regset (%lld) " | |
1727 | "regpair (%lld) verify_vtable (%lld)\n", | |
1728 | mprotect_cycles, regset_cycles, regpair_cycles, | |
1729 | verify_vtable_cycles); | |
1730 | __vtv_add_to_log (log_fd, | |
1731 | "Pages protected (1): %d\n", num_pages_protected); | |
1732 | __vtv_add_to_log (log_fd, "Pages protected (2): %d\n", page_count_2); | |
1733 | ||
1734 | close (log_fd); | |
1735 | } | |
1736 | } | |
1737 | ||
1738 | /* This function is called from __VLTVerifyVtablePointerDebug; it | |
1739 | sends as much debugging information as it can to the error log | |
1740 | file, then calls __vtv_verify_fail. SET_HANDLE_PTR is the pointer | |
1741 | to the set of valid vtable pointers, VTBL_PTR is the pointer that | |
1742 | was not found in the set, and DEBUG_MSG is the message to be | |
1743 | written to the log file before failing. n */ | |
1744 | ||
1745 | void | |
1746 | __vtv_verify_fail_debug (void **set_handle_ptr, const void *vtbl_ptr, | |
1747 | const char *debug_msg) | |
1748 | { | |
1749 | __vtv_log_verification_failure (debug_msg, false); | |
1750 | ||
1751 | /* Call the public interface in case it has been overwritten by | |
1752 | user. */ | |
1753 | __vtv_verify_fail (set_handle_ptr, vtbl_ptr); | |
1754 | ||
1755 | __vtv_log_verification_failure ("Returned from __vtv_verify_fail." | |
1756 | " Secondary verification succeeded.\n", false); | |
1757 | } | |
1758 | ||
1759 | /* This function calls __fortify_fail with a FAILURE_MSG and then | |
1760 | calls abort. */ | |
1761 | ||
1762 | void | |
1763 | __vtv_really_fail (const char *failure_msg) | |
1764 | { | |
1765 | __fortify_fail (failure_msg); | |
1766 | ||
1767 | /* We should never get this far; __fortify_fail calls __libc_message | |
1768 | which prints out a back trace and a memory dump and then is | |
1769 | supposed to call abort, but let's play it safe anyway and call abort | |
1770 | ourselves. */ | |
1771 | abort (); | |
1772 | } | |
1773 | ||
1774 | /* This function takes an error MSG, a vtable map variable | |
1775 | (DATA_SET_PTR) and a vtable pointer (VTBL_PTR). It is called when | |
1776 | an attempt to verify VTBL_PTR with the set pointed to by | |
1777 | DATA_SET_PTR failed. It outputs a failure message with the | |
1778 | addresses involved, and calls __vtv_really_fail. */ | |
1779 | ||
1780 | static void | |
1781 | vtv_fail (const char *msg, void **data_set_ptr, const void *vtbl_ptr) | |
1782 | { | |
1783 | char buffer[128]; | |
1784 | int buf_len; | |
1785 | const char *format_str = | |
1786 | "*** Unable to verify vtable pointer (%p) in set (%p) *** \n"; | |
1787 | ||
1788 | snprintf (buffer, sizeof (buffer), format_str, vtbl_ptr, | |
1789 | is_set_handle_handle(*data_set_ptr) ? | |
1790 | ptr_from_set_handle_handle (*data_set_ptr) : | |
1791 | *data_set_ptr); | |
1792 | buf_len = strlen (buffer); | |
1793 | /* Send this to to stderr. */ | |
1794 | write (2, buffer, buf_len); | |
1795 | ||
1796 | #ifndef VTV_NO_ABORT | |
1797 | __vtv_really_fail (msg); | |
1798 | #endif | |
1799 | } | |
1800 | ||
1801 | /* Send information about what we were trying to do when verification | |
1802 | failed to the error log, then call vtv_fail. This function can be | |
1803 | overwritten/replaced by the user, to implement a secondary | |
1804 | verification function instead. DATA_SET_PTR is the vtable map | |
1805 | variable used for the failed verification, and VTBL_PTR is the | |
1806 | vtable pointer that was not found in the set. */ | |
1807 | ||
1808 | void | |
1809 | __vtv_verify_fail (void **data_set_ptr, const void *vtbl_ptr) | |
1810 | { | |
1811 | char log_msg[256]; | |
1812 | snprintf (log_msg, sizeof (log_msg), "Looking for vtable %p in set %p.\n", | |
1813 | vtbl_ptr, | |
1814 | is_set_handle_handle (*data_set_ptr) ? | |
1815 | ptr_from_set_handle_handle (*data_set_ptr) : | |
1816 | *data_set_ptr); | |
1817 | __vtv_log_verification_failure (log_msg, false); | |
1818 | ||
1819 | const char *format_str = | |
1820 | "*** Unable to verify vtable pointer (%p) in set (%p) *** \n"; | |
1821 | snprintf (log_msg, sizeof (log_msg), format_str, vtbl_ptr, *data_set_ptr); | |
1822 | __vtv_log_verification_failure (log_msg, false); | |
1823 | __vtv_log_verification_failure (" Backtrace: \n", true); | |
1824 | ||
1825 | const char *fail_msg = "Potential vtable pointer corruption detected!!\n"; | |
1826 | vtv_fail (fail_msg, data_set_ptr, vtbl_ptr); | |
1827 | } |