From: Omar Sandoval Date: Mon, 26 Feb 2024 19:32:51 +0000 (-0800) Subject: libdw: Handle overflowed DW_SECT_INFO offsets in DWARF package file indexes X-Git-Tag: elfutils-0.191~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3979362ef98db16d1c6fe8fb12ee6a966305da49;p=thirdparty%2Felfutils.git libdw: Handle overflowed DW_SECT_INFO offsets in DWARF package file indexes Meta uses DWARF package files for our large, statically-linked C++ applications. Some of our largest applications have more than 4GB in .debug_info.dwo, but the section offsets in .debug_cu_index and .debug_tu_index are 32 bits; see the discussion here [1]. We implemented a workaround/extension for this in LLVM. Implement the equivalent in libdw. To test this, we need files with more than 4GB in .debug_info.dwo. I created these artificially by editing GCC's assembly output. They compress down to 6KB. I test them from run-large-elf-file.sh to take advantage of the existing checks for large file support. 1: https://discourse.llvm.org/t/dwarf-dwp-4gb-limit/63902. * libdw/dwarf_end.c (dwarf_package_index_free): New function. * tests/testfile-dwp-4-cu-index-overflow.bz2: New test file. * tests/testfile-dwp-4-cu-index-overflow.dwp.bz2: New test file. * tests/testfile-dwp-5-cu-index-overflow.bz2: New test file. * tests/testfile-dwp-5-cu-index-overflow.dwp.bz2: New test file. * tests/testfile-dwp-cu-index-overflow.source: New file. * tests/run-large-elf-file.sh: Check testfile-dwp-5-cu-index-overflow and testfile-dwp-4-cu-index-overflow. Signed-off-by: Omar Sandoval --- diff --git a/libdw/dwarf_cu_dwp_section_info.c b/libdw/dwarf_cu_dwp_section_info.c index 298f36f9..9fdc15bf 100644 --- a/libdw/dwarf_cu_dwp_section_info.c +++ b/libdw/dwarf_cu_dwp_section_info.c @@ -30,6 +30,8 @@ # include #endif +#include + #include "libdwP.h" static Dwarf_Package_Index * @@ -110,7 +112,9 @@ __libdw_read_package_index (Dwarf *dbg, bool tu) index->dbg = dbg; /* Set absent sections to UINT32_MAX. */ - memset (index->sections, 0xff, sizeof (index->sections)); + for (size_t i = 0; + i < sizeof (index->sections) / sizeof (index->sections[0]); i++) + index->sections[i] = UINT32_MAX; for (size_t i = 0; i < section_count; i++) { uint32_t section = read_4ubyte_unaligned (dbg, sections + i * 4); @@ -161,6 +165,7 @@ __libdw_read_package_index (Dwarf *dbg, bool tu) index->indices = indices; index->section_offsets = section_offsets; index->section_sizes = section_sizes; + index->debug_info_offsets = NULL; return index; } @@ -177,6 +182,138 @@ __libdw_package_index (Dwarf *dbg, bool tu) if (index == NULL) return NULL; + /* Offsets in the section offset table are 32-bit unsigned integers. In + practice, the .debug_info.dwo section for very large executables can be + larger than 4GB. GNU dwp as of binutils 2.41 and llvm-dwp before LLVM 15 + both accidentally truncate offsets larger than 4GB. + + LLVM 15 detects the overflow and errors out instead; see LLVM commit + f8df8114715b ("[DWP][DWARF] Detect and error on debug info offset + overflow"). However, lldb in LLVM 16 supports using dwp files with + truncated offsets by recovering them directly from the unit headers in the + .debug_info.dwo section; see LLVM commit c0db06227721 ("[DWARFLibrary] Add + support to re-construct cu-index"). Since LLVM 17, the overflow error can + be turned into a warning instead; see LLVM commit 53a483cee801 ("[DWP] add + overflow check for llvm-dwp tools if offset overflow"). + + LLVM's support for > 4GB offsets is effectively an extension to the DWARF + package file format, which we implement here. The strategy is to walk the + unit headers in .debug_info.dwo in lockstep with the DW_SECT_INFO columns + in the section offset tables. As long as they are in the same order + (which they are in practice for both GNU dwp and llvm-dwp), we can + correlate the truncated offset and produce a corrected array of offsets. + + Note that this will be fixed properly in DWARF 6: + https://dwarfstd.org/issues/220708.2.html. */ + if (index->sections[DW_SECT_INFO - 1] != UINT32_MAX + && dbg->sectiondata[IDX_debug_info]->d_size > UINT32_MAX) + { + Dwarf_Package_Index *cu_index, *tu_index = NULL; + if (tu) + { + tu_index = index; + assert (dbg->cu_index == NULL); + cu_index = __libdw_read_package_index (dbg, false); + if (cu_index == NULL) + { + free(index); + return NULL; + } + } + else + { + cu_index = index; + if (dbg->sectiondata[IDX_debug_tu_index] != NULL + && dbg->sectiondata[IDX_debug_types] == NULL) + { + assert (dbg->tu_index == NULL); + tu_index = __libdw_read_package_index (dbg, true); + if (tu_index == NULL) + { + free(index); + return NULL; + } + } + } + + cu_index->debug_info_offsets = malloc (cu_index->unit_count + * sizeof (Dwarf_Off)); + if (cu_index->debug_info_offsets == NULL) + { + free (tu_index); + free (cu_index); + __libdw_seterrno (DWARF_E_NOMEM); + return NULL; + } + if (tu_index != NULL) + { + tu_index->debug_info_offsets = malloc (tu_index->unit_count + * sizeof (Dwarf_Off)); + if (tu_index->debug_info_offsets == NULL) + { + free (tu_index); + free (cu_index->debug_info_offsets); + free (cu_index); + __libdw_seterrno (DWARF_E_NOMEM); + return NULL; + } + } + + Dwarf_Off off = 0; + uint32_t cui = 0, tui = 0; + uint32_t cu_count = cu_index->unit_count; + const unsigned char *cu_offset + = cu_index->section_offsets + cu_index->sections[DW_SECT_INFO - 1] * 4; + uint32_t tu_count = 0; + const unsigned char *tu_offset; + if (tu_index != NULL) + { + tu_count = tu_index->unit_count; + tu_offset = tu_index->section_offsets + + tu_index->sections[DW_SECT_INFO - 1] * 4; + } + while (cui < cu_count || tui < tu_count) + { + Dwarf_Off next_off; + uint8_t unit_type; + if (__libdw_next_unit (dbg, false, off, &next_off, NULL, NULL, + &unit_type, NULL, NULL, NULL, NULL, NULL) + != 0) + { + not_sorted: + free (cu_index->debug_info_offsets); + cu_index->debug_info_offsets = NULL; + if (tu_index != NULL) + { + free (tu_index->debug_info_offsets); + tu_index->debug_info_offsets = NULL; + } + break; + } + if (unit_type != DW_UT_split_type && cui < cu_count) + { + if ((off & UINT32_MAX) != read_4ubyte_unaligned (dbg, cu_offset)) + goto not_sorted; + cu_index->debug_info_offsets[cui++] = off; + cu_offset += cu_index->section_count * 4; + } + else if (unit_type == DW_UT_split_type && tu_index != NULL + && tui < tu_count) + { + if ((off & UINT32_MAX) != read_4ubyte_unaligned (dbg, tu_offset)) + goto not_sorted; + tu_index->debug_info_offsets[tui++] = off; + tu_offset += tu_index->section_count * 4; + } + off = next_off; + } + + if (tu) + dbg->cu_index = cu_index; + else if (tu_index != NULL) + dbg->tu_index = tu_index; + } + if (tu) dbg->tu_index = index; else @@ -244,8 +381,13 @@ __libdw_dwp_section_info (Dwarf_Package_Index *index, uint32_t unit_row, size_t i = (size_t)(unit_row - 1) * index->section_count + index->sections[section - 1]; if (offsetp != NULL) - *offsetp = read_4ubyte_unaligned (index->dbg, - index->section_offsets + i * 4); + { + if (section == DW_SECT_INFO && index->debug_info_offsets != NULL) + *offsetp = index->debug_info_offsets[unit_row - 1]; + else + *offsetp = read_4ubyte_unaligned (index->dbg, + index->section_offsets + i * 4); + } if (sizep != NULL) *sizep = read_4ubyte_unaligned (index->dbg, index->section_sizes + i * 4); diff --git a/libdw/dwarf_end.c b/libdw/dwarf_end.c index 78224ddb..ed8d27be 100644 --- a/libdw/dwarf_end.c +++ b/libdw/dwarf_end.c @@ -40,6 +40,17 @@ #include "cfi.h" +static void +dwarf_package_index_free (Dwarf_Package_Index *index) +{ + if (index != NULL) + { + free (index->debug_info_offsets); + free (index); + } +} + + static void noop_free (void *arg __attribute__ ((unused))) { @@ -79,8 +90,8 @@ dwarf_end (Dwarf *dwarf) { if (dwarf != NULL) { - free (dwarf->tu_index); - free (dwarf->cu_index); + dwarf_package_index_free (dwarf->tu_index); + dwarf_package_index_free (dwarf->cu_index); if (dwarf->cfi != NULL) /* Clean up the CFI cache. */ diff --git a/libdw/libdwP.h b/libdw/libdwP.h index c8041f15..8b2f06fa 100644 --- a/libdw/libdwP.h +++ b/libdw/libdwP.h @@ -374,6 +374,9 @@ typedef struct Dwarf_Package_Index_s const unsigned char *indices; const unsigned char *section_offsets; const unsigned char *section_sizes; + /* If DW_SECT_INFO section offsets were truncated to 32 bits, recovered + 64-bit offsets. */ + Dwarf_Off *debug_info_offsets; } Dwarf_Package_Index; /* CU representation. */ diff --git a/tests/Makefile.am b/tests/Makefile.am index 3f80c451..9141074f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -641,7 +641,12 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \ testfile-dwp-4.bz2 testfile-dwp-4.dwp.bz2 \ testfile-dwp-4-strict.bz2 testfile-dwp-4-strict.dwp.bz2 \ testfile-dwp-5.bz2 testfile-dwp-5.dwp.bz2 testfile-dwp.source \ - run-cu-dwp-section-info.sh run-declfiles.sh + run-cu-dwp-section-info.sh run-declfiles.sh \ + testfile-dwp-5-cu-index-overflow.bz2 \ + testfile-dwp-5-cu-index-overflow.dwp.bz2 \ + testfile-dwp-4-cu-index-overflow.bz2 \ + testfile-dwp-4-cu-index-overflow.dwp.bz2 \ + testfile-dwp-cu-index-overflow.source if USE_VALGRIND diff --git a/tests/run-large-elf-file.sh b/tests/run-large-elf-file.sh index 7116de53..8108cb4b 100755 --- a/tests/run-large-elf-file.sh +++ b/tests/run-large-elf-file.sh @@ -122,4 +122,178 @@ test_file testfile38 # 64bit, big endian, non-rel test_file testfile27 +# See testfile-dwp-cu-index-overflow.source +testfiles testfile-dwp-5-cu-index-overflow testfile-dwp-5-cu-index-overflow.dwp + +testrun_compare ${abs_builddir}/cu-dwp-section-info testfile-dwp-5-cu-index-overflow.dwp << EOF +file: testfile-dwp-5-cu-index-overflow.dwp +INFO: 0x0 0x8000004c +TYPES: 0x0 0x0 +ABBREV: 0x0 0x50 +LINE: 0x0 0x61 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x0 0x1c +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x8000004c 0x6f +TYPES: 0x0 0x0 +ABBREV: 0x50 0x15e +LINE: 0x61 0x63 +LOCLISTS: 0x0 0xd4 +STR_OFFSETS: 0x1c 0x24 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x22 + +INFO: 0x800000bb 0xff +TYPES: 0x0 0x0 +ABBREV: 0x50 0x15e +LINE: 0x61 0x63 +LOCLISTS: 0x0 0xd4 +STR_OFFSETS: 0x1c 0x24 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x22 + +INFO: 0x800001ba 0x8000004c +TYPES: 0x0 0x0 +ABBREV: 0x1ae 0x50 +LINE: 0xc4 0x61 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x40 0x1c +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x100000206 0x6c +TYPES: 0x0 0x0 +ABBREV: 0x1fe 0xc8 +LINE: 0x125 0x63 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x5c 0x20 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x100000272 0x6f +TYPES: 0x0 0x0 +ABBREV: 0x1fe 0xc8 +LINE: 0x125 0x63 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x5c 0x20 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x1000002e1 0x182 +TYPES: 0x0 0x0 +ABBREV: 0x2c6 0x188 +LINE: 0x188 0x65 +LOCLISTS: 0xd4 0xee +STR_OFFSETS: 0x7c 0x44 +MACRO: 0x0 0x0 +RNGLISTS: 0x22 0x43 + +EOF + +testrun_compare ${abs_builddir}/get-units-split testfile-dwp-5-cu-index-overflow << EOF +file: testfile-dwp-5-cu-index-overflow +Got cudie unit_type: 4 +Found a skeleton unit, with split die: filler1.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: foo.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: filler2.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: bar.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: main.cc + +EOF + +rm -f testfile-dwp-5-cu-index-overflow testfile-dwp-5-cu-index-overflow.dwp + +# See testfile-dwp-cu-index-overflow.source +testfiles testfile-dwp-4-cu-index-overflow testfile-dwp-4-cu-index-overflow.dwp + +testrun_compare ${abs_builddir}/cu-dwp-section-info testfile-dwp-4-cu-index-overflow.dwp << EOF +file: testfile-dwp-4-cu-index-overflow.dwp +INFO: 0x0 0x8000004b +TYPES: 0x0 0x0 +ABBREV: 0x0 0x58 +LINE: 0x0 0x2c +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x0 0x14 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x8000004b 0x116 +TYPES: 0x0 0x0 +ABBREV: 0x58 0x16f +LINE: 0x2c 0x34 +LOCLISTS: 0x0 0x110 +STR_OFFSETS: 0x14 0x1c +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x80000161 0x8000004b +TYPES: 0x0 0x0 +ABBREV: 0x1c7 0x58 +LINE: 0x60 0x2c +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x30 0x14 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x1000001ac 0x6e +TYPES: 0x0 0x0 +ABBREV: 0x21f 0xd4 +LINE: 0x8c 0x34 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x44 0x18 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x10000021a 0x1b5 +TYPES: 0x0 0x0 +ABBREV: 0x2f3 0x19b +LINE: 0xc0 0x35 +LOCLISTS: 0x110 0x12a +STR_OFFSETS: 0x5c 0x3c +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x0 0x0 +TYPES: 0x0 0x6e +ABBREV: 0x58 0x16f +LINE: 0x2c 0x34 +LOCLISTS: 0x0 0x110 +STR_OFFSETS: 0x14 0x1c +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +INFO: 0x0 0x0 +TYPES: 0x6e 0x6b +ABBREV: 0x21f 0xd4 +LINE: 0x8c 0x34 +LOCLISTS: 0x0 0x0 +STR_OFFSETS: 0x44 0x18 +MACRO: 0x0 0x0 +RNGLISTS: 0x0 0x0 + +EOF + +testrun_compare ${abs_builddir}/get-units-split testfile-dwp-4-cu-index-overflow << EOF +file: testfile-dwp-4-cu-index-overflow +Got cudie unit_type: 4 +Found a skeleton unit, with split die: filler1.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: foo.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: filler2.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: bar.cc +Got cudie unit_type: 4 +Found a skeleton unit, with split die: main.cc + +EOF + +rm -f testfile-dwp-4-cu-index-overflow testfile-dwp-4-cu-index-overflow.dwp + exit 0 diff --git a/tests/testfile-dwp-4-cu-index-overflow.bz2 b/tests/testfile-dwp-4-cu-index-overflow.bz2 new file mode 100755 index 00000000..2aaa4f40 Binary files /dev/null and b/tests/testfile-dwp-4-cu-index-overflow.bz2 differ diff --git a/tests/testfile-dwp-4-cu-index-overflow.dwp.bz2 b/tests/testfile-dwp-4-cu-index-overflow.dwp.bz2 new file mode 100644 index 00000000..0718c1b2 Binary files /dev/null and b/tests/testfile-dwp-4-cu-index-overflow.dwp.bz2 differ diff --git a/tests/testfile-dwp-5-cu-index-overflow.bz2 b/tests/testfile-dwp-5-cu-index-overflow.bz2 new file mode 100755 index 00000000..07185fe5 Binary files /dev/null and b/tests/testfile-dwp-5-cu-index-overflow.bz2 differ diff --git a/tests/testfile-dwp-5-cu-index-overflow.dwp.bz2 b/tests/testfile-dwp-5-cu-index-overflow.dwp.bz2 new file mode 100644 index 00000000..dce34e3f Binary files /dev/null and b/tests/testfile-dwp-5-cu-index-overflow.dwp.bz2 differ diff --git a/tests/testfile-dwp-cu-index-overflow.source b/tests/testfile-dwp-cu-index-overflow.source new file mode 100644 index 00000000..2de15441 --- /dev/null +++ b/tests/testfile-dwp-cu-index-overflow.source @@ -0,0 +1,86 @@ +# Dummy program that we patch to generate a dwp file with more than 4GB of +# .debug_info. + +# Generate 2 dummy files that result in DWARF blocks. +$ for (( i = 1; i <= 2; i++ )); do echo 'constexpr int filler'$i'[] = { 1 };' > filler$i.cc; done +$ g++ -O2 -g -gsplit-dwarf -fdebug-types-section -dA -S filler{1,2}.cc foo.cc bar.cc main.cc +# Patch the DWARF blocks to be 2GB. +$ for (( i = 1; i <= 2; i++ )); do patch -p1 << EOF +--- a/filler$i.s ++++ b/filler$i.s +@@ -7,5 +7,5 @@ + .section .debug_info.dwo,"e",@progbits + .Ldebug_info0: +- .long 0x49 # Length of Compilation Unit Info ++ .long 0x80000048 # Length of Compilation Unit Info + .value 0x5 # DWARF version number + .byte 0x5 # DW_UT_split_compile +@@ -51,9 +51,6 @@ + .long 0x29 # DW_AT_type + # DW_AT_const_expr +- .byte 0x4 # DW_AT_const_value +- .byte 0x1 # fp or vector constant word 0 +- .byte 0 # fp or vector constant word 1 +- .byte 0 # fp or vector constant word 2 +- .byte 0 # fp or vector constant word 3 ++ .long 0x80000000 # DW_AT_const_value ++ .fill 0x80000000 + .byte 0 # end of children of DIE 0x14 + .section .debug_info,"",@progbits +@@ -171,5 +168,5 @@ + .uleb128 0x19 # (DW_FORM_flag_present) + .uleb128 0x1c # (DW_AT_const_value) +- .uleb128 0xa # (DW_FORM_block1) ++ .uleb128 0x4 # (DW_FORM_block4) + .byte 0 + .byte 0 +EOF +done +$ for (( i = 1; i <= 2; i++ )); do as filler$i.s -o filler$i.o; done +$ as foo.s -o foo.o +$ as bar.s -o bar.o +$ as main.s -o main.o +$ g++ filler1.o foo.o filler2.o bar.o main.o -o testfile-dwp-5-cu-index-overflow +# -continue-on-cu-index-overflow was added in LLVM 17: +# https://reviews.llvm.org/D144565. +$ llvm-dwp -continue-on-cu-index-overflow filler1.o foo.o filler2.o bar.o main.o -o testfile-dwp-5-cu-index-overflow.dwp + +# Same thing for DWARF 4. +$ g++ -O2 -g -gdwarf-4 -gsplit-dwarf -fdebug-types-section -dA -S filler{1,2}.cc foo.cc bar.cc main.cc +$ for (( i = 1; i <= 2; i++ )); do patch -p1 << EOF +--- a/filler$i.s ++++ b/filler$i.s +@@ -6,5 +6,5 @@ + .section .debug_info.dwo,"e",@progbits + .Ldebug_info0: +- .long 0x48 # Length of Compilation Unit Info ++ .long 0x80000047 # Length of Compilation Unit Info + .value 0x4 # DWARF version number + .long .Ldebug_abbrev0 # Offset Into Abbrev. Section +@@ -49,9 +49,6 @@ + .long 0x28 # DW_AT_type + # DW_AT_const_expr +- .byte 0x4 # DW_AT_const_value +- .byte 0x1 # fp or vector constant word 0 +- .byte 0 # fp or vector constant word 1 +- .byte 0 # fp or vector constant word 2 +- .byte 0 # fp or vector constant word 3 ++ .long 0x80000000 # DW_AT_const_value ++ .fill 0x80000000 + .byte 0 # end of children of DIE 0xb + .section .debug_info,"",@progbits +@@ -172,5 +169,5 @@ + .uleb128 0x19 # (DW_FORM_flag_present) + .uleb128 0x1c # (DW_AT_const_value) +- .uleb128 0xa # (DW_FORM_block1) ++ .uleb128 0x4 # (DW_FORM_block4) + .byte 0 + .byte 0 +EOF +done +$ for (( i = 1; i <= 2; i++ )); do as filler$i.s -o filler$i.o; done +$ as foo.s -o foo.o +$ as bar.s -o bar.o +$ as main.s -o main.o +$ g++ filler1.o foo.o filler2.o bar.o main.o -o testfile-dwp-4-cu-index-overflow +$ llvm-dwp -continue-on-cu-index-overflow filler1.o foo.o filler2.o bar.o main.o -o testfile-dwp-4-cu-index-overflow.dwp