fix: Work around endianness problem in Util::read_text_file

The code in Util::read_text_file for converting UTF-16LE to UTF-8 only
works on little-endian machines. This makes the unit test fail on
big-endian machines.

Since the conversion is only needed on Windows (for Visual Studio, which
creates UTF-16LE .rsp files) in practice, work around the problem by
only doing the conversion in Windows builds.

Fixes #1014.

diff --git a/src/Util.cpp b/src/Util.cpp
index 146ae7a9aee26765426eb69512d7cbbd89015b5b..0a4d4304113031913f998e7310a80acddd790319 100644 (file)
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -209,6 +209,7 @@ rewrite_stderr_to_absolute_paths(string_view text)
   return result;
 }
 
+#ifdef _WIN32
 bool
 has_utf16_le_bom(string_view text)
 {
@@ -216,6 +217,7 @@ has_utf16_le_bom(string_view text)
          && ((static_cast<uint8_t>(text[0]) == 0xff
               && static_cast<uint8_t>(text[1]) == 0xfe));
 }
+#endif
 
 } // namespace
 
@@ -1190,7 +1192,12 @@ std::string
 read_text_file(const std::string& path, size_t size_hint)
 {
   std::string result = read_file(path, size_hint);
+#ifdef _WIN32
   // Convert to UTF-8 if the content starts with a UTF-16 little-endian BOM.
+  //
+  // Note that this code assumes a little-endian machine, which is why it's
+  // #ifdef-ed to only run on Windows (which is always little-endian) where it's
+  // actually needed.
   if (has_utf16_le_bom(result)) {
     result.erase(0, 2); // Remove BOM.
     std::u16string result_as_u16((result.size() / 2) + 1, '\0');
@@ -1198,6 +1205,7 @@ read_text_file(const std::string& path, size_t size_hint)
     std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
     result = converter.to_bytes(result_as_u16);
   }
+#endif
   return result;
 }
 

diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp
index 913b736246dbab3cd378b49145c8f7fd5288bc22..4ea9903e52a720ed68bfd9504bbc3d351a31ff09 100644 (file)
--- a/unittest/test_Util.cpp
+++ b/unittest/test_Util.cpp
@@ -689,6 +689,7 @@ TEST_CASE("Util::{read,write,copy}_file with binary files")
   CHECK(Util::read_file("copy") == data);
 }
 
+#ifdef _WIN32
 TEST_CASE("Util::read_text_file with UTF-16 little endian encoding")
 {
   TestContext test_context;
@@ -706,6 +707,7 @@ TEST_CASE("Util::read_text_file with UTF-16 little endian encoding")
   Util::write_file("test", data);
   CHECK(Util::read_text_file("test") == "abc");
 }
+#endif
 
 TEST_CASE("Util::remove_extension")
 {