fix: Support UTF-16LE .rsp files (#1005)

diff --git a/src/Args.cpp b/src/Args.cpp
index 3731e26b66f29e91994c3c60df068b66c6e6b6dd..572911ecea26c28a65a6afce60720cd01776b7f8 100644 (file)
--- a/src/Args.cpp
+++ b/src/Args.cpp
@@ -54,7 +54,7 @@ Args::from_atfile(const std::string& filename, bool ignore_backslash)
 {
   std::string argtext;
   try {
-    argtext = Util::read_file(filename);
+    argtext = Util::read_text_file(filename);
   } catch (core::Error&) {
     return nullopt;
   }

diff --git a/src/Util.cpp b/src/Util.cpp
index f03ba6fbea50b67c71cf1e0a156c908ea64aec74..d9c5fdbf2fcc5363c92320eef3ea83e6ad7965e7 100644 (file)
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -48,7 +48,9 @@ extern "C" {
 
 #include <algorithm>
 #include <climits>
+#include <codecvt>
 #include <fstream>
+#include <locale>
 
 #ifndef HAVE_DIRENT_H
 #  include <filesystem>
@@ -207,6 +209,14 @@ rewrite_stderr_to_absolute_paths(string_view text)
   return result;
 }
 
+bool
+has_utf16_le_bom(string_view text)
+{
+  return text.size() > 1
+         && ((static_cast<uint8_t>(text[0]) == 0xff
+              && static_cast<uint8_t>(text[1]) == 0xfe));
+}
+
 } // namespace
 
 namespace Util {
@@ -1078,6 +1088,21 @@ read_file(const std::string& path, size_t size_hint)
   return result;
 }
 
+std::string
+read_text_file(const std::string& path, size_t size_hint)
+{
+  std::string result = read_file(path, size_hint);
+  // Convert to UTF-8 if the contents start with UTF-16 little-endian BOM
+  if (has_utf16_le_bom(result)) {
+    result.erase(0, 2); // Remove BOM
+    std::u16string result_as_u16((result.size() / 2) + 1, '\0');
+    result_as_u16 = reinterpret_cast<const char16_t*>(result.c_str());
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+    result = converter.to_bytes(result_as_u16);
+  }
+  return result;
+}
+
 #ifndef _WIN32
 std::string
 read_link(const std::string& path)

diff --git a/src/Util.hpp b/src/Util.hpp
index 3647f3f30801796bec727d1cba10208767b3a370..de191e7cb0d9b65a64fbf8566c414971a5b28b43 100644 (file)
--- a/src/Util.hpp
+++ b/src/Util.hpp
@@ -293,6 +293,12 @@ bool read_fd(int fd, DataReceiver data_receiver);
 // without the path.
 std::string read_file(const std::string& path, size_t size_hint = 0);
 
+// Return contents of a text file as UTF-8 encoded string.
+//
+// Throws `core::Error` on error. The description contains the error message
+// without the path.
+std::string read_text_file(const std::string& path, size_t size_hint = 0);
+
 #ifndef _WIN32
 // Like readlink(2) but returns the string (or the empty string on failure).
 std::string read_link(const std::string& path);

diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp
index f2398c4fb83534bfa93e03cdb696c6243e821e4f..3d59b71fe742ad6789c51b7396a19d88bfb622f9 100644 (file)
--- a/unittest/test_Util.cpp
+++ b/unittest/test_Util.cpp
@@ -667,6 +667,24 @@ TEST_CASE("Util::{read,write,copy}_file with binary files")
   CHECK(Util::read_file("copy") == data);
 }
 
+TEST_CASE("Util::read_text_file with UTF-16 little endian encoding")
+{
+  TestContext test_context;
+
+  std::string data;
+  data.push_back(static_cast<unsigned char>(0xff));
+  data.push_back(static_cast<unsigned char>(0xfe));
+  data.push_back('a');
+  data.push_back('\0');
+  data.push_back('b');
+  data.push_back('\0');
+  data.push_back('c');
+  data.push_back('\0');
+
+  Util::write_file("test", data);
+  CHECK(Util::read_text_file("test") == "abc");
+}
+
 TEST_CASE("Util::remove_extension")
 {
   CHECK(Util::remove_extension("") == "");