X-Git-Url: http://git.ipfire.org/?a=blobdiff_plain;f=utf8.c;h=5b39361ada0bbab3dc4df90e64cc6173bb465ff4;hb=9892dc83f6f943d1ee5ba39038c4c3e9c5d0f239;hp=d55e20c6415cb53ef66e1edd0a5e7dc470e18052;hpb=ba6f0905fdb9e65c1ac5fbc79c9a4ef0b59b3e68;p=thirdparty%2Fgit.git

diff --git a/utf8.c b/utf8.c
index d55e20c641..5b39361ada 100644
--- a/utf8.c
+++ b/utf8.c
@@ -4,6 +4,11 @@
 
 /* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */
 
+static const char utf16_be_bom[] = {'\xFE', '\xFF'};
+static const char utf16_le_bom[] = {'\xFF', '\xFE'};
+static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'};
+static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'};
+
 struct interval {
 	ucs_char_t first;
 	ucs_char_t last;
@@ -90,13 +95,11 @@ static int git_wcwidth(ucs_char_t ch)
 		return -1;
 
 	/* binary search in table of non-spacing characters */
-	if (bisearch(ch, zero_width, sizeof(zero_width)
-				/ sizeof(struct interval) - 1))
+	if (bisearch(ch, zero_width, ARRAY_SIZE(zero_width) - 1))
 		return 0;
 
 	/* binary search in table of double width characters */
-	if (bisearch(ch, double_width, sizeof(double_width)
-				/ sizeof(struct interval) - 1))
+	if (bisearch(ch, double_width, ARRAY_SIZE(double_width) - 1))
 		return 2;
 
 	return 1;
@@ -408,11 +411,10 @@ out:
  */
 static int same_utf_encoding(const char *src, const char *dst)
 {
-	if (istarts_with(src, "utf") && istarts_with(dst, "utf")) {
-		/* src[3] or dst[3] might be '\0' */
-		int i = (src[3] == '-' ? 4 : 3);
-		int j = (dst[3] == '-' ? 4 : 3);
-		return !strcasecmp(src+i, dst+j);
+	if (skip_iprefix(src, "utf", &src) && skip_iprefix(dst, "utf", &dst)) {
+		skip_prefix(src, "-", &src);
+		skip_prefix(dst, "-", &dst);
+		return !strcasecmp(src, dst);
 	}
 	return 0;
 }
@@ -470,16 +472,17 @@ int utf8_fprintf(FILE *stream, const char *format, ...)
 #else
 	typedef char * iconv_ibp;
 #endif
-char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, int *outsz_p)
+char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv,
+			    size_t bom_len, size_t *outsz_p)
 {
 	size_t outsz, outalloc;
 	char *out, *outpos;
 	iconv_ibp cp;
 
 	outsz = insz;
-	outalloc = outsz + 1; /* for terminating NUL */
+	outalloc = st_add(outsz, 1 + bom_len); /* for terminating NUL */
 	out = xmalloc(outalloc);
-	outpos = out;
+	outpos = out + bom_len;
 	cp = (iconv_ibp)in;
 
 	while (1) {
@@ -497,7 +500,7 @@ char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, int *outs
 			 * converting the rest.
 			 */
 			sofar = outpos - out;
-			outalloc = sofar + insz * 2 + 32;
+			outalloc = st_add3(sofar, st_mult(insz, 2), 32);
 			out = xrealloc(out, outalloc);
 			outpos = out + sofar;
 			outsz = outalloc - sofar - 1;
@@ -534,16 +537,50 @@ static const char *fallback_encoding(const char *name)
 	return name;
 }
 
-char *reencode_string_len(const char *in, int insz,
+char *reencode_string_len(const char *in, size_t insz,
 			  const char *out_encoding, const char *in_encoding,
-			  int *outsz)
+			  size_t *outsz)
 {
 	iconv_t conv;
 	char *out;
+	const char *bom_str = NULL;
+	size_t bom_len = 0;
 
 	if (!in_encoding)
 		return NULL;
 
+	/* UTF-16LE-BOM is the same as UTF-16 for reading */
+	if (same_utf_encoding("UTF-16LE-BOM", in_encoding))
+		in_encoding = "UTF-16";
+
+	/*
+	 * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
+	 * Some users under Windows want the little endian version
+	 *
+	 * We handle UTF-16 and UTF-32 ourselves only if the platform does not
+	 * provide a BOM (which we require), since we want to match the behavior
+	 * of the system tools and libc as much as possible.
+	 */
+	if (same_utf_encoding("UTF-16LE-BOM", out_encoding)) {
+		bom_str = utf16_le_bom;
+		bom_len = sizeof(utf16_le_bom);
+		out_encoding = "UTF-16LE";
+	} else if (same_utf_encoding("UTF-16BE-BOM", out_encoding)) {
+		bom_str = utf16_be_bom;
+		bom_len = sizeof(utf16_be_bom);
+		out_encoding = "UTF-16BE";
+#ifdef ICONV_OMITS_BOM
+	} else if (same_utf_encoding("UTF-16", out_encoding)) {
+		bom_str = utf16_be_bom;
+		bom_len = sizeof(utf16_be_bom);
+		out_encoding = "UTF-16BE";
+	} else if (same_utf_encoding("UTF-32", out_encoding)) {
+		bom_str = utf32_be_bom;
+		bom_len = sizeof(utf32_be_bom);
+		out_encoding = "UTF-32BE";
+#endif
+	}
+
 	conv = iconv_open(out_encoding, in_encoding);
 	if (conv == (iconv_t) -1) {
 		in_encoding = fallback_encoding(in_encoding);
@@ -553,9 +590,10 @@ char *reencode_string_len(const char *in, int insz,
 		if (conv == (iconv_t) -1)
 			return NULL;
 	}
-
-	out = reencode_string_iconv(in, insz, conv, outsz);
+	out = reencode_string_iconv(in, insz, conv, bom_len, outsz);
 	iconv_close(conv);
+	if (out && bom_str && bom_len)
+		memcpy(out, bom_str, bom_len);
 	return out;
 }
 #endif
@@ -566,11 +604,6 @@ static int has_bom_prefix(const char *data, size_t len,
 	return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len);
 }
 
-static const char utf16_be_bom[] = {0xFE, 0xFF};
-static const char utf16_le_bom[] = {0xFF, 0xFE};
-static const char utf32_be_bom[] = {0x00, 0x00, 0xFE, 0xFF};
-static const char utf32_le_bom[] = {0xFF, 0xFE, 0x00, 0x00};
-
 int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
 {
 	return (