* MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4) /* Incorrect UTF-8 made by libarchive
* 2.x in the wrong assumption. */
-#define SCONV_COPY_UTF8_TO_UTF8 (1<<5) /* Copy UTF-8 characters in checking
- * CESU-8. */
#define SCONV_NORMALIZATION_C (1<<6) /* Need normalization to be Form C.
* Before UTF-8 characters are actually
* processed. */
#define SCONV_FROM_UTF8 (1<<9) /* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE (1<<10) /* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE (1<<11) /* "from charset" side is UTF-16BE. */
+#define SCONV_TO_UTF16LE (1<<12) /* "to charset" side is UTF-16LE. */
#if HAVE_ICONV
iconv_t cd;
#endif
/* A temporary buffer for a conversion of UTF-8 NFD. */
struct archive_string utf8;
+ struct archive_string utf16;
#if defined(__APPLE__)
UnicodeToTextInfo uniInfo;
struct archive_string utf16nfc;
static unsigned make_codepage_from_charset(const char *);
static unsigned get_current_codepage(void);
static unsigned get_current_oemcp(void);
-static size_t la_strnlen(const void *, size_t);
+static size_t mbsnbytes(const void *, size_t);
static size_t utf16nbytes(const void *, size_t);
#if defined(_WIN32) && !defined(__CYGWIN__)
static int archive_wstring_append_from_mbs_in_codepage(
#endif
static int best_effort_strncat_in_locale(struct archive_string *, const void *,
size_t, struct archive_string_conv *);
+static int _utf8_to_unicode(uint32_t *, const char *, size_t);
+static int utf8_to_unicode(uint32_t *, const char *, size_t);
+static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t);
+static int cesu8_to_unicode(uint32_t *, const char *, size_t);
+static size_t unicode_to_utf8(char *, size_t, uint32_t);
+static int utf16_to_unicode(uint32_t *, const char *, size_t, int);
+static size_t unicode_to_utf16be(char *, size_t, uint32_t);
+static size_t unicode_to_utf16le(char *, size_t, uint32_t);
static int strncat_from_utf8_libarchive2(struct archive_string *,
const char *, size_t);
static int strncat_from_utf8_to_utf8(struct archive_string *, const char *,
size_t);
static int archive_string_normalize_C(struct archive_string *, const char *,
- size_t);
+ size_t, struct archive_string_conv *);
#if defined(__APPLE__)
static int archive_string_normalize_D(struct archive_string *, const char *,
size_t, struct archive_string_conv *);
*ws++ = (wchar_t)*mp++;
count++;
}
+ } else if (sc != NULL && (sc->flag & SCONV_NORMALIZATION_C)) {
+ /*
+ * Normalize UTF-8 and UTF-16BE and convert it directly
+ * to UTF-16 as wchar_t.
+ */
+ struct archive_string u16;
+ int saved_flag = sc->flag;/* save current flag. */
+
+ if (is_big_endian())
+ sc->flag |= SCONV_TO_UTF16BE;
+ else
+ sc->flag |= SCONV_TO_UTF16LE;
+
+ if (sc->flag & SCONV_FROM_UTF16BE) {
+ /*
+ * UTF-16BE NFD ===> UTF-16 NFC
+ */
+ count = utf16nbytes(s, length);
+ } else {
+ /*
+ * UTF-8 NFD ===> UTF-16 NFC
+ */
+ count = mbsnbytes(s, length);
+ }
+ u16.s = (char *)dest->s;
+ u16.length = dest->length << 1;;
+ u16.buffer_length = dest->buffer_length;
+ ret = archive_string_normalize_C(&u16, s, count, sc);
+ dest->s = (wchar_t *)u16.s;
+ dest->length = u16.length >> 1;
+ dest->buffer_length = u16.buffer_length;
+ sc->flag = saved_flag;/* restore the saved flag. */
+ return (ret);
} else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16BE)) {
count = utf16nbytes(s, length);
count >>= 1; /* to be WCS length */
} else {
DWORD mbflag;
- /*
- * Need normalization to UTF-8 since
- * MultiByteToWideChar() API does not support.
- * NOTE: When from_cp is CP_UTF8, neither MB_PRECOMPOSED nor
- * MB_COMPOSITE can be used; MultiByteToWideChar fail.
- */
- if (sc != NULL && (sc->flag & SCONV_NORMALIZATION_C)) {
- archive_string_empty(&(sc->utf8));
- if (archive_string_normalize_C(&(sc->utf8), s,
- length) != 0)
- ret = -1;
- s = sc->utf8.s;
- length = sc->utf8.length;
- }
- if (sc == NULL || (sc->flag & SCONV_FROM_CHARSET))
+ if (sc == NULL)
mbflag = 0;
- else
+ else if (sc->flag & SCONV_FROM_CHARSET) {
+ /* Do not trust the length which comes from
+ * an archive file. */
+ length = mbsnbytes(s, length);
+ mbflag = 0;
+ } else
mbflag = MB_PRECOMPOSED;
/*
return (NULL);
}
archive_string_init(&sc->utf8);
+ archive_string_init(&sc->utf16);
#if defined(__APPLE__)
archive_string_init(&sc->utf16nfc);
archive_string_init(&sc->utf16nfd);
* sight of two filenames, one is NFC and other is NFD,
* would be in its directory.
*/
- if ((flag & (SCONV_FROM_CHARSET | SCONV_FROM_UTF8)) ==
- (SCONV_FROM_CHARSET | SCONV_FROM_UTF8)){
+ if ((flag & SCONV_FROM_CHARSET) &&
+ (flag & (SCONV_FROM_UTF16BE | SCONV_FROM_UTF8))) {
#if defined(__APPLE__)
flag |= SCONV_NORMALIZATION_D;
#else
#endif
}
- /*
- * Copy UTF-8 string in checking CESU-8 including surrogate pair.
- */
- if ((flag & (SCONV_TO_UTF8 | SCONV_FROM_UTF8)) ==
- (SCONV_TO_UTF8 | SCONV_FROM_UTF8))
- flag |= SCONV_COPY_UTF8_TO_UTF8;
+#if defined(__APPLE__)
+ if (flag & SCONV_NORMALIZATION_D) {
+ if (createUniInfo(sc) != 0)
+ flag &= ~SCONV_NORMALIZATION_D;
+ }
+#endif
#if defined(HAVE_ICONV)
/*
* Create an iconv object.
*/
-#if defined(__APPLE__)
- if (flag & SCONV_COPY_UTF8_TO_UTF8) {
+ if ((flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16BE)) &&
+ (flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16BE))) {
/* This case does not use iconv. */
sc->cd = (iconv_t)-1;
- if (flag & SCONV_NORMALIZATION_D) {
- if (createUniInfo(sc) != 0)
- flag &= ~SCONV_NORMALIZATION_D;
- }
- } else if (flag & SCONV_FROM_UTF8) {
- sc->cd = iconv_open(tc, "UTF-8-MAC");
- if (sc->cd == (iconv_t)-1) {
- sc->cd = iconv_open(tc, fc);
- flag |= SCONV_NORMALIZATION_C;
- }
- } else if (flag & SCONV_TO_UTF8) {
- sc->cd = iconv_open("UTF-8-MAC", fc);
- if (sc->cd == (iconv_t)-1)
- sc->cd = iconv_open(tc, fc);
- }
-#else
- if (flag & SCONV_COPY_UTF8_TO_UTF8)
- /* This case does not use iconv. */
- sc->cd = (iconv_t)-1;
-#endif
- else {
+ } else {
sc->cd = iconv_open(tc, fc);
}
#endif /* HAVE_ICONV */
free(sc->from_charset);
free(sc->to_charset);
archive_string_free(&sc->utf8);
+ archive_string_free(&sc->utf16);
#if HAVE_ICONV
if (sc->cd != (iconv_t)-1)
iconv_close(sc->cd);
add_sconv_object(a, sc);
return (sc);
}
+ /* This case we can use our conversion routine. */
+ if ((sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16BE)) &&
+ (sc->flag & (SCONV_FROM_UTF8 | SCONV_FROM_UTF16BE))) {
+ if (a != NULL)
+ add_sconv_object(a, sc);
+ return (sc);
+ }
#if HAVE_ICONV
- if (sc->cd == (iconv_t)-1 &&
- (sc->flag & (SCONV_BEST_EFFORT | SCONV_COPY_UTF8_TO_UTF8)) == 0) {
+ if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT) == 0) {
free_sconv_object(sc);
if (a != NULL)
archive_set_error(a, ARCHIVE_ERRNO_MISC,
*/
static size_t
-la_strnlen(const void *_p, size_t n)
+mbsnbytes(const void *_p, size_t n)
{
size_t s;
const char *p, *pp;
struct archive_string_conv *sc)
{
size_t length;
- int r;
+ int r, r2 = 0;
/*
* If sc is NULL, we just make a copy without conversion.
*/
if (sc == NULL) {
- length = la_strnlen(_p, n);
+ length = mbsnbytes(_p, n);
/*
* archive_string_append() will call archive_string_ensure()
* but we cannot know if that call is failed or not. so
* libarchive2.x.
*/
if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) {
- length = la_strnlen(_p, n);
+ length = mbsnbytes(_p, n);
return (strncat_from_utf8_libarchive2(as, _p, length));
}
/*
- * Converting string from/to UTF-16.
+ * Convert a string to UTF-16BE.
*/
if (sc->flag & SCONV_TO_UTF16BE) {
- length = la_strnlen(_p, n);
+ length = mbsnbytes(_p, n);
/*
* If the current locale is UTF-8, we can translate
* a UTF-8 string into a UTF-16BE string.
r = strncat_to_utf16be(as, _p, length, sc);
return (r);
}
+
+ /*
+ * Convert a string from UTF-16BE.
+ */
if (sc->flag & SCONV_FROM_UTF16BE) {
length = utf16nbytes(_p, n);
- /*
- * If the current locale is UTF-8, we can translate
- * a UTF-16BE string into a UTF-8 string.
- */
- if (sc->flag & SCONV_TO_UTF8)
- r = strncat_from_utf16_to_utf8(as, _p, length, 1);
- else
+ if (sc->flag & SCONV_TO_UTF8) {
+ /*
+ * If the current locale is UTF-8, we can translate
+ * a UTF-16BE string into a UTF-8 string directly.
+ */
+#if defined(__APPLE__)
+ if (sc->flag & SCONV_NORMALIZATION_D)
+ /* Additionally it nees normalization. */
+ r = archive_string_normalize_D(as, _p,
+ length, sc);
+ else
+#endif
+ if (sc->flag & SCONV_NORMALIZATION_C)
+ /* Additionally it nees normalization. */
+ r = archive_string_normalize_C(as, _p,
+ length, sc);
+ else
+ r = strncat_from_utf16_to_utf8(as, _p,
+ length, 1);
+ } else {
+ /*
+ * At least we should normalize a UTF-16BE string.
+ */
+#if defined(__APPLE__)
+ if (sc->flag & SCONV_NORMALIZATION_D) {
+ archive_string_empty(&(sc->utf16));
+ r = archive_string_normalize_D(
+ &(sc->utf16), _p, length, sc);
+ if (r != 0 && errno == ENOMEM)
+ return (r);
+ _p = sc->utf16.s;
+ length = sc->utf16.length;
+ r = r2;
+ } else
+#endif
+ if (sc->flag & SCONV_NORMALIZATION_C) {
+ archive_string_empty(&(sc->utf16));
+ r = archive_string_normalize_C(
+ &(sc->utf16), _p, length, sc);
+ if (r != 0 && errno == ENOMEM)
+ return (r);
+ _p = sc->utf16.s;
+ length = sc->utf16.length;
+ r = r2;
+ }
r = strncat_from_utf16be(as, _p, length, sc);
+ if (r > r2)
+ r = r2;
+ }
return (r);
}
- length = la_strnlen(_p, n);
+ length = mbsnbytes(_p, n);
- /*
- * Copy UTF-8 string with a check of CESU-8.
- * Apparently, iconv does not check surrogate pairs in UTF-8
- * when both from-charset and to-charset are UTF-8, and then
- * we use our UTF-8 copy code.
- */
- if (sc->flag & SCONV_COPY_UTF8_TO_UTF8) {
+ if (sc->flag & SCONV_FROM_UTF8) {
+ /*
+ * Copy UTF-8 string with a check of CESU-8.
+ * Apparently, iconv does not check surrogate pairs in UTF-8
+ * when both from-charset and to-charset are UTF-8, and then
+ * we use our UTF-8 copy code.
+ */
+ if (sc->flag & SCONV_TO_UTF8) {
#if defined(__APPLE__)
- if (sc->flag & SCONV_NORMALIZATION_D)
- /* Additionally it nees normalization. */
- r = archive_string_normalize_D(as, _p, length, sc);
- else
+ if (sc->flag & SCONV_NORMALIZATION_D)
+ /* Additionally it nees normalization. */
+ r = archive_string_normalize_D(
+ as, _p, length, sc);
+ else
#endif
- if (sc->flag & SCONV_NORMALIZATION_C)
- /* Additionally it nees normalization. */
- r = archive_string_normalize_C(as, _p, length);
- else
- r = strncat_from_utf8_to_utf8(as, _p, length);
- return (r);
+ if (sc->flag & SCONV_NORMALIZATION_C)
+ /* Additionally it nees normalization. */
+ r = archive_string_normalize_C(
+ as, _p, length, sc);
+ else
+ r = strncat_from_utf8_to_utf8(as, _p, length);
+ return (r);
+ } else if ((sc->flag & SCONV_WIN_CP) == 0) {
+ /*
+ * At least we should normalize a UTF-8 string.
+ * because iconv cannot correctly translate UTF-8
+ * NFD characters to other charset.
+ */
+#if defined(__APPLE__)
+ if (sc->flag & SCONV_NORMALIZATION_D) {
+ archive_string_empty(&(sc->utf8));
+ r = archive_string_normalize_D(
+ &(sc->utf8), _p, length, sc);
+ if (r != 0 && errno == ENOMEM)
+ return (r);
+ _p = sc->utf8.s;
+ length = sc->utf8.length;
+ r2 = r;
+ } else
+#endif
+ if (sc->flag & SCONV_NORMALIZATION_C) {
+ archive_string_empty(&(sc->utf8));
+ r = archive_string_normalize_C(
+ &(sc->utf8), _p, length, sc);
+ if (r != 0 && errno == ENOMEM)
+ return (r);
+ _p = sc->utf8.s;
+ length = sc->utf8.length;
+ r2 = r;
+ }
+ }
}
-#if HAVE_ICONV
- return (iconv_strncat_in_locale(as, _p, length, sc));
-#else /* HAVE_ICONV */
#if defined(_WIN32) && !defined(__CYGWIN__)
/*
- * On Windows we can use Windows API for string conversion.
+ * On Windows we can use Windows API for a string conversion.
*/
if (sc->flag & SCONV_WIN_CP)
return (strncat_in_codepage(as, _p, length, sc));
#endif
- return (best_effort_strncat_in_locale(as, _p, length, sc));
+
+#if HAVE_ICONV
+ r = iconv_strncat_in_locale(as, _p, length, sc);
+#else /* HAVE_ICONV */
+ r = best_effort_strncat_in_locale(as, _p, length, sc);
#endif /* HAVE_ICONV */
+ if (r > r2)
+ r = r2;
+ return (r);
}
#if HAVE_ICONV
if (cd == (iconv_t)-1)
return (best_effort_strncat_in_locale(as, _p, length, sc));
- /*
- * Need normalization to UTF-8 because iconv cannot correctly
- * translate UTF-8 NFD characters to other charset.
- */
- if (sc->flag & SCONV_NORMALIZATION_C) {
- archive_string_empty(&(sc->utf8));
- if (archive_string_normalize_C(&(sc->utf8), _p, length) != 0)
- return_value = -1; /* failure */
- src = sc->utf8.s;
- length = sc->utf8.length;
- }
-
if (archive_string_ensure(as, as->length + length*2+1) == NULL)
return (-1);
* NOTE:This function does not check if the Unicode is leagal or not.
* Please you definitely check it before calling this.
*/
-static int
-unicode_to_utf8(char *p, uint32_t uc)
+static size_t
+unicode_to_utf8(char *p, size_t remaining, uint32_t uc)
{
char *_p = p;
/* Translate code point to UTF8 */
if (uc <= 0x7f) {
+ if (remaining == 0)
+ return (0);
*p++ = (char)uc;
} else if (uc <= 0x7ff) {
+ if (remaining < 2)
+ return (0);
*p++ = 0xc0 | ((uc >> 6) & 0x1f);
*p++ = 0x80 | (uc & 0x3f);
} else if (uc <= 0xffff) {
+ if (remaining < 3)
+ return (0);
*p++ = 0xe0 | ((uc >> 12) & 0x0f);
*p++ = 0x80 | ((uc >> 6) & 0x3f);
*p++ = 0x80 | (uc & 0x3f);
} else if (uc <= UNICODE_MAX) {
+ if (remaining < 4)
+ return (0);
*p++ = 0xf0 | ((uc >> 18) & 0x07);
*p++ = 0x80 | ((uc >> 12) & 0x3f);
*p++ = 0x80 | ((uc >> 6) & 0x3f);
* Undescribed code point should be U+FFFD
* (replacement character).
*/
+ if (remaining < UTF8_R_CHAR_SIZE)
+ return (0);
UTF8_SET_R_CHAR(p);
p += UTF8_R_CHAR_SIZE;
}
- return ((int)(p - _p));
+ return (p - _p);
+}
+
+static int
+utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n)
+{
+ return (utf16_to_unicode(pwc, s, n, 1));
}
static int
return ((int)(utf16 - s));
}
-static int
-unicode_to_utf16(char *p, uint32_t uc, int be)
+static size_t
+unicode_to_utf16be(char *p, size_t remaining, uint32_t uc)
{
char *utf16 = p;
if (uc > 0xffff) {
/* We have a code point that won't fit into a
* wchar_t; convert it to a surrogate pair. */
+ if (remaining < 4)
+ return (0);
uc -= 0x10000;
- if (be) {
- archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
- archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
- } else {
- archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
- archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
- }
- utf16 += 4;
+ archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
+ archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
+ return (4);
} else {
- if (be)
- archive_be16enc(utf16, uc);
- else
- archive_le16enc(utf16, uc);
- utf16 += 2;
+ if (remaining < 2)
+ return (0);
+ archive_be16enc(utf16, uc);
+ return (2);
+ }
+}
+
+static size_t
+unicode_to_utf16le(char *p, size_t remaining, uint32_t uc)
+{
+ char *utf16 = p;
+
+ if (uc > 0xffff) {
+ /* We have a code point that won't fit into a
+ * wchar_t; convert it to a surrogate pair. */
+ if (remaining < 4)
+ return (0);
+ uc -= 0x10000;
+ archive_le16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
+ archive_le16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
+ return (4);
+ } else {
+ if (remaining < 2)
+ return (0);
+ archive_le16enc(utf16, uc);
+ return (2);
}
- return ((int)(utf16 - p));
}
/*
return (-1);
p = as->s + as->length;
- endp = as->s + as->buffer_length -1 -4;
+ endp = as->s + as->buffer_length -1;
do {
uint32_t uc;
const char *ss = s;
-
- if (p + len > endp) {
- as->length = p - as->s;
- if (archive_string_ensure(as,
- as->buffer_length + len + 1) == NULL)
- return (-1);
- p = as->s + as->length;
- endp = as->s + as->buffer_length -1 -4;
- }
+ size_t w;
/*
* Forward byte sequence until a conversion of that is needed.
len -= n;
}
if (ss < s) {
+ if (p + (s - ss) > endp) {
+ as->length = p - as->s;
+ if (archive_string_ensure(as,
+ as->buffer_length + len + 1) == NULL)
+ return (-1);
+ p = as->s + as->length;
+ endp = as->s + as->buffer_length -1;
+ }
+
memcpy(p, ss, s - ss);
p += s - ss;
}
n *= -1;/* Use a replaced unicode character. */
}
- if (p + 4 > endp) {
+ /* Rebuild UTF-8 byte sequence. */
+ while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) {
as->length = p - as->s;
if (archive_string_ensure(as,
as->buffer_length + len + 1) == NULL)
return (-1);
p = as->s + as->length;
- endp = as->s + as->buffer_length -1 -4;
+ endp = as->s + as->buffer_length -1;
}
- /* Rebuild UTF-8 byte sequence. */
- p += unicode_to_utf8(p, uc);
+ p += w;
s += n;
len -= n;
}
n = n2; \
} while (0)
+#define EXPAND_BUFFER() do { \
+ as->length = p - as->s; \
+ if (archive_string_ensure(as, \
+ as->buffer_length + len * tm + ts) == NULL)\
+ return (-1); \
+ p = as->s + as->length; \
+ endp = as->s + as->buffer_length - ts; \
+} while (0)
+
+#define UNPARSE(p, endp, uc) do { \
+ while ((w = unparse(p, (endp) - (p), uc)) == 0) {\
+ EXPAND_BUFFER(); \
+ } \
+ p += w; \
+} while (0)
+
/*
* Write first code point.
* If the code point has not be changed from its original code,
*/
#define WRITE_UC() do { \
if (ucptr) { \
+ if (p + n > endp) \
+ EXPAND_BUFFER(); \
switch (n) { \
case 4: \
*p++ = *ucptr++; \
} \
ucptr = NULL; \
} else { \
- p += unicode_to_utf8(p, uc); \
+ UNPARSE(p, endp, uc); \
} \
} while (0)
#define COLLECT_CPS(start) do { \
int _i; \
for (_i = start; _i < FDC_MAX ; _i++) { \
- nx = cesu8_to_unicode(&ucx[_i], s, len);\
+ nx = parse(&ucx[_i], s, len); \
if (nx <= 0) \
break; \
cx = CCC(ucx[_i]); \
ucx_size = _i; \
} while (0)
-#define MAKE_SURE_ENOUGH_BUFFER() do { \
- if (p + len > endp) { \
- as->length = p - as->s; \
- if (archive_string_ensure(as, \
- as->buffer_length + len + 1) == NULL)\
- return (-1); \
- p = as->s + as->length; \
- endp = as->s + as->buffer_length -1 -4;\
- }\
-} while (0)
-
/*
- * Normalize UTF-8 characters to Form C and copy the result.
+ * Normalize UTF-8/UTF-16BE characters to Form C and copy the result.
*
* TODO: Convert composition exclusions,which are never converted
* from NFC,NFD,NFKC and NFKD, to Form C.
*/
static int
archive_string_normalize_C(struct archive_string *as, const char *s,
- size_t len)
+ size_t len, struct archive_string_conv *sc)
{
char *p, *endp;
uint32_t uc, uc2;
- int n, n2, ret = 0;
+ size_t w;
+ int always_replace, n, n2, ret = 0, spair, ts, tm;
+ int (*parse)(uint32_t *, const char *, size_t);
+ size_t (*unparse)(char *, size_t, uint32_t);
- /*
- * The process normalizing NFD characters to NFC will not expand
- * the length of an NFC UTF-8 string more than the length of an
- * NFD one unless we normalize the composition exclusion characters.
- */
- if (archive_string_ensure(as, as->length + len + 1) == NULL)
+ always_replace = 1;
+ ts = 1;/* text size. */
+ if (sc->flag & SCONV_TO_UTF16BE) {
+ unparse = unicode_to_utf16be;
+ ts = 2;
+ if (sc->flag & SCONV_FROM_UTF16BE)
+ always_replace = 0;
+ } else if (sc->flag & SCONV_TO_UTF16LE) {
+ unparse = unicode_to_utf16le;
+ ts = 2;
+ } else if (sc->flag & SCONV_TO_UTF8) {
+ unparse = unicode_to_utf8;
+ if (sc->flag & SCONV_FROM_UTF8)
+ always_replace = 0;
+ } else {
+ /*
+ * This case is going to be converted to another
+ * character-set through iconv.
+ */
+ always_replace = 0;
+ if (sc->flag & SCONV_FROM_UTF16BE) {
+ unparse = unicode_to_utf16be;
+ ts = 2;
+ } else {
+ unparse = unicode_to_utf8;
+ }
+ }
+
+ if (sc->flag & SCONV_FROM_UTF16BE) {
+ parse = utf16be_to_unicode;
+ tm = 1;
+ spair = 4;/* surrogate pair size in UTF-16. */
+ } else {
+ parse = cesu8_to_unicode;
+ tm = ts;
+ spair = 6;/* surrogate pair size in UTF-8. */
+ }
+
+ if (archive_string_ensure(as, as->length + len * tm + ts) == NULL)
return (-1);
p = as->s + as->length;
- endp = as->s + as->buffer_length -1 -4;
- while ((n = cesu8_to_unicode(&uc, s, len)) != 0) {
+ endp = as->s + as->buffer_length - ts;
+ while ((n = parse(&uc, s, len)) != 0) {
const char *ucptr, *uc2ptr;
if (n < 0) {
/* Use a replaced unicode character. */
- MAKE_SURE_ENOUGH_BUFFER();
- p += unicode_to_utf8(p, uc);
+ UNPARSE(p, endp, uc);
s += n*-1;
len -= n*-1;
ret = -1;
continue;
- } else if (n == 6)
+ } else if (n == spair || always_replace)
/* uc is converted from a surrogate pair.
* this should be treated as a changed code. */
ucptr = NULL;
len -= n;
/* Read second code point. */
- while ((n2 = cesu8_to_unicode(&uc2, s, len)) > 0) {
+ while ((n2 = parse(&uc2, s, len)) > 0) {
uint32_t ucx[FDC_MAX];
int ccx[FDC_MAX];
int cl, cx, i, nx, ucx_size;
int LIndex,SIndex;
uint32_t nfc;
- if (n2 == 6)
+ if (n2 == spair || always_replace)
/* uc2 is converted from a surrogate pair.
* this should be treated as a changed code. */
uc2ptr = NULL;
*/
WRITE_UC();
for (i = 0; i < ucx_size; i++)
- p += unicode_to_utf8(p, ucx[i]);
+ UNPARSE(p, endp, ucx[i]);
/*
* Flush out remaining canonical combining characters.
*/
if (nx > 0 && cx == cl && len > 0) {
- while ((nx = cesu8_to_unicode(&ucx[0], s, len))
+ while ((nx = parse(&ucx[0], s, len))
> 0) {
cx = CCC(ucx[0]);
if (cl > cx)
s += nx;
len -= nx;
cl = cx;
- p += unicode_to_utf8(p, ucx[0]);
+ UNPARSE(p, endp, ucx[0]);
}
}
break;
if (n2 < 0) {
WRITE_UC();
/* Use a replaced unicode character. */
- MAKE_SURE_ENOUGH_BUFFER();
- p += unicode_to_utf8(p, uc2);
+ UNPARSE(p, endp, uc2);
s += n2*-1;
len -= n2*-1;
ret = -1;
}
as->length = p - as->s;
as->s[as->length] = '\0';
+ if (ts == 2)
+ as->s[as->length+1] = '\0';
return (ret);
}
ByteCount inAvail, outAvail;
OSStatus err;
int return_val;
+ enum {
+ RT_8,
+ RT_16LE,
+ RT_16BE
+ } rt;
- /*
- * Convert UTF-8 to UTF-16.
- */
archive_string_empty(&(sc->utf16nfc));
- return_val = strncat_from_utf8_to_utf16(&(sc->utf16nfc), s, len, 0);
- if (archive_strlen(&(sc->utf16nfc)) == 0) {
- if (archive_string_ensure(as, as->length + 1) == NULL)
+ if (sc->flag & SCONV_FROM_UTF16BE) {
+ size_t i;
+ char *d;
+
+ if (archive_string_ensure(&(sc->utf16nfc),
+ sc->utf16nfc.length + length + 1) == NULL)
return (-1);
- return (return_val);
+ d = sc->utf16nfc.s;
+ for (i = 0; i < len; i += 2) {
+ uint16_t val = archive_be16dec(s+i);
+ archive_le16enc(d+i, val);
+ }
+ sc->utf16nfc.length = i;
+ sc->utf16nfc.s[i] = 0;
+ sc->utf16nfc.s[i+1] = 0;
+ } else {
+ /*
+ * Convert UTF-8 to UTF-16.
+ */
+ return_val = strncat_from_utf8_to_utf16(&(sc->utf16nfc),
+ s, len, 0);
+ if (archive_strlen(&(sc->utf16nfc)) == 0) {
+ if (archive_string_ensure(as, as->length + 1) == NULL)
+ return (-1);
+ return (return_val);
+ }
}
/*
if (err != noErr)
return_val = -1;
} while (0);
- /*
- * Convert UTF-16 to UTF-8.
- */
- archive_string_empty(as);
- if (strncat_from_utf16_to_utf8(as, sc->utf16nfd.s,
- sc->utf16nfd.length, 0) != 0)
- return_val = -1;
+
+ if (sc->flag & SCONV_TO_UTF16BE)
+ rt = RT_16BE;
+ else if (sc->flag & SCONV_TO_UTF16LE)
+ rt = RT_16LE;/* This may not happen. */
+ else if (sc->flag & SCONV_TO_UTF8)
+ rt = RT_8;
+ else {
+ /*
+ * This case is going to be converted to another
+ * character-set through iconv.
+ */
+ if (sc->flag & SCONV_FROM_UTF16BE)
+ rt = RT_16BE;
+ else
+ rt = RT_8;
+ }
+
+ if (rt == RT_8) {
+ /*
+ * Convert UTF-16 to UTF-8.
+ */
+ archive_string_empty(as);
+ if (strncat_from_utf16_to_utf8(as, sc->utf16nfd.s,
+ sc->utf16nfd.length, 0) != 0)
+ return_val = -1;
+ } else {
+ /*
+ * Convert UTF-16 to UTF-16BE/LE.
+ */
+ char *dst, *src;
+ if (archive_string_ensure(as,
+ as->length + sc->utf16nfd.length + 2) == NULL)
+ return (-1);
+ dst = as->s + as->length;
+ src = sc->utf16nfd.s;
+ if (rt == RT_16BE) {
+ size_t i;
+ for (i = 0; i < sc->utf16nfd.length; i += 2) {
+ uint16_t val = archive_le16dec(src+i);
+ archive_be16enc(dst+i, val);
+ }
+ } else
+ memcpy(dst, src, sc->utf16nfd.length);
+ as->length += sc->utf16nfd.length;
+ as->s[as->length] = 0;
+ as->s[as->length+1] = 0;
+ }
return (return_val);
}
const char *utf16;
char *p, *end;
uint32_t uc;
+ size_t w;
int n, return_val = 0; /* success */
utf16 = (const char *)_p;
p = as->s + as->length;
end = as->s + as->buffer_length -1;
while ((n = utf16_to_unicode(&uc, utf16, bytes, be)) != 0) {
- /* Expand the buffer when we have <4 bytes free. */
- if (end - p < 4) {
- as->length = p - as->s;
- if (NULL == archive_string_ensure(as,
- as->buffer_length + bytes + 1))
- return (-1);
- p = as->s + as->length;
- end = as->s + as->buffer_length -1;
- }
-
if (n < 0) {
n *= -1;
return_val = -1;
utf16 += n;
bytes -= n;
/* Translate code point to UTF8 */
- p += unicode_to_utf8(p, uc);
+ while ((w = unicode_to_utf8(p, end - p, uc)) == 0) {
+ /* Expand a destination buffer. */
+ as->length = p - as->s;
+ if (NULL == archive_string_ensure(as,
+ as->buffer_length + bytes + 1))
+ return (-1);
+ p = as->s + as->length;
+ end = as->s + as->buffer_length -1;
+ }
+ p += w;
}
as->length = p - as->s;
as->s[as->length] = '\0';
{
char *s, *end;
uint32_t uc;/* Must be large enough for a 21-bit Unicode code point. */
+ size_t w;
int n;
int return_val = 0; /* success */
+ size_t (*to_utf16)(char *, size_t, uint32_t);
if (NULL == archive_string_ensure(as, (len+1) * sizeof(uint16_t)))
return (-1);
- as->length = 0;
- s = as->s;
+ to_utf16 = (be)?unicode_to_utf16be:unicode_to_utf16le;
+ s = as->s + as->length;
end = as->s + as->buffer_length - sizeof(uint16_t);
while ((n = _utf8_to_unicode(&uc, p, len)) != 0) {
- /* Expand the buffer when we have <2 bytes free. */
- if (end - s < 2) {
+ if (n < 0) {
+ return_val = -1;
+ n *= -1; /* Use a replaced unicode character. */
+ }
+ p += n;
+ len -= n;
+ while ((w = to_utf16(s, end - s, uc)) == 0) {
+ /* Expand a destination buffer. */
as->length = s - as->s;
if (NULL == archive_string_ensure(as,
as->buffer_length + (len+1) * sizeof(uint16_t)))
s = as->s + as->length;
end = as->s + as->buffer_length - sizeof(uint16_t);
}
- if (n < 0) {
- return_val = -1;
- n *= -1; /* Use a replaced unicode character. */
- }
- p += n;
- len -= n;
- s += unicode_to_utf16(s, uc, be);
+ s += w;
}
as->length = s - as->s;
as->s[as->length] = 0;
return ((int)(p - _p));
}
+static void
+archive_be16enc(void *pp, uint16_t u)
+{
+ unsigned char *p = (unsigned char *)pp;
+
+ p[0] = (u >> 8) & 0xff;
+ p[1] = u & 0xff;
+}
+
+static int
+unicode_to_utf16be(char *p, uint32_t uc)
+{
+ char *utf16 = p;
+
+ if (uc > 0xffff) {
+ /* We have a code point that won't fit into a
+ * wchar_t; convert it to a surrogate pair. */
+ uc -= 0x10000;
+ archive_be16enc(utf16, ((uc >> 10) & 0x3ff) + 0xD800);
+ archive_be16enc(utf16+2, (uc & 0x3ff) + 0xDC00);
+ return (4);
+ } else {
+ archive_be16enc(utf16, uc);
+ return (2);
+ }
+}
+
+static int
+wc_size()
+{
+ return (sizeof(wchar_t));
+}
+
+static int
+unicode_to_wc(wchar_t *wp, uint32_t uc)
+{
+ if (wc_size() == 4) {
+ *wp = (wchar_t)uc;
+ return (1);
+ }
+ if (uc > 0xffff) {
+ /* We have a code point that won't fit into a
+ * wchar_t; convert it to a surrogate pair. */
+ uc -= 0x10000;
+ *wp++ = (wchar_t)(((uc >> 10) & 0x3ff) + 0xD800);
+ *wp = (wchar_t)((uc & 0x3ff) + 0xDC00);
+ return (2);
+ } else {
+ *wp = (wchar_t)uc;
+ return (1);
+ }
+}
+
/*
* Note: U+2000 - U+2FFF, U+F900 - U+FAFF and U+2F800 - U+2FAFF are not
* converted to NFD on Mac OS.
* see also http://developer.apple.com/library/mac/#qa/qa2001/qa1173.html
*/
static int
-scan_unicode_pattern(char *out, const char *pattern, int exclude_mac_nfd)
+scan_unicode_pattern(char *out, wchar_t *wout, char *u16be,
+ const char *pattern, int exclude_mac_nfd)
{
unsigned uc = 0;
const char *p = pattern;
char *op = out;
+ wchar_t *owp = wout;
+ char *op16be = u16be;
for (;;) {
if (*p >= '0' && *p <= '9')
uc == 0x110AB)
return (-1);
}
+ op16be += unicode_to_utf16be(op16be, uc);
+ owp += unicode_to_wc(owp, uc);
op += unicode_to_utf8(op, uc);
if (!*p) {
+ *op16be++ = 0;
+ *op16be = 0;
+ *owp = L'\0';
*op = '\0';
break;
}
return (0);
}
+static int
+is_wc_is_unicode()
+{
+#if (defined(_WIN32) && !defined(__CYGWIN__)) \
+ || defined(__STDC_ISO_10646__) || defined(__APPLE__)
+ return (1);
+#else
+ return (0);
+#endif
+}
+
/*
* A conversion test that we correctly normalize UTF-8 characters.
* On Mac OS, the characters to be Form D.
struct archive *a;
struct archive_entry *ae;
struct archive_string utf8;
+ struct archive_mstring mstr;
struct archive_string_conv *sconv;
+ struct archive_string_conv *sconv16;
FILE *fp;
char buff[512];
static const char reffile[] = "test_archive_string_conversion.txt.gz";
ssize_t size;
int line = 0;
+ int locale_is_utf8, wc_is_unicode;
+ locale_is_utf8 = (NULL != setlocale(LC_ALL, "en_US.UTF-8"));
+ wc_is_unicode = is_wc_is_unicode();
/* If it doesn't exist, just warn and return. */
- if (NULL == setlocale(LC_ALL, "en_US.UTF-8")) {
+ if (!locale_is_utf8 && !wc_is_unicode) {
skipping("invalid encoding tests require a suitable locale;"
" en_US.UTF-8 not available on this system");
return;
}
archive_string_init(&utf8);
+ memset(&mstr, 0, sizeof(mstr));
+
extract_reference_file(reffile);
assert((a = archive_read_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
assert((fp = fopen("testdata.txt", "r")) != NULL);
assertA(NULL != (sconv =
archive_string_conversion_from_charset(a, "UTF-8", 0)));
- if (sconv == NULL || fp == NULL) {
+ assertA(NULL != (sconv16 =
+ archive_string_conversion_from_charset(a, "UTF-16BE", 0)));
+ if (sconv == NULL || sconv16 == NULL || fp == NULL) {
/* We cannot continue this test. */
if (fp != NULL)
fclose(fp);
+ assertEqualInt(ARCHIVE_OK, archive_read_free(a));
return;
}
/*
while (fgets(buff, sizeof(buff), fp) != NULL) {
char nfc[80], nfd[80];
char utf8_nfc[80], utf8_nfd[80];
+ char utf16be_nfc[80], utf16be_nfd[80];
+ wchar_t wc_nfc[40], wc_nfd[40];
char *e, *p;
line++;
* Convert an NFC pattern to UTF-8 bytes.
*/
#if defined(__APPLE__)
- if (scan_unicode_pattern(utf8_nfc, nfc, 1) != 0)
+ if (scan_unicode_pattern(utf8_nfc, wc_nfc, utf16be_nfc,
+ nfc, 1) != 0)
continue;
#else
- scan_unicode_pattern(utf8_nfc, nfc, 0);
+ scan_unicode_pattern(utf8_nfc, wc_nfc, utf16be_nfc, nfc, 0);
#endif
/*
* Convert an NFD pattern to UTF-8 bytes.
*/
- scan_unicode_pattern(utf8_nfd, nfd, 0);
+ scan_unicode_pattern(utf8_nfd, wc_nfd, utf16be_nfd, nfd, 0);
+ if (locale_is_utf8) {
#if defined(__APPLE__)
- /*
- * Normalize an NFC string.
- */
- assertEqualInt(0,
- archive_strcpy_in_locale(&utf8, utf8_nfc, sconv));
- failure("NFC(%s) should be converted to NFD(%s):%d",
- nfc, nfd, line);
- assertEqualUTF8String(utf8_nfd, utf8.s);
+ /*
+ * Normalize an NFC string.
+ */
+ assertEqualInt(0,
+ archive_strcpy_in_locale(&utf8, utf8_nfc, sconv));
+ failure("NFC(%s) should be converted to NFD(%s):%d",
+ nfc, nfd, line);
+ assertEqualUTF8String(utf8_nfd, utf8.s);
- /*
- * Normalize an NFD string.
- */
- assertEqualInt(0,
- archive_strcpy_in_locale(&utf8, utf8_nfd, sconv));
- failure("NFD(%s) should not be any changed:%d",
- nfd, line);
- assertEqualUTF8String(utf8_nfd, utf8.s);
+ /*
+ * Normalize an NFD string.
+ */
+ assertEqualInt(0,
+ archive_strcpy_in_locale(&utf8, utf8_nfd, sconv));
+ failure("NFD(%s) should not be any changed:%d",
+ nfd, line);
+ assertEqualUTF8String(utf8_nfd, utf8.s);
+
+ /*
+ * Normalize an NFC string in UTF-16BE.
+ */
+ assertEqualInt(0, archive_strncpy_in_locale(&utf8,
+ utf16be_nfc, 100000, sconv16));
+ failure("NFC(%s) should be converted to NFD(%s):%d",
+ nfc, nfd, line);
+ assertEqualUTF8String(utf8_nfd, utf8.s);
#else
- /*
- * Normalize an NFD string.
- */
- assertEqualInt(0,
- archive_strcpy_in_locale(&utf8, utf8_nfd, sconv));
- failure("NFD(%s) should be converted to NFC(%s):%d",
- nfd, nfc, line);
- assertEqualUTF8String(utf8_nfc, utf8.s);
+ /*
+ * Normalize an NFD string.
+ */
+ assertEqualInt(0,
+ archive_strcpy_in_locale(&utf8, utf8_nfd, sconv));
+ failure("NFD(%s) should be converted to NFC(%s):%d",
+ nfd, nfc, line);
+ assertEqualUTF8String(utf8_nfc, utf8.s);
+
+ /*
+ * Normalize an NFC string.
+ */
+ assertEqualInt(0,
+ archive_strcpy_in_locale(&utf8, utf8_nfc, sconv));
+ failure("NFC(%s) should not be any changed:%d",
+ nfc, line);
+ assertEqualUTF8String(utf8_nfc, utf8.s);
+
+ /*
+ * Normalize an NFD string in UTF-16BE.
+ */
+ assertEqualInt(0, archive_strncpy_in_locale(&utf8,
+ utf16be_nfd, 100000, sconv16));
+ failure("NFD(%s) should be converted to NFC(%s):%d",
+ nfd, nfc, line);
+ assertEqualUTF8String(utf8_nfc, utf8.s);
+#endif
+ }
/*
- * Normalize an NFC string.
+ * Test for archive_mstring interface.
+ * In specific, Windows platform UTF-16BE is directly
+ * converted to wide-character to avoid the effect of
+ * current locale since windows platform cannot make
+ * locale UTF-8.
*/
- assertEqualInt(0,
- archive_strcpy_in_locale(&utf8, utf8_nfc, sconv));
- failure("NFC(%s) should not be any changed:%d",
- nfc, line);
- assertEqualUTF8String(utf8_nfc, utf8.s);
+ if (locale_is_utf8 || wc_is_unicode) {
+ const wchar_t *wp;
+
+#if defined(__APPLE__)
+ /*
+ * Normalize an NFD string in UTF-8.
+ */
+ assertEqualInt(0, archive_mstring_copy_mbs_len_l(
+ &mstr, utf8_nfc, 100000, sconv));
+ failure("UTF-8 NFC(%s) should be converted "
+ "to WCS NFD(%s):%d", nfc, nfd, line);
+ assertEqualInt(0,
+ archive_mstring_get_wcs(a, &mstr, &wp));
+ assertEqualWString(wc_nfd, wp);
+
+ /*
+ * Normalize an NFD string in UTF-16BE.
+ */
+ assertEqualInt(0, archive_mstring_copy_mbs_len_l(
+ &mstr, utf16be_nfc, 100000, sconv16));
+ failure("UTF-16BE NFC(%s) should be converted "
+ "to WCS NFD(%s):%d", nfc, nfd, line);
+ assertEqualInt(0,
+ archive_mstring_get_wcs(a, &mstr, &wp));
+ assertEqualWString(wc_nfd, wp);
+#else
+ /*
+ * Normalize an NFD string in UTF-8.
+ */
+ assertEqualInt(0, archive_mstring_copy_mbs_len_l(
+ &mstr, utf8_nfd, 100000, sconv));
+ failure("UTF-8 NFD(%s) should be converted "
+ "to WCS NFC(%s):%d", nfd, nfc, line);
+ assertEqualInt(0,
+ archive_mstring_get_wcs(a, &mstr, &wp));
+ assertEqualWString(wc_nfc, wp);
+
+ /*
+ * Normalize an NFD string in UTF-16BE.
+ */
+ assertEqualInt(0, archive_mstring_copy_mbs_len_l(
+ &mstr, utf16be_nfd, 100000, sconv16));
+ failure("UTF-8 NFD(%s) should be converted "
+ "to WCS NFC(%s):%d", nfd, nfc, line);
+ assertEqualInt(0,
+ archive_mstring_get_wcs(a, &mstr, &wp));
+ assertEqualWString(wc_nfc, wp);
#endif
+ }
}
archive_string_free(&utf8);
+ archive_mstring_clean(&mstr);
fclose(fp);
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
}
projects / thirdparty / libarchive.git / commitdiff
