From: Michihiro NAKAJIMA Date: Tue, 19 Apr 2011 07:55:44 +0000 (-0400) Subject: Introudce "tar:utf8type=libarchive2x" option for the incorrect UTF-8 string X-Git-Tag: v3.0.0a~432 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=446b093fa739cd77524d84a7233d9889ee5075cb;p=thirdparty%2Flibarchive.git Introudce "tar:utf8type=libarchive2x" option for the incorrect UTF-8 string which libarchive 2.x makes in wrong assumption about wchar_t. The option works only for pax format. Currently libarchive 3 correctly translates UTF-8 string from/to current locale string, but we cannot accordingly handle the incorrect UTF-8 on the some platforms wchar_t of which is not Unicode and users are not using UTF-8 locale. So we should support the UTF-8 string to be properly translated to current locale string. SVN-Revision: 3254 --- diff --git a/Makefile.am b/Makefile.am index 62ada7929..c1108ef0a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -270,6 +270,7 @@ libarchive_test_SOURCES= \ libarchive/test/test_compat_lzip.c \ libarchive/test/test_compat_lzma.c \ libarchive/test/test_compat_mac_gnutar.c \ + libarchive/test/test_compat_pax_libarchive_2x.c \ libarchive/test/test_compat_solaris_tar_acl.c \ libarchive/test/test_compat_tar_hardlink.c \ libarchive/test/test_compat_xz.c \ @@ -421,6 +422,7 @@ libarchive_test_EXTRA_DIST=\ libarchive/test/test_compat_lzma_2.tlz.uu \ libarchive/test/test_compat_lzma_3.tlz.uu \ libarchive/test/test_compat_mac_gnutar.tar.Z.uu \ + libarchive/test/test_compat_pax_libarchive_2x.tar.Z.uu \ libarchive/test/test_compat_solaris_tar_acl.tar.uu \ libarchive/test/test_compat_tar_hardlink_1.tar.uu \ libarchive/test/test_compat_xz_1.txz.uu \ diff --git a/libarchive/archive_read_support_format_tar.c b/libarchive/archive_read_support_format_tar.c index ebc79df0b..f0a504dfd 100644 --- a/libarchive/archive_read_support_format_tar.c +++ b/libarchive/archive_read_support_format_tar.c @@ -148,6 +148,7 @@ struct tar { struct archive_string_conv *sconv_acl; struct archive_string_conv *sconv_default; int init_default_conversion; + int utf8_made_by_libarchive2x; }; static int archive_block_is_null(const unsigned char *p); @@ -372,6 +373,16 @@ archive_read_format_tar_options(struct archive_read *a, else ret = ARCHIVE_FATAL; } + } else if (strcmp(key, "utf8type") == 0) { + if (val == NULL || val[0] == 0) + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "tar: utf8type option needs value"); + else if (strcmp(val, "libarchive2x") == 0) { + tar->utf8_made_by_libarchive2x = 1; + ret = ARCHIVE_OK; + } else + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "tar: incorrect value for ``%s''", key); } else archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "tar: unknown keyword ``%s''", key); @@ -1490,8 +1501,13 @@ pax_header(struct archive_read *a, struct tar *tar, if (tar->pax_hdrcharset_binary) sconv = tar->opt_sconv; else { + const char *cs; + if (tar->utf8_made_by_libarchive2x) + cs = "UTF-8-MADE_BY_LIBARCHIVE2"; + else + cs = "UTF-8"; sconv = archive_string_conversion_from_charset( - &(a->archive), "UTF-8", 1); + &(a->archive), cs, 1); if (sconv == NULL) return (ARCHIVE_FATAL); } diff --git a/libarchive/archive_string.c b/libarchive/archive_string.c index 18be7fc92..f17cee9ef 100644 --- a/libarchive/archive_string.c +++ b/libarchive/archive_string.c @@ -86,6 +86,8 @@ struct archive_string_conv { #define SCONV_UTF16BE 16 /* Consideration to UTF-16BE; one side * is single byte character, other is * double bytes character. */ +#define SCONV_UTF8_LIBARCHIVE_2 32 /* Incorrect UTF-8 made by libarchive + * 2.x in the wrong assumption. */ #if HAVE_ICONV iconv_t cd; @@ -111,6 +113,8 @@ static int strncpy_to_utf16be(struct archive_string *, const void *, size_t, struct archive_string_conv *); static int best_effort_strncat_in_locale(struct archive_string *, const void *, size_t, struct archive_string_conv *); +static int strncat_from_utf8_libarchive2(struct archive_string *, + const char *, size_t); static struct archive_string * archive_string_append(struct archive_string *as, const char *p, size_t s) @@ -668,6 +672,18 @@ create_sconv_object(const char *fc, const char *tc, { struct archive_string_conv *sc; + /* + * Special conversion for the incorrect UTF-8 made by libarchive 2.x + * only for the platform WCS of which is not Unicode. + */ + if (strcmp(fc, "UTF-8-MADE_BY_LIBARCHIVE2") == 0) +#if (defined(_WIN32) && !defined(__CYGWIN__)) \ + || defined(__STDC_ISO_10646__) || defined(__APPLE__) + fc = "UTF-8";/* Ignore special sequence. */ +#else + flag |= SCONV_UTF8_LIBARCHIVE_2; +#endif + sc = malloc(sizeof(*sc)); if (sc == NULL) return (NULL); @@ -683,6 +699,11 @@ create_sconv_object(const char *fc, const char *tc, free(sc->from_charset); return (NULL); } + + if (flag & SCONV_UTF8_LIBARCHIVE_2) { + sc->flag = flag; + return (sc); + } #if HAVE_ICONV sc->cd = iconv_open(tc, fc); #endif @@ -1070,6 +1091,14 @@ get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag) "Could not allocate memory for a string conversion object"); return (NULL); } + + /* We have to specially treat a string conversion so that + * we can correctly translate the wrong format UTF-8 string. */ + if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) { + if (a != NULL) + add_sconv_object(a, sc); + return (sc); + } #if HAVE_ICONV if (sc->cd == (iconv_t)-1 && (flag & SCONV_BEST_EFFORT) == 0) { free_sconv_object(sc); @@ -1317,6 +1346,11 @@ archive_strncat_in_locale(struct archive_string *as, const void *_p, size_t n, return (0); } + /* Perform special sequence for the incorrect UTF-8 made by + * libarchive2.x. */ + if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) + return (strncat_from_utf8_libarchive2(as, _p, length)); + archive_string_ensure(as, as->length + length*2+1); cd = sc->cd; @@ -1567,8 +1601,8 @@ is_all_ascii_code(struct archive_string *as) * Returns 0 if sc is NULL. */ static int -best_effort_strncat_in_locale(struct archive_string *as, const void *_p, size_t n, - struct archive_string_conv *sc) +best_effort_strncat_in_locale(struct archive_string *as, const void *_p, + size_t n, struct archive_string_conv *sc) { size_t length = la_strnlen(_p, n); @@ -1576,6 +1610,11 @@ best_effort_strncat_in_locale(struct archive_string *as, const void *_p, size_t if (sc != NULL && (sc->flag & SCONV_WIN_CP) != 0) return (strncat_in_codepage(as, _p, length, sc)); #endif + /* Perform special sequence for the incorrect UTF-8 made by + * libarchive2.x. */ + if (sc != NULL && (sc->flag & SCONV_UTF8_LIBARCHIVE_2) != 0) + return (strncat_from_utf8_libarchive2(as, _p, length)); + archive_string_append(as, _p, length); /* If charset is NULL, just make a copy, so return 0 as success. */ if (sc == NULL || (sc->same && invalid_mbs(_p, n, sc) == 0)) @@ -1585,6 +1624,138 @@ best_effort_strncat_in_locale(struct archive_string *as, const void *_p, size_t return (-1); } +/* + * Utility to convert a single UTF-8 sequence. + */ +static int +utf8_to_unicode(int *pwc, const char *s, size_t n) +{ + int ch; + + /* + * Decode 1-4 bytes depending on the value of the first byte. + */ + ch = (unsigned char)*s; + if (ch == 0) { + return (0); /* Standard: return 0 for end-of-string. */ + } + if ((ch & 0x80) == 0) { + *pwc = ch & 0x7f; + return (1); + } + if ((ch & 0xe0) == 0xc0) { + if (n < 2) + return (-1); + if ((s[1] & 0xc0) != 0x80) return (-1); + *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); + return (2); + } + if ((ch & 0xf0) == 0xe0) { + if (n < 3) + return (-1); + if ((s[1] & 0xc0) != 0x80) return (-1); + if ((s[2] & 0xc0) != 0x80) return (-1); + *pwc = ((ch & 0x0f) << 12) + | ((s[1] & 0x3f) << 6) + | (s[2] & 0x3f); + return (3); + } + if ((ch & 0xf8) == 0xf0) { + if (n < 4) + return (-1); + if ((s[1] & 0xc0) != 0x80) return (-1); + if ((s[2] & 0xc0) != 0x80) return (-1); + if ((s[3] & 0xc0) != 0x80) return (-1); + *pwc = ((ch & 0x07) << 18) + | ((s[1] & 0x3f) << 12) + | ((s[2] & 0x3f) << 6) + | (s[3] & 0x3f); + return (4); + } + /* Invalid first byte. */ + return (-1); +} + +/* + * libarchive 2.x made incorrect UTF-8 strings in the wrong assumuption + * that WCS is Unicode. it is true for servel platforms but some are false. + * And then people who did not use UTF-8 locale on the non Unicode WCS + * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those + * now cannot get right filename from libarchive 3.x and later since we + * fixed the wrong assumption and it is incompatible to older its versions. + * So we provide special option, "utf8type=libarchive2.x", for resolving it. + * That option enable the string conversion of libarchive 2.x. + * + * Translates the wrong UTF-8 string made by libarchive 2.x into current + * locale character set and appends to the archive_string. + * Note: returns -1 if conversion fails. + */ +static int +strncat_from_utf8_libarchive2(struct archive_string *as, + const char *s, size_t len) +{ + int n; + char *p; + char *end; +#if HAVE_WCRTOMB + mbstate_t shift_state; + + memset(&shift_state, 0, sizeof(shift_state)); +#else + /* Clear the shift state before starting. */ + wctomb(NULL, L'\0'); +#endif + /* + * Allocate buffer for MBS. + * We need this allocation here since it is possible that + * as->s is still NULL. + */ + if (archive_string_ensure(as, as->length + len + 1) == NULL) + __archive_errx(1, "Out of memory"); + + p = as->s + as->length; + end = as->s + as->buffer_length - MB_CUR_MAX -1; + while (*s != '\0' && len > 0) { + wchar_t wc; + int unicode; + + if (p >= end) { + as->length = p - as->s; + /* Re-allocate buffer for MBS. */ + if (archive_string_ensure(as, + as->length + len * 2 + 1) == NULL) + __archive_errx(1, "Out of memory"); + p = as->s + as->length; + end = as->s + as->buffer_length - MB_CUR_MAX -1; + } + + /* + * As libarchie 2.x, translates the wrong UTF-8 MBS into + * a wide-character in the assumption that WCS is Unicode. + */ + n = utf8_to_unicode(&unicode, s, len); + if (n == -1) + return (-1); + s += n; + len -= n; + + /* + * Translates the wide-character into the current locale MBS. + */ + wc = (wchar_t)unicode; +#if HAVE_WCRTOMB + n = wcrtomb(p, wc, &shift_state); +#else + n = wctomb(p, wc); +#endif + if (n == -1) + return (-1); + p += n; + } + as->length = p - as->s; + as->s[as->length] = '\0'; + return (0); +} /* @@ -1880,58 +2051,6 @@ string_append_from_utf16be_to_utf8(struct archive_string *as, return (return_val); } -/* - * Utility to convert a single UTF-8 sequence. - */ -static int -utf8_to_unicode(int *pwc, const char *s, size_t n) -{ - int ch; - - /* - * Decode 1-4 bytes depending on the value of the first byte. - */ - ch = (unsigned char)*s; - if (ch == 0) { - return (0); /* Standard: return 0 for end-of-string. */ - } - if ((ch & 0x80) == 0) { - *pwc = ch & 0x7f; - return (1); - } - if ((ch & 0xe0) == 0xc0) { - if (n < 2) - return (-1); - if ((s[1] & 0xc0) != 0x80) return (-1); - *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f); - return (2); - } - if ((ch & 0xf0) == 0xe0) { - if (n < 3) - return (-1); - if ((s[1] & 0xc0) != 0x80) return (-1); - if ((s[2] & 0xc0) != 0x80) return (-1); - *pwc = ((ch & 0x0f) << 12) - | ((s[1] & 0x3f) << 6) - | (s[2] & 0x3f); - return (3); - } - if ((ch & 0xf8) == 0xf0) { - if (n < 4) - return (-1); - if ((s[1] & 0xc0) != 0x80) return (-1); - if ((s[2] & 0xc0) != 0x80) return (-1); - if ((s[3] & 0xc0) != 0x80) return (-1); - *pwc = ((ch & 0x07) << 18) - | ((s[1] & 0x3f) << 12) - | ((s[2] & 0x3f) << 6) - | (s[3] & 0x3f); - return (4); - } - /* Invalid first byte. */ - return (-1); -} - /* * Return a UTF-16BE string by converting this archive_string from UTF-8. * Returns 0 on success, non-zero if conversion fails. diff --git a/libarchive/test/CMakeLists.txt b/libarchive/test/CMakeLists.txt index ca2f93482..ca5e67f0a 100644 --- a/libarchive/test/CMakeLists.txt +++ b/libarchive/test/CMakeLists.txt @@ -40,6 +40,7 @@ IF(ENABLE_TEST) test_compat_lzip.c test_compat_lzma.c test_compat_mac_gnutar.c + test_compat_pax_libarchive_2x.c test_compat_solaris_tar_acl.c test_compat_tar_hardlink.c test_compat_xz.c diff --git a/libarchive/test/test_compat_pax_libarchive_2x.c b/libarchive/test/test_compat_pax_libarchive_2x.c new file mode 100644 index 000000000..3ada94038 --- /dev/null +++ b/libarchive/test/test_compat_pax_libarchive_2x.c @@ -0,0 +1,147 @@ +/*- + * Copyright (c) 2011 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD"); + +#include + +/* + * Test "tar:utf8type=libarchive2x" option. That enalbe the string + * conversion of libarchive 2.x, which made incorrect UTF-8 string for + * pax on some platform wchar_t of which is non Unicode. + * The option is unneeded if people are using UTF-8 locale during making + * tar files. + * + * NOTE: The sample tar file was made with bsdtar 2.x in LANG=KOI8-R on + * FreeBSD. + */ + +static void +test_compat_pax_libarchive_2x_KOI8R(const char *refname) +{ + struct archive *a; + struct archive_entry *ae; + char c; + wchar_t wc; + + /* + * Read incorrect format UTF-8 filename in ru_RU.KOI8-R with + * "tar:utf8type=libarchive2x" option. We should correctly + * read two filenames. + */ + if (NULL == setlocale(LC_ALL, "ru_RU.KOI8-R")) { + skipping("ru_RU.KOI8-R locale not available on this system."); + return; + } + + /* + * Test if wchar_t format is the same as FreeBSD wchar_t. + */ + wctomb(NULL, L'\0'); + wc = (wchar_t)0xd0; + c = 0; + if (wctomb(&c, wc) != 1 || (unsigned char)c != 0xd0) { + skipping("wchar_t format is different on this platform."); + return; + } + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_set_options(a, "tar:utf8type=libarchive2x")); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_filename(a, refname, 10240)); + + /* Verify regular first file. */ + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("\xd0\xd2\xc9\xd7\xc5\xd4", + archive_entry_pathname(ae)); + assertEqualInt(6, archive_entry_size(ae)); + + /* Verify regular second file. */ + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("\xf0\xf2\xe9\xf7\xe5\xf4", + archive_entry_pathname(ae)); + assertEqualInt(6, archive_entry_size(ae)); + + + /* End of archive. */ + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + + /* Verify archive format. */ + assertEqualIntA(a, ARCHIVE_FILTER_COMPRESS, archive_filter_code(a, 0)); + assertEqualIntA(a, ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE, + archive_format(a)); + + /* Close the archive. */ + assertEqualInt(ARCHIVE_OK, archive_read_close(a)); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Without "tar:utf8type=libarchive2x" option. + * Neither first file name nor second file name can be translated + * to KOI8-R. + */ + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_filename(a, refname, 10240)); + + /* We cannot correctly read the filename. */ + assertEqualIntA(a, ARCHIVE_WARN, archive_read_next_header(a, &ae)); + assert(strcmp("\xd0\xd2\xc9\xd7\xc5\xd4", + archive_entry_pathname(ae)) != 0); + assertEqualInt(6, archive_entry_size(ae)); + + /* We cannot correctly read the filename. */ + assertEqualIntA(a, ARCHIVE_WARN, archive_read_next_header(a, &ae)); + assert(strcmp("\xf0\xf2\xe9\xf7\xe5\xf4", + archive_entry_pathname(ae)) != 0); + assertEqualInt(6, archive_entry_size(ae)); + + + /* End of archive. */ + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + + /* Verify archive format. */ + assertEqualIntA(a, ARCHIVE_FILTER_COMPRESS, archive_filter_code(a, 0)); + assertEqualIntA(a, ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE, + archive_format(a)); + + /* Close the archive. */ + assertEqualInt(ARCHIVE_OK, archive_read_close(a)); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); +} + + +DEFINE_TEST(test_compat_pax_libarchive_2x) +{ + const char *refname = "test_compat_pax_libarchive_2x.tar.Z"; + + extract_reference_file(refname); + test_compat_pax_libarchive_2x_KOI8R(refname); +} diff --git a/libarchive/test/test_compat_pax_libarchive_2x.tar.Z.uu b/libarchive/test/test_compat_pax_libarchive_2x.tar.Z.uu new file mode 100644 index 000000000..f44054118 --- /dev/null +++ b/libarchive/test/test_compat_pax_libarchive_2x.tar.Z.uu @@ -0,0 +1,15 @@ +begin 644 test_compat_pax_libarchive_2x.tar.Z +M'YV04,+@05(F#)DR3.FQ94D;-D#$B%&#QHT9,CS.B`DCADT;-P"`P,.QJ-&C2)-:K#.' +M3A@Y)&&,J5-&:<:I5:U>=.F1IV5P]@Q:=!HW;P87IE'C!@W-.%9_#NW"#9O7:P8K,$N\ +MN/'CR),K7\Z\N?.)"QL^?$[=:$N0(J.:K(%2^\JH7%O&G%GS9DX8.T'T+`PC +MJ/KJ\)$R=0K5(];B]XF']]H2;/S_`"H'CSSIW%,./0$FJ.""##;HX(,01@A` +M0`,5=%!"`Q9XH(3Z?1022]MUIQ)W_+D4`TPRT6033CKQ1),,0`E%%(?PS?=4 +M5/F9E6-9^X'7'XU`-H@6"&JQY18LP\@R3"K#W#),*.*E6!Z+Z+DX0V'NP4!HJ,<%NB&IJ*:JZJJLMNKJJ[#&*NNLM-9JZZVXYDH< +` +end