libarchive/test/test_compat_lzip.c \
libarchive/test/test_compat_lzma.c \
libarchive/test/test_compat_mac_gnutar.c \
+ libarchive/test/test_compat_pax_libarchive_2x.c \
libarchive/test/test_compat_solaris_tar_acl.c \
libarchive/test/test_compat_tar_hardlink.c \
libarchive/test/test_compat_xz.c \
libarchive/test/test_compat_lzma_2.tlz.uu \
libarchive/test/test_compat_lzma_3.tlz.uu \
libarchive/test/test_compat_mac_gnutar.tar.Z.uu \
+ libarchive/test/test_compat_pax_libarchive_2x.tar.Z.uu \
libarchive/test/test_compat_solaris_tar_acl.tar.uu \
libarchive/test/test_compat_tar_hardlink_1.tar.uu \
libarchive/test/test_compat_xz_1.txz.uu \
struct archive_string_conv *sconv_acl;
struct archive_string_conv *sconv_default;
int init_default_conversion;
+ int utf8_made_by_libarchive2x;
};
static int archive_block_is_null(const unsigned char *p);
else
ret = ARCHIVE_FATAL;
}
+ } else if (strcmp(key, "utf8type") == 0) {
+ if (val == NULL || val[0] == 0)
+ archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+ "tar: utf8type option needs value");
+ else if (strcmp(val, "libarchive2x") == 0) {
+ tar->utf8_made_by_libarchive2x = 1;
+ ret = ARCHIVE_OK;
+ } else
+ archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+ "tar: incorrect value for ``%s''", key);
} else
archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
"tar: unknown keyword ``%s''", key);
if (tar->pax_hdrcharset_binary)
sconv = tar->opt_sconv;
else {
+ const char *cs;
+ if (tar->utf8_made_by_libarchive2x)
+ cs = "UTF-8-MADE_BY_LIBARCHIVE2";
+ else
+ cs = "UTF-8";
sconv = archive_string_conversion_from_charset(
- &(a->archive), "UTF-8", 1);
+ &(a->archive), cs, 1);
if (sconv == NULL)
return (ARCHIVE_FATAL);
}
#define SCONV_UTF16BE 16 /* Consideration to UTF-16BE; one side
* is single byte character, other is
* double bytes character. */
+#define SCONV_UTF8_LIBARCHIVE_2 32 /* Incorrect UTF-8 made by libarchive
+ * 2.x in the wrong assumption. */
#if HAVE_ICONV
iconv_t cd;
struct archive_string_conv *);
static int best_effort_strncat_in_locale(struct archive_string *, const void *,
size_t, struct archive_string_conv *);
+static int strncat_from_utf8_libarchive2(struct archive_string *,
+ const char *, size_t);
static struct archive_string *
archive_string_append(struct archive_string *as, const char *p, size_t s)
{
struct archive_string_conv *sc;
+ /*
+ * Special conversion for the incorrect UTF-8 made by libarchive 2.x
+ * only for the platform WCS of which is not Unicode.
+ */
+ if (strcmp(fc, "UTF-8-MADE_BY_LIBARCHIVE2") == 0)
+#if (defined(_WIN32) && !defined(__CYGWIN__)) \
+ || defined(__STDC_ISO_10646__) || defined(__APPLE__)
+ fc = "UTF-8";/* Ignore special sequence. */
+#else
+ flag |= SCONV_UTF8_LIBARCHIVE_2;
+#endif
+
sc = malloc(sizeof(*sc));
if (sc == NULL)
return (NULL);
free(sc->from_charset);
return (NULL);
}
+
+ if (flag & SCONV_UTF8_LIBARCHIVE_2) {
+ sc->flag = flag;
+ return (sc);
+ }
#if HAVE_ICONV
sc->cd = iconv_open(tc, fc);
#endif
"Could not allocate memory for a string conversion object");
return (NULL);
}
+
+ /* We have to specially treat a string conversion so that
+ * we can correctly translate the wrong format UTF-8 string. */
+ if (sc->flag & SCONV_UTF8_LIBARCHIVE_2) {
+ if (a != NULL)
+ add_sconv_object(a, sc);
+ return (sc);
+ }
#if HAVE_ICONV
if (sc->cd == (iconv_t)-1 && (flag & SCONV_BEST_EFFORT) == 0) {
free_sconv_object(sc);
return (0);
}
+ /* Perform special sequence for the incorrect UTF-8 made by
+ * libarchive2.x. */
+ if (sc->flag & SCONV_UTF8_LIBARCHIVE_2)
+ return (strncat_from_utf8_libarchive2(as, _p, length));
+
archive_string_ensure(as, as->length + length*2+1);
cd = sc->cd;
* Returns 0 if sc is NULL.
*/
static int
-best_effort_strncat_in_locale(struct archive_string *as, const void *_p, size_t n,
- struct archive_string_conv *sc)
+best_effort_strncat_in_locale(struct archive_string *as, const void *_p,
+ size_t n, struct archive_string_conv *sc)
{
size_t length = la_strnlen(_p, n);
if (sc != NULL && (sc->flag & SCONV_WIN_CP) != 0)
return (strncat_in_codepage(as, _p, length, sc));
#endif
+ /* Perform special sequence for the incorrect UTF-8 made by
+ * libarchive2.x. */
+ if (sc != NULL && (sc->flag & SCONV_UTF8_LIBARCHIVE_2) != 0)
+ return (strncat_from_utf8_libarchive2(as, _p, length));
+
archive_string_append(as, _p, length);
/* If charset is NULL, just make a copy, so return 0 as success. */
if (sc == NULL || (sc->same && invalid_mbs(_p, n, sc) == 0))
return (-1);
}
+/*
+ * Utility to convert a single UTF-8 sequence.
+ */
+static int
+utf8_to_unicode(int *pwc, const char *s, size_t n)
+{
+ int ch;
+
+ /*
+ * Decode 1-4 bytes depending on the value of the first byte.
+ */
+ ch = (unsigned char)*s;
+ if (ch == 0) {
+ return (0); /* Standard: return 0 for end-of-string. */
+ }
+ if ((ch & 0x80) == 0) {
+ *pwc = ch & 0x7f;
+ return (1);
+ }
+ if ((ch & 0xe0) == 0xc0) {
+ if (n < 2)
+ return (-1);
+ if ((s[1] & 0xc0) != 0x80) return (-1);
+ *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f);
+ return (2);
+ }
+ if ((ch & 0xf0) == 0xe0) {
+ if (n < 3)
+ return (-1);
+ if ((s[1] & 0xc0) != 0x80) return (-1);
+ if ((s[2] & 0xc0) != 0x80) return (-1);
+ *pwc = ((ch & 0x0f) << 12)
+ | ((s[1] & 0x3f) << 6)
+ | (s[2] & 0x3f);
+ return (3);
+ }
+ if ((ch & 0xf8) == 0xf0) {
+ if (n < 4)
+ return (-1);
+ if ((s[1] & 0xc0) != 0x80) return (-1);
+ if ((s[2] & 0xc0) != 0x80) return (-1);
+ if ((s[3] & 0xc0) != 0x80) return (-1);
+ *pwc = ((ch & 0x07) << 18)
+ | ((s[1] & 0x3f) << 12)
+ | ((s[2] & 0x3f) << 6)
+ | (s[3] & 0x3f);
+ return (4);
+ }
+ /* Invalid first byte. */
+ return (-1);
+}
+
+/*
+ * libarchive 2.x made incorrect UTF-8 strings in the wrong assumuption
+ * that WCS is Unicode. it is true for servel platforms but some are false.
+ * And then people who did not use UTF-8 locale on the non Unicode WCS
+ * platform and made a tar file with libarchive(mostly bsdtar) 2.x. Those
+ * now cannot get right filename from libarchive 3.x and later since we
+ * fixed the wrong assumption and it is incompatible to older its versions.
+ * So we provide special option, "utf8type=libarchive2.x", for resolving it.
+ * That option enable the string conversion of libarchive 2.x.
+ *
+ * Translates the wrong UTF-8 string made by libarchive 2.x into current
+ * locale character set and appends to the archive_string.
+ * Note: returns -1 if conversion fails.
+ */
+static int
+strncat_from_utf8_libarchive2(struct archive_string *as,
+ const char *s, size_t len)
+{
+ int n;
+ char *p;
+ char *end;
+#if HAVE_WCRTOMB
+ mbstate_t shift_state;
+
+ memset(&shift_state, 0, sizeof(shift_state));
+#else
+ /* Clear the shift state before starting. */
+ wctomb(NULL, L'\0');
+#endif
+ /*
+ * Allocate buffer for MBS.
+ * We need this allocation here since it is possible that
+ * as->s is still NULL.
+ */
+ if (archive_string_ensure(as, as->length + len + 1) == NULL)
+ __archive_errx(1, "Out of memory");
+
+ p = as->s + as->length;
+ end = as->s + as->buffer_length - MB_CUR_MAX -1;
+ while (*s != '\0' && len > 0) {
+ wchar_t wc;
+ int unicode;
+
+ if (p >= end) {
+ as->length = p - as->s;
+ /* Re-allocate buffer for MBS. */
+ if (archive_string_ensure(as,
+ as->length + len * 2 + 1) == NULL)
+ __archive_errx(1, "Out of memory");
+ p = as->s + as->length;
+ end = as->s + as->buffer_length - MB_CUR_MAX -1;
+ }
+
+ /*
+ * As libarchie 2.x, translates the wrong UTF-8 MBS into
+ * a wide-character in the assumption that WCS is Unicode.
+ */
+ n = utf8_to_unicode(&unicode, s, len);
+ if (n == -1)
+ return (-1);
+ s += n;
+ len -= n;
+
+ /*
+ * Translates the wide-character into the current locale MBS.
+ */
+ wc = (wchar_t)unicode;
+#if HAVE_WCRTOMB
+ n = wcrtomb(p, wc, &shift_state);
+#else
+ n = wctomb(p, wc);
+#endif
+ if (n == -1)
+ return (-1);
+ p += n;
+ }
+ as->length = p - as->s;
+ as->s[as->length] = '\0';
+ return (0);
+}
/*
return (return_val);
}
-/*
- * Utility to convert a single UTF-8 sequence.
- */
-static int
-utf8_to_unicode(int *pwc, const char *s, size_t n)
-{
- int ch;
-
- /*
- * Decode 1-4 bytes depending on the value of the first byte.
- */
- ch = (unsigned char)*s;
- if (ch == 0) {
- return (0); /* Standard: return 0 for end-of-string. */
- }
- if ((ch & 0x80) == 0) {
- *pwc = ch & 0x7f;
- return (1);
- }
- if ((ch & 0xe0) == 0xc0) {
- if (n < 2)
- return (-1);
- if ((s[1] & 0xc0) != 0x80) return (-1);
- *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f);
- return (2);
- }
- if ((ch & 0xf0) == 0xe0) {
- if (n < 3)
- return (-1);
- if ((s[1] & 0xc0) != 0x80) return (-1);
- if ((s[2] & 0xc0) != 0x80) return (-1);
- *pwc = ((ch & 0x0f) << 12)
- | ((s[1] & 0x3f) << 6)
- | (s[2] & 0x3f);
- return (3);
- }
- if ((ch & 0xf8) == 0xf0) {
- if (n < 4)
- return (-1);
- if ((s[1] & 0xc0) != 0x80) return (-1);
- if ((s[2] & 0xc0) != 0x80) return (-1);
- if ((s[3] & 0xc0) != 0x80) return (-1);
- *pwc = ((ch & 0x07) << 18)
- | ((s[1] & 0x3f) << 12)
- | ((s[2] & 0x3f) << 6)
- | (s[3] & 0x3f);
- return (4);
- }
- /* Invalid first byte. */
- return (-1);
-}
-
/*
* Return a UTF-16BE string by converting this archive_string from UTF-8.
* Returns 0 on success, non-zero if conversion fails.
test_compat_lzip.c
test_compat_lzma.c
test_compat_mac_gnutar.c
+ test_compat_pax_libarchive_2x.c
test_compat_solaris_tar_acl.c
test_compat_tar_hardlink.c
test_compat_xz.c
--- /dev/null
+/*-
+ * Copyright (c) 2011 Michihiro NAKAJIMA
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD");
+
+#include <locale.h>
+
+/*
+ * Test "tar:utf8type=libarchive2x" option. That enalbe the string
+ * conversion of libarchive 2.x, which made incorrect UTF-8 string for
+ * pax on some platform wchar_t of which is non Unicode.
+ * The option is unneeded if people are using UTF-8 locale during making
+ * tar files.
+ *
+ * NOTE: The sample tar file was made with bsdtar 2.x in LANG=KOI8-R on
+ * FreeBSD.
+ */
+
+static void
+test_compat_pax_libarchive_2x_KOI8R(const char *refname)
+{
+ struct archive *a;
+ struct archive_entry *ae;
+ char c;
+ wchar_t wc;
+
+ /*
+ * Read incorrect format UTF-8 filename in ru_RU.KOI8-R with
+ * "tar:utf8type=libarchive2x" option. We should correctly
+ * read two filenames.
+ */
+ if (NULL == setlocale(LC_ALL, "ru_RU.KOI8-R")) {
+ skipping("ru_RU.KOI8-R locale not available on this system.");
+ return;
+ }
+
+ /*
+ * Test if wchar_t format is the same as FreeBSD wchar_t.
+ */
+ wctomb(NULL, L'\0');
+ wc = (wchar_t)0xd0;
+ c = 0;
+ if (wctomb(&c, wc) != 1 || (unsigned char)c != 0xd0) {
+ skipping("wchar_t format is different on this platform.");
+ return;
+ }
+
+ assert((a = archive_read_new()) != NULL);
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+ assertEqualIntA(a, ARCHIVE_OK,
+ archive_read_set_options(a, "tar:utf8type=libarchive2x"));
+ assertEqualIntA(a, ARCHIVE_OK,
+ archive_read_open_filename(a, refname, 10240));
+
+ /* Verify regular first file. */
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));
+ assertEqualString("\xd0\xd2\xc9\xd7\xc5\xd4",
+ archive_entry_pathname(ae));
+ assertEqualInt(6, archive_entry_size(ae));
+
+ /* Verify regular second file. */
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));
+ assertEqualString("\xf0\xf2\xe9\xf7\xe5\xf4",
+ archive_entry_pathname(ae));
+ assertEqualInt(6, archive_entry_size(ae));
+
+
+ /* End of archive. */
+ assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
+
+ /* Verify archive format. */
+ assertEqualIntA(a, ARCHIVE_FILTER_COMPRESS, archive_filter_code(a, 0));
+ assertEqualIntA(a, ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE,
+ archive_format(a));
+
+ /* Close the archive. */
+ assertEqualInt(ARCHIVE_OK, archive_read_close(a));
+ assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+ /*
+ * Without "tar:utf8type=libarchive2x" option.
+ * Neither first file name nor second file name can be translated
+ * to KOI8-R.
+ */
+ assert((a = archive_read_new()) != NULL);
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+ assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+ assertEqualIntA(a, ARCHIVE_OK,
+ archive_read_open_filename(a, refname, 10240));
+
+ /* We cannot correctly read the filename. */
+ assertEqualIntA(a, ARCHIVE_WARN, archive_read_next_header(a, &ae));
+ assert(strcmp("\xd0\xd2\xc9\xd7\xc5\xd4",
+ archive_entry_pathname(ae)) != 0);
+ assertEqualInt(6, archive_entry_size(ae));
+
+ /* We cannot correctly read the filename. */
+ assertEqualIntA(a, ARCHIVE_WARN, archive_read_next_header(a, &ae));
+ assert(strcmp("\xf0\xf2\xe9\xf7\xe5\xf4",
+ archive_entry_pathname(ae)) != 0);
+ assertEqualInt(6, archive_entry_size(ae));
+
+
+ /* End of archive. */
+ assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
+
+ /* Verify archive format. */
+ assertEqualIntA(a, ARCHIVE_FILTER_COMPRESS, archive_filter_code(a, 0));
+ assertEqualIntA(a, ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE,
+ archive_format(a));
+
+ /* Close the archive. */
+ assertEqualInt(ARCHIVE_OK, archive_read_close(a));
+ assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+}
+
+
+DEFINE_TEST(test_compat_pax_libarchive_2x)
+{
+ const char *refname = "test_compat_pax_libarchive_2x.tar.Z";
+
+ extract_reference_file(refname);
+ test_compat_pax_libarchive_2x_KOI8R(refname);
+}
--- /dev/null
+begin 644 test_compat_pax_libarchive_2x.tar.Z
+M'YV04,+@05(F#)DR<EY`DY;L6C%J`")*G$BQHL6+&#-JW%@1AL<;-6J``.`Q
+M!L@8(TN>3.FQ94D;-D#$B%&#QHT9,CS.B`DCADT;-P"`P,.QJ-&C2)-:K#.'
+M3A@Y)&&,J5-&:<:I5:U>=.F1I<N16L.*'4NV;%D9*.&$H8.FQS!(PR0-2S3L
+MTK!"PR@IR`EB#)TT;<KTB#$#!EJ0.&+@V`L#Q%K`@@D;QG$C!PX9.128!#%E
+M")(D3+*X.&BG1XX:>V5P]@Q:=!HW;P87IE'C!@W-.%9_#NW"#9O7:P8K,$N\
+MN/'CR),K7\Z\N?.)"QL^?$[=:$N0(J.:K(%2^\JH7%O&G%GS9DX8.T'T+`PC
+MJ/KJ\)$R=0K5(];B]XF']]H2;/S_`"H'CSSIW%,./0$FJ.""##;HX(,01@A`
+M0`,5=%!"`Q9XH(3Z?1022]MUIQ)W_+D4`TPRT6033CKQ1),,0`E%%(?PS?=4
+M5/F9E6-9^X'7'XU`-H@6"&JQY18LP\@R3"K#W#),*</0PEA??P4FFV';);88
+M7X]9*9D,E%F&F68WZ-;::&64=EIJ9O+V6FR2T082;FV*YAMPP@6IYYY\]NFG
+M4AD:B."?9EWWH7<D(BIB>.*E6!Z+Z+DX0V'NP4!H<C;6)Q55^'':X5<^=G7I
+>J,<%NB&IJ*:JZJJLMNKJJ[#&*NNLM-9JZZVXYDH<
+`
+end
projects / thirdparty / libarchive.git / commitdiff
