From: Tim Kientzle Date: Mon, 30 Dec 2013 04:57:17 +0000 (-0800) Subject: Support experimental "at" extra block for better streaming. X-Git-Tag: v3.1.900a~327^2~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fbda7af0e529c53a8265af1f2d70a02bbea50ecd;p=thirdparty%2Flibarchive.git Support experimental "at" extra block for better streaming. The writer now writes this extra block with every local file header; the reader recognizes it and uses it. This allows streaming extraction to properly restore file permissions and symlinks. Without this, streaming extraction of Zip archives is somewhat hobbled by the lack of full information in the local file header. Here's a detailed description of the new extra block. The details here are subject to change at any time. -Extended Local File Header Extra Field (0x7461): The following is the layout of the extended local file header "extra" block. This allows information to be included with the local file header that could previously only be stored with the central directory file header. Note: all fields stored in Intel low-byte/high-byte order. Value Size Description ----- ---- ----------- 0x7461 2 bytes Tag for this "extra" block type Size 2 bytes Size of this "extra" block Version Made By 2 bytes See "Version Made By" above Internal File Attributes 2 bytes See "Internal File Attributes" above External File Attributes 4 bytes See "External File Attributes" above This extra block should only be used with the local file header. The values stored should exactly match the corresponding values in the central directory file header. --- diff --git a/libarchive/archive_read_support_format_zip.c b/libarchive/archive_read_support_format_zip.c index 447ca947c..b0d7858d3 100644 --- a/libarchive/archive_read_support_format_zip.c +++ b/libarchive/archive_read_support_format_zip.c @@ -339,6 +339,30 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) } break; } + case 0x7461: + /* Experimental 'at' field */ + if (datasize >= 2) { + zip_entry->system + = archive_le16dec(p + offset) >> 8; + offset += 2; + datasize -= 2; + } + if (datasize >= 2) { + // 2 byte "internal file attributes" + offset += 2; + datasize -= 2; + } + if (datasize >= 4) { + uint32_t external_attributes + = archive_le32dec(p + offset); + if (zip_entry->system == 3) { + zip_entry->mode + = external_attributes >> 16; + } + offset += 4; + datasize -= 4; + } + break; case 0x7855: /* Info-ZIP Unix Extra Field (type 2) "Ux". */ #ifdef DEBUG @@ -599,11 +623,61 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, archive_entry_set_mtime(entry, zip_entry->mtime, 0); archive_entry_set_ctime(entry, zip_entry->ctime, 0); archive_entry_set_atime(entry, zip_entry->atime, 0); - /* Set the size only if it's meaningful. */ - if (0 == (zip_entry->zip_flags & ZIP_LENGTH_AT_END) - || zip_entry->uncompressed_size > 0) - archive_entry_set_size(entry, zip_entry->uncompressed_size); + if ((zip->entry->mode & AE_IFMT) == AE_IFLNK) { + size_t linkname_length = zip_entry->compressed_size; + + archive_entry_set_size(entry, 0); + p = __archive_read_ahead(a, linkname_length, NULL); + if (p == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Truncated Zip file"); + return ARCHIVE_FATAL; + } + if (__archive_read_consume(a, linkname_length) < 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Read error skipping symlink target name"); + return ARCHIVE_FATAL; + } + + sconv = zip->sconv; + if (sconv == NULL && (zip->entry->zip_flags & ZIP_UTF8_NAME)) + sconv = zip->sconv_utf8; + if (sconv == NULL) + sconv = zip->sconv_default; + if (archive_entry_copy_symlink_l(entry, p, linkname_length, + sconv) != 0) { + if (errno != ENOMEM && sconv == zip->sconv_utf8 && + (zip->entry->zip_flags & ZIP_UTF8_NAME)) + archive_entry_copy_symlink_l(entry, p, + linkname_length, NULL); + if (errno == ENOMEM) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate memory for Symlink"); + return (ARCHIVE_FATAL); + } + /* + * Since there is no character-set regulation for + * symlink name, do not report the conversion error + * in an automatic conversion. + */ + if (sconv != zip->sconv_utf8 || + (zip->entry->zip_flags & ZIP_UTF8_NAME) == 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Symlink cannot be converted " + "from %s to current locale.", + archive_string_conversion_charset_name( + sconv)); + ret = ARCHIVE_WARN; + } + } + zip_entry->uncompressed_size = zip_entry->compressed_size = 0; + } else if (0 == (zip_entry->zip_flags & ZIP_LENGTH_AT_END) + || zip_entry->uncompressed_size > 0) { + /* Set the size only if it's meaningful. */ + archive_entry_set_size(entry, zip_entry->uncompressed_size); + } zip->entry_bytes_remaining = zip_entry->compressed_size; /* If there's no body, force read_data() to return EOF immediately. */ @@ -1956,52 +2030,6 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a, r = zip_read_local_file_header(a, entry, zip); if (r != ARCHIVE_OK) return r; - if ((zip->entry->mode & AE_IFMT) == AE_IFLNK) { - const void *p; - struct archive_string_conv *sconv; - size_t linkname_length = (size_t)archive_entry_size(entry); - - archive_entry_set_size(entry, 0); - p = __archive_read_ahead(a, linkname_length, NULL); - if (p == NULL) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Truncated Zip file"); - return ARCHIVE_FATAL; - } - - sconv = zip->sconv; - if (sconv == NULL && (zip->entry->zip_flags & ZIP_UTF8_NAME)) - sconv = zip->sconv_utf8; - if (sconv == NULL) - sconv = zip->sconv_default; - if (archive_entry_copy_symlink_l(entry, p, linkname_length, - sconv) != 0) { - if (errno != ENOMEM && sconv == zip->sconv_utf8 && - (zip->entry->zip_flags & ZIP_UTF8_NAME)) - archive_entry_copy_symlink_l(entry, p, - linkname_length, NULL); - if (errno == ENOMEM) { - archive_set_error(&a->archive, ENOMEM, - "Can't allocate memory for Symlink"); - return (ARCHIVE_FATAL); - } - /* - * Since there is no character-set regulation for - * symlink name, do not report the conversion error - * in an automatic conversion. - */ - if (sconv != zip->sconv_utf8 || - (zip->entry->zip_flags & ZIP_UTF8_NAME) == 0) { - archive_set_error(&a->archive, - ARCHIVE_ERRNO_FILE_FORMAT, - "Symlink cannot be converted " - "from %s to current locale.", - archive_string_conversion_charset_name( - sconv)); - ret = ARCHIVE_WARN; - } - } - } if (rsrc) { int ret2 = zip_read_mac_metadata(a, entry, rsrc); if (ret2 < ret) diff --git a/libarchive/archive_write_set_format_zip.c b/libarchive/archive_write_set_format_zip.c index 1731a1f1b..f28c59ea7 100644 --- a/libarchive/archive_write_set_format_zip.c +++ b/libarchive/archive_write_set_format_zip.c @@ -363,7 +363,7 @@ static int archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) { unsigned char local_header[32]; - unsigned char local_extra[64]; + unsigned char local_extra[128]; struct zip *zip = a->format_data; unsigned char *e; unsigned char *cd_extra; @@ -632,6 +632,21 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) archive_le16enc(zip64_start + 2, e - (zip64_start + 4)); } + { /* Experimental 'at' extension to support streaming. */ + unsigned char *external_info = e; + memcpy(e, "at\000\000", 4); + e += 4; + archive_le16enc(e, /* system + version written by */ + 3 * 256 + version_needed); + e += 2; + archive_le16enc(e, 0); /* internal file attributes */ + e += 2; + archive_le32enc(e, /* external file attributes */ + archive_entry_mode(zip->entry) << 16); + e += 4; + archive_le16enc(external_info + 2, e - (external_info + 4)); + } + /* Update local header with size of extra data and write it all out: */ archive_le16enc(local_header + 28, e - local_extra); diff --git a/libarchive/test/test_write_format_zip.c b/libarchive/test/test_write_format_zip.c index a21184402..3a2a129fb 100644 --- a/libarchive/test/test_write_format_zip.c +++ b/libarchive/test/test_write_format_zip.c @@ -337,16 +337,9 @@ verify_contents(struct archive *a, int seeking) assertEqualInt(0, archive_entry_atime(ae)); assertEqualInt(0, archive_entry_ctime(ae)); assertEqualString("symlink", archive_entry_pathname(ae)); - if (seeking) { - assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); - assertEqualInt(0, archive_entry_size(ae)); - assertEqualString("file1", archive_entry_symlink(ae)); - } else { - /* Streaming cannot read file type, so - * symlink body shows as regular file contents. */ - assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae)); - assertEqualInt(5, archive_entry_size(ae)); - } + assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); + assertEqualInt(0, archive_entry_size(ae)); + assertEqualString("file1", archive_entry_symlink(ae)); /* Read the dir entry back. */ assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); @@ -421,16 +414,9 @@ verify_contents(struct archive *a, int seeking) assertEqualInt(0, archive_entry_atime(ae)); assertEqualInt(0, archive_entry_ctime(ae)); assertEqualString("symlink_deflate", archive_entry_pathname(ae)); - if (seeking) { - assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); - assertEqualInt(0, archive_entry_size(ae)); - assertEqualString("file1", archive_entry_symlink(ae)); - } else { - assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae)); - assertEqualInt(5, archive_entry_size(ae)); - assertEqualIntA(a, 5, archive_read_data(a, filedata, 10)); - assertEqualMem(filedata, "file1", 5); - } + assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); + assertEqualInt(0, archive_entry_size(ae)); + assertEqualString("file1", archive_entry_symlink(ae)); /* Read the dir entry back. */ assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); @@ -506,16 +492,9 @@ verify_contents(struct archive *a, int seeking) assertEqualInt(0, archive_entry_atime(ae)); assertEqualInt(0, archive_entry_ctime(ae)); assertEqualString("symlink_stored", archive_entry_pathname(ae)); - if (seeking) { - assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); - assertEqualInt(0, archive_entry_size(ae)); - assertEqualString("file1", archive_entry_symlink(ae)); - } else { - assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae)); - assertEqualInt(5, archive_entry_size(ae)); - assertEqualIntA(a, 5, archive_read_data(a, filedata, 10)); - assertEqualMem(filedata, "file1", 5); - } + assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae)); + assertEqualInt(0, archive_entry_size(ae)); + assertEqualString("file1", archive_entry_symlink(ae)); /* Read the dir entry back. */ assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); diff --git a/libarchive/test/test_write_format_zip_compression_store.c b/libarchive/test/test_write_format_zip_compression_store.c index 4562acd1e..2aa4146e9 100644 --- a/libarchive/test/test_write_format_zip_compression_store.c +++ b/libarchive/test/test_write_format_zip_compression_store.c @@ -71,7 +71,7 @@ DEFINE_TEST(test_write_format_zip_compression_store) const char *buffend; /* p is the pointer to walk over the central directory, * q walks over the local headers, the data and the data descriptors. */ - const char *p, *q; + const char *p, *q, *local_header, *extra_start; size_t used; /* File data */ @@ -198,7 +198,7 @@ DEFINE_TEST(test_write_format_zip_compression_store) p = p + 4 + i2(p + 2); /* Verify local header of file entry. */ - q = buff; + local_header = q = buff; assertEqualMem(q, "PK\003\004", 4); /* Signature */ assertEqualInt(i2(q + 4), 10); /* Version needed to extract */ assertEqualInt(i2(q + 6), 8); /* Flags */ @@ -209,15 +209,16 @@ DEFINE_TEST(test_write_format_zip_compression_store) assertEqualInt(i4(q + 18), sizeof(file_data1) + sizeof(file_data2)); /* Compressed size */ assertEqualInt(i4(q + 22), sizeof(file_data1) + sizeof(file_data2)); /* Uncompressed size */ assertEqualInt(i2(q + 26), strlen(file_name)); /* Pathname length */ - assertEqualInt(i2(q + 28), 28); /* Extra field length */ + assertEqualInt(i2(q + 28), 40); /* Extra field length */ assertEqualMem(q + 30, file_name, strlen(file_name)); /* Pathname */ - q = q + 30 + strlen(file_name); + extra_start = q = q + 30 + strlen(file_name); assertEqualInt(i2(q), 0x5455); /* 'UT' extension header */ assertEqualInt(i2(q + 2), 9); /* 'UT' size */ assertEqualInt(q[4], 3); /* 'UT' flags */ assertEqualInt(i4(q + 5), t); /* 'UT' mtime */ assertEqualInt(i4(q + 9), t + 3); /* 'UT' atime */ q = q + 4 + i2(q + 2); + assertEqualInt(i2(q), 0x7875); /* 'ux' extension header */ assertEqualInt(i2(q + 2), 11); /* 'ux' size */ assertEqualInt(q[4], 1); /* 'ux' version */ @@ -227,6 +228,16 @@ DEFINE_TEST(test_write_format_zip_compression_store) assertEqualInt(i4(q + 11), file_gid); /* 'Ux' GID */ q = q + 4 + i2(q + 2); + assertEqualInt(i2(q), 0x7461); /* 'at' experimental extension header */ + assertEqualInt(i2(q + 2), 8); /* 'at' size */ + assertEqualInt(i2(q + 4), 3 * 256 + 10); /* version made by */ + assertEqualInt(i2(q + 6), 0); /* internal file attributes */ + assertEqualInt(i4(q + 8) >> 16 & 01777, file_perm); /* external file attributes */ + q = q + 4 + i2(q + 2); + + assert(q == extra_start + i2(local_header + 28)); + q = extra_start + i2(local_header + 28); + /* Verify data of file entry. */ assertEqualMem(q, file_data1, sizeof(file_data1)); assertEqualMem(q + sizeof(file_data1), file_data2, sizeof(file_data2)); @@ -276,6 +287,7 @@ DEFINE_TEST(test_write_format_zip_compression_store) /*p = p + 4 + i2(p + 2);*/ /* Verify local header of folder entry. */ + local_header = q; assertEqualMem(q, "PK\003\004", 4); /* Signature */ assertEqualInt(i2(q + 4), 20); /* Version needed to extract */ assertEqualInt(i2(q + 6), 0); /* Flags */ @@ -286,9 +298,9 @@ DEFINE_TEST(test_write_format_zip_compression_store) assertEqualInt(i4(q + 18), 0); /* Compressed size */ assertEqualInt(i4(q + 22), 0); /* Uncompressed size */ assertEqualInt(i2(q + 26), strlen(folder_name)); /* Pathname length */ - assertEqualInt(i2(q + 28), 28); /* Extra field length */ + assertEqualInt(i2(q + 28), 40); /* Extra field length */ assertEqualMem(q + 30, folder_name, strlen(folder_name)); /* Pathname */ - q = q + 30 + strlen(folder_name); + extra_start = q = q + 30 + strlen(folder_name); assertEqualInt(i2(q), 0x5455); /* 'UT' extension header */ assertEqualInt(i2(q + 2), 9); /* 'UT' size */ assertEqualInt(q[4], 5); /* 'UT' flags */ @@ -304,6 +316,16 @@ DEFINE_TEST(test_write_format_zip_compression_store) assertEqualInt(i4(q + 11), folder_gid); /* 'ux' GID */ q = q + 4 + i2(q + 2); + assertEqualInt(i2(q), 0x7461); /* 'at' experimental extension header */ + assertEqualInt(i2(q + 2), 8); /* 'at' size */ + assertEqualInt(i2(q + 4), 3 * 256 + 20); /* version made by */ + assertEqualInt(i2(q + 6), 0); /* internal file attributes */ + assertEqualInt(i4(q + 8) >> 16 & 01777, folder_perm); /* external file attributes */ + q = q + 4 + i2(q + 2); + + assert(q == extra_start + i2(local_header + 28)); + q = extra_start + i2(local_header + 28); + /* There should not be any data in the folder entry, * so the first central directory entry should be next: */ assertEqualMem(q, "PK\001\002", 4); /* Signature */ diff --git a/libarchive/test/test_write_format_zip_file.c b/libarchive/test/test_write_format_zip_file.c index eef36c629..7825ef55b 100644 --- a/libarchive/test/test_write_format_zip_file.c +++ b/libarchive/test/test_write_format_zip_file.c @@ -196,7 +196,7 @@ DEFINE_TEST(test_write_format_zip_file) /* assertEqualInt(i4(p + 18), sizeof(file_data)); */ /* Compressed size */ /* assertEqualInt(i4(p + 22), sizeof(file_data)); */ /* Uncompressed size not stored because we're using length-at-end. */ assertEqualInt(i2(p + 26), strlen(file_name)); /* Pathname length */ - assertEqualInt(i2(p + 28), 24); /* Extra field length */ + assertEqualInt(i2(p + 28), 36); /* Extra field length */ assertEqualMem(p + 30, file_name, strlen(file_name)); /* Pathname */ p = extension_start = local_header + 30 + strlen(file_name); extension_end = extension_start + i2(local_header + 28); @@ -216,6 +216,13 @@ DEFINE_TEST(test_write_format_zip_file) assertEqualInt(i4(p + 11), file_gid); /* 'Ux' GID */ p += 4 + i2(p + 2); + assertEqualInt(i2(p), 0x7461); /* 'at' experimental extension header */ + assertEqualInt(i2(p + 2), 8); /* 'at' size */ + assertEqualInt(i2(p + 4), 3 * 256 + 20); /* version made by */ + assertEqualInt(i2(p + 6), 0); /* internal file attributes */ + assertEqualInt(i4(p + 8) >> 16 & 01777, file_perm); /* external file attributes */ + p += 4 + i2(p + 2); + /* Just in case: Report any extra extensions. */ while (p < extension_end) { failure("Unexpected extension 0x%04X", i2(p)); diff --git a/libarchive/test/test_write_format_zip_file_zip64.c b/libarchive/test/test_write_format_zip_file_zip64.c index 3cc2794c8..1f47040b1 100644 --- a/libarchive/test/test_write_format_zip_file_zip64.c +++ b/libarchive/test/test_write_format_zip_file_zip64.c @@ -223,7 +223,7 @@ DEFINE_TEST(test_write_format_zip_file_zip64) /* assertEqualInt(i4(p + 18), sizeof(file_data)); */ /* Compressed size */ /* assertEqualInt(i4(p + 22), sizeof(file_data)); */ /* Uncompressed size not stored because we're using length-at-end. */ assertEqualInt(i2(p + 26), strlen(file_name)); /* Pathname length */ - assertEqualInt(i2(p + 28), 44); /* Extra field length */ + assertEqualInt(i2(p + 28), 56); /* Extra field length */ assertEqualMem(p + 30, file_name, strlen(file_name)); /* Pathname */ p = extension_start = local_header + 30 + strlen(file_name); extension_end = extension_start + i2(local_header + 28); @@ -249,6 +249,13 @@ DEFINE_TEST(test_write_format_zip_file_zip64) /* compressed file size we can't verify here */ p += 4 + i2(p + 2); + assertEqualInt(i2(p), 0x7461); /* 'at' experimental extension header */ + assertEqualInt(i2(p + 2), 8); /* 'at' size */ + assertEqualInt(i2(p + 4), 3 * 256 + 45); /* version made by */ + assertEqualInt(i2(p + 6), 0); /* internal file attributes */ + assertEqualInt(i4(p + 8) >> 16 & 01777, file_perm); /* external file attributes */ + p += 4 + i2(p + 2); + /* Just in case: Report any extra extensions. */ while (p < extension_end) { failure("Unexpected extension 0x%04X", i2(p));