Support experimental "at" extra block for better streaming.

The writer now writes this extra block with every local
file header; the reader recognizes it and uses it.
This allows streaming extraction to properly restore
file permissions and symlinks.

Without this, streaming extraction of Zip archives is
somewhat hobbled by the lack of full information in
the local file header.

Here's a detailed description of the new extra block.
The details here are subject to change at any time.

   -Extended Local File Header Extra Field (0x7461):

      The following is the layout of the extended local file
      header "extra" block.  This allows information to be
      included with the local file header that could previously
      only be stored with the central directory file header.

      Note: all fields stored in Intel low-byte/high-byte order.

        Value      Size       Description
        -----      ----       -----------
        0x7461     2 bytes    Tag for this "extra" block type
        Size       2 bytes    Size of this "extra" block
        Version
        Made By    2 bytes    See "Version Made By" above
        Internal File
        Attributes 2 bytes    See "Internal File Attributes" above
        External File
        Attributes 4 bytes    See "External File Attributes" above

      This extra block should only be used with the local
      file header.  The values stored should exactly match
      the corresponding values in the central directory
      file header.

diff --git a/libarchive/archive_read_support_format_zip.c b/libarchive/archive_read_support_format_zip.c
index 447ca947cf032d81c1a5232e3f87f5734a31f500..b0d7858d318cbc301791c0cf88772d85bdae4e79 100644 (file)
--- a/libarchive/archive_read_support_format_zip.c
+++ b/libarchive/archive_read_support_format_zip.c
@@ -339,6 +339,30 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry)
                        }
                        break;
                }
+               case 0x7461:
+                       /* Experimental 'at' field */
+                       if (datasize >= 2) {
+                               zip_entry->system
+                                   = archive_le16dec(p + offset) >> 8;
+                               offset += 2;
+                               datasize -= 2;
+                       }
+                       if (datasize >= 2) {
+                               // 2 byte "internal file attributes"
+                               offset += 2;
+                               datasize -= 2;
+                       }
+                       if (datasize >= 4) {
+                               uint32_t external_attributes
+                                   = archive_le32dec(p + offset);
+                               if (zip_entry->system == 3) {
+                                       zip_entry->mode
+                                           = external_attributes >> 16;
+                               }
+                               offset += 4;
+                               datasize -= 4;
+                       }
+                       break;
                case 0x7855:
                        /* Info-ZIP Unix Extra Field (type 2) "Ux". */
 #ifdef DEBUG
@@ -599,11 +623,61 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry,
        archive_entry_set_mtime(entry, zip_entry->mtime, 0);
        archive_entry_set_ctime(entry, zip_entry->ctime, 0);
        archive_entry_set_atime(entry, zip_entry->atime, 0);
-       /* Set the size only if it's meaningful. */
-       if (0 == (zip_entry->zip_flags & ZIP_LENGTH_AT_END)
-           || zip_entry->uncompressed_size > 0)
-               archive_entry_set_size(entry, zip_entry->uncompressed_size);
 
+       if ((zip->entry->mode & AE_IFMT) == AE_IFLNK) {
+               size_t linkname_length = zip_entry->compressed_size;
+
+               archive_entry_set_size(entry, 0);
+               p = __archive_read_ahead(a, linkname_length, NULL);
+               if (p == NULL) {
+                       archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+                           "Truncated Zip file");
+                       return ARCHIVE_FATAL;
+               }
+               if (__archive_read_consume(a, linkname_length) < 0) {
+                       archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+                           "Read error skipping symlink target name");
+                       return ARCHIVE_FATAL;
+               }
+
+               sconv = zip->sconv;
+               if (sconv == NULL && (zip->entry->zip_flags & ZIP_UTF8_NAME))
+                       sconv = zip->sconv_utf8;
+               if (sconv == NULL)
+                       sconv = zip->sconv_default;
+               if (archive_entry_copy_symlink_l(entry, p, linkname_length,
+                   sconv) != 0) {
+                       if (errno != ENOMEM && sconv == zip->sconv_utf8 &&
+                           (zip->entry->zip_flags & ZIP_UTF8_NAME))
+                           archive_entry_copy_symlink_l(entry, p,
+                               linkname_length, NULL);
+                       if (errno == ENOMEM) {
+                               archive_set_error(&a->archive, ENOMEM,
+                                   "Can't allocate memory for Symlink");
+                               return (ARCHIVE_FATAL);
+                       }
+                       /*
+                        * Since there is no character-set regulation for
+                        * symlink name, do not report the conversion error
+                        * in an automatic conversion.
+                        */
+                       if (sconv != zip->sconv_utf8 ||
+                           (zip->entry->zip_flags & ZIP_UTF8_NAME) == 0) {
+                               archive_set_error(&a->archive,
+                                   ARCHIVE_ERRNO_FILE_FORMAT,
+                                   "Symlink cannot be converted "
+                                   "from %s to current locale.",
+                                   archive_string_conversion_charset_name(
+                                       sconv));
+                               ret = ARCHIVE_WARN;
+                       }
+               }
+               zip_entry->uncompressed_size = zip_entry->compressed_size = 0;
+       } else if (0 == (zip_entry->zip_flags & ZIP_LENGTH_AT_END)
+           || zip_entry->uncompressed_size > 0) {
+               /* Set the size only if it's meaningful. */
+               archive_entry_set_size(entry, zip_entry->uncompressed_size);
+       }
        zip->entry_bytes_remaining = zip_entry->compressed_size;
 
        /* If there's no body, force read_data() to return EOF immediately. */
@@ -1956,52 +2030,6 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a,
        r = zip_read_local_file_header(a, entry, zip);
        if (r != ARCHIVE_OK)
                return r;
-       if ((zip->entry->mode & AE_IFMT) == AE_IFLNK) {
-               const void *p;
-               struct archive_string_conv *sconv;
-               size_t linkname_length = (size_t)archive_entry_size(entry);
-
-               archive_entry_set_size(entry, 0);
-               p = __archive_read_ahead(a, linkname_length, NULL);
-               if (p == NULL) {
-                       archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
-                           "Truncated Zip file");
-                       return ARCHIVE_FATAL;
-               }
-
-               sconv = zip->sconv;
-               if (sconv == NULL && (zip->entry->zip_flags & ZIP_UTF8_NAME))
-                       sconv = zip->sconv_utf8;
-               if (sconv == NULL)
-                       sconv = zip->sconv_default;
-               if (archive_entry_copy_symlink_l(entry, p, linkname_length,
-                   sconv) != 0) {
-                       if (errno != ENOMEM && sconv == zip->sconv_utf8 &&
-                           (zip->entry->zip_flags & ZIP_UTF8_NAME))
-                           archive_entry_copy_symlink_l(entry, p,
-                               linkname_length, NULL);
-                       if (errno == ENOMEM) {
-                               archive_set_error(&a->archive, ENOMEM,
-                                   "Can't allocate memory for Symlink");
-                               return (ARCHIVE_FATAL);
-                       }
-                       /*
-                        * Since there is no character-set regulation for
-                        * symlink name, do not report the conversion error
-                        * in an automatic conversion.
-                        */
-                       if (sconv != zip->sconv_utf8 ||
-                           (zip->entry->zip_flags & ZIP_UTF8_NAME) == 0) {
-                               archive_set_error(&a->archive,
-                                   ARCHIVE_ERRNO_FILE_FORMAT,
-                                   "Symlink cannot be converted "
-                                   "from %s to current locale.",
-                                   archive_string_conversion_charset_name(
-                                       sconv));
-                               ret = ARCHIVE_WARN;
-                       }
-               }
-       }
        if (rsrc) {
                int ret2 = zip_read_mac_metadata(a, entry, rsrc);
                if (ret2 < ret)

diff --git a/libarchive/archive_write_set_format_zip.c b/libarchive/archive_write_set_format_zip.c
index 1731a1f1beaa2af666edf0ee3f304f6d8a83f5b9..f28c59ea707e16546c3ce91fe35c1cb782f649ec 100644 (file)
--- a/libarchive/archive_write_set_format_zip.c
+++ b/libarchive/archive_write_set_format_zip.c
@@ -363,7 +363,7 @@ static int
 archive_write_zip_header(struct archive_write *a, struct archive_entry *entry)
 {
        unsigned char local_header[32];
-       unsigned char local_extra[64];
+       unsigned char local_extra[128];
        struct zip *zip = a->format_data;
        unsigned char *e;
        unsigned char *cd_extra;
@@ -632,6 +632,21 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry)
                archive_le16enc(zip64_start + 2, e - (zip64_start + 4));
        }
 
+       { /* Experimental 'at' extension to support streaming. */
+               unsigned char *external_info = e;
+               memcpy(e, "at\000\000", 4);
+               e += 4;
+               archive_le16enc(e, /* system + version written by */
+                   3 * 256 + version_needed);
+               e += 2;
+               archive_le16enc(e, 0); /* internal file attributes */
+               e += 2;
+               archive_le32enc(e,  /* external file attributes */
+                   archive_entry_mode(zip->entry) << 16);
+               e += 4;
+               archive_le16enc(external_info + 2, e - (external_info + 4));
+       }
+
        /* Update local header with size of extra data and write it all out: */
        archive_le16enc(local_header + 28, e - local_extra);
 

diff --git a/libarchive/test/test_write_format_zip.c b/libarchive/test/test_write_format_zip.c
index a21184402506e09104cf8445bea4034937fe652b..3a2a129fbf50f8b81ef3051643cf541e5ab41532 100644 (file)
--- a/libarchive/test/test_write_format_zip.c
+++ b/libarchive/test/test_write_format_zip.c
@@ -337,16 +337,9 @@ verify_contents(struct archive *a, int seeking)
        assertEqualInt(0, archive_entry_atime(ae));
        assertEqualInt(0, archive_entry_ctime(ae));
        assertEqualString("symlink", archive_entry_pathname(ae));
-       if (seeking) {
-               assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
-               assertEqualInt(0, archive_entry_size(ae));
-               assertEqualString("file1", archive_entry_symlink(ae));
-       } else {
-               /* Streaming cannot read file type, so
-                * symlink body shows as regular file contents. */
-               assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae));
-               assertEqualInt(5, archive_entry_size(ae));
-       }
+       assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
+       assertEqualInt(0, archive_entry_size(ae));
+       assertEqualString("file1", archive_entry_symlink(ae));
 
        /* Read the dir entry back. */
        assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));
@@ -421,16 +414,9 @@ verify_contents(struct archive *a, int seeking)
        assertEqualInt(0, archive_entry_atime(ae));
        assertEqualInt(0, archive_entry_ctime(ae));
        assertEqualString("symlink_deflate", archive_entry_pathname(ae));
-       if (seeking) {
-               assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
-               assertEqualInt(0, archive_entry_size(ae));
-               assertEqualString("file1", archive_entry_symlink(ae));
-       } else {
-               assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae));
-               assertEqualInt(5, archive_entry_size(ae));
-               assertEqualIntA(a, 5, archive_read_data(a, filedata, 10));
-               assertEqualMem(filedata, "file1", 5);
-       }
+       assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
+       assertEqualInt(0, archive_entry_size(ae));
+       assertEqualString("file1", archive_entry_symlink(ae));
 
        /* Read the dir entry back. */
        assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));
@@ -506,16 +492,9 @@ verify_contents(struct archive *a, int seeking)
        assertEqualInt(0, archive_entry_atime(ae));
        assertEqualInt(0, archive_entry_ctime(ae));
        assertEqualString("symlink_stored", archive_entry_pathname(ae));
-       if (seeking) {
-               assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
-               assertEqualInt(0, archive_entry_size(ae));
-               assertEqualString("file1", archive_entry_symlink(ae));
-       } else {
-               assertEqualInt(AE_IFREG | 0664, archive_entry_mode(ae));
-               assertEqualInt(5, archive_entry_size(ae));
-               assertEqualIntA(a, 5, archive_read_data(a, filedata, 10));
-               assertEqualMem(filedata, "file1", 5);
-       }
+       assertEqualInt(AE_IFLNK | 0755, archive_entry_mode(ae));
+       assertEqualInt(0, archive_entry_size(ae));
+       assertEqualString("file1", archive_entry_symlink(ae));
 
        /* Read the dir entry back. */
        assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae));

diff --git a/libarchive/test/test_write_format_zip_compression_store.c b/libarchive/test/test_write_format_zip_compression_store.c
index 4562acd1ef08e40c5d9e2b3bfcb5332fec9e7441..2aa4146e9e2cf1260f35cf1123a106db4d42e02f 100644 (file)
--- a/libarchive/test/test_write_format_zip_compression_store.c
+++ b/libarchive/test/test_write_format_zip_compression_store.c
@@ -71,7 +71,7 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        const char *buffend;
        /* p is the pointer to walk over the central directory,
         * q walks over the local headers, the data and the data descriptors. */
-       const char *p, *q;
+       const char *p, *q, *local_header, *extra_start;
        size_t used;
 
        /* File data */
@@ -198,7 +198,7 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        p = p + 4 + i2(p + 2);
 
        /* Verify local header of file entry. */
-       q = buff;
+       local_header = q = buff;
        assertEqualMem(q, "PK\003\004", 4); /* Signature */
        assertEqualInt(i2(q + 4), 10); /* Version needed to extract */
        assertEqualInt(i2(q + 6), 8); /* Flags */
@@ -209,15 +209,16 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        assertEqualInt(i4(q + 18), sizeof(file_data1) + sizeof(file_data2)); /* Compressed size */
        assertEqualInt(i4(q + 22), sizeof(file_data1) + sizeof(file_data2)); /* Uncompressed size */
        assertEqualInt(i2(q + 26), strlen(file_name)); /* Pathname length */
-       assertEqualInt(i2(q + 28), 28); /* Extra field length */
+       assertEqualInt(i2(q + 28), 40); /* Extra field length */
        assertEqualMem(q + 30, file_name, strlen(file_name)); /* Pathname */
-       q = q + 30 + strlen(file_name);
+       extra_start = q = q + 30 + strlen(file_name);
        assertEqualInt(i2(q), 0x5455); /* 'UT' extension header */
        assertEqualInt(i2(q + 2), 9); /* 'UT' size */
        assertEqualInt(q[4], 3); /* 'UT' flags */
        assertEqualInt(i4(q + 5), t); /* 'UT' mtime */
        assertEqualInt(i4(q + 9), t + 3); /* 'UT' atime */
        q = q + 4 + i2(q + 2);
+
        assertEqualInt(i2(q), 0x7875); /* 'ux' extension header */
        assertEqualInt(i2(q + 2), 11); /* 'ux' size */
        assertEqualInt(q[4], 1); /* 'ux' version */
@@ -227,6 +228,16 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        assertEqualInt(i4(q + 11), file_gid); /* 'Ux' GID */
        q = q + 4 + i2(q + 2);
 
+       assertEqualInt(i2(q), 0x7461); /* 'at' experimental extension header */
+       assertEqualInt(i2(q + 2), 8); /* 'at' size */
+       assertEqualInt(i2(q + 4), 3 * 256 + 10); /* version made by */
+       assertEqualInt(i2(q + 6), 0); /* internal file attributes */
+       assertEqualInt(i4(q + 8) >> 16 & 01777, file_perm); /* external file attributes */
+       q = q + 4 + i2(q + 2);
+
+       assert(q == extra_start + i2(local_header + 28));
+       q = extra_start + i2(local_header + 28);
+
        /* Verify data of file entry. */
        assertEqualMem(q, file_data1, sizeof(file_data1));
        assertEqualMem(q + sizeof(file_data1), file_data2, sizeof(file_data2));
@@ -276,6 +287,7 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        /*p = p + 4 + i2(p + 2);*/
 
        /* Verify local header of folder entry. */
+       local_header = q;
        assertEqualMem(q, "PK\003\004", 4); /* Signature */
        assertEqualInt(i2(q + 4), 20); /* Version needed to extract */
        assertEqualInt(i2(q + 6), 0); /* Flags */
@@ -286,9 +298,9 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        assertEqualInt(i4(q + 18), 0); /* Compressed size */
        assertEqualInt(i4(q + 22), 0); /* Uncompressed size */
        assertEqualInt(i2(q + 26), strlen(folder_name)); /* Pathname length */
-       assertEqualInt(i2(q + 28), 28); /* Extra field length */
+       assertEqualInt(i2(q + 28), 40); /* Extra field length */
        assertEqualMem(q + 30, folder_name, strlen(folder_name)); /* Pathname */
-       q = q + 30 + strlen(folder_name);
+       extra_start = q = q + 30 + strlen(folder_name);
        assertEqualInt(i2(q), 0x5455); /* 'UT' extension header */
        assertEqualInt(i2(q + 2), 9); /* 'UT' size */
        assertEqualInt(q[4], 5); /* 'UT' flags */
@@ -304,6 +316,16 @@ DEFINE_TEST(test_write_format_zip_compression_store)
        assertEqualInt(i4(q + 11), folder_gid); /* 'ux' GID */
        q = q + 4 + i2(q + 2);
 
+       assertEqualInt(i2(q), 0x7461); /* 'at' experimental extension header */
+       assertEqualInt(i2(q + 2), 8); /* 'at' size */
+       assertEqualInt(i2(q + 4), 3 * 256 + 20); /* version made by */
+       assertEqualInt(i2(q + 6), 0); /* internal file attributes */
+       assertEqualInt(i4(q + 8) >> 16 & 01777, folder_perm); /* external file attributes */
+       q = q + 4 + i2(q + 2);
+
+       assert(q == extra_start + i2(local_header + 28));
+       q = extra_start + i2(local_header + 28);
+
        /* There should not be any data in the folder entry,
         * so the first central directory entry should be next: */
        assertEqualMem(q, "PK\001\002", 4); /* Signature */

diff --git a/libarchive/test/test_write_format_zip_file.c b/libarchive/test/test_write_format_zip_file.c
index eef36c6290be4afbb820929263491d32c1e06b9d..7825ef55b7ea40d4bfee9627bb620849146ab825 100644 (file)
--- a/libarchive/test/test_write_format_zip_file.c
+++ b/libarchive/test/test_write_format_zip_file.c
@@ -196,7 +196,7 @@ DEFINE_TEST(test_write_format_zip_file)
        /* assertEqualInt(i4(p + 18), sizeof(file_data)); */ /* Compressed size */
        /* assertEqualInt(i4(p + 22), sizeof(file_data)); */ /* Uncompressed size not stored because we're using length-at-end. */
        assertEqualInt(i2(p + 26), strlen(file_name)); /* Pathname length */
-       assertEqualInt(i2(p + 28), 24); /* Extra field length */
+       assertEqualInt(i2(p + 28), 36); /* Extra field length */
        assertEqualMem(p + 30, file_name, strlen(file_name)); /* Pathname */
        p = extension_start = local_header + 30 + strlen(file_name);
        extension_end = extension_start + i2(local_header + 28);
@@ -216,6 +216,13 @@ DEFINE_TEST(test_write_format_zip_file)
        assertEqualInt(i4(p + 11), file_gid); /* 'Ux' GID */
        p += 4 + i2(p + 2);
 
+       assertEqualInt(i2(p), 0x7461); /* 'at' experimental extension header */
+       assertEqualInt(i2(p + 2), 8); /* 'at' size */
+       assertEqualInt(i2(p + 4), 3 * 256 + 20); /* version made by */
+       assertEqualInt(i2(p + 6), 0); /* internal file attributes */
+       assertEqualInt(i4(p + 8) >> 16 & 01777, file_perm); /* external file attributes */
+       p += 4 + i2(p + 2);
+
        /* Just in case: Report any extra extensions. */
        while (p < extension_end) {
                failure("Unexpected extension 0x%04X", i2(p));

diff --git a/libarchive/test/test_write_format_zip_file_zip64.c b/libarchive/test/test_write_format_zip_file_zip64.c
index 3cc2794c8d58ea823be7a784c171f49434d14a69..1f47040b1ed4794a78dca8d6fa0c21d638b772e9 100644 (file)
--- a/libarchive/test/test_write_format_zip_file_zip64.c
+++ b/libarchive/test/test_write_format_zip_file_zip64.c
@@ -223,7 +223,7 @@ DEFINE_TEST(test_write_format_zip_file_zip64)
        /* assertEqualInt(i4(p + 18), sizeof(file_data)); */ /* Compressed size */
        /* assertEqualInt(i4(p + 22), sizeof(file_data)); */ /* Uncompressed size not stored because we're using length-at-end. */
        assertEqualInt(i2(p + 26), strlen(file_name)); /* Pathname length */
-       assertEqualInt(i2(p + 28), 44); /* Extra field length */
+       assertEqualInt(i2(p + 28), 56); /* Extra field length */
        assertEqualMem(p + 30, file_name, strlen(file_name)); /* Pathname */
        p = extension_start = local_header + 30 + strlen(file_name);
        extension_end = extension_start + i2(local_header + 28);
@@ -249,6 +249,13 @@ DEFINE_TEST(test_write_format_zip_file_zip64)
        /* compressed file size we can't verify here */
        p += 4 + i2(p + 2);
 
+       assertEqualInt(i2(p), 0x7461); /* 'at' experimental extension header */
+       assertEqualInt(i2(p + 2), 8); /* 'at' size */
+       assertEqualInt(i2(p + 4), 3 * 256 + 45); /* version made by */
+       assertEqualInt(i2(p + 6), 0); /* internal file attributes */
+       assertEqualInt(i4(p + 8) >> 16 & 01777, file_perm); /* external file attributes */
+       p += 4 + i2(p + 2);
+
        /* Just in case: Report any extra extensions. */
        while (p < extension_end) {
                failure("Unexpected extension 0x%04X", i2(p));