gunzip: Implement chunked decompression

The current gzwrite() implementation is limited to 4 GiB compressed
input buffer size due to struct z_stream_s { uInt avail_in } member,
which is of type unsigned int. Current gzwrite() implementation sets
the entire input buffer size as avail_in and performs decompression
of the whole compressed input buffer in one round, which limits the
size of input buffer to 4 GiB.

Rework the decompression loop to use chunked approach, and decompress
the input buffer in up to 4 GiB - 1 kiB avail_in chunks, possibly in
multiple decompression rounds. This way, the compressed input buffer
size is limited by gzwrite() function 'len' parameter type, which is
unsigned long.

In case of sandbox build, include parsing of 'gzwrite_chunk'
environment variable, so the chunked approach can be thoroughly tested
with non default chunk size. For non-sandbox builds, the chunk size is
4 GiB - 1 kiB.

The gzwrite test case is extended to test various chunk sizes during
gzwrite decompression test.

Signed-off-by: Marek Vasut <marek.vasut+renesas@mailbox.org>

diff --git a/lib/gunzip.c b/lib/gunzip.c
index 76f3397fcedb4cc51fe775a8690e7c562eb4238b..20cc14f96882ff452f3e4a1470cd12e9c36736c3 100644 (file)
--- a/lib/gunzip.c
+++ b/lib/gunzip.c
@@ -8,8 +8,10 @@
 #include <command.h>
 #include <console.h>
 #include <div64.h>
+#include <env.h>
 #include <gzip.h>
 #include <image.h>
+#include <linux/sizes.h>
 #include <malloc.h>
 #include <memalign.h>
 #include <u-boot/crc.h>
@@ -119,7 +121,7 @@ void gzwrite_progress_finish(int returnval,
 int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
            size_t szwritebuf, off_t startoffs, size_t szexpected)
 {
-       int i, flags;
+       int flags;
        z_stream s;
        int r = 0;
        unsigned char *writebuf;
@@ -127,13 +129,23 @@ int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
        ulong totalfilled = 0;
        lbaint_t blksperbuf, outblock;
        u32 expected_crc;
-       size_t payload_size;
+       size_t i, payload_size;
+       unsigned long blocks_written;
+       lbaint_t writeblocks;
+       int numfilled = 0;
        int iteration = 0;
-
-       if (len > 0xffffffff) {
-               log_err("Input size over 4 GiB in size not supported\n");
-               return -1;
-       }
+       /*
+        * Allow runtime configuration of decompression chunk on
+        * sandbox to better cover the chunked decompression
+        * functionality without having to use > 4 GiB files.
+        */
+       const ulong minchunk = 0x400;
+       const ulong maxchunk = SZ_4G - minchunk;
+       const ulong chunk =
+               CONFIG_IS_ENABLED(SANDBOX,
+                                 (clamp(env_get_ulong("gzwrite_chunk", 10, maxchunk),
+                                        minchunk, maxchunk)),
+                                 (maxchunk));
 
        if (!szwritebuf ||
            (szwritebuf % dev->blksz) ||
@@ -175,7 +187,7 @@ int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
                return -1;
        }
 
-       payload_size = len - i - 8;
+       payload_size = len - i;
 
        memcpy(&expected_crc, src + len - 8, sizeof(expected_crc));
        expected_crc = le32_to_cpu(expected_crc);
@@ -205,35 +217,44 @@ int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
                return -1;
        }
 
-       s.next_in = src + i;
-       s.avail_in = payload_size+8;
+       src += i;
+       s.avail_in = 0;
        writebuf = (unsigned char *)malloc_cache_aligned(szwritebuf);
 
        /* decompress until deflate stream ends or end of file */
        do {
                if (s.avail_in == 0) {
-                       printf("%s: weird termination with result %d\n",
-                              __func__, r);
-                       break;
+                       if (payload_size == 0) {
+                               printf("%s: weird termination with result %d\n",
+                                      __func__, r);
+                               break;
+                       }
+
+                       s.next_in = src;
+                       s.avail_in = (payload_size > chunk) ? chunk : payload_size;
+                       src += s.avail_in;
+                       payload_size -= s.avail_in;
                }
 
                /* run inflate() on input until output buffer not full */
                do {
-                       unsigned long blocks_written;
-                       int numfilled;
-                       lbaint_t writeblocks;
-
-                       s.avail_out = szwritebuf;
-                       s.next_out = writebuf;
+                       if (numfilled) {
+                               s.avail_out = szwritebuf - numfilled;
+                               s.next_out = writebuf + numfilled;
+                       } else {
+                               s.avail_out = szwritebuf;
+                               s.next_out = writebuf;
+                       }
                        r = inflate(&s, Z_SYNC_FLUSH);
                        if ((r != Z_OK) &&
                            (r != Z_STREAM_END)) {
                                printf("Error: inflate() returned %d\n", r);
                                goto out;
                        }
+                       crc = crc32(crc, writebuf + numfilled,
+                                   szwritebuf - s.avail_out - numfilled);
+                       totalfilled += szwritebuf - s.avail_out - numfilled;
                        numfilled = szwritebuf - s.avail_out;
-                       crc = crc32(crc, writebuf, numfilled);
-                       totalfilled += numfilled;
                        if (numfilled < szwritebuf) {
                                writeblocks = (numfilled+dev->blksz-1)
                                                / dev->blksz;
@@ -241,14 +262,17 @@ int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
                                       dev->blksz-(numfilled%dev->blksz));
                        } else {
                                writeblocks = blksperbuf;
+                               numfilled = 0;
                        }
 
                        gzwrite_progress(iteration++,
                                         totalfilled,
                                         szexpected);
-                       blocks_written = blk_dwrite(dev, outblock,
+                       if (!numfilled) {
+                               blocks_written = blk_dwrite(dev, outblock,
                                                    writeblocks, writebuf);
-                       outblock += blocks_written;
+                               outblock += blocks_written;
+                       }
                        if (ctrlc()) {
                                puts("abort\n");
                                goto out;
@@ -258,6 +282,12 @@ int gzwrite(unsigned char *src, size_t len, struct blk_desc *dev,
                /* done when inflate() says it's done */
        } while (r != Z_STREAM_END);
 
+       if (numfilled) {
+               blocks_written = blk_dwrite(dev, outblock,
+                                   writeblocks, writebuf);
+               outblock += blocks_written;
+       }
+
        if ((szexpected != totalfilled) ||
            (crc != expected_crc))
                r = -1;

diff --git a/test/cmd/unzip.c b/test/cmd/unzip.c
index b67c5ba1956d40a57aa406f3667a3807ad7e85dd..623a2785884a3f66a00f0dbb9c9ac6d0b7efe3d9 100644 (file)
--- a/test/cmd/unzip.c
+++ b/test/cmd/unzip.c
@@ -105,7 +105,7 @@ static int dm_test_cmd_zip_gzwrite(struct unit_test_state *uts)
 {
        struct udevice *dev;
        ofnode root, node;
-       int i, ret;
+       int i, j, ret;
 
        /* Enable the mmc9 node for this test */
        root = oftree_root(oftree_default());
@@ -119,6 +119,16 @@ static int dm_test_cmd_zip_gzwrite(struct unit_test_state *uts)
                        return ret;
        }
 
+       /* Test various sizes of decompression chunk sizes */
+       for (j = 0; j < ARRAY_SIZE(sizes); j++) {
+               env_set_ulong("gzwrite_chunk", sizes[j]);
+               for (i = 0; i < ARRAY_SIZE(sizes); i++) {
+                       ret = do_test_cmd_zip_unzip(uts, sizes[i], true);
+                       if (ret)
+                               return ret;
+               }
+       }
+
        return 0;
 }
 DM_TEST(dm_test_cmd_zip_gzwrite, UTF_CONSOLE);