IMPORT: slz/uslz: don't lose the bytes consumed by the header fast paths

The format autodetection in uslz_decode() has a fast path for the common
case where the first call carries enough data to parse the header directly
from the caller's buffer, and a slow path that accumulates the header into
hdr_detect.buf across calls. The fast path consumes bytes from the caller's
buffer *before* it knows whether it will be able to complete, and it had no
way to give them back, so three things went wrong:

1) Raw deflate (rfc1951) streams were losing their first two bytes. The
   fast path consumes two bytes to sniff the magic, and the fallback that
   feeds them back to the block decoder is guarded by "else if (buf_len)",
   which is false on exactly that path. Every raw stream fed more than 2
   bytes at a time therefore failed with E_CORRUPT (or E_INVALID_BLOCK_CODE,
   depending on what the two lost bytes happened to decode as), i.e. raw
   deflate was simply unusable:

       ./zdec < raw.deflate     ->  error (14)

   This is fixed by rewinding in_ptr to the start of the header, which
   also lets the fast path skip the local copy the accumulating path
   needs.

2) A gzip stream whose first call carried 3 to 9 bytes was mis-parsed.
   That is more than 2, so the fast path consumed the magic, but less
   than 10, so the 10-byte header could not be parsed, and it fell back
   to need_more_header, which starts filling hdr_detect.buf at buf_len=0,
   dropping the two bytes already consumed. The next call then parsed the
   header two bytes too far. This is fixed by rewinding before the
   fallback.
3) The tests on the remaining input used <compressed_size>, which is only
   decremented by the fast paths and never by the accumulating ones, so it
   overstates what is left in the caller's buffer as soon as the two are
   mixed. "if (compressed_size >= 2)" and "if (compressed_size >= xlen)"
   could therefore both succeed with fewer bytes actually available, read
   the FEXTRA length or payload **past in_top** and advance in_ptr beyond
   it. Fixed by testing state->in_top - state->in_ptr, which is exact on
   every path; <compressed_size> is no longer modified.

tests/uslztest.sh goes from 1108/1514 to 1438/1514, and raw, gzip, zlib
and slz streams now decode correctly at every input chunk size from 1 to
24 and beyond, for every supported ring size. The gzip variant carrying
FEXTRA is still broken at some chunk sizes, this is for a future fix.

This is libslz upstream commit cea782efdd111bf476b625051fe4d37cf28be92d

diff --git a/src/uslz.c b/src/uslz.c
index f0f513abad6251fcaeb03c0a74961209636de6b2..91b9684ea32ea0024f3ec4aef74a4ad61eec0f7e 100644 (file)
--- a/src/uslz.c
+++ b/src/uslz.c
@@ -1048,18 +1048,24 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state,
        if (state->state == USLZ_ST_INITIAL) {
                const unsigned char *input;
                unsigned int zlib_header;
+               /* First byte of the header in the caller's buffer. The fast
+                * paths below consume bytes from it before they know whether
+                * they will be able to complete the header, so they need to be
+                * able to give them back. Only meaningful while buf_len is 0,
+                * that is, as long as nothing has been accumulated yet.
+                */
+               const unsigned char *hdr_start = state->in_ptr;
 
                input = state->in_ptr;
 
                if ((state->flags & USLZ_FL_GZIP))
                        goto gzip_flags; // we already know it is gzip, keep parsing
 
-               if (compressed_size > 2 && !state->hdr_detect.buf_len) {
+               if (state->in_top - state->in_ptr > 2 && !state->hdr_detect.buf_len) {
                        /* enough data available in the header on first
                         * attempt, let's try with the input buffer directly.
                         */
                        state->in_ptr += 2;
-                       compressed_size -= 2;
                        goto detect;
                }
 
@@ -1084,11 +1090,18 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state,
                                /* go to gzip header parsing directly if there
                                 * is enough data to parse it the first time.
                                 */
-                               if (compressed_size >= 8) {
+                               if (state->in_top - state->in_ptr >= 8) {
                                        state->in_ptr += 8;
-                                       compressed_size -= 8;
                                        goto enough_gzip_header;
                                }
+                               /* Not enough data for the 10-byte header. Give
+                                * the two magic bytes consumed above back, so
+                                * that need_more_header accumulates the header
+                                * from its very first byte; otherwise they
+                                * would be dropped and the next call would
+                                * parse the header two bytes too far.
+                                */
+                               state->in_ptr = hdr_start;
                                goto need_more_header; // gzip header is 10 bytes min
                        }
 
@@ -1130,10 +1143,9 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state,
                                        /* go to gzip FEXTRA parsing directly if there is enough data
                                         * to parse it the first time.
                                         */
-                                       if (compressed_size >= 2) {
+                                       if (state->in_top - state->in_ptr >= 2) {
                                                input = state->in_ptr;
                                                state->in_ptr += 2;
-                                               compressed_size -= 2;
                                                goto enough_gzip_fextra_header;
                                        }
                                        goto need_more_header; // gzip FEXTRA FLEN is 2 bytes
@@ -1163,10 +1175,9 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state,
                                                /* go to gzip FEXTRA parsing directly if there is enough data
                                                 * to parse it the first time.
                                                 */
-                                               if (compressed_size >= xlen) {
+                                               if (state->in_top - state->in_ptr >= xlen) {
                                                        input = state->in_ptr;
                                                        state->in_ptr += xlen;
-                                                       compressed_size -= xlen;
                                                }
                                                else
                                                        goto need_more_header; // gzip FEXTRA FLEN is 2 bytes
@@ -1257,7 +1268,14 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state,
                        /* rfc1950/zlib starts with initial crc=1 */
                        state->crc = 1;
                        state->flags |= USLZ_FL_ZLIB;
-               } else if (state->hdr_detect.buf_len) {
+               } else if (!state->hdr_detect.buf_len) {
+                       /* Raw format detected on the fast path: the two bytes
+                        * consumed above are not a header at all, they are the
+                        * first two bytes of the deflate stream. Give them back
+                        * and let uslz_decode_block() read them normally.
+                        */
+                       state->in_ptr = hdr_start;
+               } else {
                        /* raw format, feed back the pending bytes to the stream,
                         * (should be 2 bytes at most) if stream is invalid it
                         * will be detected by tinflate_block().