IMPORT: slz: implement a synchronous flush() operation

In some cases it may be desirable for latency reasons to forcefully
flush the queue even if it results in suboptimal compression. In our
case the queue might contain up to almost 4 bytes, which need an EOB
and a switch to literal mode, followed by 4 bytes to encode an empty
message. This means that each call can add 5 extra bytes in the ouput
stream. And the flush may also result in the header being produced for
the first time, which can amount to 2 or 10 bytes (zlib or gzip). In
the worst case, a total of 19 bytes may be emitted at once upon a flush
with 31 pending bits and a gzip header.

This is libslz upstream commit cf8c4668e4b4216e930b56338847d8d46a6bfda9.

diff --git a/include/import/slz.h b/include/import/slz.h
index d01599ca442a008bdf2de7937c0aff313f5dbeb4..901a79027bcef4e25dcab204e8bebc4b1a1e2468 100644 (file)
--- a/include/import/slz.h
+++ b/include/import/slz.h
@@ -86,6 +86,7 @@ struct slz_stream {
 /* Functions specific to rfc1951 (deflate) */
 long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more);
 int slz_rfc1951_init(struct slz_stream *strm, int level);
+int slz_rfc1951_flush(struct slz_stream *strm, unsigned char *buf);
 int slz_rfc1951_finish(struct slz_stream *strm, unsigned char *buf);
 
 /* Functions specific to rfc1952 (gzip) */
@@ -94,6 +95,7 @@ uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len);
 long slz_rfc1952_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more);
 int slz_rfc1952_send_header(struct slz_stream *strm, unsigned char *buf);
 int slz_rfc1952_init(struct slz_stream *strm, int level);
+int slz_rfc1952_flush(struct slz_stream *strm, unsigned char *buf);
 int slz_rfc1952_finish(struct slz_stream *strm, unsigned char *buf);
 
 /* Functions specific to rfc1950 (zlib) */
@@ -102,6 +104,7 @@ uint32_t slz_adler32_block(uint32_t crc, const unsigned char *buf, long len);
 long slz_rfc1950_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more);
 int slz_rfc1950_send_header(struct slz_stream *strm, unsigned char *buf);
 int slz_rfc1950_init(struct slz_stream *strm, int level);
+int slz_rfc1950_flush(struct slz_stream *strm, unsigned char *buf);
 int slz_rfc1950_finish(struct slz_stream *strm, unsigned char *buf);
 
 /* generic functions */
@@ -170,4 +173,28 @@ static inline int slz_finish(struct slz_stream *strm, void *buf)
        return ret;
 }
 
+/* Flushes any pending data for stream <strm> into buffer <buf>, then emits an
+ * empty literal block to byte-align the output, allowing to completely flush
+ * the queue. Note that if the initial header was never sent, it will be sent
+ * first as well (0, 2 or 10 extra bytes). This requires that the output buffer
+ * still has this plus the size of the queue available (up to 4 bytes), plus
+ * one byte for (BFINAL,BTYPE), plus 4 bytes for LEN+NLEN, or a total of 19
+ * bytes in the worst case. The number of bytes emitted is returned. It is
+ * guaranteed that the queue is empty on return. This may cause some overhead
+ * by adding needless 5-byte blocks if called to often.
+ */
+static inline int slz_flush(struct slz_stream *strm, void *buf)
+{
+       int ret;
+
+       if (strm->format == SLZ_FMT_GZIP)
+               ret = slz_rfc1952_flush(strm, (unsigned char *) buf);
+       else if (strm->format == SLZ_FMT_ZLIB)
+               ret = slz_rfc1950_flush(strm, (unsigned char *) buf);
+       else /* deflate for other ones */
+               ret = slz_rfc1951_flush(strm, (unsigned char *) buf);
+
+       return ret;
+}
+
 #endif

diff --git a/src/slz.c b/src/slz.c
index 1d6032ea2e9294858b33314d998081075fbfea11..1560bac343fb76bb23d1b15b5b1ce8bfd04a9a68 100644 (file)
--- a/src/slz.c
+++ b/src/slz.c
@@ -720,6 +720,42 @@ int slz_rfc1951_init(struct slz_stream *strm, int level)
        return 0;
 }
 
+/* Flushes any pending data for stream <strm> into buffer <buf>, then emits an
+ * empty literal block to byte-align the output, allowing to completely flush
+ * the queue. This requires that the output buffer still has the size of the
+ * queue available (up to 4 bytes), plus one byte for (BFINAL,BTYPE), plus 4
+ * bytes for LEN+NLEN, or a total of 9 bytes in the worst case. The number of
+ * bytes emitted is returned. It is guaranteed that the queue is empty on
+ * return. This may cause some overhead by adding needless 5-byte blocks if
+ * called to often.
+ */
+int slz_rfc1951_flush(struct slz_stream *strm, unsigned char *buf)
+{
+       strm->outbuf = buf;
+
+       /* The queue is always empty on INIT, DONE, and END */
+       if (!strm->qbits)
+               return 0;
+
+       /* we may need to terminate a huffman output. Lit is always in EOB state */
+       if (strm->state != SLZ_ST_EOB) {
+               strm->state = (strm->state == SLZ_ST_LAST) ? SLZ_ST_DONE : SLZ_ST_EOB;
+               send_eob(strm);
+       }
+
+       /* send BFINAL according to state, and BTYPE=00 (lit) */
+       enqueue8(strm, (strm->state == SLZ_ST_DONE) ? 1 : 0, 3);
+       flush_bits(strm);             // emit pending bits
+       copy_32b(strm, 0xFFFF0000U);  // len=0, nlen=~0
+
+       /* Now the queue is empty, EOB was sent, BFINAL might have been sent if
+        * we completed the last block, and a zero-byte block was sent to byte-
+        * align the output. The last state reflects all this. Let's just
+        * return the number of bytes added to the output buffer.
+        */
+       return strm->outbuf - buf;
+}
+
 /* Flushes any pending for stream <strm> into buffer <buf>, then sends BTYPE=1
  * and BFINAL=1 if needed. The stream ends in SLZ_ST_DONE. It returns the number
  * of bytes emitted. The trailer consists in flushing the possibly pending bits
@@ -1053,6 +1089,27 @@ int slz_rfc1952_init(struct slz_stream *strm, int level)
        return 0;
 }
 
+/* Flushes any pending data for stream <strm> into buffer <buf>, then emits an
+ * empty literal block to byte-align the output, allowing to completely flush
+ * the queue. Note that if the initial header was never sent, it will be sent
+ * first as well (10 extra bytes). This requires that the output buffer still
+ * has this plus the size of the queue available (up to 4 bytes), plus one byte
+ * for (BFINAL,BTYPE), plus 4 bytes for LEN+NLEN, or a total of 19 bytes in the
+ * worst case. The number of bytes emitted is returned. It is guaranteed that
+ * the queue is empty on return. This may cause some overhead by adding
+ * needless 5-byte blocks if called to often.
+ */
+int slz_rfc1952_flush(struct slz_stream *strm, unsigned char *buf)
+{
+       int sent = 0;
+
+       if (__builtin_expect(strm->state == SLZ_ST_INIT, 0))
+               sent = slz_rfc1952_send_header(strm, buf);
+
+       sent += slz_rfc1951_flush(strm, buf + sent);
+       return sent;
+}
+
 /* Flushes pending bits and sends the gzip trailer for stream <strm> into
  * buffer <buf>. When it's done, the stream state is updated to SLZ_ST_END. It
  * returns the number of bytes emitted. The trailer consists in flushing the
@@ -1308,6 +1365,27 @@ int slz_rfc1950_init(struct slz_stream *strm, int level)
        return 0;
 }
 
+/* Flushes any pending data for stream <strm> into buffer <buf>, then emits an
+ * empty literal block to byte-align the output, allowing to completely flush
+ * the queue. Note that if the initial header was never sent, it will be sent
+ * first as well (2 extra bytes). This requires that the output buffer still
+ * has this plus the size of the queue available (up to 4 bytes), plus one byte
+ * for (BFINAL,BTYPE), plus 4 bytes for LEN+NLEN, or a total of 11 bytes in the
+ * worst case. The number of bytes emitted is returned. It is guaranteed that
+ * the queue is empty on return. This may cause some overhead by adding
+ * needless 5-byte blocks if called to often.
+ */
+int slz_rfc1950_flush(struct slz_stream *strm, unsigned char *buf)
+{
+       int sent = 0;
+
+       if (__builtin_expect(strm->state == SLZ_ST_INIT, 0))
+               sent = slz_rfc1950_send_header(strm, buf);
+
+       sent += slz_rfc1951_flush(strm, buf + sent);
+       return sent;
+}
+
 /* Flushes pending bits and sends the gzip trailer for stream <strm> into
  * buffer <buf>. When it's done, the stream state is updated to SLZ_ST_END. It
  * returns the number of bytes emitted. The trailer consists in flushing the