[libzstd] Fix bug in Huffman decompresser

The zstd format specification doesn't enforce that Huffman compressed
literals (including the table) have to be smaller than the uncompressed
literals. The compressor will never Huffman compress literals if the
compressed size is larger than the uncompressed size. The decompresser
doesn't accept Huffman compressed literals with 4 streams whose compressed
size is at least as large as the uncompressed size.

* Make the decompresser accept Huffman compressed literals whose size
  increases.
* Add a test case that exposes the bug. The compressed file has to be
  statically generated, since the compressor won't normally produce files
  that expose the bug.

diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 2a1b70ea5ef25e2fe16d567cd31c76da958941e6..0a47a3d74035aa29f9e8884c8a184e0d3f2c7163 100644 (file)
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -917,11 +917,11 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
 *   Tells which decoder is likely to decode faster,
 *   based on a set of pre-determined metrics.
 *   @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
-*   Assumption : 0 < cSrcSize < dstSize <= 128 KB */
+*   Assumption : 0 < cSrcSize, dstSize <= 128 KB */
 U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
 {
     /* decoder timing evaluation */
-    U32 const Q = (U32)(cSrcSize * 16 / dstSize);   /* Q < 16 since dstSize > cSrcSize */
+    U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize);   /* Q < 16 */
     U32 const D256 = (U32)(dstSize >> 8);
     U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
     U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
@@ -977,7 +977,7 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected);   /* invalid */
+    if (cSrcSize == 0) return ERROR(corruption_detected);
 
     {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
         return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):

diff --git a/tests/files/huffman-compressed-larger b/tests/files/huffman-compressed-larger
new file mode 100644 (file)
index 0000000..f594f1a
Binary files /dev/null and b/tests/files/huffman-compressed-larger differ

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 77853b1a4c01122d479f53aeb5bf2e2c383eb1a3..bc8584e7a9f08ca547b6c3bb034c59b4beaf13ac 100755 (executable)
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -386,6 +386,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
 ./datagen | $ZSTD -c | $ZSTD -t
 
 
+
+$ECHO "\n**** golden files tests **** "
+
+$ZSTD -t -r files
+$ZSTD -c -r files | $ZSTD -t
+
+
 $ECHO "\n**** benchmark mode tests **** "
 
 $ECHO "bench one file"