Fix off-by-one error in superblock mode (#3221)

Fixes #3212.

Long literal and match lengths had an off-by-one error in ZSTD_getSequenceLength.
Fix the off-by-one error, and add a golden compression test that catches the bug.
Also run all the golden tests in the cli-tests framework.

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 038232453f87530f0c3b255cfaa59c0a2d4173b1..e8922670283927e926d4a1783b446d9f351c28ff 100644 (file)
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -324,10 +324,10 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
     seqLen.matchLength = seq->mlBase + MINMATCH;
     if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
         if (seqStore->longLengthType == ZSTD_llt_literalLength) {
-            seqLen.litLength += 0xFFFF;
+            seqLen.litLength += 0x10000;
         }
         if (seqStore->longLengthType == ZSTD_llt_matchLength) {
-            seqLen.matchLength += 0xFFFF;
+            seqLen.matchLength += 0x10000;
         }
     }
     return seqLen;

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index 6ebda486c554243beb5a8830b31058cc030804a9..e1ff21582c5d2daeffbb695124bbf7cfe136ec25 100644 (file)
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -1578,7 +1578,7 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
     const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
     const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
     const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
-    DEBUGLOG(5, "ZSTD_decompressSequences_body");
+    DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
     (void)frame;
 
     /* Regen sequences */

diff --git a/tests/cli-tests/compression/golden.sh b/tests/cli-tests/compression/golden.sh
new file mode 100755 (executable)
index 0000000..85dd3fd
--- /dev/null
+++ b/tests/cli-tests/compression/golden.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+cp -r "$GOLDEN_DIR" golden/
+
+zstd -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/

diff --git a/tests/cli-tests/decompression/golden.sh b/tests/cli-tests/decompression/golden.sh
new file mode 100755 (executable)
index 0000000..36919e6
--- /dev/null
+++ b/tests/cli-tests/decompression/golden.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/"
+
+zstd -r -t "$GOLDEN_DIR"

diff --git a/tests/cli-tests/dictionaries/golden.sh b/tests/cli-tests/dictionaries/golden.sh
new file mode 100755 (executable)
index 0000000..85da2ee
--- /dev/null
+++ b/tests/cli-tests/dictionaries/golden.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
+
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst

diff --git a/tests/cli-tests/run.py b/tests/cli-tests/run.py
index b91aa984145f20998d26c99508ab7cc662d4c809..d726fba7f7d347ea93e0a3a59fba2ff74c44da7d 100755 (executable)
--- a/tests/cli-tests/run.py
+++ b/tests/cli-tests/run.py
@@ -699,6 +699,7 @@ if __name__ == "__main__":
     if args.exec_prefix is not None:
         env["EXEC_PREFIX"] = args.exec_prefix
     env["ZSTD_SYMLINK_DIR"] = zstd_symlink_dir
+    env["ZSTD_REPO_DIR"] = os.path.abspath(REPO_DIR)
     env["DATAGEN_BIN"] = os.path.abspath(args.datagen)
     env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep)
     env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless)

diff --git a/tests/golden-compression/large-literal-and-match-lengths b/tests/golden-compression/large-literal-and-match-lengths
new file mode 100644 (file)
index 0000000..fb63c32
Binary files /dev/null and b/tests/golden-compression/large-literal-and-match-lengths differ