[CI] Hook cli-tests up to CI

Add cli-tests to `make test`. This adds a `python3` dependency to `make
test`, but not `make check`. We could make this dependency optional by
skipping the tests if `python3` is not present.

diff --git a/tests/Makefile b/tests/Makefile
index 132fa7a081857cb18713dbb94489d9c61b28552b..cb77b0160a85a160effc984f2cfd449830a452b2 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -297,7 +297,7 @@ check: shortest
 fuzztest: test-fuzzer test-zstream test-decodecorpus
 
 .PHONY: test
-test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
 ifeq ($(QEMU_SYS),)
 test: test-pool
 endif
@@ -322,6 +322,12 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
        file $(ZSTD)
        EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
 
+test-cli-tests: ZSTD = $(PRGDIR)/zstd
+test-cli-tests: zstd datagen
+       file $(ZSTD)
+       ./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
+
+
 test-fullbench: fullbench datagen
        $(QEMU_SYS) ./fullbench -i1
        $(QEMU_SYS) ./fullbench -i1 -P0

diff --git a/tests/cli-tests/basic/help.sh b/tests/cli-tests/basic/help.sh
index c683b6a6a72aeb2d0cef951638229c89fa797b21..927c3ffbcc52118e1150eafc4fd7dee1d0d34d90 100755 (executable)
--- a/tests/cli-tests/basic/help.sh
+++ b/tests/cli-tests/basic/help.sh
@@ -1,4 +1,7 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 println "+ zstd -h"
 zstd -h
 println "+ zstd -H"

diff --git a/tests/cli-tests/basic/version.sh b/tests/cli-tests/basic/version.sh
index d50de0f3cacad3afd5db887a0f297f813a1fb582..f75eaa84fb4dfbe2ad198b2b5a9b6c014c9a125c 100755 (executable)
--- a/tests/cli-tests/basic/version.sh
+++ b/tests/cli-tests/basic/version.sh
@@ -1,3 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 zstd -V
 zstd --version

diff --git a/tests/cli-tests/bin/cmp_size b/tests/cli-tests/bin/cmp_size
index 5afa1c590adda8b727f792811ba4d765770c3c52..8e4bef88eabc2428ce4aa73f4d4f178dbb3757d3 100755 (executable)
--- a/tests/cli-tests/bin/cmp_size
+++ b/tests/cli-tests/bin/cmp_size
@@ -1,7 +1,5 @@
 #!/bin/sh
 
-# Small utility to
-
 set -e
 
 usage()

diff --git a/tests/cli-tests/bin/println b/tests/cli-tests/bin/println
index 1da2460471b89efd884e72c21fc48b7e0171a7b9..494eb18c850da801b70dded1a1d099707e039b15 100755 (executable)
--- a/tests/cli-tests/bin/println
+++ b/tests/cli-tests/bin/println
@@ -1,2 +1,2 @@
-#!/bin/env sh
+#!/bin/sh
 printf '%b\n' "${*}"

diff --git a/tests/cli-tests/common/format.sh b/tests/cli-tests/common/format.sh
index 20ff0f0567240ca2e4b689015c117afc37425a03..e574e973075c8166b5814dea515cc97e03a61cb6 100644 (file)
--- a/tests/cli-tests/common/format.sh
+++ b/tests/cli-tests/common/format.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 zstd_supports_format()
 {

diff --git a/tests/cli-tests/common/mtime.sh b/tests/cli-tests/common/mtime.sh
index 7ce931a96c0040fa2bdf1ecea155259a03eefdcb..344074d398c9c0cdc245a1f51b37c7c45dab2c52 100644 (file)
--- a/tests/cli-tests/common/mtime.sh
+++ b/tests/cli-tests/common/mtime.sh
@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 MTIME="stat -c %Y"
 case "$UNAME" in

diff --git a/tests/cli-tests/common/permissions.sh b/tests/cli-tests/common/permissions.sh
index b1f6ea3ba70461f7bd7b2c433379104c5d2981b5..6bce1f0b387dcd3ea2bd92a2623bb4fd5aa6e616 100644 (file)
--- a/tests/cli-tests/common/permissions.sh
+++ b/tests/cli-tests/common/permissions.sh
@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 GET_PERMS="stat -c %a"
 case "$UNAME" in

diff --git a/tests/cli-tests/compression/basic.sh b/tests/cli-tests/compression/basic.sh
index 6f0f87932fbabada6a58dd04e8545271dedab11b..8b63e40760cc174807a79d248f2eb932f66dcd36 100755 (executable)
--- a/tests/cli-tests/compression/basic.sh
+++ b/tests/cli-tests/compression/basic.sh
@@ -1,7 +1,9 @@
-#!/bin/sh -e
+#!/bin/sh
 
-# Uncomment the set -x line for debugging
-# set -x
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
 
 # Test compression flags and check that they work
 zstd file                 ; zstd -t file.zst

diff --git a/tests/cli-tests/compression/format.sh b/tests/cli-tests/compression/format.sh
index 86fb400809c2eef82296b9284f3a33fa509e7a60..192fa2cf29fb27c05a77879b59446ae23c787e2d 100755 (executable)
--- a/tests/cli-tests/compression/format.sh
+++ b/tests/cli-tests/compression/format.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/format.sh"
+. "$COMMON/format.sh"
 
 set -e
 

diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 6bd0aca00786bb34e9480b2382f1a388deb4a762..4837790ce5d9d7ce77447c41eb239701fec5208c 100755 (executable)
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 
 set -e
-set -x
+set -v
 
 datagen > file
 
@@ -12,7 +12,7 @@ zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
 zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-{f10,f1,1,19,22}.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
 
 cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
@@ -41,10 +41,10 @@ ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
 ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
 ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 
-cmp file-f10{,-env}.zst || die "Environment variable failed to set level"
-cmp file-1{,-env}.zst   || die "Environment variable failed to set level"
-cmp file-19{,-env}.zst  || die "Environment variable failed to set level"
-cmp file-99{,-env}.zst  || die "Environment variable failed to set level"
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
 zstd -f file
@@ -60,5 +60,5 @@ ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
 ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
 
-cmp file-1{,-env}.zst  || die "Environment variable not overridden"
-cmp file-f1{,-env}.zst || die "Environment variable not overridden"
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index c0b7066fcba67bec0e47bfb8fca6975cc37b47c8..cb00433e6e16e8bc09bee21adebfb1bbaca39c83 100644 (file)
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -1,75 +1,71 @@
-+ datagen
-+ zstd --fast=10 file -o file-f10.zst
-+ zstd --fast=1 file -o file-f1.zst
-+ zstd -1 file -o file-1.zst
-+ zstd -19 file -o file-19.zst
-+ zstd -22 --ultra file -o file-22.zst
-+ zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
-+ cmp_size -ne file-19.zst file-22.zst
-+ cmp_size -lt file-19.zst file-1.zst
-+ cmp_size -lt file-1.zst file-f1.zst
-+ cmp_size -lt file-f1.zst file-f10.zst
-+ zstd --fast file -f
-+ cmp file.zst file-f1.zst
-+ zstd -0 file -o file-0.zst
-+ zstd -f file
-+ cmp file.zst file-0.zst
-+ zstd -99 file -o file-99.zst
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst
+zstd --fast=1 file -o file-f1.zst
+zstd -1 file -o file-1.zst
+zstd -19 file -o file-19.zst
+zstd -22 --ultra file -o file-22.zst
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+
+cmp_size -ne file-19.zst file-22.zst
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst
+zstd -f file
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-19.zst file-99.zst
-+ zstd --fast=200000 file -c
-+ zstd -t
-+ zstd -5000000000 -f file
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ zstd --fast=5000000000 -f file
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ ZSTD_CLEVEL=-10
-+ zstd file -o file-f10-env.zst
-+ ZSTD_CLEVEL=1
-+ zstd file -o file-1-env.zst
-+ ZSTD_CLEVEL=+19
-+ zstd file -o file-19-env.zst
-+ ZSTD_CLEVEL=+99
-+ zstd file -o file-99-env.zst
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-f10.zst file-f10-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-19.zst file-19-env.zst
-+ cmp file-99.zst file-99-env.zst
-+ zstd -f file
-+ ZSTD_CLEVEL=-
-+ zstd -f file -o file-env.zst
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=-a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=3a7
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=5000000000
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file -1 -o file-1-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file --fast=1 -o file-f1-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-f1.zst file-f1-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

diff --git a/tests/cli-tests/dict-builder/no-inputs b/tests/cli-tests/dict-builder/no-inputs.sh
similarity index 76%
rename from tests/cli-tests/dict-builder/no-inputs
rename to tests/cli-tests/dict-builder/no-inputs.sh
index d37fdce5a205e68d6ef77f47f05878f49bd662e2..416b83742f2bd0d500ad828612bbe6a8b5154f54 100755 (executable)
--- a/tests/cli-tests/dict-builder/no-inputs
+++ b/tests/cli-tests/dict-builder/no-inputs.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
-set -x
+set -v
 zstd --train

diff --git a/tests/cli-tests/dict-builder/no-inputs.exit b/tests/cli-tests/dict-builder/no-inputs.sh.exit
similarity index 100%
rename from tests/cli-tests/dict-builder/no-inputs.exit
rename to tests/cli-tests/dict-builder/no-inputs.sh.exit

diff --git a/tests/cli-tests/dict-builder/no-inputs.stderr.exact b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
similarity index 93%
rename from tests/cli-tests/dict-builder/no-inputs.stderr.exact
rename to tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
index b3b5599d5aeaf537e1c536512413142c22a5e615..d7b3ea020f761f50bd6c522a791d030ad5b2de59 100644 (file)
--- a/tests/cli-tests/dict-builder/no-inputs.stderr.exact
+++ b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
@@ -1,4 +1,4 @@
-+ zstd --train
+zstd --train
 !  Warning : nb of samples too low for proper processing ! 
 !  Please provide _one file per sample_. 
 !  Alternatively, split files into fixed-size blocks representative of samples, with -B# 

diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
index 2b5d5a0d227267bfbe89b1f27ba91faade0086b7..8264ccca5a2f06cb7bb5d0e826e3c944b5547c66 100755 (executable)
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 set -e
 
@@ -22,7 +22,7 @@ if [ false ]; then
        datagen -g1000 -s0 > file0
 fi
 
-set -x
+set -v
 zstd files/0 -D dicts/0
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:

diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
index 399a3207d26d3e392f6e1c62fd32e0668de65e18..0afea722e984dd64296de1aecb208d8c9c2d3cf1 100644 (file)
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@@ -1,8 +1,6 @@
-+ zstd files/0 -D dicts/0
-+ zstd -t files/0.zst -D dicts/0
-+ zstd -t files/0.zst -D dicts/1
+zstd files/0 -D dicts/0
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
-+ zstd -t files/0.zst
+zstd -t files/0.zst            && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :

diff --git a/tests/cli-tests/dictionaries/setup_once b/tests/cli-tests/dictionaries/setup_once
index 6316df165fa79bc81d9fa7797ca969db0e03eb64..1241c5782147f00e0f13a00814b99e1e499257fb 100755 (executable)
--- a/tests/cli-tests/dictionaries/setup_once
+++ b/tests/cli-tests/dictionaries/setup_once
@@ -2,7 +2,7 @@
 
 set -e
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 
 mkdir files/ dicts/