make-all:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make all
run: make all
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make test
run: make test
make-test-osx:
runs-on: macos-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: OS-X test
run: make test # make -c lib all doesn't work because of the fact that it's not a tty
no-intrinsics-fuzztest:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: no intrinsics fuzztest
run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
tsan-zstreamtest:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: thread sanitizer zstreamtest
run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
+ ubsan-zstreamtest:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: undefined behavior sanitizer zstreamtest
+ run: CC=clang make uasan-test-zstream
+
# lasts ~15mn
tsan-fuzztest:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Test Zstd
run: |
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
+ clang-asan-ubsan-testzstd:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: clang + ASan + UBSan + Test Zstd
+ run: CC=clang make -j uasan-test-zstd </dev/null V=1
+
gcc-asan-ubsan-testzstd-32bit:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: ASan + UBSan + Test Zstd, 32bit mode
run: |
sudo apt-get -qqq update
gcc-8-asan-ubsan-fuzz:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Fuzz Test
run: |
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+ clang-asan-ubsan-fuzz:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: clang + ASan + UBSan + Fuzz Test
+ run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
gcc-asan-ubsan-fuzz32:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: ASan + UBSan + Fuzz Test 32bit
run: |
sudo apt-get -qqq update
make libc6install
CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+ clang-asan-ubsan-fuzz32:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: clang + ASan + UBSan + Fuzz Test 32bit
+ run: |
+ sudo apt-get -qqq update
+ make libc6install
+ CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
asan-ubsan-regression:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: ASan + UBSan + Regression Test
run: make -j uasanregressiontest
+ clang-ubsan-regression:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: clang + ASan + UBSan + Regression Test
+ run: CC=clang make -j uasanregressiontest
+
msan-regression:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: MSan + Regression Test
run: make -j msanregressiontest
clang-msan-fuzz:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
clang-msan-testzstd:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: clang + MSan + Test Zstd
run: |
sudo apt-get update
armfuzz:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Qemu ARM emulation + Fuzz Test
run: |
sudo apt-get -qqq update
valgrind-fuzz-test:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: valgrind + fuzz test stack mode # ~ 7mn
shell: 'script -q -e -c "bash {0}"'
run: |
sudo apt-get -qqq update
make valgrindinstall
- make -C tests valgrindTest
+ make -C tests test-valgrind
make clean
make -C tests test-fuzzer-stackmode
{ compiler: gcc, platform: x64, action: test, script: ""},
]
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Mingw long test
run: |
$env:PATH_ORIGINAL = $env:PATH
linux-kernel:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: linux kernel, library + build + test
run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
benchmarking:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make benchmarking
run: make benchmarking
check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make check on 32-bit
run: |
sudo apt update
check-x32:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
env:
CHECK_CONSTRAINED_MEM: true
gcc-7-libzstd:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: gcc-7 + libzstdmt compilation
run: |
sudo apt-get -qqq update
cmake-build-and-test-check:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: cmake build and test check
run: |
FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
cpp-gnu90-c99-compatibility:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: C++, gnu90 and c99 compatibility
run: |
make cxxtest
mingw-cross-compilation:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: mingw cross-compilation
run: |
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
armbuild:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: ARM Build Test
run: |
sudo apt-get -qqq update
zlib-wrapper:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: zlib wrapper test
run: |
sudo apt-get -qqq update
make valgrindinstall
make -C zlibWrapper test
- make -C zlibWrapper valgrindTest
+ make -C zlibWrapper test-valgrind
lz4-threadpool-libs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: LZ4, thread pool, and libs build testslib wrapper test
run: |
make lz4install
gcc-make-tests-32bit:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Make all, 32bit mode
run: |
sudo apt-get -qqq update
gcc-8-make:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: gcc-8 build
run: |
sudo apt-get -qqq update
implicit-fall-through:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: -Wimplicit-fallthrough build
run: |
make clean
flags: "-A Win32"
- generator: "MinGW Makefiles"
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
- name: Build
platform: [x64, Win32]
configuration: [Debug, Release]
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
- name: Build
# platform: [x64, Win32]
# configuration: [Debug, Release]
# steps:
-# - uses: actions/checkout@v2
+# - uses: actions/checkout@v3
# - name: Add MSBuild to PATH
# uses: microsoft/setup-msbuild@v1.0.2
# - name: Build
minimal-decompressor-macros:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: minimal decompressor macros
run: |
make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
dynamic-bmi2:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: dynamic bmi2 tests
run: |
make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
test-variants:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: make all variants & validate
run: |
make -j -C programs allVariants MOREFLAGS=-O0
XCC: ${{ matrix.xcc }}
XEMU: ${{ matrix.xemu }}
steps:
- - uses: actions/checkout@v2 # https://github.com/actions/checkout
+ - uses: actions/checkout@v3
- name: apt update & install
run: |
sudo apt-get update
{ compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
]
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Mingw short test
run: |
ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
platform: [x64, Win32]
configuration: [Release]
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
- name: Build and run tests
intel-cet-compatibility:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Build Zstd
run: |
make -j zstd V=1
run: |
sde-external-9.0.0-2021-11-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
+ pkg-config:
+ runs-on: ubuntu-latest
+ container:
+ image: debian:testing
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install dependencies
+ run: |
+ apt -y update
+ apt -y install --no-install-recommends gcc libc6-dev make pkg-config
+ - name: Build and install
+ run: make -C lib install
+ - name: Test pkg-config
+ run: |
+ cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
+ ./simple LICENSE
# This test currently fails on Github Actions specifically.
# versions-compatibility:
# runs-on: ubuntu-latest
# steps:
-# - uses: actions/checkout@v2
+# - uses: actions/checkout@v3
# - name: Versions Compatibility Test
# run: |
# make -C tests versionsTest
# sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
# sudo apt-get update
# sudo apt-get install -y intel-basekit intel-hpckit
-# - uses: actions/checkout@v2
+# - uses: actions/checkout@v3
# - name: make check
# run: |
# make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
types:
- published
+permissions:
+ contents: read
+
jobs:
publish-release-artifacts:
+ permissions:
+ contents: read # to fetch code (actions/checkout)
+ actions: write # to attach binaries to release artifacts (skx/github-action-publish-binaries)
+
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: Archive
env:
--- /dev/null
+name: Scorecards supply-chain security
+on:
+ # Only the default branch is supported.
+ branch_protection_rule:
+ schedule:
+ - cron: '22 21 * * 2'
+ push:
+ # TODO: Add release branch when supported?
+ branches: [ "dev" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+ analysis:
+ name: Scorecards analysis
+ if: github.repository == 'facebook/zstd'
+ runs-on: ubuntu-latest
+ permissions:
+ # Needed to upload the results to code-scanning dashboard.
+ security-events: write
+ # Used to receive a badge.
+ id-token: write
+ # Needs for private repositories.
+ contents: read
+ actions: read
+
+ steps:
+ - name: "Checkout code"
+ uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0
+ with:
+ persist-credentials: false
+
+ - name: "Run analysis"
+ uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
+ with:
+ results_file: results.sarif
+ results_format: sarif
+ # (Optional) Read-only PAT token. Uncomment the `repo_token` line below if:
+ # - you want to enable the Branch-Protection check on a *public* repository, or
+ # - you are installing Scorecards on a *private* repository
+ # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+ # repo_token: ${{ secrets.SCORECARD_READ_TOKEN }}
+
+ # Publish the results for public repositories to enable scorecard badges. For more details, see
+ # https://github.com/ossf/scorecard-action#publishing-results.
+ # For private repositories, `publish_results` will automatically be set to `false`, regardless
+ # of the value entered here.
+ publish_results: true
+
+ # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+ # format to the repository Actions tab.
+ - name: "Upload artifact"
+ uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0
+ with:
+ name: SARIF file
+ path: results.sarif
+ retention-days: 5
+
+ # Upload the results to GitHub's code scanning dashboard.
+ - name: "Upload to code-scanning"
+ uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26
+ with:
+ sarif_file: results.sarif
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
- `tests/playTests.sh --test-large-data`
- Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
-- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
+- Valgrind Test (`make -C tests test-valgrind`) (testing CLI and fuzzer under `valgrind`)
- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
Long Tests
--- /dev/null
+# This module provides function for joining paths
+# known from most languages
+#
+# SPDX-License-Identifier: (MIT OR CC0-1.0)
+# Copyright 2020 Jan Tojnar
+# https://github.com/jtojnar/cmake-snips
+#
+# Modelled after Python’s os.path.join
+# https://docs.python.org/3.7/library/os.path.html#os.path.join
+# Windows not supported
+function(join_paths joined_path first_path_segment)
+ set(temp_path "${first_path_segment}")
+ foreach(current_segment IN LISTS ARGN)
+ if(NOT ("${current_segment}" STREQUAL ""))
+ if(IS_ABSOLUTE "${current_segment}")
+ set(temp_path "${current_segment}")
+ else()
+ set(temp_path "${temp_path}/${current_segment}")
+ endif()
+ endif()
+ endforeach()
+ set(${joined_path} "${temp_path}" PARENT_SCOPE)
+endfunction()
OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
endif ()
-if (UNIX OR MINGW)
- # pkg-config
- set(PREFIX "${CMAKE_INSTALL_PREFIX}")
- set(EXEC_PREFIX "\${prefix}")
- set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
- set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
- set(VERSION "${zstd_VERSION}")
-
- string(LENGTH "${PREFIX}" PREFIX_LENGTH)
- string(SUBSTRING "${LIBDIR}" 0 ${PREFIX_LENGTH} LIBDIR_PREFIX)
- string(SUBSTRING "${LIBDIR}" ${PREFIX_LENGTH} -1 LIBDIR_SUFFIX)
- string(SUBSTRING "${INCLUDEDIR}" 0 ${PREFIX_LENGTH} INCLUDEDIR_PREFIX)
- string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
-
- if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
- set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
- endif()
- if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
- set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
- endif()
-
- configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
- install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-endif ()
+# pkg-config
+include(JoinPaths) # can be replaced by cmake_path(APPEND) in CMake 3.20
+set(PREFIX "${CMAKE_INSTALL_PREFIX}")
+set(EXEC_PREFIX "\${prefix}")
+join_paths(LIBDIR "\${exec_prefix}" "${CMAKE_INSTALL_LIBDIR}")
+join_paths(INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+set(LIBS_PRIVATE "${THREADS_LIBS}")
+set(VERSION "${zstd_VERSION}")
+
+configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
# install target
install(FILES
# Tool to bundle multiple C/C++ source files, inlining any includes.
#
-# Note: this POSIX-compliant script is many times slower than the original bash
-# implementation (due to the grep calls) but it runs and works everywhere.
-#
# TODO: ROOTS, FOUND, etc., as arrays (since they fail on paths with spaces)
-# TODO: revert to Bash-only regex (the grep ones being too slow)
#
# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
fi
}
+# Test if glob pattern $1 matches subject $2 (see fnmatch(3))
+fnmatch() {
+ case "$2" in
+ $1)
+ return 0
+ ;;
+ esac
+ return 1
+}
+
+# Test if line $1 is local include directive
+is_include_line() {
+ fnmatch "*#*include*" "$1" || return 1
+ printf "%s\n" "$1" | grep -Eq '^\s*#\s*include\s*".+"'
+}
+
+# Test if line $1 is pragma once directive
+is_pragma_once_line() {
+ fnmatch "*#*pragma*once*" "$1" || return 1
+ printf "%s\n" "$1" | grep -Eq '^\s*#\s*pragma\s*once\s*'
+}
+
# Tests if list $1 has item $2 (returning zero on a match)
+# (originally used grep -Eq "(^|\s*)$2(\$|\s*))
+readonly list_FS="$IFS"
list_has_item() {
- if echo "$1" | grep -Eq "(^|\s*)$2(\$|\s*)"; then
- return 0
- else
- return 1
+ # Re: escaping glob pattern special characters in item string:
+ #
+ # bash (tested 3.2.57, 5.1.4), dash (tested 0.5.10.2), NetBSD /bin/sh
+ # (tested 8.2), and Solaris /bin/sh (tested 11.4) require escaping
+ # backslashes in a bracket expression despite POSIX specifying that
+ # backslash loses significance in a bracket expression.
+ #
+ # Conversely, neither FreeBSD /bin/sh (tested 12.2) nor OpenBSD /bin/sh
+ # (tested 7.1) obey backslash-escaping in case statement patterns even
+ # outside bracket expressions, so escape special characters using bracket
+ # expressions.
+ #
+ # Solaris /bin/sh (tested 11.4) requires vertical bar (|) to be escaped.
+ #
+ # All accommodations should behave as expected under strict POSIX semantics.
+ if fnmatch "*[\\*?[|]*" "$2"; then
+ set -- "$1" "$(printf '%s\n' "$2" | sed -e 's/[*?[|]/[&]/g; s/[\]/[\\&]/g')"
fi
+ for item_P in "*[$list_FS]$2[$list_FS]*" "*[$list_FS]$2" "$2[$list_FS]*" "$2"; do
+ fnmatch "${item_P}" "$1" && return 0
+ done
+ return 1
}
# Adds a new line with the supplied arguments to $DESTN (or stdout)
# Read the file
local line=
while IFS= read -r line; do
- if echo "$line" | grep -Eq '^\s*#\s*include\s*".+"'; then
+ if is_include_line "$line"; then
# We have an include directive so strip the (first) file
local inc=$(echo "$line" | grep -Eo '".*"' | sed -E 's/"([^"]+)"/\1/' | head -1)
local res_inc="$(resolve_include "$srcdir" "$inc")"
# Skip any 'pragma once' directives, otherwise write the source line
local write=$PONCE
if [ $write -eq 0 ]; then
- if echo "$line" | grep -Eqv '^\s*#\s*pragma\s*once\s*'; then
+ if ! is_pragma_once_line "$line"; then
write=1
fi
fi
* Constants
***************************************/
-/* Just a sential for the entires of the diagonal matrix */
+/* Just a sential for the entries of the diagonal matrix */
#define ZSTD_EDIST_DIAG_MAX (S32)(1 << 30)
/* How large should a snake be to be considered a 'big' snake.
const BYTE* src;
size_t dictSize;
size_t srcSize;
- S32* forwardDiag; /* Entires of the forward diagonal stored here */
- S32* backwardDiag; /* Entires of the backward diagonal stored here.
+ S32* forwardDiag; /* Entries of the forward diagonal stored here */
+ S32* backwardDiag; /* Entries of the backward diagonal stored here.
* Note: this buffer and the 'forwardDiag' buffer
- * are contiguous. See the ZSTD_eDist_genSequences */
+ * are contiguous. See the ZSTD_eDist_genSequences */
ZSTD_eDist_match* matches; /* Accumulate matches of length 1 in this buffer.
* In a subsequence post-processing step, we combine
* contiguous matches. */
}
/*-*************************************
-* Interal utils
+* Internal utils
***************************************/
static size_t ZSTD_eDist_hamingDist(const BYTE* const a,
--- /dev/null
+# ################################################################
+# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS ?= -O3 -g
+CFLAGS += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum \
+ -Wstrict-prototypes -Wundef -Wpointer-arith \
+ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+ -Wredundant-decls
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: seqBench
+
+all : seqBench
+
+seqBench: util.o timefn.o benchfn.o datagen.o xxhash.o seqBench.c $(LIBZSTD)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+ $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+ $(RM) *.o
+ $(MAKE) -C $(LIBDIR) clean > /dev/null
+ $(RM) seqBench
--- /dev/null
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+ ZSTD_CCtx* zc = ZSTD_createCCtx();
+
+ if (argc != 2) {
+ printf("Usage: seqBench <file>\n"); // TODO provide the block delim option here
+ return 1;
+ }
+
+ FILE *f = fopen(argv[1], "rb");
+ fseek(f, 0, SEEK_END);
+ long inBufSize = ftell(f);
+ fseek(f, 0, SEEK_SET);
+
+ char *inBuf = malloc(inBufSize + 1);
+ fread(inBuf, inBufSize, 1, f);
+ fclose(f);
+
+ size_t seqsSize = ZSTD_sequenceBound(inBufSize);
+ ZSTD_Sequence *seqs = (ZSTD_Sequence*)malloc(seqsSize * sizeof(ZSTD_Sequence));
+ char *outBuf = malloc(ZSTD_compressBound(inBufSize));
+
+ ZSTD_generateSequences(zc, seqs, seqsSize, inBuf, inBufSize);
+ ZSTD_CCtx_setParameter(zc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
+ size_t outBufSize = ZSTD_compressSequences(zc, outBuf, inBufSize, seqs, seqsSize, inBuf, inBufSize);
+ if (ZSTD_isError(outBufSize)) {
+ printf("ERROR: %lu\n", outBufSize);
+ return 1;
+ }
+
+ char *validationBuf = malloc(inBufSize);
+ ZSTD_decompress(validationBuf, inBufSize, outBuf, outBufSize);
+
+ if (memcmp(inBuf, validationBuf, inBufSize) == 0) {
+ printf("Compression and decompression were successful!\n");
+ } else {
+ printf("ERROR: input and validation buffers don't match!\n");
+ for (int i = 0; i < inBufSize; i++) {
+ if (inBuf[i] != validationBuf[i]) {
+ printf("First bad index: %d\n", i);
+ break;
+ }
+ }
+ }
+
+ return 0;
+}
#include <sys/stat.h> // stat
#include <zstd.h>
+
+/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
+#if defined(__GNUC__)
+# define UNUSED_ATTR __attribute__((unused))
+#else
+# define UNUSED_ATTR
+#endif
+
+#define HEADER_FUNCTION static UNUSED_ATTR
+
+
/*
* Define the returned error code from utility functions.
*/
*
* @return The size of a given file path.
*/
-static size_t fsize_orDie(const char *filename)
+HEADER_FUNCTION size_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) != 0) {
* @return If successful this function will return a FILE pointer to an
* opened file otherwise it sends an error to stderr and exits.
*/
-static FILE* fopen_orDie(const char *filename, const char *instruction)
+HEADER_FUNCTION FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/*! fclose_orDie() :
* Close an opened file using given FILE pointer.
*/
-static void fclose_orDie(FILE* file)
+HEADER_FUNCTION void fclose_orDie(FILE* file)
{
if (!fclose(file)) { return; };
/* error */
*
* @return The number of bytes read.
*/
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+HEADER_FUNCTION size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
*
* @return The number of bytes written.
*/
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+HEADER_FUNCTION size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
{
size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
* cated memory. If there is an error, this function will send that
* error to stderr and exit.
*/
-static void* malloc_orDie(size_t size)
+HEADER_FUNCTION void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
* @return If successful this function will load file into buffer and
* return file size, otherwise it will printout an error to stderr and exit.
*/
-static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
+HEADER_FUNCTION size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
{
size_t const fileSize = fsize_orDie(fileName);
CHECK(fileSize <= bufferSize, "File too large!");
* @return If successful this function will return buffer and bufferSize(=fileSize),
* otherwise it will printout an error to stderr and exit.
*/
-static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
+HEADER_FUNCTION void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize)
+{
size_t const fileSize = fsize_orDie(fileName);
*bufferSize = fileSize;
void* const buffer = malloc_orDie(*bufferSize);
* Note: This function will send an error to stderr and exit if it
* cannot write to a given file.
*/
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+HEADER_FUNCTION void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
{
FILE* const oFile = fopen_orDie(fileName, "wb");
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
- */
+**/
+
+/* This example deals with Dictionary compression,
+ * its counterpart is `examples/dictionary_decompression.c` .
+ * These examples presume that a dictionary already exists.
+ * The main method to create a dictionary is `zstd --train`,
+ * look at the CLI documentation for details.
+ * Another possible method is to employ dictionary training API,
+ * published in `lib/zdict.h` .
+**/
+
#include <stdio.h> // printf
#include <stdlib.h> // free
#include <string.h> // memset, strcat
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/* createDict() :
- `dictFileName` is supposed to have been created using `zstd --train` */
+** `dictFileName` is supposed already created using `zstd --train` */
static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
{
size_t dictSize;
libzstd.pc: libzstd.pc.in
@echo creating pkgconfig
- @sed $(SED_ERE_OPT) \
+ @sed \
-e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
-e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
/* ****************************************
* HUF detailed API
* ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD 1
+typedef enum {
+ HUF_depth_fast, /** Use heuristic to find the table depth**/
+ HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
+ } HUF_depth_mode;
/*! HUF_compress() does the following:
* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
* For example, it's possible to compress several blocks using the same 'CTable',
* or to save and regenerate 'CTable' using external methods.
*/
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+unsigned HUF_minTableLog(size_t srcSize, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode);
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
} HUF_repeat;
+
/** HUF_compress4X_repeat() :
* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
* If it uses hufTable it does not modify hufTable or repeat.
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned tableLog,
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+ unsigned suspectUncompressible, HUF_depth_mode depthMode);
/** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned tableLog,
void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+ unsigned suspectUncompressible, HUF_depth_mode depthMode);
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
return (size_t)(op-ostart);
}
-
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
-{
- unsigned tableLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
- assert(tableLog <= HUF_TABLELOG_MAX);
-
- return tableLog;
-}
-
typedef struct {
unsigned count[HUF_SYMBOLVALUE_MAX + 1];
HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)];
#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */
+unsigned HUF_minTableLog(size_t srcSize, unsigned maxSymbolValue)
+{
+ U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
+ U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+ if (minBits < FSE_MIN_TABLELOG) minBits = FSE_MIN_TABLELOG;
+ assert(srcSize > 1); /* Not supported, RLE should be used instead */
+ return minBits;
+}
+
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
+{
+ unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+
+ if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
+ BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+ size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
+ size_t optSize = ((size_t) ~0);
+ unsigned huffLog;
+ size_t maxBits, hSize, newSize;
+
+ if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
+
+ for (huffLog = HUF_minTableLog(srcSize, maxSymbolValue); huffLog <= maxTableLog; huffLog++) {
+ maxBits = HUF_buildCTable_wksp(table, count,
+ maxSymbolValue, huffLog,
+ workSpace, wkspSize);
+ if (ERR_isError(maxBits)) continue;
+
+ hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
+ workSpace, wkspSize);
+ if (ERR_isError(hSize)) continue;
+
+ newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+ if (newSize < optSize) {
+ optSize = newSize;
+ optLog = huffLog;
+ }
+ }
+ }
+ assert(optLog <= HUF_TABLELOG_MAX);
+ return optLog;
+}
+
/* HUF_compress_internal() :
* `workSpace_align4` must be aligned on 4-bytes boundaries,
* and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
HUF_nbStreams_e nbStreams,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
- const int bmi2, unsigned suspectUncompressible)
+ const int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
{
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
BYTE* const ostart = (BYTE*)dst;
}
/* Build Huffman Tree */
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, depthMode);
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
maxSymbolValue, huffLog,
&table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize,
- NULL, NULL, 0, 0 /*bmi2*/, 0);
+ NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
}
size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
- int bmi2, unsigned suspectUncompressible)
+ int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
{
DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize, hufTable,
- repeat, preferRepeat, bmi2, suspectUncompressible);
+ repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
}
/* HUF_compress4X_repeat():
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
- NULL, NULL, 0, 0 /*bmi2*/, 0);
+ NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
}
/* HUF_compress4X_repeat():
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize,
- HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible)
+ HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+ unsigned suspectUncompressible, HUF_depth_mode depthMode)
{
DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
- hufTable, repeat, preferRepeat, bmi2, suspectUncompressible);
+ hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
}
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
/* Base suspicion of uncompressibility on ratio of literals to sequences */
unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
size_t const litSize = (size_t)(seqStorePtr->lit - literals);
+
+ HUF_depth_mode depthMode = cctxParams->cParams.strategy > HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
size_t const cSize = ZSTD_compressLiterals(
&prevEntropy->huf, &nextEntropy->huf,
cctxParams->cParams.strategy,
op, dstCapacity,
literals, litSize,
entropyWorkspace, entropyWkspSize,
- bmi2, suspectUncompressible);
+ bmi2, suspectUncompressible, depthMode);
FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
assert(cSize <= dstCapacity);
op += cSize;
zc->seqCollector.seqIndex += seqStoreSeqSize;
}
+size_t ZSTD_sequenceBound(size_t srcSize) {
+ return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize)
{
ZSTD_hufCTables_t* nextHuf,
ZSTD_hufCTablesMetadata_t* hufMetadata,
const int literalsCompressionIsDisabled,
- void* workspace, size_t wkspSize)
+ void* workspace, size_t wkspSize, HUF_depth_mode depthMode)
{
BYTE* const wkspStart = (BYTE*)workspace;
BYTE* const wkspEnd = wkspStart + wkspSize;
/* Build Huffman Tree */
ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, depthMode);
assert(huffLog <= LitHufLog);
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
maxSymbolValue, huffLog,
void* workspace, size_t wkspSize)
{
size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
+ HUF_depth_mode depthMode = cctxParams->cParams.strategy > HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
+
entropyMetadata->hufMetadata.hufDesSize =
ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
&prevEntropy->huf, &nextEntropy->huf,
&entropyMetadata->hufMetadata,
ZSTD_literalsCompressionIsDisabled(cctxParams),
- workspace, wkspSize);
+ workspace, wkspSize, depthMode);
+
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
entropyMetadata->fseMetadata.fseTablesSize =
ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
const void* src, size_t srcSize,
void* entropyWorkspace, size_t entropyWorkspaceSize,
const int bmi2,
- unsigned suspectUncompressible)
+ unsigned suspectUncompressible, HUF_depth_mode depthMode)
{
size_t const minGain = ZSTD_minGain(srcSize, strategy);
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
{ HUF_repeat repeat = prevHuf->repeatMode;
int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
- typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned);
+ typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
huf_compress_f huf_compress;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
entropyWorkspace, entropyWorkspaceSize,
(HUF_CElt*)nextHuf->CTable,
&repeat, preferRepeat,
- bmi2, suspectUncompressible);
+ bmi2, suspectUncompressible, depthMode);
if (repeat != HUF_repeat_none) {
/* reused the existing table */
DEBUGLOG(5, "Reusing previous huffman table");
const void* src, size_t srcSize,
void* entropyWorkspace, size_t entropyWorkspaceSize,
const int bmi2,
- unsigned suspectUncompressible);
+ unsigned suspectUncompressible, HUF_depth_mode depthMode);
#endif /* ZSTD_COMPRESS_LITERALS_H */
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
- if (match[ml] == ip[ml]) /* potentially better */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
const BYTE* const match = dictBase + matchIndex;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
- if (match[ml] == ip[ml]) /* potentially better */
+ /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+ if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
const BYTE* const match = dictBase + matchIndex;
if (ZSTD_isError(decompressedSize)) return decompressedSize;
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
ip = istart + cSize;
- op += decompressedSize;
+ op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
zds->expected = 0;
zds->streamStage = zdss_init;
someMoreWork = 0;
break;
}
case zdss_flush:
- { size_t const toFlushSize = zds->outEnd - zds->outStart;
+ {
+ size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
- op += flushedSize;
+
+ op = op ? op + flushedSize : op;
+
zds->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */
zds->streamStage = zdss_read;
if ( (zds->outBuffSize < zds->fParams.frameContentSize)
- && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+ && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
(int)(zds->outBuffSize - zds->outStart),
(U32)zds->fParams.blockSizeMax);
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
GREP_OPTIONS += --color=never
endif
GREP = grep $(GREP_OPTIONS)
-SED_ERE_OPT ?= -E
ZSTD_COMMON_FILES := $(sort $(wildcard $(LIBZSTD)/common/*.c))
ZSTD_COMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/compress/*.c))
* "empty", "unknown" and "error" results to the same return value (0),
* while ZSTD_getFrameContentSize() gives them separate return values.
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
-ZSTDLIB_API
ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
/*===== Streaming decompression functions =====*/
-/* This function is redundant with the advanced API and equivalent to:
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
*
+ * Note : This function is redundant with the advanced API and equivalent to:
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
* ZSTD_DCtx_refDDict(zds, NULL);
*/
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ * on the next call.
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+ * call ZSTD_decompressStream() again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ * or an error code, which can be tested using ZSTD_isError(),
+ * or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ */
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
} ZSTD_sequenceFormat_e;
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ * @return : upper-bound for the number of sequences that can be generated
+ * from a buffer of srcSize bytes
+ *
+ * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
/*! ZSTD_generateSequences() :
* Generate sequences using ZSTD_compress2(), given a source buffer.
*
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
* This prototype will generate compilation warnings. */
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
* This prototype will generate compilation warnings. */
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
* This instruction is mandatory to decode data without a fully-formed header,
* such ZSTD_f_zstd1_magicless for example.
* @return : 0, or an error code (which can be tested using ZSTD_isError()). */
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
/*! ZSTD_decompressStream_simpleArgs() :
* "0" also disables frame content size field. It may be enabled in the future.
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel,
unsigned long long pledgedSrcSize);
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
int compressionLevel);
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params,
* note : cdict will just be referenced, and must outlive compression session
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
/*! ZSTD_initCStream_usingCDict_advanced() :
* value ZSTD_CONTENTSIZE_UNKNOWN.
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams,
* @return : 0, or an error code (which can be tested using ZSTD_isError())
* This prototype will generate compilation warnings.
*/
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
-ZSTDLIB_STATIC_API
ZSTD_DEPRECATED("use advanced API to access custom parameters")
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
/**
Buffer-less streaming decompression (synchronous mode)
.PHONY: all
-all: zstd
+all: zstd zstd-compress zstd-decompress zstd-small
.PHONY: allVariants
-allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-frugal zstd-nolegacy zstd-dictBuilder
+allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder
.PHONY: zstd # must always be run
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
endif # BUILD_DIR
+CLEAN += zstd
.PHONY: zstd-release
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
zstd-release: DEBUGFLAGS_LD :=
zstd-release: zstd
+CLEAN += zstd32
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
+## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
+CLEAN += zstd-nolegacy
zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
## zstd-pgo: zstd executable optimized with PGO.
.PHONY: zstd-pgo
zstd-pgo :
- $(MAKE) clean
- $(MAKE) zstd MOREFLAGS=-fprofile-generate
+ $(MAKE) clean HASH_DIR=$(HASH_DIR)
+ $(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
./zstd -b19i1 $(PROFILE_WITH)
./zstd -b16i1 $(PROFILE_WITH)
./zstd -b9i2 $(PROFILE_WITH)
./zstd -b $(PROFILE_WITH)
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
- $(RM) zstd *.o
+ifndef BUILD_DIR
+ $(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
+else
+ $(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
+endif
case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
- $(MAKE) zstd MOREFLAGS=-fprofile-use
+ $(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
+CLEAN += zstd-small zstd-frugal
zstd-small: CFLAGS = -Os -Wl,-s
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+CLEAN += zstd-decompress
zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
+CLEAN += zstd-compress
zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
+CLEAN += zstd-dictBuilder
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
+CLEAN += zstdmt
zstdmt: zstd
ln -sf zstd zstdmt
.PHONY: clean
clean:
- $(RM) core *.o tmp* result* *.gcda dictionary *.zst \
- zstd$(EXT) zstd32$(EXT) zstd-dll$(EXT) \
- zstd-compress$(EXT) zstd-decompress$(EXT) \
- zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
- zstd-dictBuilder$(EXT) \
- *.gcda default*.profraw default.profdata have_zlib$(EXT)
+ $(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
+ *.gcda default*.profraw default.profdata have_zlib
$(RM) -r obj/*
@echo Cleaning completed
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
ret->excludeCompressedFiles = 0;
ret->allowBlockDevices = 0;
ret->asyncIO = AIO_supported();
+ ret->passThrough = -1;
return ret;
}
void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
-void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
+void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
prefs->overlapLog = overlapLog;
}
-void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
+void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
if ((adapt>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
prefs->adaptiveMode = adapt;
prefs->contentSize = value != 0;
}
-void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) {
+void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#ifdef ZSTD_MULTITHREAD
prefs->asyncIO = value;
#else
#endif
}
+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
+ prefs->passThrough = (value != 0);
+}
+
/* FIO_ctx_t functions */
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
} else {
const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0);
- windowLog = cParams.windowLog;
+ windowLog = (int)cParams.windowLog;
}
}
windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
return result;
}
-static const char* checked_index(const char* options[], size_t length, size_t index) {
+static const char*
+checked_index(const char* options[], size_t length, size_t index) {
assert(index < length);
/* Necessary to avoid warnings since -O3 will omit the above `assert` */
(void) length;
return options[index];
}
-#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index))
+#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (size_t)(index))
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
{
unsigned readSomething = 0;
unsigned long long filesize = 0;
+ int passThrough = prefs->passThrough;
+
+ if (passThrough == -1) {
+ /* If pass-through mode is not explicitly enabled or disabled,
+ * default to the legacy behavior of enabling it if we are writing
+ * to stdout with the overwrite flag enabled.
+ */
+ passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
+ }
+ assert(passThrough == 0 || passThrough == 1);
/* for each frame */
for ( ; ; ) {
}
readSomething = 1; /* there is at least 1 byte in srcFile */
if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
- if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
+ if (passThrough) {
return FIO_passThrough(&ress);
}
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
return 1;
#endif
- } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
+ } else if (passThrough) {
return FIO_passThrough(&ress);
} else {
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
#ifndef FILEIO_H_23981798732
#define FILEIO_H_23981798732
-#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "fileio_types.h"
+#include "util.h" /* FileNamesTable */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "../lib/zstd.h" /* ZSTD_* */
#if defined (__cplusplus)
/* FIO_prefs_t functions */
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
void FIO_overwriteMode(FIO_prefs_t* const prefs);
-void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
+void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
-void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
-void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value);
+void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
/* Algorithm preferences */
FIO_compressionType_t compressionType;
- U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+ int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
int dictIDFlag;
int checksumFlag;
int blockSize;
int overlapLog;
- U32 adaptiveMode;
- U32 useRowMatchFinder;
+ int adaptiveMode;
+ int useRowMatchFinder;
int rsyncable;
int minAdaptLevel;
int maxAdaptLevel;
ZSTD_paramSwitch_e literalCompressionMode;
/* IO preferences */
- U32 removeSrcFile;
- U32 overwrite;
- U32 asyncIO;
+ int removeSrcFile;
+ int overwrite;
+ int asyncIO;
/* Computation resources preferences */
unsigned memLimit;
int patchFromMode;
int contentSize;
int allowBlockDevices;
+ int passThrough;
} FIO_prefs_t;
#endif /* FILEIO_TYPES_HEADER */
.
-.TH "ZSTD" "1" "June 2022" "zstd 1.5.3" "User Commands"
+.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
\fB\-o FILE\fR: save result into \fBFILE\fR
.
.IP "\(bu" 4
-\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\.
+\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
.
.IP "\(bu" 4
\fB\-c\fR, \fB\-\-stdout\fR: write to standard output (even if it is the console); keep original files unchanged\.
\fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
.
.IP "\(bu" 4
+\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
+.
+.IP "\(bu" 4
\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
.
.IP "\(bu" 4
* `-f`, `--force`:
disable input and output checks. Allows overwriting existing files, input
from console, output to stdout, operating on links, block devices, etc.
+ During decompression and when the output destination is stdout, pass-through
+ unrecognized formats as-is.
* `-c`, `--stdout`:
write to standard output (even if it is the console); keep original files unchanged.
* `--[no-]sparse`:
default: enabled when output is into a file,
and disabled when output is stdout.
This setting overrides default and can force sparse mode over stdout.
+* `--[no-]pass-through`
+ enable / disable passing through uncompressed files as-is. During
+ decompression when pass-through is enabled, unrecognized formats will be
+ copied as-is from the input to the output. By default, pass-through will
+ occur when the output destination is stdout and the force (-f) option is
+ set.
* `--rm`:
remove source file(s) after successful compression or decompression. If used in combination with
-o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
#endif
DISPLAY_F(f, " -f, --force disable input and output checks. Allows overwriting existing files,\n");
DISPLAY_F(f, " input from console, output to stdout, operating on links,\n");
- DISPLAY_F(f, " block devices, etc.\n");
+ DISPLAY_F(f, " block devices, etc. During decompression and when the output\n");
+ DISPLAY_F(f, " destination is stdout, pass-through unrecognized formats as-is.\n");
DISPLAY_F(f, " --rm remove source file(s) after successful de/compression\n");
DISPLAY_F(f, " -k, --keep preserve source file(s) (default) \n");
#ifdef ZSTD_GZCOMPRESS
# else
DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n");
# endif
+ {
+ char const* passThroughDefault = "disabled";
+ if (exeNameMatch(programName, ZSTD_CAT) ||
+ exeNameMatch(programName, ZSTD_ZCAT) ||
+ exeNameMatch(programName, ZSTD_GZCAT)) {
+ passThroughDefault = "enabled";
+ }
+ DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
+ }
#endif /* ZSTD_NODECOMPRESS */
#ifndef ZSTD_NODICT
const char* __nb; \
NEXT_FIELD(__nb); \
val32 = readU32FromChar(&__nb); \
+ if(*__nb != 0) { \
+ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+ } \
+}
+
+#define NEXT_TSIZE(valTsize) { \
+ const char* __nb; \
+ NEXT_FIELD(__nb); \
+ valTsize = readSizeTFromChar(&__nb); \
+ if(*__nb != 0) { \
+ errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+ } \
}
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
hasStdout = 0,
ldmFlag = 0,
main_pause = 0,
- nbWorkers = 0,
adapt = 0,
useRowMatchFinder = 0,
adaptMin = MINCLEVEL,
showDefaultCParams = 0,
ultra=0,
contentSize=1;
+ unsigned nbWorkers = 0;
double compressibility = 0.5;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
- if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
- if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
+ if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
+ if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
dictCLevel = cLevel = 6; /* gzip default is -6 */
}
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
- if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
+ if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
+ if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
+ if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
- if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
- if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
+ if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
+ if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
if (longCommandWArg(&argument, "--output-dir-flat")) {
NEXT_FIELD(outDirName);
if (strlen(outDirName) == 0) {
/* Invalid character following --long */
badusage(programName);
CLEAN_RETURN(1);
+ } else {
+ ldmWindowLog = g_defaultMaxWindowLog;
}
/* Only set windowLog if not already set by --zstd */
if (compressionParams.windowLog == 0)
/* nb of threads (hidden option) */
case 'T':
argument++;
- nbWorkers = (int)readU32FromChar(&argument);
+ nbWorkers = readU32FromChar(&argument);
break;
/* Dictionary Selection level */
if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) {
- nbWorkers = UTIL_countLogicalCores();
+ nbWorkers = (unsigned)UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
} else {
- nbWorkers = UTIL_countPhysicalCores();
+ nbWorkers = (unsigned)UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
benchParams.blockSize = blockSize;
- benchParams.nbWorkers = nbWorkers;
+ benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio;
benchParams.nbSeconds = bench_nbSeconds;
benchParams.ldmFlag = ldmFlag;
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
FIO_setContentSize(prefs, contentSize);
- FIO_setNbWorkers(prefs, nbWorkers);
+ FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
- FIO_setAdaptiveMode(prefs, (unsigned)adapt);
+ FIO_setAdaptiveMode(prefs, adapt);
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax);
-.TH "ZSTDGREP" "1" "April 2022" "zstd 1.5.2" "User Commands"
+.
+.TH "ZSTDGREP" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
+.
.SH "SYNOPSIS"
-\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \|\.\|\.\|\.]
+\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.]
+.
.SH "DESCRIPTION"
\fBzstdgrep\fR runs \fBgrep (1)\fR on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
+.
.P
The grep\-flags and pattern arguments are passed on to \fBgrep (1)\fR\. If an \fB\-e\fR flag is found in the \fBgrep\-flags\fR, \fBzstdgrep\fR will not look for a pattern argument\.
+.
.P
Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\.
+.
.SH "EXIT STATUS"
In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
+.
.SH "SEE ALSO"
\fBzstd (1)\fR
+.
.SH "AUTHORS"
Thomas Klausner \fIwiz@NetBSD\.org\fR
-.TH "ZSTDLESS" "1" "April 2022" "zstd 1.5.2" "User Commands"
+.
+.TH "ZSTDLESS" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
+.
.SH "SYNOPSIS"
-\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \|\.\|\.\|\.]
+\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.]
+.
.SH "DESCRIPTION"
\fBzstdless\fR runs \fBless (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
+.
.SH "SEE ALSO"
\fBzstd (1)\fR
zstreamtest32
zstreamtest_asan
zstreamtest_tsan
+zstreamtest_ubsan
zstreamtest-dll
datagen
paramgrill
- # ################################################################
+# ################################################################
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
DEBUGFLAGS += -g -Wno-c++-compat
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
- -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
+ -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES))
zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
+FULLBENCHS := fullbench fullbench32
+CLEAN += $(FULLBENCHS)
fullbench32: CPPFLAGS += -m32
-fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
-fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
-fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
-fullbench fullbench32 : $(ZSTD_FILES)
-fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
+$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
+$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
+$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
+$(FULLBENCHS) : $(ZSTD_FILES)
+$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(LINK.c) $^ -o $@$(EXT)
+CLEAN += fullbench-lib
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c
$(LINK.c) $^ -o $@$(EXT)
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
+CLEAN += fuzzer fuzzer32
fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer : $(ZSTDMT_OBJECTS)
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
+CLEAN += zstreamtest zstreamtest32
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
zstreamtest zstreamtest32 :
$(LINK.c) $^ -o $@$(EXT)
+CLEAN += zstreamtest_asan
zstreamtest_asan : CFLAGS += -fsanitize=address
zstreamtest_asan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+CLEAN += zstreamtest_tsan
zstreamtest_tsan : CFLAGS += -fsanitize=thread
zstreamtest_tsan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+CLEAN += zstreamtest_ubsan
+zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
+zstreamtest_ubsan : $(ZSTREAMFILES)
+ $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
# note : broken : requires symbols unavailable from dynamic library
zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
+CLEAN += paramgrill
paramgrill : DEBUGFLAGS = # turn off debug for speed measurements
paramgrill : LDLIBS += -lm
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
+CLEAN += datagen
datagen : $(PRGDIR)/datagen.c datagencli.c
$(LINK.c) $^ -o $@$(EXT)
+CLEAN += roundTripCrash
roundTripCrash: CFLAGS += $(MULTITHREAD)
roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
+CLEAN += longmatch
longmatch : $(ZSTD_OBJECTS) longmatch.c
+CLEAN += bigdict
bigdict: CFLAGS += $(MULTITHREAD)
bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
+CLEAN += invalidDictionaries
invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
+CLEAN += legacy
legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c
+CLEAN += decodecorpus
decodecorpus : LDLIBS += -lm
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
+CLEAN += poolTests
poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
automated_benchmarking: clean
$(PYTHON) automated_benchmarking.py
-# make checkTag
+# make checkTag : check that release tag corresponds to release version
+CLEAN += checkTag
checkTag.o : $(ZSTDDIR)/zstd.h
.PHONY: clean
$(MAKE) -C $(PRGDIR) clean
$(RM) -fR $(TESTARTEFACT)
$(RM) -rf tmp* # some test directories are named tmp*
- $(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
+ $(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
- fullbench$(EXT) fullbench32$(EXT) \
- fullbench-lib$(EXT) fullbench-dll$(EXT) \
- fuzzer$(EXT) fuzzer32$(EXT) \
- fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
- zstreamtest$(EXT) zstreamtest32$(EXT) \
- datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
- symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
- decodecorpus$(EXT) checkTag$(EXT) bigdict$(EXT)
+ fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT)
@echo Cleaning completed
#----------------------------------------------------------------------------------
-# valgrind tests are validated only for some posix platforms
+# valgrind tests validated only for some posix platforms
#----------------------------------------------------------------------------------
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX))
HOST_OS = POSIX
-valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
-valgrindTest: zstd datagen fuzzer fullbench
+.PHONY: test-valgrind
+test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
+test-valgrind: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
- @rm tmp
+ $(RM) tmp
$(VALGRIND) ./fuzzer -T1mn -t1
$(VALGRIND) ./fullbench -i1
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
.PHONY: test-all
-test-all: test test32 valgrindTest test-decodecorpus-cli
+test-all: test test32 test-decodecorpus-cli
-.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
+.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd
file $(ZSTD)
EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
+.PHONY: test-cli-tests
test-cli-tests: ZSTD = $(PRGDIR)/zstd
test-cli-tests: zstd datagen
file $(ZSTD)
./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
-
+.PHONY: test-fullbench
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
$(QEMU_SYS) ./fullbench -i1 -P0
+.PHONY: test-fullbench32
test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1
$(QEMU_SYS) ./fullbench32 -i1 -P0
+.PHONY: test-fuzzer
test-fuzzer: fuzzer
$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
+# Note : this test presumes `fuzzer` will be built
+.PHONY: test-fuzzer-stackmode
test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
test-fuzzer-stackmode: test-fuzzer
+.PHONY: test-fuzzer32
test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
+.PHONY: test-zstream
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
-d, --decompress decompression
-f, --force disable input and output checks. Allows overwriting existing files,
input from console, output to stdout, operating on links,
- block devices, etc.
+ block devices, etc. During decompression and when the output
+ destination is stdout, pass-through unrecognized formats as-is.
--rm remove source file(s) after successful de/compression
-k, --keep preserve source file(s) (default)
-D DICT use DICT as Dictionary for compression or decompression
--- /dev/null
+#!/bin/sh
+
+echo "some data" > file
+
+println "+ zstd --memory=32LB file"
+zstd --memory=32LB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32LiB file"
+zstd --memory=32LiB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32A file"
+zstd --memory=32A file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32r82347dn83 file"
+zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32asbdf file"
+zstd --memory=32asbdf file && die "Should not allow bogus suffix"
+println "+ zstd --memory=hello file"
+zstd --memory=hello file && die "Should not allow non-numeric parameter"
+println "+ zstd --memory=1 file"
+zstd --memory=1 file && die "Should allow numeric parameter without suffix"
+rm file.zst
+println "+ zstd --memory=1K file"
+zstd --memory=1K file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KB file"
+zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KiB file"
+zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1M file"
+zstd --memory=1M file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MB file"
+zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MiB file"
+zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+
+rm file
+exit 0
--- /dev/null
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
+Should allow numeric parameter without suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
--- /dev/null
++ zstd --memory=32LB file
++ zstd --memory=32LiB file
++ zstd --memory=32A file
++ zstd --memory=32r82347dn83 file
++ zstd --memory=32asbdf file
++ zstd --memory=hello file
++ zstd --memory=1 file
++ zstd --memory=1K file
++ zstd --memory=1KB file
++ zstd --memory=1KiB file
++ zstd --memory=1M file
++ zstd --memory=1MB file
++ zstd --memory=1MiB file
--- /dev/null
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+zstd < file -vv -19 -o file.19.zst
+zstd -vv -l file.19.zst
+
+zstd < file -vv -19 --long -o file.19.long.zst
+zstd -vv -l file.19.long.zst
--- /dev/null
+...
+*wlog=23*
+...
+*wlog=27*
+...
--- /dev/null
+...
+*Window Size: 8388608 B*
+...
+*Window Size: 134217728 B*
+...
--- /dev/null
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+echo "" > 1
+echo "2" > 2
+echo "23" > 3
+echo "234" > 4
+echo "some data" > file
+
+println "+ passthrough enabled"
+
+zstd file
+
+# Test short files
+zstd -dc --pass-through 1 2 3 4
+
+# Test *cat symlinks
+zstdcat file
+"$ZSTD_SYMLINK_DIR/zcat" file
+"$ZSTD_SYMLINK_DIR/gzcat" file
+
+# Test multiple files with mix of compressed & not
+zstdcat file file.zst
+zstdcat file.zst file
+
+# Test --pass-through
+zstd -dc --pass-through file
+zstd -d --pass-through file -o pass-through-file
+
+# Test legacy implicit passthrough with -fc
+zstd -dcf file
+zstd -dcf file file.zst
+zstd -df < file
+zstd -dcf < file file.zst -
+zstd -dcf < file.zst file -
+
+$DIFF file pass-through-file
+
+println "+ passthrough disabled"
+
+# Test *cat
+zstdcat --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
+# Test zstd without implicit passthrough
+zstd -d file -o no-pass-through-file && die "should fail"
+zstd -d < file && die "should fail"
+
+# Test legacy implicit passthrough with -fc
+zstd --no-pass-through -dcf file && die "should fail"
+zstd --no-pass-through -dcf file file.zst && die "should fail"
+zstd --no-pass-through -df < file && die "should fail"
+zstd --no-pass-through -dcf < file file.zst - && die "should fail"
+zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:
--- /dev/null
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: file: unsupported format
+zstd: file: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: /*stdin*\: unsupported format
+zstd: file: unsupported format
--- /dev/null
++ passthrough enabled
+
+2
+23
+234
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
++ passthrough disabled
+some data
+some data
+some data
HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
dt[0] = tableLog * 0x01000001;
- tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol);
+ HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
+
+ tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
FUZZ_ASSERT(tableLog <= 12);
tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
FUZZ_ZASSERT(tableLog);
}
DISPLAYLEVEL(3, "OK \n");
- DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
+ DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
inBuff.src = NULL;
inBuff.size = 0;
inBuff.pos = 0;
{ size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ret != 0) goto _output_error;
}
+ DISPLAYLEVEL(3, "OK\n");
+
+ DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
+ {
+ const char* test = "aa";
+ inBuff.src = test;
+ inBuff.size = 2;
+ inBuff.pos = 0;
+ outBuff.dst = NULL;
+ outBuff.size = 0;
+ outBuff.pos = 0;
+ CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+ CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
+ CHECK_Z( ZSTD_endStream(zc, &outBuff) );
+ outBuff.dst = (char*)(compressedBuffer);
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
+ }
+ inBuff.src = outBuff.dst;
+ inBuff.size = outBuff.pos;
+ inBuff.pos = 0;
+ outBuff.dst = NULL;
+ outBuff.size = 0;
+ outBuff.pos = 0;
+ CHECK_Z( ZSTD_initDStream(zd) );
+ CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
+ }
+
DISPLAYLEVEL(3, "OK\n");
/* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
- U32 n;
- for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
+ cSize=0;
+ totalTestSize=0;
+ while(totalTestSize < maxTestSize) {
/* compress random chunks into randomly sized dst buffers */
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
./zwrapbench -qi1b3B1K $(TEST_FILE)
./zwrapbench -rqi1b1e3 ../lib
-#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
-valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
-valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
+.PHONY: test-valgrind
+#test-valgrind: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
+test-valgrind: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
+test-valgrind: clean example fitblk example_zstd fitblk_zstd zwrapbench
@echo "\n ---- valgrind tests ----"
$(VALGRIND) ./example
$(VALGRIND) ./example_zstd
projects / thirdparty / zstd.git / commitdiff
