From: Joel Rosdahl Date: Sun, 11 Sep 2022 11:27:40 +0000 (+0200) Subject: enhance: Add util::zstd_* functions X-Git-Tag: v4.7~59 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e8def923d74dc68b4bbcb80177563f8ff23b5767;p=thirdparty%2Fccache.git enhance: Add util::zstd_* functions --- diff --git a/src/util/zstd.cpp b/src/util/zstd.cpp new file mode 100644 index 000000000..10264cc27 --- /dev/null +++ b/src/util/zstd.cpp @@ -0,0 +1,90 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "zstd.hpp" + +#include + +namespace util { + +nonstd::expected +zstd_compress(nonstd::span input, + util::Bytes& output, + int8_t compression_level) +{ + const size_t original_output_size = output.size(); + const size_t compress_bound = zstd_compress_bound(input.size()); + output.resize(original_output_size + compress_bound); + + const size_t ret = ZSTD_compress(&output[original_output_size], + compress_bound, + input.data(), + input.size(), + compression_level); + if (ZSTD_isError(ret)) { + return nonstd::make_unexpected(ZSTD_getErrorName(ret)); + } + + output.resize(original_output_size + ret); + return {}; +} + +nonstd::expected +zstd_decompress(nonstd::span input, + util::Bytes& output, + size_t original_size) +{ + const size_t original_output_size = output.size(); + + output.resize(original_output_size + original_size); + const size_t ret = ZSTD_decompress( + &output[original_output_size], original_size, input.data(), input.size()); + if (ZSTD_isError(ret)) { + return nonstd::make_unexpected(ZSTD_getErrorName(ret)); + } + + output.resize(original_output_size + ret); + + return {}; +} + +size_t +zstd_compress_bound(size_t input_size) +{ + return ZSTD_compressBound(input_size); +} + +std::tuple +zstd_supported_compression_level(int8_t wanted_level) +{ + // libzstd 1.3.4 and newer support negative levels. However, the query + // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection + // based on version instead. + if (ZSTD_versionNumber() < 10304 && wanted_level < 1) { + return {1, "minimum level supported by libzstd"}; + } + + const int8_t level = std::min(wanted_level, ZSTD_maxCLevel()); + if (level != wanted_level) { + return {level, "max libzstd level"}; + } + + return {level, {}}; +} + +} // namespace util diff --git a/src/util/zstd.hpp b/src/util/zstd.hpp new file mode 100644 index 000000000..82f4be01b --- /dev/null +++ b/src/util/zstd.hpp @@ -0,0 +1,45 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include + +#include +#include +#include + +namespace util { + +[[nodiscard]] nonstd::expected +zstd_compress(nonstd::span input, + util::Bytes& output, + int8_t compression_level); + +[[nodiscard]] nonstd::expected zstd_decompress( + nonstd::span input, util::Bytes& output, size_t original_size); + +size_t zstd_compress_bound(size_t input_size); + +std::tuple +zstd_supported_compression_level(int8_t wanted_level); + +} // namespace util diff --git a/unittest/test_util_zstd.cpp b/unittest/test_util_zstd.cpp new file mode 100644 index 000000000..962501757 --- /dev/null +++ b/unittest/test_util_zstd.cpp @@ -0,0 +1,77 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include +#include +#include + +#include + +#include + +using TestUtil::TestContext; + +const util::Bytes compressed_ab{ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x02, 0x11, 0x00, 0x00, 0x61, 0x62}; + +TEST_CASE("util::zstd_compress") +{ + TestContext test_context; + + util::Bytes output{'x'}; + auto result = util::zstd_compress(util::Bytes{'a', 'b'}, output, 1); + CHECK(result); + CHECK(output.size() == 12); + util::Bytes expected{'x'}; + expected.insert(expected.end(), compressed_ab.begin(), compressed_ab.end()); + CHECK(output == expected); +} + +TEST_CASE("util::zstd_decompress") +{ + TestContext test_context; + + util::Bytes input = compressed_ab; + util::Bytes output{'x'}; + auto result = util::zstd_decompress(input, output, 2); + CHECK(result); + CHECK(output == util::Bytes{'x', 'a', 'b'}); +} + +TEST_CASE("ZSTD roundtrip") +{ + TestContext test_context; + + const util::Bytes data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + const size_t copies = 10000; + util::Bytes original_input; + for (size_t i = 0; i < copies; i++) { + original_input.insert(original_input.end(), data.begin(), data.end()); + } + + util::Bytes output; + auto result = util::zstd_compress(original_input, output, 1); + CHECK(result); + CHECK(output.size() < 100); + + util::Bytes decompressed_input; + result = + util::zstd_decompress(output, decompressed_input, copies * data.size()); + CHECK(result); + CHECK(decompressed_input == original_input); +}