From: Joel Rosdahl Date: Sun, 11 Sep 2022 11:48:05 +0000 (+0200) Subject: chore: Simplify cache entry reading and writing X-Git-Tag: v4.7~58 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a03f14b7560f12eb28d6da1a60efc2bde4d2faf5;p=thirdparty%2Fccache.git chore: Simplify cache entry reading and writing Cache entries are now fully read into memory before (de)compressing, checksumming and parsing, instead of streaming data like before. While this increases memory usage when working with large object files, it also simplifies the code a lot. Another motivation for this change is that cache entry data is not streamed from secondary storage anyway, and it makes sense to keep the architecture simple and similar for primary and secondary storage code paths. The cache entry format has modified so that the checksum covers the potentially compressed payload (plus the header), not the uncompressed payload (plus the header) like before. The checksum is now also stored in an uncompressed epilogue. Since the cache entry format has been changed, the input hash has been changed as well. --- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 20292bf1b..25ed9a205 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,7 +25,6 @@ to understand and work with. In other words, this is work in progress. preprocessor output format, etc. Ideally this code should in the future be refactored into compiler-specific frontends, such as GCC, Clang, NVCC, MSVC, etc. -* `compression`: Compression formats. * `core`: Everything not part of other directories. * `storage`: Storage backends. * `storage/primary`: Code for the primary storage backend. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c1bc5e3c8..6a7cb3f79 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,7 +69,6 @@ endif() add_executable(test-lockfile test_lockfile.cpp) target_link_libraries(test-lockfile PRIVATE ccache_framework) -add_subdirectory(compression) add_subdirectory(core) add_subdirectory(storage) add_subdirectory(third_party) diff --git a/src/Config.cpp b/src/Config.cpp index ad342220c..2c7ca6282 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -24,8 +24,8 @@ #include "assertions.hpp" #include -#include #include +#include #include #include #include diff --git a/src/ccache.cpp b/src/ccache.cpp index 66978f4bf..0d1c4b027 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -42,11 +42,7 @@ #include "language.hpp" #include -#include -#include -#include -#include -#include +#include #include #include #include @@ -760,44 +756,27 @@ static core::Manifest read_manifest(const std::string& path) { core::Manifest manifest; - File file(path, "rb"); - if (file) { - try { - core::FileReader file_reader(*file); - core::CacheEntryReader reader(file_reader); - std::vector payload; - payload.resize(reader.header().payload_size()); - reader.read(payload.data(), payload.size()); - manifest.read(payload); - reader.finalize(); - } catch (const core::Error& e) { - LOG("Error reading {}: {}", path, e.what()); - } + try { + const auto cache_entry_data = + util::value_or_throw(util::read_file(path)); + core::CacheEntry cache_entry(cache_entry_data); + cache_entry.verify_checksum(); + manifest.read(cache_entry.payload()); + } catch (const core::Error& e) { + LOG("Error reading {}: {}", path, e.what()); } return manifest; } static void save_manifest(const Config& config, - const core::Manifest& manifest, + core::Manifest& manifest, const std::string& path) { + core::CacheEntry::Header header(config, core::CacheEntryType::manifest); + const auto cache_entry_data = core::CacheEntry::serialize(header, manifest); AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary); - core::FileWriter file_writer(atomic_manifest_file.stream()); - core::CacheEntryHeader header(core::CacheEntryType::manifest, - compression::type_from_config(config), - compression::level_from_config(config), - time(nullptr), - CCACHE_VERSION, - config.namespace_()); - header.set_entry_size_from_payload_size(manifest.serialized_size()); - - core::CacheEntryWriter writer(file_writer, header); - util::Bytes payload; - payload.reserve(header.payload_size()); - manifest.serialize(payload); - writer.write(payload.data(), payload.size()); - writer.finalize(); + atomic_manifest_file.write(cache_entry_data); atomic_manifest_file.commit(); } @@ -933,22 +912,14 @@ write_result(Context& ctx, ctx.args_info.output_al); } - AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary); - core::CacheEntryHeader header(core::CacheEntryType::result, - compression::type_from_config(ctx.config), - compression::level_from_config(ctx.config), - time(nullptr), - CCACHE_VERSION, - ctx.config.namespace_()); - header.set_entry_size_from_payload_size(serializer.serialized_size()); - - core::FileWriter file_writer(atomic_result_file.stream()); - core::CacheEntryWriter writer(file_writer, header); - - util::Bytes payload; - payload.reserve(serializer.serialized_size()); - const auto serialize_result = serializer.serialize(payload); - for (auto [file_number, source_path] : serialize_result.raw_files) { + core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result); + const auto cache_entry_data = core::CacheEntry::serialize(header, serializer); + + const auto raw_files = serializer.get_raw_files(); + if (!raw_files.empty()) { + Util::ensure_dir_exists(Util::dir_name(result_path)); + } + for (auto [file_number, source_path] : raw_files) { const auto dest_path = storage::primary::PrimaryStorage::get_raw_file_path( result_path, file_number); const auto old_stat = Stat::stat(dest_path); @@ -970,8 +941,8 @@ write_result(Context& ctx, Statistic::files_in_cache, (new_stat ? 1 : 0) - (old_stat ? 1 : 0)); } - writer.write(payload.data(), payload.size()); - writer.finalize(); + AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary); + atomic_result_file.write(cache_entry_data); atomic_result_file.commit(); return true; @@ -1937,8 +1908,11 @@ calculate_result_and_manifest_key(Context& ctx, { bool found_ccbin = false; + hash.hash_delimiter("cache entry version"); + hash.hash(core::k_cache_entry_format_version); + hash.hash_delimiter("result version"); - hash.hash(core::Result::k_version); + hash.hash(core::Result::k_format_version); if (direct_mode) { hash.hash_delimiter("manifest version"); @@ -2046,20 +2020,15 @@ from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) } try { - File file(*result_path, "rb"); - if (!file) { - throw core::Error( - FMT("Failed to open {}: {}", *result_path, strerror(errno))); - } - core::FileReader file_reader(file.get()); - core::CacheEntryReader cache_entry_reader(file_reader); - std::vector payload; - payload.resize(cache_entry_reader.header().payload_size()); - cache_entry_reader.read(payload.data(), payload.size()); - core::Result::Deserializer deserializer(payload); + const auto cache_entry_data = util::value_or_throw( + util::read_file(*result_path), + FMT("Failed to read {}: ", *result_path)); + + core::CacheEntry cache_entry(cache_entry_data); + cache_entry.verify_checksum(); + core::Result::Deserializer deserializer(cache_entry.payload()); core::ResultRetriever result_retriever(ctx, result_path); deserializer.visit(result_retriever); - cache_entry_reader.finalize(); } catch (core::ResultRetriever::WriteError& e) { LOG( "Write error when retrieving result from {}: {}", *result_path, e.what()); diff --git a/src/compression/CMakeLists.txt b/src/compression/CMakeLists.txt deleted file mode 100644 index 61aeb9fac..000000000 --- a/src/compression/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set( - sources - Compressor.cpp - Decompressor.cpp - NullCompressor.cpp - NullDecompressor.cpp - ZstdCompressor.cpp - ZstdDecompressor.cpp - types.cpp -) - -target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/compression/Compressor.cpp b/src/compression/Compressor.cpp deleted file mode 100644 index dbcc665cf..000000000 --- a/src/compression/Compressor.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Compressor.hpp" - -#include "NullCompressor.hpp" -#include "ZstdCompressor.hpp" -#include "assertions.hpp" - -#include - -#include - -namespace compression { - -std::unique_ptr -Compressor::create_from_type(const Type type, - core::Writer& writer, - const int8_t compression_level) -{ - switch (type) { - case compression::Type::none: - return std::make_unique(writer); - - case compression::Type::zstd: - return std::make_unique(writer, compression_level); - } - - ASSERT(false); -} - -} // namespace compression diff --git a/src/compression/Compressor.hpp b/src/compression/Compressor.hpp deleted file mode 100644 index deebbd997..000000000 --- a/src/compression/Compressor.hpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include -#include - -namespace core { - -class Writer; - -} - -namespace compression { - -class Compressor : public core::Writer -{ -public: - virtual ~Compressor() = default; - - static std::unique_ptr - create_from_type(Type type, core::Writer& writer, int8_t compression_level); - - virtual int8_t actual_compression_level() const = 0; -}; - -} // namespace compression diff --git a/src/compression/Decompressor.cpp b/src/compression/Decompressor.cpp deleted file mode 100644 index 6bfa713df..000000000 --- a/src/compression/Decompressor.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Decompressor.hpp" - -#include "NullDecompressor.hpp" -#include "ZstdDecompressor.hpp" -#include "assertions.hpp" - -namespace compression { - -std::unique_ptr -Decompressor::create_from_type(Type type, core::Reader& reader) -{ - switch (type) { - case compression::Type::none: - return std::make_unique(reader); - - case compression::Type::zstd: - return std::make_unique(reader); - } - - ASSERT(false); -} - -} // namespace compression diff --git a/src/compression/Decompressor.hpp b/src/compression/Decompressor.hpp deleted file mode 100644 index 2223d59e0..000000000 --- a/src/compression/Decompressor.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include - -namespace compression { - -class Decompressor : public core::Reader -{ -public: - virtual ~Decompressor() = default; - - // Create a decompressor for the specified type. - static std::unique_ptr create_from_type(Type type, - core::Reader& reader); - - // Finalize decompression. - // - // This method checks that the end state of the compressed stream is correct - // and throws Error if not. - virtual void finalize() = 0; -}; - -} // namespace compression diff --git a/src/compression/NullCompressor.cpp b/src/compression/NullCompressor.cpp deleted file mode 100644 index a2cb76517..000000000 --- a/src/compression/NullCompressor.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullCompressor.hpp" - -#include - -namespace compression { - -NullCompressor::NullCompressor(core::Writer& writer) : m_writer(writer) -{ -} - -int8_t -NullCompressor::actual_compression_level() const -{ - return 0; -} - -void -NullCompressor::write(const void* const data, const size_t count) -{ - m_writer.write(data, count); -} - -void -NullCompressor::finalize() -{ - m_writer.finalize(); -} - -} // namespace compression diff --git a/src/compression/NullCompressor.hpp b/src/compression/NullCompressor.hpp deleted file mode 100644 index 154b568c7..000000000 --- a/src/compression/NullCompressor.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "Compressor.hpp" - -#include - -namespace compression { - -// A compressor of an uncompressed stream. -class NullCompressor : public Compressor, NonCopyable -{ -public: - explicit NullCompressor(core::Writer& writer); - - int8_t actual_compression_level() const override; - void write(const void* data, size_t count) override; - void finalize() override; - -private: - core::Writer& m_writer; -}; - -} // namespace compression diff --git a/src/compression/NullDecompressor.cpp b/src/compression/NullDecompressor.cpp deleted file mode 100644 index 81693e4c2..000000000 --- a/src/compression/NullDecompressor.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullDecompressor.hpp" - -#include - -namespace compression { - -NullDecompressor::NullDecompressor(core::Reader& reader) : m_reader(reader) -{ -} - -size_t -NullDecompressor::read(void* const data, const size_t count) -{ - return m_reader.read(data, count); -} - -void -NullDecompressor::finalize() -{ - bool eof; - try { - m_reader.read_int(); - eof = false; - } catch (core::Error&) { - eof = true; - } - if (!eof) { - throw core::Error("Garbage data at end of uncompressed stream"); - } -} - -} // namespace compression diff --git a/src/compression/NullDecompressor.hpp b/src/compression/NullDecompressor.hpp deleted file mode 100644 index c2c8f5deb..000000000 --- a/src/compression/NullDecompressor.hpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "Decompressor.hpp" - -#include - -namespace compression { - -// A decompressor of an uncompressed stream. -class NullDecompressor : public Decompressor, NonCopyable -{ -public: - explicit NullDecompressor(core::Reader& reader); - - size_t read(void* data, size_t count) override; - void finalize() override; - -private: - core::Reader& m_reader; -}; - -} // namespace compression diff --git a/src/compression/ZstdCompressor.cpp b/src/compression/ZstdCompressor.cpp deleted file mode 100644 index 73abe3b07..000000000 --- a/src/compression/ZstdCompressor.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ZstdCompressor.hpp" - -#include "Logging.hpp" -#include "assertions.hpp" - -#include - -#include - -#include - -namespace compression { - -ZstdCompressor::ZstdCompressor(core::Writer& writer, int8_t compression_level) - : m_writer(writer), - m_zstd_stream(ZSTD_createCStream()), - m_zstd_in(std::make_unique()), - m_zstd_out(std::make_unique()) -{ - if (compression_level == 0) { - compression_level = default_compression_level; - LOG("Using default compression level {}", compression_level); - } - - // libzstd 1.3.4 and newer support negative levels. However, the query - // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection - // based on version instead. - if (ZSTD_versionNumber() < 10304 && compression_level < 1) { - LOG( - "Using compression level 1 (minimum level supported by libzstd) instead" - " of {}", - compression_level); - compression_level = 1; - } - - m_compression_level = std::min(compression_level, ZSTD_maxCLevel()); - if (m_compression_level != compression_level) { - LOG("Using compression level {} (max libzstd level) instead of {}", - m_compression_level, - compression_level); - } - - size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level); - if (ZSTD_isError(ret)) { - ZSTD_freeCStream(m_zstd_stream); - throw core::Error("error initializing zstd compression stream"); - } -} - -ZstdCompressor::~ZstdCompressor() -{ - ZSTD_freeCStream(m_zstd_stream); -} - -int8_t -ZstdCompressor::actual_compression_level() const -{ - return m_compression_level; -} - -void -ZstdCompressor::write(const void* const data, const size_t count) -{ - m_zstd_in->src = data; - m_zstd_in->size = count; - m_zstd_in->pos = 0; - - int flush = data ? 0 : 1; - - size_t ret; - while (m_zstd_in->pos < m_zstd_in->size) { - uint8_t buffer[CCACHE_READ_BUFFER_SIZE]; - m_zstd_out->dst = buffer; - m_zstd_out->size = sizeof(buffer); - m_zstd_out->pos = 0; - ret = ZSTD_compressStream(m_zstd_stream, m_zstd_out.get(), m_zstd_in.get()); - ASSERT(!(ZSTD_isError(ret))); - const size_t compressed_bytes = m_zstd_out->pos; - if (compressed_bytes > 0) { - m_writer.write(buffer, compressed_bytes); - } - } - ret = flush; - while (ret > 0) { - uint8_t buffer[CCACHE_READ_BUFFER_SIZE]; - m_zstd_out->dst = buffer; - m_zstd_out->size = sizeof(buffer); - m_zstd_out->pos = 0; - ret = ZSTD_endStream(m_zstd_stream, m_zstd_out.get()); - const size_t compressed_bytes = m_zstd_out->pos; - if (compressed_bytes > 0) { - m_writer.write(buffer, compressed_bytes); - } - } -} - -void -ZstdCompressor::finalize() -{ - write(nullptr, 0); - m_writer.finalize(); -} - -} // namespace compression diff --git a/src/compression/ZstdCompressor.hpp b/src/compression/ZstdCompressor.hpp deleted file mode 100644 index efb6bfc08..000000000 --- a/src/compression/ZstdCompressor.hpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "Compressor.hpp" - -#include - -#include -#include - -struct ZSTD_CCtx_s; -struct ZSTD_inBuffer_s; -struct ZSTD_outBuffer_s; - -namespace compression { - -// A compressor of a Zstandard stream. -class ZstdCompressor : public Compressor, NonCopyable -{ -public: - ZstdCompressor(core::Writer& writer, int8_t compression_level); - - ~ZstdCompressor() override; - - int8_t actual_compression_level() const override; - void write(const void* data, size_t count) override; - void finalize() override; - - constexpr static uint8_t default_compression_level = 1; - -private: - core::Writer& m_writer; - ZSTD_CCtx_s* m_zstd_stream; - std::unique_ptr m_zstd_in; - std::unique_ptr m_zstd_out; - int8_t m_compression_level; -}; - -} // namespace compression diff --git a/src/compression/ZstdDecompressor.cpp b/src/compression/ZstdDecompressor.cpp deleted file mode 100644 index 9ceab28ba..000000000 --- a/src/compression/ZstdDecompressor.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ZstdDecompressor.hpp" - -#include "assertions.hpp" - -#include - -namespace compression { - -ZstdDecompressor::ZstdDecompressor(core::Reader& reader) - : m_reader(reader), - m_input_size(0), - m_input_consumed(0), - m_zstd_stream(ZSTD_createDStream()), - m_reached_stream_end(false) -{ - const size_t ret = ZSTD_initDStream(m_zstd_stream); - if (ZSTD_isError(ret)) { - ZSTD_freeDStream(m_zstd_stream); - throw core::Error("failed to initialize zstd decompression stream"); - } -} - -ZstdDecompressor::~ZstdDecompressor() -{ - ZSTD_freeDStream(m_zstd_stream); -} - -size_t -ZstdDecompressor::read(void* const data, const size_t count) -{ - size_t bytes_read = 0; - while (bytes_read < count) { - ASSERT(m_input_size >= m_input_consumed); - if (m_input_size == m_input_consumed) { - m_input_size = m_reader.read(m_input_buffer, sizeof(m_input_buffer)); - m_input_consumed = 0; - } - - m_zstd_in.src = (m_input_buffer + m_input_consumed); - m_zstd_in.size = m_input_size - m_input_consumed; - m_zstd_in.pos = 0; - - m_zstd_out.dst = static_cast(data) + bytes_read; - m_zstd_out.size = count - bytes_read; - m_zstd_out.pos = 0; - const size_t ret = - ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); - if (ZSTD_isError(ret)) { - throw core::Error("Failed to read from zstd input stream"); - } - if (ret == 0) { - m_reached_stream_end = true; - break; - } - bytes_read += m_zstd_out.pos; - m_input_consumed += m_zstd_in.pos; - } - - return count; -} - -void -ZstdDecompressor::finalize() -{ - if (!m_reached_stream_end) { - throw core::Error("Garbage data at end of zstd input stream"); - } -} - -} // namespace compression diff --git a/src/compression/ZstdDecompressor.hpp b/src/compression/ZstdDecompressor.hpp deleted file mode 100644 index 0006ec407..000000000 --- a/src/compression/ZstdDecompressor.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "Decompressor.hpp" - -#include - -#include - -namespace compression { - -// A decompressor of a Zstandard stream. -class ZstdDecompressor : public Decompressor -{ -public: - explicit ZstdDecompressor(core::Reader& reader); - - ~ZstdDecompressor() override; - - size_t read(void* data, size_t count) override; - void finalize() override; - -private: - core::Reader& m_reader; - char m_input_buffer[CCACHE_READ_BUFFER_SIZE]; - size_t m_input_size; - size_t m_input_consumed; - ZSTD_DStream* m_zstd_stream; - ZSTD_inBuffer m_zstd_in; - ZSTD_outBuffer m_zstd_out; - bool m_reached_stream_end; -}; - -} // namespace compression diff --git a/src/compression/types.cpp b/src/compression/types.cpp deleted file mode 100644 index 04e0e9df7..000000000 --- a/src/compression/types.cpp +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) 2019-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "types.hpp" - -#include -#include -#include -#include -#include - -namespace compression { - -int8_t -level_from_config(const Config& config) -{ - return config.compression() ? config.compression_level() : 0; -} - -Type -type_from_config(const Config& config) -{ - return config.compression() ? Type::zstd : Type::none; -} - -Type -type_from_int(const uint8_t type) -{ - switch (type) { - case static_cast(Type::none): - return Type::none; - - case static_cast(Type::zstd): - return Type::zstd; - } - - throw core::Error(FMT("Unknown type: {}", type)); -} - -std::string -type_to_string(const Type type) -{ - switch (type) { - case Type::none: - return "none"; - - case Type::zstd: - return "zstd"; - } - - ASSERT(false); -} - -} // namespace compression diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 83a6d4b08..d2431be34 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -1,8 +1,6 @@ set( sources - CacheEntryHeader.cpp - CacheEntryReader.cpp - CacheEntryWriter.cpp + CacheEntry.cpp Manifest.cpp Result.cpp ResultExtractor.cpp diff --git a/src/core/CacheEntry.cpp b/src/core/CacheEntry.cpp new file mode 100644 index 000000000..fda2e1111 --- /dev/null +++ b/src/core/CacheEntry.cpp @@ -0,0 +1,327 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntry.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +const size_t k_static_header_fields_size = + sizeof(core::CacheEntry::Header::magic) + + sizeof(core::CacheEntry::Header::entry_format_version) + + sizeof(core::CacheEntry::Header::entry_type) + + sizeof(core::CacheEntry::Header::compression_type) + + sizeof(core::CacheEntry::Header::compression_level) + + sizeof(core::CacheEntry::Header::self_contained) + + sizeof(core::CacheEntry::Header::creation_time) + + sizeof(core::CacheEntry::Header::entry_size) + // ccache_version length field: + + 1 + // namespace_ length field: + + 1; + +const size_t k_epilogue_fields_size = sizeof(uint64_t) + sizeof(uint64_t); + +core::CacheEntryType +cache_entry_type_from_int(const uint8_t entry_type) +{ + switch (entry_type) { + case 0: + return core::CacheEntryType::result; + break; + case 1: + return core::CacheEntryType::manifest; + break; + default: + throw core::Error(FMT("Unknown entry type: {}", entry_type)); + } +} + +} // namespace + +namespace core { + +CacheEntry::Header::Header(const Config& config, + core::CacheEntryType entry_type) + : magic(k_ccache_magic), + entry_format_version(k_cache_entry_format_version), + entry_type(entry_type), + compression_type(compression_type_from_config(config)), + compression_level(compression_level_from_config(config)), + self_contained(entry_type != CacheEntryType::result + || !core::Result::Serializer::use_raw_files(config)), + creation_time(time(nullptr)), + ccache_version(CCACHE_VERSION), + namespace_(config.namespace_()), + entry_size(0) +{ + if (compression_level == 0) { + compression_level = default_compression_level; + LOG("Using default compression level {}", compression_level); + } +} + +CacheEntry::Header::Header(nonstd::span data) +{ + parse(data); +} + +CacheEntry::Header::Header(const std::string& path) +{ + const auto data = util::read_file_part(path, 0, 1000); + if (!data) { + throw core::Error(data.error()); + } + parse(*data); +} + +std::string +CacheEntry::Header::inspect() const +{ + std::string result; + result += result += FMT("Magic: {:04x}\n", magic); + result += FMT("Entry format version: {}\n", entry_format_version); + result += FMT("Entry type: {} ({})\n", + static_cast(entry_type), + to_string(entry_type)); + result += FMT("Compression type: {}\n", to_string(compression_type)); + result += FMT("Compression level: {}\n", compression_level); + result += FMT("Self-contained: {}\n", self_contained ? "yes" : "no"); + result += FMT("Creation time: {}\n", creation_time); + result += FMT("Ccache version: {}\n", ccache_version); + result += FMT("Namespace: {}\n", namespace_); + result += FMT("Entry size: {}\n", entry_size); + return result; +} + +void +CacheEntry::Header::parse(nonstd::span data) +{ + CacheEntryDataReader reader(data); + reader.read_int(magic); + if (magic != core::k_ccache_magic) { + throw core::Error(FMT("Bad magic value: 0x{:04x}", magic)); + } + + reader.read_int(entry_format_version); + if (entry_format_version != core::k_cache_entry_format_version) { + throw core::Error( + FMT("Unknown entry format version: {}", entry_format_version)); + } + + entry_type = cache_entry_type_from_int(reader.read_int()); + compression_type = compression_type_from_int(reader.read_int()); + reader.read_int(compression_level); + self_contained = bool(reader.read_int()); + reader.read_int(creation_time); + ccache_version = reader.read_str(reader.read_int()); + namespace_ = reader.read_str(reader.read_int()); + reader.read_int(entry_size); +} + +size_t +CacheEntry::Header::serialized_size() const +{ + return k_static_header_fields_size + ccache_version.length() + + namespace_.length(); +} + +void +CacheEntry::Header::serialize(util::Bytes& output) const +{ + core::CacheEntryDataWriter writer(output); + writer.write_int(magic); + writer.write_int(entry_format_version); + writer.write_int(static_cast(entry_type)); + writer.write_int(static_cast(compression_type)); + writer.write_int(compression_level); + writer.write_int(self_contained); + writer.write_int(creation_time); + writer.write_int(ccache_version.length()); + writer.write_str(ccache_version); + writer.write_int(namespace_.length()); + writer.write_str(namespace_); + writer.write_int(entry_size); +} + +uint32_t +CacheEntry::Header::uncompressed_payload_size() const +{ + return entry_size - serialized_size() - k_epilogue_fields_size; +} + +CacheEntry::CacheEntry(nonstd::span data) : m_header(data) +{ + const size_t non_payload_size = + m_header.serialized_size() + k_epilogue_fields_size; + if (data.size() <= non_payload_size) { + throw core::Error("CacheEntry data underflow"); + } + m_payload = + data.subspan(m_header.serialized_size(), data.size() - non_payload_size); + m_checksum = data.last(k_epilogue_fields_size); + + switch (m_header.compression_type) { + case CompressionType::none: + break; + + case CompressionType::zstd: + m_uncompressed_payload.reserve(m_header.uncompressed_payload_size()); + util::throw_on_error( + util::zstd_decompress( + m_payload, m_uncompressed_payload, m_uncompressed_payload.capacity()), + "Cache entry payload decompression error: "); + + break; + } +} + +void +CacheEntry::verify_checksum() const +{ + util::Bytes header_data; + m_header.serialize(header_data); + + util::XXH3_128 checksum; + checksum.update(header_data); + checksum.update(m_payload); + const auto actual = checksum.digest(); + + if (actual != m_checksum) { + throw core::Error( + FMT("Incorrect checksum (actual {}, expected {})", + Util::format_base16(actual.data(), actual.size()), + Util::format_base16(m_checksum.data(), m_checksum.size()))); + } +} + +const CacheEntry::Header& +CacheEntry::header() const +{ + return m_header; +} + +nonstd::span +CacheEntry::payload() const +{ + return m_header.compression_type == CompressionType::none + ? m_payload + : nonstd::span(m_uncompressed_payload); +} + +util::Bytes +CacheEntry::serialize(const CacheEntry::Header& header, + Serializer& payload_serializer) +{ + return do_serialize( + header, + payload_serializer.serialized_size(), + [&payload_serializer](util::Bytes& result, const CacheEntry::Header& hdr) { + switch (hdr.compression_type) { + case CompressionType::none: + payload_serializer.serialize(result); + break; + + case CompressionType::zstd: + util::Bytes payload; + payload_serializer.serialize(payload); + util::throw_on_error( + util::zstd_compress(payload, result, hdr.compression_level), + "Cache entry payload compression error: "); + break; + } + }); +} + +util::Bytes +CacheEntry::serialize(const CacheEntry::Header& header, + nonstd::span payload) +{ + return do_serialize( + header, + payload.size(), + [&payload](util::Bytes& result, const CacheEntry::Header& hdr) { + switch (hdr.compression_type) { + case CompressionType::none: + result.insert(result.end(), payload.begin(), payload.end()); + break; + + case CompressionType::zstd: + util::throw_on_error( + util::zstd_compress(payload, result, hdr.compression_level), + "Cache entry payload compression error: "); + break; + } + }); +} + +util::Bytes +CacheEntry::do_serialize( + const CacheEntry::Header& header, + size_t serialized_payload_size, + std::function serialize_payload) +{ + CacheEntry::Header hdr(header); + const size_t non_payload_size = + hdr.serialized_size() + k_epilogue_fields_size; + hdr.entry_size = non_payload_size + serialized_payload_size; + + if (hdr.compression_type == CompressionType::zstd) { + const auto [level, explanation] = + util::zstd_supported_compression_level(hdr.compression_level); + if (!explanation.empty()) { + LOG("Using ZSTD compression level {} ({}) instead of {}", + level, + explanation, + hdr.compression_level); + } + hdr.compression_level = level; + } + + const size_t max_serialized_size = + hdr.compression_type == CompressionType::zstd + ? (non_payload_size + util::zstd_compress_bound(serialized_payload_size)) + : hdr.entry_size; + util::Bytes result; + result.reserve(max_serialized_size); + + hdr.serialize(result); + serialize_payload(result, hdr); + + util::XXH3_128 checksum; + checksum.update(result); + const auto digest = checksum.digest(); + result.insert(result.end(), digest.begin(), digest.end()); + + return result; +} + +} // namespace core diff --git a/src/core/CacheEntry.hpp b/src/core/CacheEntry.hpp new file mode 100644 index 000000000..c4adecfc4 --- /dev/null +++ b/src/core/CacheEntry.hpp @@ -0,0 +1,135 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + +// Cache entry format +// ================== +// +// Integers are big-endian. +// +// ::=
+//
::= +// +// +// ::= uint16_t (0xccac) +// ::= uint8_t +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= 0/1 (uint8_t) ; whether suitable for secondary storage +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= int8_t +// ::= uint64_t (Unix epoch time when entry was created) +// ::= string length (uint8_t) + string data +// ::= string length (uint8_t) + string data +// ::= uint64_t ; = size of entry in uncompressed form +// ::= depends on entry_type; potentially compressed +// ::= +// ::= uint64_t ; XXH3-128 (high bits) of
+ +// ::= uint64_t ; XXH3-128 (low bits) of
+ + +class Config; + +namespace core { + +const uint16_t k_ccache_magic = 0xccac; + +// Version 0: +// - First version. +// Version 1: +// - Added self_contained field. +// - The checksum is now for the (potentially) compressed payload instead of +// the uncompressed payload, and the checksum is now always stored +// uncompressed. +const uint16_t k_cache_entry_format_version = 1; + +class CacheEntry +{ +public: + constexpr static uint8_t default_compression_level = 1; + + class Header + { + public: + Header(const Config& config, CacheEntryType entry_type); + explicit Header(nonstd::span data); + explicit Header(const std::string& path); + + std::string inspect() const; + + uint16_t magic; + uint8_t entry_format_version; + CacheEntryType entry_type; + CompressionType compression_type; + int8_t compression_level; + bool self_contained; + uint64_t creation_time; + std::string ccache_version; + std::string namespace_; + uint64_t entry_size; + + size_t serialized_size() const; + void serialize(util::Bytes& output) const; + uint32_t uncompressed_payload_size() const; + + private: + void parse(nonstd::span data); + }; + + explicit CacheEntry(nonstd::span data); + + void verify_checksum() const; + const Header& header() const; + + // Return uncompressed payload. + nonstd::span payload() const; + + static util::Bytes serialize(const Header& header, + Serializer& payload_serializer); + static util::Bytes serialize(const Header& header, + nonstd::span payload); + +private: + Header m_header; + nonstd::span m_payload; // Potentially compressed + util::Bytes m_checksum; + + mutable util::Bytes m_uncompressed_payload; + + static util::Bytes + do_serialize(const Header& header, + size_t serialized_payload_size, + std::function + serialize_payload); +}; + +} // namespace core diff --git a/src/core/CacheEntryHeader.cpp b/src/core/CacheEntryHeader.cpp deleted file mode 100644 index b1bbe9a28..000000000 --- a/src/core/CacheEntryHeader.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (C) 2019-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryHeader.hpp" - -#include -#include - -const size_t k_static_header_fields_size = - sizeof(core::CacheEntryHeader::magic) - + sizeof(core::CacheEntryHeader::entry_format_version) - + sizeof(core::CacheEntryHeader::entry_type) - + sizeof(core::CacheEntryHeader::compression_type) - + sizeof(core::CacheEntryHeader::compression_level) - + sizeof(core::CacheEntryHeader::creation_time) - + sizeof(core::CacheEntryHeader::entry_size) - // ccache_version length field: - + 1 - // namespace_ length field: - + 1; - -const size_t k_static_epilogue_fields_size = - sizeof(uint64_t) + sizeof(uint64_t); - -namespace core { - -CacheEntryHeader::CacheEntryHeader(const core::CacheEntryType entry_type_, - const compression::Type compression_type_, - const int8_t compression_level_, - const uint64_t creation_time_, - const std::string& ccache_version_, - const std::string& namespace_arg, - const uint64_t entry_size_) - : magic(k_ccache_magic), - entry_format_version(k_entry_format_version), - entry_type(entry_type_), - compression_type(compression_type_), - compression_level(compression_level_), - creation_time(creation_time_), - ccache_version(ccache_version_), - namespace_(namespace_arg), - entry_size(entry_size_) -{ -} - -uint32_t -CacheEntryHeader::payload_size() const -{ - const auto payload_size = entry_size - non_payload_size(); - // In order to support 32-bit ccache builds, restrict size to uint32_t for - // now. This restriction can be lifted when we drop 32-bit support. - const auto max = std::numeric_limits::max(); - if (payload_size > max) { - throw core::Error( - FMT("Serialized result too large ({} > {})", payload_size, max)); - } - - return payload_size; -} - -void -CacheEntryHeader::set_entry_size_from_payload_size(const uint64_t payload_size) -{ - entry_size = non_payload_size() + payload_size; -} - -void -CacheEntryHeader::inspect(FILE* const stream) const -{ - PRINT(stream, "Magic: {:04x}\n", magic); - PRINT(stream, "Entry format version: {}\n", entry_format_version); - PRINT(stream, - "Entry type: {} ({})\n", - static_cast(entry_type), - to_string(entry_type)); - PRINT(stream, - "Compression type: {}\n", - compression::type_to_string(compression_type)); - PRINT(stream, "Compression level: {}\n", compression_level); - PRINT(stream, "Creation time: {}\n", creation_time); - PRINT(stream, "Ccache version: {}\n", ccache_version); - PRINT(stream, "Namespace: {}\n", namespace_); - PRINT(stream, "Entry size: {}\n", entry_size); -} - -size_t -CacheEntryHeader::non_payload_size() const -{ - return k_static_header_fields_size + ccache_version.length() - + namespace_.length() + k_static_epilogue_fields_size; -} - -} // namespace core diff --git a/src/core/CacheEntryHeader.hpp b/src/core/CacheEntryHeader.hpp deleted file mode 100644 index a0f689188..000000000 --- a/src/core/CacheEntryHeader.hpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (C) 2021-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include - -// Cache entry format -// ================== -// -// Integers are big-endian. -// -// ::=
-//
::= -// -// -// ::= uint16_t (0xccac) -// ::= uint8_t -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= int8_t -// ::= uint64_t (Unix epoch time when entry was created) -// ::= string length (uint8_t) + string data -// ::= string length (uint8_t) + string data -// ::= uint64_t ; = size of file if stored uncompressed -// ; potentially compressed from here -// ::= depends on entry_type -// ::= -// ::= uint64_t ; XXH3-128 (high bits) of entry bytes -// ::= uint64_t ; XXH3-128 (low bits) of entry bytes - -namespace core { - -const uint16_t k_ccache_magic = 0xccac; -const uint16_t k_entry_format_version = 0; - -struct CacheEntryHeader -{ - CacheEntryHeader(core::CacheEntryType entry_type, - compression::Type compression_type, - int8_t compression_level, - uint64_t creation_time, - const std::string& ccache_version, - const std::string& namespace_, - uint64_t entry_size = 0); - - uint16_t magic; - uint8_t entry_format_version; - core::CacheEntryType entry_type; - compression::Type compression_type; - int8_t compression_level; - uint64_t creation_time; - std::string ccache_version; - std::string namespace_; - uint64_t entry_size; - - uint32_t payload_size() const; - void set_entry_size_from_payload_size(uint64_t payload_size); - void inspect(FILE* stream) const; - -private: - size_t non_payload_size() const; -}; - -} // namespace core diff --git a/src/core/CacheEntryReader.cpp b/src/core/CacheEntryReader.cpp deleted file mode 100644 index ea166d6d0..000000000 --- a/src/core/CacheEntryReader.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (C) 2019-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryReader.hpp" - -#include -#include - -namespace { - -core::CacheEntryType -cache_entry_type_from_int(const uint8_t entry_type) -{ - switch (entry_type) { - case 0: - return core::CacheEntryType::result; - break; - case 1: - return core::CacheEntryType::manifest; - break; - default: - throw core::Error(FMT("Unknown entry type: {}", entry_type)); - } -} - -} // namespace - -namespace core { - -CacheEntryReader::CacheEntryReader(core::Reader& reader) - : m_checksumming_reader(reader) -{ - const auto magic = m_checksumming_reader.read_int(); - if (magic != core::k_ccache_magic) { - throw core::Error(FMT("Bad magic value: 0x{:04x}", magic)); - } - - const auto entry_format_version = m_checksumming_reader.read_int(); - if (entry_format_version != core::k_entry_format_version) { - throw core::Error( - FMT("Unknown entry format version: {}", entry_format_version)); - } - - const auto entry_type = m_checksumming_reader.read_int(); - const auto compression_type = m_checksumming_reader.read_int(); - const auto compression_level = m_checksumming_reader.read_int(); - const auto creation_time = m_checksumming_reader.read_int(); - const auto ccache_version = - m_checksumming_reader.read_str(m_checksumming_reader.read_int()); - const auto tag = - m_checksumming_reader.read_str(m_checksumming_reader.read_int()); - const auto entry_size = m_checksumming_reader.read_int(); - - m_header = std::make_unique( - cache_entry_type_from_int(entry_type), - compression::type_from_int(compression_type), - compression_level, - creation_time, - ccache_version, - tag, - entry_size); - - m_decompressor = compression::Decompressor::create_from_type( - m_header->compression_type, reader); - m_checksumming_reader.set_reader(*m_decompressor); -} - -size_t -CacheEntryReader::read(void* const data, const size_t count) -{ - return m_checksumming_reader.read(data, count); -} - -void -CacheEntryReader::finalize() -{ - const util::XXH3_128::Digest actual = m_checksumming_reader.digest(); - util::XXH3_128::Digest expected; - m_decompressor->read(expected.bytes(), expected.size()); - - // actual == null_digest: Checksumming is not enabled now. - // expected == null_digest: Checksumming was not enabled when the entry was - // created. - const util::XXH3_128::Digest null_digest; - - if (actual != expected && actual != null_digest && expected != null_digest) { - throw core::Error( - FMT("Incorrect checksum (actual {}, expected {})", - Util::format_base16(actual.bytes(), actual.size()), - Util::format_base16(expected.bytes(), expected.size()))); - } - - m_decompressor->finalize(); -} - -} // namespace core diff --git a/src/core/CacheEntryReader.hpp b/src/core/CacheEntryReader.hpp deleted file mode 100644 index a4e518191..000000000 --- a/src/core/CacheEntryReader.hpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include -#include -#include -#include - -namespace core { - -// This class knows how to read a cache entry with a format described in -// CacheEntryHeader. -class CacheEntryReader : public Reader -{ -public: - // Read cache entry data from `reader`. - CacheEntryReader(Reader& reader); - - size_t read(void* data, size_t count) override; - using Reader::read; - - // Close for reading. - // - // This method potentially verifies the end state after reading the cache - // entry and throws `core::Error` if any integrity issues are found. - void finalize(); - - const CacheEntryHeader& header() const; - -private: - ChecksummingReader m_checksumming_reader; - std::unique_ptr m_header; - util::XXH3_128 m_checksum; - std::unique_ptr m_decompressor; -}; - -inline const CacheEntryHeader& -CacheEntryReader::header() const -{ - return *m_header; -} - -} // namespace core diff --git a/src/core/CacheEntryWriter.cpp b/src/core/CacheEntryWriter.cpp deleted file mode 100644 index b2de99d11..000000000 --- a/src/core/CacheEntryWriter.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryWriter.hpp" - -#include - -namespace core { - -CacheEntryWriter::CacheEntryWriter(core::Writer& writer, - const CacheEntryHeader& header) - : m_checksumming_writer(writer), - m_compressor(compression::Compressor::create_from_type( - header.compression_type, writer, header.compression_level)) -{ - m_checksumming_writer.write_int(header.magic); - m_checksumming_writer.write_int(header.entry_format_version); - m_checksumming_writer.write_int(static_cast(header.entry_type)); - m_checksumming_writer.write_int( - static_cast(header.compression_type)); - m_checksumming_writer.write_int(m_compressor->actual_compression_level()); - m_checksumming_writer.write_int(header.creation_time); - m_checksumming_writer.write_int(header.ccache_version.length()); - m_checksumming_writer.write_str(header.ccache_version); - m_checksumming_writer.write_int(header.namespace_.length()); - m_checksumming_writer.write_str(header.namespace_); - m_checksumming_writer.write_int(header.entry_size); - - m_checksumming_writer.set_writer(*m_compressor); -} - -void -CacheEntryWriter::write(const void* const data, const size_t count) -{ - m_checksumming_writer.write(data, count); -} - -void -CacheEntryWriter::finalize() -{ - const auto digest = m_checksumming_writer.digest(); - m_compressor->write(digest.bytes(), digest.size()); - m_compressor->finalize(); -} - -} // namespace core diff --git a/src/core/CacheEntryWriter.hpp b/src/core/CacheEntryWriter.hpp deleted file mode 100644 index 789eb0c85..000000000 --- a/src/core/CacheEntryWriter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include -#include - -namespace core { - -struct CacheEntryHeader; - -// This class knows how to write a cache entry with a format described in -// CacheEntryHeader. -class CacheEntryWriter : public Writer -{ -public: - CacheEntryWriter(Writer& writer, const CacheEntryHeader& header); - - void write(const void* data, size_t count) override; - using Writer::write; - - // Close for writing. - // - // This method potentially verifies the end state after writing the cache - // entry and throws `core::Error` if any integrity issues are found. - void finalize() override; - -private: - ChecksummingWriter m_checksumming_writer; - std::unique_ptr m_compressor; -}; - -} // namespace core diff --git a/src/core/ChecksummingReader.hpp b/src/core/ChecksummingReader.hpp deleted file mode 100644 index 41c569fa2..000000000 --- a/src/core/ChecksummingReader.hpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -namespace core { - -class ChecksummingReader : public Reader -{ -public: - ChecksummingReader(core::Reader& reader); - - using core::Reader::read; - size_t read(void* data, size_t count) override; - - void set_reader(core::Reader& reader); - - util::XXH3_128::Digest digest() const; - -private: - core::Reader* m_reader; - util::XXH3_128 m_checksum; -}; - -inline ChecksummingReader::ChecksummingReader(core::Reader& reader) - : m_reader(&reader) -{ -} - -inline size_t -ChecksummingReader::read(void* const data, const size_t count) -{ - const auto bytes_read = m_reader->read(data, count); - m_checksum.update(data, bytes_read); - return bytes_read; -} - -inline void -ChecksummingReader::set_reader(core::Reader& reader) -{ - m_reader = &reader; -} - -inline util::XXH3_128::Digest -ChecksummingReader::digest() const -{ - return m_checksum.digest(); -} - -} // namespace core diff --git a/src/core/ChecksummingWriter.hpp b/src/core/ChecksummingWriter.hpp deleted file mode 100644 index 1d6b009f6..000000000 --- a/src/core/ChecksummingWriter.hpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -namespace core { - -class ChecksummingWriter : public Writer -{ -public: - ChecksummingWriter(core::Writer& writer); - - using core::Writer::write; - void write(const void* data, size_t count) override; - void finalize() override; - - void set_writer(core::Writer& writer); - - util::XXH3_128::Digest digest() const; - -private: - core::Writer* m_writer; - util::XXH3_128 m_checksum; -}; - -inline ChecksummingWriter::ChecksummingWriter(core::Writer& writer) - : m_writer(&writer) -{ -} - -inline void -ChecksummingWriter::write(const void* const data, const size_t count) -{ - m_writer->write(data, count); - m_checksum.update(data, count); -} - -inline void -ChecksummingWriter::finalize() -{ - m_writer->finalize(); -} - -inline void -ChecksummingWriter::set_writer(core::Writer& writer) -{ - m_writer = &writer; -} - -inline util::XXH3_128::Digest -ChecksummingWriter::digest() const -{ - return m_checksum.digest(); -} - -} // namespace core diff --git a/src/core/FileReader.hpp b/src/core/FileReader.hpp deleted file mode 100644 index c7c7b8ced..000000000 --- a/src/core/FileReader.hpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include - -namespace core { - -class FileReader : public Reader -{ -public: - FileReader(FILE* stream); - - size_t read(void* data, size_t size) override; - -private: - FILE* m_stream; -}; - -inline FileReader::FileReader(FILE* stream) : m_stream(stream) -{ -} - -inline size_t -FileReader::read(void* const data, const size_t size) -{ - if (size == 0) { - return 0; - } - const auto bytes_read = fread(data, 1, size, m_stream); - if (bytes_read == 0) { - throw core::Error("Failed to read from file stream"); - } - return bytes_read; -} - -} // namespace core diff --git a/src/core/FileWriter.hpp b/src/core/FileWriter.hpp deleted file mode 100644 index f4cdc0bd4..000000000 --- a/src/core/FileWriter.hpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include - -namespace core { - -class FileWriter : public Writer -{ -public: - FileWriter(FILE* stream); - - void write(const void* data, size_t size) override; - void finalize() override; - -private: - FILE* m_stream; -}; - -inline FileWriter::FileWriter(FILE* const stream) : m_stream(stream) -{ -} - -inline void -FileWriter::write(const void* const data, const size_t size) -{ - if (size > 0 && fwrite(data, size, 1, m_stream) != 1) { - throw core::Error("Failed to write to stream"); - } -} - -inline void -FileWriter::finalize() -{ - fflush(m_stream); -} - -} // namespace core diff --git a/src/core/Manifest.cpp b/src/core/Manifest.cpp index 9b46be191..f2a3e9cc2 100644 --- a/src/core/Manifest.cpp +++ b/src/core/Manifest.cpp @@ -94,8 +94,7 @@ Manifest::read(nonstd::span data) const auto file_count = reader.read_int(); for (uint32_t i = 0; i < file_count; ++i) { - m_files.push_back( - std::string(reader.read_str(reader.read_int()))); + m_files.emplace_back(reader.read_str(reader.read_int())); } const auto file_info_count = reader.read_int(); @@ -231,7 +230,7 @@ Manifest::serialized_size() const } void -Manifest::serialize(util::Bytes& output) const +Manifest::serialize(util::Bytes& output) { core::CacheEntryDataWriter writer(output); diff --git a/src/core/Manifest.hpp b/src/core/Manifest.hpp index 267055a81..f277093f3 100644 --- a/src/core/Manifest.hpp +++ b/src/core/Manifest.hpp @@ -19,7 +19,7 @@ #pragma once #include -#include +#include #include @@ -33,7 +33,7 @@ class Context; namespace core { -class Manifest +class Manifest : public Serializer { public: static const uint8_t k_format_version; @@ -49,8 +49,9 @@ public: time_t time_of_compilation, bool save_timestamp); - uint32_t serialized_size() const; - void serialize(util::Bytes& output) const; + // core::Serializer + uint32_t serialized_size() const override; + void serialize(util::Bytes& output) override; void inspect(FILE* stream) const; diff --git a/src/core/Reader.hpp b/src/core/Reader.hpp deleted file mode 100644 index 317005ef3..000000000 --- a/src/core/Reader.hpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (C) 2021-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include -#include -#include - -namespace core { - -class Reader -{ -public: - virtual ~Reader() = default; - - // Read `count` bytes into `data`, returning the actual number of bytes read - // if not enough data is available. Throws `core::Error` on failure, e.g. if - // no bytes could be read. - virtual size_t read(void* data, size_t count) = 0; - - // Read an integer. Throws Error on failure. - template T read_int(); - - // Read an integer into `value`. Throws Error on failure. - template void read_int(T& value); - - // Read a string of length `length`. Throws `core::Error` on failure. - std::string read_str(size_t length); -}; - -template -inline T -Reader::read_int() -{ - uint8_t buffer[sizeof(T)]; - const auto bytes_read = read(buffer, sizeof(T)); - if (bytes_read != sizeof(T)) { - throw core::Error("Read underflow"); - } - T value; - Util::big_endian_to_int(buffer, value); - return value; -} - -template -inline void -Reader::read_int(T& value) -{ - value = read_int(); -} - -inline std::string -Reader::read_str(const size_t length) -{ - std::string value(length, 0); - const auto bytes_read = read(&value[0], length); - if (bytes_read != length) { - throw core::Error("Read underflow"); - } - return value; -} - -} // namespace core diff --git a/src/core/Result.cpp b/src/core/Result.cpp index 9e454b7f3..e2733778c 100644 --- a/src/core/Result.cpp +++ b/src/core/Result.cpp @@ -70,8 +70,6 @@ namespace { -const uint8_t k_result_format_version = 0; - // File data stored inside the result file. const uint8_t k_embedded_file_marker = 0; @@ -83,7 +81,7 @@ const uint8_t k_max_raw_file_entries = 10; bool should_store_raw_file(const Config& config, core::Result::FileType type) { - if (!config.file_clone() && !config.hard_link()) { + if (!core::Result::Serializer::use_raw_files(config)) { return false; } @@ -108,11 +106,9 @@ should_store_raw_file(const Config& config, core::Result::FileType type) } // namespace -namespace core { - -namespace Result { +namespace core::Result { -const uint8_t k_version = 1; +const uint8_t k_format_version = 0; const char* const k_unknown_file_type = ""; @@ -182,10 +178,10 @@ Deserializer::visit(Deserializer::Visitor& visitor) const { CacheEntryDataReader reader(m_data); const auto result_format_version = reader.read_int(); - if (result_format_version != k_result_format_version) { + if (result_format_version != k_format_version) { throw Error(FMT("Unknown result format version: {} != {}", result_format_version, - k_result_format_version)); + k_format_version)); } const auto n_files = reader.read_int(); @@ -262,13 +258,12 @@ Serializer::serialized_size() const return m_serialized_size; } -Serializer::SerializeResult +void Serializer::serialize(util::Bytes& output) { - SerializeResult serialize_result; CacheEntryDataWriter writer(output); - writer.write_int(k_result_format_version); + writer.write_int(k_format_version); writer.write_int(m_file_entries.size()); uint8_t file_number = 0; @@ -296,8 +291,8 @@ Serializer::serialize(util::Bytes& output) writer.write_int(file_size); if (store_raw) { - serialize_result.raw_files.emplace(file_number, - std::get(entry.data)); + m_raw_files.push_back( + RawFile{file_number, std::get(entry.data)}); } else if (is_file_entry) { const auto& path = std::get(entry.data); const auto data = util::value_or_throw( @@ -309,10 +304,18 @@ Serializer::serialize(util::Bytes& output) ++file_number; } +} - return serialize_result; +bool +Serializer::use_raw_files(const Config& config) +{ + return config.file_clone() || config.hard_link(); } -} // namespace Result +const std::vector& +Serializer::get_raw_files() const +{ + return m_raw_files; +} -} // namespace core +} // namespace core::Result diff --git a/src/core/Result.hpp b/src/core/Result.hpp index 140e9b236..805bcd0f8 100644 --- a/src/core/Result.hpp +++ b/src/core/Result.hpp @@ -18,14 +18,13 @@ #pragma once -#include +#include #include #include #include #include -#include #include #include @@ -38,7 +37,7 @@ class CacheEntryDataParser; namespace Result { -extern const uint8_t k_version; +extern const uint8_t k_format_version; extern const char* const k_unknown_file_type; @@ -120,7 +119,7 @@ private: }; // This class knows how to serialize a result cache entry. -class Serializer +class Serializer : public core::Serializer { public: Serializer(const Config& config); @@ -132,15 +131,20 @@ public: // Register a file path whose content should be included in the result. void add_file(FileType file_type, const std::string& path); - uint32_t serialized_size() const; + // core::Serializer + uint32_t serialized_size() const override; + void serialize(util::Bytes& output) override; - struct SerializeResult + static bool use_raw_files(const Config& config); + + struct RawFile { - // Raw files to store in primary storage. - std::unordered_map raw_files; + uint8_t file_number; + std::string path; }; - SerializeResult serialize(util::Bytes& output); + // Get raw files to store in primary storage. + const std::vector& get_raw_files() const; private: const Config& m_config; @@ -152,6 +156,8 @@ private: std::variant, std::string> data; }; std::vector m_file_entries; + + std::vector m_raw_files; }; } // namespace Result diff --git a/src/compression/types.hpp b/src/core/Serializer.hpp similarity index 69% rename from src/compression/types.hpp rename to src/core/Serializer.hpp index 4b0e35e63..bb5ddd034 100644 --- a/src/compression/types.hpp +++ b/src/core/Serializer.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,24 +18,20 @@ #pragma once -#include -#include - -class Config; +#include -namespace compression { - -enum class Type : uint8_t { - none = 0, - zstd = 1, -}; +#include -int8_t level_from_config(const Config& config); - -Type type_from_config(const Config& config); +#include -Type type_from_int(uint8_t type); +namespace core { -std::string type_to_string(Type type); +class Serializer +{ +public: + virtual ~Serializer() = default; + virtual uint32_t serialized_size() const = 0; + virtual void serialize(util::Bytes& output) = 0; +}; -} // namespace compression +} // namespace core diff --git a/src/core/Writer.hpp b/src/core/Writer.hpp deleted file mode 100644 index 7474a5ee6..000000000 --- a/src/core/Writer.hpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include -#include - -#include -#include -#include - -namespace core { - -class Writer -{ -public: - virtual ~Writer() = default; - - // Write `count` bytes from `data`. Throws `core::Error` on failure. - virtual void write(const void* data, size_t count) = 0; - - // Write integer `value`. Throws `core::Error` on failure. - template void write_int(T value); - - // Write `value`. Throws `core::Error` on failure. - void write_str(const std::string& value); - - // Finalize writing, e.g. flush written bytes and potentially check for error - // states. Throws `core::Error` on failure. - virtual void finalize() = 0; -}; - -template -inline void -Writer::write_int(const T value) -{ - uint8_t buffer[sizeof(T)]; - Util::int_to_big_endian(value, buffer); - write(buffer, sizeof(T)); -} - -inline void -Writer::write_str(const std::string& value) -{ - write(value.data(), value.length()); -} - -} // namespace core diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index 28b7fc8c8..717ca8e5c 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -25,8 +25,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -157,39 +156,59 @@ configuration_printer(const std::string& key, PRINT(stdout, "({}) {} = {}\n", origin, key, value); } +static nonstd::expected, std::string> +read_from_path_or_stdin(const std::string& path) +{ + if (path == "-") { + std::vector output; + const auto result = + util::read_fd(STDIN_FILENO, [&](const uint8_t* data, size_t size) { + output.insert(output.end(), data, data + size); + }); + if (!result) { + return nonstd::make_unexpected( + FMT("Failed to read from stdin: {}", result.error())); + } + return output; + } else { + const auto result = util::read_file>(path); + if (!result) { + return nonstd::make_unexpected( + FMT("Failed to read from {}: {}", path, result.error())); + } + return *result; + } +} + static int inspect_path(const std::string& path) { - File file = path == "-" ? File(stdin) : File(path, "rb"); - if (!file) { - PRINT(stderr, "Error: Failed to open \"{}\"", path); + const auto cache_entry_data = read_from_path_or_stdin(path); + if (!cache_entry_data) { + PRINT(stderr, "Error: {}\n", cache_entry_data.error()); return EXIT_FAILURE; } - core::FileReader file_reader(file.get()); - core::CacheEntryReader cache_entry_reader(file_reader); - const auto& header = cache_entry_reader.header(); - header.inspect(stdout); + core::CacheEntry cache_entry(*cache_entry_data); + fputs(cache_entry.header().inspect().c_str(), stdout); - std::vector data; - data.resize(cache_entry_reader.header().payload_size()); - cache_entry_reader.read(data.data(), data.size()); + const auto payload = cache_entry.payload(); - switch (header.entry_type) { + switch (cache_entry.header().entry_type) { case core::CacheEntryType::manifest: { core::Manifest manifest; - manifest.read(data); + manifest.read(payload); manifest.inspect(stdout); break; } case core::CacheEntryType::result: - Result::Deserializer result_deserializer(data); + Result::Deserializer result_deserializer(payload); ResultInspector result_inspector(stdout); result_deserializer.visit(result_inspector); break; } - cache_entry_reader.finalize(); + cache_entry.verify_checksum(); return EXIT_SUCCESS; } @@ -423,12 +442,12 @@ process_main_options(int argc, const char* const* argv) util::XXH3_128 checksum; Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY)); if (fd) { - util::read_fd(*fd, [&checksum](const void* data, size_t size) { - checksum.update(data, size); + util::read_fd(*fd, [&checksum](const uint8_t* data, size_t size) { + checksum.update({data, size}); }); const auto digest = checksum.digest(); PRINT( - stdout, "{}\n", Util::format_base16(digest.bytes(), digest.size())); + stdout, "{}\n", Util::format_base16(digest.data(), digest.size())); } else { PRINT(stderr, "Error: Failed to checksum {}\n", arg); } @@ -446,27 +465,25 @@ process_main_options(int argc, const char* const* argv) } case EXTRACT_RESULT: { - File file = arg == "-" ? File(stdin) : File(arg, "rb"); - if (!file) { - PRINT(stderr, "Error: Failed to open \"{}\"", arg); + const auto cache_entry_data = read_from_path_or_stdin(arg); + if (!cache_entry_data) { + PRINT(stderr, "Error: \"{}\"", cache_entry_data.error()); return EXIT_FAILURE; } std::optional get_raw_file_path; - if (arg == "-") { + if (arg != "-") { get_raw_file_path = [&](uint8_t file_number) { return storage::primary::PrimaryStorage::get_raw_file_path( arg, file_number); }; } ResultExtractor result_extractor(".", get_raw_file_path); - core::FileReader file_reader(file.get()); - core::CacheEntryReader cache_entry_reader(file_reader); - std::vector data; - data.resize(cache_entry_reader.header().payload_size()); - cache_entry_reader.read(data.data(), data.size()); - Result::Deserializer result_deserializer(data); + core::CacheEntry cache_entry(*cache_entry_data); + const auto payload = cache_entry.payload(); + + Result::Deserializer result_deserializer(payload); result_deserializer.visit(result_extractor); - cache_entry_reader.finalize(); + cache_entry.verify_checksum(); return EXIT_SUCCESS; } diff --git a/src/core/types.cpp b/src/core/types.cpp index 6529a2f77..ecd05e520 100644 --- a/src/core/types.cpp +++ b/src/core/types.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,6 +18,11 @@ #include "types.hpp" +#include +#include +#include +#include + namespace core { std::string @@ -35,4 +40,44 @@ to_string(const CacheEntryType type) } } +int8_t +compression_level_from_config(const Config& config) +{ + return config.compression() ? config.compression_level() : 0; +} + +CompressionType +compression_type_from_config(const Config& config) +{ + return config.compression() ? CompressionType::zstd : CompressionType::none; +} + +CompressionType +compression_type_from_int(const uint8_t type) +{ + switch (type) { + case static_cast(CompressionType::none): + return CompressionType::none; + + case static_cast(CompressionType::zstd): + return CompressionType::zstd; + } + + throw core::Error(FMT("Unknown type: {}", type)); +} + +std::string +to_string(const CompressionType type) +{ + switch (type) { + case CompressionType::none: + return "none"; + + case CompressionType::zstd: + return "zstd"; + } + + ASSERT(false); +} + } // namespace core diff --git a/src/core/types.hpp b/src/core/types.hpp index f993e2569..592eb4fe3 100644 --- a/src/core/types.hpp +++ b/src/core/types.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,10 +21,25 @@ #include #include +class Config; + namespace core { enum class CacheEntryType : uint8_t { result = 0, manifest = 1 }; std::string to_string(CacheEntryType type); +enum class CompressionType : uint8_t { + none = 0, + zstd = 1, +}; + +int8_t compression_level_from_config(const Config& config); + +CompressionType compression_type_from_config(const Config& config); + +CompressionType compression_type_from_int(uint8_t type); + +std::string to_string(CompressionType type); + } // namespace core diff --git a/src/storage/primary/PrimaryStorage_cleanup.cpp b/src/storage/primary/PrimaryStorage_cleanup.cpp index 18671ac6f..d3a0b1b93 100644 --- a/src/storage/primary/PrimaryStorage_cleanup.cpp +++ b/src/storage/primary/PrimaryStorage_cleanup.cpp @@ -24,12 +24,13 @@ #include #include #include -#include -#include +#include +#include #include #include #include #include +#include #include #ifdef INODE_CACHE_SUPPORTED @@ -171,10 +172,8 @@ PrimaryStorage::clean_dir(const std::string& subdir, if (namespace_) { try { - File file_stream(file.path(), "rb"); - core::FileReader file_reader(*file_stream); - core::CacheEntryReader reader(file_reader); - if (reader.header().namespace_ != *namespace_) { + core::CacheEntry::Header header(file.path()); + if (header.namespace_ != *namespace_) { continue; } } catch (core::Error&) { diff --git a/src/storage/primary/PrimaryStorage_compress.cpp b/src/storage/primary/PrimaryStorage_compress.cpp index 333fc1f40..bceed4555 100644 --- a/src/storage/primary/PrimaryStorage_compress.cpp +++ b/src/storage/primary/PrimaryStorage_compress.cpp @@ -24,17 +24,14 @@ #include #include #include -#include -#include -#include -#include -#include +#include #include #include #include #include #include #include +#include #include #include @@ -115,80 +112,39 @@ RecompressionStatistics::incompressible_size() const } // namespace -static File -open_file(const std::string& path, const char* const mode) -{ - File f(path, mode); - if (!f) { - throw core::Error( - FMT("failed to open {} for reading: {}", path, strerror(errno))); - } - return f; -} - -static std::unique_ptr -create_reader(const CacheFile& cache_file, core::Reader& reader) -{ - if (cache_file.type() == CacheFile::Type::unknown) { - throw core::Error(FMT("unknown file type for {}", cache_file.path())); - } - - return std::make_unique(reader); -} - -static std::unique_ptr -create_writer(core::Writer& writer, const core::CacheEntryHeader& header) -{ - return std::make_unique(writer, header); -} - static void recompress_file(RecompressionStatistics& statistics, const std::string& stats_file, const CacheFile& cache_file, const std::optional level) { - auto file = open_file(cache_file.path(), "rb"); - core::FileReader file_reader(file.get()); - auto reader = create_reader(cache_file, file_reader); + core::CacheEntry::Header header(cache_file.path()); - const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - const uint64_t content_size = reader->header().entry_size; const int8_t wanted_level = - level - ? (*level == 0 ? compression::ZstdCompressor::default_compression_level - : *level) - : 0; + level ? (*level == 0 ? core::CacheEntry::default_compression_level : *level) + : 0; + const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - if (reader->header().compression_level == wanted_level) { - statistics.update(content_size, old_stat.size(), old_stat.size(), 0); + if (header.compression_level == wanted_level) { + statistics.update(header.entry_size, old_stat.size(), old_stat.size(), 0); return; } - LOG("Recompressing {} to {}", - cache_file.path(), - level ? FMT("level {}", wanted_level) : "uncompressed"); - AtomicFile atomic_new_file(cache_file.path(), AtomicFile::Mode::binary); - core::FileWriter file_writer(atomic_new_file.stream()); - auto header = reader->header(); + const auto cache_file_data = util::value_or_throw( + util::read_file(cache_file.path()), + FMT("Failed to read {}: ", cache_file.path())); + core::CacheEntry cache_entry(cache_file_data); + cache_entry.verify_checksum(); + + header.entry_format_version = core::k_cache_entry_format_version; header.compression_type = - level ? compression::Type::zstd : compression::Type::none; + level ? core::CompressionType::zstd : core::CompressionType::none; header.compression_level = wanted_level; - auto writer = create_writer(file_writer, header); - - char buffer[CCACHE_READ_BUFFER_SIZE]; - size_t bytes_left = reader->header().payload_size(); - while (bytes_left > 0) { - size_t bytes_to_read = std::min(bytes_left, sizeof(buffer)); - reader->read(buffer, bytes_to_read); - writer->write(buffer, bytes_to_read); - bytes_left -= bytes_to_read; - } - reader->finalize(); - writer->finalize(); - file.close(); - atomic_new_file.commit(); + AtomicFile new_cache_file(cache_file.path(), AtomicFile::Mode::binary); + new_cache_file.write( + core::CacheEntry::serialize(header, cache_entry.payload())); + new_cache_file.commit(); // Restore mtime/atime to keep cache LRU cleanup working as expected: util::set_timestamps(cache_file.path(), old_stat.mtim(), old_stat.atim()); @@ -198,9 +154,7 @@ recompress_file(RecompressionStatistics& statistics, cs.increment(core::Statistic::cache_size_kibibyte, Util::size_change_kibibyte(old_stat, new_stat)); }); - statistics.update(content_size, old_stat.size(), new_stat.size(), 0); - - LOG("Recompression of {} done", cache_file.path()); + statistics.update(header.entry_size, old_stat.size(), new_stat.size(), 0); } CompressionStatistics @@ -220,11 +174,9 @@ PrimaryStorage::get_compression_statistics( cs.on_disk_size += cache_file.lstat().size_on_disk(); try { - auto file = open_file(cache_file.path(), "rb"); - core::FileReader file_reader(file.get()); - auto reader = create_reader(cache_file, file_reader); + core::CacheEntry::Header header(cache_file.path()); cs.compr_size += cache_file.lstat().size(); - cs.content_size += reader->header().entry_size; + cs.content_size += header.entry_size; } catch (core::Error&) { cs.incompr_size += cache_file.lstat().size(); } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index d02d3ca6b..59d72f6ac 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -7,6 +7,7 @@ set( file.cpp path.cpp string.cpp + zstd.cpp ) target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/util/XXH3_128.hpp b/src/util/XXH3_128.hpp index e7edbb3f6..309ac7cc3 100644 --- a/src/util/XXH3_128.hpp +++ b/src/util/XXH3_128.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,14 +18,16 @@ #pragma once +#include +#include + +#include #ifdef USE_XXH_DISPATCH -# include "third_party/xxh_x86dispatch.h" +# include #else -# include "third_party/xxhash.h" +# include #endif -#include - #include #include @@ -34,61 +36,19 @@ namespace util { class XXH3_128 { public: - struct Digest - { - public: - const uint8_t* bytes() const; - uint8_t* bytes(); - constexpr static size_t size(); - - bool operator==(const Digest& other) const; - bool operator!=(const Digest& other) const; - - private: - uint8_t m_bytes[16] = {}; - }; + static constexpr size_t k_digest_size = 16; XXH3_128(); ~XXH3_128(); void reset(); - void update(const void* data, size_t length); - Digest digest() const; + void update(nonstd::span data); + util::Bytes digest() const; private: XXH3_state_t* m_state; }; -inline const uint8_t* -XXH3_128::Digest::bytes() const -{ - return m_bytes; -} - -inline uint8_t* -XXH3_128::Digest::bytes() -{ - return m_bytes; -} - -inline constexpr size_t -XXH3_128::Digest::size() -{ - return sizeof(m_bytes); -} - -inline bool -XXH3_128::Digest::operator==(const XXH3_128::Digest& other) const -{ - return memcmp(bytes(), other.bytes(), size()) == 0; -} - -inline bool -XXH3_128::Digest::operator!=(const XXH3_128::Digest& other) const -{ - return !(*this == other); -} - inline XXH3_128::XXH3_128() : m_state(XXH3_createState()) { reset(); @@ -106,18 +66,18 @@ XXH3_128::reset() } inline void -XXH3_128::update(const void* data, size_t length) +XXH3_128::update(nonstd::span data) { - XXH3_128bits_update(m_state, data, length); + XXH3_128bits_update(m_state, data.data(), data.size()); } -inline XXH3_128::Digest +inline util::Bytes XXH3_128::digest() const { const auto result = XXH3_128bits_digest(m_state); - XXH3_128::Digest digest; - Util::int_to_big_endian(result.high64, digest.bytes()); - Util::int_to_big_endian(result.low64, digest.bytes() + 8); + util::Bytes digest(k_digest_size); + Util::int_to_big_endian(result.high64, &digest[0]); + Util::int_to_big_endian(result.low64, &digest[8]); return digest; } diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index bdfb0270a..5c5ab65d5 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -7,10 +7,8 @@ set( test_Config.cpp test_Depfile.cpp test_Hash.cpp - test_NullCompression.cpp test_Stat.cpp test_Util.cpp - test_ZstdCompression.cpp test_argprocessing.cpp test_ccache.cpp test_compopt.cpp @@ -31,6 +29,7 @@ set( test_util_file.cpp test_util_path.cpp test_util_string.cpp + test_util_zstd.cpp ) if(INODE_CACHE_SUPPORTED) diff --git a/unittest/test_NullCompression.cpp b/unittest/test_NullCompression.cpp deleted file mode 100644 index d733fa9cd..000000000 --- a/unittest/test_NullCompression.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "../src/File.hpp" -#include "TestUtil.hpp" - -#include -#include -#include -#include -#include - -#include "third_party/doctest.h" - -#include - -using compression::Compressor; -using compression::Decompressor; -using TestUtil::TestContext; - -TEST_SUITE_BEGIN("NullCompression"); - -TEST_CASE("compression::Type::none roundtrip") -{ - TestContext test_context; - - File f("data.uncompressed", "w"); - core::FileWriter fw(f.get()); - auto compressor = - Compressor::create_from_type(compression::Type::none, fw, 1); - CHECK(compressor->actual_compression_level() == 0); - compressor->write("foobar", 6); - compressor->finalize(); - - f.open("data.uncompressed", "r"); - core::FileReader fr(f.get()); - auto decompressor = - Decompressor::create_from_type(compression::Type::none, fr); - - char buffer[4]; - decompressor->read(buffer, 4); - CHECK(memcmp(buffer, "foob", 4) == 0); - - SUBCASE("Garbage data") - { - // Not reached the end. - CHECK_THROWS_WITH(decompressor->finalize(), - "Garbage data at end of uncompressed stream"); - } - - SUBCASE("Read to end") - { - decompressor->read(buffer, 2); - CHECK(memcmp(buffer, "ar", 2) == 0); - - // Reached the end. - decompressor->finalize(); - - // Nothing left to read. - CHECK_THROWS_WITH(decompressor->read(buffer, 1), - "Failed to read from file stream"); - } -} - -TEST_SUITE_END(); diff --git a/unittest/test_ZstdCompression.cpp b/unittest/test_ZstdCompression.cpp deleted file mode 100644 index 59c169f02..000000000 --- a/unittest/test_ZstdCompression.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "../src/File.hpp" -#include "TestUtil.hpp" - -#include -#include -#include -#include -#include - -#include "third_party/doctest.h" - -#include - -using compression::Compressor; -using compression::Decompressor; -using TestUtil::TestContext; - -TEST_SUITE_BEGIN("ZstdCompression"); - -TEST_CASE("Small compression::Type::zstd roundtrip") -{ - TestContext test_context; - - File f("data.zstd", "wb"); - core::FileWriter fw(f.get()); - auto compressor = - Compressor::create_from_type(compression::Type::zstd, fw, 1); - CHECK(compressor->actual_compression_level() == 1); - compressor->write("foobar", 6); - compressor->finalize(); - - f.open("data.zstd", "rb"); - core::FileReader fr(f.get()); - auto decompressor = - Decompressor::create_from_type(compression::Type::zstd, fr); - - char buffer[4]; - decompressor->read(buffer, 4); - CHECK(memcmp(buffer, "foob", 4) == 0); - - // Not reached the end. - CHECK_THROWS_WITH(decompressor->finalize(), - "Garbage data at end of zstd input stream"); - - decompressor->read(buffer, 2); - CHECK(memcmp(buffer, "ar", 2) == 0); - - // Reached the end. - decompressor->finalize(); - - // Nothing left to read. - CHECK_THROWS_WITH(decompressor->read(buffer, 1), - "Failed to read from file stream"); -} - -TEST_CASE("Large compressible compression::Type::zstd roundtrip") -{ - TestContext test_context; - - char data[] = "The quick brown fox jumps over the lazy dog"; - - File f("data.zstd", "wb"); - core::FileWriter fw(f.get()); - auto compressor = - Compressor::create_from_type(compression::Type::zstd, fw, 1); - for (size_t i = 0; i < 1000; i++) { - compressor->write(data, sizeof(data)); - } - compressor->finalize(); - - f.open("data.zstd", "rb"); - core::FileReader fr(f.get()); - auto decompressor = - Decompressor::create_from_type(compression::Type::zstd, fr); - - char buffer[sizeof(data)]; - for (size_t i = 0; i < 1000; i++) { - decompressor->read(buffer, sizeof(buffer)); - CHECK(memcmp(buffer, data, sizeof(data)) == 0); - } - - // Reached the end. - decompressor->finalize(); - - // Nothing left to read. - CHECK_THROWS_WITH(decompressor->read(buffer, 1), - "Failed to read from file stream"); -} - -TEST_CASE("Large uncompressible compression::Type::zstd roundtrip") -{ - TestContext test_context; - - char data[100000]; - for (char& c : data) { - c = rand() % 256; - } - - File f("data.zstd", "wb"); - core::FileWriter fw(f.get()); - auto compressor = - Compressor::create_from_type(compression::Type::zstd, fw, 1); - compressor->write(data, sizeof(data)); - compressor->finalize(); - - f.open("data.zstd", "rb"); - core::FileReader fr(f.get()); - auto decompressor = - Decompressor::create_from_type(compression::Type::zstd, fr); - - char buffer[sizeof(data)]; - decompressor->read(buffer, sizeof(buffer)); - CHECK(memcmp(buffer, data, sizeof(data)) == 0); - - decompressor->finalize(); -} - -TEST_SUITE_END(); diff --git a/unittest/test_compression_types.cpp b/unittest/test_compression_types.cpp index 82edb0e5d..639a81c8f 100644 --- a/unittest/test_compression_types.cpp +++ b/unittest/test_compression_types.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,35 +18,36 @@ #include "../src/Config.hpp" -#include +#include #include "third_party/doctest.h" TEST_SUITE_BEGIN("compression"); -TEST_CASE("compression::level_from_config") +TEST_CASE("compression_level_from_config") { Config config; - CHECK(compression::level_from_config(config) == 0); + CHECK(core::compression_level_from_config(config) == 0); } -TEST_CASE("compression::type_from_config") +TEST_CASE("compression_type_from_config") { Config config; - CHECK(compression::type_from_config(config) == compression::Type::zstd); + CHECK(core::compression_type_from_config(config) + == core::CompressionType::zstd); } -TEST_CASE("compression::type_from_int") +TEST_CASE("compression_type_from_int") { - CHECK(compression::type_from_int(0) == compression::Type::none); - CHECK(compression::type_from_int(1) == compression::Type::zstd); - CHECK_THROWS_WITH(compression::type_from_int(2), "Unknown type: 2"); + CHECK(core::compression_type_from_int(0) == core::CompressionType::none); + CHECK(core::compression_type_from_int(1) == core::CompressionType::zstd); + CHECK_THROWS_WITH(core::compression_type_from_int(2), "Unknown type: 2"); } -TEST_CASE("compression::type_to_string") +TEST_CASE("to_string(CompressionType)") { - CHECK(compression::type_to_string(compression::Type::none) == "none"); - CHECK(compression::type_to_string(compression::Type::zstd) == "zstd"); + CHECK(core::to_string(core::CompressionType::none) == "none"); + CHECK(core::to_string(core::CompressionType::zstd) == "zstd"); } TEST_SUITE_END(); diff --git a/unittest/test_util_XXH3_128.cpp b/unittest/test_util_XXH3_128.cpp index 3845fc582..23b411880 100644 --- a/unittest/test_util_XXH3_128.cpp +++ b/unittest/test_util_XXH3_128.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2011-2021 Joel Rosdahl and other contributors +// Copyright (C) 2011-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -17,6 +17,7 @@ // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include +#include #include @@ -26,22 +27,22 @@ TEST_CASE("util::XXH3_128") { util::XXH3_128 checksum; auto digest = checksum.digest(); - CHECK(Util::format_base16(digest.bytes(), 16) + CHECK(Util::format_base16(digest.data(), 16) == "99aa06d3014798d86001c324468d497f"); - checksum.update("foo", 3); + checksum.update(util::to_span("foo")); digest = checksum.digest(); - CHECK(Util::format_base16(digest.bytes(), 16) + CHECK(Util::format_base16(digest.data(), 16) == "79aef92e83454121ab6e5f64077e7d8a"); - checksum.update("t", 1); + checksum.update(util::to_span("t")); digest = checksum.digest(); - CHECK(Util::format_base16(digest.bytes(), 16) + CHECK(Util::format_base16(digest.data(), 16) == "e6045075b5bf1ae7a3e4c87775e6c97f"); checksum.reset(); digest = checksum.digest(); - CHECK(Util::format_base16(digest.bytes(), 16) + CHECK(Util::format_base16(digest.data(), 16) == "99aa06d3014798d86001c324468d497f"); }