preprocessor output format, etc. Ideally this code should in the future be
refactored into compiler-specific frontends, such as GCC, Clang, NVCC, MSVC,
etc.
-* `compression`: Compression formats.
* `core`: Everything not part of other directories.
* `storage`: Storage backends.
* `storage/primary`: Code for the primary storage backend.
add_executable(test-lockfile test_lockfile.cpp)
target_link_libraries(test-lockfile PRIVATE ccache_framework)
-add_subdirectory(compression)
add_subdirectory(core)
add_subdirectory(storage)
add_subdirectory(third_party)
#include "assertions.hpp"
#include <UmaskScope.hpp>
-#include <compression/types.hpp>
#include <core/exceptions.hpp>
+#include <core/types.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
#include <util/expected.hpp>
#include "language.hpp"
#include <AtomicFile.hpp>
-#include <compression/types.hpp>
-#include <core/CacheEntryReader.hpp>
-#include <core/CacheEntryWriter.hpp>
-#include <core/FileReader.hpp>
-#include <core/FileWriter.hpp>
+#include <core/CacheEntry.hpp>
#include <core/Manifest.hpp>
#include <core/Result.hpp>
#include <core/ResultRetriever.hpp>
read_manifest(const std::string& path)
{
core::Manifest manifest;
- File file(path, "rb");
- if (file) {
- try {
- core::FileReader file_reader(*file);
- core::CacheEntryReader reader(file_reader);
- std::vector<uint8_t> payload;
- payload.resize(reader.header().payload_size());
- reader.read(payload.data(), payload.size());
- manifest.read(payload);
- reader.finalize();
- } catch (const core::Error& e) {
- LOG("Error reading {}: {}", path, e.what());
- }
+ try {
+ const auto cache_entry_data =
+ util::value_or_throw<core::Error>(util::read_file<util::Bytes>(path));
+ core::CacheEntry cache_entry(cache_entry_data);
+ cache_entry.verify_checksum();
+ manifest.read(cache_entry.payload());
+ } catch (const core::Error& e) {
+ LOG("Error reading {}: {}", path, e.what());
}
return manifest;
}
static void
save_manifest(const Config& config,
- const core::Manifest& manifest,
+ core::Manifest& manifest,
const std::string& path)
{
+ core::CacheEntry::Header header(config, core::CacheEntryType::manifest);
+ const auto cache_entry_data = core::CacheEntry::serialize(header, manifest);
AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary);
- core::FileWriter file_writer(atomic_manifest_file.stream());
- core::CacheEntryHeader header(core::CacheEntryType::manifest,
- compression::type_from_config(config),
- compression::level_from_config(config),
- time(nullptr),
- CCACHE_VERSION,
- config.namespace_());
- header.set_entry_size_from_payload_size(manifest.serialized_size());
-
- core::CacheEntryWriter writer(file_writer, header);
- util::Bytes payload;
- payload.reserve(header.payload_size());
- manifest.serialize(payload);
- writer.write(payload.data(), payload.size());
- writer.finalize();
+ atomic_manifest_file.write(cache_entry_data);
atomic_manifest_file.commit();
}
ctx.args_info.output_al);
}
- AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary);
- core::CacheEntryHeader header(core::CacheEntryType::result,
- compression::type_from_config(ctx.config),
- compression::level_from_config(ctx.config),
- time(nullptr),
- CCACHE_VERSION,
- ctx.config.namespace_());
- header.set_entry_size_from_payload_size(serializer.serialized_size());
-
- core::FileWriter file_writer(atomic_result_file.stream());
- core::CacheEntryWriter writer(file_writer, header);
-
- util::Bytes payload;
- payload.reserve(serializer.serialized_size());
- const auto serialize_result = serializer.serialize(payload);
- for (auto [file_number, source_path] : serialize_result.raw_files) {
+ core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result);
+ const auto cache_entry_data = core::CacheEntry::serialize(header, serializer);
+
+ const auto raw_files = serializer.get_raw_files();
+ if (!raw_files.empty()) {
+ Util::ensure_dir_exists(Util::dir_name(result_path));
+ }
+ for (auto [file_number, source_path] : raw_files) {
const auto dest_path = storage::primary::PrimaryStorage::get_raw_file_path(
result_path, file_number);
const auto old_stat = Stat::stat(dest_path);
Statistic::files_in_cache, (new_stat ? 1 : 0) - (old_stat ? 1 : 0));
}
- writer.write(payload.data(), payload.size());
- writer.finalize();
+ AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary);
+ atomic_result_file.write(cache_entry_data);
atomic_result_file.commit();
return true;
{
bool found_ccbin = false;
+ hash.hash_delimiter("cache entry version");
+ hash.hash(core::k_cache_entry_format_version);
+
hash.hash_delimiter("result version");
- hash.hash(core::Result::k_version);
+ hash.hash(core::Result::k_format_version);
if (direct_mode) {
hash.hash_delimiter("manifest version");
}
try {
- File file(*result_path, "rb");
- if (!file) {
- throw core::Error(
- FMT("Failed to open {}: {}", *result_path, strerror(errno)));
- }
- core::FileReader file_reader(file.get());
- core::CacheEntryReader cache_entry_reader(file_reader);
- std::vector<uint8_t> payload;
- payload.resize(cache_entry_reader.header().payload_size());
- cache_entry_reader.read(payload.data(), payload.size());
- core::Result::Deserializer deserializer(payload);
+ const auto cache_entry_data = util::value_or_throw<core::Error>(
+ util::read_file<util::Bytes>(*result_path),
+ FMT("Failed to read {}: ", *result_path));
+
+ core::CacheEntry cache_entry(cache_entry_data);
+ cache_entry.verify_checksum();
+ core::Result::Deserializer deserializer(cache_entry.payload());
core::ResultRetriever result_retriever(ctx, result_path);
deserializer.visit(result_retriever);
- cache_entry_reader.finalize();
} catch (core::ResultRetriever::WriteError& e) {
LOG(
"Write error when retrieving result from {}: {}", *result_path, e.what());
+++ /dev/null
-set(
- sources
- Compressor.cpp
- Decompressor.cpp
- NullCompressor.cpp
- NullDecompressor.cpp
- ZstdCompressor.cpp
- ZstdDecompressor.cpp
- types.cpp
-)
-
-target_sources(ccache_framework PRIVATE ${sources})
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "Compressor.hpp"
-
-#include "NullCompressor.hpp"
-#include "ZstdCompressor.hpp"
-#include "assertions.hpp"
-
-#include <core/Writer.hpp>
-
-#include <memory>
-
-namespace compression {
-
-std::unique_ptr<Compressor>
-Compressor::create_from_type(const Type type,
- core::Writer& writer,
- const int8_t compression_level)
-{
- switch (type) {
- case compression::Type::none:
- return std::make_unique<NullCompressor>(writer);
-
- case compression::Type::zstd:
- return std::make_unique<ZstdCompressor>(writer, compression_level);
- }
-
- ASSERT(false);
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <compression/types.hpp>
-#include <core/Writer.hpp>
-
-#include <cstdint>
-#include <memory>
-
-namespace core {
-
-class Writer;
-
-}
-
-namespace compression {
-
-class Compressor : public core::Writer
-{
-public:
- virtual ~Compressor() = default;
-
- static std::unique_ptr<Compressor>
- create_from_type(Type type, core::Writer& writer, int8_t compression_level);
-
- virtual int8_t actual_compression_level() const = 0;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "Decompressor.hpp"
-
-#include "NullDecompressor.hpp"
-#include "ZstdDecompressor.hpp"
-#include "assertions.hpp"
-
-namespace compression {
-
-std::unique_ptr<Decompressor>
-Decompressor::create_from_type(Type type, core::Reader& reader)
-{
- switch (type) {
- case compression::Type::none:
- return std::make_unique<NullDecompressor>(reader);
-
- case compression::Type::zstd:
- return std::make_unique<ZstdDecompressor>(reader);
- }
-
- ASSERT(false);
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <compression/types.hpp>
-#include <core/Reader.hpp>
-
-#include <memory>
-
-namespace compression {
-
-class Decompressor : public core::Reader
-{
-public:
- virtual ~Decompressor() = default;
-
- // Create a decompressor for the specified type.
- static std::unique_ptr<Decompressor> create_from_type(Type type,
- core::Reader& reader);
-
- // Finalize decompression.
- //
- // This method checks that the end state of the compressed stream is correct
- // and throws Error if not.
- virtual void finalize() = 0;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "NullCompressor.hpp"
-
-#include <core/exceptions.hpp>
-
-namespace compression {
-
-NullCompressor::NullCompressor(core::Writer& writer) : m_writer(writer)
-{
-}
-
-int8_t
-NullCompressor::actual_compression_level() const
-{
- return 0;
-}
-
-void
-NullCompressor::write(const void* const data, const size_t count)
-{
- m_writer.write(data, count);
-}
-
-void
-NullCompressor::finalize()
-{
- m_writer.finalize();
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Compressor.hpp"
-
-#include <NonCopyable.hpp>
-
-namespace compression {
-
-// A compressor of an uncompressed stream.
-class NullCompressor : public Compressor, NonCopyable
-{
-public:
- explicit NullCompressor(core::Writer& writer);
-
- int8_t actual_compression_level() const override;
- void write(const void* data, size_t count) override;
- void finalize() override;
-
-private:
- core::Writer& m_writer;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "NullDecompressor.hpp"
-
-#include <core/exceptions.hpp>
-
-namespace compression {
-
-NullDecompressor::NullDecompressor(core::Reader& reader) : m_reader(reader)
-{
-}
-
-size_t
-NullDecompressor::read(void* const data, const size_t count)
-{
- return m_reader.read(data, count);
-}
-
-void
-NullDecompressor::finalize()
-{
- bool eof;
- try {
- m_reader.read_int<uint8_t>();
- eof = false;
- } catch (core::Error&) {
- eof = true;
- }
- if (!eof) {
- throw core::Error("Garbage data at end of uncompressed stream");
- }
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Decompressor.hpp"
-
-#include <NonCopyable.hpp>
-
-namespace compression {
-
-// A decompressor of an uncompressed stream.
-class NullDecompressor : public Decompressor, NonCopyable
-{
-public:
- explicit NullDecompressor(core::Reader& reader);
-
- size_t read(void* data, size_t count) override;
- void finalize() override;
-
-private:
- core::Reader& m_reader;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ZstdCompressor.hpp"
-
-#include "Logging.hpp"
-#include "assertions.hpp"
-
-#include <core/exceptions.hpp>
-
-#include <zstd.h>
-
-#include <algorithm>
-
-namespace compression {
-
-ZstdCompressor::ZstdCompressor(core::Writer& writer, int8_t compression_level)
- : m_writer(writer),
- m_zstd_stream(ZSTD_createCStream()),
- m_zstd_in(std::make_unique<ZSTD_inBuffer_s>()),
- m_zstd_out(std::make_unique<ZSTD_outBuffer_s>())
-{
- if (compression_level == 0) {
- compression_level = default_compression_level;
- LOG("Using default compression level {}", compression_level);
- }
-
- // libzstd 1.3.4 and newer support negative levels. However, the query
- // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection
- // based on version instead.
- if (ZSTD_versionNumber() < 10304 && compression_level < 1) {
- LOG(
- "Using compression level 1 (minimum level supported by libzstd) instead"
- " of {}",
- compression_level);
- compression_level = 1;
- }
-
- m_compression_level = std::min<int>(compression_level, ZSTD_maxCLevel());
- if (m_compression_level != compression_level) {
- LOG("Using compression level {} (max libzstd level) instead of {}",
- m_compression_level,
- compression_level);
- }
-
- size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level);
- if (ZSTD_isError(ret)) {
- ZSTD_freeCStream(m_zstd_stream);
- throw core::Error("error initializing zstd compression stream");
- }
-}
-
-ZstdCompressor::~ZstdCompressor()
-{
- ZSTD_freeCStream(m_zstd_stream);
-}
-
-int8_t
-ZstdCompressor::actual_compression_level() const
-{
- return m_compression_level;
-}
-
-void
-ZstdCompressor::write(const void* const data, const size_t count)
-{
- m_zstd_in->src = data;
- m_zstd_in->size = count;
- m_zstd_in->pos = 0;
-
- int flush = data ? 0 : 1;
-
- size_t ret;
- while (m_zstd_in->pos < m_zstd_in->size) {
- uint8_t buffer[CCACHE_READ_BUFFER_SIZE];
- m_zstd_out->dst = buffer;
- m_zstd_out->size = sizeof(buffer);
- m_zstd_out->pos = 0;
- ret = ZSTD_compressStream(m_zstd_stream, m_zstd_out.get(), m_zstd_in.get());
- ASSERT(!(ZSTD_isError(ret)));
- const size_t compressed_bytes = m_zstd_out->pos;
- if (compressed_bytes > 0) {
- m_writer.write(buffer, compressed_bytes);
- }
- }
- ret = flush;
- while (ret > 0) {
- uint8_t buffer[CCACHE_READ_BUFFER_SIZE];
- m_zstd_out->dst = buffer;
- m_zstd_out->size = sizeof(buffer);
- m_zstd_out->pos = 0;
- ret = ZSTD_endStream(m_zstd_stream, m_zstd_out.get());
- const size_t compressed_bytes = m_zstd_out->pos;
- if (compressed_bytes > 0) {
- m_writer.write(buffer, compressed_bytes);
- }
- }
-}
-
-void
-ZstdCompressor::finalize()
-{
- write(nullptr, 0);
- m_writer.finalize();
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Compressor.hpp"
-
-#include <NonCopyable.hpp>
-
-#include <cstdint>
-#include <memory>
-
-struct ZSTD_CCtx_s;
-struct ZSTD_inBuffer_s;
-struct ZSTD_outBuffer_s;
-
-namespace compression {
-
-// A compressor of a Zstandard stream.
-class ZstdCompressor : public Compressor, NonCopyable
-{
-public:
- ZstdCompressor(core::Writer& writer, int8_t compression_level);
-
- ~ZstdCompressor() override;
-
- int8_t actual_compression_level() const override;
- void write(const void* data, size_t count) override;
- void finalize() override;
-
- constexpr static uint8_t default_compression_level = 1;
-
-private:
- core::Writer& m_writer;
- ZSTD_CCtx_s* m_zstd_stream;
- std::unique_ptr<ZSTD_inBuffer_s> m_zstd_in;
- std::unique_ptr<ZSTD_outBuffer_s> m_zstd_out;
- int8_t m_compression_level;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ZstdDecompressor.hpp"
-
-#include "assertions.hpp"
-
-#include <core/exceptions.hpp>
-
-namespace compression {
-
-ZstdDecompressor::ZstdDecompressor(core::Reader& reader)
- : m_reader(reader),
- m_input_size(0),
- m_input_consumed(0),
- m_zstd_stream(ZSTD_createDStream()),
- m_reached_stream_end(false)
-{
- const size_t ret = ZSTD_initDStream(m_zstd_stream);
- if (ZSTD_isError(ret)) {
- ZSTD_freeDStream(m_zstd_stream);
- throw core::Error("failed to initialize zstd decompression stream");
- }
-}
-
-ZstdDecompressor::~ZstdDecompressor()
-{
- ZSTD_freeDStream(m_zstd_stream);
-}
-
-size_t
-ZstdDecompressor::read(void* const data, const size_t count)
-{
- size_t bytes_read = 0;
- while (bytes_read < count) {
- ASSERT(m_input_size >= m_input_consumed);
- if (m_input_size == m_input_consumed) {
- m_input_size = m_reader.read(m_input_buffer, sizeof(m_input_buffer));
- m_input_consumed = 0;
- }
-
- m_zstd_in.src = (m_input_buffer + m_input_consumed);
- m_zstd_in.size = m_input_size - m_input_consumed;
- m_zstd_in.pos = 0;
-
- m_zstd_out.dst = static_cast<uint8_t*>(data) + bytes_read;
- m_zstd_out.size = count - bytes_read;
- m_zstd_out.pos = 0;
- const size_t ret =
- ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in);
- if (ZSTD_isError(ret)) {
- throw core::Error("Failed to read from zstd input stream");
- }
- if (ret == 0) {
- m_reached_stream_end = true;
- break;
- }
- bytes_read += m_zstd_out.pos;
- m_input_consumed += m_zstd_in.pos;
- }
-
- return count;
-}
-
-void
-ZstdDecompressor::finalize()
-{
- if (!m_reached_stream_end) {
- throw core::Error("Garbage data at end of zstd input stream");
- }
-}
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "Decompressor.hpp"
-
-#include <zstd.h>
-
-#include <cstdint>
-
-namespace compression {
-
-// A decompressor of a Zstandard stream.
-class ZstdDecompressor : public Decompressor
-{
-public:
- explicit ZstdDecompressor(core::Reader& reader);
-
- ~ZstdDecompressor() override;
-
- size_t read(void* data, size_t count) override;
- void finalize() override;
-
-private:
- core::Reader& m_reader;
- char m_input_buffer[CCACHE_READ_BUFFER_SIZE];
- size_t m_input_size;
- size_t m_input_consumed;
- ZSTD_DStream* m_zstd_stream;
- ZSTD_inBuffer m_zstd_in;
- ZSTD_outBuffer m_zstd_out;
- bool m_reached_stream_end;
-};
-
-} // namespace compression
+++ /dev/null
-// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "types.hpp"
-
-#include <Config.hpp>
-#include <Context.hpp>
-#include <assertions.hpp>
-#include <core/exceptions.hpp>
-#include <fmtmacros.hpp>
-
-namespace compression {
-
-int8_t
-level_from_config(const Config& config)
-{
- return config.compression() ? config.compression_level() : 0;
-}
-
-Type
-type_from_config(const Config& config)
-{
- return config.compression() ? Type::zstd : Type::none;
-}
-
-Type
-type_from_int(const uint8_t type)
-{
- switch (type) {
- case static_cast<uint8_t>(Type::none):
- return Type::none;
-
- case static_cast<uint8_t>(Type::zstd):
- return Type::zstd;
- }
-
- throw core::Error(FMT("Unknown type: {}", type));
-}
-
-std::string
-type_to_string(const Type type)
-{
- switch (type) {
- case Type::none:
- return "none";
-
- case Type::zstd:
- return "zstd";
- }
-
- ASSERT(false);
-}
-
-} // namespace compression
set(
sources
- CacheEntryHeader.cpp
- CacheEntryReader.cpp
- CacheEntryWriter.cpp
+ CacheEntry.cpp
Manifest.cpp
Result.cpp
ResultExtractor.cpp
--- /dev/null
+// Copyright (C) 2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "CacheEntry.hpp"
+
+#include <Logging.hpp>
+#include <ccache.hpp>
+#include <core/CacheEntryDataReader.hpp>
+#include <core/CacheEntryDataWriter.hpp>
+#include <core/Result.hpp>
+#include <core/exceptions.hpp>
+#include <core/types.hpp>
+#include <fmtmacros.hpp>
+#include <util/expected.hpp>
+#include <util/file.hpp>
+#include <util/zstd.hpp>
+
+#include <cstring>
+
+namespace {
+
+const size_t k_static_header_fields_size =
+ sizeof(core::CacheEntry::Header::magic)
+ + sizeof(core::CacheEntry::Header::entry_format_version)
+ + sizeof(core::CacheEntry::Header::entry_type)
+ + sizeof(core::CacheEntry::Header::compression_type)
+ + sizeof(core::CacheEntry::Header::compression_level)
+ + sizeof(core::CacheEntry::Header::self_contained)
+ + sizeof(core::CacheEntry::Header::creation_time)
+ + sizeof(core::CacheEntry::Header::entry_size)
+ // ccache_version length field:
+ + 1
+ // namespace_ length field:
+ + 1;
+
+const size_t k_epilogue_fields_size = sizeof(uint64_t) + sizeof(uint64_t);
+
+core::CacheEntryType
+cache_entry_type_from_int(const uint8_t entry_type)
+{
+ switch (entry_type) {
+ case 0:
+ return core::CacheEntryType::result;
+ break;
+ case 1:
+ return core::CacheEntryType::manifest;
+ break;
+ default:
+ throw core::Error(FMT("Unknown entry type: {}", entry_type));
+ }
+}
+
+} // namespace
+
+namespace core {
+
+CacheEntry::Header::Header(const Config& config,
+ core::CacheEntryType entry_type)
+ : magic(k_ccache_magic),
+ entry_format_version(k_cache_entry_format_version),
+ entry_type(entry_type),
+ compression_type(compression_type_from_config(config)),
+ compression_level(compression_level_from_config(config)),
+ self_contained(entry_type != CacheEntryType::result
+ || !core::Result::Serializer::use_raw_files(config)),
+ creation_time(time(nullptr)),
+ ccache_version(CCACHE_VERSION),
+ namespace_(config.namespace_()),
+ entry_size(0)
+{
+ if (compression_level == 0) {
+ compression_level = default_compression_level;
+ LOG("Using default compression level {}", compression_level);
+ }
+}
+
+CacheEntry::Header::Header(nonstd::span<const uint8_t> data)
+{
+ parse(data);
+}
+
+CacheEntry::Header::Header(const std::string& path)
+{
+ const auto data = util::read_file_part<util::Bytes>(path, 0, 1000);
+ if (!data) {
+ throw core::Error(data.error());
+ }
+ parse(*data);
+}
+
+std::string
+CacheEntry::Header::inspect() const
+{
+ std::string result;
+ result += result += FMT("Magic: {:04x}\n", magic);
+ result += FMT("Entry format version: {}\n", entry_format_version);
+ result += FMT("Entry type: {} ({})\n",
+ static_cast<uint8_t>(entry_type),
+ to_string(entry_type));
+ result += FMT("Compression type: {}\n", to_string(compression_type));
+ result += FMT("Compression level: {}\n", compression_level);
+ result += FMT("Self-contained: {}\n", self_contained ? "yes" : "no");
+ result += FMT("Creation time: {}\n", creation_time);
+ result += FMT("Ccache version: {}\n", ccache_version);
+ result += FMT("Namespace: {}\n", namespace_);
+ result += FMT("Entry size: {}\n", entry_size);
+ return result;
+}
+
+void
+CacheEntry::Header::parse(nonstd::span<const uint8_t> data)
+{
+ CacheEntryDataReader reader(data);
+ reader.read_int(magic);
+ if (magic != core::k_ccache_magic) {
+ throw core::Error(FMT("Bad magic value: 0x{:04x}", magic));
+ }
+
+ reader.read_int(entry_format_version);
+ if (entry_format_version != core::k_cache_entry_format_version) {
+ throw core::Error(
+ FMT("Unknown entry format version: {}", entry_format_version));
+ }
+
+ entry_type = cache_entry_type_from_int(reader.read_int<uint8_t>());
+ compression_type = compression_type_from_int(reader.read_int<uint8_t>());
+ reader.read_int(compression_level);
+ self_contained = bool(reader.read_int<uint8_t>());
+ reader.read_int(creation_time);
+ ccache_version = reader.read_str(reader.read_int<uint8_t>());
+ namespace_ = reader.read_str(reader.read_int<uint8_t>());
+ reader.read_int(entry_size);
+}
+
+size_t
+CacheEntry::Header::serialized_size() const
+{
+ return k_static_header_fields_size + ccache_version.length()
+ + namespace_.length();
+}
+
+void
+CacheEntry::Header::serialize(util::Bytes& output) const
+{
+ core::CacheEntryDataWriter writer(output);
+ writer.write_int(magic);
+ writer.write_int(entry_format_version);
+ writer.write_int(static_cast<uint8_t>(entry_type));
+ writer.write_int(static_cast<uint8_t>(compression_type));
+ writer.write_int(compression_level);
+ writer.write_int<uint8_t>(self_contained);
+ writer.write_int(creation_time);
+ writer.write_int<uint8_t>(ccache_version.length());
+ writer.write_str(ccache_version);
+ writer.write_int<uint8_t>(namespace_.length());
+ writer.write_str(namespace_);
+ writer.write_int(entry_size);
+}
+
+uint32_t
+CacheEntry::Header::uncompressed_payload_size() const
+{
+ return entry_size - serialized_size() - k_epilogue_fields_size;
+}
+
+CacheEntry::CacheEntry(nonstd::span<const uint8_t> data) : m_header(data)
+{
+ const size_t non_payload_size =
+ m_header.serialized_size() + k_epilogue_fields_size;
+ if (data.size() <= non_payload_size) {
+ throw core::Error("CacheEntry data underflow");
+ }
+ m_payload =
+ data.subspan(m_header.serialized_size(), data.size() - non_payload_size);
+ m_checksum = data.last(k_epilogue_fields_size);
+
+ switch (m_header.compression_type) {
+ case CompressionType::none:
+ break;
+
+ case CompressionType::zstd:
+ m_uncompressed_payload.reserve(m_header.uncompressed_payload_size());
+ util::throw_on_error<core::Error>(
+ util::zstd_decompress(
+ m_payload, m_uncompressed_payload, m_uncompressed_payload.capacity()),
+ "Cache entry payload decompression error: ");
+
+ break;
+ }
+}
+
+void
+CacheEntry::verify_checksum() const
+{
+ util::Bytes header_data;
+ m_header.serialize(header_data);
+
+ util::XXH3_128 checksum;
+ checksum.update(header_data);
+ checksum.update(m_payload);
+ const auto actual = checksum.digest();
+
+ if (actual != m_checksum) {
+ throw core::Error(
+ FMT("Incorrect checksum (actual {}, expected {})",
+ Util::format_base16(actual.data(), actual.size()),
+ Util::format_base16(m_checksum.data(), m_checksum.size())));
+ }
+}
+
+const CacheEntry::Header&
+CacheEntry::header() const
+{
+ return m_header;
+}
+
+nonstd::span<const uint8_t>
+CacheEntry::payload() const
+{
+ return m_header.compression_type == CompressionType::none
+ ? m_payload
+ : nonstd::span<const uint8_t>(m_uncompressed_payload);
+}
+
+util::Bytes
+CacheEntry::serialize(const CacheEntry::Header& header,
+ Serializer& payload_serializer)
+{
+ return do_serialize(
+ header,
+ payload_serializer.serialized_size(),
+ [&payload_serializer](util::Bytes& result, const CacheEntry::Header& hdr) {
+ switch (hdr.compression_type) {
+ case CompressionType::none:
+ payload_serializer.serialize(result);
+ break;
+
+ case CompressionType::zstd:
+ util::Bytes payload;
+ payload_serializer.serialize(payload);
+ util::throw_on_error<core::Error>(
+ util::zstd_compress(payload, result, hdr.compression_level),
+ "Cache entry payload compression error: ");
+ break;
+ }
+ });
+}
+
+util::Bytes
+CacheEntry::serialize(const CacheEntry::Header& header,
+ nonstd::span<const uint8_t> payload)
+{
+ return do_serialize(
+ header,
+ payload.size(),
+ [&payload](util::Bytes& result, const CacheEntry::Header& hdr) {
+ switch (hdr.compression_type) {
+ case CompressionType::none:
+ result.insert(result.end(), payload.begin(), payload.end());
+ break;
+
+ case CompressionType::zstd:
+ util::throw_on_error<core::Error>(
+ util::zstd_compress(payload, result, hdr.compression_level),
+ "Cache entry payload compression error: ");
+ break;
+ }
+ });
+}
+
+util::Bytes
+CacheEntry::do_serialize(
+ const CacheEntry::Header& header,
+ size_t serialized_payload_size,
+ std::function<void(util::Bytes& result, const Header& hdr)> serialize_payload)
+{
+ CacheEntry::Header hdr(header);
+ const size_t non_payload_size =
+ hdr.serialized_size() + k_epilogue_fields_size;
+ hdr.entry_size = non_payload_size + serialized_payload_size;
+
+ if (hdr.compression_type == CompressionType::zstd) {
+ const auto [level, explanation] =
+ util::zstd_supported_compression_level(hdr.compression_level);
+ if (!explanation.empty()) {
+ LOG("Using ZSTD compression level {} ({}) instead of {}",
+ level,
+ explanation,
+ hdr.compression_level);
+ }
+ hdr.compression_level = level;
+ }
+
+ const size_t max_serialized_size =
+ hdr.compression_type == CompressionType::zstd
+ ? (non_payload_size + util::zstd_compress_bound(serialized_payload_size))
+ : hdr.entry_size;
+ util::Bytes result;
+ result.reserve(max_serialized_size);
+
+ hdr.serialize(result);
+ serialize_payload(result, hdr);
+
+ util::XXH3_128 checksum;
+ checksum.update(result);
+ const auto digest = checksum.digest();
+ result.insert(result.end(), digest.begin(), digest.end());
+
+ return result;
+}
+
+} // namespace core
--- /dev/null
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <core/Serializer.hpp>
+#include <core/types.hpp>
+#include <util/Bytes.hpp>
+#include <util/XXH3_128.hpp>
+
+#include <third_party/nonstd/span.hpp>
+
+#include <cstdint>
+#include <functional>
+#include <string>
+
+// Cache entry format
+// ==================
+//
+// Integers are big-endian.
+//
+// <entry> ::= <header> <payload> <epilogue>
+// <header> ::= <magic> <format_ver> <entry_type> <compr_type>
+// <compr_level> <creation_time> <ccache_ver> <namespace>
+// <entry_size>
+// <magic> ::= uint16_t (0xccac)
+// <format_ver> ::= uint8_t
+// <entry_type> ::= <result_entry> | <manifest_entry>
+// <result_entry> ::= 0 (uint8_t)
+// <manifest_entry> ::= 1 (uint8_t)
+// <self_contained> ::= 0/1 (uint8_t) ; whether suitable for secondary storage
+// <compr_type> ::= <compr_none> | <compr_zstd>
+// <compr_none> ::= 0 (uint8_t)
+// <compr_zstd> ::= 1 (uint8_t)
+// <compr_level> ::= int8_t
+// <creation_time> ::= uint64_t (Unix epoch time when entry was created)
+// <ccache_ver> ::= string length (uint8_t) + string data
+// <namespace> ::= string length (uint8_t) + string data
+// <entry_size> ::= uint64_t ; = size of entry in uncompressed form
+// <payload> ::= depends on entry_type; potentially compressed
+// <epilogue> ::= <checksum_high> <checksum_low>
+// <checksum_high> ::= uint64_t ; XXH3-128 (high bits) of <header>+<payload>
+// <checksum_low> ::= uint64_t ; XXH3-128 (low bits) of <header>+<payload>
+
+class Config;
+
+namespace core {
+
+const uint16_t k_ccache_magic = 0xccac;
+
+// Version 0:
+// - First version.
+// Version 1:
+// - Added self_contained field.
+// - The checksum is now for the (potentially) compressed payload instead of
+// the uncompressed payload, and the checksum is now always stored
+// uncompressed.
+const uint16_t k_cache_entry_format_version = 1;
+
+class CacheEntry
+{
+public:
+ constexpr static uint8_t default_compression_level = 1;
+
+ class Header
+ {
+ public:
+ Header(const Config& config, CacheEntryType entry_type);
+ explicit Header(nonstd::span<const uint8_t> data);
+ explicit Header(const std::string& path);
+
+ std::string inspect() const;
+
+ uint16_t magic;
+ uint8_t entry_format_version;
+ CacheEntryType entry_type;
+ CompressionType compression_type;
+ int8_t compression_level;
+ bool self_contained;
+ uint64_t creation_time;
+ std::string ccache_version;
+ std::string namespace_;
+ uint64_t entry_size;
+
+ size_t serialized_size() const;
+ void serialize(util::Bytes& output) const;
+ uint32_t uncompressed_payload_size() const;
+
+ private:
+ void parse(nonstd::span<const uint8_t> data);
+ };
+
+ explicit CacheEntry(nonstd::span<const uint8_t> data);
+
+ void verify_checksum() const;
+ const Header& header() const;
+
+ // Return uncompressed payload.
+ nonstd::span<const uint8_t> payload() const;
+
+ static util::Bytes serialize(const Header& header,
+ Serializer& payload_serializer);
+ static util::Bytes serialize(const Header& header,
+ nonstd::span<const uint8_t> payload);
+
+private:
+ Header m_header;
+ nonstd::span<const uint8_t> m_payload; // Potentially compressed
+ util::Bytes m_checksum;
+
+ mutable util::Bytes m_uncompressed_payload;
+
+ static util::Bytes
+ do_serialize(const Header& header,
+ size_t serialized_payload_size,
+ std::function<void(util::Bytes& result, const Header& header)>
+ serialize_payload);
+};
+
+} // namespace core
+++ /dev/null
-// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "CacheEntryHeader.hpp"
-
-#include <core/exceptions.hpp>
-#include <fmtmacros.hpp>
-
-const size_t k_static_header_fields_size =
- sizeof(core::CacheEntryHeader::magic)
- + sizeof(core::CacheEntryHeader::entry_format_version)
- + sizeof(core::CacheEntryHeader::entry_type)
- + sizeof(core::CacheEntryHeader::compression_type)
- + sizeof(core::CacheEntryHeader::compression_level)
- + sizeof(core::CacheEntryHeader::creation_time)
- + sizeof(core::CacheEntryHeader::entry_size)
- // ccache_version length field:
- + 1
- // namespace_ length field:
- + 1;
-
-const size_t k_static_epilogue_fields_size =
- sizeof(uint64_t) + sizeof(uint64_t);
-
-namespace core {
-
-CacheEntryHeader::CacheEntryHeader(const core::CacheEntryType entry_type_,
- const compression::Type compression_type_,
- const int8_t compression_level_,
- const uint64_t creation_time_,
- const std::string& ccache_version_,
- const std::string& namespace_arg,
- const uint64_t entry_size_)
- : magic(k_ccache_magic),
- entry_format_version(k_entry_format_version),
- entry_type(entry_type_),
- compression_type(compression_type_),
- compression_level(compression_level_),
- creation_time(creation_time_),
- ccache_version(ccache_version_),
- namespace_(namespace_arg),
- entry_size(entry_size_)
-{
-}
-
-uint32_t
-CacheEntryHeader::payload_size() const
-{
- const auto payload_size = entry_size - non_payload_size();
- // In order to support 32-bit ccache builds, restrict size to uint32_t for
- // now. This restriction can be lifted when we drop 32-bit support.
- const auto max = std::numeric_limits<uint32_t>::max();
- if (payload_size > max) {
- throw core::Error(
- FMT("Serialized result too large ({} > {})", payload_size, max));
- }
-
- return payload_size;
-}
-
-void
-CacheEntryHeader::set_entry_size_from_payload_size(const uint64_t payload_size)
-{
- entry_size = non_payload_size() + payload_size;
-}
-
-void
-CacheEntryHeader::inspect(FILE* const stream) const
-{
- PRINT(stream, "Magic: {:04x}\n", magic);
- PRINT(stream, "Entry format version: {}\n", entry_format_version);
- PRINT(stream,
- "Entry type: {} ({})\n",
- static_cast<uint8_t>(entry_type),
- to_string(entry_type));
- PRINT(stream,
- "Compression type: {}\n",
- compression::type_to_string(compression_type));
- PRINT(stream, "Compression level: {}\n", compression_level);
- PRINT(stream, "Creation time: {}\n", creation_time);
- PRINT(stream, "Ccache version: {}\n", ccache_version);
- PRINT(stream, "Namespace: {}\n", namespace_);
- PRINT(stream, "Entry size: {}\n", entry_size);
-}
-
-size_t
-CacheEntryHeader::non_payload_size() const
-{
- return k_static_header_fields_size + ccache_version.length()
- + namespace_.length() + k_static_epilogue_fields_size;
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <compression/types.hpp>
-#include <core/types.hpp>
-
-#include <cstdint>
-
-// Cache entry format
-// ==================
-//
-// Integers are big-endian.
-//
-// <entry> ::= <header> <payload> <epilogue>
-// <header> ::= <magic> <format_ver> <entry_type> <compr_type>
-// <compr_level> <creation_time> <ccache_ver> <namespace>
-// <entry_size>
-// <magic> ::= uint16_t (0xccac)
-// <format_ver> ::= uint8_t
-// <entry_type> ::= <result_entry> | <manifest_entry>
-// <result_entry> ::= 0 (uint8_t)
-// <manifest_entry> ::= 1 (uint8_t)
-// <compr_type> ::= <compr_none> | <compr_zstd>
-// <compr_none> ::= 0 (uint8_t)
-// <compr_zstd> ::= 1 (uint8_t)
-// <compr_level> ::= int8_t
-// <creation_time> ::= uint64_t (Unix epoch time when entry was created)
-// <ccache_ver> ::= string length (uint8_t) + string data
-// <namespace> ::= string length (uint8_t) + string data
-// <entry_size> ::= uint64_t ; = size of file if stored uncompressed
-// ; potentially compressed from here
-// <payload> ::= depends on entry_type
-// <epilogue> ::= <checksum_high> <checksum_low>
-// <checksum_high> ::= uint64_t ; XXH3-128 (high bits) of entry bytes
-// <checksum_low> ::= uint64_t ; XXH3-128 (low bits) of entry bytes
-
-namespace core {
-
-const uint16_t k_ccache_magic = 0xccac;
-const uint16_t k_entry_format_version = 0;
-
-struct CacheEntryHeader
-{
- CacheEntryHeader(core::CacheEntryType entry_type,
- compression::Type compression_type,
- int8_t compression_level,
- uint64_t creation_time,
- const std::string& ccache_version,
- const std::string& namespace_,
- uint64_t entry_size = 0);
-
- uint16_t magic;
- uint8_t entry_format_version;
- core::CacheEntryType entry_type;
- compression::Type compression_type;
- int8_t compression_level;
- uint64_t creation_time;
- std::string ccache_version;
- std::string namespace_;
- uint64_t entry_size;
-
- uint32_t payload_size() const;
- void set_entry_size_from_payload_size(uint64_t payload_size);
- void inspect(FILE* stream) const;
-
-private:
- size_t non_payload_size() const;
-};
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "CacheEntryReader.hpp"
-
-#include <core/exceptions.hpp>
-#include <fmtmacros.hpp>
-
-namespace {
-
-core::CacheEntryType
-cache_entry_type_from_int(const uint8_t entry_type)
-{
- switch (entry_type) {
- case 0:
- return core::CacheEntryType::result;
- break;
- case 1:
- return core::CacheEntryType::manifest;
- break;
- default:
- throw core::Error(FMT("Unknown entry type: {}", entry_type));
- }
-}
-
-} // namespace
-
-namespace core {
-
-CacheEntryReader::CacheEntryReader(core::Reader& reader)
- : m_checksumming_reader(reader)
-{
- const auto magic = m_checksumming_reader.read_int<uint16_t>();
- if (magic != core::k_ccache_magic) {
- throw core::Error(FMT("Bad magic value: 0x{:04x}", magic));
- }
-
- const auto entry_format_version = m_checksumming_reader.read_int<uint8_t>();
- if (entry_format_version != core::k_entry_format_version) {
- throw core::Error(
- FMT("Unknown entry format version: {}", entry_format_version));
- }
-
- const auto entry_type = m_checksumming_reader.read_int<uint8_t>();
- const auto compression_type = m_checksumming_reader.read_int<uint8_t>();
- const auto compression_level = m_checksumming_reader.read_int<int8_t>();
- const auto creation_time = m_checksumming_reader.read_int<uint64_t>();
- const auto ccache_version =
- m_checksumming_reader.read_str(m_checksumming_reader.read_int<uint8_t>());
- const auto tag =
- m_checksumming_reader.read_str(m_checksumming_reader.read_int<uint8_t>());
- const auto entry_size = m_checksumming_reader.read_int<uint64_t>();
-
- m_header = std::make_unique<CacheEntryHeader>(
- cache_entry_type_from_int(entry_type),
- compression::type_from_int(compression_type),
- compression_level,
- creation_time,
- ccache_version,
- tag,
- entry_size);
-
- m_decompressor = compression::Decompressor::create_from_type(
- m_header->compression_type, reader);
- m_checksumming_reader.set_reader(*m_decompressor);
-}
-
-size_t
-CacheEntryReader::read(void* const data, const size_t count)
-{
- return m_checksumming_reader.read(data, count);
-}
-
-void
-CacheEntryReader::finalize()
-{
- const util::XXH3_128::Digest actual = m_checksumming_reader.digest();
- util::XXH3_128::Digest expected;
- m_decompressor->read(expected.bytes(), expected.size());
-
- // actual == null_digest: Checksumming is not enabled now.
- // expected == null_digest: Checksumming was not enabled when the entry was
- // created.
- const util::XXH3_128::Digest null_digest;
-
- if (actual != expected && actual != null_digest && expected != null_digest) {
- throw core::Error(
- FMT("Incorrect checksum (actual {}, expected {})",
- Util::format_base16(actual.bytes(), actual.size()),
- Util::format_base16(expected.bytes(), expected.size())));
- }
-
- m_decompressor->finalize();
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <compression/Decompressor.hpp>
-#include <core/CacheEntryHeader.hpp>
-#include <core/ChecksummingReader.hpp>
-#include <core/Reader.hpp>
-#include <util/XXH3_128.hpp>
-
-namespace core {
-
-// This class knows how to read a cache entry with a format described in
-// CacheEntryHeader.
-class CacheEntryReader : public Reader
-{
-public:
- // Read cache entry data from `reader`.
- CacheEntryReader(Reader& reader);
-
- size_t read(void* data, size_t count) override;
- using Reader::read;
-
- // Close for reading.
- //
- // This method potentially verifies the end state after reading the cache
- // entry and throws `core::Error` if any integrity issues are found.
- void finalize();
-
- const CacheEntryHeader& header() const;
-
-private:
- ChecksummingReader m_checksumming_reader;
- std::unique_ptr<CacheEntryHeader> m_header;
- util::XXH3_128 m_checksum;
- std::unique_ptr<compression::Decompressor> m_decompressor;
-};
-
-inline const CacheEntryHeader&
-CacheEntryReader::header() const
-{
- return *m_header;
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "CacheEntryWriter.hpp"
-
-#include <core/CacheEntryHeader.hpp>
-
-namespace core {
-
-CacheEntryWriter::CacheEntryWriter(core::Writer& writer,
- const CacheEntryHeader& header)
- : m_checksumming_writer(writer),
- m_compressor(compression::Compressor::create_from_type(
- header.compression_type, writer, header.compression_level))
-{
- m_checksumming_writer.write_int(header.magic);
- m_checksumming_writer.write_int(header.entry_format_version);
- m_checksumming_writer.write_int(static_cast<uint8_t>(header.entry_type));
- m_checksumming_writer.write_int(
- static_cast<uint8_t>(header.compression_type));
- m_checksumming_writer.write_int(m_compressor->actual_compression_level());
- m_checksumming_writer.write_int(header.creation_time);
- m_checksumming_writer.write_int<uint8_t>(header.ccache_version.length());
- m_checksumming_writer.write_str(header.ccache_version);
- m_checksumming_writer.write_int<uint8_t>(header.namespace_.length());
- m_checksumming_writer.write_str(header.namespace_);
- m_checksumming_writer.write_int(header.entry_size);
-
- m_checksumming_writer.set_writer(*m_compressor);
-}
-
-void
-CacheEntryWriter::write(const void* const data, const size_t count)
-{
- m_checksumming_writer.write(data, count);
-}
-
-void
-CacheEntryWriter::finalize()
-{
- const auto digest = m_checksumming_writer.digest();
- m_compressor->write(digest.bytes(), digest.size());
- m_compressor->finalize();
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <compression/Compressor.hpp>
-#include <core/ChecksummingWriter.hpp>
-#include <core/Writer.hpp>
-
-namespace core {
-
-struct CacheEntryHeader;
-
-// This class knows how to write a cache entry with a format described in
-// CacheEntryHeader.
-class CacheEntryWriter : public Writer
-{
-public:
- CacheEntryWriter(Writer& writer, const CacheEntryHeader& header);
-
- void write(const void* data, size_t count) override;
- using Writer::write;
-
- // Close for writing.
- //
- // This method potentially verifies the end state after writing the cache
- // entry and throws `core::Error` if any integrity issues are found.
- void finalize() override;
-
-private:
- ChecksummingWriter m_checksumming_writer;
- std::unique_ptr<compression::Compressor> m_compressor;
-};
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <core/Reader.hpp>
-#include <util/XXH3_128.hpp>
-
-namespace core {
-
-class ChecksummingReader : public Reader
-{
-public:
- ChecksummingReader(core::Reader& reader);
-
- using core::Reader::read;
- size_t read(void* data, size_t count) override;
-
- void set_reader(core::Reader& reader);
-
- util::XXH3_128::Digest digest() const;
-
-private:
- core::Reader* m_reader;
- util::XXH3_128 m_checksum;
-};
-
-inline ChecksummingReader::ChecksummingReader(core::Reader& reader)
- : m_reader(&reader)
-{
-}
-
-inline size_t
-ChecksummingReader::read(void* const data, const size_t count)
-{
- const auto bytes_read = m_reader->read(data, count);
- m_checksum.update(data, bytes_read);
- return bytes_read;
-}
-
-inline void
-ChecksummingReader::set_reader(core::Reader& reader)
-{
- m_reader = &reader;
-}
-
-inline util::XXH3_128::Digest
-ChecksummingReader::digest() const
-{
- return m_checksum.digest();
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <core/Writer.hpp>
-#include <util/XXH3_128.hpp>
-
-namespace core {
-
-class ChecksummingWriter : public Writer
-{
-public:
- ChecksummingWriter(core::Writer& writer);
-
- using core::Writer::write;
- void write(const void* data, size_t count) override;
- void finalize() override;
-
- void set_writer(core::Writer& writer);
-
- util::XXH3_128::Digest digest() const;
-
-private:
- core::Writer* m_writer;
- util::XXH3_128 m_checksum;
-};
-
-inline ChecksummingWriter::ChecksummingWriter(core::Writer& writer)
- : m_writer(&writer)
-{
-}
-
-inline void
-ChecksummingWriter::write(const void* const data, const size_t count)
-{
- m_writer->write(data, count);
- m_checksum.update(data, count);
-}
-
-inline void
-ChecksummingWriter::finalize()
-{
- m_writer->finalize();
-}
-
-inline void
-ChecksummingWriter::set_writer(core::Writer& writer)
-{
- m_writer = &writer;
-}
-
-inline util::XXH3_128::Digest
-ChecksummingWriter::digest() const
-{
- return m_checksum.digest();
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <core/Reader.hpp>
-#include <core/exceptions.hpp>
-
-#include <cstdio>
-
-namespace core {
-
-class FileReader : public Reader
-{
-public:
- FileReader(FILE* stream);
-
- size_t read(void* data, size_t size) override;
-
-private:
- FILE* m_stream;
-};
-
-inline FileReader::FileReader(FILE* stream) : m_stream(stream)
-{
-}
-
-inline size_t
-FileReader::read(void* const data, const size_t size)
-{
- if (size == 0) {
- return 0;
- }
- const auto bytes_read = fread(data, 1, size, m_stream);
- if (bytes_read == 0) {
- throw core::Error("Failed to read from file stream");
- }
- return bytes_read;
-}
-
-} // namespace core
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <core/Writer.hpp>
-#include <core/exceptions.hpp>
-
-#include <cstdio>
-
-namespace core {
-
-class FileWriter : public Writer
-{
-public:
- FileWriter(FILE* stream);
-
- void write(const void* data, size_t size) override;
- void finalize() override;
-
-private:
- FILE* m_stream;
-};
-
-inline FileWriter::FileWriter(FILE* const stream) : m_stream(stream)
-{
-}
-
-inline void
-FileWriter::write(const void* const data, const size_t size)
-{
- if (size > 0 && fwrite(data, size, 1, m_stream) != 1) {
- throw core::Error("Failed to write to stream");
- }
-}
-
-inline void
-FileWriter::finalize()
-{
- fflush(m_stream);
-}
-
-} // namespace core
const auto file_count = reader.read_int<uint32_t>();
for (uint32_t i = 0; i < file_count; ++i) {
- m_files.push_back(
- std::string(reader.read_str(reader.read_int<uint16_t>())));
+ m_files.emplace_back(reader.read_str(reader.read_int<uint16_t>()));
}
const auto file_info_count = reader.read_int<uint32_t>();
}
void
-Manifest::serialize(util::Bytes& output) const
+Manifest::serialize(util::Bytes& output)
{
core::CacheEntryDataWriter writer(output);
#pragma once
#include <Digest.hpp>
-#include <util/Bytes.hpp>
+#include <core/Serializer.hpp>
#include <third_party/nonstd/span.hpp>
namespace core {
-class Manifest
+class Manifest : public Serializer
{
public:
static const uint8_t k_format_version;
time_t time_of_compilation,
bool save_timestamp);
- uint32_t serialized_size() const;
- void serialize(util::Bytes& output) const;
+ // core::Serializer
+ uint32_t serialized_size() const override;
+ void serialize(util::Bytes& output) override;
void inspect(FILE* stream) const;
+++ /dev/null
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <Util.hpp>
-#include <core/exceptions.hpp>
-
-#include <cstddef>
-#include <cstdint>
-#include <string>
-
-namespace core {
-
-class Reader
-{
-public:
- virtual ~Reader() = default;
-
- // Read `count` bytes into `data`, returning the actual number of bytes read
- // if not enough data is available. Throws `core::Error` on failure, e.g. if
- // no bytes could be read.
- virtual size_t read(void* data, size_t count) = 0;
-
- // Read an integer. Throws Error on failure.
- template<typename T> T read_int();
-
- // Read an integer into `value`. Throws Error on failure.
- template<typename T> void read_int(T& value);
-
- // Read a string of length `length`. Throws `core::Error` on failure.
- std::string read_str(size_t length);
-};
-
-template<typename T>
-inline T
-Reader::read_int()
-{
- uint8_t buffer[sizeof(T)];
- const auto bytes_read = read(buffer, sizeof(T));
- if (bytes_read != sizeof(T)) {
- throw core::Error("Read underflow");
- }
- T value;
- Util::big_endian_to_int(buffer, value);
- return value;
-}
-
-template<typename T>
-inline void
-Reader::read_int(T& value)
-{
- value = read_int<T>();
-}
-
-inline std::string
-Reader::read_str(const size_t length)
-{
- std::string value(length, 0);
- const auto bytes_read = read(&value[0], length);
- if (bytes_read != length) {
- throw core::Error("Read underflow");
- }
- return value;
-}
-
-} // namespace core
namespace {
-const uint8_t k_result_format_version = 0;
-
// File data stored inside the result file.
const uint8_t k_embedded_file_marker = 0;
bool
should_store_raw_file(const Config& config, core::Result::FileType type)
{
- if (!config.file_clone() && !config.hard_link()) {
+ if (!core::Result::Serializer::use_raw_files(config)) {
return false;
}
} // namespace
-namespace core {
-
-namespace Result {
+namespace core::Result {
-const uint8_t k_version = 1;
+const uint8_t k_format_version = 0;
const char* const k_unknown_file_type = "<unknown type>";
{
CacheEntryDataReader reader(m_data);
const auto result_format_version = reader.read_int<uint8_t>();
- if (result_format_version != k_result_format_version) {
+ if (result_format_version != k_format_version) {
throw Error(FMT("Unknown result format version: {} != {}",
result_format_version,
- k_result_format_version));
+ k_format_version));
}
const auto n_files = reader.read_int<uint8_t>();
return m_serialized_size;
}
-Serializer::SerializeResult
+void
Serializer::serialize(util::Bytes& output)
{
- SerializeResult serialize_result;
CacheEntryDataWriter writer(output);
- writer.write_int(k_result_format_version);
+ writer.write_int(k_format_version);
writer.write_int<uint8_t>(m_file_entries.size());
uint8_t file_number = 0;
writer.write_int(file_size);
if (store_raw) {
- serialize_result.raw_files.emplace(file_number,
- std::get<std::string>(entry.data));
+ m_raw_files.push_back(
+ RawFile{file_number, std::get<std::string>(entry.data)});
} else if (is_file_entry) {
const auto& path = std::get<std::string>(entry.data);
const auto data = util::value_or_throw<Error>(
++file_number;
}
+}
- return serialize_result;
+bool
+Serializer::use_raw_files(const Config& config)
+{
+ return config.file_clone() || config.hard_link();
}
-} // namespace Result
+const std::vector<Serializer::RawFile>&
+Serializer::get_raw_files() const
+{
+ return m_raw_files;
+}
-} // namespace core
+} // namespace core::Result
#pragma once
-#include <util/Bytes.hpp>
+#include <core/Serializer.hpp>
#include <util/types.hpp>
#include <third_party/nonstd/span.hpp>
#include <cstdint>
#include <string>
-#include <unordered_map>
#include <variant>
#include <vector>
namespace Result {
-extern const uint8_t k_version;
+extern const uint8_t k_format_version;
extern const char* const k_unknown_file_type;
};
// This class knows how to serialize a result cache entry.
-class Serializer
+class Serializer : public core::Serializer
{
public:
Serializer(const Config& config);
// Register a file path whose content should be included in the result.
void add_file(FileType file_type, const std::string& path);
- uint32_t serialized_size() const;
+ // core::Serializer
+ uint32_t serialized_size() const override;
+ void serialize(util::Bytes& output) override;
- struct SerializeResult
+ static bool use_raw_files(const Config& config);
+
+ struct RawFile
{
- // Raw files to store in primary storage.
- std::unordered_map<uint8_t /*index*/, std::string /*path*/> raw_files;
+ uint8_t file_number;
+ std::string path;
};
- SerializeResult serialize(util::Bytes& output);
+ // Get raw files to store in primary storage.
+ const std::vector<RawFile>& get_raw_files() const;
private:
const Config& m_config;
std::variant<nonstd::span<const uint8_t>, std::string> data;
};
std::vector<FileEntry> m_file_entries;
+
+ std::vector<RawFile> m_raw_files;
};
} // namespace Result
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
+// Copyright (C) 2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
-#include <cstdint>
-#include <string>
-
-class Config;
+#include <util/Bytes.hpp>
-namespace compression {
-
-enum class Type : uint8_t {
- none = 0,
- zstd = 1,
-};
+#include <third_party/nonstd/span.hpp>
-int8_t level_from_config(const Config& config);
-
-Type type_from_config(const Config& config);
+#include <cstdint>
-Type type_from_int(uint8_t type);
+namespace core {
-std::string type_to_string(Type type);
+class Serializer
+{
+public:
+ virtual ~Serializer() = default;
+ virtual uint32_t serialized_size() const = 0;
+ virtual void serialize(util::Bytes& output) = 0;
+};
-} // namespace compression
+} // namespace core
+++ /dev/null
-// Copyright (C) 2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <Util.hpp>
-#include <assertions.hpp>
-
-#include <cstddef>
-#include <cstdint>
-#include <string>
-
-namespace core {
-
-class Writer
-{
-public:
- virtual ~Writer() = default;
-
- // Write `count` bytes from `data`. Throws `core::Error` on failure.
- virtual void write(const void* data, size_t count) = 0;
-
- // Write integer `value`. Throws `core::Error` on failure.
- template<typename T> void write_int(T value);
-
- // Write `value`. Throws `core::Error` on failure.
- void write_str(const std::string& value);
-
- // Finalize writing, e.g. flush written bytes and potentially check for error
- // states. Throws `core::Error` on failure.
- virtual void finalize() = 0;
-};
-
-template<typename T>
-inline void
-Writer::write_int(const T value)
-{
- uint8_t buffer[sizeof(T)];
- Util::int_to_big_endian(value, buffer);
- write(buffer, sizeof(T));
-}
-
-inline void
-Writer::write_str(const std::string& value)
-{
- write(value.data(), value.length());
-}
-
-} // namespace core
#include <InodeCache.hpp>
#include <ProgressBar.hpp>
#include <ccache.hpp>
-#include <core/CacheEntryReader.hpp>
-#include <core/FileReader.hpp>
+#include <core/CacheEntry.hpp>
#include <core/Manifest.hpp>
#include <core/Result.hpp>
#include <core/ResultExtractor.hpp>
PRINT(stdout, "({}) {} = {}\n", origin, key, value);
}
+static nonstd::expected<std::vector<uint8_t>, std::string>
+read_from_path_or_stdin(const std::string& path)
+{
+ if (path == "-") {
+ std::vector<uint8_t> output;
+ const auto result =
+ util::read_fd(STDIN_FILENO, [&](const uint8_t* data, size_t size) {
+ output.insert(output.end(), data, data + size);
+ });
+ if (!result) {
+ return nonstd::make_unexpected(
+ FMT("Failed to read from stdin: {}", result.error()));
+ }
+ return output;
+ } else {
+ const auto result = util::read_file<std::vector<uint8_t>>(path);
+ if (!result) {
+ return nonstd::make_unexpected(
+ FMT("Failed to read from {}: {}", path, result.error()));
+ }
+ return *result;
+ }
+}
+
static int
inspect_path(const std::string& path)
{
- File file = path == "-" ? File(stdin) : File(path, "rb");
- if (!file) {
- PRINT(stderr, "Error: Failed to open \"{}\"", path);
+ const auto cache_entry_data = read_from_path_or_stdin(path);
+ if (!cache_entry_data) {
+ PRINT(stderr, "Error: {}\n", cache_entry_data.error());
return EXIT_FAILURE;
}
- core::FileReader file_reader(file.get());
- core::CacheEntryReader cache_entry_reader(file_reader);
- const auto& header = cache_entry_reader.header();
- header.inspect(stdout);
+ core::CacheEntry cache_entry(*cache_entry_data);
+ fputs(cache_entry.header().inspect().c_str(), stdout);
- std::vector<uint8_t> data;
- data.resize(cache_entry_reader.header().payload_size());
- cache_entry_reader.read(data.data(), data.size());
+ const auto payload = cache_entry.payload();
- switch (header.entry_type) {
+ switch (cache_entry.header().entry_type) {
case core::CacheEntryType::manifest: {
core::Manifest manifest;
- manifest.read(data);
+ manifest.read(payload);
manifest.inspect(stdout);
break;
}
case core::CacheEntryType::result:
- Result::Deserializer result_deserializer(data);
+ Result::Deserializer result_deserializer(payload);
ResultInspector result_inspector(stdout);
result_deserializer.visit(result_inspector);
break;
}
- cache_entry_reader.finalize();
+ cache_entry.verify_checksum();
return EXIT_SUCCESS;
}
util::XXH3_128 checksum;
Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY));
if (fd) {
- util::read_fd(*fd, [&checksum](const void* data, size_t size) {
- checksum.update(data, size);
+ util::read_fd(*fd, [&checksum](const uint8_t* data, size_t size) {
+ checksum.update({data, size});
});
const auto digest = checksum.digest();
PRINT(
- stdout, "{}\n", Util::format_base16(digest.bytes(), digest.size()));
+ stdout, "{}\n", Util::format_base16(digest.data(), digest.size()));
} else {
PRINT(stderr, "Error: Failed to checksum {}\n", arg);
}
}
case EXTRACT_RESULT: {
- File file = arg == "-" ? File(stdin) : File(arg, "rb");
- if (!file) {
- PRINT(stderr, "Error: Failed to open \"{}\"", arg);
+ const auto cache_entry_data = read_from_path_or_stdin(arg);
+ if (!cache_entry_data) {
+ PRINT(stderr, "Error: \"{}\"", cache_entry_data.error());
return EXIT_FAILURE;
}
std::optional<ResultExtractor::GetRawFilePathFunction> get_raw_file_path;
- if (arg == "-") {
+ if (arg != "-") {
get_raw_file_path = [&](uint8_t file_number) {
return storage::primary::PrimaryStorage::get_raw_file_path(
arg, file_number);
};
}
ResultExtractor result_extractor(".", get_raw_file_path);
- core::FileReader file_reader(file.get());
- core::CacheEntryReader cache_entry_reader(file_reader);
- std::vector<uint8_t> data;
- data.resize(cache_entry_reader.header().payload_size());
- cache_entry_reader.read(data.data(), data.size());
- Result::Deserializer result_deserializer(data);
+ core::CacheEntry cache_entry(*cache_entry_data);
+ const auto payload = cache_entry.payload();
+
+ Result::Deserializer result_deserializer(payload);
result_deserializer.visit(result_extractor);
- cache_entry_reader.finalize();
+ cache_entry.verify_checksum();
return EXIT_SUCCESS;
}
-// Copyright (C) 2021 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include "types.hpp"
+#include <Config.hpp>
+#include <assertions.hpp>
+#include <core/exceptions.hpp>
+#include <fmtmacros.hpp>
+
namespace core {
std::string
}
}
+int8_t
+compression_level_from_config(const Config& config)
+{
+ return config.compression() ? config.compression_level() : 0;
+}
+
+CompressionType
+compression_type_from_config(const Config& config)
+{
+ return config.compression() ? CompressionType::zstd : CompressionType::none;
+}
+
+CompressionType
+compression_type_from_int(const uint8_t type)
+{
+ switch (type) {
+ case static_cast<uint8_t>(CompressionType::none):
+ return CompressionType::none;
+
+ case static_cast<uint8_t>(CompressionType::zstd):
+ return CompressionType::zstd;
+ }
+
+ throw core::Error(FMT("Unknown type: {}", type));
+}
+
+std::string
+to_string(const CompressionType type)
+{
+ switch (type) {
+ case CompressionType::none:
+ return "none";
+
+ case CompressionType::zstd:
+ return "zstd";
+ }
+
+ ASSERT(false);
+}
+
} // namespace core
-// Copyright (C) 2021 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include <cstdint>
#include <string>
+class Config;
+
namespace core {
enum class CacheEntryType : uint8_t { result = 0, manifest = 1 };
std::string to_string(CacheEntryType type);
+enum class CompressionType : uint8_t {
+ none = 0,
+ zstd = 1,
+};
+
+int8_t compression_level_from_config(const Config& config);
+
+CompressionType compression_type_from_config(const Config& config);
+
+CompressionType compression_type_from_int(uint8_t type);
+
+std::string to_string(CompressionType type);
+
} // namespace core
#include <File.hpp>
#include <Logging.hpp>
#include <Util.hpp>
-#include <core/CacheEntryReader.hpp>
-#include <core/FileReader.hpp>
+#include <core/CacheEntry.hpp>
+#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
#include <storage/primary/CacheFile.hpp>
#include <storage/primary/StatsFile.hpp>
#include <storage/primary/util.hpp>
+#include <util/file.hpp>
#include <util/string.hpp>
#ifdef INODE_CACHE_SUPPORTED
if (namespace_) {
try {
- File file_stream(file.path(), "rb");
- core::FileReader file_reader(*file_stream);
- core::CacheEntryReader reader(file_reader);
- if (reader.header().namespace_ != *namespace_) {
+ core::CacheEntry::Header header(file.path());
+ if (header.namespace_ != *namespace_) {
continue;
}
} catch (core::Error&) {
#include <Logging.hpp>
#include <ThreadPool.hpp>
#include <assertions.hpp>
-#include <compression/ZstdCompressor.hpp>
-#include <core/CacheEntryReader.hpp>
-#include <core/CacheEntryWriter.hpp>
-#include <core/FileReader.hpp>
-#include <core/FileWriter.hpp>
+#include <core/CacheEntry.hpp>
#include <core/Manifest.hpp>
#include <core/Result.hpp>
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
#include <storage/primary/StatsFile.hpp>
+#include <util/expected.hpp>
#include <util/file.hpp>
#include <util/string.hpp>
} // namespace
-static File
-open_file(const std::string& path, const char* const mode)
-{
- File f(path, mode);
- if (!f) {
- throw core::Error(
- FMT("failed to open {} for reading: {}", path, strerror(errno)));
- }
- return f;
-}
-
-static std::unique_ptr<core::CacheEntryReader>
-create_reader(const CacheFile& cache_file, core::Reader& reader)
-{
- if (cache_file.type() == CacheFile::Type::unknown) {
- throw core::Error(FMT("unknown file type for {}", cache_file.path()));
- }
-
- return std::make_unique<core::CacheEntryReader>(reader);
-}
-
-static std::unique_ptr<core::CacheEntryWriter>
-create_writer(core::Writer& writer, const core::CacheEntryHeader& header)
-{
- return std::make_unique<core::CacheEntryWriter>(writer, header);
-}
-
static void
recompress_file(RecompressionStatistics& statistics,
const std::string& stats_file,
const CacheFile& cache_file,
const std::optional<int8_t> level)
{
- auto file = open_file(cache_file.path(), "rb");
- core::FileReader file_reader(file.get());
- auto reader = create_reader(cache_file, file_reader);
+ core::CacheEntry::Header header(cache_file.path());
- const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
- const uint64_t content_size = reader->header().entry_size;
const int8_t wanted_level =
- level
- ? (*level == 0 ? compression::ZstdCompressor::default_compression_level
- : *level)
- : 0;
+ level ? (*level == 0 ? core::CacheEntry::default_compression_level : *level)
+ : 0;
+ const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log);
- if (reader->header().compression_level == wanted_level) {
- statistics.update(content_size, old_stat.size(), old_stat.size(), 0);
+ if (header.compression_level == wanted_level) {
+ statistics.update(header.entry_size, old_stat.size(), old_stat.size(), 0);
return;
}
- LOG("Recompressing {} to {}",
- cache_file.path(),
- level ? FMT("level {}", wanted_level) : "uncompressed");
- AtomicFile atomic_new_file(cache_file.path(), AtomicFile::Mode::binary);
- core::FileWriter file_writer(atomic_new_file.stream());
- auto header = reader->header();
+ const auto cache_file_data = util::value_or_throw<core::Error>(
+ util::read_file<util::Bytes>(cache_file.path()),
+ FMT("Failed to read {}: ", cache_file.path()));
+ core::CacheEntry cache_entry(cache_file_data);
+ cache_entry.verify_checksum();
+
+ header.entry_format_version = core::k_cache_entry_format_version;
header.compression_type =
- level ? compression::Type::zstd : compression::Type::none;
+ level ? core::CompressionType::zstd : core::CompressionType::none;
header.compression_level = wanted_level;
- auto writer = create_writer(file_writer, header);
-
- char buffer[CCACHE_READ_BUFFER_SIZE];
- size_t bytes_left = reader->header().payload_size();
- while (bytes_left > 0) {
- size_t bytes_to_read = std::min(bytes_left, sizeof(buffer));
- reader->read(buffer, bytes_to_read);
- writer->write(buffer, bytes_to_read);
- bytes_left -= bytes_to_read;
- }
- reader->finalize();
- writer->finalize();
- file.close();
- atomic_new_file.commit();
+ AtomicFile new_cache_file(cache_file.path(), AtomicFile::Mode::binary);
+ new_cache_file.write(
+ core::CacheEntry::serialize(header, cache_entry.payload()));
+ new_cache_file.commit();
// Restore mtime/atime to keep cache LRU cleanup working as expected:
util::set_timestamps(cache_file.path(), old_stat.mtim(), old_stat.atim());
cs.increment(core::Statistic::cache_size_kibibyte,
Util::size_change_kibibyte(old_stat, new_stat));
});
- statistics.update(content_size, old_stat.size(), new_stat.size(), 0);
-
- LOG("Recompression of {} done", cache_file.path());
+ statistics.update(header.entry_size, old_stat.size(), new_stat.size(), 0);
}
CompressionStatistics
cs.on_disk_size += cache_file.lstat().size_on_disk();
try {
- auto file = open_file(cache_file.path(), "rb");
- core::FileReader file_reader(file.get());
- auto reader = create_reader(cache_file, file_reader);
+ core::CacheEntry::Header header(cache_file.path());
cs.compr_size += cache_file.lstat().size();
- cs.content_size += reader->header().entry_size;
+ cs.content_size += header.entry_size;
} catch (core::Error&) {
cs.incompr_size += cache_file.lstat().size();
}
file.cpp
path.cpp
string.cpp
+ zstd.cpp
)
target_sources(ccache_framework PRIVATE ${sources})
-// Copyright (C) 2021 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
+#include <Util.hpp>
+#include <util/Bytes.hpp>
+
+#include <third_party/nonstd/span.hpp>
#ifdef USE_XXH_DISPATCH
-# include "third_party/xxh_x86dispatch.h"
+# include <third_party/xxh_x86dispatch.h>
#else
-# include "third_party/xxhash.h"
+# include <third_party/xxhash.h>
#endif
-#include <Util.hpp>
-
#include <cstdint>
#include <cstring>
class XXH3_128
{
public:
- struct Digest
- {
- public:
- const uint8_t* bytes() const;
- uint8_t* bytes();
- constexpr static size_t size();
-
- bool operator==(const Digest& other) const;
- bool operator!=(const Digest& other) const;
-
- private:
- uint8_t m_bytes[16] = {};
- };
+ static constexpr size_t k_digest_size = 16;
XXH3_128();
~XXH3_128();
void reset();
- void update(const void* data, size_t length);
- Digest digest() const;
+ void update(nonstd::span<const uint8_t> data);
+ util::Bytes digest() const;
private:
XXH3_state_t* m_state;
};
-inline const uint8_t*
-XXH3_128::Digest::bytes() const
-{
- return m_bytes;
-}
-
-inline uint8_t*
-XXH3_128::Digest::bytes()
-{
- return m_bytes;
-}
-
-inline constexpr size_t
-XXH3_128::Digest::size()
-{
- return sizeof(m_bytes);
-}
-
-inline bool
-XXH3_128::Digest::operator==(const XXH3_128::Digest& other) const
-{
- return memcmp(bytes(), other.bytes(), size()) == 0;
-}
-
-inline bool
-XXH3_128::Digest::operator!=(const XXH3_128::Digest& other) const
-{
- return !(*this == other);
-}
-
inline XXH3_128::XXH3_128() : m_state(XXH3_createState())
{
reset();
}
inline void
-XXH3_128::update(const void* data, size_t length)
+XXH3_128::update(nonstd::span<const uint8_t> data)
{
- XXH3_128bits_update(m_state, data, length);
+ XXH3_128bits_update(m_state, data.data(), data.size());
}
-inline XXH3_128::Digest
+inline util::Bytes
XXH3_128::digest() const
{
const auto result = XXH3_128bits_digest(m_state);
- XXH3_128::Digest digest;
- Util::int_to_big_endian(result.high64, digest.bytes());
- Util::int_to_big_endian(result.low64, digest.bytes() + 8);
+ util::Bytes digest(k_digest_size);
+ Util::int_to_big_endian(result.high64, &digest[0]);
+ Util::int_to_big_endian(result.low64, &digest[8]);
return digest;
}
test_Config.cpp
test_Depfile.cpp
test_Hash.cpp
- test_NullCompression.cpp
test_Stat.cpp
test_Util.cpp
- test_ZstdCompression.cpp
test_argprocessing.cpp
test_ccache.cpp
test_compopt.cpp
test_util_file.cpp
test_util_path.cpp
test_util_string.cpp
+ test_util_zstd.cpp
)
if(INODE_CACHE_SUPPORTED)
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "../src/File.hpp"
-#include "TestUtil.hpp"
-
-#include <compression/Compressor.hpp>
-#include <compression/Decompressor.hpp>
-#include <compression/types.hpp>
-#include <core/FileReader.hpp>
-#include <core/FileWriter.hpp>
-
-#include "third_party/doctest.h"
-
-#include <cstring>
-
-using compression::Compressor;
-using compression::Decompressor;
-using TestUtil::TestContext;
-
-TEST_SUITE_BEGIN("NullCompression");
-
-TEST_CASE("compression::Type::none roundtrip")
-{
- TestContext test_context;
-
- File f("data.uncompressed", "w");
- core::FileWriter fw(f.get());
- auto compressor =
- Compressor::create_from_type(compression::Type::none, fw, 1);
- CHECK(compressor->actual_compression_level() == 0);
- compressor->write("foobar", 6);
- compressor->finalize();
-
- f.open("data.uncompressed", "r");
- core::FileReader fr(f.get());
- auto decompressor =
- Decompressor::create_from_type(compression::Type::none, fr);
-
- char buffer[4];
- decompressor->read(buffer, 4);
- CHECK(memcmp(buffer, "foob", 4) == 0);
-
- SUBCASE("Garbage data")
- {
- // Not reached the end.
- CHECK_THROWS_WITH(decompressor->finalize(),
- "Garbage data at end of uncompressed stream");
- }
-
- SUBCASE("Read to end")
- {
- decompressor->read(buffer, 2);
- CHECK(memcmp(buffer, "ar", 2) == 0);
-
- // Reached the end.
- decompressor->finalize();
-
- // Nothing left to read.
- CHECK_THROWS_WITH(decompressor->read(buffer, 1),
- "Failed to read from file stream");
- }
-}
-
-TEST_SUITE_END();
+++ /dev/null
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "../src/File.hpp"
-#include "TestUtil.hpp"
-
-#include <compression/Compressor.hpp>
-#include <compression/Decompressor.hpp>
-#include <compression/types.hpp>
-#include <core/FileReader.hpp>
-#include <core/FileWriter.hpp>
-
-#include "third_party/doctest.h"
-
-#include <cstring>
-
-using compression::Compressor;
-using compression::Decompressor;
-using TestUtil::TestContext;
-
-TEST_SUITE_BEGIN("ZstdCompression");
-
-TEST_CASE("Small compression::Type::zstd roundtrip")
-{
- TestContext test_context;
-
- File f("data.zstd", "wb");
- core::FileWriter fw(f.get());
- auto compressor =
- Compressor::create_from_type(compression::Type::zstd, fw, 1);
- CHECK(compressor->actual_compression_level() == 1);
- compressor->write("foobar", 6);
- compressor->finalize();
-
- f.open("data.zstd", "rb");
- core::FileReader fr(f.get());
- auto decompressor =
- Decompressor::create_from_type(compression::Type::zstd, fr);
-
- char buffer[4];
- decompressor->read(buffer, 4);
- CHECK(memcmp(buffer, "foob", 4) == 0);
-
- // Not reached the end.
- CHECK_THROWS_WITH(decompressor->finalize(),
- "Garbage data at end of zstd input stream");
-
- decompressor->read(buffer, 2);
- CHECK(memcmp(buffer, "ar", 2) == 0);
-
- // Reached the end.
- decompressor->finalize();
-
- // Nothing left to read.
- CHECK_THROWS_WITH(decompressor->read(buffer, 1),
- "Failed to read from file stream");
-}
-
-TEST_CASE("Large compressible compression::Type::zstd roundtrip")
-{
- TestContext test_context;
-
- char data[] = "The quick brown fox jumps over the lazy dog";
-
- File f("data.zstd", "wb");
- core::FileWriter fw(f.get());
- auto compressor =
- Compressor::create_from_type(compression::Type::zstd, fw, 1);
- for (size_t i = 0; i < 1000; i++) {
- compressor->write(data, sizeof(data));
- }
- compressor->finalize();
-
- f.open("data.zstd", "rb");
- core::FileReader fr(f.get());
- auto decompressor =
- Decompressor::create_from_type(compression::Type::zstd, fr);
-
- char buffer[sizeof(data)];
- for (size_t i = 0; i < 1000; i++) {
- decompressor->read(buffer, sizeof(buffer));
- CHECK(memcmp(buffer, data, sizeof(data)) == 0);
- }
-
- // Reached the end.
- decompressor->finalize();
-
- // Nothing left to read.
- CHECK_THROWS_WITH(decompressor->read(buffer, 1),
- "Failed to read from file stream");
-}
-
-TEST_CASE("Large uncompressible compression::Type::zstd roundtrip")
-{
- TestContext test_context;
-
- char data[100000];
- for (char& c : data) {
- c = rand() % 256;
- }
-
- File f("data.zstd", "wb");
- core::FileWriter fw(f.get());
- auto compressor =
- Compressor::create_from_type(compression::Type::zstd, fw, 1);
- compressor->write(data, sizeof(data));
- compressor->finalize();
-
- f.open("data.zstd", "rb");
- core::FileReader fr(f.get());
- auto decompressor =
- Decompressor::create_from_type(compression::Type::zstd, fr);
-
- char buffer[sizeof(data)];
- decompressor->read(buffer, sizeof(buffer));
- CHECK(memcmp(buffer, data, sizeof(data)) == 0);
-
- decompressor->finalize();
-}
-
-TEST_SUITE_END();
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
+// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include "../src/Config.hpp"
-#include <compression/types.hpp>
+#include <core/types.hpp>
#include "third_party/doctest.h"
TEST_SUITE_BEGIN("compression");
-TEST_CASE("compression::level_from_config")
+TEST_CASE("compression_level_from_config")
{
Config config;
- CHECK(compression::level_from_config(config) == 0);
+ CHECK(core::compression_level_from_config(config) == 0);
}
-TEST_CASE("compression::type_from_config")
+TEST_CASE("compression_type_from_config")
{
Config config;
- CHECK(compression::type_from_config(config) == compression::Type::zstd);
+ CHECK(core::compression_type_from_config(config)
+ == core::CompressionType::zstd);
}
-TEST_CASE("compression::type_from_int")
+TEST_CASE("compression_type_from_int")
{
- CHECK(compression::type_from_int(0) == compression::Type::none);
- CHECK(compression::type_from_int(1) == compression::Type::zstd);
- CHECK_THROWS_WITH(compression::type_from_int(2), "Unknown type: 2");
+ CHECK(core::compression_type_from_int(0) == core::CompressionType::none);
+ CHECK(core::compression_type_from_int(1) == core::CompressionType::zstd);
+ CHECK_THROWS_WITH(core::compression_type_from_int(2), "Unknown type: 2");
}
-TEST_CASE("compression::type_to_string")
+TEST_CASE("to_string(CompressionType)")
{
- CHECK(compression::type_to_string(compression::Type::none) == "none");
- CHECK(compression::type_to_string(compression::Type::zstd) == "zstd");
+ CHECK(core::to_string(core::CompressionType::none) == "none");
+ CHECK(core::to_string(core::CompressionType::zstd) == "zstd");
}
TEST_SUITE_END();
-// Copyright (C) 2011-2021 Joel Rosdahl and other contributors
+// Copyright (C) 2011-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <util/XXH3_128.hpp>
+#include <util/string.hpp>
#include <third_party/doctest.h>
{
util::XXH3_128 checksum;
auto digest = checksum.digest();
- CHECK(Util::format_base16(digest.bytes(), 16)
+ CHECK(Util::format_base16(digest.data(), 16)
== "99aa06d3014798d86001c324468d497f");
- checksum.update("foo", 3);
+ checksum.update(util::to_span("foo"));
digest = checksum.digest();
- CHECK(Util::format_base16(digest.bytes(), 16)
+ CHECK(Util::format_base16(digest.data(), 16)
== "79aef92e83454121ab6e5f64077e7d8a");
- checksum.update("t", 1);
+ checksum.update(util::to_span("t"));
digest = checksum.digest();
- CHECK(Util::format_base16(digest.bytes(), 16)
+ CHECK(Util::format_base16(digest.data(), 16)
== "e6045075b5bf1ae7a3e4c87775e6c97f");
checksum.reset();
digest = checksum.digest();
- CHECK(Util::format_base16(digest.bytes(), 16)
+ CHECK(Util::format_base16(digest.data(), 16)
== "99aa06d3014798d86001c324468d497f");
}
projects / thirdparty / ccache.git / commitdiff
