* Converted compression code into C++ interfaces and implementations.
* Extracted code for reading/writing results/manifests into
reader/writer classes.
* C++-ified result/manifest parsing code.
non_third_party_sources = \
src/AtomicFile.cpp \
+ src/CacheEntryReader.cpp \
+ src/CacheEntryWriter.cpp \
src/CacheFile.cpp \
+ src/Compression.cpp \
+ src/Compressor.cpp \
src/Config.cpp \
+ src/Decompressor.cpp \
+ src/NullCompressor.cpp \
+ src/NullDecompressor.cpp \
src/ProgressBar.cpp \
src/Util.cpp \
+ src/ZstdCompressor.cpp \
+ src/ZstdDecompressor.cpp \
src/args.cpp \
src/ccache.cpp \
src/cleanup.cpp \
- src/common_header.cpp \
src/compopt.cpp \
- src/compr_none.cpp \
- src/compr_zstd.cpp \
src/compress.cpp \
- src/compression.cpp \
src/counters.cpp \
- src/decompr_none.cpp \
- src/decompr_zstd.cpp \
src/execute.cpp \
src/exitfn.cpp \
src/hash.cpp \
test_suites += unittest/test_AtomicFile.cpp
test_suites += unittest/test_Checksum.cpp
+test_suites += unittest/test_Compression.cpp
test_suites += unittest/test_Config.cpp
+test_suites += unittest/test_NullCompression.cpp
test_suites += unittest/test_Util.cpp
+test_suites += unittest/test_ZstdCompression.cpp
test_suites += unittest/test_args.cpp
test_suites += unittest/test_argument_processing.cpp
test_suites += unittest/test_compopt.cpp
-test_suites += unittest/test_compr_none.cpp
-test_suites += unittest/test_compr_zstd.cpp
test_suites += unittest/test_counters.cpp
test_suites += unittest/test_hash.cpp
test_suites += unittest/test_hashutil.cpp
non_third_party_headers = \
src/AtomicFile.hpp \
+ src/CacheEntryReader.hpp \
+ src/CacheEntryWriter.hpp \
src/CacheFile.hpp \
src/Checksum.hpp \
+ src/Compression.hpp \
+ src/Compressor.hpp \
+ src/Decompressor.hpp \
src/Config.hpp \
src/Error.hpp \
+ src/File.hpp \
+ src/NonCopyable.hpp \
+ src/NullCompressor.hpp \
+ src/NullDecompressor.hpp \
src/ProgressBar.hpp \
src/Util.hpp \
+ src/ZstdCompressor.hpp \
+ src/ZstdDecompressor.hpp \
src/ccache.hpp \
src/cleanup.hpp \
- src/common_header.hpp \
src/compopt.hpp \
src/compress.hpp \
- src/compression.hpp \
src/counters.hpp \
src/hash.hpp \
src/hashutil.hpp \
- src/int_bytes_conversion.hpp \
src/language.hpp \
src/macroskip.hpp \
src/manifest.hpp \
files using the real-time compression algorithm Zstandard. The setting only
has effect if <<config_compression,*compression*>> is enabled (which it is
by default). Zstandard is extremely fast for decompression and very fast
- for compression for lower
- compression levels. The default is 0.
+ for compression for lower compression levels. The default is 0.
+
Semantics of *compression_level*:
+
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "CacheEntryReader.hpp"
+
+#include "Checksum.hpp"
+#include "Compressor.hpp"
+#include "Error.hpp"
+#include "ccache.hpp"
+
+#include <fmt/core.h>
+
+CacheEntryReader::CacheEntryReader(FILE* stream,
+ const uint8_t expected_magic[4],
+ uint8_t expected_version,
+ Checksum* checksum)
+ : m_checksum(checksum)
+{
+ uint8_t header_bytes[15];
+ if (fread(header_bytes, sizeof(header_bytes), 1, stream) != 1) {
+ throw Error("Error reading header");
+ }
+
+ memcpy(m_magic, header_bytes, sizeof(m_magic));
+ m_version = header_bytes[4];
+ m_compression_type = Compression::type_from_int(header_bytes[5]);
+ m_compression_level = header_bytes[6];
+ Util::big_endian_to_int(header_bytes + 7, m_content_size);
+
+ if (memcmp(m_magic, expected_magic, sizeof(m_magic)) != 0) {
+ throw Error(fmt::format("Bad magic value 0x{:02x}{:02x}{:02x}{:02x}",
+ m_magic[0],
+ m_magic[1],
+ m_magic[2],
+ m_magic[3]));
+ }
+ if (m_version != expected_version) {
+ throw Error(fmt::format(
+ "Unknown version (actual {}, expected {})", m_version, expected_version));
+ }
+ if (m_compression_type == Compression::Type::none) {
+ // Since we have the size available, let's use it as a super primitive
+ // consistency check for the non-compressed case. (A real checksum is used
+ // for compressed data.)
+ struct stat st;
+ if (x_fstat(fileno(stream), &st) != 0) {
+ throw Error(fmt::format("Failed to fstat: {}", strerror(errno)));
+ }
+ if (static_cast<uint64_t>(st.st_size) != m_content_size) {
+ throw Error(fmt::format(
+ "Bad uncompressed file size (actual {} bytes, expected {} bytes)",
+ st.st_size,
+ m_content_size));
+ }
+ }
+
+ if (m_checksum) {
+ m_checksum->update(header_bytes, sizeof(header_bytes));
+ }
+
+ m_decompressor = Decompressor::create_from_type(m_compression_type, stream);
+}
+
+void
+CacheEntryReader::dump_header(FILE* dump_stream)
+{
+ fmt::print(dump_stream, "Magic: {:.4}\n", m_magic);
+ fmt::print(dump_stream, "Version: {}\n", m_version);
+ fmt::print(dump_stream,
+ "Compression type: {}\n",
+ Compression::type_to_string(m_compression_type));
+ fmt::print(dump_stream, "Compression level: {}\n", m_compression_level);
+ fmt::print(dump_stream, "Content size: {}\n", m_content_size);
+}
+
+void
+CacheEntryReader::read(void* data, size_t count)
+{
+ m_decompressor->read(data, count);
+ m_checksum->update(data, count);
+}
+
+void
+CacheEntryReader::finalize()
+{
+ m_decompressor->finalize();
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Decompressor.hpp"
+#include "Util.hpp"
+
+#include <cstdio>
+#include <memory>
+
+class Checksum;
+
+// This class knows how to read a cache entry with a common header and a
+// payload part that is different depending on the cache entry type (result or
+// manifest).
+class CacheEntryReader
+{
+public:
+ // Constructor.
+ //
+ // Parameters:
+ // - stream: Stream to read header and payload from.
+ // - expected_magic: Expected magic bytes (first four bytes of the file).
+ // - expected_version: Expected file format version.
+ // - checksum: Checksum state that will be updated with the read bytes, or
+ // nullptr for no checksumming.
+ CacheEntryReader(FILE* stream,
+ const uint8_t expected_magic[4],
+ uint8_t expected_version,
+ Checksum* checksum = nullptr);
+
+ // Dump header information in text format.
+ //
+ // Parameters:
+ // - dump_stream: Stream to write to.
+ void dump_header(FILE* dump_stream);
+
+ // Read data into a buffer from the payload.
+ //
+ // Parameters:
+ // - data: Buffer to write data to.
+ // - count: How many bytes to write.
+ //
+ // Throws Error on failure.
+ void read(void* data, size_t count);
+
+ // Read an unsigned integer from the payload.
+ //
+ // Parameters:
+ // - value: Variable to write to.
+ //
+ // Throws Error on failure.
+ template<typename T> void read(T& value);
+
+ // Close for reading.
+ //
+ // This method potentially verifies the end state after reading the cache
+ // entry and throws Error if any integrity issues are found.
+ void finalize();
+
+ // Get size of the content (header + payload).
+ uint64_t content_size() const;
+
+private:
+ std::unique_ptr<Decompressor> m_decompressor;
+ Checksum* m_checksum;
+ char m_magic[4];
+ uint8_t m_version;
+ Compression::Type m_compression_type;
+ int8_t m_compression_level;
+ uint64_t m_content_size;
+};
+
+template<typename T>
+inline void
+CacheEntryReader::read(T& value)
+{
+ uint8_t buffer[sizeof(T)];
+ read(buffer, sizeof(T));
+ Util::big_endian_to_int(buffer, value);
+}
+
+inline uint64_t
+CacheEntryReader::content_size() const
+{
+ return m_content_size;
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "CacheEntryWriter.hpp"
+
+#include "Checksum.hpp"
+
+CacheEntryWriter::CacheEntryWriter(FILE* stream,
+ const uint8_t magic[4],
+ uint8_t version,
+ Compression::Type compression_type,
+ int8_t compression_level,
+ uint64_t content_size,
+ Checksum& checksum)
+ : m_compressor(
+ Compressor::create_from_type(compression_type, stream, compression_level)),
+ m_checksum(checksum)
+{
+ uint8_t header_bytes[15];
+ memcpy(header_bytes, magic, 4);
+ header_bytes[4] = version;
+ header_bytes[5] = static_cast<uint8_t>(compression_type);
+ header_bytes[6] = m_compressor->actual_compression_level();
+ Util::int_to_big_endian(content_size, header_bytes + 7);
+ if (fwrite(header_bytes, sizeof(header_bytes), 1, stream) != 1) {
+ throw Error("Failed to write cache entry header");
+ }
+ checksum.update(header_bytes, sizeof(header_bytes));
+}
+
+void
+CacheEntryWriter::write(const void* data, size_t count)
+{
+ m_compressor->write(data, count);
+ m_checksum.update(data, count);
+}
+
+void
+CacheEntryWriter::finalize()
+{
+ m_compressor->finalize();
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Compressor.hpp"
+#include "Util.hpp"
+
+#include <cstdio>
+#include <memory>
+
+class Checksum;
+
+// This class knows how to write a cache entry with a common header and a
+// payload part that is different depending on the cache entry type (result or
+// manifest).
+class CacheEntryWriter
+{
+public:
+ // Constructor.
+ //
+ // Parameters:
+ // - stream: Stream to write header + payload to.
+ // - magic: File format magic bytes.
+ // - version: File format version.
+ // - compression_type: Compression type to use.
+ // - compression_level: Compression level to use.
+ // - content_size: Content size.
+ // - checksum: Checksum state that will be updated with the written bytes.
+ CacheEntryWriter(FILE* stream,
+ const uint8_t magic[4],
+ uint8_t version,
+ Compression::Type compression_type,
+ int8_t compression_level,
+ uint64_t content_size,
+ Checksum& checksum);
+
+ // Write data to the payload from a buffer.
+ //
+ // Parameters:
+ // - data: Data to write.
+ // - count: Size of data to write.
+ //
+ // Throws Error on failure.
+ void write(const void* data, size_t count);
+
+ // Write an unsigned integer to the payload.
+ //
+ // Parameters:
+ // - value: Value to write.
+ //
+ // Throws Error on failure.
+ template<typename T> void write(T value);
+
+ // Close for writing.
+ //
+ // This method potentially verifies the end state after writing the cache
+ // entry and throws Error if any integrity issues are found.
+ void finalize();
+
+private:
+ std::unique_ptr<Compressor> m_compressor;
+ Checksum& m_checksum;
+};
+
+template<typename T>
+inline void
+CacheEntryWriter::write(T value)
+{
+ uint8_t buffer[sizeof(T)];
+ Util::int_to_big_endian(value, buffer);
+ write(buffer, sizeof(T));
+}
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "compression.hpp"
+#include "Compression.hpp"
#include "Config.hpp"
+#include "Error.hpp"
+
+namespace Compression {
int8_t
-compression_level_from_config(void)
+level_from_config()
{
return g_config.compression() ? g_config.compression_level() : 0;
}
-enum compression_type
-compression_type_from_config(void)
+Type
+type_from_config()
{
- return g_config.compression() ? COMPR_TYPE_ZSTD : COMPR_TYPE_NONE;
+ return g_config.compression() ? Type::zstd : Type::none;
}
-const char*
-compression_type_to_string(uint8_t type)
+Type
+type_from_int(uint8_t type)
{
switch (type) {
- case COMPR_TYPE_NONE:
- return "none";
+ case static_cast<uint8_t>(Type::none):
+ return Type::none;
- case COMPR_TYPE_ZSTD:
- return "zstd";
+ case static_cast<uint8_t>(Type::zstd):
+ return Type::zstd;
}
- return "unknown";
+ throw Error(fmt::format("Unknown type: {}", type));
}
-struct compressor*
-compressor_from_type(uint8_t type)
+std::string
+type_to_string(Type type)
{
switch (type) {
- case COMPR_TYPE_NONE:
- return &compressor_none_impl;
+ case Type::none:
+ return "none";
- case COMPR_TYPE_ZSTD:
- return &compressor_zstd_impl;
+ case Type::zstd:
+ return "zstd";
}
- return NULL;
+ assert(false);
+ return {};
}
-struct decompressor*
-decompressor_from_type(uint8_t type)
-{
- switch (type) {
- case COMPR_TYPE_NONE:
- return &decompressor_none_impl;
-
- case COMPR_TYPE_ZSTD:
- return &decompressor_zstd_impl;
- }
-
- return NULL;
-}
+} // namespace Compression
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "system.hpp"
+
+#include <string>
+
+namespace Compression {
+
+enum class Type : uint8_t {
+ none = 0,
+ zstd = 1,
+};
+
+int8_t level_from_config();
+
+Type type_from_config();
+
+Type type_from_int(uint8_t type);
+
+std::string type_to_string(Compression::Type type);
+
+} // namespace Compression
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "Compressor.hpp"
+
+#include "NullCompressor.hpp"
+#include "StdMakeUnique.hpp"
+#include "ZstdCompressor.hpp"
+
+std::unique_ptr<Compressor>
+Compressor::create_from_type(Compression::Type type,
+ FILE* stream,
+ int8_t compression_level)
+{
+ switch (type) {
+ case Compression::Type::none:
+ return std::make_unique<NullCompressor>(stream);
+
+ case Compression::Type::zstd:
+ return std::make_unique<ZstdCompressor>(stream, compression_level);
+ }
+
+ assert(false);
+ return {};
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Compression.hpp"
+
+#include <memory>
+
+class Compressor
+{
+public:
+ virtual ~Compressor() = default;
+
+ // Create a compressor for the specified type.
+ //
+ // Parameters:
+ // - type: The type.
+ // - stream: The stream to write to.
+ // - compression_level: Desired compression level.
+ static std::unique_ptr<Compressor> create_from_type(Compression::Type type,
+ FILE* stream,
+ int8_t compression_level);
+
+ // Get the actual compression level used for the compressed stream.
+ virtual int8_t actual_compression_level() const = 0;
+
+ // Write data from a buffer to the compressed stream.
+ //
+ // Parameters:
+ // - data: Data to write.
+ // - count: Size of data to write.
+ //
+ // Throws Error on failure.
+ virtual void write(const void* data, size_t count) = 0;
+
+ // Write an unsigned integer to the compressed stream.
+ //
+ // Parameters:
+ // - value: Value to write.
+ //
+ // Throws Error on failure.
+ template<typename T> void write(T value);
+
+ // Finalize compression.
+ //
+ // This method checks that the end state of the compressed stream is correct
+ // and throws Error if not.
+ virtual void finalize() = 0;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "Decompressor.hpp"
+
+#include "NullDecompressor.hpp"
+#include "StdMakeUnique.hpp"
+#include "ZstdDecompressor.hpp"
+
+std::unique_ptr<Decompressor>
+Decompressor::create_from_type(Compression::Type type, FILE* stream)
+{
+ switch (type) {
+ case Compression::Type::none:
+ return std::make_unique<NullDecompressor>(stream);
+
+ case Compression::Type::zstd:
+ return std::make_unique<ZstdDecompressor>(stream);
+ }
+
+ assert(false);
+ return {};
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Compression.hpp"
+
+#include <cstdio>
+#include <memory>
+
+class Decompressor
+{
+public:
+ virtual ~Decompressor() = default;
+
+ // Create a decompressor for the specified type.
+ //
+ // Parameters:
+ // - type: The type.
+ // - stream: The stream to read from.
+ static std::unique_ptr<Decompressor> create_from_type(Compression::Type type,
+ FILE* stream);
+
+ // Read data into a buffer from the compressed stream.
+ //
+ // Parameters:
+ // - data: Buffer to write decompressed data to.
+ // - count: How many bytes to write.
+ //
+ // Throws Error on failure.
+ virtual void read(void* data, size_t count) = 0;
+
+ // Finalize decompression.
+ //
+ // This method checks that the end state of the compressed stream is correct
+ // and throws Error if not.
+ virtual void finalize() = 0;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include <cstdio>
+#include <string>
+
+class File
+{
+public:
+ File(const std::string& path, const char* mode)
+ {
+ open(path, mode);
+ }
+
+ void
+ open(const std::string& path, const char* mode)
+ {
+ close();
+ m_file = fopen(path.c_str(), mode);
+ }
+
+ void
+ close()
+ {
+ if (m_file) {
+ fclose(m_file);
+ m_file = nullptr;
+ }
+ }
+
+ ~File()
+ {
+ close();
+ }
+
+ operator bool() const
+ {
+ return m_file;
+ }
+
+ FILE*
+ get()
+ {
+ return m_file;
+ }
+
+private:
+ FILE* m_file = nullptr;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+class NonCopyable
+{
+protected:
+ NonCopyable() = default;
+
+private:
+ NonCopyable(const NonCopyable&) = delete;
+ NonCopyable& operator=(const NonCopyable&) = delete;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "NullCompressor.hpp"
+
+#include "Error.hpp"
+
+NullCompressor::NullCompressor(FILE* stream) : m_stream(stream)
+{
+}
+
+int8_t
+NullCompressor::actual_compression_level() const
+{
+ return 0;
+}
+
+void
+NullCompressor::write(const void* data, size_t count)
+{
+ if (fwrite(data, 1, count, m_stream) != count) {
+ throw Error("failed to write to uncompressed stream");
+ }
+}
+
+void
+NullCompressor::finalize()
+{
+ if (fflush(m_stream) != 0) {
+ throw Error("failed to finalize uncompressed stream");
+ }
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Compressor.hpp"
+#include "NonCopyable.hpp"
+
+// A compressor of an uncompressed stream.
+class NullCompressor : public Compressor, NonCopyable
+{
+public:
+ // Parameters:
+ // - stream: The file to write data to.
+ explicit NullCompressor(FILE* stream);
+
+ int8_t actual_compression_level() const override;
+ void write(const void* data, size_t count) override;
+ void finalize() override;
+
+private:
+ FILE* m_stream;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "NullDecompressor.hpp"
+
+#include "Error.hpp"
+
+NullDecompressor::NullDecompressor(FILE* stream) : m_stream(stream)
+{
+}
+
+void
+NullDecompressor::read(void* data, size_t count)
+{
+ if (fread(data, count, 1, m_stream) != 1) {
+ throw Error("failed to read from uncompressed stream");
+ }
+}
+
+void
+NullDecompressor::finalize()
+{
+ if (fgetc(m_stream) != EOF) {
+ throw Error("garbage data at end of uncompressed stream");
+ }
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Decompressor.hpp"
+#include "NonCopyable.hpp"
+
+// A decompressor of an uncompressed stream.
+class NullDecompressor : public Decompressor, NonCopyable
+{
+public:
+ // Parameters:
+ // - stream: The file to read data from.
+ explicit NullDecompressor(FILE* stream);
+
+ void read(void* data, size_t count) override;
+ void finalize() override;
+
+private:
+ FILE* m_stream;
+};
#include "Util.hpp"
#include "ccache.hpp"
-#include "util.hpp"
#include <algorithm>
#include <fstream>
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ZstdCompressor.hpp"
+
+#include "Error.hpp"
+#include "ccache.hpp"
+
+const uint8_t k_default_zstd_compression_level = -1;
+
+ZstdCompressor::ZstdCompressor(FILE* stream, int8_t compression_level)
+ : m_stream(stream), m_zstd_stream(ZSTD_createCStream())
+{
+ if (compression_level == 0) {
+ compression_level = k_default_zstd_compression_level;
+ cc_log("Using default compression level %d", compression_level);
+ }
+
+ // libzstd 1.3.4 and newer support negative levels. However, the query
+ // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection
+ // based on version instead.
+ if (ZSTD_versionNumber() < 10304 && compression_level < 1) {
+ cc_log(
+ "Using compression level 1 (minimum level supported by libzstd) instead"
+ " of %d",
+ compression_level);
+ compression_level = 1;
+ }
+
+ m_compression_level = std::min<int>(compression_level, ZSTD_maxCLevel());
+ if (m_compression_level != compression_level) {
+ cc_log("Using compression level %d (max libzstd level) instead of %d",
+ m_compression_level,
+ compression_level);
+ }
+
+ size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level);
+ if (ZSTD_isError(ret)) {
+ ZSTD_freeCStream(m_zstd_stream);
+ throw Error("error initializing zstd compression stream");
+ }
+}
+
+ZstdCompressor::~ZstdCompressor()
+{
+ ZSTD_freeCStream(m_zstd_stream);
+}
+
+int8_t
+ZstdCompressor::actual_compression_level() const
+{
+ return m_compression_level;
+}
+
+void
+ZstdCompressor::write(const void* data, size_t count)
+{
+ m_zstd_in.src = data;
+ m_zstd_in.size = count;
+ m_zstd_in.pos = 0;
+
+ int flush = data ? 0 : 1;
+
+ size_t ret;
+ while (m_zstd_in.pos < m_zstd_in.size) {
+ uint8_t buffer[READ_BUFFER_SIZE];
+ m_zstd_out.dst = buffer;
+ m_zstd_out.size = sizeof(buffer);
+ m_zstd_out.pos = 0;
+ ret = ZSTD_compressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in);
+ assert(!(ZSTD_isError(ret)));
+ size_t compressed_bytes = m_zstd_out.pos;
+ if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes
+ || ferror(m_stream)) {
+ throw Error("failed to write to zstd output stream ");
+ }
+ }
+ ret = flush;
+ while (ret > 0) {
+ uint8_t buffer[READ_BUFFER_SIZE];
+ m_zstd_out.dst = buffer;
+ m_zstd_out.size = sizeof(buffer);
+ m_zstd_out.pos = 0;
+ ret = ZSTD_endStream(m_zstd_stream, &m_zstd_out);
+ size_t compressed_bytes = m_zstd_out.pos;
+ if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes
+ || ferror(m_stream)) {
+ throw Error("failed to write to zstd output stream");
+ }
+ }
+}
+
+void
+ZstdCompressor::finalize()
+{
+ write(nullptr, 0);
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Compressor.hpp"
+#include "NonCopyable.hpp"
+
+#include <zstd.h>
+
+// A compressor of a Zstandard stream.
+class ZstdCompressor : public Compressor, NonCopyable
+{
+public:
+ // Parameters:
+ // - stream: The file to write data to.
+ // - compression_level: Desired compression level.
+ ZstdCompressor(FILE* stream, int8_t compression_level);
+
+ ~ZstdCompressor() override;
+
+ int8_t actual_compression_level() const override;
+ void write(const void* data, size_t count) override;
+ void finalize() override;
+
+private:
+ FILE* m_stream;
+ ZSTD_CStream* m_zstd_stream;
+ ZSTD_inBuffer m_zstd_in;
+ ZSTD_outBuffer m_zstd_out;
+ int8_t m_compression_level;
+};
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ZstdDecompressor.hpp"
+
+#include "Error.hpp"
+
+ZstdDecompressor::ZstdDecompressor(FILE* stream)
+ : m_stream(stream),
+ m_input_size(0),
+ m_input_consumed(0),
+ m_zstd_stream(ZSTD_createDStream()),
+ m_reached_stream_end(false)
+{
+ size_t ret = ZSTD_initDStream(m_zstd_stream);
+ if (ZSTD_isError(ret)) {
+ ZSTD_freeDStream(m_zstd_stream);
+ throw Error("failed to initialize zstd decompression stream");
+ }
+}
+
+ZstdDecompressor::~ZstdDecompressor()
+{
+ ZSTD_freeDStream(m_zstd_stream);
+}
+
+void
+ZstdDecompressor::read(void* data, size_t count)
+{
+ size_t bytes_read = 0;
+ while (bytes_read < count) {
+ assert(m_input_size >= m_input_consumed);
+ if (m_input_size == m_input_consumed) {
+ m_input_size = fread(m_input_buffer, 1, sizeof(m_input_buffer), m_stream);
+ if (m_input_size == 0) {
+ throw Error("failed to read from zstd input stream");
+ }
+ m_input_consumed = 0;
+ }
+
+ m_zstd_in.src = (m_input_buffer + m_input_consumed);
+ m_zstd_in.size = m_input_size - m_input_consumed;
+ m_zstd_in.pos = 0;
+
+ m_zstd_out.dst = static_cast<uint8_t*>(data) + bytes_read;
+ m_zstd_out.size = count - bytes_read;
+ m_zstd_out.pos = 0;
+ size_t ret = ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in);
+ if (ZSTD_isError(ret)) {
+ throw Error("failed to read from zstd input stream");
+ }
+ if (ret == 0) {
+ m_reached_stream_end = true;
+ break;
+ }
+ bytes_read += m_zstd_out.pos;
+ m_input_consumed += m_zstd_in.pos;
+ }
+}
+
+void
+ZstdDecompressor::finalize()
+{
+ if (!m_reached_stream_end) {
+ throw Error("garbage data at end of zstd input stream");
+ }
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "Decompressor.hpp"
+#include "ccache.hpp"
+
+#include <fstream>
+#include <zstd.h>
+
+// A decompressor of a Zstandard stream.
+class ZstdDecompressor : public Decompressor
+{
+public:
+ // Parameters:
+ // - stream: The file to read data from.
+ explicit ZstdDecompressor(FILE* stream);
+
+ ~ZstdDecompressor() override;
+
+ void read(void* data, size_t count) override;
+ void finalize() override;
+
+private:
+ FILE* m_stream;
+ char m_input_buffer[READ_BUFFER_SIZE];
+ size_t m_input_size;
+ size_t m_input_consumed;
+ ZSTD_DStream* m_zstd_stream;
+ ZSTD_inBuffer m_zstd_in;
+ ZSTD_outBuffer m_zstd_out;
+ bool m_reached_stream_end;
+};
MTR_BEGIN("manifest", "manifest_put");
cc_log("Adding result name to %s", manifest_path);
- if (manifest_put(manifest_path, cached_result_name, g_included_files)) {
- if (x_stat(manifest_path, &st) == 0) {
- stats_update_size(
- manifest_stats_file, file_size(&st) - old_size, old_size == 0 ? 1 : 0);
- }
- } else {
+ if (!manifest_put(manifest_path, *cached_result_name, g_included_files)) {
cc_log("Failed to add result name to %s", manifest_path);
+ } else if (x_stat(manifest_path, &st) == 0) {
+ stats_update_size(
+ manifest_stats_file, file_size(&st) - old_size, old_size == 0 ? 1 : 0);
}
MTR_END("manifest", "manifest_put");
}
stats_update(STATS_ERROR);
failed();
}
- struct result_files* result_files = result_files_init();
+ ResultFileMap result_file_map;
if (st.st_size > 0) {
- result_files_add(result_files, tmp_stderr, RESULT_STDERR_NAME);
+ result_file_map.emplace(k_result_stderr_name, tmp_stderr);
}
- result_files_add(result_files, output_obj, ".o");
+ result_file_map.emplace(".o", output_obj);
if (generating_dependencies) {
- result_files_add(result_files, output_dep, ".d");
+ result_file_map.emplace(".d", output_dep);
}
if (generating_coverage) {
- result_files_add(result_files, output_cov, ".gcno");
+ result_file_map.emplace(".gcno", output_cov);
}
if (generating_stackusage) {
- result_files_add(result_files, output_su, ".su");
+ result_file_map.emplace(".su", output_su);
}
if (generating_diagnostics) {
- result_files_add(result_files, output_dia, ".dia");
+ result_file_map.emplace(".dia", output_dia);
}
if (seen_split_dwarf && stat(output_dwo, &st) == 0) {
// Only copy .dwo file if it was created by the compiler (GCC and Clang
// behave differently e.g. for "-gsplit-dwarf -g1").
- result_files_add(result_files, output_dwo, ".dwo");
+ result_file_map.emplace(".dwo", output_dwo);
}
struct stat orig_dest_st;
bool orig_dest_existed = stat(cached_result_path, &orig_dest_st) == 0;
- result_put(cached_result_path, result_files);
- result_files_free(result_files);
+ result_put(cached_result_path, result_file_map);
cc_log("Stored in cache: %s", cached_result_path);
bool found_ccbin = false;
hash_delimiter(hash, "result version");
- hash_int(hash, RESULT_VERSION);
+ hash_int(hash, k_result_version);
if (direct_mode) {
hash_delimiter(hash, "manifest version");
- hash_int(hash, MANIFEST_VERSION);
+ hash_int(hash, k_manifest_version);
}
// clang will emit warnings for unused linker flags, so we shouldn't skip
int tmp_stderr_fd = create_tmp_fd(&tmp_stderr);
close(tmp_stderr_fd);
- struct result_files* result_files = result_files_init();
+ ResultFileMap result_file_map;
if (!str_eq(output_obj, "/dev/null")) {
- result_files_add(result_files, output_obj, ".o");
+ result_file_map.emplace(".o", output_obj);
if (seen_split_dwarf) {
- result_files_add(result_files, output_dwo, ".dwo");
+ result_file_map.emplace(".dwo", output_dwo);
}
}
- result_files_add(result_files, tmp_stderr, RESULT_STDERR_NAME);
+ result_file_map.emplace(k_result_stderr_name, tmp_stderr);
if (produce_dep_file) {
- result_files_add(result_files, output_dep, ".d");
+ result_file_map.emplace(".d", output_dep);
}
if (generating_coverage) {
- result_files_add(result_files, output_cov, ".gcno");
+ result_file_map.emplace(".gcno", output_cov);
}
if (generating_stackusage) {
- result_files_add(result_files, output_su, ".su");
+ result_file_map.emplace(".su", output_su);
}
if (generating_diagnostics) {
- result_files_add(result_files, output_dia, ".dia");
+ result_file_map.emplace(".dia", output_dia);
}
- bool ok = result_get(cached_result_path, result_files);
- result_files_free(result_files);
+ bool ok = result_get(cached_result_path, result_file_map);
if (!ok) {
cc_log("Failed to get result from cache");
tmp_unlink(tmp_stderr);
switch (c) {
case DUMP_MANIFEST:
initialize();
- manifest_dump(optarg, stdout);
- break;
+ return manifest_dump(optarg, stdout) ? 0 : 1;
case DUMP_RESULT:
initialize();
- if (!result_dump(optarg, stdout)) {
- return 1;
- }
- break;
+ return result_dump(optarg, stdout) ? 0 : 1;
case HASH_FILE: {
initialize();
#include "system.hpp"
#include "counters.hpp"
-#include "util.hpp"
#include "third_party/minitrace.h"
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "common_header.hpp"
-
-#include "ccache.hpp"
-#include "int_bytes_conversion.hpp"
-
-bool
-common_header_initialize_for_writing(struct common_header* header,
- FILE* output,
- const char magic[4],
- uint8_t version,
- uint8_t compression_type,
- int8_t compression_level,
- uint64_t content_size,
- Checksum& checksum,
- struct compressor** compressor,
- struct compr_state** compr_state)
-{
- memcpy(header->magic, magic, 4);
- header->version = version;
- header->compression_type = compression_type;
- header->compression_level = compression_level;
- header->content_size = content_size;
-
- *compressor = compressor_from_type(header->compression_type);
- assert(*compressor);
- *compr_state =
- (*compressor)->init(output, header->compression_level, &checksum);
- if (!*compr_state) {
- cc_log("Failed to initialize compressor");
- return false;
- }
- header->compression_level =
- (*compressor)->get_actual_compression_level(*compr_state);
-
- uint8_t header_bytes[COMMON_HEADER_SIZE];
- memcpy(header_bytes, header->magic, 4);
- header_bytes[4] = header->version;
- header_bytes[5] = header->compression_type;
- header_bytes[6] = header->compression_level;
- BYTES_FROM_UINT64(header_bytes + 7, header->content_size);
- if (fwrite(header_bytes, sizeof(header_bytes), 1, output) != 1) {
- cc_log("Failed to write common file header");
- return false;
- }
- checksum.update(header_bytes, sizeof(header_bytes));
- return true;
-}
-
-bool
-common_header_initialize_for_reading(struct common_header* header,
- FILE* input,
- const char expected_magic[4],
- uint8_t expected_version,
- struct decompressor** decompressor,
- struct decompr_state** decompr_state,
- Checksum* checksum,
- char** errmsg)
-{
- uint8_t header_bytes[COMMON_HEADER_SIZE];
- if (fread(header_bytes, sizeof(header_bytes), 1, input) != 1) {
- *errmsg = format("Failed to read common header");
- return false;
- }
-
- memcpy(header->magic, header_bytes, 4);
- header->version = header_bytes[4];
- header->compression_type = header_bytes[5];
- header->compression_level = header_bytes[6];
- header->content_size = UINT64_FROM_BYTES(header_bytes + 7);
-
- if (memcmp(header->magic, expected_magic, sizeof(header->magic)) != 0) {
- *errmsg = format("Bad magic value 0x%x%x%x%x",
- header->magic[0],
- header->magic[1],
- header->magic[2],
- header->magic[3]);
- return false;
- }
-
- if (header->version != expected_version) {
- *errmsg = format("Unknown version (actual %u, expected %u)",
- header->version,
- expected_version);
- return false;
- }
-
- if (header->compression_type == COMPR_TYPE_NONE) {
- // Since we have the size available, let's use it as a super primitive
- // consistency check for the non-compressed case. (A real checksum is used
- // for compressed data.)
- struct stat st;
- if (x_fstat(fileno(input), &st) != 0) {
- return false;
- }
- if ((uint64_t)st.st_size != header->content_size) {
- *errmsg = format(
- "Bad uncompressed file size (actual %llu bytes, expected %llu bytes)",
- (unsigned long long)st.st_size,
- (unsigned long long)header->content_size);
- return false;
- }
- }
-
- if (!decompressor) {
- return true;
- }
-
- *decompressor = decompressor_from_type(header->compression_type);
- if (!*decompressor) {
- *errmsg = format("Unknown compression type: %u", header->compression_type);
- return false;
- }
-
- if (checksum) {
- checksum->update(header_bytes, sizeof(header_bytes));
- }
-
- *decompr_state = (*decompressor)->init(input, checksum);
- if (!*decompr_state) {
- *errmsg = x_strdup("Failed to initialize decompressor");
- return false;
- }
-
- return true;
-}
-
-void
-common_header_dump(const struct common_header* header, FILE* f)
-{
- fprintf(f,
- "Magic: %c%c%c%c\n",
- header->magic[0],
- header->magic[1],
- header->magic[2],
- header->magic[3]);
- fprintf(f, "Version: %u\n", header->version);
- fprintf(f,
- "Compression type: %s\n",
- compression_type_to_string(header->compression_type));
- fprintf(f, "Compression level: %d\n", header->compression_level);
- fprintf(f, "Content size: %" PRIu64 "\n", header->content_size);
-}
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "system.hpp"
-
-#include "Checksum.hpp"
-#include "compression.hpp"
-
-#define COMMON_HEADER_SIZE 15
-
-struct common_header
-{
- char magic[4];
- uint8_t version;
- uint8_t compression_type;
- int8_t compression_level;
- uint64_t content_size;
-};
-
-// Initialize a common_header and write the header data to a file.
-//
-// header: Header to initialize.
-// output: Open file to write to.
-// magic: File format magic bytes.
-// version: File format version.
-// compression_type: Compression type to use.
-// compression_level: Compression level to use.
-// content_size: Content size.
-// compressor: Compressor created from compression_type.
-// compressor_state: State for the compressor.
-// checksum: Checksum state that will be updated with the written
-// bytes.
-bool common_header_initialize_for_writing(struct common_header* header,
- FILE* output,
- const char magic[4],
- uint8_t version,
- uint8_t compression_type,
- int8_t compression_level,
- uint64_t content_size,
- Checksum& checksum,
- struct compressor** compressor,
- struct compr_state** compr_state);
-
-// Initialize a common_header by reading header data from a file.
-//
-// header: Header to initialize.
-// input: Open file to read from.
-// expected_magic: Expected file format magic bytes.
-// expected_version: Expected file format version.
-// decompressor: Decompressor created from the compression type field in
-// the header. Pass NULL to not create a decompressor.
-// decompressor_state: State for the decompressor. Should be NULL if
-// decompressor is NULL.
-// checksum: Checksum state that will be updated with the read bytes.
-// May be NULL for no checksumming.
-bool common_header_initialize_for_reading(struct common_header* header,
- FILE* input,
- const char expected_magic[4],
- uint8_t expected_version,
- struct decompressor** decompressor,
- struct decompr_state** decompr_state,
- Checksum* checksum,
- char** errmsg);
-
-void common_header_dump(const struct common_header* header, FILE* f);
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "compression.hpp"
-
-struct state
-{
- FILE* output;
- Checksum* checksum;
-};
-
-static struct compr_state*
-compr_none_init(FILE* output, int8_t level, Checksum* checksum)
-{
- auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
- state->output = output;
- state->checksum = checksum;
- (void)level;
- return (struct compr_state*)state;
-}
-
-static int8_t
-compr_none_get_actual_compression_level(struct compr_state* handle)
-{
- (void)handle;
- return 0;
-}
-
-static bool
-compr_none_write(struct compr_state* handle, const void* data, size_t size)
-{
- struct state* state = (struct state*)handle;
- size_t ret = fwrite(data, size, 1, state->output);
- if (state->checksum) {
- state->checksum->update(data, size);
- }
- return ret == 1;
-}
-
-static bool
-compr_none_free(struct compr_state* handle)
-{
- struct state* state = (struct state*)handle;
- bool result = ferror(state->output) == 0;
- free(state);
- return result;
-}
-
-struct compressor compressor_none_impl = {
- compr_none_init,
- compr_none_get_actual_compression_level,
- compr_none_write,
- compr_none_free};
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ccache.hpp"
-#include "compression.hpp"
-
-#include <algorithm>
-#include <zstd.h>
-
-#define DEFAULT_ZSTD_COMPRESSION_LEVEL -1
-
-struct state
-{
- FILE* output;
- Checksum* checksum;
- ZSTD_CStream* stream;
- ZSTD_inBuffer in;
- ZSTD_outBuffer out;
- bool failed;
- int8_t compression_level;
-};
-
-static struct compr_state*
-compr_zstd_init(FILE* output, int8_t level, Checksum* checksum)
-{
- auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
- state->output = output;
- state->checksum = checksum;
- state->stream = ZSTD_createCStream();
- state->failed = false;
-
- if (level == 0) {
- level = DEFAULT_ZSTD_COMPRESSION_LEVEL;
- cc_log("Using default compression level %d", level);
- }
-
- // libzstd 1.3.4 and newer support negative levels. However, the query
- // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection
- // based on version instead.
- if (ZSTD_versionNumber() < 10304 && level < 1) {
- cc_log(
- "Using compression level 1 (minimum level supported by libzstd) instead"
- " of %d",
- level);
- level = 1;
- }
-
- state->compression_level = std::min<int>(level, ZSTD_maxCLevel());
- if (state->compression_level != level) {
- cc_log("Using compression level %d (max libzstd level) instead of %d",
- state->compression_level,
- level);
- }
-
- size_t ret = ZSTD_initCStream(state->stream, state->compression_level);
- if (ZSTD_isError(ret)) {
- ZSTD_freeCStream(state->stream);
- free(state);
- return NULL;
- }
- return (struct compr_state*)state;
-}
-
-static int8_t
-compr_zstd_get_actual_compression_level(struct compr_state* handle)
-{
- struct state* state = (struct state*)handle;
- return state->compression_level;
-}
-
-static bool
-compr_zstd_write(struct compr_state* handle, const void* data, size_t size)
-{
- if (!handle) {
- return false;
- }
- struct state* state = (struct state*)handle;
-
- if (state->checksum) {
- state->checksum->update(data, size);
- }
-
- state->in.src = data;
- state->in.size = size;
- state->in.pos = 0;
-
- int flush = data ? 0 : 1;
-
- size_t ret;
- while (state->in.pos < state->in.size) {
- unsigned char buffer[READ_BUFFER_SIZE];
- state->out.dst = buffer;
- state->out.size = sizeof(buffer);
- state->out.pos = 0;
- ret = ZSTD_compressStream(state->stream, &state->out, &state->in);
- assert(!(ZSTD_isError(ret)));
- size_t compressed_bytes = state->out.pos;
- if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes
- || ferror(state->output)) {
- state->failed = true;
- return false;
- }
- }
- ret = flush;
- while (ret > 0) {
- unsigned char buffer[READ_BUFFER_SIZE];
- state->out.dst = buffer;
- state->out.size = sizeof(buffer);
- state->out.pos = 0;
- ret = ZSTD_endStream(state->stream, &state->out);
- size_t compressed_bytes = state->out.pos;
- if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes
- || ferror(state->output)) {
- state->failed = true;
- return false;
- }
- }
-
- return true;
-}
-
-static bool
-compr_zstd_free(struct compr_state* handle)
-{
- if (!handle) {
- return false;
- }
-
- struct state* state = (struct state*)handle;
-
- compr_zstd_write(handle, NULL, 0);
- ZSTD_freeCStream(state->stream);
- bool success = !state->failed;
- free(state);
- return success;
-}
-
-struct compressor compressor_zstd_impl = {
- compr_zstd_init,
- compr_zstd_get_actual_compression_level,
- compr_zstd_write,
- compr_zstd_free};
#include "compress.hpp"
+#include "CacheEntryReader.hpp"
+#include "File.hpp"
#include "ccache.hpp"
-#include "common_header.hpp"
#include "manifest.hpp"
#include "result.hpp"
static bool
get_content_size(const std::string& path,
- const char* magic,
+ const uint8_t magic[4],
uint8_t version,
- size_t* size)
+ uint64_t& size)
{
- char* errmsg;
- FILE* f = fopen(path.c_str(), "rb");
+ File f(path, "rb");
if (!f) {
cc_log("Failed to open %s for reading: %s", path.c_str(), strerror(errno));
return false;
}
- struct common_header header;
- bool success = common_header_initialize_for_reading(
- &header, f, magic, version, NULL, NULL, NULL, &errmsg);
- fclose(f);
- if (success) {
- *size = header.content_size;
- }
- return success;
+ try {
+ size = CacheEntryReader(f.get(), magic, version).content_size();
+ return true;
+ } catch (const Error&) {
+ return false;
+ }
}
void
on_disk_size += file_size(&file->stat());
- size_t content_size = 0;
+ uint64_t content_size = 0;
bool is_compressible;
if (file->type() == CacheFile::Type::manifest) {
is_compressible = get_content_size(
- file->path(), MANIFEST_MAGIC, MANIFEST_VERSION, &content_size);
+ file->path(), k_manifest_magic, k_manifest_version, content_size);
} else if (file->type() == CacheFile::Type::result) {
is_compressible = get_content_size(
- file->path(), RESULT_MAGIC, RESULT_VERSION, &content_size);
+ file->path(), k_result_magic, k_result_version, content_size);
} else {
is_compressible = false;
}
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include "system.hpp"
-
-#include "Checksum.hpp"
-
-struct compr_state;
-
-struct compressor
-{
- // Create and initialize a compressor.
- //
- // output: The file to write compressed data to.
- // compression_level: Desired compression level.
- // checksum: Checksum state to update (NULL for no checksum).
- struct compr_state* (*init)(FILE* output,
- int8_t compression_level,
- Checksum* checksum);
-
- // Get the actual compression level that will be used.
- int8_t (*get_actual_compression_level)(struct compr_state* state);
-
- // Compress data.
- //
- // data: Buffer to read decompressed data from.
- // size: How many bytes to read.
- //
- // Returns false on failure, otherwise true.
- bool (*write)(struct compr_state* state, const void* data, size_t size);
-
- // Finalize compressor and free its state.
- //
- // Returns false if finalization failed or if any previous operation failed,
- // otherwise true.
- bool (*free)(struct compr_state* state);
-};
-
-struct decompr_state;
-
-struct decompressor
-{
- // Create and initialize a decompressor.
- //
- // input: The file to read compressed data from.
- // checksum: Checksum state to update (NULL for no checksum).
- struct decompr_state* (*init)(FILE* input, Checksum* checksum);
-
- // Decompress data.
- //
- // data: Buffer to write decompressed data to.
- // size: How many bytes to write.
- //
- // Returns false on failure, otherwise true.
- bool (*read)(struct decompr_state* state, void* data, size_t size);
-
- // Finalize the decompressor and free its state.
- //
- // Returns false if finalization failed or if any previous operation failed,
- // otherwise true.
- bool (*free)(struct decompr_state* state);
-};
-
-enum compression_type { COMPR_TYPE_NONE = 0, COMPR_TYPE_ZSTD = 1 };
-
-extern struct compressor compressor_none_impl;
-extern struct decompressor decompressor_none_impl;
-
-extern struct compressor compressor_zstd_impl;
-extern struct decompressor decompressor_zstd_impl;
-
-int8_t compression_level_from_config(void);
-enum compression_type compression_type_from_config(void);
-const char* compression_type_to_string(uint8_t type);
-struct compressor* compressor_from_type(uint8_t type);
-struct decompressor* decompressor_from_type(uint8_t type);
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "compression.hpp"
-
-struct state
-{
- FILE* input;
- Checksum* checksum;
- bool failed;
-};
-
-static struct decompr_state*
-decompr_none_init(FILE* input, Checksum* checksum)
-{
- auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
- state->input = input;
- state->checksum = checksum;
- state->failed = false;
- return (struct decompr_state*)state;
-}
-
-static bool
-decompr_none_read(struct decompr_state* handle, void* data, size_t size)
-{
- struct state* state = (struct state*)handle;
-
- bool result = fread(data, 1, size, state->input) == size;
- if (result && state->checksum) {
- state->checksum->update(data, size);
- }
- if (!result) {
- state->failed = true;
- }
- return result;
-}
-
-static bool
-decompr_none_free(struct decompr_state* handle)
-{
- struct state* state = (struct state*)handle;
- bool result = !state->failed;
- free(state);
- return result;
-}
-
-struct decompressor decompressor_none_impl = {
- decompr_none_init, decompr_none_read, decompr_none_free};
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ccache.hpp"
-#include "compression.hpp"
-
-#include <zstd.h>
-
-enum stream_state {
- STREAM_STATE_READING,
- STREAM_STATE_FAILED,
- STREAM_STATE_END
-};
-
-struct state
-{
- FILE* input;
- Checksum* checksum;
- char input_buffer[READ_BUFFER_SIZE];
- size_t input_size;
- size_t input_consumed;
- ZSTD_DStream* stream;
- ZSTD_inBuffer in;
- ZSTD_outBuffer out;
- enum stream_state stream_state;
-};
-
-static struct decompr_state*
-decompr_zstd_init(FILE* input, Checksum* checksum)
-{
- auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
-
- state->input = input;
- state->checksum = checksum;
- state->input_size = 0;
- state->input_consumed = 0;
- state->stream = ZSTD_createDStream();
- state->stream_state = STREAM_STATE_READING;
-
- size_t ret = ZSTD_initDStream(state->stream);
- if (ZSTD_isError(ret)) {
- ZSTD_freeDStream(state->stream);
- free(state);
- return NULL;
- }
- return (struct decompr_state*)state;
-}
-
-static bool
-decompr_zstd_read(struct decompr_state* handle, void* data, size_t size)
-{
- if (!handle) {
- return false;
- }
- struct state* state = (struct state*)handle;
-
- size_t bytes_read = 0;
- while (bytes_read < size) {
- assert(state->input_size >= state->input_consumed);
- if (state->input_size == state->input_consumed) {
- state->input_size = fread(
- state->input_buffer, 1, sizeof(state->input_buffer), state->input);
- if (state->input_size == 0) {
- state->stream_state = STREAM_STATE_FAILED;
- return false;
- }
- state->input_consumed = 0;
- }
-
- state->in.src = (state->input_buffer + state->input_consumed);
- state->in.size = state->input_size - state->input_consumed;
- state->in.pos = 0;
-
- state->out.dst = ((char*)data + bytes_read);
- state->out.size = size - bytes_read;
- state->out.pos = 0;
- size_t ret = ZSTD_decompressStream(state->stream, &state->out, &state->in);
- if (ZSTD_isError(ret)) {
- state->stream_state = STREAM_STATE_FAILED;
- return false;
- }
- if (state->checksum) {
- state->checksum->update(state->out.dst, state->out.pos);
- }
- if (ret == 0) {
- state->stream_state = STREAM_STATE_END;
- break;
- }
- bytes_read += state->out.pos;
- state->input_consumed += state->in.pos;
- }
-
- return true;
-}
-
-static bool
-decompr_zstd_free(struct decompr_state* handle)
-{
- if (!handle) {
- return false;
- }
- struct state* state = (struct state*)handle;
- ZSTD_freeDStream(state->stream);
- bool success = state->stream_state == STREAM_STATE_END;
- free(handle);
- return success;
-}
-
-struct decompressor decompressor_zstd_impl = {
- decompr_zstd_init, decompr_zstd_read, decompr_zstd_free};
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#define BYTES_FROM_UINT16(bytes, uint16) \
- do { \
- (bytes)[0] = (uint16) >> 8 & 0xFF; \
- (bytes)[1] = (uint16) >> 0 & 0xFF; \
- } while (false)
-
-#define UINT16_FROM_BYTES(bytes) \
- ((uint16_t)(uint8_t)(bytes)[0] << 8 | (uint16_t)(uint8_t)(bytes)[1] << 0)
-
-#define BYTES_FROM_UINT32(bytes, uint32) \
- do { \
- (bytes)[0] = (uint32) >> 24 & 0xFF; \
- (bytes)[1] = (uint32) >> 16 & 0xFF; \
- (bytes)[2] = (uint32) >> 8 & 0xFF; \
- (bytes)[3] = (uint32) >> 0 & 0xFF; \
- } while (false)
-
-#define UINT32_FROM_BYTES(bytes) \
- ((uint32_t)(uint8_t)(bytes)[0] << 24 | (uint32_t)(uint8_t)(bytes)[1] << 16 \
- | (uint32_t)(uint8_t)(bytes)[2] << 8 | (uint32_t)(uint8_t)(bytes)[3] << 0)
-
-#define BYTES_FROM_INT64(bytes, int64) \
- do { \
- (bytes)[0] = (int64) >> 56 & 0xFF; \
- (bytes)[1] = (int64) >> 48 & 0xFF; \
- (bytes)[2] = (int64) >> 40 & 0xFF; \
- (bytes)[3] = (int64) >> 32 & 0xFF; \
- (bytes)[4] = (int64) >> 24 & 0xFF; \
- (bytes)[5] = (int64) >> 16 & 0xFF; \
- (bytes)[6] = (int64) >> 8 & 0xFF; \
- (bytes)[7] = (int64) >> 0 & 0xFF; \
- } while (false)
-
-#define INT64_FROM_BYTES(bytes) \
- ((int64_t)(uint8_t)(bytes)[0] << 56 | (int64_t)(uint8_t)(bytes)[1] << 48 \
- | (int64_t)(uint8_t)(bytes)[2] << 40 | (int64_t)(uint8_t)(bytes)[3] << 32 \
- | (int64_t)(uint8_t)(bytes)[4] << 24 | (int64_t)(uint8_t)(bytes)[5] << 16 \
- | (int64_t)(uint8_t)(bytes)[6] << 8 | (int64_t)(uint8_t)(bytes)[7] << 0)
-
-#define BYTES_FROM_UINT64(bytes, uint64) \
- do { \
- (bytes)[0] = (uint64) >> 56 & 0xFF; \
- (bytes)[1] = (uint64) >> 48 & 0xFF; \
- (bytes)[2] = (uint64) >> 40 & 0xFF; \
- (bytes)[3] = (uint64) >> 32 & 0xFF; \
- (bytes)[4] = (uint64) >> 24 & 0xFF; \
- (bytes)[5] = (uint64) >> 16 & 0xFF; \
- (bytes)[6] = (uint64) >> 8 & 0xFF; \
- (bytes)[7] = (uint64) >> 0 & 0xFF; \
- } while (false)
-
-#define UINT64_FROM_BYTES(bytes) \
- ((uint64_t)(uint8_t)(bytes)[0] << 56 | (uint64_t)(uint8_t)(bytes)[1] << 48 \
- | (uint64_t)(uint8_t)(bytes)[2] << 40 | (uint64_t)(uint8_t)(bytes)[3] << 32 \
- | (uint64_t)(uint8_t)(bytes)[4] << 24 | (uint64_t)(uint8_t)(bytes)[5] << 16 \
- | (uint64_t)(uint8_t)(bytes)[6] << 8 | (uint64_t)(uint8_t)(bytes)[7] << 0)
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "util.hpp"
-
#include "Config.hpp"
#include "Error.hpp"
#include "Util.hpp"
#include "manifest.hpp"
+#include "AtomicFile.hpp"
+#include "CacheEntryReader.hpp"
+#include "CacheEntryWriter.hpp"
#include "Checksum.hpp"
+#include "Config.hpp"
+#include "File.hpp"
+#include "StdMakeUnique.hpp"
#include "ccache.hpp"
-#include "common_header.hpp"
-#include "compression.hpp"
+#include "hash.hpp"
#include "hashutil.hpp"
-#include "int_bytes_conversion.hpp"
// Manifest data format
// ====================
// 1: Introduced in ccache 3.0. (Files are always compressed with gzip.)
// 2: Introduced in ccache 4.0.
-const char MANIFEST_MAGIC[4] = {'c', 'C', 'm', 'F'};
-static const uint32_t MAX_MANIFEST_ENTRIES = 100;
-static const uint32_t MAX_MANIFEST_FILE_INFO_ENTRIES = 10000;
+const uint8_t k_manifest_magic[4] = {'c', 'C', 'm', 'F'};
+const uint8_t k_manifest_version = 2;
+const uint32_t k_max_manifest_entries = 100;
+const uint32_t k_max_manifest_file_info_entries = 10000;
namespace {
-struct file
-{
- uint16_t path_len; // strlen(path)
- char* path; // NUL-terminated
-};
-
struct FileInfo
{
// Index to n_files.
} // namespace std
-struct result
+struct ResultEntry
{
- // Number of entries in file_info_indexes.
- uint32_t n_file_info_indexes;
// Indexes to file_infos.
- uint32_t* file_info_indexes;
+ std::vector<uint32_t> file_info_indexes;
+
// Name of the result.
struct digest name;
};
-struct manifest
+struct ManifestData
{
- struct common_header header;
-
// Referenced include files.
- uint32_t n_files;
- struct file* files;
+ std::vector<std::string> files;
// Information about referenced include files.
- uint32_t n_file_infos;
- struct FileInfo* file_infos;
+ std::vector<FileInfo> file_infos;
// Result names plus references to include file infos.
- uint32_t n_results;
- struct result* results;
+ std::vector<ResultEntry> results;
+
+ void
+ add_result_entry(
+ const struct digest& result_digest,
+ const std::unordered_map<std::string, digest>& included_files)
+ {
+ std::unordered_map<std::string, uint32_t /*index*/> mf_files;
+ for (uint32_t i = 0; i < files.size(); ++i) {
+ mf_files.emplace(files[i], i);
+ }
+
+ std::unordered_map<FileInfo, uint32_t /*index*/> mf_file_infos;
+ for (uint32_t i = 0; i < file_infos.size(); ++i) {
+ mf_file_infos.emplace(file_infos[i], i);
+ }
+
+ std::vector<uint32_t> file_info_indexes;
+ for (const auto& item : included_files) {
+ file_info_indexes.push_back(
+ get_file_info_index(item.first, item.second, mf_files, mf_file_infos));
+ }
+
+ results.push_back(ResultEntry{std::move(file_info_indexes), result_digest});
+ }
+
+private:
+ uint32_t
+ get_file_info_index(
+ const std::string& path,
+ const digest& digest,
+ const std::unordered_map<std::string, uint32_t>& mf_files,
+ const std::unordered_map<FileInfo, uint32_t>& mf_file_infos)
+ {
+ struct FileInfo fi;
+
+ auto f_it = mf_files.find(path);
+ if (f_it != mf_files.end()) {
+ fi.index = f_it->second;
+ } else {
+ files.push_back(path);
+ fi.index = files.size() - 1;
+ }
+
+ fi.digest = digest;
+
+ // file_stat.st_{m,c}time have a resolution of 1 second, so we can cache
+ // the file's mtime and ctime only if they're at least one second older
+ // than time_of_compilation.
+ //
+ // st->ctime may be 0, so we have to check time_of_compilation against
+ // MAX(mtime, ctime).
+
+ struct stat file_stat;
+ if (stat(path.c_str(), &file_stat) != -1) {
+ if (time_of_compilation
+ > std::max(file_stat.st_mtime, file_stat.st_ctime)) {
+ fi.mtime = file_stat.st_mtime;
+ fi.ctime = file_stat.st_ctime;
+ } else {
+ fi.mtime = -1;
+ fi.ctime = -1;
+ }
+ fi.fsize = file_stat.st_size;
+ } else {
+ fi.mtime = -1;
+ fi.ctime = -1;
+ fi.fsize = 0;
+ }
+
+ auto fi_it = mf_file_infos.find(fi);
+ if (fi_it != mf_file_infos.end()) {
+ return fi_it->second;
+ } else {
+ file_infos.push_back(fi);
+ return file_infos.size() - 1;
+ }
+ }
};
-struct file_stats
+struct FileStats
{
uint64_t size;
int64_t mtime;
int64_t ctime;
};
-static void
-free_manifest(struct manifest* mf)
+static std::unique_ptr<ManifestData>
+read_manifest(const std::string& path, FILE* dump_stream = nullptr)
{
- for (uint32_t i = 0; i < mf->n_files; i++) {
- free(mf->files[i].path);
- }
- free(mf->files);
- free(mf->file_infos);
- for (uint32_t i = 0; i < mf->n_results; i++) {
- free(mf->results[i].file_info_indexes);
+ File file(path, "rb");
+ if (!file) {
+ return {};
}
- free(mf->results);
- free(mf);
-}
-
-#define READ_BYTES(buf, length) \
- do { \
- if (!decompressor->read(decompr_state, buf, length)) { \
- goto out; \
- } \
- } while (false)
-
-#define READ_UINT16(var) \
- do { \
- char buf_[2]; \
- READ_BYTES(buf_, sizeof(buf_)); \
- (var) = UINT16_FROM_BYTES(buf_); \
- } while (false)
-
-#define READ_UINT32(var) \
- do { \
- char buf_[4]; \
- READ_BYTES(buf_, sizeof(buf_)); \
- (var) = UINT32_FROM_BYTES(buf_); \
- } while (false)
-
-#define READ_INT64(var) \
- do { \
- char buf_[8]; \
- READ_BYTES(buf_, sizeof(buf_)); \
- (var) = INT64_FROM_BYTES(buf_); \
- } while (false)
-
-#define READ_UINT64(var) \
- do { \
- char buf_[8]; \
- READ_BYTES(buf_, sizeof(buf_)); \
- (var) = UINT64_FROM_BYTES(buf_); \
- } while (false)
-
-#define READ_STR(str_var, len_var) \
- do { \
- READ_UINT16(len_var); \
- (str_var) = static_cast<char*>(x_malloc(len_var + 1)); \
- READ_BYTES(str_var, len_var); \
- str_var[len_var] = '\0'; \
- } while (false)
-
-static struct manifest*
-create_empty_manifest(void)
-{
- auto mf = static_cast<manifest*>(x_malloc(sizeof(manifest)));
- mf->n_files = 0;
- mf->files = NULL;
- mf->n_file_infos = 0;
- mf->file_infos = NULL;
- mf->n_results = 0;
- mf->results = NULL;
-
- return mf;
-}
-static struct manifest*
-read_manifest(const char* path, char** errmsg)
-{
- bool success = false;
- struct manifest* mf = create_empty_manifest();
- struct decompressor* decompressor = NULL;
- struct decompr_state* decompr_state = NULL;
- *errmsg = NULL;
Checksum checksum;
- uint64_t actual_checksum;
- uint64_t expected_checksum;
+ CacheEntryReader reader(
+ file.get(), k_manifest_magic, k_manifest_version, &checksum);
- FILE* f = fopen(path, "rb");
- if (!f) {
- *errmsg = x_strdup("No such manifest file");
- goto out;
+ if (dump_stream) {
+ reader.dump_header(dump_stream);
}
- if (!common_header_initialize_for_reading(&mf->header,
- f,
- MANIFEST_MAGIC,
- MANIFEST_VERSION,
- &decompressor,
- &decompr_state,
- &checksum,
- errmsg)) {
- goto out;
- }
+ auto mf = std::make_unique<ManifestData>();
- READ_UINT32(mf->n_files);
- mf->files = static_cast<file*>(x_calloc(mf->n_files, sizeof(file)));
- for (uint32_t i = 0; i < mf->n_files; i++) {
- READ_STR(mf->files[i].path, mf->files[i].path_len);
+ uint32_t entry_count;
+ reader.read(entry_count);
+ for (uint32_t i = 0; i < entry_count; ++i) {
+ mf->files.emplace_back();
+ auto& entry = mf->files.back();
+
+ uint16_t length;
+ reader.read(length);
+ entry.assign(length, 0);
+ reader.read(&entry[0], length);
}
- READ_UINT32(mf->n_file_infos);
- mf->file_infos =
- static_cast<FileInfo*>(x_calloc(mf->n_file_infos, sizeof(FileInfo)));
- for (uint32_t i = 0; i < mf->n_file_infos; i++) {
- READ_UINT32(mf->file_infos[i].index);
- READ_BYTES(mf->file_infos[i].digest.bytes, DIGEST_SIZE);
- READ_UINT64(mf->file_infos[i].fsize);
- READ_INT64(mf->file_infos[i].mtime);
- READ_INT64(mf->file_infos[i].ctime);
+ reader.read(entry_count);
+ for (uint32_t i = 0; i < entry_count; ++i) {
+ mf->file_infos.emplace_back();
+ auto& entry = mf->file_infos.back();
+
+ reader.read(entry.index);
+ reader.read(entry.digest.bytes, DIGEST_SIZE);
+ reader.read(entry.fsize);
+ reader.read(entry.mtime);
+ reader.read(entry.ctime);
}
- READ_UINT32(mf->n_results);
- mf->results = static_cast<result*>(x_calloc(mf->n_results, sizeof(result)));
- for (uint32_t i = 0; i < mf->n_results; i++) {
- READ_UINT32(mf->results[i].n_file_info_indexes);
- mf->results[i].file_info_indexes = static_cast<uint32_t*>(
- x_calloc(mf->results[i].n_file_info_indexes, sizeof(uint32_t)));
- for (uint32_t j = 0; j < mf->results[i].n_file_info_indexes; j++) {
- READ_UINT32(mf->results[i].file_info_indexes[j]);
+ reader.read(entry_count);
+ for (uint32_t i = 0; i < entry_count; ++i) {
+ mf->results.emplace_back();
+ auto& entry = mf->results.back();
+
+ uint32_t file_info_count;
+ reader.read(file_info_count);
+ for (uint32_t j = 0; j < file_info_count; ++j) {
+ uint32_t file_info_index;
+ reader.read(file_info_index);
+ entry.file_info_indexes.push_back(file_info_index);
}
- READ_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
+ reader.read(entry.name.bytes, DIGEST_SIZE);
}
- actual_checksum = checksum.digest();
- READ_UINT64(expected_checksum);
- if (actual_checksum == expected_checksum) {
- success = true;
- } else {
- *errmsg = format("Incorrect checksum (actual %016llx, expected %016llx)",
- (unsigned long long)actual_checksum,
- (unsigned long long)expected_checksum);
+ uint64_t actual_checksum = checksum.digest();
+ uint64_t expected_checksum;
+ reader.read(expected_checksum);
+ if (actual_checksum != expected_checksum) {
+ throw Error(
+ fmt::format("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})",
+ actual_checksum,
+ expected_checksum));
}
-out:
- if (decompressor && !decompressor->free(decompr_state)) {
- success = false;
- }
- if (f) {
- fclose(f);
- }
- if (!success) {
- if (!*errmsg) {
- *errmsg = x_strdup("Corrupt manifest file");
- }
- free_manifest(mf);
- mf = NULL;
- }
+ reader.finalize();
return mf;
}
-#define WRITE_BYTES(buf, length) \
- do { \
- if (!compressor->write(compr_state, buf, length)) { \
- goto out; \
- } \
- } while (false)
-
-#define WRITE_UINT16(var) \
- do { \
- char buf_[2]; \
- BYTES_FROM_UINT16(buf_, (var)); \
- WRITE_BYTES(buf_, sizeof(buf_)); \
- } while (false)
-
-#define WRITE_UINT32(var) \
- do { \
- char buf_[4]; \
- BYTES_FROM_UINT32(buf_, (var)); \
- WRITE_BYTES(buf_, sizeof(buf_)); \
- } while (false)
-
-#define WRITE_INT64(var) \
- do { \
- char buf_[8]; \
- BYTES_FROM_INT64(buf_, (var)); \
- WRITE_BYTES(buf_, sizeof(buf_)); \
- } while (false)
-
-#define WRITE_UINT64(var) \
- do { \
- char buf_[8]; \
- BYTES_FROM_UINT64(buf_, (var)); \
- WRITE_BYTES(buf_, sizeof(buf_)); \
- } while (false)
-
static bool
-write_manifest(FILE* f, const struct manifest* mf)
+write_manifest(const std::string& path, const ManifestData& mf)
{
- int ret = false;
-
- uint64_t content_size = COMMON_HEADER_SIZE;
+ uint64_t content_size = 15;
content_size += 4; // n_files
- for (size_t i = 0; i < mf->n_files; i++) {
- content_size += 2 + mf->files[i].path_len;
+ for (size_t i = 0; i < mf.files.size(); ++i) {
+ content_size += 2 + mf.files[i].length();
}
content_size += 4; // n_file_infos
- content_size += mf->n_file_infos * (4 + DIGEST_SIZE + 8 + 8 + 8);
+ content_size += mf.file_infos.size() * (4 + DIGEST_SIZE + 8 + 8 + 8);
content_size += 4; // n_results
- for (size_t i = 0; i < mf->n_results; i++) {
+ for (size_t i = 0; i < mf.results.size(); ++i) {
content_size += 4; // n_file_info_indexes
- content_size += mf->results[i].n_file_info_indexes * 4;
+ content_size += mf.results[i].file_info_indexes.size() * 4;
content_size += DIGEST_SIZE;
}
content_size += 8; // checksum
Checksum checksum;
- struct common_header header;
- struct compressor* compressor;
- struct compr_state* compr_state;
- if (!common_header_initialize_for_writing(&header,
- f,
- MANIFEST_MAGIC,
- MANIFEST_VERSION,
- compression_type_from_config(),
- compression_level_from_config(),
- content_size,
- checksum,
- &compressor,
- &compr_state)) {
- goto out;
- }
-
- WRITE_UINT32(mf->n_files);
- for (uint32_t i = 0; i < mf->n_files; i++) {
- WRITE_UINT16(mf->files[i].path_len);
- WRITE_BYTES(mf->files[i].path, mf->files[i].path_len);
- }
-
- WRITE_UINT32(mf->n_file_infos);
- for (uint32_t i = 0; i < mf->n_file_infos; i++) {
- WRITE_UINT32(mf->file_infos[i].index);
- WRITE_BYTES(mf->file_infos[i].digest.bytes, DIGEST_SIZE);
- WRITE_UINT64(mf->file_infos[i].fsize);
- WRITE_INT64(mf->file_infos[i].mtime);
- WRITE_INT64(mf->file_infos[i].ctime);
- }
-
- WRITE_UINT32(mf->n_results);
- for (uint32_t i = 0; i < mf->n_results; i++) {
- WRITE_UINT32(mf->results[i].n_file_info_indexes);
- for (uint32_t j = 0; j < mf->results[i].n_file_info_indexes; j++) {
- WRITE_UINT32(mf->results[i].file_info_indexes[j]);
+ AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary);
+ CacheEntryWriter writer(atomic_manifest_file.stream(),
+ k_manifest_magic,
+ k_manifest_version,
+ Compression::type_from_config(),
+ Compression::level_from_config(),
+ content_size,
+ checksum);
+ writer.write<uint32_t>(mf.files.size());
+ for (uint32_t i = 0; i < mf.files.size(); ++i) {
+ writer.write<uint16_t>(mf.files[i].length());
+ writer.write(mf.files[i].data(), mf.files[i].length());
+ }
+
+ writer.write<uint32_t>(mf.file_infos.size());
+ for (uint32_t i = 0; i < mf.file_infos.size(); ++i) {
+ writer.write<uint32_t>(mf.file_infos[i].index);
+ writer.write(mf.file_infos[i].digest.bytes, DIGEST_SIZE);
+ writer.write(mf.file_infos[i].fsize);
+ writer.write(mf.file_infos[i].mtime);
+ writer.write(mf.file_infos[i].ctime);
+ }
+
+ writer.write<uint32_t>(mf.results.size());
+ for (uint32_t i = 0; i < mf.results.size(); ++i) {
+ writer.write<uint32_t>(mf.results[i].file_info_indexes.size());
+ for (uint32_t j = 0; j < mf.results[i].file_info_indexes.size(); ++j) {
+ writer.write(mf.results[i].file_info_indexes[j]);
}
- WRITE_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
+ writer.write(mf.results[i].name.bytes, DIGEST_SIZE);
}
- WRITE_UINT64(checksum.digest());
-
- ret = compressor->free(compr_state);
-
-out:
- if (!ret) {
- cc_log("Error writing to manifest file");
- }
- return ret;
+ writer.write(checksum.digest());
+ writer.finalize();
+ atomic_manifest_file.commit();
+ return true;
}
static bool
verify_result(const Config& config,
- struct manifest* mf,
- struct result* result,
- std::unordered_map<std::string, file_stats>* stated_files,
- std::unordered_map<std::string, digest>* hashed_files)
+ const ManifestData& mf,
+ const ResultEntry& result,
+ std::unordered_map<std::string, FileStats>& stated_files,
+ std::unordered_map<std::string, digest>& hashed_files)
{
- for (uint32_t i = 0; i < result->n_file_info_indexes; i++) {
- struct FileInfo* fi = &mf->file_infos[result->file_info_indexes[i]];
- char* path = mf->files[fi->index].path;
- auto stated_files_iter = stated_files->find(path);
- if (stated_files_iter == stated_files->end()) {
+ for (uint32_t i = 0; i < result.file_info_indexes.size(); ++i) {
+ const auto& fi = mf.file_infos[result.file_info_indexes[i]];
+ const auto& path = mf.files[fi.index];
+
+ auto stated_files_iter = stated_files.find(path);
+ if (stated_files_iter == stated_files.end()) {
struct stat file_stat;
- if (x_stat(path, &file_stat) != 0) {
+ if (x_stat(path.c_str(), &file_stat) != 0) {
return false;
}
- file_stats st;
+ FileStats st;
st.size = file_stat.st_size;
st.mtime = file_stat.st_mtime;
st.ctime = file_stat.st_ctime;
- stated_files_iter = stated_files->emplace(path, st).first;
+ stated_files_iter = stated_files.emplace(path, st).first;
}
- const file_stats& fs = stated_files_iter->second;
+ const FileStats& fs = stated_files_iter->second;
- if (fi->fsize != fs.size) {
+ if (fi.fsize != fs.size) {
return false;
}
// and will error out if that header is later used without rebuilding.
if ((guessed_compiler == GUESSED_CLANG
|| guessed_compiler == GUESSED_UNKNOWN)
- && output_is_precompiled_header && fi->mtime != fs.mtime) {
- cc_log("Precompiled header includes %s, which has a new mtime", path);
+ && output_is_precompiled_header && fi.mtime != fs.mtime) {
+ cc_log("Precompiled header includes %s, which has a new mtime",
+ path.c_str());
return false;
}
if (config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) {
if (!(config.sloppiness() & SLOPPY_FILE_STAT_MATCHES_CTIME)) {
- if (fi->mtime == fs.mtime && fi->ctime == fs.ctime) {
- cc_log("mtime/ctime hit for %s", path);
+ if (fi.mtime == fs.mtime && fi.ctime == fs.ctime) {
+ cc_log("mtime/ctime hit for %s", path.c_str());
continue;
} else {
- cc_log("mtime/ctime miss for %s", path);
+ cc_log("mtime/ctime miss for %s", path.c_str());
}
} else {
- if (fi->mtime == fs.mtime) {
- cc_log("mtime hit for %s", path);
+ if (fi.mtime == fs.mtime) {
+ cc_log("mtime hit for %s", path.c_str());
continue;
} else {
- cc_log("mtime miss for %s", path);
+ cc_log("mtime miss for %s", path.c_str());
}
}
}
- auto hashed_files_iter = hashed_files->find(path);
- if (hashed_files_iter == hashed_files->end()) {
+ auto hashed_files_iter = hashed_files.find(path);
+ if (hashed_files_iter == hashed_files.end()) {
struct hash* hash = hash_init();
- int ret = hash_source_code_file(config, hash, path);
+ int ret = hash_source_code_file(config, hash, path.c_str());
if (ret & HASH_SOURCE_CODE_ERROR) {
- cc_log("Failed hashing %s", path);
+ cc_log("Failed hashing %s", path.c_str());
hash_free(hash);
return false;
}
digest actual;
hash_result_as_bytes(hash, &actual);
hash_free(hash);
- hashed_files_iter = hashed_files->emplace(path, actual).first;
+ hashed_files_iter = hashed_files.emplace(path, actual).first;
}
- if (!digests_equal(&fi->digest, &hashed_files_iter->second)) {
+ if (!digests_equal(&fi.digest, &hashed_files_iter->second)) {
return false;
}
}
return true;
}
-static void
-create_file_index_map(
- struct file* files,
- uint32_t len,
- std::unordered_map<std::string, uint32_t /*index*/>* mf_files)
-{
- for (uint32_t i = 0; i < len; i++) {
- mf_files->emplace(files[i].path, i);
- }
-}
-
-static void
-create_file_info_index_map(
- struct FileInfo* infos,
- uint32_t len,
- std::unordered_map<FileInfo, uint32_t /*index*/>* mf_file_infos)
-{
- for (uint32_t i = 0; i < len; i++) {
- mf_file_infos->emplace(infos[i], i);
- }
-}
-
-static uint32_t
-get_include_file_index(
- struct manifest* mf,
- const std::string& path,
- const std::unordered_map<std::string, uint32_t>& mf_files)
-{
- auto it = mf_files.find(path);
- if (it != mf_files.end()) {
- return it->second;
- }
-
- uint32_t n = mf->n_files;
- mf->files = static_cast<file*>(x_realloc(mf->files, (n + 1) * sizeof(file)));
- mf->n_files++;
- mf->files[n].path_len = path.size();
- mf->files[n].path = x_strdup(path.c_str());
- return n;
-}
-
-static uint32_t
-get_file_info_index(struct manifest* mf,
- const std::string& path,
- const digest& digest,
- const std::unordered_map<std::string, uint32_t>& mf_files,
- const std::unordered_map<FileInfo, uint32_t>& mf_file_infos)
-{
- struct FileInfo fi;
- fi.index = get_include_file_index(mf, path, mf_files);
- fi.digest = digest;
-
- // file_stat.st_{m,c}time has a resolution of 1 second, so we can cache the
- // file's mtime and ctime only if they're at least one second older than
- // time_of_compilation.
- //
- // st->ctime may be 0, so we have to check time_of_compilation against
- // MAX(mtime, ctime).
-
- struct stat file_stat;
- if (stat(path.c_str(), &file_stat) != -1) {
- if (time_of_compilation
- > std::max(file_stat.st_mtime, file_stat.st_ctime)) {
- fi.mtime = file_stat.st_mtime;
- fi.ctime = file_stat.st_ctime;
- } else {
- fi.mtime = -1;
- fi.ctime = -1;
- }
- fi.fsize = file_stat.st_size;
- } else {
- fi.mtime = -1;
- fi.ctime = -1;
- fi.fsize = 0;
- }
-
- auto it = mf_file_infos.find(fi);
- if (it != mf_file_infos.end()) {
- return it->second;
- }
-
- uint32_t n = mf->n_file_infos;
- mf->file_infos = static_cast<FileInfo*>(
- x_realloc(mf->file_infos, (n + 1) * sizeof(FileInfo)));
- mf->n_file_infos++;
- mf->file_infos[n] = fi;
- return n;
-}
-
-static void
-add_file_info_indexes(
- uint32_t* indexes,
- uint32_t size,
- struct manifest* mf,
- const std::unordered_map<std::string, digest>& included_files)
-{
- if (size == 0) {
- return;
- }
-
- std::unordered_map<std::string, uint32_t /*index*/> mf_files;
- create_file_index_map(mf->files, mf->n_files, &mf_files);
-
- std::unordered_map<FileInfo, uint32_t /*index*/> mf_file_infos;
- create_file_info_index_map(mf->file_infos, mf->n_file_infos, &mf_file_infos);
-
- uint32_t i = 0;
- for (const auto& item : included_files) {
- const auto& path = item.first;
- const auto& digest = item.second;
- indexes[i] = get_file_info_index(mf, path, digest, mf_files, mf_file_infos);
- i++;
- }
- assert(i == size);
-}
-
-static void
-add_result_entry(struct manifest* mf,
- struct digest* result_digest,
- const std::unordered_map<std::string, digest>& included_files)
-{
- uint32_t n_results = mf->n_results;
- mf->results = static_cast<result*>(
- x_realloc(mf->results, (n_results + 1) * sizeof(result)));
- mf->n_results++;
- struct result* result = &mf->results[n_results];
-
- uint32_t n_fii = included_files.size();
- result->n_file_info_indexes = n_fii;
- result->file_info_indexes =
- static_cast<uint32_t*>(x_malloc(n_fii * sizeof(uint32_t)));
- add_file_info_indexes(result->file_info_indexes, n_fii, mf, included_files);
- result->name = *result_digest;
-}
-
// Try to get the result name from a manifest file. Caller frees. Returns NULL
// on failure.
struct digest*
-manifest_get(const Config& config, const char* manifest_path)
+manifest_get(const Config& config, const std::string& path)
{
- char* errmsg;
- struct manifest* mf = read_manifest(manifest_path, &errmsg);
- if (!mf) {
- cc_log("%s", errmsg);
- return NULL;
+ std::unique_ptr<ManifestData> mf;
+ try {
+ mf = read_manifest(path);
+ if (mf) {
+ // Update modification timestamp to save files from LRU cleanup.
+ update_mtime(path.c_str());
+ } else {
+ cc_log("No such result file");
+ return nullptr;
+ }
+ } catch (const Error& e) {
+ cc_log("Error: %s", e.what());
+ return nullptr;
}
- std::unordered_map<std::string, file_stats> stated_files;
+ std::unordered_map<std::string, FileStats> stated_files;
std::unordered_map<std::string, digest> hashed_files;
// Check newest result first since it's a bit more likely to match.
struct digest* name = NULL;
- for (uint32_t i = mf->n_results; i > 0; i--) {
+ for (uint32_t i = mf->results.size(); i > 0; i--) {
if (verify_result(
- config, mf, &mf->results[i - 1], &stated_files, &hashed_files)) {
+ config, *mf, mf->results[i - 1], stated_files, hashed_files)) {
name = static_cast<digest*>(x_malloc(sizeof(digest)));
*name = mf->results[i - 1].name;
- goto out;
+ break;
}
}
-out:
- free_manifest(mf);
- if (name) {
- // Update modification timestamp to save files from LRU cleanup.
- update_mtime(manifest_path);
- }
return name;
}
// Put the result name into a manifest file given a set of included files.
// Returns true on success, otherwise false.
bool
-manifest_put(const char* manifest_path,
- struct digest* result_name,
+manifest_put(const std::string& path,
+ const struct digest& result_name,
const std::unordered_map<std::string, digest>& included_files)
{
// We don't bother to acquire a lock when writing the manifest to disk. A
// race between two processes will only result in one lost entry, which is
// not a big deal, and it's also very unlikely.
- char* errmsg;
- struct manifest* mf = read_manifest(manifest_path, &errmsg);
- if (!mf) {
- // New or corrupt file.
- mf = create_empty_manifest();
- free(errmsg); // Ignore.
+ std::unique_ptr<ManifestData> mf;
+ try {
+ mf = read_manifest(path);
+ if (!mf) {
+ // Manifest file didn't exist.
+ mf = std::make_unique<ManifestData>();
+ }
+ } catch (const Error& e) {
+ cc_log("Error: %s", e.what());
+ // Manifest file was corrupt, ignore.
+ mf = std::make_unique<ManifestData>();
}
- if (mf->n_results > MAX_MANIFEST_ENTRIES) {
+ if (mf->results.size() > k_max_manifest_entries) {
// Normally, there shouldn't be many result entries in the manifest since
// new entries are added only if an include file has changed but not the
// source file, and you typically change source files more often than
// LRU order and discarding the old ones. An easy way is to throw away all
// entries when there are too many. Let's do that for now.
cc_log("More than %u entries in manifest file; discarding",
- MAX_MANIFEST_ENTRIES);
- free_manifest(mf);
- mf = create_empty_manifest();
- } else if (mf->n_file_infos > MAX_MANIFEST_FILE_INFO_ENTRIES) {
+ k_max_manifest_entries);
+ mf = std::make_unique<ManifestData>();
+ } else if (mf->file_infos.size() > k_max_manifest_file_info_entries) {
// Rarely, FileInfo entries can grow large in pathological cases where
// many included files change, but the main file does not. This also puts
// an upper bound on the number of FileInfo entries.
cc_log("More than %u FileInfo entries in manifest file; discarding",
- MAX_MANIFEST_FILE_INFO_ENTRIES);
- free_manifest(mf);
- mf = create_empty_manifest();
+ k_max_manifest_file_info_entries);
+ mf = std::make_unique<ManifestData>();
}
- add_result_entry(mf, result_name, included_files);
-
- int ret = false;
- char* tmp_file = format("%s.tmp", manifest_path);
- int fd = create_tmp_fd(&tmp_file);
- FILE* f = fdopen(fd, "wb");
- if (!f) {
- cc_log("Failed to fdopen %s", tmp_file);
- goto out;
- }
- if (write_manifest(f, mf)) {
- fclose(f);
- f = NULL;
- if (x_rename(tmp_file, manifest_path) == 0) {
- ret = true;
- } else {
- cc_log("Failed to rename %s to %s", tmp_file, manifest_path);
- }
- } else {
- cc_log("Failed to write manifest file %s", tmp_file);
- }
+ mf->add_result_entry(result_name, included_files);
-out:
- if (f) {
- fclose(f);
- }
- if (mf) {
- free_manifest(mf);
- }
- if (tmp_file) {
- free(tmp_file);
+ try {
+ write_manifest(path, *mf);
+ return true;
+ } catch (const Error& e) {
+ cc_log("Error: %s", e.what());
+ return false;
}
- return ret;
}
bool
-manifest_dump(const char* manifest_path, FILE* stream)
+manifest_dump(const std::string& path, FILE* stream)
{
- char* errmsg;
- struct manifest* mf = read_manifest(manifest_path, &errmsg);
- if (!mf) {
- assert(errmsg);
- fprintf(stream, "Error: %s\n", errmsg);
- free(errmsg);
+ std::unique_ptr<ManifestData> mf;
+ try {
+ mf = read_manifest(path, stream);
+ } catch (const Error& e) {
+ fmt::print(stream, "Error: {}\n", e.what());
return false;
}
- common_header_dump(&mf->header, stream);
+ if (!mf) {
+ fmt::print(stream, "Error: No such file: {}\n", path);
+ return false;
+ }
- fprintf(stream, "File paths (%u):\n", (unsigned)mf->n_files);
- for (unsigned i = 0; i < mf->n_files; ++i) {
- fprintf(stream, " %u: %s\n", i, mf->files[i].path);
+ fmt::print(stream, "File paths ({}):\n", mf->files.size());
+ for (unsigned i = 0; i < mf->files.size(); ++i) {
+ fmt::print(stream, " {}: {}\n", i, mf->files[i]);
}
- fprintf(stream, "File infos (%u):\n", (unsigned)mf->n_file_infos);
- for (unsigned i = 0; i < mf->n_file_infos; ++i) {
+ fmt::print(stream, "File infos ({}):\n", mf->file_infos.size());
+ for (unsigned i = 0; i < mf->file_infos.size(); ++i) {
char digest[DIGEST_STRING_BUFFER_SIZE];
- fprintf(stream, " %u:\n", i);
- fprintf(stream, " Path index: %u\n", mf->file_infos[i].index);
+ fmt::print(stream, " {}:\n", i);
+ fmt::print(stream, " Path index: {}\n", mf->file_infos[i].index);
digest_as_string(&mf->file_infos[i].digest, digest);
- fprintf(stream, " Hash: %s\n", digest);
- fprintf(stream, " File size: %" PRIu64 "\n", mf->file_infos[i].fsize);
- fprintf(stream, " Mtime: %lld\n", (long long)mf->file_infos[i].mtime);
- fprintf(stream, " Ctime: %lld\n", (long long)mf->file_infos[i].ctime);
+ fmt::print(stream, " Hash: {}\n", digest);
+ fmt::print(stream, " File size: {}\n", mf->file_infos[i].fsize);
+ fmt::print(stream, " Mtime: {}\n", mf->file_infos[i].mtime);
+ fmt::print(stream, " Ctime: {}\n", mf->file_infos[i].ctime);
}
- fprintf(stream, "Results (%u):\n", (unsigned)mf->n_results);
- for (unsigned i = 0; i < mf->n_results; ++i) {
+ fmt::print(stream, "Results ({}):\n", mf->results.size());
+ for (unsigned i = 0; i < mf->results.size(); ++i) {
char name[DIGEST_STRING_BUFFER_SIZE];
- fprintf(stream, " %u:\n", i);
- fprintf(stream, " File info indexes:");
- for (unsigned j = 0; j < mf->results[i].n_file_info_indexes; ++j) {
- fprintf(stream, " %u", mf->results[i].file_info_indexes[j]);
+ fmt::print(stream, " {}:\n", i);
+ fmt::print(stream, " File info indexes:");
+ for (unsigned j = 0; j < mf->results[i].file_info_indexes.size(); ++j) {
+ fmt::print(stream, " {}", mf->results[i].file_info_indexes[j]);
}
- fprintf(stream, "\n");
+ fmt::print(stream, "\n");
digest_as_string(&mf->results[i].name, name);
- fprintf(stream, " Name: %s\n", name);
+ fmt::print(stream, " Name: {}\n", name);
}
- free_manifest(mf);
return true;
}
#include "system.hpp"
-#include "Config.hpp"
-#include "hashutil.hpp"
+#include <string>
+#include <unordered_map>
-extern const char MANIFEST_MAGIC[4];
-#define MANIFEST_VERSION 2
+class Config;
+struct digest;
-struct digest* manifest_get(const Config& config, const char* manifest_path);
+extern const uint8_t k_manifest_magic[4];
+extern const uint8_t k_manifest_version;
+
+struct digest* manifest_get(const Config& config, const std::string& path);
bool
-manifest_put(const char* manifest_path,
- struct digest* result_digest,
+manifest_put(const std::string& path,
+ const struct digest& result_name,
const std::unordered_map<std::string, digest>& included_files);
-bool manifest_dump(const char* manifest_path, FILE* stream);
+bool manifest_dump(const std::string& path, FILE* stream);
#include "result.hpp"
+#include "AtomicFile.hpp"
+#include "CacheEntryReader.hpp"
+#include "CacheEntryWriter.hpp"
+#include "Checksum.hpp"
#include "Config.hpp"
+#include "Error.hpp"
+#include "File.hpp"
+#include "Util.hpp"
#include "ccache.hpp"
-#include "common_header.hpp"
-#include "compression.hpp"
-#include "hash.hpp"
-#include "int_bytes_conversion.hpp"
// Result data format
// ==================
extern char* stats_file;
-const char RESULT_MAGIC[4] = {'c', 'C', 'r', 'S'};
+const uint8_t k_result_magic[4] = {'c', 'C', 'r', 'S'};
+const uint8_t k_result_version = 1;
+const std::string k_result_stderr_name = "<stderr>";
-enum {
- // File data stored inside the result file.
- EMBEDDED_FILE_MARKER = 0,
+// File data stored inside the result file.
+const uint8_t k_embedded_file_marker = 0;
- // File stored as-is in the file system.
- RAW_FILE_MARKER = 1
-};
+// File stored as-is in the file system.
+const uint8_t k_raw_file_marker = 1;
-struct result_file
-{
- char* suffix;
- char* path;
- uint64_t size;
-};
-
-struct result_files
-{
- uint32_t n_files;
- struct result_file* files;
- uint64_t* sizes;
-};
-
-typedef bool (*read_entry_fn)(struct decompressor* decompressor,
- struct decompr_state* decompr_state,
- const char* result_path_in_cache,
- uint32_t entry_number,
- const struct result_files* list,
- FILE* dump_stream);
-
-typedef bool (*write_entry_fn)(struct compressor* compressor,
- struct compr_state* compr_state,
- const char* result_path_in_cache,
- uint32_t entry_number,
- const struct result_file* file);
-
-struct result_files*
-result_files_init(void)
-{
- auto list = static_cast<result_files*>(x_malloc(sizeof(result_files)));
- list->n_files = 0;
- list->files = NULL;
- list->sizes = NULL;
-
- return list;
-}
-
-void
-result_files_add(struct result_files* list,
- const char* path,
- const char* suffix)
-{
- uint32_t n = list->n_files;
- list->files = static_cast<result_file*>(
- x_realloc(list->files, (n + 1) * sizeof(result_file)));
- list->sizes =
- static_cast<uint64_t*>(x_realloc(list->sizes, (n + 1) * sizeof(uint64_t)));
- struct result_file* f = &list->files[list->n_files];
- list->n_files++;
-
- struct stat st;
- x_stat(path, &st);
-
- f->suffix = x_strdup(suffix);
- f->path = x_strdup(path);
- f->size = st.st_size;
-}
-
-void
-result_files_free(struct result_files* list)
-{
- for (uint32_t i = 0; i < list->n_files; i++) {
- free(list->files[i].suffix);
- free(list->files[i].path);
- }
- free(list->files);
- list->files = NULL;
- free(list->sizes);
- list->sizes = NULL;
+using ReadEntryFunction = void (*)(CacheEntryReader& reader,
+ const std::string& result_path_in_cache,
+ uint32_t entry_number,
+ const ResultFileMap* result_file_map,
+ FILE* dump_stream);
- free(list);
-}
-
-#define READ_BYTES(buf, length) \
- do { \
- if (!decompressor->read(decompr_state, buf, length)) { \
- goto out; \
- } \
- } while (false)
-
-#define READ_BYTE(var) READ_BYTES(&var, 1)
+using WriteEntryFunction =
+ void (*)(CacheEntryWriter& writer,
+ const std::string& result_path_in_cache,
+ uint32_t entry_number,
+ const ResultFileMap::value_type& suffix_and_path);
-#define READ_UINT64(var) \
- do { \
- char buf_[8]; \
- READ_BYTES(buf_, sizeof(buf_)); \
- (var) = UINT64_FROM_BYTES(buf_); \
- } while (false)
-
-static bool
-read_embedded_file_entry(struct decompressor* decompressor,
- struct decompr_state* decompr_state,
- const char* result_path_in_cache,
+static void
+read_embedded_file_entry(CacheEntryReader& reader,
+ const std::string& /*result_path_in_cache*/,
uint32_t entry_number,
- const struct result_files* list,
+ const ResultFileMap* result_file_map,
FILE* dump_stream)
{
- (void)result_path_in_cache;
- bool found = false;
- bool success = false;
- FILE* subfile = NULL;
-
uint8_t suffix_len;
- READ_BYTE(suffix_len);
+ reader.read(suffix_len);
- char suffix[256 + 1];
- READ_BYTES(suffix, suffix_len);
- suffix[suffix_len] = '\0';
+ char suffix[256];
+ reader.read(suffix, suffix_len);
uint64_t file_len;
- READ_UINT64(file_len);
+ reader.read(file_len);
+ bool content_read = false;
if (dump_stream) {
- fprintf(dump_stream,
- "Embedded file #%u: %s (%" PRIu64 " bytes)\n",
- entry_number,
- suffix,
- file_len);
+ fmt::print(dump_stream,
+ "Embedded file #{}: {} ({} bytes)\n",
+ entry_number,
+ suffix,
+ file_len);
} else {
cc_log("Retrieving embedded file #%u %s (%llu bytes)",
entry_number,
suffix,
(unsigned long long)file_len);
- for (uint32_t i = 0; i < list->n_files; i++) {
- if (str_eq(suffix, list->files[i].suffix)) {
- found = true;
+ std::string suffix_str(suffix, suffix_len);
+ const auto it = result_file_map->find(suffix_str);
+ if (it != result_file_map->end()) {
+ content_read = true;
- cc_log("Copying to %s", list->files[i].path);
+ const auto& path = it->second;
+ cc_log("Copying to %s", path.c_str());
- subfile = fopen(list->files[i].path, "wb");
- if (!subfile) {
- cc_log("Failed to open %s for writing", list->files[i].path);
- goto out;
- }
- char buf[READ_BUFFER_SIZE];
- size_t remain = file_len;
- while (remain > 0) {
- size_t n = std::min(remain, sizeof(buf));
- READ_BYTES(buf, n);
- if (fwrite(buf, 1, n, subfile) != n) {
- goto out;
- }
- remain -= n;
+ File subfile(path, "wb");
+ if (!subfile) {
+ throw Error(fmt::format(
+ "Failed to open {} for writing: {}", path, strerror(errno)));
+ }
+ uint8_t buf[READ_BUFFER_SIZE];
+ size_t remain = file_len;
+ while (remain > 0) {
+ size_t n = std::min(remain, sizeof(buf));
+ reader.read(buf, n);
+ if (fwrite(buf, n, 1, subfile.get()) != 1) {
+ throw Error(fmt::format("Failed to write to {}", path));
}
- fclose(subfile);
- subfile = NULL;
-
- break;
+ remain -= n;
}
}
}
- if (!found) {
+ if (!content_read) {
// Discard the file data.
- char buf[READ_BUFFER_SIZE];
+ uint8_t buf[READ_BUFFER_SIZE];
size_t remain = file_len;
while (remain > 0) {
size_t n = std::min(remain, sizeof(buf));
- READ_BYTES(buf, n);
+ reader.read(buf, n);
remain -= n;
}
}
-
- success = true;
-
-out:
- if (subfile) {
- fclose(subfile);
- }
-
- return success;
}
-static char*
-get_raw_file_path(const char* result_path_in_cache, uint32_t entry_number)
+static std::string
+get_raw_file_path(const std::string& result_path_in_cache,
+ uint32_t entry_number)
{
- return format("%.*s_%u.raw",
- (int)strlen(result_path_in_cache) - 7, // .result
- result_path_in_cache,
- entry_number);
+ return fmt::format("{:{}}_{}.raw",
+ result_path_in_cache.c_str(),
+ result_path_in_cache.length() - 7, // .result
+ entry_number);
}
static bool
-copy_raw_file(const char* source, const char* dest, bool to_cache)
+copy_raw_file(const std::string& source, const std::string& dest, bool to_cache)
{
if (g_config.file_clone()) {
- cc_log("Cloning %s to %s", source, dest);
- if (clone_file(source, dest, to_cache)) {
+ cc_log("Cloning %s to %s", source.c_str(), dest.c_str());
+ if (clone_file(source.c_str(), dest.c_str(), to_cache)) {
return true;
}
cc_log("Failed to clone: %s", strerror(errno));
}
if (g_config.hard_link()) {
- x_try_unlink(dest);
- cc_log("Hard linking %s to %s", source, dest);
- int ret = link(source, dest);
+ x_try_unlink(dest.c_str());
+ cc_log("Hard linking %s to %s", source.c_str(), dest.c_str());
+ int ret = link(source.c_str(), dest.c_str());
if (ret == 0) {
return true;
}
cc_log("Failed to hard link: %s", strerror(errno));
}
- cc_log("Copying %s to %s", source, dest);
- return copy_file(source, dest, to_cache);
+ cc_log("Copying %s to %s", source.c_str(), dest.c_str());
+ return copy_file(source.c_str(), dest.c_str(), to_cache);
}
-static bool
-read_raw_file_entry(struct decompressor* decompressor,
- struct decompr_state* decompr_state,
- const char* result_path_in_cache,
+static void
+read_raw_file_entry(CacheEntryReader& reader,
+ const std::string& result_path_in_cache,
uint32_t entry_number,
- const struct result_files* list,
- FILE* dump_stream)
+ const ResultFileMap* result_file_map,
+ std::FILE* dump_stream)
{
- bool success = false;
- char* raw_path = get_raw_file_path(result_path_in_cache, entry_number);
-
uint8_t suffix_len;
- READ_BYTE(suffix_len);
+ reader.read(suffix_len);
- char suffix[256 + 1];
- READ_BYTES(suffix, suffix_len);
- suffix[suffix_len] = '\0';
+ char suffix[256];
+ reader.read(suffix, suffix_len);
uint64_t file_len;
- READ_UINT64(file_len);
+ reader.read(file_len);
if (dump_stream) {
- fprintf(dump_stream,
- "Raw file #%u: %s (%" PRIu64 " bytes)\n",
- entry_number,
- suffix,
- file_len);
+ fmt::print(dump_stream,
+ "Raw file #{}: {} ({} bytes)\n",
+ entry_number,
+ suffix,
+ file_len);
} else {
cc_log("Retrieving raw file #%u %s (%llu bytes)",
entry_number,
suffix,
(unsigned long long)file_len);
+ auto raw_path = get_raw_file_path(result_path_in_cache, entry_number);
struct stat st;
- if (x_stat(raw_path, &st) != 0) {
- goto out;
+ if (x_stat(raw_path.c_str(), &st) != 0) {
+ throw Error(
+ fmt::format("Failed to stat {}: {}", raw_path, strerror(errno)));
}
if ((uint64_t)st.st_size != file_len) {
- cc_log("Bad file size of %s (actual %llu bytes, expected %llu bytes)",
- raw_path,
- (unsigned long long)st.st_size,
- (unsigned long long)file_len);
- goto out;
+ throw Error(
+ fmt::format("Bad file size of {} (actual {} bytes, expected {} bytes)",
+ raw_path,
+ st.st_size,
+ file_len));
}
- for (uint32_t i = 0; i < list->n_files; i++) {
- if (str_eq(suffix, list->files[i].suffix)) {
- if (!copy_raw_file(raw_path, list->files[i].path, false)) {
- goto out;
- }
- // Update modification timestamp to save the file from LRU cleanup
- // (and, if hard-linked, to make the object file newer than the source
- // file).
- update_mtime(raw_path);
- break;
+ std::string suffix_str(suffix, suffix_len);
+ const auto it = result_file_map->find(suffix_str);
+ if (it != result_file_map->end()) {
+ const auto& dest_path = it->second;
+ if (!copy_raw_file(raw_path, dest_path, false)) {
+ throw Error(
+ fmt::format("Failed to copy raw file {} to {}", raw_path, dest_path));
}
+ // Update modification timestamp to save the file from LRU cleanup
+ // (and, if hard-linked, to make the object file newer than the source
+ // file).
+ update_mtime(raw_path.c_str());
}
}
-
- success = true;
-
-out:
- free(raw_path);
- return success;
}
static bool
-read_result(const char* path,
- struct result_files* list,
- FILE* dump_stream,
- char** errmsg)
+read_result(const std::string& path,
+ const ResultFileMap* result_file_map,
+ FILE* dump_stream)
{
- *errmsg = NULL;
- bool cache_miss = false;
- bool success = false;
- struct decompressor* decompressor = NULL;
- struct decompr_state* decompr_state = NULL;
- Checksum checksum;
-
- FILE* f = fopen(path, "rb");
- if (!f) {
- cache_miss = true;
- goto out;
+ File file(path, "rb");
+ if (!file) {
+ // Cache miss.
+ return false;
}
- struct common_header header;
- if (!common_header_initialize_for_reading(&header,
- f,
- RESULT_MAGIC,
- RESULT_VERSION,
- &decompressor,
- &decompr_state,
- &checksum,
- errmsg)) {
- goto out;
- }
+ Checksum checksum;
+ CacheEntryReader reader(
+ file.get(), k_result_magic, k_result_version, &checksum);
if (dump_stream) {
- common_header_dump(&header, dump_stream);
+ reader.dump_header(dump_stream);
}
uint8_t n_entries;
- READ_BYTE(n_entries);
+ reader.read(n_entries);
uint32_t i;
- for (i = 0; i < n_entries; i++) {
+ for (i = 0; i < n_entries; ++i) {
uint8_t marker;
- READ_BYTE(marker);
+ reader.read(marker);
- read_entry_fn read_entry;
+ ReadEntryFunction read_entry;
switch (marker) {
- case EMBEDDED_FILE_MARKER:
+ case k_embedded_file_marker:
read_entry = read_embedded_file_entry;
break;
- case RAW_FILE_MARKER:
+ case k_raw_file_marker:
read_entry = read_raw_file_entry;
break;
default:
- *errmsg = format("Unknown entry type: %u", marker);
- goto out;
+ throw Error(fmt::format("Unknown entry type: {}", marker));
}
- if (!read_entry(decompressor, decompr_state, path, i, list, dump_stream)) {
- goto out;
- }
+ read_entry(reader, path, i, result_file_map, dump_stream);
}
if (i != n_entries) {
- *errmsg = format("Too few entries (read %u, expected %u)", i, n_entries);
- goto out;
+ throw Error(
+ fmt::format("Too few entries (read {}, expected {})", i, n_entries));
}
- {
- uint64_t actual_checksum = checksum.digest();
- uint64_t expected_checksum;
- READ_UINT64(expected_checksum);
-
- if (actual_checksum == expected_checksum) {
- success = true;
- } else {
- *errmsg = format("Incorrect checksum (actual %016llx, expected %016llx)",
- (unsigned long long)actual_checksum,
- (unsigned long long)expected_checksum);
- }
+ uint64_t actual_checksum = checksum.digest();
+ uint64_t expected_checksum;
+ reader.read(expected_checksum);
+ if (actual_checksum != expected_checksum) {
+ throw Error(
+ fmt::format("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})",
+ actual_checksum,
+ expected_checksum));
}
-out:
- if (decompressor && !decompressor->free(decompr_state)) {
- success = false;
- }
- if (f) {
- fclose(f);
- }
- if (!success && !cache_miss && !*errmsg) {
- *errmsg = x_strdup("Corrupt result");
- }
- return success;
+ reader.finalize();
+ return true;
}
-#define WRITE_BYTES(buf, length) \
- do { \
- if (!compressor->write(compr_state, buf, length)) { \
- goto error; \
- } \
- } while (false)
-
-#define WRITE_BYTE(var) \
- do { \
- char ch_ = var; \
- WRITE_BYTES(&ch_, 1); \
- } while (false)
-
-#define WRITE_UINT64(var) \
- do { \
- char buf_[8]; \
- BYTES_FROM_UINT64(buf_, (var)); \
- WRITE_BYTES(buf_, sizeof(buf_)); \
- } while (false)
-
-static bool
-write_embedded_file_entry(struct compressor* compressor,
- struct compr_state* compr_state,
- const char* result_path_in_cache,
+static void
+write_embedded_file_entry(CacheEntryWriter& writer,
+ const std::string& /*result_path_in_cache*/,
uint32_t entry_number,
- const struct result_file* file)
+ const ResultFileMap::value_type& suffix_and_path)
{
- (void)result_path_in_cache;
- bool success = false;
- size_t suffix_len;
- FILE* f;
- char buf[READ_BUFFER_SIZE];
- size_t remain;
+ const auto& suffix = suffix_and_path.first;
+ const auto& source_path = suffix_and_path.second;
+
+ uint64_t source_file_size;
+ if (!Util::get_file_size(source_path, source_file_size)) {
+ throw Error(
+ fmt::format("Failed to stat {}: {}", source_path, strerror(errno)));
+ }
cc_log("Storing embedded file #%u %s (%llu bytes) from %s",
entry_number,
- file->suffix,
- (unsigned long long)file->size,
- file->path);
-
- WRITE_BYTE(EMBEDDED_FILE_MARKER);
- suffix_len = strlen(file->suffix);
- WRITE_BYTE(suffix_len);
- WRITE_BYTES(file->suffix, suffix_len);
- WRITE_UINT64(file->size);
-
- f = fopen(file->path, "rb");
- if (!f) {
- cc_log("Failed to open %s for reading", file->path);
- goto error;
+ suffix.c_str(),
+ (unsigned long long)source_file_size,
+ source_path.c_str());
+
+ writer.write<uint8_t>(k_embedded_file_marker);
+ writer.write<uint8_t>(suffix.length());
+ writer.write(suffix.data(), suffix.length());
+ writer.write(source_file_size);
+
+ File file(source_path, "rb");
+ if (!file) {
+ throw Error(fmt::format("Failed to open {} for reading", source_path));
}
- remain = file->size;
+
+ size_t remain = source_file_size;
while (remain > 0) {
+ uint8_t buf[READ_BUFFER_SIZE];
size_t n = std::min(remain, sizeof(buf));
- if (fread(buf, 1, n, f) != n) {
- goto error;
+ if (fread(buf, n, 1, file.get()) != 1) {
+ throw Error(fmt::format("Error reading from {}", source_path));
}
- WRITE_BYTES(buf, n);
+ writer.write(buf, n);
remain -= n;
}
- fclose(f);
-
- success = true;
-
-error:
- return success;
}
-static bool
-write_raw_file_entry(struct compressor* compressor,
- struct compr_state* compr_state,
- const char* result_path_in_cache,
+static void
+write_raw_file_entry(CacheEntryWriter& writer,
+ const std::string& result_path_in_cache,
uint32_t entry_number,
- const struct result_file* file)
+ const ResultFileMap::value_type& suffix_and_path)
{
- bool success = false;
- size_t suffix_len;
- char* raw_file;
- struct stat old_stat;
- bool old_existed;
- struct stat new_stat;
- bool new_exists;
+ const auto& suffix = suffix_and_path.first;
+ const auto& source_path = suffix_and_path.second;
+
+ uint64_t source_file_size;
+ if (!Util::get_file_size(source_path, source_file_size)) {
+ throw Error(
+ fmt::format("Failed to stat {}: {}", source_path, strerror(errno)));
+ }
+
size_t old_size;
size_t new_size;
cc_log("Storing raw file #%u %s (%llu bytes) from %s",
entry_number,
- file->suffix,
- (unsigned long long)file->size,
- file->path);
-
- WRITE_BYTE(RAW_FILE_MARKER);
- suffix_len = strlen(file->suffix);
- WRITE_BYTE(suffix_len);
- WRITE_BYTES(file->suffix, suffix_len);
- WRITE_UINT64(file->size);
-
- raw_file = get_raw_file_path(result_path_in_cache, entry_number);
- old_existed = stat(raw_file, &old_stat) == 0;
- success = copy_raw_file(file->path, raw_file, true);
- new_exists = stat(raw_file, &new_stat) == 0;
- free(raw_file);
+ suffix.c_str(),
+ (unsigned long long)source_file_size,
+ source_path.c_str());
+
+ writer.write<uint8_t>(k_raw_file_marker);
+ writer.write<uint8_t>(suffix.length());
+ writer.write(suffix.data(), suffix.length());
+ writer.write(source_file_size);
+
+ auto raw_file = get_raw_file_path(result_path_in_cache, entry_number);
+ struct stat old_stat;
+ bool old_existed = stat(raw_file.c_str(), &old_stat) == 0;
+ if (!copy_raw_file(source_path, raw_file, true)) {
+ throw Error(
+ fmt::format("Failed to store {} as raw file {}", source_path, raw_file));
+ }
+ struct stat new_stat;
+ bool new_exists = stat(raw_file.c_str(), &new_stat) == 0;
old_size = old_existed ? file_size(&old_stat) : 0;
new_size = new_exists ? file_size(&new_stat) : 0;
stats_update_size(stats_file,
new_size - old_size,
(new_exists ? 1 : 0) - (old_existed ? 1 : 0));
-
-error:
- return success;
}
static bool
-should_store_raw_file(const char* suffix)
+should_store_raw_file(const std::string& suffix)
{
if (!g_config.file_clone() && !g_config.hard_link()) {
return false;
// could be fixed by letting read_raw_file_entry refuse to hard link .d
// files, but it's easier to simply always store them embedded. This will
// also save i-nodes in the cache.
- return !str_eq(suffix, RESULT_STDERR_NAME) && !str_eq(suffix, ".d");
+ return suffix != k_result_stderr_name && suffix != ".d";
}
-static bool
-write_result(const struct result_files* list,
- struct compressor* compressor,
- struct compr_state* compr_state,
- Checksum& checksum,
- const char* result_path_in_cache)
+static void
+write_result(const std::string& path, const ResultFileMap& result_file_map)
{
- WRITE_BYTE(list->n_files);
-
- for (uint32_t i = 0; i < list->n_files; i++) {
- write_entry_fn write_entry = should_store_raw_file(list->files[i].suffix)
- ? write_raw_file_entry
- : write_embedded_file_entry;
- if (!write_entry(
- compressor, compr_state, result_path_in_cache, i, &list->files[i])) {
- goto error;
+ uint64_t content_size = 15;
+ content_size += 1; // n_entries
+ for (const auto& pair : result_file_map) {
+ const auto& suffix = pair.first;
+ const auto& result_file = pair.second;
+ uint64_t source_file_size;
+ if (!Util::get_file_size(result_file, source_file_size)) {
+ throw Error(
+ fmt::format("Failed to stat {}: {}", result_file, strerror(errno)));
}
+ content_size += 1; // embedded_file_marker
+ content_size += 1; // suffix_len
+ content_size += suffix.length(); // suffix
+ content_size += 8; // data_len
+ content_size += source_file_size; // data
}
+ content_size += 8; // checksum
- WRITE_UINT64(checksum.digest());
-
- return true;
+ Checksum checksum;
+ AtomicFile atomic_result_file(path, AtomicFile::Mode::binary);
+ CacheEntryWriter writer(atomic_result_file.stream(),
+ k_result_magic,
+ k_result_version,
+ Compression::type_from_config(),
+ Compression::level_from_config(),
+ content_size,
+ checksum);
+
+ writer.write<uint8_t>(result_file_map.size());
+
+ size_t entry_number = 0;
+ for (const auto& pair : result_file_map) {
+ const auto& suffix = pair.first;
+ WriteEntryFunction write_entry = should_store_raw_file(suffix)
+ ? write_raw_file_entry
+ : write_embedded_file_entry;
+ write_entry(writer, path, entry_number, pair);
+ ++entry_number;
+ }
-error:
- cc_log("Error writing to result file");
- return false;
+ writer.write(checksum.digest());
+ writer.finalize();
+ atomic_result_file.commit();
}
bool
-result_get(const char* path, struct result_files* list)
+result_get(const std::string& path, const ResultFileMap& result_file_map)
{
- cc_log("Getting result %s", path);
-
- char* errmsg;
- bool success = read_result(path, list, NULL, &errmsg);
- if (success) {
- // Update modification timestamp to save files from LRU cleanup.
- update_mtime(path);
- } else if (errmsg) {
- cc_log("Error: %s", errmsg);
- free(errmsg);
- } else {
- cc_log("No such result file");
+ cc_log("Getting result %s", path.c_str());
+
+ try {
+ bool cache_hit = read_result(path, &result_file_map, nullptr);
+ if (cache_hit) {
+ // Update modification timestamp to save files from LRU cleanup.
+ update_mtime(path.c_str());
+ } else {
+ cc_log("No such result file");
+ }
+ return cache_hit;
+ } catch (const Error& e) {
+ cc_log("Error: %s", e.what());
+ return false;
}
- return success;
}
bool
-result_put(const char* path, struct result_files* list)
+result_put(const std::string& path, const ResultFileMap& result_file_map)
{
- bool ret = false;
- Checksum checksum;
- bool ok;
- uint64_t content_size;
-
- char* tmp_file = format("%s.tmp", path);
- int fd = create_tmp_fd(&tmp_file);
- FILE* f = fdopen(fd, "wb");
- if (!f) {
- cc_log("Failed to fdopen %s", tmp_file);
- goto out;
- }
-
- content_size = COMMON_HEADER_SIZE;
- content_size += 1; // n_entries
- for (uint32_t i = 0; i < list->n_files; i++) {
- content_size += 1; // embedded_file_marker
- content_size += 1; // suffix_len
- content_size += strlen(list->files[i].suffix); // suffix
- content_size += 8; // data_len
- content_size += list->files[i].size; // data
- }
- content_size += 8; // checksum
-
- struct common_header header;
- struct compressor* compressor;
- struct compr_state* compr_state;
- if (!common_header_initialize_for_writing(&header,
- f,
- RESULT_MAGIC,
- RESULT_VERSION,
- compression_type_from_config(),
- compression_level_from_config(),
- content_size,
- checksum,
- &compressor,
- &compr_state)) {
- goto out;
- }
+ cc_log("Storing result %s", path.c_str());
- ok = write_result(list, compressor, compr_state, checksum, path)
- && compressor->free(compr_state);
- if (!ok) {
- cc_log("Failed to write result file");
- goto out;
- }
-
- fclose(f);
- f = NULL;
- if (x_rename(tmp_file, path) == 0) {
- ret = true;
- } else {
- cc_log("Failed to rename %s to %s", tmp_file, path);
- }
-
-out:
- free(tmp_file);
- if (f) {
- fclose(f);
+ try {
+ write_result(path, result_file_map);
+ return true;
+ } catch (const Error& e) {
+ cc_log("Error: %s", e.what());
+ return false;
}
- return ret;
}
bool
-result_dump(const char* path, FILE* stream)
+result_dump(const std::string& path, FILE* stream)
{
assert(stream);
- char* errmsg;
- bool success = read_result(path, NULL, stream, &errmsg);
- if (errmsg) {
- fprintf(stream, "Error: %s\n", errmsg);
- free(errmsg);
+ try {
+ if (read_result(path, nullptr, stream)) {
+ return true;
+ } else {
+ fmt::print(stream, "Error: No such file: {}\n", path);
+ }
+ } catch (const Error& e) {
+ fmt::print(stream, "Error: {}\n", e.what());
}
- return success;
+
+ return false;
}
#include "system.hpp"
-#include <cstdio>
+#include <map>
+#include <string>
-extern const char RESULT_MAGIC[4];
-#define RESULT_VERSION 1
-#define RESULT_STDERR_NAME "<stderr>"
+extern const uint8_t k_result_magic[4];
+extern const uint8_t k_result_version;
+extern const std::string k_result_stderr_name;
-struct result_files;
+typedef std::map<std::string /*suffix*/, std::string /*path*/> ResultFileMap;
-struct result_files* result_files_init(void);
-void
-result_files_add(struct result_files* c, const char* path, const char* suffix);
-void result_files_free(struct result_files* c);
-
-bool result_get(const char* path, struct result_files* list);
-bool result_put(const char* path, struct result_files* list);
-bool result_dump(const char* path, FILE* stream);
+bool result_get(const std::string& path, const ResultFileMap& result_file_map);
+bool result_put(const std::string& path, const ResultFileMap& result_file_map);
+bool result_dump(const std::string& path, FILE* stream);
unsigned suite_args(unsigned);
unsigned suite_argument_processing(unsigned);
unsigned suite_compopt(unsigned);
-unsigned suite_compr_type_none(unsigned);
-unsigned suite_compr_type_zstd(unsigned);
unsigned suite_conf(unsigned);
unsigned suite_counters(unsigned);
unsigned suite_hash(unsigned);
&suite_args,
&suite_argument_processing,
&suite_compopt,
- &suite_compr_type_none,
- &suite_compr_type_zstd,
&suite_counters,
&suite_hash,
&suite_hashutil,
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/Compression.hpp"
+
+#include <catch.hpp>
+
+using Catch::Equals;
+
+TEST_CASE("Compression::level_from_config")
+{
+ CHECK(Compression::level_from_config() == 0);
+}
+
+TEST_CASE("Compression::type_from_config")
+{
+ CHECK(Compression::type_from_config() == Compression::Type::zstd);
+}
+
+TEST_CASE("Compression::type_from_int")
+{
+ CHECK(Compression::type_from_int(0) == Compression::Type::none);
+ CHECK(Compression::type_from_int(1) == Compression::Type::zstd);
+ CHECK_THROWS_WITH(Compression::type_from_int(2), Equals("Unknown type: 2"));
+}
+
+TEST_CASE("Compression::type_to_string")
+{
+ CHECK(Compression::type_to_string(Compression::Type::none) == "none");
+ CHECK(Compression::type_to_string(Compression::Type::zstd) == "zstd");
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/Compression.hpp"
+#include "../src/Compressor.hpp"
+#include "../src/Decompressor.hpp"
+#include "../src/File.hpp"
+
+#include <catch.hpp>
+
+using Catch::Equals;
+
+TEST_CASE("Compression::Type::none roundtrip")
+{
+ File f("data.uncompressed", "w");
+ auto compressor =
+ Compressor::create_from_type(Compression::Type::none, f.get(), 1);
+ CHECK(compressor->actual_compression_level() == 0);
+ compressor->write("foobar", 6);
+ compressor->finalize();
+
+ f.open("data.uncompressed", "r");
+ auto decompressor =
+ Decompressor::create_from_type(Compression::Type::none, f.get());
+
+ char buffer[4];
+ decompressor->read(buffer, 4);
+ CHECK(memcmp(buffer, "foob", 4) == 0);
+
+ SECTION("Garbage data")
+ {
+ // Not reached the end.
+ CHECK_THROWS_WITH(decompressor->finalize(),
+ Equals("garbage data at end of uncompressed stream"));
+ }
+
+ SECTION("Read to end")
+ {
+ decompressor->read(buffer, 2);
+ CHECK(memcmp(buffer, "ar", 2) == 0);
+
+ // Reached the end.
+ decompressor->finalize();
+
+ // Nothing left to read.
+ CHECK_THROWS_WITH(decompressor->read(buffer, 1),
+ Equals("failed to read from uncompressed stream"));
+ }
+}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/Compression.hpp"
+#include "../src/Compressor.hpp"
+#include "../src/Decompressor.hpp"
+#include "../src/File.hpp"
+
+#include <catch.hpp>
+
+using Catch::Equals;
+
+TEST_CASE("Small Compression::Type::zstd roundtrip")
+{
+ File f("data.zstd", "w");
+ auto compressor =
+ Compressor::create_from_type(Compression::Type::zstd, f.get(), 1);
+ CHECK(compressor->actual_compression_level() == 1);
+ compressor->write("foobar", 6);
+ compressor->finalize();
+
+ f.open("data.zstd", "r");
+ auto decompressor =
+ Decompressor::create_from_type(Compression::Type::zstd, f.get());
+
+ char buffer[4];
+ decompressor->read(buffer, 4);
+ CHECK(memcmp(buffer, "foob", 4) == 0);
+
+ // Not reached the end.
+ CHECK_THROWS_WITH(decompressor->finalize(),
+ Equals("garbage data at end of zstd input stream"));
+
+ decompressor->read(buffer, 2);
+ CHECK(memcmp(buffer, "ar", 2) == 0);
+
+ // Reached the end.
+ decompressor->finalize();
+
+ // Nothing left to read.
+ CHECK_THROWS_WITH(decompressor->read(buffer, 1),
+ Equals("failed to read from zstd input stream"));
+}
+
+TEST_CASE("Large compressible Compression::Type::zstd roundtrip")
+{
+ char data[] = "The quick brown fox jumps over the lazy dog";
+
+ File f("data.zstd", "w");
+ auto compressor =
+ Compressor::create_from_type(Compression::Type::zstd, f.get(), 1);
+ for (size_t i = 0; i < 1000; i++) {
+ compressor->write(data, sizeof(data));
+ }
+ compressor->finalize();
+
+ f.open("data.zstd", "r");
+ auto decompressor =
+ Decompressor::create_from_type(Compression::Type::zstd, f.get());
+
+ char buffer[sizeof(data)];
+ for (size_t i = 0; i < 1000; i++) {
+ decompressor->read(buffer, sizeof(buffer));
+ CHECK(memcmp(buffer, data, sizeof(data)) == 0);
+ }
+
+ // Reached the end.
+ decompressor->finalize();
+
+ // Nothing left to read.
+ CHECK_THROWS_WITH(decompressor->read(buffer, 1),
+ Equals("failed to read from zstd input stream"));
+}
+
+TEST_CASE("Large uncompressible Compression::Type::zstd roundtrip")
+{
+ char data[100000];
+ for (size_t i = 0; i < sizeof(data); i++) {
+ data[i] = rand() % 256;
+ }
+
+ File f("data.zstd", "w");
+ auto compressor =
+ Compressor::create_from_type(Compression::Type::zstd, f.get(), 1);
+ compressor->write(data, sizeof(data));
+ compressor->finalize();
+
+ f.open("data.zstd", "r");
+ auto decompressor =
+ Decompressor::create_from_type(Compression::Type::zstd, f.get());
+
+ char buffer[sizeof(data)];
+ decompressor->read(buffer, sizeof(buffer));
+ CHECK(memcmp(buffer, data, sizeof(data)) == 0);
+
+ decompressor->finalize();
+}
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "../src/compression.hpp"
-#include "framework.hpp"
-#include "util.hpp"
-
-TEST_SUITE(compr_type_none)
-
-TEST(small_roundtrip)
-{
- const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
-
- Checksum checksum;
-
- FILE* f = fopen("data.uncompressed", "w");
- struct compressor* compr_none = compressor_from_type(COMPR_TYPE_NONE);
- struct compr_state* c_state = compr_none->init(f, -1, &checksum);
- CHECK(c_state);
-
- CHECK(compr_none->write(c_state, "foobar", 6));
-
- CHECK(compr_none->free(c_state));
- fclose(f);
-
- CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
-
- checksum.reset();
- f = fopen("data.uncompressed", "r");
- struct decompressor* decompr_none = decompressor_from_type(COMPR_TYPE_NONE);
- struct decompr_state* d_state = decompr_none->init(f, &checksum);
- CHECK(d_state);
-
- char buffer[4];
- CHECK(decompr_none->read(d_state, buffer, 4));
- CHECK(memcmp(buffer, "foob", 4) == 0);
- CHECK(decompr_none->read(d_state, buffer, 2));
- CHECK(memcmp(buffer, "ar", 2) == 0);
-
- // Nothing left to read.
- CHECK(!decompr_none->read(d_state, buffer, 1));
-
- // Error state is remembered.
- CHECK(!decompr_none->free(d_state));
- fclose(f);
-
- CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
-}
-
-TEST_SUITE_END
+++ /dev/null
-// Copyright (C) 2019 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "../src/compression.hpp"
-#include "framework.hpp"
-#include "util.hpp"
-
-TEST_SUITE(compr_type_zstd)
-
-TEST(small_roundtrip)
-{
- const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
-
- Checksum checksum;
-
- FILE* f = fopen("data.zstd", "w");
- struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state* c_state = compr_zstd->init(f, -1, &checksum);
- CHECK(c_state);
-
- CHECK(compr_zstd->write(c_state, "foobar", 6));
-
- CHECK(compr_zstd->free(c_state));
- fclose(f);
-
- CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
-
- checksum.reset();
- f = fopen("data.zstd", "r");
- struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state* d_state = decompr_zstd->init(f, &checksum);
- CHECK(d_state);
-
- char buffer[4];
- CHECK(decompr_zstd->read(d_state, buffer, 4));
- CHECK(memcmp(buffer, "foob", 4) == 0);
- CHECK(decompr_zstd->read(d_state, buffer, 2));
- CHECK(memcmp(buffer, "ar", 2) == 0);
-
- // Nothing left to read.
- CHECK(!decompr_zstd->read(d_state, buffer, 1));
-
- // Error state is remembered.
- CHECK(!decompr_zstd->free(d_state));
- fclose(f);
-
- CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
-}
-
-TEST(large_compressible_roundtrip)
-{
- char data[] = "The quick brown fox jumps over the lazy dog";
-
- FILE* f = fopen("data.zstd", "w");
- struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state* c_state = compr_zstd->init(f, 1, NULL);
- CHECK(c_state);
-
- for (size_t i = 0; i < 1000; i++) {
- CHECK(compr_zstd->write(c_state, data, sizeof(data)));
- }
-
- CHECK(compr_zstd->free(c_state));
- fclose(f);
-
- f = fopen("data.zstd", "r");
- struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state* d_state = decompr_zstd->init(f, NULL);
- CHECK(d_state);
-
- char buffer[sizeof(data)];
- for (size_t i = 0; i < 1000; i++) {
- CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer)));
- CHECK(memcmp(buffer, data, sizeof(data)) == 0);
- }
-
- // Nothing left to read.
- CHECK(!decompr_zstd->read(d_state, buffer, 1));
-
- // Error state is remembered.
- CHECK(!decompr_zstd->free(d_state));
- fclose(f);
-}
-
-TEST(large_uncompressible_roundtrip)
-{
- char data[100000];
- for (size_t i = 0; i < sizeof(data); i++) {
- data[i] = rand() % 256;
- }
-
- FILE* f = fopen("data.zstd", "w");
- struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state* c_state = compr_zstd->init(f, 1, NULL);
- CHECK(c_state);
-
- CHECK(compr_zstd->write(c_state, data, sizeof(data)));
-
- CHECK(compr_zstd->free(c_state));
- fclose(f);
-
- f = fopen("data.zstd", "r");
- struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state* d_state = decompr_zstd->init(f, NULL);
- CHECK(d_state);
-
- char buffer[sizeof(data)];
- CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer)));
- CHECK(memcmp(buffer, data, sizeof(data)) == 0);
-
- CHECK(decompr_zstd->free(d_state));
- fclose(f);
-}
-
-TEST_SUITE_END
projects / thirdparty / ccache.git / commitdiff
