compiler: gcc
version: "10"
- - os: ubuntu-18.04
- compiler: clang
- version: "5.0"
-
- - os: ubuntu-18.04
- compiler: clang
- version: "6.0"
-
- - os: ubuntu-18.04
- compiler: clang
- version: "7"
-
- os: ubuntu-18.04
compiler: clang
version: "8"
Hash.cpp
Logging.cpp
ProgressBar.cpp
- Result.cpp
- ResultExtractor.cpp
- ResultInspector.cpp
- ResultRetriever.cpp
SignalHandler.cpp
Stat.cpp
TemporaryFile.cpp
+++ /dev/null
-// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "Result.hpp"
-
-#include "AtomicFile.hpp"
-#include "Config.hpp"
-#include "Context.hpp"
-#include "Fd.hpp"
-#include "File.hpp"
-#include "Logging.hpp"
-#include "Stat.hpp"
-#include "Util.hpp"
-
-#include <ccache.hpp>
-#include <core/CacheEntryReader.hpp>
-#include <core/CacheEntryWriter.hpp>
-#include <core/FileReader.hpp>
-#include <core/FileWriter.hpp>
-#include <core/Statistic.hpp>
-#include <core/exceptions.hpp>
-#include <core/wincompat.hpp>
-#include <fmtmacros.hpp>
-#include <util/path.hpp>
-
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-
-#include <algorithm>
-
-// Result data format
-// ==================
-//
-// Integers are big-endian.
-//
-// <payload> ::= <format_ver> <n_entries> <entry>*
-// <format_ver> ::= uint8_t
-// <n_entries> ::= uint8_t
-// <entry> ::= <embedded_file_entry> | <raw_file_entry>
-// <embedded_file_entry> ::= <embedded_file_marker> <suffix_len> <suffix>
-// <data_len> <data>
-// <embedded_file_marker> ::= 0 (uint8_t)
-// <embedded_file_type> ::= uint8_t
-// <data_len> ::= uint64_t
-// <data> ::= data_len bytes
-// <raw_file_entry> ::= <raw_file_marker> <suffix_len> <suffix> <file_len>
-// <raw_file_marker> ::= 1 (uint8_t)
-// <file_len> ::= uint64_t
-// <epilogue> ::= <checksum>
-// <checksum> ::= uint64_t ; XXH3 of content bytes
-
-namespace {
-
-const uint8_t k_result_format_version = 0;
-
-// File data stored inside the result file.
-const uint8_t k_embedded_file_marker = 0;
-
-// File stored as-is in the file system.
-const uint8_t k_raw_file_marker = 1;
-
-const uint8_t k_max_raw_file_entries = 10;
-
-std::string
-get_raw_file_path(std::string_view result_path, uint8_t entry_number)
-{
- if (entry_number >= 10) {
- // To support more entries in the future, encode to [0-9a-z]. Note that
- // PrimaryStorage::evict currently assumes that the entry number is
- // represented as one character.
- throw core::Error(FMT("Too high raw file entry number: {}", entry_number));
- }
-
- const auto prefix = result_path.substr(
- 0, result_path.length() - Result::k_file_suffix.length());
- return FMT("{}{}W", prefix, entry_number);
-}
-
-bool
-should_store_raw_file(const Config& config, Result::FileType type)
-{
- if (!config.file_clone() && !config.hard_link()) {
- return false;
- }
-
- // Only store object files as raw files since there are several problems with
- // storing other file types:
- //
- // 1. The compiler unlinks object files before writing to them but it doesn't
- // unlink .d files, so just it's possible to corrupt .d files just by
- // running the compiler (see ccache issue 599).
- // 2. .d files cause trouble for automake if hard-linked (see ccache issue
- // 378).
- // 3. It's unknown how the compiler treats other file types, so better safe
- // than sorry.
- //
- // It would be possible to store all files in raw form for the file_clone case
- // and only hard link object files. However, most likely it's only object
- // files that become large enough that it's of interest to clone or hard link
- // them, so we keep things simple for now. This will also save i-nodes in the
- // cache.
- return type == Result::FileType::object;
-}
-
-} // namespace
-
-namespace Result {
-
-const std::string k_file_suffix = "R";
-const uint8_t k_magic[4] = {'c', 'C', 'r', 'S'};
-const uint8_t k_version = 1;
-const char* const k_unknown_file_type = "<unknown type>";
-
-const char*
-file_type_to_string(FileType type)
-{
- switch (type) {
- case FileType::object:
- return ".o";
-
- case FileType::dependency:
- return ".d";
-
- case FileType::stderr_output:
- return "<stderr>";
-
- case FileType::coverage_unmangled:
- return ".gcno-unmangled";
-
- case FileType::stackusage:
- return ".su";
-
- case FileType::diagnostic:
- return ".dia";
-
- case FileType::dwarf_object:
- return ".dwo";
-
- case FileType::coverage_mangled:
- return ".gcno-mangled";
-
- case FileType::stdout_output:
- return "<stdout>";
-
- case FileType::assembler_listing:
- return ".al";
- }
-
- return k_unknown_file_type;
-}
-
-std::string
-gcno_file_in_mangled_form(const Context& ctx)
-{
- const auto& output_obj = ctx.args_info.output_obj;
- const std::string abs_output_obj =
- util::is_absolute_path(output_obj)
- ? output_obj
- : FMT("{}/{}", ctx.apparent_cwd, output_obj);
- std::string hashified_obj = abs_output_obj;
- std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#');
- return Util::change_extension(hashified_obj, ".gcno");
-}
-
-std::string
-gcno_file_in_unmangled_form(const Context& ctx)
-{
- return Util::change_extension(ctx.args_info.output_obj, ".gcno");
-}
-
-FileSizeAndCountDiff&
-FileSizeAndCountDiff::operator+=(const FileSizeAndCountDiff& other)
-{
- size_kibibyte += other.size_kibibyte;
- count += other.count;
- return *this;
-}
-
-Result::Reader::Reader(core::CacheEntryReader& cache_entry_reader,
- const std::string& result_path)
- : m_reader(cache_entry_reader),
- m_result_path(result_path)
-{
-}
-
-void
-Reader::read(Consumer& consumer)
-{
- if (m_reader.header().entry_type != core::CacheEntryType::result) {
- throw core::Error(FMT("Unexpected cache entry type: {}",
- to_string(m_reader.header().entry_type)));
- }
-
- const auto result_format_version = m_reader.read_int<uint8_t>();
- if (result_format_version != k_result_format_version) {
- throw core::Error(
- FMT("Unknown result format version: {}", result_format_version));
- }
-
- const auto n_entries = m_reader.read_int<uint8_t>();
- if (n_entries >= k_max_raw_file_entries) {
- throw core::Error(FMT(
- "Too many raw file entries: {} > {}", n_entries, k_max_raw_file_entries));
- }
-
- uint8_t i;
- for (i = 0; i < n_entries; ++i) {
- read_entry(i, consumer);
- }
-
- if (i != n_entries) {
- throw core::Error(
- FMT("Too few entries (read {}, expected {})", i, n_entries));
- }
-
- m_reader.finalize();
-}
-
-void
-Reader::read_entry(uint8_t entry_number, Reader::Consumer& consumer)
-{
- const auto marker = m_reader.read_int<uint8_t>();
-
- switch (marker) {
- case k_embedded_file_marker:
- case k_raw_file_marker:
- break;
-
- default:
- throw core::Error(FMT("Unknown entry type: {}", marker));
- }
-
- const auto type = m_reader.read_int<UnderlyingFileTypeInt>();
- const auto file_type = FileType(type);
- const auto file_len = m_reader.read_int<uint64_t>();
-
- if (marker == k_embedded_file_marker) {
- consumer.on_entry_start(entry_number, file_type, file_len, std::nullopt);
-
- uint8_t buf[CCACHE_READ_BUFFER_SIZE];
- size_t remain = file_len;
- while (remain > 0) {
- size_t n = std::min(remain, sizeof(buf));
- m_reader.read(buf, n);
- consumer.on_entry_data(buf, n);
- remain -= n;
- }
- } else {
- ASSERT(marker == k_raw_file_marker);
-
- std::string raw_path;
- if (m_result_path != "-") {
- raw_path = get_raw_file_path(m_result_path, entry_number);
- auto st = Stat::stat(raw_path, Stat::OnError::throw_error);
- if (st.size() != file_len) {
- throw core::Error(
- FMT("Bad file size of {} (actual {} bytes, expected {} bytes)",
- raw_path,
- st.size(),
- file_len));
- }
- }
-
- consumer.on_entry_start(entry_number, file_type, file_len, raw_path);
- }
-
- consumer.on_entry_end();
-}
-
-Writer::Writer(const Config& config, const std::string& result_path)
- : m_config(config),
- m_result_path(result_path)
-{
-}
-
-void
-Writer::write_data(const FileType file_type, const std::string& data)
-{
- m_entries_to_write.push_back(Entry{file_type, ValueType::data, data});
-}
-
-void
-Writer::write_file(const FileType file_type, const std::string& path)
-{
- m_entries_to_write.push_back(Entry{file_type, ValueType::path, path});
-}
-
-nonstd::expected<FileSizeAndCountDiff, std::string>
-Writer::finalize()
-{
- try {
- return do_finalize();
- } catch (const core::Error& e) {
- return nonstd::make_unexpected(e.what());
- }
-}
-
-FileSizeAndCountDiff
-Writer::do_finalize()
-{
- FileSizeAndCountDiff file_size_and_count_diff{0, 0};
- uint64_t payload_size = 0;
- payload_size += 1; // format_ver
- payload_size += 1; // n_entries
- for (const auto& entry : m_entries_to_write) {
- payload_size += 1; // embedded_file_marker
- payload_size += 1; // embedded_file_type
- payload_size += 8; // data_len
- payload_size += // data
- entry.value_type == ValueType::data
- ? entry.value.size()
- : Stat::stat(entry.value, Stat::OnError::throw_error).size();
- }
-
- AtomicFile atomic_result_file(m_result_path, AtomicFile::Mode::binary);
- core::CacheEntryHeader header(core::CacheEntryType::result,
- compression::type_from_config(m_config),
- compression::level_from_config(m_config),
- time(nullptr),
- CCACHE_VERSION,
- m_config.namespace_());
- header.set_entry_size_from_payload_size(payload_size);
-
- core::FileWriter file_writer(atomic_result_file.stream());
- core::CacheEntryWriter writer(file_writer, header);
-
- writer.write_int(k_result_format_version);
- writer.write_int<uint8_t>(m_entries_to_write.size());
-
- uint8_t entry_number = 0;
- for (const auto& entry : m_entries_to_write) {
- const bool store_raw = entry.value_type == ValueType::path
- && should_store_raw_file(m_config, entry.file_type);
- const uint64_t entry_size =
- entry.value_type == ValueType::data
- ? entry.value.size()
- : Stat::stat(entry.value, Stat::OnError::throw_error).size();
-
- LOG("Storing {} entry #{} {} ({} bytes){}",
- store_raw ? "raw" : "embedded",
- entry_number,
- file_type_to_string(entry.file_type),
- entry_size,
- entry.value_type == ValueType::data ? ""
- : FMT(" from {}", entry.value));
-
- writer.write_int<uint8_t>(store_raw ? k_raw_file_marker
- : k_embedded_file_marker);
- writer.write_int(UnderlyingFileTypeInt(entry.file_type));
- writer.write_int(entry_size);
-
- if (store_raw) {
- file_size_and_count_diff +=
- write_raw_file_entry(entry.value, entry_number);
- } else if (entry.value_type == ValueType::data) {
- writer.write(entry.value.data(), entry.value.size());
- } else {
- write_embedded_file_entry(writer, entry.value, entry_size);
- }
-
- ++entry_number;
- }
-
- writer.finalize();
- atomic_result_file.commit();
-
- return file_size_and_count_diff;
-}
-
-void
-Result::Writer::write_embedded_file_entry(core::CacheEntryWriter& writer,
- const std::string& path,
- const uint64_t file_size)
-{
- Fd file(open(path.c_str(), O_RDONLY | O_BINARY));
- if (!file) {
- throw core::Error(FMT("Failed to open {} for reading", path));
- }
-
- uint64_t remain = file_size;
- while (remain > 0) {
- uint8_t buf[CCACHE_READ_BUFFER_SIZE];
- size_t n = std::min(remain, static_cast<uint64_t>(sizeof(buf)));
- auto bytes_read = read(*file, buf, n);
- if (bytes_read == -1) {
- if (errno == EINTR) {
- continue;
- }
- throw core::Error(
- FMT("Error reading from {}: {}", path, strerror(errno)));
- }
- if (bytes_read == 0) {
- throw core::Error(FMT("Error reading from {}: end of file", path));
- }
- writer.write(buf, bytes_read);
- remain -= bytes_read;
- }
-}
-
-FileSizeAndCountDiff
-Result::Writer::write_raw_file_entry(const std::string& path,
- uint8_t entry_number)
-{
- const auto raw_file = get_raw_file_path(m_result_path, entry_number);
- const auto old_stat = Stat::stat(raw_file);
- try {
- Util::clone_hard_link_or_copy_file(m_config, path, raw_file, true);
- } catch (core::Error& e) {
- throw core::Error(
- FMT("Failed to store {} as raw file {}: {}", path, raw_file, e.what()));
- }
- const auto new_stat = Stat::stat(raw_file);
- return {
- Util::size_change_kibibyte(old_stat, new_stat),
- (new_stat ? 1 : 0) - (old_stat ? 1 : 0),
- };
-}
-
-} // namespace Result
+++ /dev/null
-// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ResultExtractor.hpp"
-
-#include "Util.hpp"
-#include "fmtmacros.hpp"
-
-#include <core/exceptions.hpp>
-#include <core/wincompat.hpp>
-#include <fmtmacros.hpp>
-#include <util/file.hpp>
-
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-ResultExtractor::ResultExtractor(const std::string& directory)
- : m_directory(directory)
-{
-}
-
-void
-ResultExtractor::on_entry_start(uint8_t /*entry_number*/,
- Result::FileType file_type,
- uint64_t /*file_len*/,
- std::optional<std::string> raw_file)
-{
- std::string suffix = Result::file_type_to_string(file_type);
- if (suffix == Result::k_unknown_file_type) {
- suffix =
- FMT(".type_{}", static_cast<Result::UnderlyingFileTypeInt>(file_type));
- } else if (suffix[0] == '<') {
- suffix[0] = '.';
- suffix.resize(suffix.length() - 1);
- }
-
- m_dest_path = FMT("{}/ccache-result{}", m_directory, suffix);
-
- if (!raw_file) {
- m_dest_fd = Fd(
- open(m_dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
- if (!m_dest_fd) {
- throw core::Error(
- FMT("Failed to open {} for writing: {}", m_dest_path, strerror(errno)));
- }
- } else if (raw_file->empty()) {
- PRINT_RAW(stderr,
- "Note: Can't extract raw file since reading from stdin\n");
- } else {
- try {
- Util::copy_file(*raw_file, m_dest_path, false);
- } catch (core::Error& e) {
- throw core::Error(
- FMT("Failed to copy {} to {}: {}", *raw_file, m_dest_path, e.what()));
- }
- }
-}
-
-void
-ResultExtractor::on_entry_data(const uint8_t* data, size_t size)
-{
- ASSERT(m_dest_fd);
-
- const auto result = util::write_fd(*m_dest_fd, data, size);
- if (!result) {
- throw core::Error(
- FMT("Failed to write to {}: {}", m_dest_path, result.error()));
- }
-}
-
-void
-ResultExtractor::on_entry_end()
-{
- if (m_dest_fd) {
- m_dest_fd.close();
- }
-}
+++ /dev/null
-// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "ResultRetriever.hpp"
-
-#include "Context.hpp"
-#include "Depfile.hpp"
-#include "Logging.hpp"
-
-#include <core/exceptions.hpp>
-#include <core/wincompat.hpp>
-#include <fmtmacros.hpp>
-#include <util/file.hpp>
-
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-
-using Result::FileType;
-
-ResultRetriever::ResultRetriever(Context& ctx) : m_ctx(ctx)
-{
-}
-
-void
-ResultRetriever::on_entry_start(uint8_t entry_number,
- FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file)
-{
- LOG("Reading {} entry #{} {} ({} bytes)",
- raw_file ? "raw" : "embedded",
- entry_number,
- Result::file_type_to_string(file_type),
- file_len);
-
- std::string dest_path;
- m_dest_file_type = file_type;
-
- switch (file_type) {
- case FileType::object:
- dest_path = m_ctx.args_info.output_obj;
- break;
-
- case FileType::dependency:
- // Dependency file: Open destination file but accumulate data in m_dest_data
- // and write it in on_entry_end.
- if (m_ctx.args_info.generating_dependencies) {
- dest_path = m_ctx.args_info.output_dep;
- m_dest_data.reserve(file_len);
- }
- break;
-
- case FileType::stdout_output:
- case FileType::stderr_output:
- // Stdout/stderr data: Don't open a destination file. Instead accumulate it
- // in m_dest_data and write it in on_entry_end.
- m_dest_data.reserve(file_len);
- break;
-
- case FileType::coverage_unmangled:
- if (m_ctx.args_info.generating_coverage) {
- dest_path = Util::change_extension(m_ctx.args_info.output_obj, ".gcno");
- }
- break;
-
- case FileType::stackusage:
- if (m_ctx.args_info.generating_stackusage) {
- dest_path = m_ctx.args_info.output_su;
- }
- break;
-
- case FileType::diagnostic:
- if (m_ctx.args_info.generating_diagnostics) {
- dest_path = m_ctx.args_info.output_dia;
- }
- break;
-
- case FileType::dwarf_object:
- if (m_ctx.args_info.seen_split_dwarf
- && m_ctx.args_info.output_obj != "/dev/null") {
- dest_path = m_ctx.args_info.output_dwo;
- }
- break;
-
- case FileType::coverage_mangled:
- if (m_ctx.args_info.generating_coverage) {
- dest_path = Result::gcno_file_in_mangled_form(m_ctx);
- }
- break;
-
- case FileType::assembler_listing:
- dest_path = m_ctx.args_info.output_al;
- break;
- }
-
- if (file_type == FileType::stdout_output
- || file_type == FileType::stderr_output) {
- // Written in on_entry_end.
- } else if (dest_path.empty()) {
- LOG_RAW("Not writing");
- } else if (dest_path == "/dev/null") {
- LOG_RAW("Not writing to /dev/null");
- } else if (raw_file) {
- Util::clone_hard_link_or_copy_file(
- m_ctx.config, *raw_file, dest_path, false);
-
- // Update modification timestamp to save the file from LRU cleanup (and, if
- // hard-linked, to make the object file newer than the source file).
- util::set_timestamps(*raw_file);
- } else {
- LOG("Writing to {}", dest_path);
- m_dest_fd = Fd(
- open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
- if (!m_dest_fd) {
- throw WriteError(
- FMT("Failed to open {} for writing: {}", dest_path, strerror(errno)));
- }
- m_dest_path = dest_path;
- }
-}
-
-void
-ResultRetriever::on_entry_data(const uint8_t* data, size_t size)
-{
- ASSERT(!((m_dest_file_type == FileType::stdout_output
- || m_dest_file_type == FileType::stderr_output)
- && m_dest_fd));
-
- if (m_dest_file_type == FileType::stdout_output
- || m_dest_file_type == FileType::stderr_output
- || (m_dest_file_type == FileType::dependency && !m_dest_path.empty())) {
- m_dest_data.append(reinterpret_cast<const char*>(data), size);
- } else if (m_dest_fd) {
- const auto result = util::write_fd(*m_dest_fd, data, size);
- if (!result) {
- throw WriteError(
- FMT("Failed to write to {}: {}", m_dest_path, result.error()));
- }
- }
-}
-
-void
-ResultRetriever::on_entry_end()
-{
- if (m_dest_file_type == FileType::stdout_output) {
- LOG("Writing to file descriptor {}", STDOUT_FILENO);
- Util::send_to_fd(m_ctx, m_dest_data, STDOUT_FILENO);
- } else if (m_dest_file_type == FileType::stderr_output) {
- LOG("Writing to file descriptor {}", STDERR_FILENO);
- Util::send_to_fd(m_ctx, m_dest_data, STDERR_FILENO);
- } else if (m_dest_file_type == FileType::dependency && !m_dest_path.empty()) {
- write_dependency_file();
- }
-
- if (m_dest_fd) {
- m_dest_fd.close();
- }
- m_dest_path.clear();
- m_dest_data.clear();
-}
-
-void
-ResultRetriever::write_dependency_file()
-{
- ASSERT(m_ctx.args_info.dependency_target);
-
- auto write_data = [&](auto data, auto size) {
- const auto result = util::write_fd(*m_dest_fd, data, size);
- if (!result) {
- throw WriteError(
- FMT("Failed to write to {}: {}", m_dest_path, result.error()));
- }
- };
-
- size_t start_pos = 0;
- const size_t colon_pos = m_dest_data.find(": ");
- if (colon_pos != std::string::npos) {
- const auto obj_in_dep_file =
- std::string_view(m_dest_data).substr(0, colon_pos);
- const auto& dep_target = *m_ctx.args_info.dependency_target;
- if (obj_in_dep_file != dep_target) {
- write_data(dep_target.data(), dep_target.length());
- start_pos = colon_pos;
- }
- }
-
- write_data(m_dest_data.data() + start_pos, m_dest_data.length() - start_pos);
-}
#include "Hash.hpp"
#include "Logging.hpp"
#include "MiniTrace.hpp"
-#include "Result.hpp"
-#include "ResultRetriever.hpp"
#include "SignalHandler.hpp"
#include "TemporaryFile.hpp"
#include "UmaskScope.hpp"
#include <core/FileReader.hpp>
#include <core/FileWriter.hpp>
#include <core/Manifest.hpp>
+#include <core/Result.hpp>
+#include <core/ResultRetriever.hpp>
#include <core/Statistics.hpp>
#include <core/StatsLog.hpp>
#include <core/exceptions.hpp>
// (in CWD) if -fprofile-dir=DIR is present (regardless of DIR) instead of the
// traditional /dir/to/example.gcno.
- std::string mangled_form = Result::gcno_file_in_mangled_form(ctx);
- std::string unmangled_form = Result::gcno_file_in_unmangled_form(ctx);
+ std::string mangled_form = core::Result::gcno_file_in_mangled_form(ctx);
+ std::string unmangled_form = core::Result::gcno_file_in_unmangled_form(ctx);
std::string found_file;
if (Stat::stat(mangled_form)) {
LOG("Found coverage file {}", mangled_form);
const std::string& stdout_data,
const std::string& stderr_data)
{
- Result::Writer result_writer(ctx.config, result_path);
+ core::Result::Serializer serializer(ctx.config);
if (!stderr_data.empty()) {
- result_writer.write_data(Result::FileType::stderr_output, stderr_data);
+ serializer.add_data(core::Result::FileType::stderr_output, stderr_data);
}
// Write stdout only after stderr (better with MSVC), as ResultRetriever
// will later print process them in the order they are read.
if (!stdout_data.empty()) {
- result_writer.write_data(Result::FileType::stdout_output, stdout_data);
+ serializer.add_data(core::Result::FileType::stdout_output, stdout_data);
}
if (obj_stat) {
- result_writer.write_file(Result::FileType::object,
- ctx.args_info.output_obj);
+ serializer.add_file(core::Result::FileType::object,
+ ctx.args_info.output_obj);
}
if (ctx.args_info.generating_dependencies) {
- result_writer.write_file(Result::FileType::dependency,
- ctx.args_info.output_dep);
+ serializer.add_file(core::Result::FileType::dependency,
+ ctx.args_info.output_dep);
}
if (ctx.args_info.generating_coverage) {
const auto coverage_file = find_coverage_file(ctx);
if (!coverage_file.found) {
return false;
}
- result_writer.write_file(coverage_file.mangled
- ? Result::FileType::coverage_mangled
- : Result::FileType::coverage_unmangled,
- coverage_file.path);
+ serializer.add_file(coverage_file.mangled
+ ? core::Result::FileType::coverage_mangled
+ : core::Result::FileType::coverage_unmangled,
+ coverage_file.path);
}
if (ctx.args_info.generating_stackusage) {
- result_writer.write_file(Result::FileType::stackusage,
- ctx.args_info.output_su);
+ serializer.add_file(core::Result::FileType::stackusage,
+ ctx.args_info.output_su);
}
if (ctx.args_info.generating_diagnostics) {
- result_writer.write_file(Result::FileType::diagnostic,
- ctx.args_info.output_dia);
+ serializer.add_file(core::Result::FileType::diagnostic,
+ ctx.args_info.output_dia);
}
if (ctx.args_info.seen_split_dwarf && Stat::stat(ctx.args_info.output_dwo)) {
// Only store .dwo file if it was created by the compiler (GCC and Clang
// behave differently e.g. for "-gsplit-dwarf -g1").
- result_writer.write_file(Result::FileType::dwarf_object,
- ctx.args_info.output_dwo);
+ serializer.add_file(core::Result::FileType::dwarf_object,
+ ctx.args_info.output_dwo);
}
if (!ctx.args_info.output_al.empty()) {
- result_writer.write_file(Result::FileType::assembler_listing,
- ctx.args_info.output_al);
+ serializer.add_file(core::Result::FileType::assembler_listing,
+ ctx.args_info.output_al);
}
- const auto file_size_and_count_diff = result_writer.finalize();
- if (file_size_and_count_diff) {
+ AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary);
+ core::CacheEntryHeader header(core::CacheEntryType::result,
+ compression::type_from_config(ctx.config),
+ compression::level_from_config(ctx.config),
+ time(nullptr),
+ CCACHE_VERSION,
+ ctx.config.namespace_());
+ header.set_entry_size_from_payload_size(serializer.serialized_size());
+
+ core::FileWriter file_writer(atomic_result_file.stream());
+ core::CacheEntryWriter writer(file_writer, header);
+
+ std::vector<uint8_t> payload;
+ payload.reserve(serializer.serialized_size());
+ const auto serialize_result = serializer.serialize(payload);
+ for (auto [file_number, source_path] : serialize_result.raw_files) {
+ const auto dest_path = storage::primary::PrimaryStorage::get_raw_file_path(
+ result_path, file_number);
+ const auto old_stat = Stat::stat(dest_path);
+ try {
+ Util::clone_hard_link_or_copy_file(
+ ctx.config, source_path, dest_path, true);
+ } catch (core::Error& e) {
+ LOG("Failed to store {} as raw file {}: {}",
+ source_path,
+ dest_path,
+ e.what());
+ throw;
+ }
+ const auto new_stat = Stat::stat(dest_path);
ctx.storage.primary.increment_statistic(
- Statistic::cache_size_kibibyte, file_size_and_count_diff->size_kibibyte);
- ctx.storage.primary.increment_statistic(Statistic::files_in_cache,
- file_size_and_count_diff->count);
- } else {
- LOG("Error: {}", file_size_and_count_diff.error());
- return false;
+ Statistic::cache_size_kibibyte,
+ Util::size_change_kibibyte(old_stat, new_stat));
+ ctx.storage.primary.increment_statistic(
+ Statistic::files_in_cache, (new_stat ? 1 : 0) - (old_stat ? 1 : 0));
}
+ writer.write(payload.data(), payload.size());
+ writer.finalize();
+ atomic_result_file.commit();
+
return true;
}
MTR_BEGIN("result", "result_put");
const bool added = ctx.storage.put(
*result_key, core::CacheEntryType::result, [&](const auto& path) {
- return write_result(
- ctx, path, obj_stat, result->stdout_data, result->stderr_data);
+ try {
+ return write_result(
+ ctx, path, obj_stat, result->stdout_data, result->stderr_data);
+ } catch (core::Error& e) {
+ LOG("Error writing to {}: {}", path, e.what());
+ return false;
+ }
});
MTR_END("result", "result_put");
if (!added) {
bool found_ccbin = false;
hash.hash_delimiter("result version");
- hash.hash(Result::k_version);
+ hash.hash(core::Result::k_version);
if (direct_mode) {
hash.hash_delimiter("manifest version");
}
core::FileReader file_reader(file.get());
core::CacheEntryReader cache_entry_reader(file_reader);
- Result::Reader result_reader(cache_entry_reader, *result_path);
- ResultRetriever result_retriever(ctx);
-
- result_reader.read(result_retriever);
- } catch (ResultRetriever::WriteError& e) {
+ std::vector<uint8_t> payload;
+ payload.resize(cache_entry_reader.header().payload_size());
+ cache_entry_reader.read(payload.data(), payload.size());
+ core::Result::Deserializer deserializer(payload);
+ core::ResultRetriever result_retriever(ctx, result_path);
+ deserializer.visit(result_retriever);
+ cache_entry_reader.finalize();
+ } catch (core::ResultRetriever::WriteError& e) {
LOG(
"Write error when retrieving result from {}: {}", *result_path, e.what());
return nonstd::make_unexpected(Statistic::bad_output_file);
CacheEntryReader.cpp
CacheEntryWriter.cpp
Manifest.cpp
+ Result.cpp
+ ResultExtractor.cpp
+ ResultInspector.cpp
+ ResultRetriever.cpp
Statistics.cpp
StatisticsCounters.cpp
StatsLog.cpp
--- /dev/null
+// Copyright (C) 2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <Util.hpp>
+#include <core/exceptions.hpp>
+#include <fmtmacros.hpp>
+#include <util/string.hpp>
+
+#include <third_party/nonstd/span.hpp>
+
+#include <cstddef>
+#include <string_view>
+
+namespace core {
+
+class CacheEntryDataReader
+{
+public:
+ CacheEntryDataReader(nonstd::span<const uint8_t> data);
+
+ // Read `size` bytes. Throws `core::Error` on failure.
+ nonstd::span<const uint8_t> read_bytes(size_t size);
+
+ // Read a string of length `length`. Throws `core::Error` on failure.
+ std::string_view read_str(size_t length);
+
+ // Read an integer. Throws `core::Error` on failure.
+ template<typename T> T read_int();
+
+ // Read an integer into `value`. Throws `core::Error` on failure.
+ template<typename T> void read_int(T& value);
+
+private:
+ nonstd::span<const uint8_t> m_data;
+};
+
+inline CacheEntryDataReader::CacheEntryDataReader(
+ nonstd::span<const uint8_t> data)
+ : m_data(data)
+{
+}
+
+inline nonstd::span<const uint8_t>
+CacheEntryDataReader::read_bytes(size_t size)
+{
+ if (size > m_data.size()) {
+ throw core::Error(FMT("CacheEntryDataReader: data underflow of {} bytes",
+ size - m_data.size()));
+ }
+ const auto bytes = m_data.first(size);
+ m_data = m_data.subspan(size);
+ return bytes;
+}
+
+inline std::string_view
+CacheEntryDataReader::read_str(const size_t length)
+{
+ return util::to_string_view(read_bytes(length));
+}
+
+template<typename T>
+inline T
+CacheEntryDataReader::read_int()
+{
+ const auto buffer = read_bytes(sizeof(T));
+ T value;
+ Util::big_endian_to_int(buffer.data(), value);
+ return value;
+}
+
+template<typename T>
+inline void
+CacheEntryDataReader::read_int(T& value)
+{
+ value = read_int<T>();
+}
+
+} // namespace core
--- /dev/null
+// Copyright (C) 2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <Util.hpp>
+#include <core/exceptions.hpp>
+#include <fmtmacros.hpp>
+#include <util/string.hpp>
+
+#include <third_party/nonstd/span.hpp>
+
+#include <cstddef>
+#include <cstring>
+#include <string_view>
+#include <vector>
+
+namespace core {
+
+class CacheEntryDataWriter
+{
+public:
+ CacheEntryDataWriter(std::vector<uint8_t>& output);
+
+ // Write `data`. Throws `core::Error` on failure.
+ void write_bytes(nonstd::span<const uint8_t> data);
+
+ // Write `data`. Throws `core::Error` on failure.
+ void write_str(std::string_view data);
+
+ // Write integer `value`. Throws `core::Error` on failure.
+ template<typename T> void write_int(T value);
+
+private:
+ std::vector<uint8_t>& m_output;
+};
+
+inline CacheEntryDataWriter::CacheEntryDataWriter(std::vector<uint8_t>& output)
+ : m_output(output)
+{
+}
+
+inline void
+CacheEntryDataWriter::write_bytes(nonstd::span<const uint8_t> data)
+{
+ m_output.insert(m_output.end(), data.begin(), data.end());
+}
+
+template<typename T>
+inline void
+CacheEntryDataWriter::write_int(const T value)
+{
+ uint8_t buffer[sizeof(T)];
+ Util::int_to_big_endian(value, buffer);
+ write_bytes(buffer);
+}
+
+inline void
+CacheEntryDataWriter::write_str(std::string_view value)
+{
+ write_bytes(util::to_span(value));
+}
+
+} // namespace core
#include "CacheEntryHeader.hpp"
+#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
const size_t k_static_header_fields_size =
{
}
-uint64_t
+uint32_t
CacheEntryHeader::payload_size() const
{
- return entry_size - non_payload_size();
+ const auto payload_size = entry_size - non_payload_size();
+ // In order to support 32-bit ccache builds, restrict size to uint32_t for
+ // now. This restriction can be lifted when we drop 32-bit support.
+ const auto max = std::numeric_limits<uint32_t>::max();
+ if (payload_size > max) {
+ throw core::Error(
+ FMT("Serialized result too large ({} > {})", payload_size, max));
+ }
+
+ return payload_size;
}
void
std::string namespace_;
uint64_t entry_size;
- uint64_t payload_size() const;
+ uint32_t payload_size() const;
void set_entry_size_from_payload_size(uint64_t payload_size);
void inspect(FILE* stream) const;
--- /dev/null
+// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "Result.hpp"
+
+#include "Config.hpp"
+#include "Context.hpp"
+#include "Fd.hpp"
+#include "File.hpp"
+#include "Logging.hpp"
+#include "Stat.hpp"
+#include "Util.hpp"
+
+#include <ccache.hpp>
+#include <core/CacheEntryDataReader.hpp>
+#include <core/CacheEntryDataWriter.hpp>
+#include <core/Statistic.hpp>
+#include <core/exceptions.hpp>
+#include <core/wincompat.hpp>
+#include <fmtmacros.hpp>
+#include <util/file.hpp>
+#include <util/path.hpp>
+#include <util/string.hpp>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
+
+#include <algorithm>
+
+// Result data format
+// ==================
+//
+// Integers are big-endian.
+//
+// <payload> ::= <format_ver> <n_files> <file_entry>*
+// <format_ver> ::= uint8_t
+// <n_files> ::= uint8_t
+// <file_entry> ::= <embedded_file_entry> | <raw_file_entry>
+// <embedded_file_entry> ::= <embedded_file_marker> <file_type> <file_size>
+// <file_data>
+// <embedded_file_marker> ::= 0 (uint8_t)
+// <file_type> ::= uint8_t (see Result::FileType)
+// <file_size> ::= uint64_t
+// <file_data> ::= file_size bytes
+// <raw_file_entry> ::= <raw_file_marker> <file_type> <file_size>
+// <raw_file_marker> ::= 1 (uint8_t)
+// <file_size> ::= uint64_t
+
+namespace {
+
+const uint8_t k_result_format_version = 0;
+
+// File data stored inside the result file.
+const uint8_t k_embedded_file_marker = 0;
+
+// File stored as-is in the file system.
+const uint8_t k_raw_file_marker = 1;
+
+const uint8_t k_max_raw_file_entries = 10;
+
+bool
+should_store_raw_file(const Config& config, core::Result::FileType type)
+{
+ if (!config.file_clone() && !config.hard_link()) {
+ return false;
+ }
+
+ // Only store object files as raw files since there are several problems with
+ // storing other file types:
+ //
+ // 1. The compiler unlinks object files before writing to them but it doesn't
+ // unlink .d files, so just it's possible to corrupt .d files just by
+ // running the compiler (see ccache issue 599).
+ // 2. .d files cause trouble for automake if hard-linked (see ccache issue
+ // 378).
+ // 3. It's unknown how the compiler treats other file types, so better safe
+ // than sorry.
+ //
+ // It would be possible to store all files in raw form for the file_clone case
+ // and only hard link object files. However, most likely it's only object
+ // files that become large enough that it's of interest to clone or hard link
+ // them, so we keep things simple for now. This will also save i-nodes in the
+ // cache.
+ return type == core::Result::FileType::object;
+}
+
+} // namespace
+
+namespace core {
+
+namespace Result {
+
+const uint8_t k_version = 1;
+
+const char* const k_unknown_file_type = "<unknown type>";
+
+const char*
+file_type_to_string(FileType type)
+{
+ switch (type) {
+ case FileType::object:
+ return ".o";
+
+ case FileType::dependency:
+ return ".d";
+
+ case FileType::stderr_output:
+ return "<stderr>";
+
+ case FileType::coverage_unmangled:
+ return ".gcno-unmangled";
+
+ case FileType::stackusage:
+ return ".su";
+
+ case FileType::diagnostic:
+ return ".dia";
+
+ case FileType::dwarf_object:
+ return ".dwo";
+
+ case FileType::coverage_mangled:
+ return ".gcno-mangled";
+
+ case FileType::stdout_output:
+ return "<stdout>";
+
+ case FileType::assembler_listing:
+ return ".al";
+ }
+
+ return k_unknown_file_type;
+}
+
+std::string
+gcno_file_in_mangled_form(const Context& ctx)
+{
+ const auto& output_obj = ctx.args_info.output_obj;
+ const std::string abs_output_obj =
+ util::is_absolute_path(output_obj)
+ ? output_obj
+ : FMT("{}/{}", ctx.apparent_cwd, output_obj);
+ std::string hashified_obj = abs_output_obj;
+ std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#');
+ return Util::change_extension(hashified_obj, ".gcno");
+}
+
+std::string
+gcno_file_in_unmangled_form(const Context& ctx)
+{
+ return Util::change_extension(ctx.args_info.output_obj, ".gcno");
+}
+
+Deserializer::Deserializer(nonstd::span<const uint8_t> data) : m_data(data)
+{
+}
+
+void
+Deserializer::visit(Deserializer::Visitor& visitor) const
+{
+ CacheEntryDataReader reader(m_data);
+ const auto result_format_version = reader.read_int<uint8_t>();
+ if (result_format_version != k_result_format_version) {
+ throw Error(FMT("Unknown result format version: {} != {}",
+ result_format_version,
+ k_result_format_version));
+ }
+
+ const auto n_files = reader.read_int<uint8_t>();
+ if (n_files >= k_max_raw_file_entries) {
+ throw Error(FMT(
+ "Too many raw file entries: {} > {}", n_files, k_max_raw_file_entries));
+ }
+
+ uint8_t file_number;
+ for (file_number = 0; file_number < n_files; ++file_number) {
+ const auto marker = reader.read_int<uint8_t>();
+ switch (marker) {
+ case k_embedded_file_marker:
+ case k_raw_file_marker:
+ break;
+
+ default:
+ throw Error(FMT("Unknown entry type: {}", marker));
+ }
+
+ const auto type = reader.read_int<UnderlyingFileTypeInt>();
+ const auto file_type = FileType(type);
+ const auto file_size = reader.read_int<uint64_t>();
+
+ if (marker == k_embedded_file_marker) {
+ visitor.on_embedded_file(
+ file_number, file_type, reader.read_bytes(file_size));
+ } else {
+ ASSERT(marker == k_raw_file_marker);
+ visitor.on_raw_file(file_number, file_type, file_size);
+ }
+ }
+
+ if (file_number != n_files) {
+ throw Error(
+ FMT("Too few entries (read {}, expected {})", file_number, n_files));
+ }
+}
+
+Serializer::Serializer(const Config& config)
+ : m_config(config),
+ m_serialized_size(1 + 1) // format_ver + n_files
+{
+}
+
+void
+Serializer::add_data(const FileType file_type, std::string_view data)
+{
+ m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size
+ m_serialized_size += data.size();
+ m_file_entries.push_back(FileEntry{file_type, util::to_span(data)});
+}
+
+void
+Serializer::add_file(const FileType file_type, const std::string& path)
+{
+ m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size
+ if (!should_store_raw_file(m_config, file_type)) {
+ m_serialized_size += Stat::stat(path, Stat::OnError::throw_error).size();
+ }
+ m_file_entries.push_back(FileEntry{file_type, path});
+}
+
+uint32_t
+Serializer::serialized_size() const
+{
+ // In order to support 32-bit ccache builds, restrict size to uint32_t for
+ // now. This restriction can be lifted when we drop 32-bit support.
+ const auto max = std::numeric_limits<uint32_t>::max();
+ if (m_serialized_size > max) {
+ throw Error(
+ FMT("Serialized result too large ({} > {})", m_serialized_size, max));
+ }
+ return m_serialized_size;
+}
+
+Serializer::SerializeResult
+Serializer::serialize(std::vector<uint8_t>& output)
+{
+ SerializeResult serialize_result;
+ CacheEntryDataWriter writer(output);
+
+ writer.write_int(k_result_format_version);
+ writer.write_int<uint8_t>(m_file_entries.size());
+
+ uint8_t file_number = 0;
+ for (const auto& entry : m_file_entries) {
+ const bool is_file_entry = std::holds_alternative<std::string>(entry.data);
+ const bool store_raw =
+ is_file_entry && should_store_raw_file(m_config, entry.file_type);
+ const uint64_t file_size =
+ is_file_entry ? Stat::stat(std::get<std::string>(entry.data),
+ Stat::OnError::throw_error)
+ .size()
+ : std::get<nonstd::span<const uint8_t>>(entry.data).size();
+
+ LOG("Storing {} entry #{} {} ({} bytes){}",
+ store_raw ? "raw" : "embedded",
+ file_number,
+ file_type_to_string(entry.file_type),
+ file_size,
+ is_file_entry ? FMT(" from {}", std::get<std::string>(entry.data))
+ : "");
+
+ writer.write_int<uint8_t>(store_raw ? k_raw_file_marker
+ : k_embedded_file_marker);
+ writer.write_int(UnderlyingFileTypeInt(entry.file_type));
+ writer.write_int(file_size);
+
+ if (store_raw) {
+ serialize_result.raw_files.emplace(file_number,
+ std::get<std::string>(entry.data));
+ } else if (is_file_entry) {
+ const auto& path = std::get<std::string>(entry.data);
+ const auto data = util::read_file<std::vector<uint8_t>>(path);
+ if (!data) {
+ throw Error(FMT("Failed to read {}: {}", path, data.error()));
+ }
+ writer.write_bytes(*data);
+ } else {
+ writer.write_bytes(std::get<nonstd::span<const uint8_t>>(entry.data));
+ }
+
+ ++file_number;
+ }
+
+ return serialize_result;
+}
+
+} // namespace Result
+
+} // namespace core
#pragma once
-#include <core/Reader.hpp>
+#include <util/types.hpp>
-#include "third_party/nonstd/expected.hpp"
+#include <third_party/nonstd/span.hpp>
#include <cstdint>
-#include <map>
-#include <optional>
#include <string>
+#include <unordered_map>
+#include <variant>
#include <vector>
-namespace core {
-
-class CacheEntryReader;
-class CacheEntryWriter;
+class Config;
+class Context;
-} // namespace core
+namespace core {
-class Context;
+class CacheEntryDataParser;
namespace Result {
-extern const std::string k_file_suffix;
-extern const uint8_t k_magic[4];
extern const uint8_t k_version;
extern const char* const k_unknown_file_type;
std::string gcno_file_in_mangled_form(const Context& ctx);
std::string gcno_file_in_unmangled_form(const Context& ctx);
-struct FileSizeAndCountDiff
-{
- int64_t size_kibibyte;
- int64_t count;
-
- FileSizeAndCountDiff& operator+=(const FileSizeAndCountDiff& other);
-};
-
-// This class knows how to read a result cache entry.
-class Reader
+// This class knows how to deserializer a result cache entry.
+class Deserializer
{
public:
- Reader(core::CacheEntryReader& cache_entry_reader,
- const std::string& result_path);
+ // Read a result from `data`.
+ Deserializer(nonstd::span<const uint8_t> data);
- class Consumer
+ class Visitor
{
public:
- virtual ~Consumer() = default;
-
- virtual void on_entry_start(uint8_t entry_number,
- FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file) = 0;
- virtual void on_entry_data(const uint8_t* data, size_t size) = 0;
- virtual void on_entry_end() = 0;
+ virtual ~Visitor() = default;
+
+ virtual void on_embedded_file(uint8_t file_number,
+ FileType file_type,
+ nonstd::span<const uint8_t> data) = 0;
+ virtual void on_raw_file(uint8_t file_number,
+ FileType file_type,
+ uint64_t file_size) = 0;
};
// Throws core::Error on error.
- void read(Consumer& consumer);
+ void visit(Visitor& visitor) const;
private:
- core::CacheEntryReader& m_reader;
- const std::string m_result_path;
+ nonstd::span<const uint8_t> m_data;
- void read_entry(uint8_t entry_number, Reader::Consumer& consumer);
+ void parse_file_entry(CacheEntryDataParser& parser,
+ uint8_t file_number) const;
};
-// This class knows how to write a result cache entry.
-class Writer
+// This class knows how to serialize a result cache entry.
+class Serializer
{
public:
- Writer(const Config& config, const std::string& result_path);
+ Serializer(const Config& config);
- // Register content to include in the result. Does not throw.
- void write_data(FileType file_type, const std::string& data);
+ // Register data to include in the result. The data must live until
+ // serialize() has been called.
+ void add_data(FileType file_type, std::string_view data);
- // Register a file path whose content should be included in the result. Does
- // not throw.
- void write_file(FileType file_type, const std::string& path);
+ // Register a file path whose content should be included in the result.
+ void add_file(FileType file_type, const std::string& path);
- // Write registered entries to the result. Returns an error message on error.
- nonstd::expected<FileSizeAndCountDiff, std::string> finalize();
+ uint32_t serialized_size() const;
-private:
- enum class ValueType { data, path };
- struct Entry
+ struct SerializeResult
{
- FileType file_type;
- ValueType value_type;
- std::string value;
+ // Raw files to store in primary storage.
+ std::unordered_map<uint8_t /*index*/, std::string /*path*/> raw_files;
};
+ SerializeResult serialize(std::vector<uint8_t>& output);
+
+private:
const Config& m_config;
- const std::string m_result_path;
- std::vector<Entry> m_entries_to_write;
-
- FileSizeAndCountDiff do_finalize();
- static void write_embedded_file_entry(core::CacheEntryWriter& writer,
- const std::string& path,
- uint64_t file_size);
- FileSizeAndCountDiff write_raw_file_entry(const std::string& path,
- uint8_t entry_number);
+ uint64_t m_serialized_size;
+
+ struct FileEntry
+ {
+ FileType file_type;
+ std::variant<nonstd::span<const uint8_t>, std::string> data;
+ };
+ std::vector<FileEntry> m_file_entries;
};
} // namespace Result
+
+} // namespace core
--- /dev/null
+// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ResultExtractor.hpp"
+
+#include "Util.hpp"
+#include "fmtmacros.hpp"
+
+#include <core/exceptions.hpp>
+#include <core/wincompat.hpp>
+#include <fmtmacros.hpp>
+#include <util/file.hpp>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <vector>
+
+namespace core {
+
+ResultExtractor::ResultExtractor(
+ const std::string& output_directory,
+ std::optional<GetRawFilePathFunction> get_raw_file_path)
+ : m_output_directory(output_directory),
+ m_get_raw_file_path(get_raw_file_path)
+{
+}
+
+void
+ResultExtractor::on_embedded_file(uint8_t /*file_number*/,
+ Result::FileType file_type,
+ nonstd::span<const uint8_t> data)
+{
+ std::string suffix = Result::file_type_to_string(file_type);
+ if (suffix == Result::k_unknown_file_type) {
+ suffix =
+ FMT(".type_{}", static_cast<Result::UnderlyingFileTypeInt>(file_type));
+ } else if (suffix[0] == '<') {
+ suffix[0] = '.';
+ suffix.resize(suffix.length() - 1);
+ }
+
+ const auto dest_path = FMT("{}/ccache-result{}", m_output_directory, suffix);
+ const auto result = util::write_file(dest_path, data);
+ if (!result) {
+ throw Error(FMT("Failed to write to {}: {}", dest_path, result.error()));
+ }
+}
+
+void
+ResultExtractor::on_raw_file(uint8_t file_number,
+ Result::FileType file_type,
+ uint64_t file_size)
+{
+ if (!m_get_raw_file_path) {
+ throw Error("Raw entry for non-local result");
+ }
+ const auto raw_file_path = (*m_get_raw_file_path)(file_number);
+ const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error);
+ if (st.size() != file_size) {
+ throw Error(FMT("Bad file size of {} (actual {} bytes, expected {} bytes)",
+ raw_file_path,
+ st.size(),
+ file_size));
+ }
+
+ const auto data =
+ util::read_file<std::vector<uint8_t>>(raw_file_path, file_size);
+ if (!data) {
+ throw Error(FMT("Failed to read {}: {}", raw_file_path, data.error()));
+ }
+ on_embedded_file(file_number, file_type, *data);
+}
+
+} // namespace core
#pragma once
#include "Fd.hpp"
-#include "Result.hpp"
-class Context;
+#include <core/Result.hpp>
+
+#include <functional>
+#include <optional>
+#include <string>
+
+namespace core {
// This class extracts the parts of a result entry to a directory.
-class ResultExtractor : public Result::Reader::Consumer
+class ResultExtractor : public Result::Deserializer::Visitor
{
public:
- ResultExtractor(const std::string& directory);
+ using GetRawFilePathFunction = std::function<std::string(uint8_t)>;
- void on_entry_start(uint8_t entry_number,
- Result::FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file) override;
- void on_entry_data(const uint8_t* data, size_t size) override;
- void on_entry_end() override;
+ //`result_path` should be the path to the local result entry file if the
+ // result comes from primary storage.
+ ResultExtractor(
+ const std::string& output_directory,
+ std::optional<GetRawFilePathFunction> get_raw_file_path = std::nullopt);
+
+ void on_embedded_file(uint8_t file_number,
+ Result::FileType file_type,
+ nonstd::span<const uint8_t> data) override;
+ void on_raw_file(uint8_t file_number,
+ Result::FileType file_type,
+ uint64_t file_size) override;
private:
- const std::string m_directory;
- Fd m_dest_fd;
- std::string m_dest_path;
+ std::string m_output_directory;
+ std::optional<GetRawFilePathFunction> m_get_raw_file_path;
};
+
+} // namespace core
#include "Logging.hpp"
#include "fmtmacros.hpp"
+namespace core {
+
ResultInspector::ResultInspector(FILE* stream) : m_stream(stream)
{
}
void
-ResultInspector::on_entry_start(uint8_t entry_number,
- Result::FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file)
+ResultInspector::on_embedded_file(uint8_t file_number,
+ Result::FileType file_type,
+ nonstd::span<const uint8_t> data)
{
PRINT(m_stream,
- "{} file #{}: {} ({} bytes)\n",
- raw_file ? "Raw" : "Embedded",
- entry_number,
+ "Embedded file #{}: {} ({} bytes)\n",
+ file_number,
Result::file_type_to_string(file_type),
- file_len);
+ data.size());
}
void
-ResultInspector::on_entry_data(const uint8_t* /*data*/, size_t /*size*/)
+ResultInspector::on_raw_file(uint8_t file_number,
+ Result::FileType file_type,
+ uint64_t file_size)
{
+ PRINT(m_stream,
+ "Raw file #{}: {} ({} bytes)\n",
+ file_number,
+ Result::file_type_to_string(file_type),
+ file_size);
}
-void
-ResultInspector::on_entry_end()
-{
-}
+} // namespace core
#pragma once
-#include "Result.hpp"
+#include <core/Result.hpp>
#include <cstdint>
#include <cstdio>
+namespace core {
+
// This class writes information about the result entry to `stream`.
-class ResultInspector : public Result::Reader::Consumer
+class ResultInspector : public Result::Deserializer::Visitor
{
public:
ResultInspector(FILE* stream);
- void on_entry_start(uint8_t entry_number,
- Result::FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file) override;
- void on_entry_data(const uint8_t* data, size_t size) override;
- void on_entry_end() override;
+ void on_embedded_file(uint8_t file_number,
+ Result::FileType file_type,
+ nonstd::span<const uint8_t> data) override;
+ void on_raw_file(uint8_t file_number,
+ Result::FileType file_type,
+ uint64_t file_size) override;
private:
FILE* m_stream;
};
+
+} // namespace core
--- /dev/null
+// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ResultRetriever.hpp"
+
+#include "Context.hpp"
+#include "Depfile.hpp"
+#include "Logging.hpp"
+
+#include <Context.hpp>
+#include <Stat.hpp>
+#include <core/exceptions.hpp>
+#include <core/wincompat.hpp>
+#include <fmtmacros.hpp>
+#include <util/file.hpp>
+#include <util/string.hpp>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
+
+namespace core {
+
+using Result::FileType;
+
+ResultRetriever::ResultRetriever(const Context& ctx,
+ std::optional<std::string> path)
+ : m_ctx(ctx),
+ m_path(path)
+{
+}
+
+void
+ResultRetriever::on_embedded_file(uint8_t file_number,
+ FileType file_type,
+ nonstd::span<const uint8_t> data)
+{
+ LOG("Reading embedded entry #{} {} ({} bytes)",
+ file_number,
+ Result::file_type_to_string(file_type),
+ data.size());
+
+ if (file_type == FileType::stdout_output) {
+ Util::send_to_fd(m_ctx, util::to_string_view(data), STDOUT_FILENO);
+ } else if (file_type == FileType::stderr_output) {
+ Util::send_to_fd(m_ctx, util::to_string_view(data), STDERR_FILENO);
+ } else {
+ const auto dest_path = get_dest_path(file_type);
+ if (dest_path.empty()) {
+ LOG_RAW("Not writing");
+ } else if (dest_path == "/dev/null") {
+ LOG_RAW("Not writing to /dev/null");
+ } else {
+ LOG("Writing to {}", dest_path);
+ if (file_type == FileType::dependency) {
+ write_dependency_file(dest_path, data);
+ } else {
+ const auto result = util::write_file(dest_path, data);
+ if (!result) {
+ throw WriteError(
+ FMT("Failed to write to {}: {}", dest_path, result.error()));
+ }
+ }
+ }
+ }
+}
+
+void
+ResultRetriever::on_raw_file(uint8_t file_number,
+ FileType file_type,
+ uint64_t file_size)
+{
+ LOG("Reading raw entry #{} {} ({} bytes)",
+ file_number,
+ Result::file_type_to_string(file_type),
+ file_size);
+
+ if (!m_path) {
+ throw core::Error("Raw entry for non-local result");
+ }
+ const auto raw_file_path =
+ storage::primary::PrimaryStorage::get_raw_file_path(*m_path, file_number);
+ const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error);
+ if (st.size() != file_size) {
+ throw core::Error(
+ FMT("Bad file size of {} (actual {} bytes, expected {} bytes)",
+ raw_file_path,
+ st.size(),
+ file_size));
+ }
+
+ const auto dest_path = get_dest_path(file_type);
+ if (!dest_path.empty()) {
+ Util::clone_hard_link_or_copy_file(
+ m_ctx.config, raw_file_path, dest_path, false);
+
+ // Update modification timestamp to save the file from LRU cleanup (and, if
+ // hard-linked, to make the object file newer than the source file).
+ util::set_timestamps(raw_file_path);
+ } else {
+ // Should never happen.
+ LOG("Did not copy {} since destination path is unknown for type {}",
+ raw_file_path,
+ static_cast<Result::UnderlyingFileTypeInt>(file_type));
+ }
+}
+
+std::string
+ResultRetriever::get_dest_path(FileType file_type) const
+{
+ switch (file_type) {
+ case FileType::object:
+ return m_ctx.args_info.output_obj;
+
+ case FileType::dependency:
+ if (m_ctx.args_info.generating_dependencies) {
+ return m_ctx.args_info.output_dep;
+ }
+ break;
+
+ case FileType::stdout_output:
+ case FileType::stderr_output:
+ // Should never get here.
+ break;
+
+ case FileType::coverage_unmangled:
+ if (m_ctx.args_info.generating_coverage) {
+ return Util::change_extension(m_ctx.args_info.output_obj, ".gcno");
+ }
+ break;
+
+ case FileType::stackusage:
+ if (m_ctx.args_info.generating_stackusage) {
+ return m_ctx.args_info.output_su;
+ }
+ break;
+
+ case FileType::diagnostic:
+ if (m_ctx.args_info.generating_diagnostics) {
+ return m_ctx.args_info.output_dia;
+ }
+ break;
+
+ case FileType::dwarf_object:
+ if (m_ctx.args_info.seen_split_dwarf
+ && m_ctx.args_info.output_obj != "/dev/null") {
+ return m_ctx.args_info.output_dwo;
+ }
+ break;
+
+ case FileType::coverage_mangled:
+ if (m_ctx.args_info.generating_coverage) {
+ return Result::gcno_file_in_mangled_form(m_ctx);
+ }
+ break;
+
+ case FileType::assembler_listing:
+ return m_ctx.args_info.output_al;
+ }
+
+ return {};
+}
+
+void
+ResultRetriever::write_dependency_file(const std::string& path,
+ nonstd::span<const uint8_t> data)
+{
+ ASSERT(m_ctx.args_info.dependency_target);
+
+ Fd fd(open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
+ if (!fd) {
+ throw WriteError(FMT("Failed to open {} for writing", path));
+ }
+
+ auto write_data = [&](auto data, auto size) {
+ const auto result = util::write_fd(*fd, data, size);
+ if (!result) {
+ throw WriteError(FMT("Failed to write to {}: {}", path, result.error()));
+ }
+ };
+
+ std::string_view str_data = util::to_string_view(data);
+ size_t start_pos = 0;
+ const size_t colon_pos = str_data.find(": ");
+ if (colon_pos != std::string::npos) {
+ const auto obj_in_dep_file = str_data.substr(0, colon_pos);
+ const auto& dep_target = *m_ctx.args_info.dependency_target;
+ if (obj_in_dep_file != dep_target) {
+ write_data(dep_target.data(), dep_target.length());
+ start_pos = colon_pos;
+ }
+ }
+
+ write_data(str_data.data() + start_pos, str_data.length() - start_pos);
+}
+
+} // namespace core
#pragma once
#include "Fd.hpp"
-#include "Result.hpp"
+#include <core/Result.hpp>
#include <core/exceptions.hpp>
+#include <optional>
+
class Context;
+namespace core {
+
// This class retrieves a result entry to the local file system.
-class ResultRetriever : public Result::Reader::Consumer
+class ResultRetriever : public Result::Deserializer::Visitor
{
public:
- class WriteError : public core::Error
+ class WriteError : public Error
{
- using core::Error::Error;
+ using Error::Error;
};
- ResultRetriever(Context& ctx);
+ //`path` should be the path to the local result entry file if the result comes
+ // from primary storage.
+ ResultRetriever(const Context& ctx,
+ std::optional<std::string> path = std::nullopt);
- void on_entry_start(uint8_t entry_number,
- Result::FileType file_type,
- uint64_t file_len,
- std::optional<std::string> raw_file) override;
- void on_entry_data(const uint8_t* data, size_t size) override;
- void on_entry_end() override;
+ void on_embedded_file(uint8_t file_number,
+ Result::FileType file_type,
+ nonstd::span<const uint8_t> data) override;
+ void on_raw_file(uint8_t file_number,
+ Result::FileType file_type,
+ uint64_t file_size) override;
private:
- Context& m_ctx;
- Result::FileType m_dest_file_type{};
- Fd m_dest_fd;
- std::string m_dest_path;
-
- // Collects the full data of stderr output (since we want to potentially strip
- // color codes which could span chunk boundaries) or dependency data (since we
- // potentially want to rewrite the dependency target which in theory can span
- // a chunk boundary).
- std::string m_dest_data;
-
- void write_dependency_file();
+ const Context& m_ctx;
+ std::optional<std::string> m_path;
+
+ std::string get_dest_path(Result::FileType file_type) const;
+
+ void write_dependency_file(const std::string& path,
+ nonstd::span<const uint8_t> data);
};
+
+} // namespace core
#include <Hash.hpp>
#include <InodeCache.hpp>
#include <ProgressBar.hpp>
-#include <Result.hpp>
-#include <ResultExtractor.hpp>
-#include <ResultInspector.hpp>
#include <ccache.hpp>
#include <core/CacheEntryReader.hpp>
#include <core/FileReader.hpp>
#include <core/Manifest.hpp>
+#include <core/Result.hpp>
+#include <core/ResultExtractor.hpp>
+#include <core/ResultInspector.hpp>
#include <core/Statistics.hpp>
#include <core/StatsLog.hpp>
#include <core/exceptions.hpp>
case core::CacheEntryType::manifest: {
core::Manifest manifest;
manifest.read(cache_entry_reader);
- cache_entry_reader.finalize();
manifest.dump(stdout);
break;
}
case core::CacheEntryType::result:
- Result::Reader result_reader(cache_entry_reader, path);
+ std::vector<uint8_t> data;
+ data.resize(cache_entry_reader.header().payload_size());
+ cache_entry_reader.read(data.data(), data.size());
+ Result::Deserializer result_deserializer(data);
ResultInspector result_inspector(stdout);
- result_reader.read(result_inspector);
+ result_deserializer.visit(result_inspector);
break;
}
+ cache_entry_reader.finalize();
+
return EXIT_SUCCESS;
}
}
case EXTRACT_RESULT: {
- ResultExtractor result_extractor(".");
File file = arg == "-" ? File(stdin) : File(arg, "rb");
if (!file) {
PRINT(stderr, "Error: Failed to open \"{}\"", arg);
return EXIT_FAILURE;
}
+ std::optional<ResultExtractor::GetRawFilePathFunction> get_raw_file_path;
+ if (arg == "-") {
+ get_raw_file_path = [&](uint8_t file_number) {
+ return storage::primary::PrimaryStorage::get_raw_file_path(
+ arg, file_number);
+ };
+ }
+ ResultExtractor result_extractor(".", get_raw_file_path);
core::FileReader file_reader(file.get());
core::CacheEntryReader cache_entry_reader(file_reader);
- Result::Reader result_reader(cache_entry_reader, arg);
- result_reader.read(result_extractor);
+ std::vector<uint8_t> data;
+ data.resize(cache_entry_reader.header().payload_size());
+ cache_entry_reader.read(data.data(), data.size());
+ Result::Deserializer result_deserializer(data);
+ result_deserializer.visit(result_extractor);
+ cache_entry_reader.finalize();
return EXIT_SUCCESS;
}
-// Copyright (C) 2019-2021 Joel Rosdahl and other contributors
+// Copyright (C) 2019-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include "CacheFile.hpp"
-#include <Result.hpp>
#include <core/Manifest.hpp>
+#include <core/Result.hpp>
#include <util/string.hpp>
const Stat&
{
if (util::ends_with(m_path, "M")) {
return Type::manifest;
- } else if (util::ends_with(m_path, Result::k_file_suffix)) {
+ } else if (util::ends_with(m_path, "R")) {
return Type::result;
} else if (util::ends_with(m_path, "W")) {
return Type::raw;
}
}
+std::string
+PrimaryStorage::get_raw_file_path(std::string_view result_path,
+ uint8_t file_number)
+{
+ if (file_number >= 10) {
+ // To support more entries in the future, encode to [0-9a-z]. Note that
+ // PrimaryStorage::evict currently assumes that the entry number is
+ // represented as one character.
+ throw core::Error(FMT("Too high raw file entry number: {}", file_number));
+ }
+
+ const auto prefix = result_path.substr(0, result_path.length() - 1);
+ return FMT("{}{}W", prefix, file_number);
+}
+
void
PrimaryStorage::increment_statistic(const Statistic statistic,
const int64_t value)
void remove(const Digest& key, core::CacheEntryType type);
+ static std::string get_raw_file_path(std::string_view result_path,
+ uint8_t file_number);
+
// --- Statistics ---
void increment_statistic(core::Statistic statistic, int64_t value = 1);
#include <Context.hpp>
#include <File.hpp>
#include <Logging.hpp>
-#include <Result.hpp>
#include <ThreadPool.hpp>
#include <assertions.hpp>
#include <compression/ZstdCompressor.hpp>
#include <core/FileReader.hpp>
#include <core/FileWriter.hpp>
#include <core/Manifest.hpp>
+#include <core/Result.hpp>
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
expect_stat files_in_cache 2
result_file=$(find $CCACHE_DIR -name '*R')
- printf foo | dd of=$result_file bs=3 count=1 seek=20 conv=notrunc >&/dev/null
+ # Write BAD at byte 300.
+ printf BAD | dd of=$result_file bs=3 count=1 seek=100 conv=notrunc >&/dev/null
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
projects / thirdparty / ccache.git / commitdiff
