# include "InodeCache.hpp"
#endif
+#include <core/Manifest.hpp>
#include <storage/Storage.hpp>
#include <ctime>
// Storage (fronting primary and secondary storage backends).
storage::Storage storage;
+ // Direct mode manifest.
+ core::Manifest manifest;
+
#ifdef INODE_CACHE_SUPPORTED
// InodeCache that caches source file hashes when enabled.
mutable InodeCache inode_cache;
return DoExecuteResult{status, stdout_data, *stderr_data_result};
}
-static core::Manifest
-read_manifest(const std::string& path)
+static void
+read_manifest(Context& ctx, nonstd::span<const uint8_t> cache_entry_data)
{
- core::Manifest manifest;
try {
- const auto cache_entry_data =
- util::value_or_throw<core::Error>(util::read_file<util::Bytes>(path));
core::CacheEntry cache_entry(cache_entry_data);
cache_entry.verify_checksum();
- manifest.read(cache_entry.payload());
+ ctx.manifest.read(cache_entry.payload());
} catch (const core::Error& e) {
- LOG("Error reading {}: {}", path, e.what());
+ LOG("Error reading manifest: {}", e.what());
}
- return manifest;
}
static void
-save_manifest(const Config& config,
- core::Manifest& manifest,
- const std::string& path)
-{
- core::CacheEntry::Header header(config, core::CacheEntryType::manifest);
- const auto cache_entry_data = core::CacheEntry::serialize(header, manifest);
- AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary);
- atomic_manifest_file.write(cache_entry_data);
- atomic_manifest_file.commit();
-}
-
-// Create or update the manifest file.
-static void
-update_manifest_file(Context& ctx,
- const Digest& manifest_key,
- const Digest& result_key)
+update_manifest(Context& ctx,
+ const Digest& manifest_key,
+ const Digest& result_key)
{
if (ctx.config.read_only() || ctx.config.read_only_direct()) {
return;
(ctx.config.sloppiness().is_enabled(core::Sloppy::file_stat_matches))
|| ctx.args_info.output_is_precompiled_header;
- ctx.storage.put(
- manifest_key, core::CacheEntryType::manifest, [&](const auto& path) {
- LOG("Adding result key to {}", path);
- try {
- auto manifest = read_manifest(path);
- const bool added = manifest.add_result(result_key,
- ctx.included_files,
- ctx.time_of_compilation,
- save_timestamp);
- if (added) {
- save_manifest(ctx.config, manifest, path);
- }
- return added;
- } catch (const core::Error& e) {
- LOG("Failed to add result key to {}: {}", path, e.what());
- return false;
- }
- });
+ const bool added = ctx.manifest.add_result(
+ result_key, ctx.included_files, ctx.time_of_compilation, save_timestamp);
+ if (added) {
+ core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
+ ctx.storage.put(manifest_key,
+ core::CacheEntryType::manifest,
+ core::CacheEntry::serialize(header, ctx.manifest));
+ LOG("Added result key to manifest {}", manifest_key.to_string());
+ } else {
+ LOG("Did not add result key to manifest {}", manifest_key.to_string());
+ }
}
struct FindCoverageFileResult
static bool
write_result(Context& ctx,
- const std::string& result_path,
+ const Digest& result_key,
const Stat& obj_stat,
const std::string& stdout_data,
const std::string& stderr_data)
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result);
const auto cache_entry_data = core::CacheEntry::serialize(header, serializer);
-
const auto raw_files = serializer.get_raw_files();
if (!raw_files.empty()) {
- Util::ensure_dir_exists(Util::dir_name(result_path));
+ ctx.storage.primary.put_raw_files(result_key, raw_files);
}
- for (auto [file_number, source_path] : raw_files) {
- const auto dest_path = storage::primary::PrimaryStorage::get_raw_file_path(
- result_path, file_number);
- const auto old_stat = Stat::stat(dest_path);
- try {
- Util::clone_hard_link_or_copy_file(
- ctx.config, source_path, dest_path, true);
- } catch (core::Error& e) {
- LOG("Failed to store {} as raw file {}: {}",
- source_path,
- dest_path,
- e.what());
- throw;
- }
- const auto new_stat = Stat::stat(dest_path);
- ctx.storage.primary.increment_statistic(
- Statistic::cache_size_kibibyte,
- Util::size_change_kibibyte(old_stat, new_stat));
- ctx.storage.primary.increment_statistic(
- Statistic::files_in_cache, (new_stat ? 1 : 0) - (old_stat ? 1 : 0));
- }
-
- AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary);
- atomic_result_file.write(cache_entry_data);
- atomic_result_file.commit();
+ ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data);
return true;
}
}
MTR_BEGIN("result", "result_put");
- const bool added = ctx.storage.put(
- *result_key, core::CacheEntryType::result, [&](const auto& path) {
- try {
- return write_result(
- ctx, path, obj_stat, result->stdout_data, result->stderr_data);
- } catch (core::Error& e) {
- LOG("Error writing to {}: {}", path, e.what());
- return false;
- }
- });
+ write_result(
+ ctx, *result_key, obj_stat, result->stdout_data, result->stderr_data);
MTR_END("result", "result_put");
- if (!added) {
- return nonstd::make_unexpected(Statistic::internal_error);
- }
// Everything OK.
Util::send_to_fd(ctx, result->stderr_data, STDERR_FILENO);
manifest_key = hash.digest();
- auto manifest_path = ctx.storage.get(*manifest_key,
- core::CacheEntryType::manifest,
- storage::Storage::Mode::primary_only);
+ auto cache_entry_data = ctx.storage.get(*manifest_key,
+ core::CacheEntryType::manifest,
+ storage::Storage::Mode::primary_only);
- if (manifest_path) {
- LOG("Looking for result key in {}", *manifest_path);
+ if (cache_entry_data) {
MTR_BEGIN("manifest", "manifest_get");
try {
- const auto manifest = read_manifest(*manifest_path);
- result_key = manifest.look_up_result_digest(ctx);
+ read_manifest(ctx, *cache_entry_data);
+ result_key = ctx.manifest.look_up_result_digest(ctx);
} catch (const core::Error& e) {
- LOG("Failed to look up result key in {}: {}", *manifest_path, e.what());
+ LOG("Failed to look up result key in manifest: {}", e.what());
}
MTR_END("manifest", "manifest_get");
if (result_key) {
}
// Check secondary storage if not found in primary
if (!result_key) {
- manifest_path = ctx.storage.get(*manifest_key,
- core::CacheEntryType::manifest,
- storage::Storage::Mode::secondary_only);
- if (manifest_path) {
- LOG("Looking for result key in fetched secondary manifest {}",
- *manifest_path);
+ cache_entry_data = ctx.storage.get(*manifest_key,
+ core::CacheEntryType::manifest,
+ storage::Storage::Mode::secondary_only);
+ if (cache_entry_data) {
+ LOG_RAW("Looking for result key in fetched secondary manifest");
MTR_BEGIN("manifest", "secondary_manifest_get");
try {
- const auto manifest = read_manifest(*manifest_path);
- result_key = manifest.look_up_result_digest(ctx);
+ read_manifest(ctx, *cache_entry_data);
+ result_key = ctx.manifest.look_up_result_digest(ctx);
} catch (const core::Error& e) {
- LOG("Failed to look up result key in {}: {}", *manifest_path, e.what());
+ LOG("Failed to look up result key in manifest: {}", e.what());
}
MTR_END("manifest", "secondary_manifest_get");
if (result_key) {
MTR_SCOPE("cache", "from_cache");
// Get result from cache.
- const auto result_path =
+ const auto cache_entry_data =
ctx.storage.get(result_key, core::CacheEntryType::result);
- if (!result_path) {
+ if (!cache_entry_data) {
return false;
}
try {
- const auto cache_entry_data = util::value_or_throw<core::Error>(
- util::read_file<util::Bytes>(*result_path),
- FMT("Failed to read {}: ", *result_path));
-
- core::CacheEntry cache_entry(cache_entry_data);
+ core::CacheEntry cache_entry(*cache_entry_data);
cache_entry.verify_checksum();
core::Result::Deserializer deserializer(cache_entry.payload());
- core::ResultRetriever result_retriever(ctx, result_path);
+ core::ResultRetriever result_retriever(ctx, result_key);
deserializer.visit(result_retriever);
} catch (core::ResultRetriever::WriteError& e) {
- LOG(
- "Write error when retrieving result from {}: {}", *result_path, e.what());
+ LOG("Write error when retrieving result from {}: {}",
+ result_key.to_string(),
+ e.what());
return nonstd::make_unexpected(Statistic::bad_output_file);
} catch (core::Error& e) {
- LOG("Failed to get result from {}: {}", *result_path, e.what());
+ LOG("Failed to get result from {}: {}", result_key.to_string(), e.what());
return false;
}
return nonstd::make_unexpected(from_cache_result.error());
} else if (*from_cache_result) {
if (ctx.config.direct_mode() && manifest_key && put_result_in_manifest) {
- update_manifest_file(ctx, *manifest_key, *result_key);
+ MTR_SCOPE("cache", "update_manifest");
+ update_manifest(ctx, *manifest_key, *result_key);
}
return Statistic::preprocessed_cache_hit;
}
if (ctx.config.direct_mode()) {
ASSERT(manifest_key);
MTR_SCOPE("cache", "update_manifest");
- update_manifest_file(ctx, *manifest_key, *result_key);
+ update_manifest(ctx, *manifest_key, *result_key);
}
return ctx.config.recache() ? Statistic::recache : Statistic::cache_miss;
using Result::FileType;
ResultRetriever::ResultRetriever(const Context& ctx,
- std::optional<std::string> path)
+ std::optional<Digest> result_key)
: m_ctx(ctx),
- m_path(path)
+ m_result_key(result_key)
{
}
Result::file_type_to_string(file_type),
file_size);
- if (!m_path) {
+ if (!m_result_key) {
throw core::Error("Raw entry for non-local result");
}
const auto raw_file_path =
- storage::primary::PrimaryStorage::get_raw_file_path(*m_path, file_number);
+ m_ctx.storage.primary.get_raw_file_path(*m_result_key, file_number);
const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error);
if (st.size() != file_size) {
throw core::Error(
#include "Fd.hpp"
+#include <Digest.hpp>
#include <core/Result.hpp>
#include <core/exceptions.hpp>
//`path` should be the path to the local result entry file if the result comes
// from primary storage.
ResultRetriever(const Context& ctx,
- std::optional<std::string> path = std::nullopt);
+ std::optional<Digest> result_key = std::nullopt);
void on_embedded_file(uint8_t file_number,
Result::FileType file_type,
private:
const Context& m_ctx;
- std::optional<std::string> m_path;
+ std::optional<Digest> m_result_key;
std::string get_dest_path(Result::FileType file_type) const;
#include <third_party/url.hpp>
#include <cmath>
+#include <memory>
+#include <string>
#include <unordered_map>
#include <vector>
{
}
+// Define the destructor in the implementation file to avoid having to declare
+// SecondaryStorageEntry and its constituents in the header file.
+// NOLINTNEXTLINE(modernize-use-equals-default)
Storage::~Storage()
{
- for (const auto& tmp_file : m_tmp_files) {
- Util::unlink_tmp(tmp_file);
- }
}
void
primary.finalize();
}
-std::optional<std::string>
+std::optional<util::Bytes>
Storage::get(const Digest& key,
const core::CacheEntryType type,
const Mode mode)
MTR_SCOPE("storage", "get");
if (mode != Mode::secondary_only) {
- auto path = primary.get(key, type);
- primary.increment_statistic(path ? core::Statistic::primary_storage_hit
- : core::Statistic::primary_storage_miss);
- if (path) {
+ auto value = primary.get(key, type);
+ primary.increment_statistic(value ? core::Statistic::primary_storage_hit
+ : core::Statistic::primary_storage_miss);
+ if (value) {
if (m_config.reshare()) {
- // Temporary optimization until primary storage API has been refactored
- // to pass data via memory instead of files.
- const bool should_put_in_secondary_storage = std::any_of(
- m_secondary_storages.begin(),
- m_secondary_storages.end(),
- [](const auto& entry) { return !entry->config.read_only; });
- if (should_put_in_secondary_storage) {
- const auto value = util::read_file<util::Bytes>(*path);
- if (!value) {
- LOG("Failed to read {}: {}", *path, value.error());
- return path; // Don't indicate failure since primary storage was OK.
- }
- put_in_secondary_storage(key, *value, true);
- }
+ put_in_secondary_storage(key, *value, true);
}
-
- return path;
+ return value;
}
}
return std::nullopt;
}
- const auto value = get_from_secondary_storage(key);
- if (!value) {
- return std::nullopt;
+ auto value = get_from_secondary_storage(key);
+ if (value) {
+ primary.put(key, type, *value);
}
-
- TemporaryFile tmp_file(FMT("{}/tmp.get", m_config.temporary_dir()));
- m_tmp_files.push_back(tmp_file.path);
- try {
- util::write_file(tmp_file.path, *value);
- } catch (const core::Error& e) {
- throw core::Fatal(FMT("Error writing to {}: {}", tmp_file.path, e.what()));
- }
-
- primary.put(key, type, [&](const auto& path) {
- try {
- Util::ensure_dir_exists(Util::dir_name(path));
- Util::copy_file(tmp_file.path, path);
- } catch (const core::Error& e) {
- LOG("Failed to copy {} to {}: {}", tmp_file.path, path, e.what());
- // Don't indicate failure since get from primary storage was OK.
- }
- return true;
- });
-
- return tmp_file.path;
+ return value;
}
-bool
+void
Storage::put(const Digest& key,
const core::CacheEntryType type,
- const storage::EntryWriter& entry_writer)
+ nonstd::span<const uint8_t> value)
{
MTR_SCOPE("storage", "put");
- const auto path = primary.put(key, type, entry_writer);
- if (!path) {
- return false;
- }
-
- // Temporary optimization until primary storage API has been refactored to
- // pass data via memory instead of files.
- const bool should_put_in_secondary_storage =
- std::any_of(m_secondary_storages.begin(),
- m_secondary_storages.end(),
- [](const auto& entry) { return !entry->config.read_only; });
- if (should_put_in_secondary_storage) {
- const auto value = util::read_file<util::Bytes>(*path);
- if (!value) {
- LOG("Failed to read {}: {}", *path, value.error());
- return true; // Don't indicate failure since primary storage was OK.
- }
- put_in_secondary_storage(key, *value, false);
- }
-
- return true;
+ primary.put(key, type, value);
+ put_in_secondary_storage(key, value, false);
}
void
#include <storage/primary/PrimaryStorage.hpp>
#include <storage/secondary/SecondaryStorage.hpp>
#include <storage/types.hpp>
+#include <util/Bytes.hpp>
#include <third_party/nonstd/span.hpp>
-#include <functional>
#include <memory>
#include <optional>
#include <string>
primary::PrimaryStorage primary;
- // Returns a path to a file containing the value.
enum class Mode { secondary_fallback, secondary_only, primary_only };
- std::optional<std::string> get(const Digest& key,
+ std::optional<util::Bytes> get(const Digest& key,
core::CacheEntryType type,
const Mode mode = Mode::secondary_fallback);
- bool put(const Digest& key,
+ void put(const Digest& key,
core::CacheEntryType type,
- const storage::EntryWriter& entry_writer);
+ nonstd::span<const uint8_t> value);
void remove(const Digest& key, core::CacheEntryType type);
private:
const Config& m_config;
std::vector<std::unique_ptr<SecondaryStorageEntry>> m_secondary_storages;
- std::vector<std::string> m_tmp_files;
void add_secondary_storages();
#include "PrimaryStorage.hpp"
+#include <AtomicFile.hpp>
#include <Config.hpp>
#include <Logging.hpp>
#include <MiniTrace.hpp>
}
}
-std::optional<std::string>
+std::optional<util::Bytes>
PrimaryStorage::get(const Digest& key, const core::CacheEntryType type) const
{
MTR_SCOPE("primary_storage", "get");
const auto cache_file = look_up_cache_file(key, type);
if (!cache_file.stat) {
- LOG("No {} in primary storage", key.to_string());
+ LOG("No {} {} in primary storage", key.to_string(), core::to_string(type));
+ return std::nullopt;
+ }
+ const auto value = util::read_file<util::Bytes>(cache_file.path);
+ if (!value) {
+ LOG("Failed to read {}: {}", cache_file.path, value.error());
return std::nullopt;
}
// Update modification timestamp to save file from LRU cleanup.
util::set_timestamps(cache_file.path);
- return cache_file.path;
+
+ return *value;
}
-std::optional<std::string>
+void
PrimaryStorage::put(const Digest& key,
const core::CacheEntryType type,
- const storage::EntryWriter& entry_writer)
+ nonstd::span<const uint8_t> value)
{
MTR_SCOPE("primary_storage", "put");
break;
}
- if (!entry_writer(cache_file.path)) {
- LOG("Did not store {} in primary storage", key.to_string());
- return std::nullopt;
+ try {
+ AtomicFile result_file(cache_file.path, AtomicFile::Mode::binary);
+ result_file.write(value);
+ result_file.commit();
+ }
+
+ catch (core::Error& e) {
+ LOG("Failed to write to {}: {}", cache_file.path, e.what());
+ return;
}
const auto new_stat = Stat::stat(cache_file.path, Stat::OnError::log);
if (!new_stat) {
LOG("Failed to stat {}: {}", cache_file.path, strerror(errno));
- return std::nullopt;
+ return;
}
LOG("Stored {} in primary storage ({})", key.to_string(), cache_file.path);
// the stat call if we exit early.
util::create_cachedir_tag(
FMT("{}/{}", m_config.cache_dir(), key.to_string()[0]));
-
- return cache_file.path;
}
void
return FMT("{}{}W", prefix, file_number);
}
+std::string
+PrimaryStorage::get_raw_file_path(const Digest& result_key,
+ uint8_t file_number) const
+{
+ const auto cache_file =
+ look_up_cache_file(result_key, core::CacheEntryType::result);
+ return get_raw_file_path(cache_file.path, file_number);
+}
+
+void
+PrimaryStorage::put_raw_files(
+ const Digest& key,
+ const std::vector<core::Result::Serializer::RawFile> raw_files)
+{
+ const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result);
+ Util::ensure_dir_exists(Util::dir_name(cache_file.path));
+
+ for (auto [file_number, source_path] : raw_files) {
+ const auto dest_path = get_raw_file_path(cache_file.path, file_number);
+ const auto old_stat = Stat::stat(dest_path);
+ try {
+ Util::clone_hard_link_or_copy_file(
+ m_config, source_path, dest_path, true);
+ m_added_raw_files.push_back(dest_path);
+ } catch (core::Error& e) {
+ LOG("Failed to store {} as raw file {}: {}",
+ source_path,
+ dest_path,
+ e.what());
+ throw;
+ }
+ const auto new_stat = Stat::stat(dest_path);
+ increment_statistic(Statistic::cache_size_kibibyte,
+ Util::size_change_kibibyte(old_stat, new_stat));
+ increment_statistic(Statistic::files_in_cache,
+ (new_stat ? 1 : 0) - (old_stat ? 1 : 0));
+ }
+}
+
void
PrimaryStorage::increment_statistic(const Statistic statistic,
const int64_t value)
// Two ccache processes may move the file at the same time, so failure
// to rename is OK.
}
+ for (const auto& raw_file : m_added_raw_files) {
+ try {
+ Util::rename(raw_file,
+ FMT("{}/{}",
+ Util::dir_name(wanted_path),
+ Util::base_name(raw_file)));
+ } catch (const core::Error&) {
+ // Two ccache processes may move the file at the same time, so failure
+ // to rename is OK.
+ }
+ }
}
}
return counters;
#pragma once
#include <Digest.hpp>
+#include <core/Result.hpp>
#include <core/StatisticsCounters.hpp>
#include <core/types.hpp>
#include <storage/primary/util.hpp>
#include <storage/types.hpp>
+#include <util/Bytes.hpp>
+
+#include <third_party/nonstd/span.hpp>
#include <cstdint>
#include <optional>
+#include <vector>
class Config;
// --- Cache entry handling ---
- // Returns a path to a file containing the value.
- std::optional<std::string> get(const Digest& key,
+ std::optional<util::Bytes> get(const Digest& key,
core::CacheEntryType type) const;
- std::optional<std::string> put(const Digest& key,
- core::CacheEntryType type,
- const storage::EntryWriter& entry_writer);
+ void put(const Digest& key,
+ core::CacheEntryType type,
+ nonstd::span<const uint8_t> value);
void remove(const Digest& key, core::CacheEntryType type);
static std::string get_raw_file_path(std::string_view result_path,
uint8_t file_number);
+ std::string get_raw_file_path(const Digest& result_key,
+ uint8_t file_number) const;
+
+ void
+ put_raw_files(const Digest& key,
+ const std::vector<core::Result::Serializer::RawFile> raw_files);
// --- Statistics ---
std::string m_manifest_path;
std::string m_result_path;
+ std::vector<std::string> m_added_raw_files;
+
struct LookUpCacheFileResult
{
std::string path;
projects / thirdparty / ccache.git / commitdiff
