#include "Args.hpp"
#include "ArgsInfo.hpp"
#include "Config.hpp"
-#include "Digest.hpp"
#include "File.hpp"
#include "MiniTrace.hpp"
#include "NonCopyable.hpp"
# include "InodeCache.hpp"
#endif
+#include <Hash.hpp>
#include <core/Manifest.hpp>
#include <storage/Storage.hpp>
#include <util/TimePoint.hpp>
util::TimePoint time_of_compilation;
// Files included by the preprocessor and their hashes.
- std::unordered_map<std::string, Digest> included_files;
+ std::unordered_map<std::string, Hash::Digest> included_files;
// Have we tried and failed to get colored diagnostics?
bool diagnostics_color_failed = false;
-// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2020-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
class Context;
class Hash;
-#include "Digest.hpp"
-
#include <optional>
#include <string>
#include <string_view>
+++ /dev/null
-// Copyright (C) 2020-2023 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#pragma once
-
-#include <util/string.hpp>
-
-#include "third_party/fmt/core.h"
-
-#include <cstdint>
-#include <string>
-
-// Digest represents the binary form of the final digest (AKA hash or checksum)
-// produced by the hash algorithm.
-class Digest
-{
-public:
- Digest() = default;
-
- // Access the raw byte buffer, which is `size()` bytes long.
- const uint8_t* bytes() const;
- uint8_t* bytes();
-
- // Size of the digest in bytes.
- constexpr static size_t size();
-
- // Format the digest as hex string.
- std::string to_string() const;
-
- bool operator==(const Digest& other) const;
- bool operator!=(const Digest& other) const;
-
-private:
- constexpr static size_t k_digest_size = 20;
- uint8_t m_bytes[k_digest_size];
-};
-
-inline const uint8_t*
-Digest::bytes() const
-{
- return m_bytes;
-}
-
-inline uint8_t*
-Digest::bytes()
-{
- return m_bytes;
-}
-
-inline constexpr size_t
-Digest::size()
-{
- return k_digest_size;
-}
-
-inline std::string
-Digest::to_string() const
-{
- // The first two bytes are encoded as four lowercase base16 digits to maintain
- // compatibility with the cleanup algorithm in older ccache versions and to
- // allow for up to four uniform cache levels. The rest are encoded as
- // lowercase base32hex digits without padding characters.
- const size_t base16_bytes = 2;
- return util::format_base16({m_bytes, base16_bytes})
- + util::format_base32hex(
- {m_bytes + base16_bytes, size() - base16_bytes});
-}
-
-inline bool
-Digest::operator==(const Digest& other) const
-{
- return memcmp(bytes(), other.bytes(), size()) == 0;
-}
-
-inline bool
-Digest::operator!=(const Digest& other) const
-{
- return !(*this == other);
-}
add_debug_text(" ===\n");
}
-Digest
+Hash::Digest
Hash::digest() const
{
// Note that blake3_hasher_finalize doesn't modify the hasher itself, thus it
// is possible to finalize again after more data has been added.
Digest digest;
- blake3_hasher_finalize(&m_hasher, digest.bytes(), digest.size());
+ blake3_hasher_finalize(&m_hasher, digest.data(), digest.size());
return digest;
}
-// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2020-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
-#include "Digest.hpp"
-
#include "third_party/blake3/blake3.h"
#include <third_party/nonstd/expected.hpp>
+#include <array>
#include <cstdint>
#include <cstdio>
#include <string_view>
public:
enum class HashType { binary, text };
+ using Digest = std::array<uint8_t, 20>;
+
Hash();
Hash(const Hash& other) = default;
#include "InodeCache.hpp"
#include "Config.hpp"
-#include "Digest.hpp"
#include "Finalizer.hpp"
#include "Hash.hpp"
#include "Logging.hpp"
// How long a filesystem space check is valid before we make a new one.
const util::Duration k_fs_space_check_valid_duration(1);
-static_assert(Digest::size() == 20,
+static_assert(std::tuple_size<Hash::Digest>() == 20,
"Increment version number if size of digest is changed.");
-static_assert(std::is_trivially_copyable<Digest>::value,
+static_assert(std::is_trivially_copyable<Hash::Digest>::value,
"Digest is expected to be trivially copyable.");
static_assert(
struct InodeCache::Entry
{
- Digest key_digest; // Hashed key
- Digest file_digest; // Cached file hash
- int return_value; // Cached return value
+ Hash::Digest key_digest; // Hashed key
+ Hash::Digest file_digest; // Cached file hash
+ int return_value; // Cached return value
};
struct InodeCache::Bucket
bool
InodeCache::hash_inode(const std::string& path,
ContentType type,
- Digest& digest)
+ Hash::Digest& digest)
{
Stat stat = Stat::stat(path);
if (!stat) {
}
bool
-InodeCache::with_bucket(const Digest& key_digest,
+InodeCache::with_bucket(const Hash::Digest& key_digest,
const BucketHandler& bucket_handler)
{
uint32_t hash;
- Util::big_endian_to_int(key_digest.bytes(), hash);
+ Util::big_endian_to_int(key_digest.data(), hash);
const uint32_t index = hash % k_num_buckets;
Bucket* bucket = &m_sr->buckets[index];
bool acquired_lock = spin_lock(bucket->owner_pid, m_self_pid);
}
std::optional<HashSourceCodeResult>
-InodeCache::get(const std::string& path, ContentType type, Digest& file_digest)
+InodeCache::get(const std::string& path,
+ ContentType type,
+ Hash::Digest& file_digest)
{
if (!initialize()) {
return std::nullopt;
}
- Digest key_digest;
+ Hash::Digest key_digest;
if (!hash_inode(path, type, key_digest)) {
return std::nullopt;
}
bool
InodeCache::put(const std::string& path,
ContentType type,
- const Digest& file_digest,
+ const Hash::Digest& file_digest,
HashSourceCodeResult return_value)
{
if (!initialize()) {
return false;
}
- Digest key_digest;
+ Hash::Digest key_digest;
if (!hash_inode(path, type, key_digest)) {
return false;
}
#pragma once
#include <Fd.hpp>
+#include <Hash.hpp>
#include <hashutil.hpp>
#include <util/Duration.hpp>
#include <util/TimePoint.hpp>
class Config;
class Context;
-class Digest;
class InodeCache
{
// Returns true if saved values could be retrieved from the cache, false
// otherwise.
std::optional<HashSourceCodeResult>
- get(const std::string& path, ContentType type, Digest& file_digest);
+ get(const std::string& path, ContentType type, Hash::Digest& file_digest);
// Put hash digest and return value from a successful call to do_hash_file()
// in hashutil.cpp.
// Returns true if values could be stored in the cache, false otherwise.
bool put(const std::string& path,
ContentType type,
- const Digest& file_digest,
+ const Hash::Digest& file_digest,
HashSourceCodeResult return_value);
// Unmaps the current cache and removes the mapped file from disk.
using BucketHandler = std::function<void(Bucket* bucket)>;
bool mmap_file(const std::string& inode_cache_file);
- bool hash_inode(const std::string& path, ContentType type, Digest& digest);
- bool with_bucket(const Digest& key_digest,
+
+ bool
+ hash_inode(const std::string& path, ContentType type, Hash::Digest& digest);
+
+ bool with_bucket(const Hash::Digest& key_digest,
const BucketHandler& bucket_handler);
+
static bool create_new_file(const std::string& filename);
+
bool initialize();
const Config& m_config;
}
// Let's hash the include file content.
- Digest file_digest;
+ Hash::Digest file_digest;
if (is_pch) {
if (ctx.args_info.included_pch_file.empty()) {
return false;
}
cpp_hash.hash_delimiter(using_pch_sum ? "pch_sum_hash" : "pch_hash");
- cpp_hash.hash(file_digest.to_string());
+ cpp_hash.hash(util::format_digest(file_digest));
}
if (ctx.config.direct_mode()) {
if (depend_mode_hash) {
depend_mode_hash->hash_delimiter("include");
- depend_mode_hash->hash(file_digest.to_string());
+ depend_mode_hash->hash(util::format_digest(file_digest));
}
}
// Extract the used includes from the dependency file. Note that we cannot
// distinguish system headers from other includes here.
-static std::optional<Digest>
+static std::optional<Hash::Digest>
result_key_from_depfile(Context& ctx, Hash& hash)
{
// Make sure that result hash will always be different from the manifest hash
// Extract the used includes from /showIncludes output in stdout. Note that we
// cannot distinguish system headers from other includes here.
-static std::optional<Digest>
+static std::optional<Hash::Digest>
result_key_from_includes(Context& ctx, Hash& hash, std::string_view stdout_data)
{
for (std::string_view include : core::MsvcShowIncludesOutput::get_includes(
static void
update_manifest(Context& ctx,
- const Digest& manifest_key,
- const Digest& result_key)
+ const Hash::Digest& manifest_key,
+ const Hash::Digest& result_key)
{
if (ctx.config.read_only() || ctx.config.read_only_direct()) {
return;
};
});
if (added) {
- LOG("Added result key to manifest {}", manifest_key.to_string());
+ LOG("Added result key to manifest {}", util::format_digest(manifest_key));
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
ctx.storage.put(manifest_key,
core::CacheEntryType::manifest,
core::CacheEntry::serialize(header, ctx.manifest));
} else {
- LOG("Did not add result key to manifest {}", manifest_key.to_string());
+ LOG("Did not add result key to manifest {}",
+ util::format_digest(manifest_key));
}
}
[[nodiscard]] static bool
write_result(Context& ctx,
- const Digest& result_key,
+ const Hash::Digest& result_key,
const Stat& obj_stat,
const util::Bytes& stdout_data,
const util::Bytes& stderr_data)
}
// Run the real compiler and put the result in cache. Returns the result key.
-static nonstd::expected<Digest, Failure>
+static nonstd::expected<Hash::Digest, Failure>
to_cache(Context& ctx,
Args& args,
- std::optional<Digest> result_key,
+ std::optional<Hash::Digest> result_key,
const Args& depend_extra_args,
Hash* depend_mode_hash)
{
return nonstd::make_unexpected(Statistic::internal_error);
}
LOG_RAW("Got result key from dependency file");
- LOG("Result key: {}", result_key->to_string());
+ LOG("Result key: {}", util::format_digest(*result_key));
}
ASSERT(result_key);
// Find the result key by running the compiler in preprocessor mode and
// hashing the result.
-static nonstd::expected<Digest, Failure>
+static nonstd::expected<Hash::Digest, Failure>
get_result_key_from_cpp(Context& ctx, Args& args, Hash& hash)
{
ctx.time_of_compilation = util::TimePoint::now();
return {};
}
-static nonstd::expected<std::optional<Digest>, Failure>
+static nonstd::expected<std::optional<Hash::Digest>, Failure>
get_manifest_key(Context& ctx, Hash& hash)
{
// Hash environment variables that affect the preprocessor output.
hash.hash(ctx.args_info.input_file);
hash.hash_delimiter("sourcecode hash");
- Digest input_file_digest;
+ Hash::Digest input_file_digest;
auto ret =
hash_source_code_file(ctx, input_file_digest, ctx.args_info.input_file);
if (ret.contains(HashSourceCode::error)) {
ctx.config.set_direct_mode(false);
return {};
}
- hash.hash(input_file_digest.to_string());
+ hash.hash(util::format_digest(input_file_digest));
return hash.digest();
}
return {};
}
-static std::optional<Digest>
-get_result_key_from_manifest(Context& ctx, const Digest& manifest_key)
+static std::optional<Hash::Digest>
+get_result_key_from_manifest(Context& ctx, const Hash::Digest& manifest_key)
{
MTR_BEGIN("manifest", "manifest_get");
- std::optional<Digest> result_key;
+ std::optional<Hash::Digest> result_key;
size_t read_manifests = 0;
ctx.storage.get(
manifest_key, core::CacheEntryType::manifest, [&](util::Bytes&& value) {
MTR_END("manifest", "manifest_get");
if (read_manifests > 1 && !ctx.config.remote_only()) {
MTR_SCOPE("manifest", "merge");
- LOG("Storing merged manifest {} locally", manifest_key.to_string());
+ LOG("Storing merged manifest {} locally",
+ util::format_digest(manifest_key));
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
ctx.storage.local.put(manifest_key,
core::CacheEntryType::manifest,
// Update a hash sum with information specific to the direct and preprocessor
// modes and calculate the result key. Returns the result key on success, and
// if direct_mode is true also the manifest key.
-static nonstd::expected<std::pair<std::optional<Digest>, std::optional<Digest>>,
- Failure>
+static nonstd::expected<
+ std::pair<std::optional<Hash::Digest>, std::optional<Hash::Digest>>,
+ Failure>
calculate_result_and_manifest_key(Context& ctx,
const Args& args,
Hash& hash,
hash.hash(arch);
}
- std::optional<Digest> result_key;
- std::optional<Digest> manifest_key;
+ std::optional<Hash::Digest> result_key;
+ std::optional<Hash::Digest> manifest_key;
if (direct_mode) {
const auto manifest_key_result = get_manifest_key(ctx, hash);
}
manifest_key = *manifest_key_result;
if (manifest_key) {
- LOG("Manifest key: {}", manifest_key->to_string());
+ LOG("Manifest key: {}", util::format_digest(*manifest_key));
result_key = get_result_key_from_manifest(ctx, *manifest_key);
}
} else if (ctx.args_info.arch_args.empty()) {
}
if (result_key) {
- LOG("Result key: {}", result_key->to_string());
+ LOG("Result key: {}", util::format_digest(*result_key));
}
return std::make_pair(result_key, manifest_key);
}
// Try to return the compile result from cache.
static nonstd::expected<bool, Failure>
-from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key)
+from_cache(Context& ctx, FromCacheCallMode mode, const Hash::Digest& result_key)
{
// The user might be disabling cache hits.
if (ctx.config.recache()) {
deserializer.visit(result_retriever);
} catch (core::ResultRetriever::WriteError& e) {
LOG("Write error when retrieving result from {}: {}",
- result_key.to_string(),
+ util::format_digest(result_key),
e.what());
return nonstd::make_unexpected(Statistic::bad_output_file);
} catch (core::Error& e) {
- LOG("Failed to get result from {}: {}", result_key.to_string(), e.what());
+ LOG("Failed to get result from {}: {}",
+ util::format_digest(result_key),
+ e.what());
return false;
}
args_to_hash.push_back(processed.extra_args_to_hash);
bool put_result_in_manifest = false;
- std::optional<Digest> result_key;
- std::optional<Digest> result_key_from_manifest;
- std::optional<Digest> manifest_key;
+ std::optional<Hash::Digest> result_key;
+ std::optional<Hash::Digest> result_key_from_manifest;
+ std::optional<Hash::Digest> manifest_key;
if (ctx.config.direct_mode()) {
LOG_RAW("Trying direct lookup");
#include <fmtmacros.hpp>
#include <hashutil.hpp>
#include <util/XXH3_64.hpp>
+#include <util/string.hpp>
// Manifest data format
// ====================
// <n_includes> ::= uint32_t
// <include_entry> ::= <path_index> <digest> <fsize> <mtime> <ctime>
// <path_index> ::= uint32_t
-// <digest> ::= Digest::size() bytes
+// <digest> ::= Hash::Digest::size() bytes
// <fsize> ::= uint64_t ; file size
// <mtime> ::= int64_t ; modification time (ns), 0 = not recorded
// <ctime> ::= int64_t ; status change time (ns), 0 = not recorded
// <result> ::= <n_indexes> <include_index>* <key>
// <n_indexes> ::= uint32_t
// <include_index> ::= uint32_t
-// <result_key> ::= Digest::size() bytes
+// <result_key> ::= Hash::Digest::size() bytes
const uint32_t k_max_manifest_entries = 100;
const uint32_t k_max_manifest_file_info_entries = 10000;
auto& entry = file_infos.back();
reader.read_int(entry.index);
- reader.read_and_copy_bytes({entry.digest.bytes(), Digest::size()});
+ reader.read_and_copy_bytes(entry.digest);
reader.read_int(entry.fsize);
entry.mtime.set_nsec(reader.read_int<int64_t>());
entry.ctime.set_nsec(reader.read_int<int64_t>());
for (uint32_t j = 0; j < file_info_index_count; ++j) {
entry.file_info_indexes.push_back(reader.read_int<uint32_t>());
}
- reader.read_and_copy_bytes({entry.key.bytes(), Digest::size()});
+ reader.read_and_copy_bytes(entry.key);
}
if (m_results.empty()) {
m_results = std::move(results);
} else {
for (const auto& result : results) {
- std::unordered_map<std::string, Digest> included_files;
+ std::unordered_map<std::string, Hash::Digest> included_files;
std::unordered_map<std::string, FileStats> included_files_stats;
for (auto file_info_index : result.file_info_indexes) {
const auto& file_info = file_infos[file_info_index];
}
}
-std::optional<Digest>
+std::optional<Hash::Digest>
Manifest::look_up_result_digest(const Context& ctx) const
{
std::unordered_map<std::string, FileStats> stated_files;
- std::unordered_map<std::string, Digest> hashed_files;
+ std::unordered_map<std::string, Hash::Digest> hashed_files;
// Check newest result first since it's a more likely to match.
for (size_t i = m_results.size(); i > 0; i--) {
bool
Manifest::add_result(
- const Digest& result_key,
- const std::unordered_map<std::string, Digest>& included_files,
+ const Hash::Digest& result_key,
+ const std::unordered_map<std::string, Hash::Digest>& included_files,
const FileStater& stat_file_function)
{
if (m_results.size() > k_max_manifest_entries) {
size += 2 + file.length();
}
size += 4; // n_file_infos
- size += m_file_infos.size() * (4 + Digest::size() + 8 + 8 + 8);
+ size +=
+ m_file_infos.size() * (4 + std::tuple_size<Hash::Digest>() + 8 + 8 + 8);
size += 4; // n_results
for (const auto& result : m_results) {
size += 4; // n_file_info_indexes
size += result.file_info_indexes.size() * 4;
- size += Digest::size();
+ size += std::tuple_size<Hash::Digest>();
}
// In order to support 32-bit ccache builds, restrict size to uint32_t for
writer.write_int<uint32_t>(m_file_infos.size());
for (const auto& file_info : m_file_infos) {
writer.write_int<uint32_t>(file_info.index);
- writer.write_bytes({file_info.digest.bytes(), Digest::size()});
+ writer.write_bytes(file_info.digest);
writer.write_int(file_info.fsize);
writer.write_int(file_info.mtime.nsec());
writer.write_int(file_info.ctime.nsec());
for (auto index : result.file_info_indexes) {
writer.write_int(index);
}
- writer.write_bytes({result.key.bytes(), Digest::size()});
+ writer.write_bytes(result.key);
}
}
uint32_t
Manifest::get_file_info_index(
const std::string& path,
- const Digest& digest,
+ const Hash::Digest& digest,
const std::unordered_map<std::string, uint32_t>& mf_files,
const std::unordered_map<FileInfo, uint32_t>& mf_file_infos,
const FileStater& file_stater)
const Context& ctx,
const ResultEntry& result,
std::unordered_map<std::string, FileStats>& stated_files,
- std::unordered_map<std::string, Digest>& hashed_files) const
+ std::unordered_map<std::string, Hash::Digest>& hashed_files) const
{
for (uint32_t file_info_index : result.file_info_indexes) {
const auto& fi = m_file_infos[file_info_index];
auto hashed_files_iter = hashed_files.find(path);
if (hashed_files_iter == hashed_files.end()) {
- Digest actual_digest;
+ Hash::Digest actual_digest;
auto ret = hash_source_code_file(ctx, actual_digest, path, fs.size);
if (ret.contains(HashSourceCode::error)) {
LOG("Failed hashing {}", path);
for (size_t i = 0; i < m_file_infos.size(); ++i) {
PRINT(stream, " {}:\n", i);
PRINT(stream, " Path index: {}\n", m_file_infos[i].index);
- PRINT(stream, " Hash: {}\n", m_file_infos[i].digest.to_string());
+ PRINT(
+ stream, " Hash: {}\n", util::format_digest(m_file_infos[i].digest));
PRINT(stream, " File size: {}\n", m_file_infos[i].fsize);
if (m_file_infos[i].mtime == util::TimePoint()) {
PRINT_RAW(stream, " Mtime: -\n");
PRINT(stream, " {}", file_info_index);
}
PRINT_RAW(stream, "\n");
- PRINT(stream, " Key: {}\n", m_results[i].key.to_string());
+ PRINT(stream, " Key: {}\n", util::format_digest(m_results[i].key));
}
}
-// Copyright (C) 2009-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2009-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
-#include <Digest.hpp>
+#include <Hash.hpp>
#include <core/Serializer.hpp>
#include <util/TimePoint.hpp>
void read(nonstd::span<const uint8_t> data);
- std::optional<Digest> look_up_result_digest(const Context& ctx) const;
+ std::optional<Hash::Digest> look_up_result_digest(const Context& ctx) const;
- bool add_result(const Digest& result_key,
- const std::unordered_map<std::string, Digest>& included_files,
- const FileStater& stat_file);
+ bool add_result(
+ const Hash::Digest& result_key,
+ const std::unordered_map<std::string, Hash::Digest>& included_files,
+ const FileStater& stat_file);
// core::Serializer
uint32_t serialized_size() const override;
struct FileInfo
{
uint32_t index; // Index to m_files.
- Digest digest; // Digest of referenced file.
+ Hash::Digest digest; // Hash::Digest of referenced file.
uint64_t fsize; // Size of referenced file.
util::TimePoint mtime; // mtime of referenced file.
util::TimePoint ctime; // ctime of referenced file.
struct ResultEntry
{
std::vector<uint32_t> file_info_indexes; // Indexes to m_file_infos.
- Digest key; // Key of the result.
+ Hash::Digest key; // Key of the result.
bool operator==(const ResultEntry& other) const;
};
uint32_t get_file_info_index(
const std::string& path,
- const Digest& digest,
+ const Hash::Digest& digest,
const std::unordered_map<std::string, uint32_t>& mf_files,
const std::unordered_map<FileInfo, uint32_t>& mf_file_infos,
const FileStater& file_state);
- bool
- result_matches(const Context& ctx,
- const ResultEntry& result,
- std::unordered_map<std::string, FileStats>& stated_files,
- std::unordered_map<std::string, Digest>& hashed_files) const;
+ bool result_matches(
+ const Context& ctx,
+ const ResultEntry& result,
+ std::unordered_map<std::string, FileStats>& stated_files,
+ std::unordered_map<std::string, Hash::Digest>& hashed_files) const;
};
} // namespace core
using Result::FileType;
ResultRetriever::ResultRetriever(const Context& ctx,
- std::optional<Digest> result_key)
+ std::optional<Hash::Digest> result_key)
: m_ctx(ctx),
m_result_key(result_key)
{
-// Copyright (C) 2020-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2020-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include "Fd.hpp"
-#include <Digest.hpp>
+#include <Hash.hpp>
#include <core/Result.hpp>
#include <core/exceptions.hpp>
//`path` should be the path to the local result entry file if the result comes
// from local storage.
ResultRetriever(const Context& ctx,
- std::optional<Digest> result_key = std::nullopt);
+ std::optional<Hash::Digest> result_key = std::nullopt);
void on_embedded_file(uint8_t file_number,
Result::FileType file_type,
private:
const Context& m_ctx;
- std::optional<Digest> m_result_key;
+ std::optional<Hash::Digest> m_result_key;
std::string get_dest_path(Result::FileType file_type) const;
const auto result =
arg == "-" ? hash.hash_fd(STDIN_FILENO) : hash.hash_file(arg);
if (result) {
- PRINT(stdout, "{}\n", hash.digest().to_string());
+ PRINT(stdout, "{}\n", util::format_digest(hash.digest()));
} else {
PRINT(stderr, "Error: Failed to hash {}: {}\n", arg, result.error());
return EXIT_FAILURE;
#include "Args.hpp"
#include "Config.hpp"
#include "Context.hpp"
-#include "Hash.hpp"
#include "Logging.hpp"
#include "Stat.hpp"
#include "Util.hpp"
HashSourceCodeResult
do_hash_file(const Context& ctx,
- Digest& digest,
+ Hash::Digest& digest,
const std::string& path,
size_t size_hint,
bool check_temporal_macros)
HashSourceCodeResult
hash_source_code_file(const Context& ctx,
- Digest& digest,
+ Hash::Digest& digest,
const std::string& path,
size_t size_hint)
{
// macro expansions.
Hash hash;
- hash.hash(digest.to_string());
+ hash.hash(util::format_digest(digest));
if (result.contains(HashSourceCode::found_date)) {
LOG("Found __DATE__ in {}", path);
bool
hash_binary_file(const Context& ctx,
- Digest& digest,
+ Hash::Digest& digest,
const std::string& path,
size_t size_hint)
{
bool
hash_binary_file(const Context& ctx, Hash& hash, const std::string& path)
{
- Digest digest;
+ Hash::Digest digest;
const bool success = hash_binary_file(ctx, digest, path);
if (success) {
- hash.hash(digest.to_string());
+ hash.hash(util::format_digest(digest));
}
return success;
}
#pragma once
+#include <Hash.hpp>
#include <util/BitSet.hpp>
#include <cstddef>
class Config;
class Context;
-class Digest;
-class Hash;
enum class HashSourceCode {
ok = 0,
// Hash a source code file using the inode cache if enabled.
HashSourceCodeResult hash_source_code_file(const Context& ctx,
- Digest& digest,
+ Hash::Digest& digest,
const std::string& path,
size_t size_hint = 0);
//
// Returns true on success, otherwise false.
bool hash_binary_file(const Context& ctx,
- Digest& digest,
+ Hash::Digest& digest,
const std::string& path,
size_t size_hint = 0);
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include <TemporaryFile.hpp>
#include <Util.hpp>
#include <assertions.hpp>
+#include <core/CacheEntry.hpp>
#include <core/Statistic.hpp>
#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
#ifdef HAVE_REDIS_STORAGE_BACKEND
# include <storage/remote/RedisStorage.hpp>
#endif
-#include <core/CacheEntry.hpp>
#include <util/Bytes.hpp>
#include <util/Timer.hpp>
#include <util/Tokenizer.hpp>
}
void
-Storage::get(const Digest& key,
+Storage::get(const Hash::Hash::Digest& key,
const core::CacheEntryType type,
const EntryReceiver& entry_receiver)
{
}
void
-Storage::put(const Digest& key,
+Storage::put(const Hash::Digest& key,
const core::CacheEntryType type,
nonstd::span<const uint8_t> value)
{
}
void
-Storage::remove(const Digest& key, const core::CacheEntryType type)
+Storage::remove(const Hash::Digest& key, const core::CacheEntryType type)
{
MTR_SCOPE("storage", "remove");
}
static Url
-get_shard_url(const Digest& key,
+get_shard_url(const Hash::Digest& key,
const std::string& url,
const std::vector<RemoteStorageShardConfig>& shards)
{
std::string best_shard;
for (const auto& shard_config : shards) {
util::XXH3_64 hash;
- hash.update(key.bytes(), key.size());
+ hash.update(key.data(), key.size());
hash.update(shard_config.name.data(), shard_config.name.length());
const double score = to_half_open_unit_interval(hash.digest());
ASSERT(score >= 0.0 && score < 1.0);
RemoteStorageBackendEntry*
Storage::get_backend(RemoteStorageEntry& entry,
- const Digest& key,
+ const Hash::Digest& key,
const std::string_view operation_description,
const bool for_writing)
{
}
void
-Storage::get_from_remote_storage(const Digest& key,
+Storage::get_from_remote_storage(const Hash::Digest& key,
const core::CacheEntryType type,
const EntryReceiver& entry_receiver)
{
auto& value = *result;
if (value) {
LOG("Retrieved {} from {} ({:.2f} ms)",
- key.to_string(),
+ util::format_digest(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_read_hit);
}
} else {
LOG("No {} in {} ({:.2f} ms)",
- key.to_string(),
+ util::format_digest(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_read_miss);
}
void
-Storage::put_in_remote_storage(const Digest& key,
+Storage::put_in_remote_storage(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing)
{
if (!core::CacheEntry::Header(value).self_contained) {
LOG("Not putting {} in remote storage since it's not self-contained",
- key.to_string());
+ util::format_digest(key));
return;
}
const bool stored = *result;
LOG("{} {} in {} ({:.2f} ms)",
stored ? "Stored" : "Did not have to store",
- key.to_string(),
+ util::format_digest(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_write);
}
void
-Storage::remove_from_remote_storage(const Digest& key)
+Storage::remove_from_remote_storage(const Hash::Digest& key)
{
MTR_SCOPE("remote_storage", "remove");
const bool removed = *result;
if (removed) {
LOG("Removed {} from {} ({:.2f} ms)",
- key.to_string(),
+ util::format_digest(key),
backend->url_for_logging,
ms);
} else {
LOG("No {} to remove from {} ({:.2f} ms)",
- key.to_string(),
+ util::format_digest(key),
backend->url_for_logging,
ms);
}
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
+#include <Hash.hpp>
#include <core/types.hpp>
#include <storage/local/LocalStorage.hpp>
#include <storage/remote/RemoteStorage.hpp>
#include <string>
#include <vector>
-class Digest;
-
namespace storage {
std::string get_features();
using EntryReceiver = std::function<bool(util::Bytes&&)>;
- void get(const Digest& key,
+ void get(const Hash::Digest& key,
core::CacheEntryType type,
const EntryReceiver& entry_receiver);
- void put(const Digest& key,
+ void put(const Hash::Digest& key,
core::CacheEntryType type,
nonstd::span<const uint8_t> value);
- void remove(const Digest& key, core::CacheEntryType type);
+ void remove(const Hash::Digest& key, core::CacheEntryType type);
bool has_remote_storage() const;
std::string get_remote_storage_config_for_logging() const;
remote::RemoteStorage::Backend::Failure failure);
RemoteStorageBackendEntry* get_backend(RemoteStorageEntry& entry,
- const Digest& key,
+ const Hash::Digest& key,
std::string_view operation_description,
const bool for_writing);
- void get_from_remote_storage(const Digest& key,
+ void get_from_remote_storage(const Hash::Digest& key,
core::CacheEntryType type,
const EntryReceiver& entry_receiver);
- void put_in_remote_storage(const Digest& key,
+ void put_in_remote_storage(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing);
- void remove_from_remote_storage(const Digest& key);
+ void remove_from_remote_storage(const Hash::Digest& key);
};
} // namespace storage
#include <core/CacheEntry.hpp>
#include <core/FileRecompressor.hpp>
#include <core/Manifest.hpp>
-#include <core/Result.hpp>
#include <core/Statistics.hpp>
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
-#include <storage/local/StatsFile.hpp>
-#include <storage/local/util.hpp>
#include <util/Duration.hpp>
#include <util/TextTable.hpp>
#include <util/expected.hpp>
#include <util/file.hpp>
#include <util/string.hpp>
-#include <third_party/fmt/core.h>
-
#ifdef INODE_CACHE_SUPPORTED
# include <InodeCache.hpp>
#endif
+#include <third_party/fmt/core.h>
+
#include <algorithm>
#include <atomic>
#include <cstdlib>
}
std::optional<util::Bytes>
-LocalStorage::get(const Digest& key, const core::CacheEntryType type)
+LocalStorage::get(const Hash::Digest& key, const core::CacheEntryType type)
{
MTR_SCOPE("local_storage", "get");
const auto value = util::read_file<util::Bytes>(cache_file.path);
if (value) {
LOG("Retrieved {} from local storage ({})",
- key.to_string(),
+ util::format_digest(key),
cache_file.path);
// Update modification timestamp to save file from LRU cleanup.
LOG("Failed to read {}: {}", cache_file.path, value.error());
}
} else {
- LOG("No {} in local storage", key.to_string());
+ LOG("No {} in local storage", util::format_digest(key));
}
increment_statistic(return_value ? Statistic::local_storage_read_hit
}
void
-LocalStorage::put(const Digest& key,
+LocalStorage::put(const Hash::Digest& key,
const core::CacheEntryType type,
nonstd::span<const uint8_t> value,
bool only_if_missing)
return;
}
- LOG("Stored {} in local storage ({})", key.to_string(), cache_file.path);
+ LOG("Stored {} in local storage ({})",
+ util::format_digest(key),
+ cache_file.path);
m_stored_data = true;
if (!m_config.stats()) {
// be done almost anywhere, but we might as well do it near the end as we save
// the stat call if we exit early.
util::create_cachedir_tag(
- FMT("{}/{}", m_config.cache_dir(), key.to_string()[0]));
+ FMT("{}/{}", m_config.cache_dir(), util::format_digest(key)[0]));
}
void
-LocalStorage::remove(const Digest& key, const core::CacheEntryType type)
+LocalStorage::remove(const Hash::Digest& key, const core::CacheEntryType type)
{
MTR_SCOPE("local_storage", "remove");
const auto cache_file = look_up_cache_file(key, type);
if (!cache_file.stat) {
- LOG("No {} to remove from local storage", key.to_string());
+ LOG("No {} to remove from local storage", util::format_digest(key));
return;
}
Util::unlink_safe(cache_file.path);
}
- LOG("Removed {} from local storage ({})", key.to_string(), cache_file.path);
+ LOG("Removed {} from local storage ({})",
+ util::format_digest(key),
+ cache_file.path);
increment_level_2_counters(
key, -1, -static_cast<int64_t>(cache_file.stat.size_on_disk() / 1024));
}
}
std::string
-LocalStorage::get_raw_file_path(const Digest& result_key,
+LocalStorage::get_raw_file_path(const Hash::Digest& result_key,
uint8_t file_number) const
{
const auto cache_file =
void
LocalStorage::put_raw_files(
- const Digest& key,
+ const Hash::Digest& key,
const std::vector<core::Result::Serializer::RawFile> raw_files)
{
const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result);
}
LocalStorage::LookUpCacheFileResult
-LocalStorage::look_up_cache_file(const Digest& key,
+LocalStorage::look_up_cache_file(const Hash::Digest& key,
const core::CacheEntryType type) const
{
- const auto key_string = FMT("{}{}", key.to_string(), suffix_from_type(type));
+ const auto key_string =
+ FMT("{}{}", util::format_digest(key), suffix_from_type(type));
for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels;
++level) {
void
LocalStorage::move_to_wanted_cache_level(const StatisticsCounters& counters,
- const Digest& key,
+ const Hash::Digest& key,
core::CacheEntryType type,
const std::string& cache_file_path)
{
const auto wanted_level =
calculate_wanted_cache_level(counters.get(Statistic::files_in_cache));
- const auto wanted_path =
- get_path_in_cache(wanted_level, key.to_string() + suffix_from_type(type));
+ const auto wanted_path = get_path_in_cache(
+ wanted_level, util::format_digest(key) + suffix_from_type(type));
if (cache_file_path != wanted_path) {
Util::ensure_dir_exists(Util::dir_name(wanted_path));
LOG("Moving {} to {}", cache_file_path, wanted_path);
}
std::optional<core::StatisticsCounters>
-LocalStorage::increment_level_2_counters(const Digest& key,
+LocalStorage::increment_level_2_counters(const Hash::Digest& key,
int64_t files,
int64_t size_kibibyte)
{
- uint8_t l1_index = key.bytes()[0] >> 4;
- uint8_t l2_index = key.bytes()[0] & 0xF;
+ uint8_t l1_index = key[0] >> 4;
+ uint8_t l2_index = key[0] & 0xF;
const auto level_1_stats_file = get_stats_file(l1_index);
return level_1_stats_file.update([&](auto& cs) {
// Level 1 counters:
}
util::LockFile
-LocalStorage::get_level_2_content_lock(const Digest& key) const
+LocalStorage::get_level_2_content_lock(const Hash::Digest& key) const
{
- return get_level_2_content_lock(key.bytes()[0] >> 4, key.bytes()[0] & 0xF);
+ return get_level_2_content_lock(key[0] >> 4, key[0] & 0xF);
}
util::LockFile
#pragma once
-#include <Digest.hpp>
+#include <Hash.hpp>
#include <core/Result.hpp>
#include <core/StatisticsCounters.hpp>
#include <core/types.hpp>
// --- Cache entry handling ---
- std::optional<util::Bytes> get(const Digest& key, core::CacheEntryType type);
+ std::optional<util::Bytes> get(const Hash::Digest& key,
+ core::CacheEntryType type);
- void put(const Digest& key,
+ void put(const Hash::Digest& key,
core::CacheEntryType type,
nonstd::span<const uint8_t> value,
bool only_if_missing = false);
- void remove(const Digest& key, core::CacheEntryType type);
+ void remove(const Hash::Digest& key, core::CacheEntryType type);
static std::string get_raw_file_path(std::string_view result_path,
uint8_t file_number);
- std::string get_raw_file_path(const Digest& result_key,
+ std::string get_raw_file_path(const Hash::Digest& result_key,
uint8_t file_number) const;
void
- put_raw_files(const Digest& key,
+ put_raw_files(const Hash::Digest& key,
const std::vector<core::Result::Serializer::RawFile> raw_files);
// --- Statistics ---
uint8_t level;
};
- LookUpCacheFileResult look_up_cache_file(const Digest& key,
+ LookUpCacheFileResult look_up_cache_file(const Hash::Digest& key,
core::CacheEntryType type) const;
std::string get_subdir(uint8_t l1_index) const;
StatsFile get_stats_file(uint8_t l1_index, uint8_t l2_index) const;
void move_to_wanted_cache_level(const core::StatisticsCounters& counters,
- const Digest& key,
+ const Hash::Digest& key,
core::CacheEntryType type,
const std::string& cache_file_path);
uint8_t l1_index);
std::optional<core::StatisticsCounters> increment_level_2_counters(
- const Digest& key, int64_t files, int64_t size_kibibyte);
+ const Hash::Digest& key, int64_t files, int64_t size_kibibyte);
void perform_automatic_cleanup();
// in the directory (including any subdirectories). However, the lock does not
// have to be acquired to update a level 2 stats file since level 2 content
// size and file count are stored in the parent (level 1) stats file.
- util::LockFile get_level_2_content_lock(const Digest& key) const;
+ util::LockFile get_level_2_content_lock(const Hash::Digest& key) const;
util::LockFile get_level_2_content_lock(uint8_t l1_index,
uint8_t l2_index) const;
};
#include "FileStorage.hpp"
#include <AtomicFile.hpp>
-#include <Digest.hpp>
#include <Logging.hpp>
#include <UmaskScope.hpp>
#include <Util.hpp>
FileStorageBackend(const Params& params);
nonstd::expected<std::optional<util::Bytes>, Failure>
- get(const Digest& key) override;
+ get(const Hash::Digest& key) override;
- nonstd::expected<bool, Failure> put(const Digest& key,
+ nonstd::expected<bool, Failure> put(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing) override;
- nonstd::expected<bool, Failure> remove(const Digest& key) override;
+ nonstd::expected<bool, Failure> remove(const Hash::Digest& key) override;
private:
enum class Layout { flat, subdirs };
bool m_update_mtime = false;
Layout m_layout = Layout::subdirs;
- std::string get_entry_path(const Digest& key) const;
+ std::string get_entry_path(const Hash::Digest& key) const;
};
FileStorageBackend::FileStorageBackend(const Params& params)
}
nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
-FileStorageBackend::get(const Digest& key)
+FileStorageBackend::get(const Hash::Digest& key)
{
const auto path = get_entry_path(key);
const bool exists = Stat::stat(path);
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-FileStorageBackend::put(const Digest& key,
+FileStorageBackend::put(const Hash::Digest& key,
const nonstd::span<const uint8_t> value,
const bool only_if_missing)
{
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-FileStorageBackend::remove(const Digest& key)
+FileStorageBackend::remove(const Hash::Digest& key)
{
return Util::unlink_safe(get_entry_path(key));
}
std::string
-FileStorageBackend::get_entry_path(const Digest& key) const
+FileStorageBackend::get_entry_path(const Hash::Digest& key) const
{
switch (m_layout) {
case Layout::flat:
- return FMT("{}/{}", m_dir, key.to_string());
+ return FMT("{}/{}", m_dir, util::format_digest(key));
case Layout::subdirs: {
- const auto key_str = key.to_string();
+ const auto key_str = util::format_digest(key);
const uint8_t digits = 2;
ASSERT(key_str.length() > digits);
return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]);
#include "HttpStorage.hpp"
-#include <Digest.hpp>
+#include <Hash.hpp>
#include <Logging.hpp>
#include <assertions.hpp>
#include <ccache.hpp>
HttpStorageBackend(const Params& params);
nonstd::expected<std::optional<util::Bytes>, Failure>
- get(const Digest& key) override;
+ get(const Hash::Digest& key) override;
- nonstd::expected<bool, Failure> put(const Digest& key,
+ nonstd::expected<bool, Failure> put(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing) override;
- nonstd::expected<bool, Failure> remove(const Digest& key) override;
+ nonstd::expected<bool, Failure> remove(const Hash::Digest& key) override;
private:
enum class Layout { bazel, flat, subdirs };
httplib::Client m_http_client;
Layout m_layout = Layout::subdirs;
- std::string get_entry_path(const Digest& key) const;
+ std::string get_entry_path(const Hash::Digest& key) const;
};
std::string
}
nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
-HttpStorageBackend::get(const Digest& key)
+HttpStorageBackend::get(const Hash::Digest& key)
{
const auto url_path = get_entry_path(key);
const auto result = m_http_client.Get(url_path);
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-HttpStorageBackend::put(const Digest& key,
+HttpStorageBackend::put(const Hash::Digest& key,
const nonstd::span<const uint8_t> value,
const bool only_if_missing)
{
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-HttpStorageBackend::remove(const Digest& key)
+HttpStorageBackend::remove(const Hash::Digest& key)
{
const auto url_path = get_entry_path(key);
const auto result = m_http_client.Delete(url_path);
}
std::string
-HttpStorageBackend::get_entry_path(const Digest& key) const
+HttpStorageBackend::get_entry_path(const Hash::Digest& key) const
{
switch (m_layout) {
case Layout::bazel: {
// Mimic hex representation of a SHA256 hash value.
const auto sha256_hex_size = 64;
- static_assert(Digest::size() == 20, "Update below if digest size changes");
- std::string hex_digits = util::format_base16({key.bytes(), key.size()});
+ static_assert(std::tuple_size<Hash::Digest>() == 20,
+ "Update below if digest size changes");
+ std::string hex_digits = util::format_base16(key);
hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size());
- LOG("Translated key {} to Bazel layout ac/{}", key.to_string(), hex_digits);
+ LOG("Translated key {} to Bazel layout ac/{}",
+ util::format_digest(key),
+ hex_digits);
return FMT("{}ac/{}", m_url_path, hex_digits);
}
case Layout::flat:
- return m_url_path + key.to_string();
+ return m_url_path + util::format_digest(key);
case Layout::subdirs: {
- const auto key_str = key.to_string();
+ const auto key_str = util::format_digest(key);
const uint8_t digits = 2;
ASSERT(key_str.length() > digits);
return FMT("{}{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]);
#include "RedisStorage.hpp"
-#include <Digest.hpp>
+#include <Hash.hpp>
#include <Logging.hpp>
#include <assertions.hpp>
#include <core/exceptions.hpp>
RedisStorageBackend(const RemoteStorage::Backend::Params& params);
nonstd::expected<std::optional<util::Bytes>, Failure>
- get(const Digest& key) override;
+ get(const Hash::Digest& key) override;
- nonstd::expected<bool, Failure> put(const Digest& key,
+ nonstd::expected<bool, Failure> put(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing) override;
- nonstd::expected<bool, Failure> remove(const Digest& key) override;
+ nonstd::expected<bool, Failure> remove(const Hash::Digest& key) override;
private:
const std::string m_prefix;
void select_database(const Url& url);
void authenticate(const Url& url);
nonstd::expected<RedisReply, Failure> redis_command(const char* format, ...);
- std::string get_key_string(const Digest& digest) const;
+ std::string get_key_string(const Hash::Digest& digest) const;
};
timeval
}
nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
-RedisStorageBackend::get(const Digest& key)
+RedisStorageBackend::get(const Hash::Digest& key)
{
const auto key_string = get_key_string(key);
LOG("Redis GET {}", key_string);
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-RedisStorageBackend::put(const Digest& key,
+RedisStorageBackend::put(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing)
{
}
nonstd::expected<bool, RemoteStorage::Backend::Failure>
-RedisStorageBackend::remove(const Digest& key)
+RedisStorageBackend::remove(const Hash::Digest& key)
{
const auto key_string = get_key_string(key);
LOG("Redis DEL {}", key_string);
}
std::string
-RedisStorageBackend::get_key_string(const Digest& digest) const
+RedisStorageBackend::get_key_string(const Hash::Digest& digest) const
{
- return FMT("{}:{}", m_prefix, digest.to_string());
+ return FMT("{}:{}", m_prefix, util::format_digest(digest));
}
} // namespace
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#pragma once
+#include <Hash.hpp>
#include <storage/types.hpp>
#include <util/Bytes.hpp>
#include <string>
#include <vector>
-class Digest;
-
namespace storage::remote {
constexpr auto k_redacted_password = "********";
// Get the value associated with `key`. Returns the value on success or
// std::nullopt if the entry is not present.
virtual nonstd::expected<std::optional<util::Bytes>, Failure>
- get(const Digest& key) = 0;
+ get(const Hash::Digest& key) = 0;
// Put `value` associated to `key` in the storage. A true `only_if_missing`
// is a hint that the value does not have to be set if already present.
// Returns true if the entry was stored, otherwise false.
virtual nonstd::expected<bool, Failure>
- put(const Digest& key,
+ put(const Hash::Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing = false) = 0;
// Remove `key` and its associated value. Returns true if the entry was
// removed, otherwise false.
- virtual nonstd::expected<bool, Failure> remove(const Digest& key) = 0;
+ virtual nonstd::expected<bool, Failure> remove(const Hash::Digest& key) = 0;
// Determine whether an attribute is handled by the remote storage
// framework itself.
return result;
}
+std::string
+format_digest(nonstd::span<const uint8_t> data)
+{
+ const size_t base16_bytes = 2;
+ ASSERT(data.size() >= base16_bytes);
+ return util::format_base16({data.data(), base16_bytes})
+ + util::format_base32hex(
+ {data.data() + base16_bytes, data.size() - base16_bytes});
+}
+
std::string
format_human_readable_diff(int64_t diff, SizeUnitPrefixType prefix_type)
{
// characters will be added.
std::string format_base32hex(nonstd::span<const uint8_t> data);
+// Format a hash digest representing `data`.
+//
+// The first two bytes are encoded as four lowercase base16 digits to maintain
+// compatibility with the cleanup algorithm in older ccache versions and to
+// allow for up to four uniform cache levels. The rest are encoded as lowercase
+// base32hex digits without padding characters.
+std::string format_digest(nonstd::span<const uint8_t> data);
+
// Format `diff` as a human-readable string.
std::string format_human_readable_diff(int64_t diff,
SizeUnitPrefixType prefix_type);
-// Copyright (C) 2010-2020 Joel Rosdahl and other contributors
+// Copyright (C) 2010-2023 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
#include "../src/Hash.hpp"
+#include <util/string.hpp>
+
#include "third_party/doctest.h"
TEST_SUITE_BEGIN("Hash");
{
SUBCASE("initial state")
{
- CHECK(Hash().digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi");
+ CHECK(util::format_digest(Hash().digest())
+ == "af1396svbud1kqg40jfa6reciicrpcisi");
}
SUBCASE("empty string")
{
- CHECK(Hash().hash("").digest().to_string()
+ CHECK(util::format_digest(Hash().hash("").digest())
== "af1396svbud1kqg40jfa6reciicrpcisi");
}
SUBCASE("a")
{
- CHECK(Hash().hash("a").digest().to_string()
+ CHECK(util::format_digest(Hash().hash("a").digest())
== "17765vetiqd4ae95qpbhfb1ut8gj42r6m");
}
SUBCASE("message digest")
{
- CHECK(Hash().hash("message digest").digest().to_string()
+ CHECK(util::format_digest(Hash().hash("message digest").digest())
== "7bc2kbnbinerv6ruptldpdrb8ko93hcdo");
}
const char long_string[] =
"123456789012345678901234567890123456789012345678901234567890"
"12345678901234567890";
- CHECK(Hash().hash(long_string).digest().to_string()
+ CHECK(util::format_digest(Hash().hash(long_string).digest())
== "f263ljqhc8co1ee8rpeq98bt654o9o2qm");
}
}
h.hash("message");
h.digest();
h.hash(" digest");
- CHECK(h.digest().to_string() == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo");
+ CHECK(util::format_digest(h.digest()) == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo");
}
TEST_CASE("Hash::digest should be idempotent")
{
Hash h;
- CHECK(h.digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi");
- CHECK(h.digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi");
-}
-
-TEST_CASE("Digest::bytes")
-{
- Digest d = Hash().hash("message digest").digest();
- uint8_t expected[Digest::size()] = {
- 0x7b, 0xc2, 0xa2, 0xee, 0xb9, 0x5d, 0xdb, 0xf9, 0xb7, 0xec,
- 0xf6, 0xad, 0xcb, 0x76, 0xb4, 0x53, 0x09, 0x1c, 0x58, 0xdc,
- };
- CHECK(memcmp(d.bytes(), expected, Digest::size()) == 0);
+ CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi");
+ CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi");
}
TEST_SUITE_END();
config.set_inode_cache(false);
InodeCache inode_cache(config, util::Duration(0));
- Digest digest;
+ Hash::Digest digest;
CHECK(!inode_cache.get(
"a", InodeCache::ContentType::checked_for_temporal_macros, digest));
InodeCache inode_cache(config, util::Duration(0));
util::write_file("a", "");
- Digest digest;
+ Hash::Digest digest;
CHECK(!inode_cache.get(
"a", InodeCache::ContentType::checked_for_temporal_macros, digest));
result.insert(HashSourceCode::found_date);
CHECK(put(inode_cache, "a", "a text", result));
- Digest digest;
+ Hash::Digest digest;
auto return_value = inode_cache.get(
"a", InodeCache::ContentType::checked_for_temporal_macros, digest);
REQUIRE(return_value);
InodeCache inode_cache(config, util::Duration(0));
- Digest digest;
+ Hash::Digest digest;
inode_cache.get("a", InodeCache::ContentType::raw, digest);
CHECK(Stat::stat(inode_cache.get_file()));
InodeCache inode_cache(config, util::Duration(0));
util::write_file("a", "a text");
- Digest binary_digest = Hash().hash("binary").digest();
- Digest code_digest = Hash().hash("code").digest();
+ auto binary_digest = Hash().hash("binary").digest();
+ auto code_digest = Hash().hash("code").digest();
CHECK(inode_cache.put("a",
InodeCache::ContentType::raw,
code_digest,
HashSourceCodeResult(HashSourceCode::found_time)));
- Digest digest;
+ Hash::Digest digest;
auto return_value =
inode_cache.get("a", InodeCache::ContentType::raw, digest);
#include "third_party/doctest.h"
+#include <sys/stat.h>
+
using TestUtil::TestContext;
TEST_SUITE_BEGIN("hashutil");
projects / thirdparty / ccache.git / commitdiff
