From: Joel Rosdahl Date: Fri, 7 Jul 2023 10:00:24 +0000 (+0200) Subject: refactor: Use std::array for hash digests instead of custom class X-Git-Tag: v4.9~148 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0e4e4b63347408c78c2ffe36cb6f71e740e328ca;p=thirdparty%2Fccache.git refactor: Use std::array for hash digests instead of custom class --- diff --git a/src/Context.hpp b/src/Context.hpp index 3eee4e5e8..7f3535fec 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -21,7 +21,6 @@ #include "Args.hpp" #include "ArgsInfo.hpp" #include "Config.hpp" -#include "Digest.hpp" #include "File.hpp" #include "MiniTrace.hpp" #include "NonCopyable.hpp" @@ -30,6 +29,7 @@ # include "InodeCache.hpp" #endif +#include #include #include #include @@ -74,7 +74,7 @@ public: util::TimePoint time_of_compilation; // Files included by the preprocessor and their hashes. - std::unordered_map included_files; + std::unordered_map included_files; // Have we tried and failed to get colored diagnostics? bool diagnostics_color_failed = false; diff --git a/src/Depfile.hpp b/src/Depfile.hpp index af73ff8bc..a58bc37f4 100644 --- a/src/Depfile.hpp +++ b/src/Depfile.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// Copyright (C) 2020-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,8 +21,6 @@ class Context; class Hash; -#include "Digest.hpp" - #include #include #include diff --git a/src/Digest.hpp b/src/Digest.hpp deleted file mode 100644 index 6ab8b9764..000000000 --- a/src/Digest.hpp +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) 2020-2023 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include - -#include "third_party/fmt/core.h" - -#include -#include - -// Digest represents the binary form of the final digest (AKA hash or checksum) -// produced by the hash algorithm. -class Digest -{ -public: - Digest() = default; - - // Access the raw byte buffer, which is `size()` bytes long. - const uint8_t* bytes() const; - uint8_t* bytes(); - - // Size of the digest in bytes. - constexpr static size_t size(); - - // Format the digest as hex string. - std::string to_string() const; - - bool operator==(const Digest& other) const; - bool operator!=(const Digest& other) const; - -private: - constexpr static size_t k_digest_size = 20; - uint8_t m_bytes[k_digest_size]; -}; - -inline const uint8_t* -Digest::bytes() const -{ - return m_bytes; -} - -inline uint8_t* -Digest::bytes() -{ - return m_bytes; -} - -inline constexpr size_t -Digest::size() -{ - return k_digest_size; -} - -inline std::string -Digest::to_string() const -{ - // The first two bytes are encoded as four lowercase base16 digits to maintain - // compatibility with the cleanup algorithm in older ccache versions and to - // allow for up to four uniform cache levels. The rest are encoded as - // lowercase base32hex digits without padding characters. - const size_t base16_bytes = 2; - return util::format_base16({m_bytes, base16_bytes}) - + util::format_base32hex( - {m_bytes + base16_bytes, size() - base16_bytes}); -} - -inline bool -Digest::operator==(const Digest& other) const -{ - return memcmp(bytes(), other.bytes(), size()) == 0; -} - -inline bool -Digest::operator!=(const Digest& other) const -{ - return !(*this == other); -} diff --git a/src/Hash.cpp b/src/Hash.cpp index be1c16bf4..ca68e0eb2 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -54,13 +54,13 @@ Hash::enable_debug(std::string_view section_name, add_debug_text(" ===\n"); } -Digest +Hash::Digest Hash::digest() const { // Note that blake3_hasher_finalize doesn't modify the hasher itself, thus it // is possible to finalize again after more data has been added. Digest digest; - blake3_hasher_finalize(&m_hasher, digest.bytes(), digest.size()); + blake3_hasher_finalize(&m_hasher, digest.data(), digest.size()); return digest; } diff --git a/src/Hash.hpp b/src/Hash.hpp index 3e8d5573c..a27ed88a6 100644 --- a/src/Hash.hpp +++ b/src/Hash.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// Copyright (C) 2020-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,11 +18,10 @@ #pragma once -#include "Digest.hpp" - #include "third_party/blake3/blake3.h" #include +#include #include #include #include @@ -33,6 +32,8 @@ class Hash public: enum class HashType { binary, text }; + using Digest = std::array; + Hash(); Hash(const Hash& other) = default; diff --git a/src/InodeCache.cpp b/src/InodeCache.cpp index db75d58bd..3d18f37af 100644 --- a/src/InodeCache.cpp +++ b/src/InodeCache.cpp @@ -19,7 +19,6 @@ #include "InodeCache.hpp" #include "Config.hpp" -#include "Digest.hpp" #include "Finalizer.hpp" #include "Hash.hpp" #include "Logging.hpp" @@ -83,9 +82,9 @@ const uint64_t k_min_fs_mib_left = 100; // 100 MiB // How long a filesystem space check is valid before we make a new one. const util::Duration k_fs_space_check_valid_duration(1); -static_assert(Digest::size() == 20, +static_assert(std::tuple_size() == 20, "Increment version number if size of digest is changed."); -static_assert(std::is_trivially_copyable::value, +static_assert(std::is_trivially_copyable::value, "Digest is expected to be trivially copyable."); static_assert( @@ -207,9 +206,9 @@ struct InodeCache::Key struct InodeCache::Entry { - Digest key_digest; // Hashed key - Digest file_digest; // Cached file hash - int return_value; // Cached return value + Hash::Digest key_digest; // Hashed key + Hash::Digest file_digest; // Cached file hash + int return_value; // Cached return value }; struct InodeCache::Bucket @@ -275,7 +274,7 @@ InodeCache::mmap_file(const std::string& inode_cache_file) bool InodeCache::hash_inode(const std::string& path, ContentType type, - Digest& digest) + Hash::Digest& digest) { Stat stat = Stat::stat(path); if (!stat) { @@ -307,11 +306,11 @@ InodeCache::hash_inode(const std::string& path, } bool -InodeCache::with_bucket(const Digest& key_digest, +InodeCache::with_bucket(const Hash::Digest& key_digest, const BucketHandler& bucket_handler) { uint32_t hash; - Util::big_endian_to_int(key_digest.bytes(), hash); + Util::big_endian_to_int(key_digest.data(), hash); const uint32_t index = hash % k_num_buckets; Bucket* bucket = &m_sr->buckets[index]; bool acquired_lock = spin_lock(bucket->owner_pid, m_self_pid); @@ -466,13 +465,15 @@ InodeCache::available(int fd) } std::optional -InodeCache::get(const std::string& path, ContentType type, Digest& file_digest) +InodeCache::get(const std::string& path, + ContentType type, + Hash::Digest& file_digest) { if (!initialize()) { return std::nullopt; } - Digest key_digest; + Hash::Digest key_digest; if (!hash_inode(path, type, key_digest)) { return std::nullopt; } @@ -512,14 +513,14 @@ InodeCache::get(const std::string& path, ContentType type, Digest& file_digest) bool InodeCache::put(const std::string& path, ContentType type, - const Digest& file_digest, + const Hash::Digest& file_digest, HashSourceCodeResult return_value) { if (!initialize()) { return false; } - Digest key_digest; + Hash::Digest key_digest; if (!hash_inode(path, type, key_digest)) { return false; } diff --git a/src/InodeCache.hpp b/src/InodeCache.hpp index 98276a343..8d54a1fbe 100644 --- a/src/InodeCache.hpp +++ b/src/InodeCache.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include @@ -30,7 +31,6 @@ class Config; class Context; -class Digest; class InodeCache { @@ -79,7 +79,7 @@ public: // Returns true if saved values could be retrieved from the cache, false // otherwise. std::optional - get(const std::string& path, ContentType type, Digest& file_digest); + get(const std::string& path, ContentType type, Hash::Digest& file_digest); // Put hash digest and return value from a successful call to do_hash_file() // in hashutil.cpp. @@ -87,7 +87,7 @@ public: // Returns true if values could be stored in the cache, false otherwise. bool put(const std::string& path, ContentType type, - const Digest& file_digest, + const Hash::Digest& file_digest, HashSourceCodeResult return_value); // Unmaps the current cache and removes the mapped file from disk. @@ -124,10 +124,15 @@ private: using BucketHandler = std::function; bool mmap_file(const std::string& inode_cache_file); - bool hash_inode(const std::string& path, ContentType type, Digest& digest); - bool with_bucket(const Digest& key_digest, + + bool + hash_inode(const std::string& path, ContentType type, Hash::Digest& digest); + + bool with_bucket(const Hash::Digest& key_digest, const BucketHandler& bucket_handler); + static bool create_new_file(const std::string& filename); + bool initialize(); const Config& m_config; diff --git a/src/ccache.cpp b/src/ccache.cpp index fa388445c..be1984748 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -383,7 +383,7 @@ do_remember_include_file(Context& ctx, } // Let's hash the include file content. - Digest file_digest; + Hash::Digest file_digest; if (is_pch) { if (ctx.args_info.included_pch_file.empty()) { @@ -405,7 +405,7 @@ do_remember_include_file(Context& ctx, return false; } cpp_hash.hash_delimiter(using_pch_sum ? "pch_sum_hash" : "pch_hash"); - cpp_hash.hash(file_digest.to_string()); + cpp_hash.hash(util::format_digest(file_digest)); } if (ctx.config.direct_mode()) { @@ -421,7 +421,7 @@ do_remember_include_file(Context& ctx, if (depend_mode_hash) { depend_mode_hash->hash_delimiter("include"); - depend_mode_hash->hash(file_digest.to_string()); + depend_mode_hash->hash(util::format_digest(file_digest)); } } @@ -656,7 +656,7 @@ process_preprocessed_file(Context& ctx, Hash& hash, const std::string& path) // Extract the used includes from the dependency file. Note that we cannot // distinguish system headers from other includes here. -static std::optional +static std::optional result_key_from_depfile(Context& ctx, Hash& hash) { // Make sure that result hash will always be different from the manifest hash @@ -729,7 +729,7 @@ struct DoExecuteResult // Extract the used includes from /showIncludes output in stdout. Note that we // cannot distinguish system headers from other includes here. -static std::optional +static std::optional result_key_from_includes(Context& ctx, Hash& hash, std::string_view stdout_data) { for (std::string_view include : core::MsvcShowIncludesOutput::get_includes( @@ -831,8 +831,8 @@ read_manifest(Context& ctx, nonstd::span cache_entry_data) static void update_manifest(Context& ctx, - const Digest& manifest_key, - const Digest& result_key) + const Hash::Digest& manifest_key, + const Hash::Digest& result_key) { if (ctx.config.read_only() || ctx.config.read_only_direct()) { return; @@ -865,13 +865,14 @@ update_manifest(Context& ctx, }; }); if (added) { - LOG("Added result key to manifest {}", manifest_key.to_string()); + LOG("Added result key to manifest {}", util::format_digest(manifest_key)); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); ctx.storage.put(manifest_key, core::CacheEntryType::manifest, core::CacheEntry::serialize(header, ctx.manifest)); } else { - LOG("Did not add result key to manifest {}", manifest_key.to_string()); + LOG("Did not add result key to manifest {}", + util::format_digest(manifest_key)); } } @@ -915,7 +916,7 @@ find_coverage_file(const Context& ctx) [[nodiscard]] static bool write_result(Context& ctx, - const Digest& result_key, + const Hash::Digest& result_key, const Stat& obj_stat, const util::Bytes& stdout_data, const util::Bytes& stderr_data) @@ -1043,10 +1044,10 @@ rewrite_stdout_from_compiler(const Context& ctx, util::Bytes&& stdout_data) } // Run the real compiler and put the result in cache. Returns the result key. -static nonstd::expected +static nonstd::expected to_cache(Context& ctx, Args& args, - std::optional result_key, + std::optional result_key, const Args& depend_extra_args, Hash* depend_mode_hash) { @@ -1159,7 +1160,7 @@ to_cache(Context& ctx, return nonstd::make_unexpected(Statistic::internal_error); } LOG_RAW("Got result key from dependency file"); - LOG("Result key: {}", result_key->to_string()); + LOG("Result key: {}", util::format_digest(*result_key)); } ASSERT(result_key); @@ -1209,7 +1210,7 @@ to_cache(Context& ctx, // Find the result key by running the compiler in preprocessor mode and // hashing the result. -static nonstd::expected +static nonstd::expected get_result_key_from_cpp(Context& ctx, Args& args, Hash& hash) { ctx.time_of_compilation = util::TimePoint::now(); @@ -1792,7 +1793,7 @@ hash_argument(const Context& ctx, return {}; } -static nonstd::expected, Failure> +static nonstd::expected, Failure> get_manifest_key(Context& ctx, Hash& hash) { // Hash environment variables that affect the preprocessor output. @@ -1825,7 +1826,7 @@ get_manifest_key(Context& ctx, Hash& hash) hash.hash(ctx.args_info.input_file); hash.hash_delimiter("sourcecode hash"); - Digest input_file_digest; + Hash::Digest input_file_digest; auto ret = hash_source_code_file(ctx, input_file_digest, ctx.args_info.input_file); if (ret.contains(HashSourceCode::error)) { @@ -1836,7 +1837,7 @@ get_manifest_key(Context& ctx, Hash& hash) ctx.config.set_direct_mode(false); return {}; } - hash.hash(input_file_digest.to_string()); + hash.hash(util::format_digest(input_file_digest)); return hash.digest(); } @@ -1916,11 +1917,11 @@ hash_profiling_related_data(const Context& ctx, Hash& hash) return {}; } -static std::optional -get_result_key_from_manifest(Context& ctx, const Digest& manifest_key) +static std::optional +get_result_key_from_manifest(Context& ctx, const Hash::Digest& manifest_key) { MTR_BEGIN("manifest", "manifest_get"); - std::optional result_key; + std::optional result_key; size_t read_manifests = 0; ctx.storage.get( manifest_key, core::CacheEntryType::manifest, [&](util::Bytes&& value) { @@ -1942,7 +1943,8 @@ get_result_key_from_manifest(Context& ctx, const Digest& manifest_key) MTR_END("manifest", "manifest_get"); if (read_manifests > 1 && !ctx.config.remote_only()) { MTR_SCOPE("manifest", "merge"); - LOG("Storing merged manifest {} locally", manifest_key.to_string()); + LOG("Storing merged manifest {} locally", + util::format_digest(manifest_key)); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); ctx.storage.local.put(manifest_key, core::CacheEntryType::manifest, @@ -1955,8 +1957,9 @@ get_result_key_from_manifest(Context& ctx, const Digest& manifest_key) // Update a hash sum with information specific to the direct and preprocessor // modes and calculate the result key. Returns the result key on success, and // if direct_mode is true also the manifest key. -static nonstd::expected, std::optional>, - Failure> +static nonstd::expected< + std::pair, std::optional>, + Failure> calculate_result_and_manifest_key(Context& ctx, const Args& args, Hash& hash, @@ -2005,8 +2008,8 @@ calculate_result_and_manifest_key(Context& ctx, hash.hash(arch); } - std::optional result_key; - std::optional manifest_key; + std::optional result_key; + std::optional manifest_key; if (direct_mode) { const auto manifest_key_result = get_manifest_key(ctx, hash); @@ -2015,7 +2018,7 @@ calculate_result_and_manifest_key(Context& ctx, } manifest_key = *manifest_key_result; if (manifest_key) { - LOG("Manifest key: {}", manifest_key->to_string()); + LOG("Manifest key: {}", util::format_digest(*manifest_key)); result_key = get_result_key_from_manifest(ctx, *manifest_key); } } else if (ctx.args_info.arch_args.empty()) { @@ -2046,7 +2049,7 @@ calculate_result_and_manifest_key(Context& ctx, } if (result_key) { - LOG("Result key: {}", result_key->to_string()); + LOG("Result key: {}", util::format_digest(*result_key)); } return std::make_pair(result_key, manifest_key); } @@ -2055,7 +2058,7 @@ enum class FromCacheCallMode { direct, cpp }; // Try to return the compile result from cache. static nonstd::expected -from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) +from_cache(Context& ctx, FromCacheCallMode mode, const Hash::Digest& result_key) { // The user might be disabling cache hits. if (ctx.config.recache()) { @@ -2099,11 +2102,13 @@ from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) deserializer.visit(result_retriever); } catch (core::ResultRetriever::WriteError& e) { LOG("Write error when retrieving result from {}: {}", - result_key.to_string(), + util::format_digest(result_key), e.what()); return nonstd::make_unexpected(Statistic::bad_output_file); } catch (core::Error& e) { - LOG("Failed to get result from {}: {}", result_key.to_string(), e.what()); + LOG("Failed to get result from {}: {}", + util::format_digest(result_key), + e.what()); return false; } @@ -2519,9 +2524,9 @@ do_cache_compilation(Context& ctx) args_to_hash.push_back(processed.extra_args_to_hash); bool put_result_in_manifest = false; - std::optional result_key; - std::optional result_key_from_manifest; - std::optional manifest_key; + std::optional result_key; + std::optional result_key_from_manifest; + std::optional manifest_key; if (ctx.config.direct_mode()) { LOG_RAW("Trying direct lookup"); diff --git a/src/core/Manifest.cpp b/src/core/Manifest.cpp index 6f9b1b725..784075665 100644 --- a/src/core/Manifest.cpp +++ b/src/core/Manifest.cpp @@ -27,6 +27,7 @@ #include #include #include +#include // Manifest data format // ==================== @@ -44,7 +45,7 @@ // ::= uint32_t // ::= // ::= uint32_t -// ::= Digest::size() bytes +// ::= Hash::Digest::size() bytes // ::= uint64_t ; file size // ::= int64_t ; modification time (ns), 0 = not recorded // ::= int64_t ; status change time (ns), 0 = not recorded @@ -53,7 +54,7 @@ // ::= * // ::= uint32_t // ::= uint32_t -// ::= Digest::size() bytes +// ::= Hash::Digest::size() bytes const uint32_t k_max_manifest_entries = 100; const uint32_t k_max_manifest_file_info_entries = 10000; @@ -111,7 +112,7 @@ Manifest::read(nonstd::span data) auto& entry = file_infos.back(); reader.read_int(entry.index); - reader.read_and_copy_bytes({entry.digest.bytes(), Digest::size()}); + reader.read_and_copy_bytes(entry.digest); reader.read_int(entry.fsize); entry.mtime.set_nsec(reader.read_int()); entry.ctime.set_nsec(reader.read_int()); @@ -126,7 +127,7 @@ Manifest::read(nonstd::span data) for (uint32_t j = 0; j < file_info_index_count; ++j) { entry.file_info_indexes.push_back(reader.read_int()); } - reader.read_and_copy_bytes({entry.key.bytes(), Digest::size()}); + reader.read_and_copy_bytes(entry.key); } if (m_results.empty()) { @@ -135,7 +136,7 @@ Manifest::read(nonstd::span data) m_results = std::move(results); } else { for (const auto& result : results) { - std::unordered_map included_files; + std::unordered_map included_files; std::unordered_map included_files_stats; for (auto file_info_index : result.file_info_indexes) { const auto& file_info = file_infos[file_info_index]; @@ -151,11 +152,11 @@ Manifest::read(nonstd::span data) } } -std::optional +std::optional Manifest::look_up_result_digest(const Context& ctx) const { std::unordered_map stated_files; - std::unordered_map hashed_files; + std::unordered_map hashed_files; // Check newest result first since it's a more likely to match. for (size_t i = m_results.size(); i > 0; i--) { @@ -170,8 +171,8 @@ Manifest::look_up_result_digest(const Context& ctx) const bool Manifest::add_result( - const Digest& result_key, - const std::unordered_map& included_files, + const Hash::Digest& result_key, + const std::unordered_map& included_files, const FileStater& stat_file_function) { if (m_results.size() > k_max_manifest_entries) { @@ -235,12 +236,13 @@ Manifest::serialized_size() const size += 2 + file.length(); } size += 4; // n_file_infos - size += m_file_infos.size() * (4 + Digest::size() + 8 + 8 + 8); + size += + m_file_infos.size() * (4 + std::tuple_size() + 8 + 8 + 8); size += 4; // n_results for (const auto& result : m_results) { size += 4; // n_file_info_indexes size += result.file_info_indexes.size() * 4; - size += Digest::size(); + size += std::tuple_size(); } // In order to support 32-bit ccache builds, restrict size to uint32_t for @@ -268,7 +270,7 @@ Manifest::serialize(util::Bytes& output) writer.write_int(m_file_infos.size()); for (const auto& file_info : m_file_infos) { writer.write_int(file_info.index); - writer.write_bytes({file_info.digest.bytes(), Digest::size()}); + writer.write_bytes(file_info.digest); writer.write_int(file_info.fsize); writer.write_int(file_info.mtime.nsec()); writer.write_int(file_info.ctime.nsec()); @@ -280,7 +282,7 @@ Manifest::serialize(util::Bytes& output) for (auto index : result.file_info_indexes) { writer.write_int(index); } - writer.write_bytes({result.key.bytes(), Digest::size()}); + writer.write_bytes(result.key); } } @@ -308,7 +310,7 @@ Manifest::clear() uint32_t Manifest::get_file_info_index( const std::string& path, - const Digest& digest, + const Hash::Digest& digest, const std::unordered_map& mf_files, const std::unordered_map& mf_file_infos, const FileStater& file_stater) @@ -344,7 +346,7 @@ Manifest::result_matches( const Context& ctx, const ResultEntry& result, std::unordered_map& stated_files, - std::unordered_map& hashed_files) const + std::unordered_map& hashed_files) const { for (uint32_t file_info_index : result.file_info_indexes) { const auto& fi = m_file_infos[file_info_index]; @@ -402,7 +404,7 @@ Manifest::result_matches( auto hashed_files_iter = hashed_files.find(path); if (hashed_files_iter == hashed_files.end()) { - Digest actual_digest; + Hash::Digest actual_digest; auto ret = hash_source_code_file(ctx, actual_digest, path, fs.size); if (ret.contains(HashSourceCode::error)) { LOG("Failed hashing {}", path); @@ -437,7 +439,8 @@ Manifest::inspect(FILE* const stream) const for (size_t i = 0; i < m_file_infos.size(); ++i) { PRINT(stream, " {}:\n", i); PRINT(stream, " Path index: {}\n", m_file_infos[i].index); - PRINT(stream, " Hash: {}\n", m_file_infos[i].digest.to_string()); + PRINT( + stream, " Hash: {}\n", util::format_digest(m_file_infos[i].digest)); PRINT(stream, " File size: {}\n", m_file_infos[i].fsize); if (m_file_infos[i].mtime == util::TimePoint()) { PRINT_RAW(stream, " Mtime: -\n"); @@ -465,7 +468,7 @@ Manifest::inspect(FILE* const stream) const PRINT(stream, " {}", file_info_index); } PRINT_RAW(stream, "\n"); - PRINT(stream, " Key: {}\n", m_results[i].key.to_string()); + PRINT(stream, " Key: {}\n", util::format_digest(m_results[i].key)); } } diff --git a/src/core/Manifest.hpp b/src/core/Manifest.hpp index a8f3b67e7..c4b1d3b3b 100644 --- a/src/core/Manifest.hpp +++ b/src/core/Manifest.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2022 Joel Rosdahl and other contributors +// Copyright (C) 2009-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,7 +18,7 @@ #pragma once -#include +#include #include #include @@ -53,11 +53,12 @@ public: void read(nonstd::span data); - std::optional look_up_result_digest(const Context& ctx) const; + std::optional look_up_result_digest(const Context& ctx) const; - bool add_result(const Digest& result_key, - const std::unordered_map& included_files, - const FileStater& stat_file); + bool add_result( + const Hash::Digest& result_key, + const std::unordered_map& included_files, + const FileStater& stat_file); // core::Serializer uint32_t serialized_size() const override; @@ -69,7 +70,7 @@ private: struct FileInfo { uint32_t index; // Index to m_files. - Digest digest; // Digest of referenced file. + Hash::Digest digest; // Hash::Digest of referenced file. uint64_t fsize; // Size of referenced file. util::TimePoint mtime; // mtime of referenced file. util::TimePoint ctime; // ctime of referenced file. @@ -82,7 +83,7 @@ private: struct ResultEntry { std::vector file_info_indexes; // Indexes to m_file_infos. - Digest key; // Key of the result. + Hash::Digest key; // Key of the result. bool operator==(const ResultEntry& other) const; }; @@ -95,16 +96,16 @@ private: uint32_t get_file_info_index( const std::string& path, - const Digest& digest, + const Hash::Digest& digest, const std::unordered_map& mf_files, const std::unordered_map& mf_file_infos, const FileStater& file_state); - bool - result_matches(const Context& ctx, - const ResultEntry& result, - std::unordered_map& stated_files, - std::unordered_map& hashed_files) const; + bool result_matches( + const Context& ctx, + const ResultEntry& result, + std::unordered_map& stated_files, + std::unordered_map& hashed_files) const; }; } // namespace core diff --git a/src/core/ResultRetriever.cpp b/src/core/ResultRetriever.cpp index 8312b8f5a..8b2e5c625 100644 --- a/src/core/ResultRetriever.cpp +++ b/src/core/ResultRetriever.cpp @@ -46,7 +46,7 @@ namespace core { using Result::FileType; ResultRetriever::ResultRetriever(const Context& ctx, - std::optional result_key) + std::optional result_key) : m_ctx(ctx), m_result_key(result_key) { diff --git a/src/core/ResultRetriever.hpp b/src/core/ResultRetriever.hpp index 92a3af885..fb16857c7 100644 --- a/src/core/ResultRetriever.hpp +++ b/src/core/ResultRetriever.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// Copyright (C) 2020-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,7 +20,7 @@ #include "Fd.hpp" -#include +#include #include #include @@ -42,7 +42,7 @@ public: //`path` should be the path to the local result entry file if the result comes // from local storage. ResultRetriever(const Context& ctx, - std::optional result_key = std::nullopt); + std::optional result_key = std::nullopt); void on_embedded_file(uint8_t file_number, Result::FileType file_type, @@ -53,7 +53,7 @@ public: private: const Context& m_ctx; - std::optional m_result_key; + std::optional m_result_key; std::string get_dest_path(Result::FileType file_type) const; diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index f9cd2cf80..051b9fa16 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -628,7 +628,7 @@ process_main_options(int argc, const char* const* argv) const auto result = arg == "-" ? hash.hash_fd(STDIN_FILENO) : hash.hash_file(arg); if (result) { - PRINT(stdout, "{}\n", hash.digest().to_string()); + PRINT(stdout, "{}\n", util::format_digest(hash.digest())); } else { PRINT(stderr, "Error: Failed to hash {}: {}\n", arg, result.error()); return EXIT_FAILURE; diff --git a/src/hashutil.cpp b/src/hashutil.cpp index 1b019f9e9..921238ae0 100644 --- a/src/hashutil.cpp +++ b/src/hashutil.cpp @@ -21,7 +21,6 @@ #include "Args.hpp" #include "Config.hpp" #include "Context.hpp" -#include "Hash.hpp" #include "Logging.hpp" #include "Stat.hpp" #include "Util.hpp" @@ -177,7 +176,7 @@ check_for_temporal_macros_avx2(std::string_view str) HashSourceCodeResult do_hash_file(const Context& ctx, - Digest& digest, + Hash::Digest& digest, const std::string& path, size_t size_hint, bool check_temporal_macros) @@ -233,7 +232,7 @@ check_for_temporal_macros(std::string_view str) HashSourceCodeResult hash_source_code_file(const Context& ctx, - Digest& digest, + Hash::Digest& digest, const std::string& path, size_t size_hint) { @@ -263,7 +262,7 @@ hash_source_code_file(const Context& ctx, // macro expansions. Hash hash; - hash.hash(digest.to_string()); + hash.hash(util::format_digest(digest)); if (result.contains(HashSourceCode::found_date)) { LOG("Found __DATE__ in {}", path); @@ -322,7 +321,7 @@ hash_source_code_file(const Context& ctx, bool hash_binary_file(const Context& ctx, - Digest& digest, + Hash::Digest& digest, const std::string& path, size_t size_hint) { @@ -332,10 +331,10 @@ hash_binary_file(const Context& ctx, bool hash_binary_file(const Context& ctx, Hash& hash, const std::string& path) { - Digest digest; + Hash::Digest digest; const bool success = hash_binary_file(ctx, digest, path); if (success) { - hash.hash(digest.to_string()); + hash.hash(util::format_digest(digest)); } return success; } diff --git a/src/hashutil.hpp b/src/hashutil.hpp index c270b38e8..c036b429a 100644 --- a/src/hashutil.hpp +++ b/src/hashutil.hpp @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -26,8 +27,6 @@ class Config; class Context; -class Digest; -class Hash; enum class HashSourceCode { ok = 0, @@ -44,7 +43,7 @@ HashSourceCodeResult check_for_temporal_macros(std::string_view str); // Hash a source code file using the inode cache if enabled. HashSourceCodeResult hash_source_code_file(const Context& ctx, - Digest& digest, + Hash::Digest& digest, const std::string& path, size_t size_hint = 0); @@ -53,7 +52,7 @@ HashSourceCodeResult hash_source_code_file(const Context& ctx, // // Returns true on success, otherwise false. bool hash_binary_file(const Context& ctx, - Digest& digest, + Hash::Digest& digest, const std::string& path, size_t size_hint = 0); diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 775f4be77..7f15f9fe0 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// Copyright (C) 2021-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +33,6 @@ #ifdef HAVE_REDIS_STORAGE_BACKEND # include #endif -#include #include #include #include @@ -231,7 +231,7 @@ Storage::finalize() } void -Storage::get(const Digest& key, +Storage::get(const Hash::Hash::Digest& key, const core::CacheEntryType type, const EntryReceiver& entry_receiver) { @@ -258,7 +258,7 @@ Storage::get(const Digest& key, } void -Storage::put(const Digest& key, +Storage::put(const Hash::Digest& key, const core::CacheEntryType type, nonstd::span value) { @@ -271,7 +271,7 @@ Storage::put(const Digest& key, } void -Storage::remove(const Digest& key, const core::CacheEntryType type) +Storage::remove(const Hash::Digest& key, const core::CacheEntryType type) { MTR_SCOPE("storage", "remove"); @@ -346,7 +346,7 @@ to_half_open_unit_interval(uint64_t value) } static Url -get_shard_url(const Digest& key, +get_shard_url(const Hash::Digest& key, const std::string& url, const std::vector& shards) { @@ -357,7 +357,7 @@ get_shard_url(const Digest& key, std::string best_shard; for (const auto& shard_config : shards) { util::XXH3_64 hash; - hash.update(key.bytes(), key.size()); + hash.update(key.data(), key.size()); hash.update(shard_config.name.data(), shard_config.name.length()); const double score = to_half_open_unit_interval(hash.digest()); ASSERT(score >= 0.0 && score < 1.0); @@ -374,7 +374,7 @@ get_shard_url(const Digest& key, RemoteStorageBackendEntry* Storage::get_backend(RemoteStorageEntry& entry, - const Digest& key, + const Hash::Digest& key, const std::string_view operation_description, const bool for_writing) { @@ -422,7 +422,7 @@ Storage::get_backend(RemoteStorageEntry& entry, } void -Storage::get_from_remote_storage(const Digest& key, +Storage::get_from_remote_storage(const Hash::Digest& key, const core::CacheEntryType type, const EntryReceiver& entry_receiver) { @@ -445,7 +445,7 @@ Storage::get_from_remote_storage(const Digest& key, auto& value = *result; if (value) { LOG("Retrieved {} from {} ({:.2f} ms)", - key.to_string(), + util::format_digest(key), backend->url_for_logging, ms); local.increment_statistic(core::Statistic::remote_storage_read_hit); @@ -457,7 +457,7 @@ Storage::get_from_remote_storage(const Digest& key, } } else { LOG("No {} in {} ({:.2f} ms)", - key.to_string(), + util::format_digest(key), backend->url_for_logging, ms); local.increment_statistic(core::Statistic::remote_storage_read_miss); @@ -466,7 +466,7 @@ Storage::get_from_remote_storage(const Digest& key, } void -Storage::put_in_remote_storage(const Digest& key, +Storage::put_in_remote_storage(const Hash::Digest& key, nonstd::span value, bool only_if_missing) { @@ -474,7 +474,7 @@ Storage::put_in_remote_storage(const Digest& key, if (!core::CacheEntry::Header(value).self_contained) { LOG("Not putting {} in remote storage since it's not self-contained", - key.to_string()); + util::format_digest(key)); return; } @@ -496,7 +496,7 @@ Storage::put_in_remote_storage(const Digest& key, const bool stored = *result; LOG("{} {} in {} ({:.2f} ms)", stored ? "Stored" : "Did not have to store", - key.to_string(), + util::format_digest(key), backend->url_for_logging, ms); local.increment_statistic(core::Statistic::remote_storage_write); @@ -504,7 +504,7 @@ Storage::put_in_remote_storage(const Digest& key, } void -Storage::remove_from_remote_storage(const Digest& key) +Storage::remove_from_remote_storage(const Hash::Digest& key) { MTR_SCOPE("remote_storage", "remove"); @@ -525,12 +525,12 @@ Storage::remove_from_remote_storage(const Digest& key) const bool removed = *result; if (removed) { LOG("Removed {} from {} ({:.2f} ms)", - key.to_string(), + util::format_digest(key), backend->url_for_logging, ms); } else { LOG("No {} to remove from {} ({:.2f} ms)", - key.to_string(), + util::format_digest(key), backend->url_for_logging, ms); } diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp index 1b72106ef..3a63ffe94 100644 --- a/src/storage/Storage.hpp +++ b/src/storage/Storage.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// Copyright (C) 2021-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -32,8 +33,6 @@ #include #include -class Digest; - namespace storage { std::string get_features(); @@ -54,15 +53,15 @@ public: using EntryReceiver = std::function; - void get(const Digest& key, + void get(const Hash::Digest& key, core::CacheEntryType type, const EntryReceiver& entry_receiver); - void put(const Digest& key, + void put(const Hash::Digest& key, core::CacheEntryType type, nonstd::span value); - void remove(const Digest& key, core::CacheEntryType type); + void remove(const Hash::Digest& key, core::CacheEntryType type); bool has_remote_storage() const; std::string get_remote_storage_config_for_logging() const; @@ -77,19 +76,19 @@ private: remote::RemoteStorage::Backend::Failure failure); RemoteStorageBackendEntry* get_backend(RemoteStorageEntry& entry, - const Digest& key, + const Hash::Digest& key, std::string_view operation_description, const bool for_writing); - void get_from_remote_storage(const Digest& key, + void get_from_remote_storage(const Hash::Digest& key, core::CacheEntryType type, const EntryReceiver& entry_receiver); - void put_in_remote_storage(const Digest& key, + void put_in_remote_storage(const Hash::Digest& key, nonstd::span value, bool only_if_missing); - void remove_from_remote_storage(const Digest& key); + void remove_from_remote_storage(const Hash::Digest& key); }; } // namespace storage diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp index fe423989f..0ffd74ae0 100644 --- a/src/storage/local/LocalStorage.cpp +++ b/src/storage/local/LocalStorage.cpp @@ -31,25 +31,22 @@ #include #include #include -#include #include #include #include #include -#include -#include #include #include #include #include #include -#include - #ifdef INODE_CACHE_SUPPORTED # include #endif +#include + #include #include #include @@ -363,7 +360,7 @@ LocalStorage::finalize() } std::optional -LocalStorage::get(const Digest& key, const core::CacheEntryType type) +LocalStorage::get(const Hash::Digest& key, const core::CacheEntryType type) { MTR_SCOPE("local_storage", "get"); @@ -374,7 +371,7 @@ LocalStorage::get(const Digest& key, const core::CacheEntryType type) const auto value = util::read_file(cache_file.path); if (value) { LOG("Retrieved {} from local storage ({})", - key.to_string(), + util::format_digest(key), cache_file.path); // Update modification timestamp to save file from LRU cleanup. @@ -385,7 +382,7 @@ LocalStorage::get(const Digest& key, const core::CacheEntryType type) LOG("Failed to read {}: {}", cache_file.path, value.error()); } } else { - LOG("No {} in local storage", key.to_string()); + LOG("No {} in local storage", util::format_digest(key)); } increment_statistic(return_value ? Statistic::local_storage_read_hit @@ -398,7 +395,7 @@ LocalStorage::get(const Digest& key, const core::CacheEntryType type) } void -LocalStorage::put(const Digest& key, +LocalStorage::put(const Hash::Digest& key, const core::CacheEntryType type, nonstd::span value, bool only_if_missing) @@ -428,7 +425,9 @@ LocalStorage::put(const Digest& key, return; } - LOG("Stored {} in local storage ({})", key.to_string(), cache_file.path); + LOG("Stored {} in local storage ({})", + util::format_digest(key), + cache_file.path); m_stored_data = true; if (!m_config.stats()) { @@ -460,17 +459,17 @@ LocalStorage::put(const Digest& key, // be done almost anywhere, but we might as well do it near the end as we save // the stat call if we exit early. util::create_cachedir_tag( - FMT("{}/{}", m_config.cache_dir(), key.to_string()[0])); + FMT("{}/{}", m_config.cache_dir(), util::format_digest(key)[0])); } void -LocalStorage::remove(const Digest& key, const core::CacheEntryType type) +LocalStorage::remove(const Hash::Digest& key, const core::CacheEntryType type) { MTR_SCOPE("local_storage", "remove"); const auto cache_file = look_up_cache_file(key, type); if (!cache_file.stat) { - LOG("No {} to remove from local storage", key.to_string()); + LOG("No {} to remove from local storage", util::format_digest(key)); return; } @@ -484,7 +483,9 @@ LocalStorage::remove(const Digest& key, const core::CacheEntryType type) Util::unlink_safe(cache_file.path); } - LOG("Removed {} from local storage ({})", key.to_string(), cache_file.path); + LOG("Removed {} from local storage ({})", + util::format_digest(key), + cache_file.path); increment_level_2_counters( key, -1, -static_cast(cache_file.stat.size_on_disk() / 1024)); } @@ -505,7 +506,7 @@ LocalStorage::get_raw_file_path(std::string_view result_path, } std::string -LocalStorage::get_raw_file_path(const Digest& result_key, +LocalStorage::get_raw_file_path(const Hash::Digest& result_key, uint8_t file_number) const { const auto cache_file = @@ -515,7 +516,7 @@ LocalStorage::get_raw_file_path(const Digest& result_key, void LocalStorage::put_raw_files( - const Digest& key, + const Hash::Digest& key, const std::vector raw_files) { const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result); @@ -856,10 +857,11 @@ LocalStorage::get_subdir(uint8_t l1_index, uint8_t l2_index) const } LocalStorage::LookUpCacheFileResult -LocalStorage::look_up_cache_file(const Digest& key, +LocalStorage::look_up_cache_file(const Hash::Digest& key, const core::CacheEntryType type) const { - const auto key_string = FMT("{}{}", key.to_string(), suffix_from_type(type)); + const auto key_string = + FMT("{}{}", util::format_digest(key), suffix_from_type(type)); for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; ++level) { @@ -890,14 +892,14 @@ LocalStorage::get_stats_file(uint8_t l1_index, uint8_t l2_index) const void LocalStorage::move_to_wanted_cache_level(const StatisticsCounters& counters, - const Digest& key, + const Hash::Digest& key, core::CacheEntryType type, const std::string& cache_file_path) { const auto wanted_level = calculate_wanted_cache_level(counters.get(Statistic::files_in_cache)); - const auto wanted_path = - get_path_in_cache(wanted_level, key.to_string() + suffix_from_type(type)); + const auto wanted_path = get_path_in_cache( + wanted_level, util::format_digest(key) + suffix_from_type(type)); if (cache_file_path != wanted_path) { Util::ensure_dir_exists(Util::dir_name(wanted_path)); LOG("Moving {} to {}", cache_file_path, wanted_path); @@ -941,12 +943,12 @@ LocalStorage::recount_level_1_dir(util::LongLivedLockFileManager& lock_manager, } std::optional -LocalStorage::increment_level_2_counters(const Digest& key, +LocalStorage::increment_level_2_counters(const Hash::Digest& key, int64_t files, int64_t size_kibibyte) { - uint8_t l1_index = key.bytes()[0] >> 4; - uint8_t l2_index = key.bytes()[0] & 0xF; + uint8_t l1_index = key[0] >> 4; + uint8_t l2_index = key[0] & 0xF; const auto level_1_stats_file = get_stats_file(l1_index); return level_1_stats_file.update([&](auto& cs) { // Level 1 counters: @@ -1326,9 +1328,9 @@ LocalStorage::get_auto_cleanup_lock() const } util::LockFile -LocalStorage::get_level_2_content_lock(const Digest& key) const +LocalStorage::get_level_2_content_lock(const Hash::Digest& key) const { - return get_level_2_content_lock(key.bytes()[0] >> 4, key.bytes()[0] & 0xF); + return get_level_2_content_lock(key[0] >> 4, key[0] & 0xF); } util::LockFile diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp index 29f8a028f..1a49aa6d1 100644 --- a/src/storage/local/LocalStorage.hpp +++ b/src/storage/local/LocalStorage.hpp @@ -18,7 +18,7 @@ #pragma once -#include +#include #include #include #include @@ -65,22 +65,23 @@ public: // --- Cache entry handling --- - std::optional get(const Digest& key, core::CacheEntryType type); + std::optional get(const Hash::Digest& key, + core::CacheEntryType type); - void put(const Digest& key, + void put(const Hash::Digest& key, core::CacheEntryType type, nonstd::span value, bool only_if_missing = false); - void remove(const Digest& key, core::CacheEntryType type); + void remove(const Hash::Digest& key, core::CacheEntryType type); static std::string get_raw_file_path(std::string_view result_path, uint8_t file_number); - std::string get_raw_file_path(const Digest& result_key, + std::string get_raw_file_path(const Hash::Digest& result_key, uint8_t file_number) const; void - put_raw_files(const Digest& key, + put_raw_files(const Hash::Digest& key, const std::vector raw_files); // --- Statistics --- @@ -134,7 +135,7 @@ private: uint8_t level; }; - LookUpCacheFileResult look_up_cache_file(const Digest& key, + LookUpCacheFileResult look_up_cache_file(const Hash::Digest& key, core::CacheEntryType type) const; std::string get_subdir(uint8_t l1_index) const; @@ -144,7 +145,7 @@ private: StatsFile get_stats_file(uint8_t l1_index, uint8_t l2_index) const; void move_to_wanted_cache_level(const core::StatisticsCounters& counters, - const Digest& key, + const Hash::Digest& key, core::CacheEntryType type, const std::string& cache_file_path); @@ -152,7 +153,7 @@ private: uint8_t l1_index); std::optional increment_level_2_counters( - const Digest& key, int64_t files, int64_t size_kibibyte); + const Hash::Digest& key, int64_t files, int64_t size_kibibyte); void perform_automatic_cleanup(); @@ -191,7 +192,7 @@ private: // in the directory (including any subdirectories). However, the lock does not // have to be acquired to update a level 2 stats file since level 2 content // size and file count are stored in the parent (level 1) stats file. - util::LockFile get_level_2_content_lock(const Digest& key) const; + util::LockFile get_level_2_content_lock(const Hash::Digest& key) const; util::LockFile get_level_2_content_lock(uint8_t l1_index, uint8_t l2_index) const; }; diff --git a/src/storage/remote/FileStorage.cpp b/src/storage/remote/FileStorage.cpp index a860732bc..648e9504c 100644 --- a/src/storage/remote/FileStorage.cpp +++ b/src/storage/remote/FileStorage.cpp @@ -19,7 +19,6 @@ #include "FileStorage.hpp" #include -#include #include #include #include @@ -45,13 +44,13 @@ public: FileStorageBackend(const Params& params); nonstd::expected, Failure> - get(const Digest& key) override; + get(const Hash::Digest& key) override; - nonstd::expected put(const Digest& key, + nonstd::expected put(const Hash::Digest& key, nonstd::span value, bool only_if_missing) override; - nonstd::expected remove(const Digest& key) override; + nonstd::expected remove(const Hash::Digest& key) override; private: enum class Layout { flat, subdirs }; @@ -61,7 +60,7 @@ private: bool m_update_mtime = false; Layout m_layout = Layout::subdirs; - std::string get_entry_path(const Digest& key) const; + std::string get_entry_path(const Hash::Digest& key) const; }; FileStorageBackend::FileStorageBackend(const Params& params) @@ -109,7 +108,7 @@ FileStorageBackend::FileStorageBackend(const Params& params) } nonstd::expected, RemoteStorage::Backend::Failure> -FileStorageBackend::get(const Digest& key) +FileStorageBackend::get(const Hash::Digest& key) { const auto path = get_entry_path(key); const bool exists = Stat::stat(path); @@ -134,7 +133,7 @@ FileStorageBackend::get(const Digest& key) } nonstd::expected -FileStorageBackend::put(const Digest& key, +FileStorageBackend::put(const Hash::Digest& key, const nonstd::span value, const bool only_if_missing) { @@ -170,20 +169,20 @@ FileStorageBackend::put(const Digest& key, } nonstd::expected -FileStorageBackend::remove(const Digest& key) +FileStorageBackend::remove(const Hash::Digest& key) { return Util::unlink_safe(get_entry_path(key)); } std::string -FileStorageBackend::get_entry_path(const Digest& key) const +FileStorageBackend::get_entry_path(const Hash::Digest& key) const { switch (m_layout) { case Layout::flat: - return FMT("{}/{}", m_dir, key.to_string()); + return FMT("{}/{}", m_dir, util::format_digest(key)); case Layout::subdirs: { - const auto key_str = key.to_string(); + const auto key_str = util::format_digest(key); const uint8_t digits = 2; ASSERT(key_str.length() > digits); return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]); diff --git a/src/storage/remote/HttpStorage.cpp b/src/storage/remote/HttpStorage.cpp index 3512a4365..84787de9b 100644 --- a/src/storage/remote/HttpStorage.cpp +++ b/src/storage/remote/HttpStorage.cpp @@ -18,7 +18,7 @@ #include "HttpStorage.hpp" -#include +#include #include #include #include @@ -43,13 +43,13 @@ public: HttpStorageBackend(const Params& params); nonstd::expected, Failure> - get(const Digest& key) override; + get(const Hash::Digest& key) override; - nonstd::expected put(const Digest& key, + nonstd::expected put(const Hash::Digest& key, nonstd::span value, bool only_if_missing) override; - nonstd::expected remove(const Digest& key) override; + nonstd::expected remove(const Hash::Digest& key) override; private: enum class Layout { bazel, flat, subdirs }; @@ -58,7 +58,7 @@ private: httplib::Client m_http_client; Layout m_layout = Layout::subdirs; - std::string get_entry_path(const Digest& key) const; + std::string get_entry_path(const Hash::Digest& key) const; }; std::string @@ -154,7 +154,7 @@ HttpStorageBackend::HttpStorageBackend(const Params& params) } nonstd::expected, RemoteStorage::Backend::Failure> -HttpStorageBackend::get(const Digest& key) +HttpStorageBackend::get(const Hash::Digest& key) { const auto url_path = get_entry_path(key); const auto result = m_http_client.Get(url_path); @@ -176,7 +176,7 @@ HttpStorageBackend::get(const Digest& key) } nonstd::expected -HttpStorageBackend::put(const Digest& key, +HttpStorageBackend::put(const Hash::Digest& key, const nonstd::span value, const bool only_if_missing) { @@ -228,7 +228,7 @@ HttpStorageBackend::put(const Digest& key, } nonstd::expected -HttpStorageBackend::remove(const Digest& key) +HttpStorageBackend::remove(const Hash::Digest& key) { const auto url_path = get_entry_path(key); const auto result = m_http_client.Delete(url_path); @@ -252,24 +252,27 @@ HttpStorageBackend::remove(const Digest& key) } std::string -HttpStorageBackend::get_entry_path(const Digest& key) const +HttpStorageBackend::get_entry_path(const Hash::Digest& key) const { switch (m_layout) { case Layout::bazel: { // Mimic hex representation of a SHA256 hash value. const auto sha256_hex_size = 64; - static_assert(Digest::size() == 20, "Update below if digest size changes"); - std::string hex_digits = util::format_base16({key.bytes(), key.size()}); + static_assert(std::tuple_size() == 20, + "Update below if digest size changes"); + std::string hex_digits = util::format_base16(key); hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size()); - LOG("Translated key {} to Bazel layout ac/{}", key.to_string(), hex_digits); + LOG("Translated key {} to Bazel layout ac/{}", + util::format_digest(key), + hex_digits); return FMT("{}ac/{}", m_url_path, hex_digits); } case Layout::flat: - return m_url_path + key.to_string(); + return m_url_path + util::format_digest(key); case Layout::subdirs: { - const auto key_str = key.to_string(); + const auto key_str = util::format_digest(key); const uint8_t digits = 2; ASSERT(key_str.length() > digits); return FMT("{}{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]); diff --git a/src/storage/remote/RedisStorage.cpp b/src/storage/remote/RedisStorage.cpp index 2422bb0b8..ad1587df7 100644 --- a/src/storage/remote/RedisStorage.cpp +++ b/src/storage/remote/RedisStorage.cpp @@ -18,7 +18,7 @@ #include "RedisStorage.hpp" -#include +#include #include #include #include @@ -67,13 +67,13 @@ public: RedisStorageBackend(const RemoteStorage::Backend::Params& params); nonstd::expected, Failure> - get(const Digest& key) override; + get(const Hash::Digest& key) override; - nonstd::expected put(const Digest& key, + nonstd::expected put(const Hash::Digest& key, nonstd::span value, bool only_if_missing) override; - nonstd::expected remove(const Digest& key) override; + nonstd::expected remove(const Hash::Digest& key) override; private: const std::string m_prefix; @@ -84,7 +84,7 @@ private: void select_database(const Url& url); void authenticate(const Url& url); nonstd::expected redis_command(const char* format, ...); - std::string get_key_string(const Digest& digest) const; + std::string get_key_string(const Hash::Digest& digest) const; }; timeval @@ -164,7 +164,7 @@ is_timeout(int err) } nonstd::expected, RemoteStorage::Backend::Failure> -RedisStorageBackend::get(const Digest& key) +RedisStorageBackend::get(const Hash::Digest& key) { const auto key_string = get_key_string(key); LOG("Redis GET {}", key_string); @@ -182,7 +182,7 @@ RedisStorageBackend::get(const Digest& key) } nonstd::expected -RedisStorageBackend::put(const Digest& key, +RedisStorageBackend::put(const Hash::Digest& key, nonstd::span value, bool only_if_missing) { @@ -215,7 +215,7 @@ RedisStorageBackend::put(const Digest& key, } nonstd::expected -RedisStorageBackend::remove(const Digest& key) +RedisStorageBackend::remove(const Hash::Digest& key) { const auto key_string = get_key_string(key); LOG("Redis DEL {}", key_string); @@ -341,9 +341,9 @@ RedisStorageBackend::redis_command(const char* format, ...) } std::string -RedisStorageBackend::get_key_string(const Digest& digest) const +RedisStorageBackend::get_key_string(const Hash::Digest& digest) const { - return FMT("{}:{}", m_prefix, digest.to_string()); + return FMT("{}:{}", m_prefix, util::format_digest(digest)); } } // namespace diff --git a/src/storage/remote/RemoteStorage.hpp b/src/storage/remote/RemoteStorage.hpp index a80237ba2..5e93da362 100644 --- a/src/storage/remote/RemoteStorage.hpp +++ b/src/storage/remote/RemoteStorage.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// Copyright (C) 2021-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -31,8 +32,6 @@ #include #include -class Digest; - namespace storage::remote { constexpr auto k_redacted_password = "********"; @@ -81,19 +80,19 @@ public: // Get the value associated with `key`. Returns the value on success or // std::nullopt if the entry is not present. virtual nonstd::expected, Failure> - get(const Digest& key) = 0; + get(const Hash::Digest& key) = 0; // Put `value` associated to `key` in the storage. A true `only_if_missing` // is a hint that the value does not have to be set if already present. // Returns true if the entry was stored, otherwise false. virtual nonstd::expected - put(const Digest& key, + put(const Hash::Digest& key, nonstd::span value, bool only_if_missing = false) = 0; // Remove `key` and its associated value. Returns true if the entry was // removed, otherwise false. - virtual nonstd::expected remove(const Digest& key) = 0; + virtual nonstd::expected remove(const Hash::Digest& key) = 0; // Determine whether an attribute is handled by the remote storage // framework itself. diff --git a/src/util/string.cpp b/src/util/string.cpp index 3fe7d2e49..d1c98ea01 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -64,6 +64,16 @@ format_base32hex(nonstd::span data) return result; } +std::string +format_digest(nonstd::span data) +{ + const size_t base16_bytes = 2; + ASSERT(data.size() >= base16_bytes); + return util::format_base16({data.data(), base16_bytes}) + + util::format_base32hex( + {data.data() + base16_bytes, data.size() - base16_bytes}); +} + std::string format_human_readable_diff(int64_t diff, SizeUnitPrefixType prefix_type) { diff --git a/src/util/string.hpp b/src/util/string.hpp index c8e17c0dc..32e13e0b0 100644 --- a/src/util/string.hpp +++ b/src/util/string.hpp @@ -49,6 +49,14 @@ std::string format_base16(nonstd::span data); // characters will be added. std::string format_base32hex(nonstd::span data); +// Format a hash digest representing `data`. +// +// The first two bytes are encoded as four lowercase base16 digits to maintain +// compatibility with the cleanup algorithm in older ccache versions and to +// allow for up to four uniform cache levels. The rest are encoded as lowercase +// base32hex digits without padding characters. +std::string format_digest(nonstd::span data); + // Format `diff` as a human-readable string. std::string format_human_readable_diff(int64_t diff, SizeUnitPrefixType prefix_type); diff --git a/unittest/test_Hash.cpp b/unittest/test_Hash.cpp index 8107c89ad..24a57f66e 100644 --- a/unittest/test_Hash.cpp +++ b/unittest/test_Hash.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors +// Copyright (C) 2010-2023 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,6 +18,8 @@ #include "../src/Hash.hpp" +#include + #include "third_party/doctest.h" TEST_SUITE_BEGIN("Hash"); @@ -26,24 +28,25 @@ TEST_CASE("known strings") { SUBCASE("initial state") { - CHECK(Hash().digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi"); + CHECK(util::format_digest(Hash().digest()) + == "af1396svbud1kqg40jfa6reciicrpcisi"); } SUBCASE("empty string") { - CHECK(Hash().hash("").digest().to_string() + CHECK(util::format_digest(Hash().hash("").digest()) == "af1396svbud1kqg40jfa6reciicrpcisi"); } SUBCASE("a") { - CHECK(Hash().hash("a").digest().to_string() + CHECK(util::format_digest(Hash().hash("a").digest()) == "17765vetiqd4ae95qpbhfb1ut8gj42r6m"); } SUBCASE("message digest") { - CHECK(Hash().hash("message digest").digest().to_string() + CHECK(util::format_digest(Hash().hash("message digest").digest()) == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo"); } @@ -52,7 +55,7 @@ TEST_CASE("known strings") const char long_string[] = "123456789012345678901234567890123456789012345678901234567890" "12345678901234567890"; - CHECK(Hash().hash(long_string).digest().to_string() + CHECK(util::format_digest(Hash().hash(long_string).digest()) == "f263ljqhc8co1ee8rpeq98bt654o9o2qm"); } } @@ -63,24 +66,14 @@ TEST_CASE("Hash::digest should not alter state") h.hash("message"); h.digest(); h.hash(" digest"); - CHECK(h.digest().to_string() == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo"); + CHECK(util::format_digest(h.digest()) == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo"); } TEST_CASE("Hash::digest should be idempotent") { Hash h; - CHECK(h.digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi"); - CHECK(h.digest().to_string() == "af1396svbud1kqg40jfa6reciicrpcisi"); -} - -TEST_CASE("Digest::bytes") -{ - Digest d = Hash().hash("message digest").digest(); - uint8_t expected[Digest::size()] = { - 0x7b, 0xc2, 0xa2, 0xee, 0xb9, 0x5d, 0xdb, 0xf9, 0xb7, 0xec, - 0xf6, 0xad, 0xcb, 0x76, 0xb4, 0x53, 0x09, 0x1c, 0x58, 0xdc, - }; - CHECK(memcmp(d.bytes(), expected, Digest::size()) == 0); + CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi"); + CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi"); } TEST_SUITE_END(); diff --git a/unittest/test_InodeCache.cpp b/unittest/test_InodeCache.cpp index 34bd00342..5e331f639 100644 --- a/unittest/test_InodeCache.cpp +++ b/unittest/test_InodeCache.cpp @@ -76,7 +76,7 @@ TEST_CASE("Test disabled") config.set_inode_cache(false); InodeCache inode_cache(config, util::Duration(0)); - Digest digest; + Hash::Digest digest; CHECK(!inode_cache.get( "a", InodeCache::ContentType::checked_for_temporal_macros, digest)); @@ -99,7 +99,7 @@ TEST_CASE("Test lookup nonexistent") InodeCache inode_cache(config, util::Duration(0)); util::write_file("a", ""); - Digest digest; + Hash::Digest digest; CHECK(!inode_cache.get( "a", InodeCache::ContentType::checked_for_temporal_macros, digest)); @@ -122,7 +122,7 @@ TEST_CASE("Test put and lookup") result.insert(HashSourceCode::found_date); CHECK(put(inode_cache, "a", "a text", result)); - Digest digest; + Hash::Digest digest; auto return_value = inode_cache.get( "a", InodeCache::ContentType::checked_for_temporal_macros, digest); REQUIRE(return_value); @@ -166,7 +166,7 @@ TEST_CASE("Drop file") InodeCache inode_cache(config, util::Duration(0)); - Digest digest; + Hash::Digest digest; inode_cache.get("a", InodeCache::ContentType::raw, digest); CHECK(Stat::stat(inode_cache.get_file())); @@ -184,8 +184,8 @@ TEST_CASE("Test content type") InodeCache inode_cache(config, util::Duration(0)); util::write_file("a", "a text"); - Digest binary_digest = Hash().hash("binary").digest(); - Digest code_digest = Hash().hash("code").digest(); + auto binary_digest = Hash().hash("binary").digest(); + auto code_digest = Hash().hash("code").digest(); CHECK(inode_cache.put("a", InodeCache::ContentType::raw, @@ -196,7 +196,7 @@ TEST_CASE("Test content type") code_digest, HashSourceCodeResult(HashSourceCode::found_time))); - Digest digest; + Hash::Digest digest; auto return_value = inode_cache.get("a", InodeCache::ContentType::raw, digest); diff --git a/unittest/test_hashutil.cpp b/unittest/test_hashutil.cpp index bb4522c55..d8dc3b9cd 100644 --- a/unittest/test_hashutil.cpp +++ b/unittest/test_hashutil.cpp @@ -24,6 +24,8 @@ #include "third_party/doctest.h" +#include + using TestUtil::TestContext; TEST_SUITE_BEGIN("hashutil");