From: Joel Rosdahl Date: Fri, 17 Jul 2026 11:53:23 +0000 (+0200) Subject: chore: Remove last traces of legacy mixed encoding hash digests X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fbfa39e322a6135327eee632b25c0973cf0ba599;p=thirdparty%2Fccache.git chore: Remove last traces of legacy mixed encoding hash digests 7d64f3baf1e99ebbaf185aeef5426f04de0638de (feat: Use base16 encoding for all hash digest keys) transitioned from the mixed base16/base32hex encoding to just base16, but left backward-compatibility code during a transition period. This commit completes the transition by removing backward-compatibility code for looking up old entries. --- diff --git a/src/ccache/ccache.cpp b/src/ccache/ccache.cpp index 0988e406..2ff0cf9d 100644 --- a/src/ccache/ccache.cpp +++ b/src/ccache/ccache.cpp @@ -464,7 +464,7 @@ remember_include_file(Context& ctx, } file_digest = *ret; cpp_hash.hash_delimiter(using_pch_sum ? "pch_sum_hash" : "pch_hash"); - cpp_hash.hash(util::format_legacy_digest(file_digest)); + cpp_hash.hash(util::format_base16(file_digest)); } if (ctx.config.direct_mode()) { @@ -478,7 +478,7 @@ remember_include_file(Context& ctx, if (depend_mode_hash) { depend_mode_hash->hash_delimiter("include"); - depend_mode_hash->hash(util::format_legacy_digest(file_digest)); + depend_mode_hash->hash(util::format_base16(file_digest)); } } @@ -941,7 +941,6 @@ update_manifest(Context& ctx, LOG("Added result key to manifest {}", util::format_base16(manifest_key)); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); ctx.storage.put(manifest_key, - core::CacheEntryType::manifest, core::CacheEntry::serialize(header, ctx.manifest)); } else { LOG("Did not add result key to manifest {}", @@ -1095,7 +1094,7 @@ write_result(Context& ctx, } } - ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data); + ctx.storage.put(result_key, cache_entry_data); return true; } @@ -2370,7 +2369,7 @@ get_manifest_key(Context& ctx, Hash& hash) // Direct mode was disabled by hash_source_code_file. return {}; } - hash.hash(util::format_legacy_digest(*input_file_digest)); + hash.hash(util::format_base16(*input_file_digest)); return hash.digest(); } @@ -2496,7 +2495,6 @@ get_result_key_from_manifest(Context& ctx, const Hash::Digest& manifest_key) util::format_base16(manifest_key)); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); ctx.storage.local.put(manifest_key, - core::CacheEntryType::manifest, core::CacheEntry::serialize(header, ctx.manifest), storage::Overwrite::yes); } @@ -3137,7 +3135,7 @@ do_cache_compilation(Context& ctx) LOG("Hash from manifest doesn't match preprocessor output"); LOG("Likely reason: different CCACHE_BASEDIRs used"); LOG("Removing manifest as a safety measure"); - ctx.storage.remove(*manifest_key, core::CacheEntryType::manifest); + ctx.storage.remove(*manifest_key); put_result_in_manifest = true; } diff --git a/src/ccache/hashutil.cpp b/src/ccache/hashutil.cpp index 411e51d9..301caac2 100644 --- a/src/ccache/hashutil.cpp +++ b/src/ccache/hashutil.cpp @@ -426,7 +426,7 @@ hash_source_code_file(Context& ctx, const fs::path& path, size_t size_hint) // expansions. Hash hash; - hash.hash(util::format_legacy_digest(digest)); + hash.hash(util::format_base16(digest)); if (result.contains(SourceCodeScan::found_date)) { hash.hash_delimiter("date"); @@ -482,7 +482,7 @@ hash_binary_file(const Context& ctx, Hash& hash, const fs::path& path) { const auto result = hash_binary_file(ctx, path); if (result) { - hash.hash(util::format_legacy_digest(*result)); + hash.hash(util::format_base16(*result)); } return result.has_value(); } diff --git a/src/ccache/storage/local/localstorage.cpp b/src/ccache/storage/local/localstorage.cpp index e4f7bf4b..5993d41c 100644 --- a/src/ccache/storage/local/localstorage.cpp +++ b/src/ccache/storage/local/localstorage.cpp @@ -486,7 +486,7 @@ LocalStorage::get(const Hash::Digest& key, const core::CacheEntryType type) { std::optional return_value; - const auto cache_file = look_up_cache_file(key, type); + const auto cache_file = look_up_cache_file(key); if (cache_file.dir_entry.is_regular_file()) { const auto value = util::read_file(cache_file.path); if (value) { @@ -516,11 +516,10 @@ LocalStorage::get(const Hash::Digest& key, const core::CacheEntryType type) void LocalStorage::put(const Hash::Digest& key, - const core::CacheEntryType type, std::span value, Overwrite overwrite) { - const auto cache_file = look_up_cache_file(key, type); + const auto cache_file = look_up_cache_file(key); if (overwrite == Overwrite::no && cache_file.dir_entry.exists()) { LOG("Not storing {} in local storage since it already exists", cache_file.path); @@ -581,9 +580,9 @@ LocalStorage::put(const Hash::Digest& key, } void -LocalStorage::remove(const Hash::Digest& key, const core::CacheEntryType type) +LocalStorage::remove(const Hash::Digest& key) { - const auto cache_file = look_up_cache_file(key, type); + const auto cache_file = look_up_cache_file(key); if (!cache_file.dir_entry) { LOG("No {} to remove from local storage", util::format_base16(key)); return; @@ -617,8 +616,7 @@ fs::path LocalStorage::get_raw_file_path(const Hash::Digest& result_key, uint8_t file_number) const { - const auto cache_file = - look_up_cache_file(result_key, core::CacheEntryType::result); + const auto cache_file = look_up_cache_file(result_key); return get_raw_file_path(cache_file.path, file_number); } @@ -633,7 +631,7 @@ LocalStorage::put_raw_files( FMT("Too many raw files: {} > {}", raw_files.size(), k_max_raw_files)); } - const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result); + const auto cache_file = look_up_cache_file(key); core::ensure_dir_exists(cache_file.path.parent_path()); int64_t files_change = 0; @@ -1061,10 +1059,8 @@ LocalStorage::get_subdir(uint8_t l1_index, uint8_t l2_index) const } LocalStorage::LookUpCacheFileResult -LocalStorage::look_up_cache_file(const Hash::Digest& key, - const core::CacheEntryType type) const +LocalStorage::look_up_cache_file(const Hash::Digest& key) const { - // Try new format first: base16 without suffix const auto key_string = util::format_base16(key); for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; @@ -1076,27 +1072,6 @@ LocalStorage::look_up_cache_file(const Hash::Digest& key, } } - // Try old format with R/M suffix - const auto old_key_string = util::format_legacy_digest(key); - const std::string old_suffix = - type == core::CacheEntryType::manifest ? "M" : "R"; - const auto old_key_string_with_suffix = old_key_string + old_suffix; - for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; - ++level) { - const auto path = get_path_in_cache(level, old_key_string_with_suffix); - DirEntry dir_entry(path); - if (dir_entry.is_regular_file()) { - const auto new_path = get_path_in_cache(level, key_string); - LOG("Migrating {} to {}", path, new_path); - if (fs::rename(path, new_path)) { - return {new_path, DirEntry(new_path), level}; - } else { - LOG("Failed to rename {} to {}", path, new_path); - return {path, dir_entry, level}; - } - } - } - const auto shallowest_path = get_path_in_cache(k_min_cache_levels, key_string); return {shallowest_path, DirEntry(), k_min_cache_levels}; diff --git a/src/ccache/storage/local/localstorage.hpp b/src/ccache/storage/local/localstorage.hpp index 76b5f2e4..fa137a7a 100644 --- a/src/ccache/storage/local/localstorage.hpp +++ b/src/ccache/storage/local/localstorage.hpp @@ -72,11 +72,10 @@ public: core::CacheEntryType type); void put(const Hash::Digest& key, - core::CacheEntryType type, std::span value, Overwrite overwrite); - void remove(const Hash::Digest& key, core::CacheEntryType type); + void remove(const Hash::Digest& key); static std::filesystem::path get_raw_file_path(const std::filesystem::path& result_path, @@ -152,8 +151,7 @@ private: uint8_t level; }; - LookUpCacheFileResult look_up_cache_file(const Hash::Digest& key, - core::CacheEntryType type) const; + LookUpCacheFileResult look_up_cache_file(const Hash::Digest& key) const; std::filesystem::path get_subdir(uint8_t l1_index) const; std::filesystem::path get_subdir(uint8_t l1_index, uint8_t l2_index) const; diff --git a/src/ccache/storage/storage.cpp b/src/ccache/storage/storage.cpp index e5ecf6cc..a4c5574b 100644 --- a/src/ccache/storage/storage.cpp +++ b/src/ccache/storage/storage.cpp @@ -451,28 +451,26 @@ Storage::get(const Hash::Hash::Digest& key, get_from_remote_storage(key, type, [&](util::Bytes&& data) { if (!m_config.remote_only()) { - local.put(key, type, data, Overwrite::no); + local.put(key, data, Overwrite::no); } return entry_receiver(std::move(data)); }); } void -Storage::put(const Hash::Digest& key, - const core::CacheEntryType type, - std::span value) +Storage::put(const Hash::Digest& key, std::span value) { if (!m_config.remote_only()) { - local.put(key, type, value, Overwrite::yes); + local.put(key, value, Overwrite::yes); } put_in_remote_storage(key, value, Overwrite::yes); } void -Storage::remove(const Hash::Digest& key, const core::CacheEntryType type) +Storage::remove(const Hash::Digest& key) { if (!m_config.remote_only()) { - local.remove(key, type); + local.remove(key); } remove_from_remote_storage(key); } diff --git a/src/ccache/storage/storage.hpp b/src/ccache/storage/storage.hpp index a160db24..11c9cb80 100644 --- a/src/ccache/storage/storage.hpp +++ b/src/ccache/storage/storage.hpp @@ -63,11 +63,9 @@ public: core::CacheEntryType type, const EntryReceiver& entry_receiver); - void put(const Hash::Digest& key, - core::CacheEntryType type, - std::span value); + void put(const Hash::Digest& key, std::span value); - void remove(const Hash::Digest& key, core::CacheEntryType type); + void remove(const Hash::Digest& key); void stop_remote_storage_helpers(); diff --git a/src/ccache/util/string.cpp b/src/ccache/util/string.cpp index 1fdb42e1..fffc1681 100644 --- a/src/ccache/util/string.cpp +++ b/src/ccache/util/string.cpp @@ -140,40 +140,6 @@ format_base16(std::span data) return result; } -std::string -format_base32hex(std::span data) -{ - static const char digits[] = "0123456789abcdefghijklmnopqrstuv"; - std::string result; - result.reserve(data.size() * 8 / 5 + 1); - uint8_t i = 0; - uint16_t bits = 0; - for (uint8_t b : data) { - bits <<= 8; - bits |= b; - i += 8; - while (i >= 5) { - result += digits[(bits >> (i - 5)) & 0x1f]; - i -= 5; - } - } - if (i > 0) { - DEBUG_ASSERT(i < 5); - result += digits[(bits << (5 - i)) & 0x1f]; - } - return result; -} - -std::string -format_legacy_digest(std::span data) -{ - const size_t base16_bytes = 2; - ASSERT(data.size() >= base16_bytes); - return format_base16({data.data(), base16_bytes}) - + format_base32hex( - {data.data() + base16_bytes, data.size() - base16_bytes}); -} - std::string format_duration(std::chrono::milliseconds ms) { diff --git a/src/ccache/util/string.hpp b/src/ccache/util/string.hpp index b7819463..89ad8259 100644 --- a/src/ccache/util/string.hpp +++ b/src/ccache/util/string.hpp @@ -63,10 +63,6 @@ std::string format_argv_for_logging(const char* const* argv); // `2 * data.size()` long. std::string format_base16(std::span data); -// Format a lowercase base32hex string representing `data`. No padding -// characters will be added. -std::string format_base32hex(std::span data); - // Format `ms` as a duration string. std::string format_duration(std::chrono::milliseconds ms); @@ -74,14 +70,6 @@ std::string format_duration(std::chrono::milliseconds ms); std::string format_human_readable_diff(int64_t diff, SizeUnitPrefixType prefix_type); -// Format a hash digest representing `data`. -// -// The first two bytes are encoded as four lowercase base16 digits to maintain -// compatibility with the cleanup algorithm in older ccache versions and to -// allow for up to four uniform cache levels. The rest are encoded as lowercase -// base32hex digits without padding characters. -std::string format_legacy_digest(std::span data); - // Format `size` as a human-readable string. std::string format_human_readable_size(uint64_t size, SizeUnitPrefixType prefix_type); diff --git a/unittest/test_util_string.cpp b/unittest/test_util_string.cpp index 08af4910..5f70d02c 100644 --- a/unittest/test_util_string.cpp +++ b/unittest/test_util_string.cpp @@ -242,19 +242,6 @@ TEST_CASE("util::parse_base16") } } -TEST_CASE("util::format_base32hex") -{ - // Test vectors (without padding) from RFC 4648. - const uint8_t input[] = {'f', 'o', 'o', 'b', 'a', 'r'}; - CHECK(util::format_base32hex({input, 0}) == ""); - CHECK(util::format_base32hex({input, 1}) == "co"); - CHECK(util::format_base32hex({input, 2}) == "cpng"); - CHECK(util::format_base32hex({input, 3}) == "cpnmu"); - CHECK(util::format_base32hex({input, 4}) == "cpnmuog"); - CHECK(util::format_base32hex({input, 5}) == "cpnmuoj1"); - CHECK(util::format_base32hex({input, 6}) == "cpnmuoj1e8"); -} - TEST_CASE("util::format_human_readable_diff") { using SUPT = util::SizeUnitPrefixType;