This reverts the mixed encoding scheme introduced in commit
47cb00f7 ("Encode
hash digests as 4 base16 digits + 29 base32hex digits") back to pure base16
encoding.
Background: The mixed base16/base32hex encoding was introduced to reduce
filename lengths slightly, thereby reducing the number of system calls when
scanning local cache directories. At the time, "the effect is very small but
there is no real downside".
However, with the introduction of remote storage helpers implemented outside
ccache's code base, a downside has emerged: key encoding inconsistency. Remote
storage helpers must now either:
1. reimplement ccache's custom encoding algorithm to maintain key consistency
between the remote storage and ccache's local storage (and logs), or
2. use standard base16 encoding, resulting in different key representations
between ccache and the remote storage.
By standardizing on base16 encoding throughout, we ensure that:
- remote storage helpers can use simple, standard base16 encoding
- keys are represented identically in ccache's local storage and remote storage
- external implementations don't need to replicate custom encoding logic
- debugging and key correlation across systems is straightforward
The minor filesystem performance benefit of shorter filenames is outweighed by
the ecosystem benefits of a simple, standard encoding scheme.
Local storage will for now be backward compatible: to avoid invalidating
existing cache entries, ccache will write in the new format but will try
both the new and the legacy format on cache lookup. The legacy format is
also still used for hashes that are part of the input hash. When some
other input hash material is changed (which invalidates cache entries
anyway) in the future, we'll drop the legacy format completely.
return tl::unexpected(Statistic::bad_input_file);
}
cpp_hash.hash_delimiter(using_pch_sum ? "pch_sum_hash" : "pch_hash");
- cpp_hash.hash(util::format_digest(file_digest));
+ cpp_hash.hash(util::format_legacy_digest(file_digest));
}
if (ctx.config.direct_mode()) {
if (depend_mode_hash) {
depend_mode_hash->hash_delimiter("include");
- depend_mode_hash->hash(util::format_digest(file_digest));
+ depend_mode_hash->hash(util::format_legacy_digest(file_digest));
}
}
};
});
if (added) {
- LOG("Added result key to manifest {}", util::format_digest(manifest_key));
+ LOG("Added result key to manifest {}", util::format_base16(manifest_key));
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
ctx.storage.put(manifest_key,
core::CacheEntryType::manifest,
core::CacheEntry::serialize(header, ctx.manifest));
} else {
LOG("Did not add result key to manifest {}",
- util::format_digest(manifest_key));
+ util::format_base16(manifest_key));
}
}
ASSERT(false);
}
LOG_RAW("Got result key from dependency file");
- LOG("Result key: {}", util::format_digest(*result_key));
+ LOG("Result key: {}", util::format_base16(*result_key));
}
ASSERT(result_key);
ctx.config.set_direct_mode(false);
return {};
}
- hash.hash(util::format_digest(input_file_digest));
+ hash.hash(util::format_legacy_digest(input_file_digest));
return hash.digest();
}
});
if (read_manifests > 1 && !ctx.config.remote_only()) {
LOG("Storing merged manifest {} locally",
- util::format_digest(manifest_key));
+ util::format_base16(manifest_key));
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
ctx.storage.local.put(manifest_key,
core::CacheEntryType::manifest,
if (direct_mode) {
TRY_ASSIGN(manifest_key, get_manifest_key(ctx, hash));
if (manifest_key && !ctx.config.recache()) {
- LOG("Manifest key: {}", util::format_digest(*manifest_key));
+ LOG("Manifest key: {}", util::format_base16(*manifest_key));
result_key = get_result_key_from_manifest(ctx, *manifest_key);
}
} else if (ctx.args_info.arch_args.empty()) {
}
if (result_key) {
- LOG("Result key: {}", util::format_digest(*result_key));
+ LOG("Result key: {}", util::format_base16(*result_key));
}
return std::make_pair(result_key, manifest_key);
}
deserializer.visit(result_retriever);
} catch (core::ResultRetriever::WriteError& e) {
LOG("Write error when retrieving result from {}: {}",
- util::format_digest(result_key),
+ util::format_base16(result_key),
e.what());
return tl::unexpected(Statistic::bad_output_file);
} catch (core::Error& e) {
LOG("Failed to get result from {}: {}",
- util::format_digest(result_key),
+ util::format_base16(result_key),
e.what());
return false;
}
const auto result =
arg == "-" ? hash.hash_fd(STDIN_FILENO) : hash.hash_file(arg);
if (result) {
- PRINT(stdout, "{}\n", util::format_digest(hash.digest()));
+ PRINT(stdout, "{}\n", util::format_base16(hash.digest()));
} else {
PRINT(stderr, "Error: Failed to hash {}: {}\n", arg, result.error());
return EXIT_FAILURE;
const auto& result = m_results[i - 1];
LOG("Considering result entry {} ({})",
i - 1,
- util::format_digest(result.key));
+ util::format_base16(result.key));
if (result_matches(ctx, result, stated_files, hashed_files)) {
LOG("Result entry {} matched in manifest", i - 1);
return result.key;
if (hashed_files_iter->second != fi.digest) {
LOG("Mismatch for {}: hash {} != {}",
path,
- util::format_digest(hashed_files_iter->second),
- util::format_digest(fi.digest));
+ util::format_base16(hashed_files_iter->second),
+ util::format_base16(fi.digest));
return false;
}
}
PRINT(stream, " {}:\n", i);
PRINT(stream, " Path index: {}\n", m_file_infos[i].index);
PRINT(
- stream, " Hash: {}\n", util::format_digest(m_file_infos[i].digest));
+ stream, " Hash: {}\n", util::format_base16(m_file_infos[i].digest));
PRINT(stream, " File size: {}\n", m_file_infos[i].fsize);
if (m_file_infos[i].mtime == util::TimePoint()) {
PRINT_RAW(stream, " Mtime: -\n");
PRINT(stream, " {}", file_info_index);
}
PRINT_RAW(stream, "\n");
- PRINT(stream, " Key: {}\n", util::format_digest(m_results[i].key));
+ PRINT(stream, " Key: {}\n", util::format_base16(m_results[i].key));
}
}
// macro expansions.
Hash hash;
- hash.hash(util::format_digest(digest));
+ hash.hash(util::format_legacy_digest(digest));
if (result.contains(HashSourceCode::found_date)) {
LOG("Found __DATE__ in {}", path);
Hash::Digest digest;
const bool success = hash_binary_file(ctx, digest, path);
if (success) {
- hash.hash(util::format_digest(digest));
+ hash.hash(util::format_legacy_digest(digest));
}
return success;
}
const auto value = util::read_file<util::Bytes>(cache_file.path);
if (value) {
LOG("Retrieved {} from local storage ({})",
- util::format_digest(key),
+ util::format_base16(key),
cache_file.path);
// Update modification timestamp to save file from LRU cleanup.
LOG("Failed to read {}: {}", cache_file.path, value.error());
}
} else {
- LOG("No {} in local storage", util::format_digest(key));
+ LOG("No {} in local storage", util::format_base16(key));
}
increment_statistic(return_value ? Statistic::local_storage_read_hit
}
LOG("Stored {} in local storage ({})",
- util::format_digest(key),
+ util::format_base16(key),
cache_file.path);
m_stored_data = true;
// be done almost anywhere, but we might as well do it near the end as we save
// the stat call if we exit early.
util::create_cachedir_tag(
- FMT("{}/{}", m_config.cache_dir(), util::format_digest(key)[0]));
+ FMT("{}/{}", m_config.cache_dir(), util::format_base16(key)[0]));
}
void
{
const auto cache_file = look_up_cache_file(key, type);
if (!cache_file.dir_entry) {
- LOG("No {} to remove from local storage", util::format_digest(key));
+ LOG("No {} to remove from local storage", util::format_base16(key));
return;
}
}
LOG("Removed {} from local storage ({})",
- util::format_digest(key),
+ util::format_base16(key),
cache_file.path);
increment_files_and_size_counters(
key, -1, -static_cast<int64_t>(cache_file.dir_entry.size_on_disk() / 1024));
const core::CacheEntryType type) const
{
const auto key_string =
- FMT("{}{}", util::format_digest(key), suffix_from_type(type));
+ FMT("{}{}", util::format_base16(key), suffix_from_type(type));
for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels;
++level) {
const auto wanted_level =
calculate_wanted_cache_level(counters.get(Statistic::files_in_cache));
const auto wanted_path = get_path_in_cache(
- wanted_level, util::format_digest(key) + suffix_from_type(type));
+ wanted_level, util::format_base16(key) + suffix_from_type(type));
if (cache_file_path != wanted_path) {
core::ensure_dir_exists(wanted_path.parent_path());
-// Copyright (C) 2021-2025 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
{
switch (m_layout) {
case Layout::flat:
- return FMT("{}/{}", m_dir, util::format_digest(key));
+ return FMT("{}/{}", m_dir, util::format_base16(key));
case Layout::subdirs: {
- const auto key_str = util::format_digest(key);
+ const auto key_str = util::format_base16(key);
const uint8_t digits = 2;
ASSERT(key_str.length() > digits);
return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]);
-// Copyright (C) 2021-2025 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
const auto sha256_hex_size = 64;
static_assert(std::tuple_size<Hash::Digest>() == 20,
"Update below if digest size changes");
- std::string hex_digits = util::format_base16(key);
- hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size());
- LOG("Translated key {} to Bazel layout ac/{}",
- util::format_digest(key),
- hex_digits);
- return FMT("{}ac/{}", m_url_path, hex_digits);
+ std::string hex_key = util::format_base16(key);
+ std::string bazel_key =
+ FMT("ac/{}{:.{}}", hex_key, hex_key, sha256_hex_size - hex_key.size());
+ LOG("Translated key {} to Bazel layout {}", hex_key, bazel_key);
+ return FMT("{}{}", m_url_path, bazel_key);
}
case Layout::flat:
- return m_url_path + util::format_digest(key);
+ return m_url_path + util::format_base16(key);
case Layout::subdirs: {
- const auto key_str = util::format_digest(key);
+ const auto key_str = util::format_base16(key);
const uint8_t digits = 2;
ASSERT(key_str.length() > digits);
return FMT("{}{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]);
-// Copyright (C) 2021-2025 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2026 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
std::string
RedisStorageBackend::get_key_string(const Hash::Digest& digest) const
{
- return FMT("{}:{}", m_prefix, util::format_digest(digest));
+ return FMT("{}:{}", m_prefix, util::format_base16(digest));
}
} // namespace
auto& value = *result;
if (value) {
LOG("Retrieved {} from {} ({:.2f} ms)",
- util::format_digest(key),
+ util::format_base16(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_read_hit);
}
} else {
LOG("No {} in {} ({:.2f} ms)",
- util::format_digest(key),
+ util::format_base16(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_read_miss);
{
if (!core::CacheEntry::Header(value).self_contained) {
LOG("Not putting {} in remote storage since it's not self-contained",
- util::format_digest(key));
+ util::format_base16(key));
return;
}
const bool stored = *result;
LOG("{} {} in {} ({:.2f} ms)",
stored ? "Stored" : "Did not have to store",
- util::format_digest(key),
+ util::format_base16(key),
backend->url_for_logging,
ms);
local.increment_statistic(core::Statistic::remote_storage_write);
const bool removed = *result;
if (removed) {
LOG("Removed {} from {} ({:.2f} ms)",
- util::format_digest(key),
+ util::format_base16(key),
backend->url_for_logging,
ms);
} else {
LOG("No {} to remove from {} ({:.2f} ms)",
- util::format_digest(key),
+ util::format_base16(key),
backend->url_for_logging,
ms);
}
}
std::string
-format_digest(nonstd::span<const uint8_t> data)
+format_legacy_digest(nonstd::span<const uint8_t> data)
{
const size_t base16_bytes = 2;
ASSERT(data.size() >= base16_bytes);
// characters will be added.
std::string format_base32hex(nonstd::span<const uint8_t> data);
-// Format a hash digest representing `data`.
-//
-// The first two bytes are encoded as four lowercase base16 digits to maintain
-// compatibility with the cleanup algorithm in older ccache versions and to
-// allow for up to four uniform cache levels. The rest are encoded as lowercase
-// base32hex digits without padding characters.
-std::string format_digest(nonstd::span<const uint8_t> data);
-
// Format `ms` as a duration string.
std::string format_duration(std::chrono::milliseconds ms);
std::string format_human_readable_diff(int64_t diff,
SizeUnitPrefixType prefix_type);
+// Format a hash digest representing `data`.
+//
+// The first two bytes are encoded as four lowercase base16 digits to maintain
+// compatibility with the cleanup algorithm in older ccache versions and to
+// allow for up to four uniform cache levels. The rest are encoded as lowercase
+// base32hex digits without padding characters.
+std::string format_legacy_digest(nonstd::span<const uint8_t> data);
+
// Format `size` as a human-readable string.
std::string format_human_readable_size(uint64_t size,
SizeUnitPrefixType prefix_type);
fi
}
+expect_equal() {
+ if [ "$1" != "$2" ]; then
+ test_failed_internal "$1 != $2"
+ fi
+}
+
expect_equal_content() {
if [ ! -e "$1" ]; then
test_failed_internal "expect_equal_content: $1 missing"
# -------------------------------------------------------------------------
TEST "--hash-file"
- $CCACHE --hash-file /dev/null > hash.out
- printf "a" | $CCACHE --hash-file - >> hash.out
-
- hash_0='af1396svbud1kqg40jfa6reciicrpcisi'
- hash_1='17765vetiqd4ae95qpbhfb1ut8gj42r6m'
-
- if grep "$hash_0" hash.out >/dev/null 2>&1 && \
- grep "$hash_1" hash.out >/dev/null 2>&1; then
- : OK
- else
- test_failed "Unexpected output of --hash-file"
- fi
+ expect_equal $($CCACHE --hash-file /dev/null) af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9
+ expect_equal $(printf a | $CCACHE --hash-file -) 17762fddd969a453925d65717ac3eea21320b66b
}
# =============================================================================
manifest=`find $CCACHE_DIR -name '*M'`
$CCACHE --inspect $manifest >manifest.dump
- checksum_test1_h='b7273h0ksdehi0o4pitg5jeehal3i54ns'
- checksum_test2_h='24f1315jch5tcndjbm6uejtu8q3lf9100'
- checksum_test3_h='56a6dkffffv485aepk44seaq3i6lbepq2'
+ checksum_test1_h='b7271c414e35d190304ccbb02cdce8aaa391497e'
+ checksum_test2_h='24f1184b3644bd65db35d8de74fbe468757a4200'
+ checksum_test3_h='56a66d1ef7bfe44154ecd084e395a1c8d55bb3a1'
if grep "Hash: $checksum_test1_h" manifest.dump >/dev/null 2>&1 && \
grep "Hash: $checksum_test2_h" manifest.dump >/dev/null 2>&1 && \
-// Copyright (C) 2010-2024 Joel Rosdahl and other contributors
+// Copyright (C) 2010-2025 Joel Rosdahl and other contributors
//
// See doc/authors.adoc for a complete list of contributors.
//
{
SUBCASE("initial state")
{
- CHECK(util::format_digest(Hash().digest())
- == "af1396svbud1kqg40jfa6reciicrpcisi");
+ CHECK(util::format_base16(Hash().digest())
+ == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9");
}
SUBCASE("empty string")
{
- CHECK(util::format_digest(Hash().hash("").digest())
- == "af1396svbud1kqg40jfa6reciicrpcisi");
+ CHECK(util::format_base16(Hash().hash("").digest())
+ == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9");
}
SUBCASE("a")
{
- CHECK(util::format_digest(Hash().hash("a").digest())
- == "17765vetiqd4ae95qpbhfb1ut8gj42r6m");
+ CHECK(util::format_base16(Hash().hash("a").digest())
+ == "17762fddd969a453925d65717ac3eea21320b66b");
}
SUBCASE("message digest")
{
- CHECK(util::format_digest(Hash().hash("message digest").digest())
- == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo");
+ CHECK(util::format_base16(Hash().hash("message digest").digest())
+ == "7bc2a2eeb95ddbf9b7ecf6adcb76b453091c58dc");
}
SUBCASE("long string")
const char long_string[] =
"123456789012345678901234567890123456789012345678901234567890"
"12345678901234567890";
- CHECK(util::format_digest(Hash().hash(long_string).digest())
- == "f263ljqhc8co1ee8rpeq98bt654o9o2qm");
+ CHECK(util::format_base16(Hash().hash(long_string).digest())
+ == "f263acf51621980b9c8de5da4a17d314984e05ab");
}
}
h.hash("message");
h.digest();
h.hash(" digest");
- CHECK(util::format_digest(h.digest()) == "7bc2kbnbinerv6ruptldpdrb8ko93hcdo");
+ CHECK(util::format_base16(h.digest())
+ == "7bc2a2eeb95ddbf9b7ecf6adcb76b453091c58dc");
}
TEST_CASE("Hash::digest should be idempotent")
{
Hash h;
- CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi");
- CHECK(util::format_digest(h.digest()) == "af1396svbud1kqg40jfa6reciicrpcisi");
+ CHECK(util::format_base16(h.digest())
+ == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9");
+ CHECK(util::format_base16(h.digest())
+ == "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9");
}
TEST_SUITE_END();
projects / thirdparty / ccache.git / commitdiff
