From: Joel Rosdahl Date: Sat, 25 Oct 2025 21:04:02 +0000 (+0200) Subject: feat: Remove R/M suffix from files in local storage X-Git-Tag: v4.13~38^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d34019c4a4d7e9917be08129e079df3d7b50399a;p=thirdparty%2Fccache.git feat: Remove R/M suffix from files in local storage In addition to the previous commit, this commit make things even more consistent with remote storage by changing file names in local storage: 1. ${digest}R -> ${digest} 2. ${digest}M -> ${digest} 3. ${digest}${n}W -> ${digest}_${nn} (${n} is file number 0-9, ${nn} is file number in two-digit hex form.) The original reason for adding R and M suffixes was to simplify implementation of local cleanup and recompression somewhat (and also discoverability in tests), but I think that consistency trumps this. --- diff --git a/src/ccache/storage/local/localstorage.cpp b/src/ccache/storage/local/localstorage.cpp index 778e785b..c31c401c 100644 --- a/src/ccache/storage/local/localstorage.cpp +++ b/src/ccache/storage/local/localstorage.cpp @@ -189,20 +189,6 @@ set_counters(const StatsFile& stats_file, const Level1Counters& level_1_cs) }); } -static std::string -suffix_from_type(const core::CacheEntryType type) -{ - switch (type) { - case core::CacheEntryType::manifest: - return "M"; - - case core::CacheEntryType::result: - return "R"; - } - - ASSERT(false); -} - static uint8_t calculate_wanted_cache_level(const uint64_t files_in_level_1) { @@ -315,6 +301,26 @@ ratio(T numerator, T denominator) } } +static std::optional +result_path_from_raw_file(const std::string& path) +{ + if (path.length() < 3) { + return std::nullopt; + } + + if (path[path.length() - 3] == '_') { + // raw file ends with "_nn" where nn is file number in hex form + return path.substr(0, path.length() - 3); + } + + if (path[path.length() - 1] == 'W') { + // legacy: raw file ends with "nW" where n is file number (0-9) + return FMT("{}R", path.substr(0, path.length() - 2)); + } + + return std::nullopt; +} + static CleanDirResult clean_dir( const fs::path& l2_dir, @@ -352,11 +358,11 @@ clean_dir( continue; } - if (namespace_ && file_type_from_path(file.path()) == FileType::raw) { - util::PathString path_str(file.path()); - const std::string result_path = - FMT("{}R", path_str.str().substr(0, path_str.str().length() - 2)); - raw_files_map[result_path].push_back(file.path()); + if (namespace_) { + auto result_path = result_path_from_raw_file(util::pstr(file.path())); + if (result_path) { + raw_files_map[*result_path].push_back(file.path()); + } } cache_size += file.size_on_disk(); @@ -403,12 +409,10 @@ clean_dir( // For namespace eviction we need to remove raw files based on result // filename since they don't have a header. - if (file_type_from_path(file.path()) == FileType::result) { - const auto entry = raw_files_map.find(util::pstr(file.path())); - if (entry != raw_files_map.end()) { - for (const auto& raw_file : entry->second) { - delete_file(DirEntry(raw_file), cache_size, files_in_cache); - } + const auto entry = raw_files_map.find(util::pstr(file.path())); + if (entry != raw_files_map.end()) { + for (const auto& raw_file : entry->second) { + delete_file(DirEntry(raw_file), cache_size, files_in_cache); } } } @@ -429,21 +433,6 @@ clean_dir( return {counters_before, counters_after}; } -FileType -file_type_from_path(const fs::path& path) -{ - std::string filename = path.filename().string(); - if (util::ends_with(filename, "M")) { - return FileType::manifest; - } else if (util::ends_with(filename, "R")) { - return FileType::result; - } else if (util::ends_with(filename, "W")) { - return FileType::raw; - } else { - return FileType::unknown; - } -} - LocalStorage::LocalStorage(const Config& config) : m_config(config) { @@ -582,7 +571,7 @@ LocalStorage::put(const Hash::Digest& key, return; } - move_to_wanted_cache_level(*counters, key, type, cache_file.path); + move_to_wanted_cache_level(*counters, key, cache_file.path); // Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can // be done almost anywhere, but we might as well do it near the end as we save @@ -621,16 +610,7 @@ fs::path LocalStorage::get_raw_file_path(const fs::path& result_path, uint8_t file_number) { - if (file_number >= 10) { - // To support more entries in the future, encode to [0-9a-z]. Note that - // LocalStorage::evict currently assumes that the entry number is - // represented as one character. - throw core::Error(FMT("Too high raw file entry number: {}", file_number)); - } - - std::string s = result_path.string(); - s.pop_back(); - return FMT("{}{}W", s, file_number); + return FMT("{}_{:02x}", result_path, file_number); } fs::path @@ -647,6 +627,12 @@ LocalStorage::put_raw_files( const Hash::Digest& key, const std::vector& raw_files) { + const size_t k_max_raw_files = 256; + if (raw_files.size() > k_max_raw_files) { + throw core::Error( + FMT("Too many raw files: {} > {}", raw_files.size(), k_max_raw_files)); + } + const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result); core::ensure_dir_exists(cache_file.path.parent_path()); @@ -929,7 +915,7 @@ LocalStorage::recompress(const std::optional level, auto stats_file = get_stats_file(l1_index); for (const auto& file : files) { - if (file_type_from_path(file.path()) != FileType::unknown) { + if (!util::TemporaryFile::is_tmp_file(file.path())) { thread_pool.enqueue_detach([&, file, l2_index, stats_file, level] { try { DirEntry new_dir_entry = recompressor.recompress( @@ -964,8 +950,6 @@ LocalStorage::recompress(const std::optional level, incompressible_size += file.size_on_disk(); } }); - } else if (!util::TemporaryFile::is_tmp_file(file.path())) { - incompressible_size += file.size_on_disk(); } } @@ -1074,8 +1058,8 @@ LocalStorage::LookUpCacheFileResult LocalStorage::look_up_cache_file(const Hash::Digest& key, const core::CacheEntryType type) const { - const auto key_string = - FMT("{}{}", util::format_base16(key), suffix_from_type(type)); + // Try new format first: base16 without suffix + const auto key_string = util::format_base16(key); for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; ++level) { @@ -1086,6 +1070,27 @@ LocalStorage::look_up_cache_file(const Hash::Digest& key, } } + // Try old format with R/M suffix + const auto old_key_string = util::format_legacy_digest(key); + const std::string old_suffix = + type == core::CacheEntryType::manifest ? "M" : "R"; + const auto old_key_string_with_suffix = old_key_string + old_suffix; + for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; + ++level) { + const auto path = get_path_in_cache(level, old_key_string_with_suffix); + DirEntry dir_entry(path); + if (dir_entry.is_regular_file()) { + const auto new_path = get_path_in_cache(level, key_string); + LOG("Migrating {} to {}", path, new_path); + if (fs::rename(path, new_path)) { + return {new_path, DirEntry(new_path), level}; + } else { + LOG("Failed to rename {} to {}", path, new_path); + return {path, dir_entry, level}; + } + } + } + const auto shallowest_path = get_path_in_cache(k_min_cache_levels, key_string); return {shallowest_path, DirEntry(), k_min_cache_levels}; @@ -1107,13 +1112,12 @@ LocalStorage::get_stats_file(uint8_t l1_index, uint8_t l2_index) const void LocalStorage::move_to_wanted_cache_level(const StatisticsCounters& counters, const Hash::Digest& key, - core::CacheEntryType type, const fs::path& cache_file_path) { const auto wanted_level = calculate_wanted_cache_level(counters.get(Statistic::files_in_cache)); - const auto wanted_path = get_path_in_cache( - wanted_level, util::format_base16(key) + suffix_from_type(type)); + const auto wanted_path = + get_path_in_cache(wanted_level, util::format_base16(key)); if (cache_file_path != wanted_path) { core::ensure_dir_exists(wanted_path.parent_path()); diff --git a/src/ccache/storage/local/localstorage.hpp b/src/ccache/storage/local/localstorage.hpp index 3c5a5a65..5dc59856 100644 --- a/src/ccache/storage/local/localstorage.hpp +++ b/src/ccache/storage/local/localstorage.hpp @@ -60,8 +60,6 @@ struct CompressionStatistics enum class FileType { result, manifest, raw, unknown }; -FileType file_type_from_path(const std::filesystem::path& path); - class LocalStorage { public: @@ -166,7 +164,6 @@ private: void move_to_wanted_cache_level(const core::StatisticsCounters& counters, const Hash::Digest& key, - core::CacheEntryType type, const std::filesystem::path& cache_file_path); void recount_level_1_dir(util::LongLivedLockFileManager& lock_manager, diff --git a/test/run b/test/run index 1b8a781b..8b53696c 100755 --- a/test/run +++ b/test/run @@ -99,6 +99,26 @@ find_compiler() { }' $name } +_find_cache_files() { + local dir=$1 + local type=$2 + find "${dir}"/?/? -type f \! -name stats | while read path; do + if $CCACHE --inspect "${path}" 2>/dev/null | grep -q "Entry type: ${type}"; then + echo "${path}" + fi + done +} + +find_result_files() { + local dir=$1 + _find_cache_files "${dir}" 0 +} + +find_manifest_files() { + local dir=$1 + _find_cache_files "${dir}" 1 +} + generate_code() { local nlines=$1 local outfile=$2 diff --git a/test/suites/base.bash b/test/suites/base.bash index b3ce9e66..e10df373 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -366,7 +366,7 @@ EOF TEST "Result file is compressed" $CCACHE_COMPILE -c test1.c - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") if ! $CCACHE --inspect $result_file | grep 'Compression type: zstd' >/dev/null 2>&1; then test_failed "Result file not uncompressed according to metadata" fi @@ -387,7 +387,7 @@ EOF expect_stat cache_miss 1 expect_stat files_in_cache 1 - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") printf foo | dd of=$result_file bs=3 count=1 seek=20 conv=notrunc >&/dev/null $CCACHE_COMPILE -c test1.c @@ -1156,7 +1156,7 @@ EOF $CCACHE_COMPILE -MMD -c test.c expect_stat preprocessed_cache_hit 0 expect_stat cache_miss 1 - result_file=$(find "$CCACHE_DIR" -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") level_2_dir=$(dirname "$result_file") level_1_dir=$(dirname $(dirname "$result_file")) expect_perm test.o -rw-r--r-- diff --git a/test/suites/cache_levels.bash b/test/suites/cache_levels.bash index 90dbe20c..f47b787f 100644 --- a/test/suites/cache_levels.bash +++ b/test/suites/cache_levels.bash @@ -7,15 +7,19 @@ add_fake_files_counters() { } expect_on_level() { - local type="$1" - local expected_level="$2" - - slashes=$(find $CCACHE_DIR -name "*$type" \ - | sed -E -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g') - actual_level=$(echo -n "$slashes" | wc -c) - if [ "$actual_level" -ne "$expected_level" ]; then - test_failed "$type file on level $actual_level, expected level $expected_level" - fi + local expected_level="$1" + + local files=( + $(find_result_files "${CCACHE_DIR}") + $(find_manifest_files "${CCACHE_DIR}") + ) + for file in "${files[@]}"; do + slashes=$(echo $file | sed -E -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g') + actual_level=$(echo -n "$slashes" | wc -c) + if [ "$actual_level" -ne "$expected_level" ]; then + test_failed "file on level $actual_level, expected level $expected_level: $file" + fi + done } @@ -32,15 +36,13 @@ SUITE_cache_levels() { expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache 2 - expect_on_level R 2 - expect_on_level M 2 + expect_on_level 2 $CCACHE_COMPILE -c test1.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 expect_stat files_in_cache 2 - expect_on_level R 2 - expect_on_level M 2 + expect_on_level 2 # ------------------------------------------------------------------------- TEST "Many files but still level 2" @@ -52,15 +54,13 @@ SUITE_cache_levels() { expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 2 - expect_on_level M 2 + expect_on_level 2 $CCACHE_COMPILE -c test1.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 2 - expect_on_level M 2 + expect_on_level 2 # ------------------------------------------------------------------------- TEST "Level 3" @@ -72,15 +72,13 @@ SUITE_cache_levels() { expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 3 - expect_on_level M 3 + expect_on_level 3 $CCACHE_COMPILE -c test1.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 3 - expect_on_level M 3 + expect_on_level 3 # ------------------------------------------------------------------------- TEST "Level 4" @@ -92,15 +90,13 @@ SUITE_cache_levels() { expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 4 - expect_on_level M 4 + expect_on_level 4 $CCACHE_COMPILE -c test1.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 4 - expect_on_level M 4 + expect_on_level 4 # ------------------------------------------------------------------------- TEST "No deeper than 4 levels" @@ -112,13 +108,11 @@ SUITE_cache_levels() { expect_stat direct_cache_hit 0 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 4 - expect_on_level M 4 + expect_on_level 4 $CCACHE_COMPILE -c test1.c expect_stat direct_cache_hit 1 expect_stat cache_miss 1 expect_stat files_in_cache $((files + 2)) - expect_on_level R 4 - expect_on_level M 4 + expect_on_level 4 } diff --git a/test/suites/direct.bash b/test/suites/direct.bash index d6de9669..9333bf91 100644 --- a/test/suites/direct.bash +++ b/test/suites/direct.bash @@ -48,7 +48,7 @@ SUITE_direct() { expect_stat files_in_cache 2 # result + manifest expect_equal_object_files reference_test.o test.o - manifest_file=$(find $CCACHE_DIR -name '*M') + manifest_file=$(find_manifest_files "${CCACHE_DIR}") backdate $manifest_file $CCACHE_COMPILE -c test.c @@ -79,7 +79,7 @@ SUITE_direct() { expect_stat preprocessed_cache_hit 0 expect_stat cache_miss 1 - manifest_file=`find $CCACHE_DIR -name '*M'` + manifest_file=$(find_manifest_files "${CCACHE_DIR}") rm $manifest_file touch $manifest_file @@ -1229,7 +1229,7 @@ EOF CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS include_file_mtime" $CCACHE_COMPILE -c strange.c - manifest=`find $CCACHE_DIR -name '*M'` + manifest=$(find_manifest_files "${CCACHE_DIR}") if [ -n "$manifest" ]; then data="`$CCACHE --inspect $manifest | grep -E '/dev/(stdout|tty|sda|hda)'`" if [ -n "$data" ]; then @@ -1242,7 +1242,7 @@ EOF $CCACHE_COMPILE test.c -c -o test.o - manifest=`find $CCACHE_DIR -name '*M'` + manifest=$(find_manifest_files "${CCACHE_DIR}") $CCACHE --inspect $manifest >manifest.dump checksum_test1_h='b7271c414e35d190304ccbb02cdce8aaa391497e' @@ -1293,7 +1293,7 @@ int foo; EOF CCACHE_IGNOREHEADERS="subdir/ignore.h" $CCACHE_COMPILE -c ignore.c - manifest=`find $CCACHE_DIR -name '*M'` + manifest=$(find_manifest_files "${CCACHE_DIR}") data="`$CCACHE --inspect $manifest | grep subdir/ignore.h`" if [ -n "$data" ]; then test_failed "$manifest contained ignored header: $data" @@ -1313,7 +1313,7 @@ int foo; EOF CCACHE_IGNOREHEADERS="subdir" $CCACHE_COMPILE -c ignore.c - manifest=`find $CCACHE_DIR -name '*M'` + manifest=$(find_manifest_files "${CCACHE_DIR}") data="`$CCACHE --inspect $manifest | grep subdir/ignore.h`" if [ -n "$data" ]; then test_failed "$manifest contained ignored header: $data" diff --git a/test/suites/namespace.bash b/test/suites/namespace.bash index accf4001..99f5967e 100644 --- a/test/suites/namespace.bash +++ b/test/suites/namespace.bash @@ -36,7 +36,7 @@ SUITE_namespace() { TEST "--evict-namespace + --evict-older-than" CCACHE_NAMESPACE="a" $CCACHE_COMPILE -c test1.c - result_file="$(find $CCACHE_DIR -name '*R')" + result_file="$(find_result_files "${CCACHE_DIR}")" backdate "$result_file" for ns in a b c; do CCACHE_NAMESPACE="$ns" $CCACHE_COMPILE -c test2.c diff --git a/test/suites/no_compression.bash b/test/suites/no_compression.bash index ec7d7608..e71f6a37 100644 --- a/test/suites/no_compression.bash +++ b/test/suites/no_compression.bash @@ -27,7 +27,7 @@ SUITE_no_compression() { TEST "Result file is uncompressed" $CCACHE_COMPILE -c test.c - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") if ! $CCACHE --inspect $result_file | grep 'Compression type: none' >/dev/null 2>&1; then test_failed "Result file not uncompressed according to metadata" fi @@ -65,7 +65,7 @@ SUITE_no_compression() { expect_stat cache_miss 1 expect_stat files_in_cache 2 - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") # Write BAD at byte 300. printf BAD | dd of=$result_file bs=3 count=1 seek=100 conv=notrunc >&/dev/null diff --git a/test/suites/remote_file.bash b/test/suites/remote_file.bash index 9ed67e43..65bd8406 100644 --- a/test/suites/remote_file.bash +++ b/test/suites/remote_file.bash @@ -278,7 +278,7 @@ SUITE_remote_file() { $CCACHE_COMPILE -c test.c expect_perm remote drwxr-x-wx # 777 & 024 expect_perm remote/CACHEDIR.TAG -rw-r---w- # 666 & 024 - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042 expect_perm "${result_file}" -rw--w-r-- # 666 & 042 @@ -289,7 +289,7 @@ SUITE_remote_file() { $CCACHE_COMPILE -c test.c expect_perm remote drwxr-x--x # 777 & 026 expect_perm remote/CACHEDIR.TAG -rw-r----- # 666 & 026 - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042 expect_perm "${result_file}" -rw--w-r-- # 666 & 042 @@ -298,7 +298,7 @@ SUITE_remote_file() { $CCACHE_COMPILE -c test.c expect_perm remote drwxr-x--x # 777 & 026 expect_perm remote/CACHEDIR.TAG -rw-r----- # 666 & 026 - result_file=$(find $CCACHE_DIR -name '*R') + result_file=$(find_result_files "${CCACHE_DIR}") expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042 expect_perm "${result_file}" -rw--w-r-- # 666 & 042