From: Joel Rosdahl Date: Fri, 18 Aug 2023 20:17:40 +0000 (+0200) Subject: fix: Fix bookkeeping of raw files X-Git-Tag: v4.9~55 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=69f2bcc5cfe836128224acd4c215e2831f757fdf;p=thirdparty%2Fccache.git fix: Fix bookkeeping of raw files Ccache 4.8-4.8.2 notes the size and count of raw files (written to the cache if hard link or file clone mode is enabled) to a level 2 stats file. However, the new cleanup algorithm (also introduced in 4.8) assumes that all size/count statistics are placed in a level 1 stats file. After a "ccache -c", the raw files will be "rediscovered" and noted in level 1 stats files, thus duplicating bookkeeping in level 2 stats files, resulting in a too high cache size figure in "ccache -s". To fix this, ccache will now: - Note size/count of raw files in the proper level 1 stats file. - When storing an entry on a cache miss, detect if the corresponding level 2 stats file has non-zero size/count statistics and propagate them to the level 1 stats file if so. - Zero out files/count statistics for level 2 stats file on "ccache -c". This means that the reported cache size will gradually correct itself automatically. Alternatively, the user can run "ccache -c" to fix all counters immediately. Fixes #1320. --- diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp index 53a2704b4..3a4701abe 100644 --- a/src/storage/local/LocalStorage.cpp +++ b/src/storage/local/LocalStorage.cpp @@ -442,10 +442,33 @@ LocalStorage::finalize() // Pseudo-randomly choose one of the stats files in the 256 level 2 // directories. const auto bucket = getpid() % 256; - const auto stats_file = get_stats_file(bucket / 16, bucket % 16); - stats_file.update([&](auto& cs) { cs.increment(m_counter_updates); }); + const uint8_t l1_index = bucket / 16; + const uint8_t l2_index = bucket % 16; + const auto l2_stats_file = get_stats_file(l1_index, l2_index); + + uint64_t l2_files_in_cache = 0; + uint64_t l2_cache_size_kibibyte = 0; + + l2_stats_file.update([&](auto& cs) { + cs.increment(m_counter_updates); + + if (m_stored_data) { + // Ccache 4.8-4.8.2 erroneously stored files/size counters for raw files + // in L2, so move them to L1 to make the cleanup algorithm aware. + l2_files_in_cache = cs.get(Statistic::files_in_cache); + l2_cache_size_kibibyte = cs.get(Statistic::cache_size_kibibyte); + cs.set(Statistic::files_in_cache, 0); + cs.set(Statistic::cache_size_kibibyte, 0); + } + }); if (m_stored_data) { + // See comment about ccache 4.8-4.8.2 above. + if (l2_files_in_cache > 0 || l2_cache_size_kibibyte > 0) { + increment_files_and_size_counters( + l1_index, l2_index, l2_files_in_cache, l2_cache_size_kibibyte); + } + perform_automatic_cleanup(); } } @@ -618,6 +641,9 @@ LocalStorage::put_raw_files( const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result); core::ensure_dir_exists(Util::dir_name(cache_file.path)); + int64_t files_change = 0; + int64_t size_kibibyte_change = 0; + for (auto [file_number, source_path] : raw_files) { const auto dest_path = get_raw_file_path(cache_file.path, file_number); DirEntry old_dir_entry(dest_path); @@ -633,11 +659,11 @@ LocalStorage::put_raw_files( throw; } DirEntry new_dir_entry(dest_path); - increment_statistic(Statistic::cache_size_kibibyte, - kibibyte_size_diff(old_dir_entry, new_dir_entry)); - increment_statistic(Statistic::files_in_cache, - (new_dir_entry ? 1 : 0) - (old_dir_entry ? 1 : 0)); + files_change += (new_dir_entry ? 1 : 0) - (old_dir_entry ? 1 : 0); + size_kibibyte_change += kibibyte_size_diff(old_dir_entry, new_dir_entry); } + + increment_files_and_size_counters(key, files_change, size_kibibyte_change); } void @@ -746,7 +772,7 @@ LocalStorage::get_all_statistics() const }); counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp); - return std::make_pair(counters, last_updated); + return {counters, last_updated}; } void @@ -1085,12 +1111,11 @@ LocalStorage::recount_level_1_dir(util::LongLivedLockFileManager& lock_manager, } std::optional -LocalStorage::increment_files_and_size_counters(const Hash::Digest& key, +LocalStorage::increment_files_and_size_counters(uint8_t l1_index, + uint8_t l2_index, int64_t files, int64_t size_kibibyte) { - uint8_t l1_index = key[0] >> 4; - uint8_t l2_index = key[0] & 0xF; const auto level_1_stats_file = get_stats_file(l1_index); return level_1_stats_file.update([&](auto& cs) { // Level 1 counters: @@ -1104,6 +1129,15 @@ LocalStorage::increment_files_and_size_counters(const Hash::Digest& key, }); } +std::optional +LocalStorage::increment_files_and_size_counters(const Hash::Digest& key, + int64_t files, + int64_t size_kibibyte) +{ + return increment_files_and_size_counters( + key[0] >> 4, key[0] & 0xF, files, size_kibibyte); +} + static uint8_t get_largest_level_2_index(const StatisticsCounters& counters) { @@ -1289,6 +1323,16 @@ LocalStorage::do_clean_all(const ProgressReceiver& progress_receiver, if (clean_dir_result.after.files != clean_dir_result.before.files) { ++level_1_counters.cleanups; } + + // Fix erroneous files/size counters for raw files in L2 stats files. + // See also comments in finalize(). + get_stats_file(l1_index, l2_index) + .update( + [](auto& cs) { + cs.set(Statistic::cache_size_kibibyte, 0); + cs.set(Statistic::files_in_cache, 0); + }, + StatsFile::OnlyIfChanged::yes); }); set_counters(get_stats_file(l1_index), level_1_counters); diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp index 9a5af32b8..26dc828b7 100644 --- a/src/storage/local/LocalStorage.hpp +++ b/src/storage/local/LocalStorage.hpp @@ -160,6 +160,8 @@ private: void recount_level_1_dir(util::LongLivedLockFileManager& lock_manager, uint8_t l1_index); + std::optional increment_files_and_size_counters( + uint8_t l1_index, uint8_t l2_index, int64_t files, int64_t size_kibibyte); std::optional increment_files_and_size_counters( const Hash::Digest& key, int64_t files, int64_t size_kibibyte);