fix: Fix bookkeeping of raw files

Ccache 4.8-4.8.2 notes the size and count of raw files (written to the
cache if hard link or file clone mode is enabled) to a level 2 stats
file. However, the new cleanup algorithm (also introduced in 4.8)
assumes that all size/count statistics are placed in a level 1 stats
file. After a "ccache -c", the raw files will be "rediscovered" and
noted in level 1 stats files, thus duplicating bookkeeping in level 2
stats files, resulting in a too high cache size figure in "ccache -s".

To fix this, ccache will now:

- Note size/count of raw files in the proper level 1 stats file.
- When storing an entry on a cache miss, detect if the corresponding
  level 2 stats file has non-zero size/count statistics and propagate
  them to the level 1 stats file if so.
- Zero out files/count statistics for level 2 stats file on "ccache -c".

This means that the reported cache size will gradually correct itself
automatically. Alternatively, the user can run "ccache -c" to fix all
counters immediately.

Fixes #1320.

diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp
index 53a2704b47f158e27e867f37d074504bd195ef98..3a4701abe648061bbc0385f56bd8725ce04f4239 100644 (file)
--- a/src/storage/local/LocalStorage.cpp
+++ b/src/storage/local/LocalStorage.cpp
@@ -442,10 +442,33 @@ LocalStorage::finalize()
     // Pseudo-randomly choose one of the stats files in the 256 level 2
     // directories.
     const auto bucket = getpid() % 256;
-    const auto stats_file = get_stats_file(bucket / 16, bucket % 16);
-    stats_file.update([&](auto& cs) { cs.increment(m_counter_updates); });
+    const uint8_t l1_index = bucket / 16;
+    const uint8_t l2_index = bucket % 16;
+    const auto l2_stats_file = get_stats_file(l1_index, l2_index);
+
+    uint64_t l2_files_in_cache = 0;
+    uint64_t l2_cache_size_kibibyte = 0;
+
+    l2_stats_file.update([&](auto& cs) {
+      cs.increment(m_counter_updates);
+
+      if (m_stored_data) {
+        // Ccache 4.8-4.8.2 erroneously stored files/size counters for raw files
+        // in L2, so move them to L1 to make the cleanup algorithm aware.
+        l2_files_in_cache = cs.get(Statistic::files_in_cache);
+        l2_cache_size_kibibyte = cs.get(Statistic::cache_size_kibibyte);
+        cs.set(Statistic::files_in_cache, 0);
+        cs.set(Statistic::cache_size_kibibyte, 0);
+      }
+    });
 
     if (m_stored_data) {
+      // See comment about ccache 4.8-4.8.2 above.
+      if (l2_files_in_cache > 0 || l2_cache_size_kibibyte > 0) {
+        increment_files_and_size_counters(
+          l1_index, l2_index, l2_files_in_cache, l2_cache_size_kibibyte);
+      }
+
       perform_automatic_cleanup();
     }
   }
@@ -618,6 +641,9 @@ LocalStorage::put_raw_files(
   const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result);
   core::ensure_dir_exists(Util::dir_name(cache_file.path));
 
+  int64_t files_change = 0;
+  int64_t size_kibibyte_change = 0;
+
   for (auto [file_number, source_path] : raw_files) {
     const auto dest_path = get_raw_file_path(cache_file.path, file_number);
     DirEntry old_dir_entry(dest_path);
@@ -633,11 +659,11 @@ LocalStorage::put_raw_files(
       throw;
     }
     DirEntry new_dir_entry(dest_path);
-    increment_statistic(Statistic::cache_size_kibibyte,
-                        kibibyte_size_diff(old_dir_entry, new_dir_entry));
-    increment_statistic(Statistic::files_in_cache,
-                        (new_dir_entry ? 1 : 0) - (old_dir_entry ? 1 : 0));
+    files_change += (new_dir_entry ? 1 : 0) - (old_dir_entry ? 1 : 0);
+    size_kibibyte_change += kibibyte_size_diff(old_dir_entry, new_dir_entry);
   }
+
+  increment_files_and_size_counters(key, files_change, size_kibibyte_change);
 }
 
 void
@@ -746,7 +772,7 @@ LocalStorage::get_all_statistics() const
     });
 
   counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp);
-  return std::make_pair(counters, last_updated);
+  return {counters, last_updated};
 }
 
 void
@@ -1085,12 +1111,11 @@ LocalStorage::recount_level_1_dir(util::LongLivedLockFileManager& lock_manager,
 }
 
 std::optional<core::StatisticsCounters>
-LocalStorage::increment_files_and_size_counters(const Hash::Digest& key,
+LocalStorage::increment_files_and_size_counters(uint8_t l1_index,
+                                                uint8_t l2_index,
                                                 int64_t files,
                                                 int64_t size_kibibyte)
 {
-  uint8_t l1_index = key[0] >> 4;
-  uint8_t l2_index = key[0] & 0xF;
   const auto level_1_stats_file = get_stats_file(l1_index);
   return level_1_stats_file.update([&](auto& cs) {
     // Level 1 counters:
@@ -1104,6 +1129,15 @@ LocalStorage::increment_files_and_size_counters(const Hash::Digest& key,
   });
 }
 
+std::optional<core::StatisticsCounters>
+LocalStorage::increment_files_and_size_counters(const Hash::Digest& key,
+                                                int64_t files,
+                                                int64_t size_kibibyte)
+{
+  return increment_files_and_size_counters(
+    key[0] >> 4, key[0] & 0xF, files, size_kibibyte);
+}
+
 static uint8_t
 get_largest_level_2_index(const StatisticsCounters& counters)
 {
@@ -1289,6 +1323,16 @@ LocalStorage::do_clean_all(const ProgressReceiver& progress_receiver,
           if (clean_dir_result.after.files != clean_dir_result.before.files) {
             ++level_1_counters.cleanups;
           }
+
+          // Fix erroneous files/size counters for raw files in L2 stats files.
+          // See also comments in finalize().
+          get_stats_file(l1_index, l2_index)
+            .update(
+              [](auto& cs) {
+                cs.set(Statistic::cache_size_kibibyte, 0);
+                cs.set(Statistic::files_in_cache, 0);
+              },
+              StatsFile::OnlyIfChanged::yes);
         });
 
       set_counters(get_stats_file(l1_index), level_1_counters);

diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp
index 9a5af32b8731bf6b3c0431b887c96951dc12d87c..26dc828b7fbdcacaf72a8517982c6975df5d131b 100644 (file)
--- a/src/storage/local/LocalStorage.hpp
+++ b/src/storage/local/LocalStorage.hpp
@@ -160,6 +160,8 @@ private:
   void recount_level_1_dir(util::LongLivedLockFileManager& lock_manager,
                            uint8_t l1_index);
 
+  std::optional<core::StatisticsCounters> increment_files_and_size_counters(
+    uint8_t l1_index, uint8_t l2_index, int64_t files, int64_t size_kibibyte);
   std::optional<core::StatisticsCounters> increment_files_and_size_counters(
     const Hash::Digest& key, int64_t files, int64_t size_kibibyte);