Store cache statistics on level 2 and cache bookkeeping still on level 1

In a “mostly cache hits” scenario, many parallel ccache invocations may
want to update statistics at the same time. Spreading out counter
updates over 256 instead of 16 stats files reduces lock contention
significantly.

Keeping cache bookkeeping (i.e. cache size and files counters) on level
1 preserves backward compatibility with the cleanup algorithm in older
ccache versions.

Some counters displayed by “ccache -s” won’t be accurate when an older
ccache version is run on a cache directory that is also used by ccache
4.x, but that shouldn’t be much of a problem in practice. Also, the
cache statistics isn’t backward compatible anyway since new counters may
be introduced and they will be invisible to older versions.

Closes #168 (only statistics are moved to the second level, though).

diff --git a/src/Statistics.cpp b/src/Statistics.cpp
index 84e2301789f47b3f86a8d452b0ddfe4c5fde74a3..0861de63b9ca3944127f843fef4fb4b6f6c7b41d 100644 (file)
--- a/src/Statistics.cpp
+++ b/src/Statistics.cpp
@@ -79,6 +79,19 @@ hit_rate(const Counters& counters)
   return total > 0 ? (100.0 * hit) / total : 0.0;
 }
 
+static void
+for_each_level_1_and_2_stats_file(
+  const std::string& cache_dir,
+  const std::function<void(const std::string& path)> function)
+{
+  for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) {
+    function(fmt::format("{}/{:x}/stats", cache_dir, level_1));
+    for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) {
+      function(fmt::format("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2));
+    }
+  }
+}
+
 static std::pair<Counters, time_t>
 collect_counters(const Config& config)
 {
@@ -87,15 +100,14 @@ collect_counters(const Config& config)
   time_t last_updated = 0;
 
   // Add up the stats in each directory.
-  for (size_t dir = 0; dir <= 0xF; ++dir) {
-    const auto path = fmt::format("{}/{:x}/stats", config.cache_dir(), dir);
-
-    counters.set(Statistic::stats_zeroed_timestamp, 0); // Don't add
-    counters.increment(Statistics::read(path));
-    zero_timestamp =
-      std::max(counters.get(Statistic::stats_zeroed_timestamp), zero_timestamp);
-    last_updated = std::max(last_updated, Stat::stat(path).mtime());
-  }
+  for_each_level_1_and_2_stats_file(
+    config.cache_dir(), [&](const std::string& path) {
+      counters.set(Statistic::stats_zeroed_timestamp, 0); // Don't add
+      counters.increment(Statistics::read(path));
+      zero_timestamp = std::max(counters.get(Statistic::stats_zeroed_timestamp),
+                                zero_timestamp);
+      last_updated = std::max(last_updated, Stat::stat(path).mtime());
+    });
 
   counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp);
   return std::make_pair(counters, last_updated);
@@ -252,22 +264,17 @@ zero_all_counters(const Config& config)
 {
   const time_t timestamp = time(nullptr);
 
-  for (size_t dir = 0; dir <= 0xF; ++dir) {
-    auto fname = fmt::format("{}/{:x}/stats", config.cache_dir(), dir);
-    if (!Stat::stat(fname)) {
-      // No point in trying to reset the stats file if it doesn't exist.
-      continue;
-    }
-
-    Statistics::update(fname, [=](Counters& cs) {
-      for (size_t i = 0; k_statistics_fields[i].message; ++i) {
-        if (!(k_statistics_fields[i].flags & FLAG_NOZERO)) {
-          cs.set(k_statistics_fields[i].statistic, 0);
+  for_each_level_1_and_2_stats_file(
+    config.cache_dir(), [=](const std::string& path) {
+      Statistics::update(path, [=](Counters& cs) {
+        for (size_t i = 0; k_statistics_fields[i].message; ++i) {
+          if (!(k_statistics_fields[i].flags & FLAG_NOZERO)) {
+            cs.set(k_statistics_fields[i].statistic, 0);
+          }
         }
-      }
-      cs.set(Statistic::stats_zeroed_timestamp, timestamp);
+        cs.set(Statistic::stats_zeroed_timestamp, timestamp);
+      });
     });
-  }
 }
 
 std::string

diff --git a/src/ccache.cpp b/src/ccache.cpp
index 67358ba9ada2acee4b07eccf75b362e465a28079..352ecfd0fa9ca85f6a437c4f7fb4b5149de08e3e 100644 (file)
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -1971,8 +1971,19 @@ update_stats_and_maybe_move_cache_file(const Context& ctx,
                                        const Counters& counter_updates,
                                        const std::string& file_suffix)
 {
+  // Use stats file in the level one subdirectory for cache bookkeeping counters
+  // since cleanup is performed on level one. Use stats file in the level two
+  // subdirectory for other counters to reduce lock contention.
+  const bool updated_file_size_or_count =
+    counter_updates.get(Statistic::cache_size_kibibyte) != 0
+    || counter_updates.get(Statistic::files_in_cache) != 0;
+  std::string level_string = fmt::format("{:x}", name.bytes()[0] >> 4);
+  if (!updated_file_size_or_count) {
+    level_string += fmt::format("/{:x}", name.bytes()[0] & 0xF);
+  }
   const auto stats_file =
-    fmt::format("{}/{:x}/stats", ctx.config.cache_dir(), name.bytes()[0] >> 4);
+    fmt::format("{}/{}/stats", ctx.config.cache_dir(), level_string);
+
   auto counters =
     Statistics::update(stats_file, [&counter_updates](Counters& cs) {
       cs.increment(counter_updates);
@@ -2021,10 +2032,14 @@ finalize_stats_and_trigger_cleanup(Context& ctx)
   }
 
   if (!ctx.result_path()) {
+    ASSERT(ctx.counter_updates.get(Statistic::cache_size_kibibyte) == 0);
+    ASSERT(ctx.counter_updates.get(Statistic::files_in_cache) == 0);
+
     // Context::set_result_path hasn't been called yet, so we just choose one of
-    // stats files in the 16 subdirectories.
-    const auto stats_file =
-      fmt::format("{}/{:x}/stats", config.cache_dir(), getpid() % 16);
+    // the stats files in the 256 level 2 directories.
+    const auto bucket = getpid() % 256;
+    const auto stats_file = fmt::format(
+      "{}/{:x}/{:x}/stats", config.cache_dir(), bucket / 16, bucket % 16);
     Statistics::update(
       stats_file, [&ctx](Counters& cs) { cs.increment(ctx.counter_updates); });
     return;