From: Joel Rosdahl <joel@rosdahl.net>
Date: Wed, 29 Oct 2025 18:06:18 +0000 (+0100)
Subject: perf: Parallelize -x/--show-compression
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=83565e24e13f5f7f81bb8b6d93e980784dbc0b04;p=thirdparty%2Fccache.git

perf: Parallelize -x/--show-compression
---

diff --git a/src/ccache/core/mainoptions.cpp b/src/ccache/core/mainoptions.cpp
index ef3f4cc9..4e0d34e0 100644
--- a/src/ccache/core/mainoptions.cpp
+++ b/src/ccache/core/mainoptions.cpp
@@ -824,7 +824,7 @@ process_main_options(int argc, const char* const* argv)
       ProgressBar progress_bar("Scanning...");
       const auto compression_statistics =
         storage::local::LocalStorage(config).get_compression_statistics(
-          [&](double progress) { progress_bar.update(progress); });
+          threads, [&](double progress) { progress_bar.update(progress); });
       if (isatty(STDOUT_FILENO)) {
         PRINT_RAW(stdout, "\n\n");
       }
diff --git a/src/ccache/storage/local/localstorage.cpp b/src/ccache/storage/local/localstorage.cpp
index 99b18c51..d4b1bcda 100644
--- a/src/ccache/storage/local/localstorage.cpp
+++ b/src/ccache/storage/local/localstorage.cpp
@@ -837,35 +837,59 @@ LocalStorage::wipe_all(const ProgressReceiver& progress_receiver)
 
 CompressionStatistics
 LocalStorage::get_compression_statistics(
-  const ProgressReceiver& progress_receiver) const
+  const uint32_t threads, const ProgressReceiver& progress_receiver) const
 {
-  CompressionStatistics cs{};
+  std::atomic<uint64_t> content_size{0};
+  std::atomic<uint64_t> actual_size{0};
+  std::atomic<uint64_t> incompressible_size{0};
+  std::atomic<uint32_t> completed_dirs{0};
 
-  for_each_cache_subdir(
-    progress_receiver,
-    [&](const auto& l1_index, const auto& l1_progress_receiver) {
-      for_each_cache_subdir(
-        l1_progress_receiver,
-        [&](const auto& l2_index, const auto& l2_progress_receiver) {
-          auto l2_dir = get_subdir(l1_index, l2_index);
-          const auto files = get_cache_dir_files(l2_dir);
-          l2_progress_receiver(0.2);
+  const size_t read_ahead =
+    std::max(static_cast<size_t>(10), 2 * static_cast<size_t>(threads));
+  util::ThreadPool thread_pool(threads, read_ahead);
 
-          for (size_t i = 0; i < files.size(); ++i) {
-            const auto& cache_file = files[i];
-            try {
-              core::CacheEntry::Header header(cache_file.path());
-              cs.actual_size += cache_file.size_on_disk();
-              cs.content_size += util::likely_size_on_disk(header.entry_size);
-            } catch (core::Error&) {
-              cs.incompressible_size += cache_file.size_on_disk();
-            }
-            l2_progress_receiver(0.2 + 0.8 * ratio(i, files.size()));
+  std::vector<std::future<void>> futures;
+  futures.reserve(256);
+
+  for_each_cache_subdir([&](uint8_t l1_index) {
+    for_each_cache_subdir([&](uint8_t l2_index) {
+      futures.push_back(thread_pool.enqueue([&, l1_index, l2_index] {
+        auto l2_dir = get_subdir(l1_index, l2_index);
+        const auto files = get_cache_dir_files(l2_dir);
+
+        uint64_t local_content_size = 0;
+        uint64_t local_actual_size = 0;
+        uint64_t local_incompressible_size = 0;
+
+        for (const auto& cache_file : files) {
+          try {
+            core::CacheEntry::Header header(cache_file.path());
+            local_actual_size += cache_file.size_on_disk();
+            local_content_size += util::likely_size_on_disk(header.entry_size);
+          } catch (core::Error&) {
+            local_incompressible_size += cache_file.size_on_disk();
           }
-        });
+        }
+
+        // Atomic updates (fewer atomic ops by accumulating locally first).
+        content_size += local_content_size;
+        actual_size += local_actual_size;
+        incompressible_size += local_incompressible_size;
+        ++completed_dirs;
+
+        progress_receiver(completed_dirs / 256.0);
+      }));
     });
+  });
+
+  for (auto& future : futures) {
+    future.get();
+  }
+
+  thread_pool.shut_down();
 
-  return cs;
+  return CompressionStatistics{
+    content_size.load(), actual_size.load(), incompressible_size.load()};
 }
 
 void
diff --git a/src/ccache/storage/local/localstorage.hpp b/src/ccache/storage/local/localstorage.hpp
index 5ce7c793..3c5a5a65 100644
--- a/src/ccache/storage/local/localstorage.hpp
+++ b/src/ccache/storage/local/localstorage.hpp
@@ -126,7 +126,8 @@ public:
   // --- Compression ---
 
   CompressionStatistics
-  get_compression_statistics(const ProgressReceiver& progress_receiver) const;
+  get_compression_statistics(uint32_t threads,
+                             const ProgressReceiver& progress_receiver) const;
 
   void recompress(std::optional<int8_t> level,
                   uint32_t threads,