Add progress bar for -c/--cleanup and -C/--clear

diff --git a/dev.mk.in b/dev.mk.in
index 2561fe5bb0fa828464e66d63b99f54642391b3cd..3e6f3c83218cf5e48d07ba6ad00ef44b9934b7a8 100644 (file)
--- a/dev.mk.in
+++ b/dev.mk.in
@@ -41,6 +41,7 @@ non_third_party_headers = \
     src/Error.hpp \
     src/ProgressBar.hpp \
     src/ccache.hpp \
+    src/cleanup.hpp \
     src/common_header.hpp \
     src/compopt.hpp \
     src/compression.hpp \

diff --git a/src/ccache.cpp b/src/ccache.cpp
index 9142d355a915b6a6098bbb588bc6bfa0c250afe1..4b0297485cf4252c85d121fe79cae9490eddb5d3 100644 (file)
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -20,6 +20,8 @@
 #include "ccache.hpp"
 
 #include "Error.hpp"
+#include "ProgressBar.hpp"
+#include "cleanup.hpp"
 #include "compopt.hpp"
 #include "util.hpp"
 
@@ -3980,16 +3982,28 @@ ccache_main_options(int argc, char* argv[])
       break;
 
     case 'c': // --cleanup
+    {
       initialize();
-      clean_up_all(g_config);
-      printf("Cleaned cache\n");
+      ProgressBar progress_bar("Cleaning...");
+      clean_up_all(g_config,
+                   [&](double progress) { progress_bar.update(progress); });
+      if (isatty(STDOUT_FILENO)) {
+        printf("\n");
+      }
       break;
+    }
 
     case 'C': // --clear
+    {
       initialize();
-      wipe_all(g_config);
-      printf("Cleared cache\n");
+      ProgressBar progress_bar("Clearing...");
+      wipe_all(g_config,
+               [&](double progress) { progress_bar.update(progress); });
+      if (isatty(STDOUT_FILENO)) {
+        printf("\n");
+      }
       break;
+    }
 
     case 'h': // --help
       fputs(USAGE_TEXT, stdout);

diff --git a/src/ccache.hpp b/src/ccache.hpp
index f3b26309a04f5183fb7dce198c945ead364575cf..56ed97a04bf26434eebffe0f98a4a88dc2f4e40c 100644 (file)
--- a/src/ccache.hpp
+++ b/src/ccache.hpp
@@ -22,6 +22,7 @@
 #include "system.hpp"
 
 #include "counters.hpp"
+#include "util.hpp"
 
 #include "third_party/minitrace.h"
 
@@ -248,13 +249,6 @@ void exitfn_add(void (*function)(void*), void* context);
 void exitfn_add_last(void (*function)(void*), void* context);
 void exitfn_call(void);
 
-// ----------------------------------------------------------------------------
-// cleanup.c
-
-void clean_up_dir(const Config& config, const char* dir, double limit_multiple);
-void clean_up_all(const Config& config);
-void wipe_all(const Config& config);
-
 // ----------------------------------------------------------------------------
 // compress.c
 

diff --git a/src/cleanup.cpp b/src/cleanup.cpp
index 955a937bea024e26d6ebd16cebb3d62b2a97443c..88956f15aa37bb78f6e977b9dd02dee5a6417771 100644 (file)
--- a/src/cleanup.cpp
+++ b/src/cleanup.cpp
@@ -17,131 +17,107 @@
 // this program; if not, write to the Free Software Foundation, Inc., 51
 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
+#include "cleanup.hpp"
+
+#include "CacheFile.hpp"
 #include "Config.hpp"
 #include "ccache.hpp"
 
+#include <algorithm>
 #include <math.h>
 
-struct files
-{
-  char* fname;
-  time_t mtime;
-  uint64_t size;
-};
-
-static struct files** files;
-static unsigned allocated; // Size of the files array.
-static unsigned num_files; // Number of used entries in the files array.
-
-static uint64_t cache_size;
-static size_t files_in_cache;
-static uint64_t cache_size_threshold;
-static size_t files_in_cache_threshold;
-
-// File comparison function that orders files in mtime order, oldest first.
-static int
-files_compare(struct files** f1, struct files** f2)
+static void
+delete_file(const std::string& path,
+            size_t size,
+            uint64_t* cache_size,
+            uint32_t* files_in_cache)
 {
-  if ((*f2)->mtime == (*f1)->mtime) {
-    return strcmp((*f1)->fname, (*f2)->fname);
-  }
-  if ((*f2)->mtime > (*f1)->mtime) {
-    return -1;
+  bool deleted = x_try_unlink(path.c_str()) == 0;
+  if (!deleted && errno != ENOENT && errno != ESTALE) {
+    cc_log("Failed to unlink %s (%s)", path.c_str(), strerror(errno));
+  } else if (cache_size && files_in_cache) {
+    // The counters are intentionally subtracted even if there was no file to
+    // delete since the final cache size calculation will be incorrect if they
+    // aren't. (This can happen when there are several parallel ongoing
+    // cleanups of the same directory.)
+    *cache_size -= size;
+    --*files_in_cache;
   }
-  return 1;
 }
 
-// This builds the list of files in the cache.
-static void
-traverse_fn(const char* fname, struct stat* st)
+// Clean up one cache subdirectory.
+void
+clean_up_dir(const std::string& subdir,
+             uint64_t max_size,
+             uint32_t max_files,
+             const util::ProgressReceiver& progress_receiver)
 {
-  if (!S_ISREG(st->st_mode)) {
-    return;
-  }
-
-  char* p = x_basename(fname);
-  if (str_eq(p, "stats")) {
-    goto out;
-  }
+  cc_log("Cleaning up cache directory %s", subdir.c_str());
 
-  if (str_startswith(p, ".nfs")) {
-    // Ignore temporary NFS files that may be left for open but deleted files.
-    goto out;
-  }
+  std::vector<std::shared_ptr<CacheFile>> files;
+  util::get_level_1_files(
+    subdir, [&](double progress) { progress_receiver(progress / 3); }, files);
 
-  // Delete any tmp files older than 1 hour.
-  if (strstr(p, ".tmp.") && st->st_mtime + 3600 < time(NULL)) {
-    x_unlink(fname);
-    goto out;
-  }
+  uint64_t cache_size = 0;
+  uint32_t files_in_cache = 0;
+  time_t current_time = time(NULL);
 
-  if (strstr(p, "CACHEDIR.TAG")) {
-    goto out;
-  }
+  for (size_t i = 0; i < files.size();
+       ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) {
+    const auto& file = files[i];
 
-  if (num_files == allocated) {
-    allocated = 10000 + num_files * 2;
-    files = (struct files**)x_realloc(files, sizeof(struct files*) * allocated);
-  }
-
-  files[num_files] = (struct files*)x_malloc(sizeof(struct files));
-  files[num_files]->fname = x_strdup(fname);
-  files[num_files]->mtime = st->st_mtime;
-  files[num_files]->size = file_size(st);
-  cache_size += files[num_files]->size;
-  files_in_cache++;
-  num_files++;
+    if (!S_ISREG(file->stat().st_mode)) {
+      // Not a file or missing file.
+      continue;
+    }
 
-out:
-  free(p);
-}
+    // Delete any tmp files older than 1 hour right away.
+    if (file->stat().st_mtime + 3600 < current_time
+        && util::base_name(file->path()).find(".tmp.") != std::string::npos) {
+      x_unlink(file->path().c_str());
+      continue;
+    }
 
-static void
-delete_file(const char* path, size_t size, bool update_counters)
-{
-  bool deleted = x_try_unlink(path) == 0;
-  if (!deleted && errno != ENOENT && errno != ESTALE) {
-    cc_log("Failed to unlink %s (%s)", path, strerror(errno));
-  } else if (update_counters) {
-    // The counters are intentionally subtracted even if there was no file to
-    // delete since the final cache size calculation will be incorrect if they
-    // aren't. (This can happen when there are several parallel ongoing
-    // cleanups of the same directory.)
-    cache_size -= size;
-    files_in_cache--;
+    cache_size += file_size(&file->stat());
+    files_in_cache += 1;
   }
-}
 
-// Sort the files we've found and delete the oldest ones until we are below the
-// thresholds.
-static bool
-sort_and_clean(void)
-{
-  if (num_files > 1) {
-    // Sort in ascending mtime order.
-    qsort(files, num_files, sizeof(struct files*), (COMPAR_FN_T)files_compare);
-  }
+  // Sort according to modification time, oldest first.
+  std::sort(files.begin(),
+            files.end(),
+            [](const std::shared_ptr<CacheFile>& f1,
+               const std::shared_ptr<CacheFile>& f2) {
+              return f1->stat().st_mtime < f2->stat().st_mtime;
+            });
+
+  cc_log("Before cleanup: %.0f KiB, %.0f files",
+         static_cast<double>(cache_size) / 1024,
+         static_cast<double>(files_in_cache));
 
-  // Delete enough files to bring us below the threshold.
   bool cleaned = false;
-  for (unsigned i = 0; i < num_files; i++) {
-    const char* ext;
+  for (size_t i = 0; i < files.size();
+       ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) {
+    const auto& file = files[i];
 
-    if ((cache_size_threshold == 0 || cache_size <= cache_size_threshold)
-        && (files_in_cache_threshold == 0
-            || files_in_cache <= files_in_cache_threshold)) {
+    if (!S_ISREG(file->stat().st_mode)) {
+      // Not a file or missing file.
+      continue;
+    }
+
+    if ((max_size == 0 || cache_size <= max_size)
+        && (max_files == 0 || files_in_cache <= max_files)) {
       break;
     }
 
-    ext = get_extension(files[i]->fname);
-    if (str_eq(ext, ".stderr")) {
-      // Make sure that the .o file is deleted before .stderr, because if the
-      // ccache process gets killed after deleting the .stderr but before
-      // deleting the .o, the cached result will be inconsistent. (.stderr is
-      // the only file that is optional; any other file missing from the cache
-      // will be detected by get_file_from_cache.)
-      char* base = remove_extension(files[i]->fname);
-      char* o_file = format("%s.o", base);
+    if (util::ends_with(file->path(), ".stderr")) {
+      // In order to be nice to legacy ccache versions, make sure that the .o
+      // file is deleted before .stderr, because if the ccache process gets
+      // killed after deleting the .stderr but before deleting the .o, the
+      // cached result will be inconsistent. (.stderr is the only file that is
+      // optional for legacy ccache versions; any other file missing from the
+      // cache will be detected.)
+      std::string o_file =
+        file->path().substr(0, file->path().size() - 6) + "o";
 
       // Don't subtract this extra deletion from the cache size; that
       // bookkeeping will be done when the loop reaches the .o file. If the
@@ -149,130 +125,71 @@ sort_and_clean(void)
       // reached, the bookkeeping won't happen, but that small counter
       // discrepancy won't do much harm and it will correct itself in the next
       // cleanup.
-      delete_file(o_file, 0, false);
-
-      free(o_file);
-      free(base);
+      delete_file(o_file, 0, nullptr, nullptr);
     }
-    delete_file(files[i]->fname, files[i]->size, true);
+
+    delete_file(
+      file->path(), file_size(&file->stat()), &cache_size, &files_in_cache);
     cleaned = true;
   }
-  return cleaned;
-}
 
-// Clean up one cache subdirectory.
-void
-clean_up_dir(const Config& config, const char* dir, double limit_multiple)
-{
-  cc_log("Cleaning up cache directory %s", dir);
-
-  // When "max files" or "max cache size" is reached, one of the 16 cache
-  // subdirectories is cleaned up. When doing so, files are deleted (in LRU
-  // order) until the levels are below limit_multiple.
-  double cache_size_float = round(config.max_size() * limit_multiple / 16);
-  cache_size_threshold = (uint64_t)cache_size_float;
-  double files_in_cache_float = round(config.max_files() * limit_multiple / 16);
-  files_in_cache_threshold = (size_t)files_in_cache_float;
-
-  num_files = 0;
-  cache_size = 0;
-  files_in_cache = 0;
-
-  // Build a list of files.
-  traverse(dir, traverse_fn);
-
-  // Clean the cache.
-  cc_log("Before cleanup: %.0f KiB, %.0f files",
-         (double)cache_size / 1024,
-         (double)files_in_cache);
-  bool cleaned = sort_and_clean();
   cc_log("After cleanup: %.0f KiB, %.0f files",
-         (double)cache_size / 1024,
-         (double)files_in_cache);
+         static_cast<double>(cache_size) / 1024,
+         static_cast<double>(files_in_cache));
 
   if (cleaned) {
-    cc_log("Cleaned up cache directory %s", dir);
-    stats_add_cleanup(dir, 1);
+    cc_log("Cleaned up cache directory %s", subdir.c_str());
+    stats_add_cleanup(subdir.c_str(), 1);
   }
 
-  stats_set_sizes(dir, files_in_cache, cache_size);
-
-  // Free it up.
-  for (unsigned i = 0; i < num_files; i++) {
-    free(files[i]->fname);
-    free(files[i]);
-    files[i] = NULL;
-  }
-  if (files) {
-    free(files);
-  }
-  allocated = 0;
-  files = NULL;
-
-  num_files = 0;
-  cache_size = 0;
-  files_in_cache = 0;
+  stats_set_sizes(subdir.c_str(), files_in_cache, cache_size);
 }
 
 // Clean up all cache subdirectories.
 void
-clean_up_all(const Config& config)
+clean_up_all(const Config& config,
+             const util::ProgressReceiver& progress_receiver)
 {
-  for (int i = 0; i <= 0xF; i++) {
-    char* dname = format("%s/%1x", config.cache_dir().c_str(), i);
-    clean_up_dir(config, dname, 1.0);
-    free(dname);
-  }
-}
-
-// Traverse function for wiping files.
-static void
-wipe_fn(const char* fname, struct stat* st)
-{
-  if (!S_ISREG(st->st_mode)) {
-    return;
-  }
-
-  char* p = x_basename(fname);
-  if (str_eq(p, "stats")) {
-    free(p);
-    return;
-  }
-  free(p);
-
-  files_in_cache++;
-
-  x_unlink(fname);
+  util::for_each_level_1_subdir(
+    config.cache_dir(),
+    [&](const std::string& subdir,
+        const util::ProgressReceiver& sub_progress_receiver) {
+      clean_up_dir(subdir,
+                   config.max_size() / 16,
+                   config.max_files() / 16,
+                   sub_progress_receiver);
+    },
+    progress_receiver);
 }
 
 // Wipe one cache subdirectory.
 static void
-wipe_dir(const char* dir)
+wipe_dir(const std::string& subdir,
+         const util::ProgressReceiver& progress_receiver)
 {
-  cc_log("Clearing out cache directory %s", dir);
+  cc_log("Clearing out cache directory %s", subdir.c_str());
 
-  files_in_cache = 0;
+  std::vector<std::shared_ptr<CacheFile>> files;
+  util::get_level_1_files(
+    subdir, [&](double progress) { progress_receiver(progress / 2); }, files);
 
-  traverse(dir, wipe_fn);
+  for (size_t i = 0; i < files.size(); ++i) {
+    x_unlink(files[i]->path().c_str());
+    progress_receiver(0.5 + 0.5 * i / files.size());
+  }
 
-  if (files_in_cache > 0) {
-    cc_log("Cleared out cache directory %s", dir);
-    stats_add_cleanup(dir, 1);
+  if (!files.empty()) {
+    cc_log("Cleared out cache directory %s", subdir.c_str());
+    stats_add_cleanup(subdir.c_str(), 1);
   }
 
-  files_in_cache = 0;
+  stats_set_sizes(subdir.c_str(), 0, 0);
 }
 
 // Wipe all cached files in all subdirectories.
 void
-wipe_all(const Config& config)
+wipe_all(const Config& config, const util::ProgressReceiver& progress_receiver)
 {
-  for (int i = 0; i <= 0xF; i++) {
-    char* dname = format("%s/%1x", config.cache_dir().c_str(), i);
-    wipe_dir(dname);
-    free(dname);
-  }
-
-  // Fix the counters.
-  clean_up_all(config);
+  util::for_each_level_1_subdir(
+    config.cache_dir(), wipe_dir, progress_receiver);
 }

diff --git a/src/cleanup.hpp b/src/cleanup.hpp
new file mode 100644 (file)
index 0000000..7fce900
--- /dev/null
+++ b/src/cleanup.hpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include "util.hpp"
+
+#include <string>
+
+class Config;
+
+void clean_up_dir(const std::string& subdir,
+                  uint64_t max_size,
+                  uint32_t max_files,
+                  const util::ProgressReceiver& progress_receiver);
+
+void clean_up_all(const Config& config,
+                  const util::ProgressReceiver& progress_receiver);
+
+void wipe_all(const Config& config,
+              const util::ProgressReceiver& progress_receiver);

diff --git a/src/stats.cpp b/src/stats.cpp
index 9f0727c1ae2fa097dfd3a69a709eefe2d230c057..b8dae83ad8ca1bd5fb3ca68d324a200a51d5f205 100644 (file)
--- a/src/stats.cpp
+++ b/src/stats.cpp
@@ -21,8 +21,10 @@
 // subdirectory to make this more scalable.
 
 #include "ccache.hpp"
+#include "cleanup.hpp"
 #include "hashutil.hpp"
 
+#include <cmath>
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -390,7 +392,10 @@ stats_flush_to_file(const char* sfile, struct counters* updates)
   }
 
   if (need_cleanup) {
-    clean_up_dir(g_config, subdir, g_config.limit_multiple());
+    double factor = g_config.limit_multiple() / 16;
+    uint64_t max_size = round(g_config.max_size() * factor);
+    uint32_t max_files = round(g_config.max_files() * factor);
+    clean_up_dir(subdir, max_size, max_files, [](double) {});
   }
 
   free(subdir);